From c916c911ab472b83bc9d3fe85fe1fb0804635141 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 19 Mar 2019 00:46:33 -0400 Subject: [PATCH 1/6] Fix ST_ASMVT() signature for current PostGIS --- server.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server.py b/server.py index 8122558..1a9784f 100644 --- a/server.py +++ b/server.py @@ -29,7 +29,7 @@ def GeneratePrepared(layers): layer_query = layer['Datasource']['table'].lstrip().rstrip() # Remove lead and trailing whitespace layer_query = layer_query[1:len(layer_query)-6] # Remove enough characters to remove first and last () and "AS t" layer_query = layer_query.replace("geometry", "ST_AsMVTGeom(geometry,!bbox!,4096,0,true) AS mvtgeometry") - base_query = "SELECT ST_ASMVT('"+layer['id']+"', 4096, 'mvtgeometry', tile) FROM ("+layer_query+" WHERE ST_AsMVTGeom(geometry, !bbox!,4096,0,true) IS NOT NULL) AS tile" + base_query = "SELECT ST_ASMVT(tile, '"+layer['id']+"', 4096, 'mvtgeometry') FROM ("+layer_query+" WHERE ST_AsMVTGeom(geometry, !bbox!,4096,0,true) IS NOT NULL) AS tile" queries.append(base_query.replace("!bbox!","$1").replace("!scale_denominator!","$2").replace("!pixel_width!","$3").replace("!pixel_height!","$4")) prepared = prepared + " UNION ALL ".join(queries) + ";" print(prepared) @@ -80,7 +80,7 @@ def get_mvt(zoom,x,y): layers = filter(None,list(itertools.chain.from_iterable(response))) final_tile = b'' for layer in layers: - final_tile = final_tile + io.BytesIO(layer).getvalue() + final_tile = final_tile + io.BytesIO(layer).getvalue() return final_tile class GetTile(tornado.web.RequestHandler): From f6468ccc2f3507e111b4fea4b67e46ece927291c Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 21 Mar 2019 03:52:00 -0400 Subject: [PATCH 2/6] fixes --- Dockerfile | 4 +-- requirements.txt | 1 - server.py | 92 ++++++++++++++++++++---------------------------- 3 files changed, 41 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index 06bc6a4..11f1815 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.4 +FROM python:3.6 RUN mkdir -p /usr/src/app WORKDIR /usr/src/app @@ -7,4 +7,4 @@ VOLUME /mapping COPY . /usr/src/app/ RUN pip install --no-cache-dir -r requirements.txt -CMD ["python", "-u","/usr/src/app/server.py"] +CMD ["python", "-u","/usr/src/app/server.py"] diff --git a/requirements.txt b/requirements.txt index bb95b17..0d306c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,4 @@ tornado==4.4.2 sqlalchemy==1.1.5 mercantile==0.9.0 pyproj==1.9.5.1 -pyyaml==3.12 psycopg2==2.6.2 diff --git a/server.py b/server.py index 1a9784f..361d9a1 100644 --- a/server.py +++ b/server.py @@ -3,100 +3,86 @@ import tornado.web import io import os -from sqlalchemy import Column, ForeignKey, Integer, String -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship -from sqlalchemy import create_engine -from sqlalchemy import inspect -from sqlalchemy import text +from sqlalchemy import create_engine, inspect from sqlalchemy.orm import sessionmaker import mercantile import pyproj -import yaml import sys import itertools -def GetTM2Source(file): - with open(file,'r') as stream: - tm2source = yaml.load(stream) - return tm2source -def GeneratePrepared(layers): - queries = [] - prepared = "PREPARE gettile(geometry, numeric, numeric, numeric) AS " - for layer in layers['Layer']: - layer_query = layer['Datasource']['table'].lstrip().rstrip() # Remove lead and trailing whitespace - layer_query = layer_query[1:len(layer_query)-6] # Remove enough characters to remove first and last () and "AS t" - layer_query = layer_query.replace("geometry", "ST_AsMVTGeom(geometry,!bbox!,4096,0,true) AS mvtgeometry") - base_query = "SELECT ST_ASMVT(tile, '"+layer['id']+"', 4096, 'mvtgeometry') FROM ("+layer_query+" WHERE ST_AsMVTGeom(geometry, !bbox!,4096,0,true) IS NOT NULL) AS tile" - queries.append(base_query.replace("!bbox!","$1").replace("!scale_denominator!","$2").replace("!pixel_width!","$3").replace("!pixel_height!","$4")) - prepared = prepared + " UNION ALL ".join(queries) + ";" - print(prepared) - return(prepared) +def getPreparedSql(file): + with open(file, 'r') as stream: + return stream.read() -layers = GetTM2Source("/mapping/data.yml") -prepared = GeneratePrepared(layers) -engine = create_engine('postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles')) + +prepared = getPreparedSql("/mapping/mvt/maketile_prep.sql") +engine = create_engine( + 'postgresql://' + os.getenv('POSTGRES_USER', 'openmaptiles') + + ':' + os.getenv('POSTGRES_PASSWORD', 'openmaptiles') + + '@' + os.getenv('POSTGRES_HOST', 'postgres') + + ':' + os.getenv('POSTGRES_PORT', '5432') + + '/' + os.getenv('POSTGRES_DB', 'openmaptiles')) inspector = inspect(engine) DBSession = sessionmaker(bind=engine) session = DBSession() session.execute(prepared) -def bounds(zoom,x,y): + +def bounds(zoom, x, y): inProj = pyproj.Proj(init='epsg:4326') outProj = pyproj.Proj(init='epsg:3857') - lnglatbbox = mercantile.bounds(x,y,zoom) - ws = (pyproj.transform(inProj,outProj,lnglatbbox[0],lnglatbbox[1])) - en = (pyproj.transform(inProj,outProj,lnglatbbox[2],lnglatbbox[3])) - return {'w':ws[0],'s':ws[1],'e':en[0],'n':en[1]} + lnglatbbox = mercantile.bounds(x, y, zoom) + ws = (pyproj.transform(inProj, outProj, lnglatbbox[0], lnglatbbox[1])) + en = (pyproj.transform(inProj, outProj, lnglatbbox[2], lnglatbbox[3])) + return {'w': ws[0], 's': ws[1], 'e': en[0], 'n': en[1]} -def zoom_to_scale_denom(zoom): # For !scale_denominator! - # From https://github.com/openstreetmap/mapnik-stylesheets/blob/master/zoom-to-scale.txt - map_width_in_metres = 40075016.68557849 - tile_width_in_pixels = 256.0 - standardized_pixel_size = 0.00028 - map_width_in_pixels = tile_width_in_pixels*(2.0**zoom) - return str(map_width_in_metres/(map_width_in_pixels * standardized_pixel_size)) -def replace_tokens(query,s,w,n,e,scale_denom): - return query.replace("!bbox!","ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+"))").replace("!scale_denominator!",scale_denom).replace("!pixel_width!","256").replace("!pixel_height!","256") +def replace_tokens(query, s, w, n, e, zoom): + return (query + .replace("!bbox!", "ST_MakeBox2D(ST_Point(" + w + ", " + s + "), ST_Point(" + e + ", " + n + "))") + .replace("!zoom!", zoom) + .replace("!pixel_width!", "256")) -def get_mvt(zoom,x,y): - try: # Sanitize the inputs - sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y) - del zoom,x,y + +def get_mvt(zoom, x, y): + try: + # Sanitize the inputs + sani_zoom, sani_x, sani_y = float(zoom), float(x), float(y) + del zoom, x, y except: print('suspicious') return 1 - scale_denom = zoom_to_scale_denom(sani_zoom) - tilebounds = bounds(sani_zoom,sani_x,sani_y) - s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e']) - final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!);" - sent_query = replace_tokens(final_query,s,w,n,e,scale_denom) + tilebounds = bounds(sani_zoom, sani_x, sani_y) + s, w, n, e = str(tilebounds['s']), str(tilebounds['w']), str(tilebounds['n']), str(tilebounds['e']) + final_query = "EXECUTE gettile(!bbox!, !zoom!, !pixel_width!);" + sent_query = replace_tokens(final_query, s, w, n, e, sani_zoom) response = list(session.execute(sent_query)) print(sent_query) - layers = filter(None,list(itertools.chain.from_iterable(response))) + layers = filter(None, list(itertools.chain.from_iterable(response))) final_tile = b'' for layer in layers: final_tile = final_tile + io.BytesIO(layer).getvalue() return final_tile + class GetTile(tornado.web.RequestHandler): - def get(self, zoom,x,y): + def get(self, zoom, x, y): self.set_header("Content-Type", "application/x-protobuf") self.set_header("Content-Disposition", "attachment") self.set_header("Access-Control-Allow-Origin", "*") - response = get_mvt(zoom,x,y) + response = get_mvt(zoom, x, y) self.write(response) + def m(): if __name__ == "__main__": - # Make this prepared statement from the tm2source application = tornado.web.Application([(r"/tiles/([0-9]+)/([0-9]+)/([0-9]+).pbf", GetTile)]) print("Postserve started..") application.listen(8080) tornado.ioloop.IOLoop.instance().start() + m() From 05b142f3eea415d597f6bd837abdbef21f547469 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 21 Mar 2019 17:42:37 -0400 Subject: [PATCH 3/6] Rewrote server.py for simplicity, remove sql injection possibility, etc --- Dockerfile | 10 ++-- requirements.txt | 8 +-- server.py | 151 +++++++++++++++++++++++------------------------ 3 files changed, 83 insertions(+), 86 deletions(-) diff --git a/Dockerfile b/Dockerfile index 11f1815..860b894 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,12 @@ FROM python:3.6 -RUN mkdir -p /usr/src/app -WORKDIR /usr/src/app - VOLUME /mapping -COPY . /usr/src/app/ +WORKDIR /usr/src/app + +# Copy requirements.txt first to avoid pip install on every code change +COPY ./requirements.txt /usr/src/app/ RUN pip install --no-cache-dir -r requirements.txt +COPY . /usr/src/app/ + CMD ["python", "-u","/usr/src/app/server.py"] diff --git a/requirements.txt b/requirements.txt index 0d306c5..3b2a7cf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -tornado==4.4.2 -sqlalchemy==1.1.5 -mercantile==0.9.0 -pyproj==1.9.5.1 -psycopg2==2.6.2 +tornado==6.0.1 +sqlalchemy==1.3.1 +psycopg2-binary==2.7.7 diff --git a/server.py b/server.py index 361d9a1..786c175 100644 --- a/server.py +++ b/server.py @@ -1,88 +1,85 @@ +#!/usr/bin/env python +""" +This is a simple vector tile server that returns a PBF tile for /tiles/{z}/{x}/{y}.pbf requests + +Use these environment variables to configure PostgreSQL access: + POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB, POSTGRES_PASSWORD + +Usage: + server [--fname ] [--port ] + server --help + server --version + + SQL file generated by generate-sqltomvt script with the --prepared flag + +Options: + --fname= Name of the generated function [default: gettile] + -p --port= Serve on this port [default: 8080] + --help Show this screen. + --version Show version. +""" import tornado.ioloop import tornado.web import io import os - +from docopt import docopt from sqlalchemy import create_engine, inspect from sqlalchemy.orm import sessionmaker -import mercantile -import pyproj -import sys -import itertools - - -def getPreparedSql(file): - with open(file, 'r') as stream: - return stream.read() - - -prepared = getPreparedSql("/mapping/mvt/maketile_prep.sql") -engine = create_engine( - 'postgresql://' + os.getenv('POSTGRES_USER', 'openmaptiles') + - ':' + os.getenv('POSTGRES_PASSWORD', 'openmaptiles') + - '@' + os.getenv('POSTGRES_HOST', 'postgres') + - ':' + os.getenv('POSTGRES_PORT', '5432') + - '/' + os.getenv('POSTGRES_DB', 'openmaptiles')) -inspector = inspect(engine) -DBSession = sessionmaker(bind=engine) -session = DBSession() -session.execute(prepared) - - -def bounds(zoom, x, y): - inProj = pyproj.Proj(init='epsg:4326') - outProj = pyproj.Proj(init='epsg:3857') - lnglatbbox = mercantile.bounds(x, y, zoom) - ws = (pyproj.transform(inProj, outProj, lnglatbbox[0], lnglatbbox[1])) - en = (pyproj.transform(inProj, outProj, lnglatbbox[2], lnglatbbox[3])) - return {'w': ws[0], 's': ws[1], 'e': en[0], 'n': en[1]} - - -def replace_tokens(query, s, w, n, e, zoom): - return (query - .replace("!bbox!", "ST_MakeBox2D(ST_Point(" + w + ", " + s + "), ST_Point(" + e + ", " + n + "))") - .replace("!zoom!", zoom) - .replace("!pixel_width!", "256")) - - -def get_mvt(zoom, x, y): - try: - # Sanitize the inputs - sani_zoom, sani_x, sani_y = float(zoom), float(x), float(y) - del zoom, x, y - except: - print('suspicious') - return 1 - - tilebounds = bounds(sani_zoom, sani_x, sani_y) - s, w, n, e = str(tilebounds['s']), str(tilebounds['w']), str(tilebounds['n']), str(tilebounds['e']) - final_query = "EXECUTE gettile(!bbox!, !zoom!, !pixel_width!);" - sent_query = replace_tokens(final_query, s, w, n, e, sani_zoom) - response = list(session.execute(sent_query)) - print(sent_query) - layers = filter(None, list(itertools.chain.from_iterable(response))) - final_tile = b'' - for layer in layers: - final_tile = final_tile + io.BytesIO(layer).getvalue() - return final_tile - class GetTile(tornado.web.RequestHandler): - def get(self, zoom, x, y): - self.set_header("Content-Type", "application/x-protobuf") - self.set_header("Content-Disposition", "attachment") - self.set_header("Access-Control-Allow-Origin", "*") - response = get_mvt(zoom, x, y) - self.write(response) + def initialize(self, session, query): + self.db_session = session + self.db_query = query + + def get(self, z, x, y): + z, x, y = int(z), int(x), int(y) + try: + result = self.db_session.execute(self.db_query, params=dict(z=z, x=x, y=y)).fetchall() + if result: + self.set_header("Content-Type", "application/x-protobuf") + self.set_header("Content-Disposition", "attachment") + self.set_header("Access-Control-Allow-Origin", "*") + value = io.BytesIO(result[0][0]).getvalue() + self.write(value) + print('{0},{1},{2} returned {3} bytes'.format(z, x, y, len(value))) + else: + self.clear() + self.set_status(404) + print('Got NULL result for {0},{1},{2}'.format(z, x, y)) + except Exception as err: + print('{0},{1},{2} threw an exception {3}'.format(z, x, y, err)) + raise + +def main(args): + sqlfile = args[''] + with open(sqlfile, 'r') as stream: + prepared = stream.read() + + pghost = os.getenv('POSTGRES_HOST', 'localhost') + ':' + os.getenv('POSTGRES_PORT', '5432') + pgdb = os.getenv('POSTGRES_DB', 'openmaptiles') + pgcreds = os.getenv('POSTGRES_USER', 'openmaptiles') + ':' + os.getenv('POSTGRES_PASSWORD', 'openmaptiles') + engine = create_engine('postgresql://' + pgcreds + '@' + pghost + '/' + pgdb) + + print('Connecting to PostgreSQL at {0}, db={1}'.format(pghost, pgdb)) + inspector = inspect(engine) + session = sessionmaker(bind=engine)() + session.execute(prepared) + + query = "EXECUTE {0}(:z, :x, :y)".format(args['--fname']) + print('Loaded {0}, will use "{1}" to get vector tiles.'.format(sqlfile, query)) + + port = int(args['--port']) + application = tornado.web.Application([( + r"/tiles/([0-9]+)/([0-9]+)/([0-9]+).pbf", + GetTile, + dict(session=session, query=query) + )]) + application.listen(port) + + print("Postserve started, listening on 0.0.0.0:{0}".format(port)) + tornado.ioloop.IOLoop.instance().start() -def m(): - if __name__ == "__main__": - application = tornado.web.Application([(r"/tiles/([0-9]+)/([0-9]+)/([0-9]+).pbf", GetTile)]) - print("Postserve started..") - application.listen(8080) - tornado.ioloop.IOLoop.instance().start() - - -m() +if __name__ == "__main__": + main(docopt(__doc__, version="1.0")) From 1bf8d30e4a299d3f76740b9c249ee2effdd1e7b6 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 21 Mar 2019 19:44:09 -0400 Subject: [PATCH 4/6] Use direct Postgres connection, proper CLI, cleanup Minimum requirements, proper parameter processing, docs --- .gitignore | 173 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 - server.py | 57 ++++++++++------ 3 files changed, 211 insertions(+), 20 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..30618d2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,173 @@ +# Created by .ignore support plugin (hsz.mobi) +### VirtualEnv template +# Virtualenv +# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ +.Python +[Bb]in +[Ii]nclude +[Ll]ib +[Ll]ib64 +[Ll]ocal +[Ss]cripts +pyvenv.cfg +.venv +pip-selfcheck.json +### Python template +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +### JetBrains template +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/dictionaries +.idea/**/shelf + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# CMake +cmake-build-debug/ +cmake-build-release/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + diff --git a/requirements.txt b/requirements.txt index 3b2a7cf..42dcd2f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ tornado==6.0.1 -sqlalchemy==1.3.1 psycopg2-binary==2.7.7 diff --git a/server.py b/server.py index 786c175..b04b22d 100644 --- a/server.py +++ b/server.py @@ -18,62 +18,81 @@ Options: --help Show this screen. --version Show version. """ +import logging import tornado.ioloop import tornado.web import io import os from docopt import docopt -from sqlalchemy import create_engine, inspect -from sqlalchemy.orm import sessionmaker +import psycopg2 class GetTile(tornado.web.RequestHandler): - def initialize(self, session, query): - self.db_session = session + def initialize(self, fname, connection, query): + self.fname = fname + self.db_connection = connection self.db_query = query def get(self, z, x, y): z, x, y = int(z), int(x), int(y) + cursor = self.db_connection.cursor() try: - result = self.db_session.execute(self.db_query, params=dict(z=z, x=x, y=y)).fetchall() + cursor.execute(self.db_query, (z, x, y)) + result = cursor.fetchall() if result: self.set_header("Content-Type", "application/x-protobuf") self.set_header("Content-Disposition", "attachment") self.set_header("Access-Control-Allow-Origin", "*") value = io.BytesIO(result[0][0]).getvalue() self.write(value) - print('{0},{1},{2} returned {3} bytes'.format(z, x, y, len(value))) + print('{0}({1},{2},{3}) --> {4:,} bytes'.format(self.fname, z, x, y, len(value))) else: self.clear() self.set_status(404) - print('Got NULL result for {0},{1},{2}'.format(z, x, y)) + print('{0}({1},{2},{3}) is EMPTY'.format(self.fname, z, x, y)) except Exception as err: - print('{0},{1},{2} threw an exception {3}'.format(z, x, y, err)) + print('{0}({1},{2},{3}) threw an exception'.format(self.fname, z, x, y, err)) raise + finally: + cursor.close() + def main(args): + pgdb = os.getenv('POSTGRES_DB', 'openmaptiles') + pghost = os.getenv('POSTGRES_HOST', 'localhost') + pgport = os.getenv('POSTGRES_PORT', '5432') + print('Connecting to PostgreSQL at {0}:{1}, db={2}...'.format(pghost, pgport, pgdb)) + + connection = psycopg2.connect( + dbname=pgdb, + host=pghost, + port=pgport, + user=os.getenv('POSTGRES_USER', 'openmaptiles'), + password=os.getenv('POSTGRES_PASSWORD', 'openmaptiles'), + ) + sqlfile = args[''] with open(sqlfile, 'r') as stream: prepared = stream.read() - pghost = os.getenv('POSTGRES_HOST', 'localhost') + ':' + os.getenv('POSTGRES_PORT', '5432') - pgdb = os.getenv('POSTGRES_DB', 'openmaptiles') - pgcreds = os.getenv('POSTGRES_USER', 'openmaptiles') + ':' + os.getenv('POSTGRES_PASSWORD', 'openmaptiles') - engine = create_engine('postgresql://' + pgcreds + '@' + pghost + '/' + pgdb) + print('Using prepared SQL:\n\n-------\n\n' + prepared + '\n\n-------\n\n') - print('Connecting to PostgreSQL at {0}, db={1}'.format(pghost, pgdb)) - inspector = inspect(engine) - session = sessionmaker(bind=engine)() - session.execute(prepared) + cursor = connection.cursor() + try: + cursor.execute(prepared) + finally: + cursor.close() - query = "EXECUTE {0}(:z, :x, :y)".format(args['--fname']) - print('Loaded {0}, will use "{1}" to get vector tiles.'.format(sqlfile, query)) + fname = args['--fname'] + query = "EXECUTE {0}(%s, %s, %s)".format(fname) + print('Loaded {0}\nWill use "{1}" to get vector tiles.'.format(sqlfile, query)) + tornado.log.access_log.setLevel(logging.ERROR) port = int(args['--port']) application = tornado.web.Application([( r"/tiles/([0-9]+)/([0-9]+)/([0-9]+).pbf", GetTile, - dict(session=session, query=query) + dict(fname=fname, connection=connection, query=query) )]) application.listen(port) From 08e3f009205f6969ac516bf9c9c145e354173c92 Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 21 Mar 2019 20:05:29 -0400 Subject: [PATCH 5/6] Dockerfile and requirement fixes --- Dockerfile | 10 +++++++++- requirements.txt | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 860b894..9a551be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ FROM python:3.6 VOLUME /mapping +LABEL MAINTAINER "Yuri Astrakhan " + WORKDIR /usr/src/app # Copy requirements.txt first to avoid pip install on every code change @@ -9,4 +11,10 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . /usr/src/app/ -CMD ["python", "-u","/usr/src/app/server.py"] +ENTRYPOINT ["python", "server.py"] + +# Users can easily override prepared file with their own: +# +# docker run -it --rm --net=host -v "$PWD:/mapping" openmaptiles/postserve /mapping/myfile.sql +# +CMD ["/mapping/mvt/maketile_prep.sql"] diff --git a/requirements.txt b/requirements.txt index 42dcd2f..9f14259 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ tornado==6.0.1 psycopg2-binary==2.7.7 +docopt==0.6.2 From 6e74be938dd0bd457de9f656dd47e2306f81ec3d Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Thu, 21 Mar 2019 20:18:49 -0400 Subject: [PATCH 6/6] Documentation and dockerfile cleanups --- Dockerfile | 8 -------- README.md | 16 +++++++++++++++- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9a551be..21af613 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,4 @@ FROM python:3.6 -VOLUME /mapping - LABEL MAINTAINER "Yuri Astrakhan " WORKDIR /usr/src/app @@ -12,9 +10,3 @@ RUN pip install --no-cache-dir -r requirements.txt COPY . /usr/src/app/ ENTRYPOINT ["python", "server.py"] - -# Users can easily override prepared file with their own: -# -# docker run -it --rm --net=host -v "$PWD:/mapping" openmaptiles/postserve /mapping/myfile.sql -# -CMD ["/mapping/mvt/maketile_prep.sql"] diff --git a/README.md b/README.md index c581971..9796545 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,16 @@ # postserve -Use the ST_AsMVT function to render tiles directly in Postgres + +This is an OpenMapTiles map vector tile test server. It requires the prepared SQL statement +generated by the `generate-sqltovmt` tools script. + +To run, use this command, replacing `myfile.sql` with the name of the generated file in the current dir. + +``` +docker run -it --rm --net=host -v "$PWD:/data" openmaptiles/postserve /data/myfile.sql +``` + +To see help, use + +``` +docker run -it --rm --net=host -v "$PWD:/data" openmaptiles/postserve --help +```