skylarmt/postserve
1
0
Fork 0
Code Issues 6 Pull Requests 5 Releases Wiki Activity

Merge 6e74be938dd0bd457de9f656dd47e2306f81ec3d into 66d17625d85aa144e82702dbd1cf31aadf3d094d

Browse Source
This commit is contained in:
Yuri Astrakhan 2019-03-22 00:18:56 +00:00 committed by GitHub
commit 6214eef9b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 294 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

173
.gitignore vendored Normal file
View File

@ -0,0 +1,173 @@
# Created by .ignore support plugin (hsz.mobi)
### VirtualEnv template
# Virtualenv
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
.Python
[Bb]in
[Ii]nclude
[Ll]ib
[Ll]ib64
[Ll]ocal
[Ss]cripts
pyvenv.cfg
.venv
pip-selfcheck.json
### Python template
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
### JetBrains template
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
# User-specific stuff
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/dictionaries
.idea/**/shelf
# Sensitive or high-churn files
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
# Gradle
.idea/**/gradle.xml
.idea/**/libraries
# CMake
cmake-build-debug/
cmake-build-release/
# Mongo Explorer plugin
.idea/**/mongoSettings.xml
# File-based project format
*.iws
# IntelliJ
out/
# mpeltonen/sbt-idea plugin
.idea_modules/
# JIRA plugin
atlassian-ide-plugin.xml
# Cursive Clojure plugin
.idea/replstate.xml
# Crashlytics plugin (for Android Studio and IntelliJ)
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
# Editor-based Rest Client
.idea/httpRequests

14
Dockerfile
View File

@ -1,10 +1,12 @@
FROM python:3.4
RUN mkdir -p /usr/src/app
FROM python:3.6
LABEL MAINTAINER "Yuri Astrakhan <YuriAstrakhan@gmail.com>"
WORKDIR /usr/src/app
VOLUME /mapping
COPY . /usr/src/app/
# Copy requirements.txt first to avoid pip install on every code change
COPY ./requirements.txt /usr/src/app/
RUN pip install --no-cache-dir -r requirements.txt
CMD ["python", "-u","/usr/src/app/server.py"]
COPY . /usr/src/app/
ENTRYPOINT ["python", "server.py"]

16
README.md
View File

@ -1,2 +1,16 @@
# postserve
Use the ST_AsMVT function to render tiles directly in Postgres
This is an OpenMapTiles map vector tile test server. It requires the prepared SQL statement
generated by the `generate-sqltovmt` tools script.
To run, use this command, replacing `myfile.sql` with the name of the generated file in the current dir.
```
docker run -it --rm --net=host -v "$PWD:/data" openmaptiles/postserve /data/myfile.sql
```
To see help, use
```
docker run -it --rm --net=host -v "$PWD:/data" openmaptiles/postserve --help
```

9
requirements.txt
View File

@ -1,6 +1,3 @@
tornado==4.4.2
sqlalchemy==1.1.5
mercantile==0.9.0
pyproj==1.9.5.1
pyyaml==3.12
psycopg2==2.6.2
tornado==6.0.1
psycopg2-binary==2.7.7
docopt==0.6.2

188
server.py
View File

@ -1,102 +1,104 @@
#!/usr/bin/env python
"""
This is a simple vector tile server that returns a PBF tile for /tiles/{z}/{x}/{y}.pbf requests
Use these environment variables to configure PostgreSQL access:
POSTGRES_HOST, POSTGRES_PORT, POSTGRES_DB, POSTGRES_PASSWORD
Usage:
server <prepared-sql-file> [--fname <name>] [--port <port>]
server --help
server --version
<prepared-sql-file> SQL file generated by generate-sqltomvt script with the --prepared flag
Options:
--fname=<name> Name of the generated function [default: gettile]
-p --port=<port> Serve on this port [default: 8080]
--help Show this screen.
--version Show version.
"""
import logging
import tornado.ioloop
import tornado.web
import io
import os
from docopt import docopt
import psycopg2
from sqlalchemy import Column, ForeignKey, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from sqlalchemy import create_engine
from sqlalchemy import inspect
from sqlalchemy import text
from sqlalchemy.orm import sessionmaker
import mercantile
import pyproj
import yaml
import sys
import itertools
def GetTM2Source(file):
with open(file,'r') as stream:
tm2source = yaml.load(stream)
return tm2source
def GeneratePrepared(layers):
queries = []
prepared = "PREPARE gettile(geometry, numeric, numeric, numeric) AS "
for layer in layers['Layer']:
layer_query = layer['Datasource']['table'].lstrip().rstrip() # Remove lead and trailing whitespace
layer_query = layer_query[1:len(layer_query)-6] # Remove enough characters to remove first and last () and "AS t"
layer_query = layer_query.replace("geometry", "ST_AsMVTGeom(geometry,!bbox!,4096,0,true) AS mvtgeometry")
base_query = "SELECT ST_ASMVT('"+layer['id']+"', 4096, 'mvtgeometry', tile) FROM ("+layer_query+" WHERE ST_AsMVTGeom(geometry, !bbox!,4096,0,true) IS NOT NULL) AS tile"
queries.append(base_query.replace("!bbox!","$1").replace("!scale_denominator!","$2").replace("!pixel_width!","$3").replace("!pixel_height!","$4"))
prepared = prepared + " UNION ALL ".join(queries) + ";"
print(prepared)
return(prepared)
layers = GetTM2Source("/mapping/data.yml")
prepared = GeneratePrepared(layers)
engine = create_engine('postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles'))
inspector = inspect(engine)
DBSession = sessionmaker(bind=engine)
session = DBSession()
session.execute(prepared)
def bounds(zoom,x,y):
inProj = pyproj.Proj(init='epsg:4326')
outProj = pyproj.Proj(init='epsg:3857')
lnglatbbox = mercantile.bounds(x,y,zoom)
ws = (pyproj.transform(inProj,outProj,lnglatbbox[0],lnglatbbox[1]))
en = (pyproj.transform(inProj,outProj,lnglatbbox[2],lnglatbbox[3]))
return {'w':ws[0],'s':ws[1],'e':en[0],'n':en[1]}
def zoom_to_scale_denom(zoom): # For !scale_denominator!
# From https://github.com/openstreetmap/mapnik-stylesheets/blob/master/zoom-to-scale.txt
map_width_in_metres = 40075016.68557849
tile_width_in_pixels = 256.0
standardized_pixel_size = 0.00028
map_width_in_pixels = tile_width_in_pixels*(2.0**zoom)
return str(map_width_in_metres/(map_width_in_pixels * standardized_pixel_size))
def replace_tokens(query,s,w,n,e,scale_denom):
return query.replace("!bbox!","ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+"))").replace("!scale_denominator!",scale_denom).replace("!pixel_width!","256").replace("!pixel_height!","256")
def get_mvt(zoom,x,y):
try: # Sanitize the inputs
sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y)
del zoom,x,y
except:
print('suspicious')
return 1
scale_denom = zoom_to_scale_denom(sani_zoom)
tilebounds = bounds(sani_zoom,sani_x,sani_y)
s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e'])
final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!);"
sent_query = replace_tokens(final_query,s,w,n,e,scale_denom)
response = list(session.execute(sent_query))
print(sent_query)
layers = filter(None,list(itertools.chain.from_iterable(response)))
final_tile = b''
for layer in layers:
final_tile = final_tile + io.BytesIO(layer).getvalue()
return final_tile
class GetTile(tornado.web.RequestHandler):
def get(self, zoom,x,y):
self.set_header("Content-Type", "application/x-protobuf")
self.set_header("Content-Disposition", "attachment")
self.set_header("Access-Control-Allow-Origin", "*")
response = get_mvt(zoom,x,y)
self.write(response)
def initialize(self, fname, connection, query):
self.fname = fname
self.db_connection = connection
self.db_query = query
def m():
if __name__ == "__main__":
# Make this prepared statement from the tm2source
application = tornado.web.Application([(r"/tiles/([0-9]+)/([0-9]+)/([0-9]+).pbf", GetTile)])
print("Postserve started..")
application.listen(8080)
tornado.ioloop.IOLoop.instance().start()
def get(self, z, x, y):
z, x, y = int(z), int(x), int(y)
cursor = self.db_connection.cursor()
try:
cursor.execute(self.db_query, (z, x, y))
result = cursor.fetchall()
if result:
self.set_header("Content-Type", "application/x-protobuf")
self.set_header("Content-Disposition", "attachment")
self.set_header("Access-Control-Allow-Origin", "*")
value = io.BytesIO(result[0][0]).getvalue()
self.write(value)
print('{0}({1},{2},{3}) --> {4:,} bytes'.format(self.fname, z, x, y, len(value)))
else:
self.clear()
self.set_status(404)
print('{0}({1},{2},{3}) is EMPTY'.format(self.fname, z, x, y))
except Exception as err:
print('{0}({1},{2},{3}) threw an exception'.format(self.fname, z, x, y, err))
raise
finally:
cursor.close()
m()
def main(args):
pgdb = os.getenv('POSTGRES_DB', 'openmaptiles')
pghost = os.getenv('POSTGRES_HOST', 'localhost')
pgport = os.getenv('POSTGRES_PORT', '5432')
print('Connecting to PostgreSQL at {0}:{1}, db={2}...'.format(pghost, pgport, pgdb))
connection = psycopg2.connect(
dbname=pgdb,
host=pghost,
port=pgport,
user=os.getenv('POSTGRES_USER', 'openmaptiles'),
password=os.getenv('POSTGRES_PASSWORD', 'openmaptiles'),
)
sqlfile = args['<prepared-sql-file>']
with open(sqlfile, 'r') as stream:
prepared = stream.read()
print('Using prepared SQL:\n\n-------\n\n' + prepared + '\n\n-------\n\n')
cursor = connection.cursor()
try:
cursor.execute(prepared)
finally:
cursor.close()
fname = args['--fname']
query = "EXECUTE {0}(%s, %s, %s)".format(fname)
print('Loaded {0}\nWill use "{1}" to get vector tiles.'.format(sqlfile, query))
tornado.log.access_log.setLevel(logging.ERROR)
port = int(args['--port'])
application = tornado.web.Application([(
r"/tiles/([0-9]+)/([0-9]+)/([0-9]+).pbf",
GetTile,
dict(fname=fname, connection=connection, query=query)
)])
application.listen(port)
print("Postserve started, listening on 0.0.0.0:{0}".format(port))
tornado.ioloop.IOLoop.instance().start()
if __name__ == "__main__":
main(docopt(__doc__, version="1.0"))