skylarmt/postserve
1
0
Fork 0
Code Issues 6 Pull Requests 5 Releases Wiki Activity

WIP: add buffers to layers (according to layer configuration).

Browse Source 
Multithread application — this is done badly, as I don't know the proper
way to do it in Python.
This commit is contained in:
Matthew Blissett 2017-09-01 15:30:34 +02:00
parent 66d17625d8
commit 3a4b3b4408
4 changed files with 287 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Dockerfile
View File

@ -1,4 +1,4 @@
FROM python:3.4 FROM python:3.6
RUN mkdir -p /usr/src/app RUN mkdir -p /usr/src/app
WORKDIR /usr/src/app WORKDIR /usr/src/app
@ -7,4 +7,8 @@ VOLUME /mapping
COPY . /usr/src/app/ COPY . /usr/src/app/
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt
CMD ["python", "-u","/usr/src/app/server.py"] CMD ["python", "-u","/usr/src/app/server.py"]
RUN DEBIAN_FRONTEND=noninteractive apt-get update \
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
emacs-nox

35
postserve.html Normal file
View File

@ -0,0 +1,35 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8"/>
<title>EPSG:4326 map preview</title>
<link rel="stylesheet" href="https://tile.gbif.org/ui/ol.css" type="text/css">
<script src="https://tile.gbif.org/ui/ol-debug.js"></script>
<script src="https://tile.gbif.org/ui/proj4.js"></script>
<script src="https://tile.gbif.org/ui/olms.js"></script>
<script src="https://tile.gbif.org/ui/common.js"></script>
<style>
html {
font-family: sans-serif;
}
.wrapper {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
background: lavender;
}
.map {
width: 100%;
height: 100%;
}
</style>
</head>
<body>
<div class="wrapper">
<div id="map" class="map"></div>
</div>
<script src="postserve.js"></script>
</body>
</html>

106
postserve.js Normal file
View File

@ -0,0 +1,106 @@
proj4.defs('EPSG:4326', "+proj=longlat +ellps=WGS84 +datum=WGS84 +units=degrees");
var resolutions = [];
var extent = 180.0;
var tile_size = 512;
var resolutions = Array(17).fill().map((_, i) => ( extent / tile_size / Math.pow(2, i) ));
var layers = [];
densityColours = ["#FFFF00", "#FFCC00", "#FF9900", "#FF6600", "#FF3300", "#FF0000"];
function createDensityStyle2() {
var point = new ol.style.Style({
image: new ol.style.Circle({
fill: new ol.style.Fill({color: '#FF0000'}),
radius: 1
}),
fill: new ol.style.Fill({color: '#FF0000'})
});
var styles = [];
return function(feature, resolution) {
var length = 0;
//console.log(feature);
var magnitude = Math.trunc(Math.min(5, Math.floor(Math.log(feature.get('total'))))) - 1;
//console.log("Colour ", magnitude, densityColours[magnitude]);
//styles[length++] = point;
styles[length++] = new ol.style.Style({
image: new ol.style.Circle({
fill: new ol.style.Fill({color: densityColours[magnitude]}),
radius: 1
}),
fill: new ol.style.Fill({color: densityColours[magnitude]})
});
styles.length = length;
return styles;
};
}
function createStatsStyle() {
var fill = new ol.style.Fill({color: '#000000'});
var stroke = new ol.style.Stroke({color: '#000000', width: 1});
var text = new ol.style.Text({
text: 'XYXYXY',
fill: fill,
stroke: stroke,
font: '16px "Open Sans", "Arial Unicode MS"'
});
var styles = [];
return function(feature, resolution) {
var length = 0;
//console.log(feature);
text.setText('Occurrences: '+feature.get('total'));
console.log(feature.get('total'));
styles[length++] = new ol.style.Style({
stroke: new ol.style.Stroke({color: '#000000'}),
text: text
});
styles.length = length;
return styles;
};
}
var tileGrid = new ol.tilegrid.TileGrid({
extent: ol.proj.get('EPSG:4326').getExtent(),
minZoom: 0,
maxZoom: 16,
resolutions: resolutions,
tileSize: 512,
});
layers['EPSG:4326'] = new ol.layer.VectorTile({
source: new ol.source.VectorTile({
projection: 'EPSG:4326',
format: new ol.format.MVT(),
tileGrid: tileGrid,
tilePixelRatio: 8,
url: 'http://mb.gbif.org:8080/tiles/{z}_{x}_{y}.pbf',
wrapX: false
}),
style: createStyle(),
});
layers['Grid'] = new ol.layer.Tile({
extent: ol.proj.get('EPSG:4326').getExtent(),
source: new ol.source.TileDebug({
projection: 'EPSG:4326',
tileGrid: tileGrid,
wrapX: false
}),
});
var map = new ol.Map({
layers: [
layers['EPSG:4326'],
layers['Grid']
],
target: 'map',
view: new ol.View({
center: [10.7522, 59.9139],
projection: 'EPSG:4326',
zoom: 10
}),
});

182
server.py
View File

@ -1,5 +1,6 @@
import tornado.ioloop import tornado.ioloop
import tornado.web import tornado.web
import tornado.httpserver
import io import io
import os import os
@ -22,81 +23,178 @@ def GetTM2Source(file):
tm2source = yaml.load(stream) tm2source = yaml.load(stream)
return tm2source return tm2source
buffer_sizes = []
def GeneratePrepared(layers): def GeneratePrepared(layers):
queries = [] queries = []
prepared = "PREPARE gettile(geometry, numeric, numeric, numeric) AS "
for layer in layers['Layer']: for layer in layers['Layer']:
buffer_size = str(layer['properties']['buffer-size']*2)
if (buffer_size not in buffer_sizes):
buffer_sizes.append(buffer_size)
layer_query = layer['Datasource']['table'].lstrip().rstrip() # Remove lead and trailing whitespace layer_query = layer['Datasource']['table'].lstrip().rstrip() # Remove lead and trailing whitespace
layer_query = layer_query[1:len(layer_query)-6] # Remove enough characters to remove first and last () and "AS t" layer_query = layer_query[1:len(layer_query)-6] # Remove enough characters to remove first and last () and "AS t"
layer_query = layer_query.replace("geometry", "ST_AsMVTGeom(geometry,!bbox!,4096,0,true) AS mvtgeometry") layer_query = layer_query.replace("geometry", "ST_AsMVTGeom(geometry,!bbox_nobuffer!,4096,"+buffer_size+",true) AS mvtgeometry")
base_query = "SELECT ST_ASMVT('"+layer['id']+"', 4096, 'mvtgeometry', tile) FROM ("+layer_query+" WHERE ST_AsMVTGeom(geometry, !bbox!,4096,0,true) IS NOT NULL) AS tile" layer_query = layer_query.replace("!bbox!", "!bbox_"+buffer_size+"_buffer!")
queries.append(base_query.replace("!bbox!","$1").replace("!scale_denominator!","$2").replace("!pixel_width!","$3").replace("!pixel_height!","$4")) base_query = "SELECT ST_ASMVT('"+layer['id']+"', 4096, 'mvtgeometry', tile) FROM ("+layer_query+" WHERE ST_AsMVTGeom(geometry,!bbox_nobuffer!,4096,"+buffer_size+",true) IS NOT NULL) AS tile"
queries.append(base_query.replace("!bbox_nobuffer!","$1").replace("!scale_denominator!","$2").replace("!pixel_width!","$3").replace("!pixel_height!","$4").replace("!bbox_"+buffer_size+"_buffer!", "$"+str(buffer_sizes.index(buffer_size)+5)))
chunk = ""
for buffer_size in buffer_sizes:
chunk = chunk + ", geometry"
prepared = "PREPARE gettile(geometry, numeric, numeric, numeric"+chunk+") AS "
prepared = prepared + " UNION ALL ".join(queries) + ";" prepared = prepared + " UNION ALL ".join(queries) + ";"
print(prepared) print(prepared)
return(prepared) return(prepared)
layers = GetTM2Source("/mapping/data.yml") layers = GetTM2Source("/mapping/data.yml")
prepared = GeneratePrepared(layers) prepared = GeneratePrepared(layers)
engine = create_engine('postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles'))
inspector = inspect(engine)
DBSession = sessionmaker(bind=engine)
session = DBSession()
session.execute(prepared)
def bounds(zoom,x,y): def bounds(zoom,x,y,buff):
inProj = pyproj.Proj(init='epsg:4326') #inProj = pyproj.Proj(init='epsg:3575')
outProj = pyproj.Proj(init='epsg:3857') #outProj = pyproj.Proj(init='epsg:3575')
lnglatbbox = mercantile.bounds(x,y,zoom) #lnglatbbox = mercantile.bounds(x,y,zoom)
ws = (pyproj.transform(inProj,outProj,lnglatbbox[0],lnglatbbox[1])) #ws = (pyproj.transform(inProj,outProj,lnglatbbox[0],lnglatbbox[1]))
en = (pyproj.transform(inProj,outProj,lnglatbbox[2],lnglatbbox[3])) #en = (pyproj.transform(inProj,outProj,lnglatbbox[2],lnglatbbox[3]))
#map_width_in_metres = 2 * 2**0.5*6371007.2
map_width_in_metres = 180.0
#tile_width_in_pixels = 512.0
#standardized_pixel_size = 0.00028
#map_width_in_pixels = tile_width_in_pixels*(2.0**zoom)
tiles_down = 2**(zoom)
tiles_across = 2**(zoom)
#arc x = x - 2**(zoom-1)
#arctic y = -(y - 2**(zoom-1)) - 1
x = x - 2**(zoom)
y = -(y - 2**(zoom)) - 1 - 2**(zoom-1)
print(x, y);
tile_width_in_metres = (map_width_in_metres / tiles_across)
tile_height_in_metres = (map_width_in_metres / tiles_down)
ws = ((x - buff)*tile_width_in_metres, (y - buff)*tile_width_in_metres)
en = ((x+1+buff)*tile_height_in_metres, (y+1+buff)*tile_height_in_metres)
print("Zoom, buffer", zoom, buff)
print("West: ", ws[0])
print("South: ", ws[1])
print("East: ", en[0])
print("North: ", en[1])
return {'w':ws[0],'s':ws[1],'e':en[0],'n':en[1]} return {'w':ws[0],'s':ws[1],'e':en[0],'n':en[1]}
def zoom_to_scale_denom(zoom): # For !scale_denominator! def zoom_to_scale_denom(zoom): # For !scale_denominator!
# From https://github.com/openstreetmap/mapnik-stylesheets/blob/master/zoom-to-scale.txt # From https://github.com/openstreetmap/mapnik-stylesheets/blob/master/zoom-to-scale.txt
#map_width_in_metres = 2 * 2**0.5*6371007.2 # Arctic
#map_width_in_metres = 180.0
map_width_in_metres = 40075016.68557849 map_width_in_metres = 40075016.68557849
tile_width_in_pixels = 256.0 tile_width_in_pixels = 512.0 # This asks for a zoom level higher, since the tiles are doubled.
standardized_pixel_size = 0.00028 standardized_pixel_size = 0.00028
map_width_in_pixels = tile_width_in_pixels*(2.0**zoom) map_width_in_pixels = tile_width_in_pixels*(2.0**zoom)
return str(map_width_in_metres/(map_width_in_pixels * standardized_pixel_size)) return str(map_width_in_metres/(map_width_in_pixels * standardized_pixel_size))
def replace_tokens(query,s,w,n,e,scale_denom): def replace_tokens(query,tilebounds,buffered_tilebounds,scale_denom,z):
return query.replace("!bbox!","ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+"))").replace("!scale_denominator!",scale_denom).replace("!pixel_width!","256").replace("!pixel_height!","256")
def get_mvt(zoom,x,y):
try: # Sanitize the inputs
sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y)
del zoom,x,y
except:
print('suspicious')
return 1
scale_denom = zoom_to_scale_denom(sani_zoom)
tilebounds = bounds(sani_zoom,sani_x,sani_y)
s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e']) s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e'])
final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!);"
sent_query = replace_tokens(final_query,s,w,n,e,scale_denom) start = query.replace("!bbox!","ST_SetSRID(ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+")), 4326)").replace("!scale_denominator!",scale_denom).replace("!pixel_width!","512").replace("!pixel_height!","512")
response = list(session.execute(sent_query))
print(sent_query) for buffer_size in buffer_sizes:
layers = filter(None,list(itertools.chain.from_iterable(response))) token = "!bbox_"+buffer_size+"_buffer!"
final_tile = b'' idx = buffer_sizes.index(buffer_size)
for layer in layers:
final_tile = final_tile + io.BytesIO(layer).getvalue() t = buffered_tilebounds[idx]
return final_tile s,w,n,e = str(t['s']),str(t['w']),str(t['n']),str(t['e'])
start = start.replace(token,"ST_SetSRID(ST_MakeBox2D(ST_Point("+w+", "+s+"), ST_Point("+e+", "+n+")), 4326)")
return start
class GetTile(tornado.web.RequestHandler): class GetTile(tornado.web.RequestHandler):
def get(self, zoom,x,y): def get(self, zoom,x,y):
engine = create_engine('postgresql://'+os.getenv('POSTGRES_USER','openmaptiles')+':'+os.getenv('POSTGRES_PASSWORD','openmaptiles')+'@'+os.getenv('POSTGRES_HOST','postgres')+':'+os.getenv('POSTGRES_PORT','5432')+'/'+os.getenv('POSTGRES_DB','openmaptiles'))
inspector = inspect(engine)
DBSession = sessionmaker(bind=engine)
self.session = DBSession()
print("Running prepare statement")
self.session.execute(prepared)
self.set_header("Content-Type", "application/x-protobuf") self.set_header("Content-Type", "application/x-protobuf")
self.set_header("Content-Disposition", "attachment") self.set_header("Content-Disposition", "attachment")
self.set_header("Access-Control-Allow-Origin", "*") self.set_header("Access-Control-Allow-Origin", "*")
response = get_mvt(zoom,x,y) response = self.get_mvt(zoom,x,y)
self.write(response)
self.session.close()
#self.write(response)
def get_mvt(self, zoom,x,y):
try: # Sanitize the inputs
sani_zoom,sani_x,sani_y = float(zoom),float(x),float(y)
del zoom,x,y
except:
print('suspicious')
return 1
scale_denom = zoom_to_scale_denom(sani_zoom)
tilebounds = bounds(sani_zoom,sani_x,sani_y,0)
#s_nobuffer,w_nobuffer,n_nobuffer,e_nobuffer = str(tilebounds_nobuffer['s']),str(tilebounds_nobuffer['w']),str(tilebounds_nobuffer['n']),str(tilebounds_nobuffer['e'])
#tilebounds = bounds(sani_zoom,sani_x,sani_y,0.50)
#s,w,n,e = str(tilebounds['s']),str(tilebounds['w']),str(tilebounds['n']),str(tilebounds['e'])
buffered_tilebounds = []
chunk = ""
for buffer_size in buffer_sizes:
t = bounds(sani_zoom,sani_x,sani_y,int(buffer_size)/512.0)
buffered_tilebounds.append(t)
chunk = chunk + ", !bbox_"+buffer_size+"_buffer!"
final_query = "EXECUTE gettile(!bbox!, !scale_denominator!, !pixel_width!, !pixel_height!"+chunk+");"
sent_query = replace_tokens(final_query,tilebounds,buffered_tilebounds,scale_denom,sani_zoom)
print('Final query', sent_query)
response = list(self.session.execute(sent_query))
print (sani_zoom, sani_x, sani_y)
print(sent_query)
layers = filter(None,list(itertools.chain.from_iterable(response)))
final_tile = b''
for layer in layers:
final_tile = final_tile + io.BytesIO(layer).getvalue()
self.write(final_tile)
#return final_tile
# Buffers
# aeroway.yaml: buffer_size: 4
# boundary.yaml: buffer_size: 4
# building.yaml: buffer_size: 4
# landcover.yaml: buffer_size: 4
# landuse.yaml: buffer_size: 4
# park.yaml: buffer_size: 4
# transportation.yaml: buffer_size: 4
# water.yaml: buffer_size: 4
# waterway.yaml: buffer_size: 4
# housenumber.yaml: buffer_size: 8
# transportation_name.yaml: buffer_size: 8
# graticules.yaml: buffer_size: 64
# mountain_peak.yaml: buffer_size: 64
# poi.yaml: buffer_size: 64
# place.yaml: buffer_size: 256
# water_name.yaml: buffer_size: 256
# Need 4/256, 8/256, 64/256 and 256/256
def m(): def m():
if __name__ == "__main__": if __name__ == "__main__":
# Make this prepared statement from the tm2source # Make this prepared statement from the tm2source
application = tornado.web.Application([(r"/tiles/([0-9]+)/([0-9]+)/([0-9]+).pbf", GetTile)]) application = tornado.web.Application([(r"/tiles/([0-9]+)_([0-9]+)_([0-9]+).pbf", GetTile)])
server = tornado.httpserver.HTTPServer(application)
server.bind(8080)
server.start(0)
print("Postserve started..") print("Postserve started..")
application.listen(8080) #application.listen(8080)
tornado.ioloop.IOLoop.instance().start() tornado.ioloop.IOLoop.instance().start()
m() m()