From a9392359b93fedb4a67607432d2a43868f534911 Mon Sep 17 00:00:00 2001 From: Steffen Volkmann Date: Thu, 13 Jun 2019 15:39:04 +0200 Subject: [PATCH] integration of update and tile expire procedure see https://ircama.github.io/osm-carto-tutorials/updating-data/ Implements https://github.com/Overv/openstreetmap-tile-server/issues/2 --- Dockerfile | 20 +++ README.md | 57 ++++++-- openstreetmap-tiles-update-expire | 207 ++++++++++++++++++++++++++++++ run.sh | 20 +++ 4 files changed, 292 insertions(+), 12 deletions(-) create mode 100644 openstreetmap-tiles-update-expire diff --git a/Dockerfile b/Dockerfile index eb7f98e..2d8c5e2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,6 +6,7 @@ FROM ubuntu:18.04 # Set up environment ENV TZ=UTC ENV AUTOVACUUM=on +ENV UPDATES=disabled RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Install dependencies @@ -63,6 +64,10 @@ RUN echo "deb [ allow-insecure=yes ] http://apt.postgresql.org/pub/repos/apt/ bi unzip \ wget \ zlib1g-dev \ + osmosis \ + osmium-tool \ + cron \ + python-psycopg2 python-shapely python-lxml \ && apt-get clean autoclean \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ @@ -138,7 +143,22 @@ RUN chown -R postgres:postgres /var/lib/postgresql \ && chown postgres:postgres /etc/postgresql/10/main/postgresql.custom.conf.tmpl \ && echo "\ninclude 'postgresql.custom.conf'" >> /etc/postgresql/10/main/postgresql.conf +# copy update scripts +COPY openstreetmap-tiles-update-expire /usr/bin/ +RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \ + && mkdir /var/log/tiles \ + && chmod a+rw /var/log/tiles \ + && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \ + && echo "* * * * * renderer openstreetmap-tiles-update-expire\n" >> /etc/crontab + +# install trim_osc.py helper script +USER renderer +RUN cd ~/src \ + && git clone https://github.com/zverik/regional \ + && chmod u+x ~/src/regional/trim_osc.py + # Start running +USER root COPY run.sh / ENTRYPOINT ["/run.sh"] CMD [] diff --git a/README.md b/README.md index ea3ddd0..5821777 100644 --- a/README.md +++ b/README.md @@ -9,41 +9,74 @@ First create a Docker volume to hold the PostgreSQL database that will contain t docker volume create openstreetmap-data Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example: - - docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main overv/openstreetmap-tile-server import - +``` + docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ + -v openstreetmap-data:/var/lib/postgresql/10/main \ + overv/openstreetmap-tile-server \ + import +``` If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server. +### optional step to allow consecutive updates of osm extracts + +if your import is a extract of planet and is based on polygon then you should download this polynom data and use follwing command for the import procedure: +``` + docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ + -v /absolute/path/to/luxembourg.poly:/data.poly \ + -v /openstreetmap-data:/var/lib/postgresql/10/main \ + overv/openstreetmap-tile-server \ + import +``` ## Running the server Run the server like this: - - docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run - +``` + docker run -p 80:80 \ + -v openstreetmap-data:/var/lib/postgresql/10/main \ + -d overv/openstreetmap-tile-server \ + run +``` Your tiles will now be available at `http://localhost:80/tile/{z}/{x}/{y}.png`. The demo map in `leaflet-demo.html` will then be available on `http://localhost:80`. Note that it will initially quite a bit of time to render the larger tiles for the first time. ## Preserving rendered tiles Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To make sure that this data survives container restarts, you should create another volume for it: - +``` docker volume create openstreetmap-rendered-tiles - docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -v openstreetmap-rendered-tiles:/var/lib/mod_tile -d overv/openstreetmap-tile-server run + docker run -p 80:80 + -v openstreetmap-data:/var/lib/postgresql/10/main + -v openstreetmap-rendered-tiles:/var/lib/mod_tile + -d overv/openstreetmap-tile-server \ + run +``` +## Performance tuning and tweaking -## Performance tuning +### update and tile expire procedure + +The environment variable "UPDATES" (default value =disabled) allows you to run the container with consecutive updates. +``` +docker run -p 80:80 \ + -v openstreetmap-data:/var/lib/postgresql/10/main \ + -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ + -d overv/openstreetmap-tile-server \ + -e UPDATES=enabled \ + run +``` +Details for update procedure and invoked scripts can you find here [link](https://ircama.github.io/osm-carto-tutorials/updating-data/) ### THREADS The import and tile serving processes use 4 threads by default, but this number can be changed by setting the `THREADS` environment variable. For example: - +``` docker run -p 80:80 -e THREADS=24 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run - +``` ### AUTOVACUUM The database use the autovacuum feature by default. This behavior can be changed with `AUTOVACUUM` environment variable. For example: docker run -p 80:80 -e AUTOVACUUM=off -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server -## Benchmarks +### Benchmarks You can find an example of the import performance to expect with this image on the [OpenStreetMap wiki](https://wiki.openstreetmap.org/wiki/Osm2pgsql/benchmarks#debian_9_.2F_openstreetmap-tile-server). diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire new file mode 100644 index 0000000..50c3fa2 --- /dev/null +++ b/openstreetmap-tiles-update-expire @@ -0,0 +1,207 @@ +#!/bin/sh + +set -e + +#------------------------------------------------------------------------------ +# AJT - change directory to mod_tile directory so that we can run replag +# and other things directly from this script when run from cron. +# Change the actual location to wherever installed locally. +#------------------------------------------------------------------------------ +export PATH=.:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ACCOUNT=renderer +cd /home/$ACCOUNT/src/mod_tile/ + +#------------------------------------------------------------------------------ +# Extra OSM2PGSQL_OPTIONS may need setting if a tag transform script is +# in use. See https://github.com/SomeoneElseOSM/SomeoneElse-style and +# http://wiki.openstreetmap.org/wiki/User:SomeoneElse/Ubuntu_1404_tileserver_load +# The database name always needs setting. +#------------------------------------------------------------------------------ +OSMOSIS_BIN=osmosis +OSM2PGSQL_BIN=osm2pgsql +TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py + +DBNAME=gis +OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua -C 2048 --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style" + +#------------------------------------------------------------------------------ +# When using trim_osc.py we can define either a bounding box (such as this +# example for England and Wales) or a polygon. +# See https://github.com/zverik/regional . +# This area will usually correspond to the data originally loaded. +#------------------------------------------------------------------------------ +TRIM_POLY_FILE="/var/lib/mod_tile/data.poly" +TRIM_OPTIONS="-d $DBNAME" +#TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27" +TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE" + +BASE_DIR=/var/lib/mod_tile +LOG_DIR=/var/log/tiles/ +WORKOSM_DIR=$BASE_DIR/.osmosis + +LOCK_FILE=/tmp/openstreetmap-update-expire-lock.txt +CHANGE_FILE=$BASE_DIR/changes.osc.gz +EXPIRY_FILE=$BASE_DIR/dirty_tiles +STOP_FILE=$BASE_DIR/stop.txt + +OSMOSISLOG=$LOG_DIR/osmosis.log +PGSQLLOG=$LOG_DIR/osm2pgsql.log +EXPIRYLOG=$LOG_DIR/expiry.log +RUNLOG=$LOG_DIR/run.log + +#------------------------------------------------------------------------------ +# The tile expiry section below can re-render, delete or dirty expired tiles. +# By default, tiles between EXPIRY_MINZOOM and EXPIRY_MAXZOOM are rerendered. +# "render_expired" can optionally delete (and/or dirty) tiles above a certail +# threshold rather than rendering them. +# Here we expire (but don't immediately rerender) tiles between zoom levels +# 13 and 18 and delete between 19 and 20. +#------------------------------------------------------------------------------ +EXPIRY_MINZOOM=13 +EXPIRY_TOUCHFROM=13 +EXPIRY_DELETEFROM=19 +EXPIRY_MAXZOOM=20 + +#************************************************************************* +#************************************************************************* + +m_info() +{ + echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG" +} + +m_error() +{ + echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ [error] $1" >> "$RUNLOG" + + m_info "resetting state" + /bin/cp $WORKOSM_DIR/last.state.txt $WORKOSM_DIR/state.txt || true + + rm "$CHANGE_FILE" || true + rm "$EXPIRY_FILE.$$" || true + rm "$LOCK_FILE" + exit +} + +m_ok() +{ + echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG" +} + +getlock() +{ + if [ -s $1 ]; then + if [ "$(ps -p `cat $1` | wc -l)" -gt 1 ]; then + return 1 #false + fi + fi + + echo $$ >"$1" + return 0 #true +} + +freelock() +{ + rm "$1" + rm "$CHANGE_FILE" +} + + +if [ $# -eq 1 ] ; then + m_info "Initialising Osmosis replication system to $1" + mkdir $WORKOSM_DIR + $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" + wget "http://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt + mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt + sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt +else +# make sure the lockfile is removed when we exit and then claim it + + if ! getlock "$LOCK_FILE"; then + m_info "pid `cat $LOCK_FILE` still running" + exit 3 + fi + + if [ -e $STOP_FILE ]; then + m_info "stopped" + exit 2 + fi + +# ----------------------------------------------------------------------------- +# Add disk space check from https://github.com/zverik/regional +# ----------------------------------------------------------------------------- +MIN_DISK_SPACE_MB=500 + +if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_bavail*st.f_frsize/1024/1024 > $MIN_DISK_SPACE_MB else 0)"`; then + m_info "there is less than $MIN_DISK_SPACE_MB MB left" + exit 4 +fi + + seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2` + + m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`" + + /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt + m_ok "downloading diff" + + if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then + m_error "Osmosis error" + fi + +if [ -f TRIM_POLY_FILE ] ; then + m_ok "filtering diff" + /var/lib/mod_tile/data.poly + if ! $TRIM_BIN $TRIM_OPTIONS $TRIM_REGION_OPTIONS -z $CHANGE_FILE $CHANGE_FILE 1>&2 2>> "$RUNLOG"; then + m_error "Trim_osc error" + fi +else + m_ok "filtering diff skipped" +fi + m_ok "importing diff" +#------------------------------------------------------------------------------ +# Previously openstreetmap-tiles-update-expire tried to dirty layer +# "$EXPIRY_MAXZOOM - 3" (which was 15) only. Instead we write all expired +# tiles in range to the list (note the "-" rather than ":" in the "-e" +# parameter). +#------------------------------------------------------------------------------ + if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then + m_error "osm2pgsql error" + fi + +#------------------------------------------------------------------------------ +# The lockfile is normally removed before we expire tiles because that is +# something thatcan be done in parallel with further processing. In order to +# avoid rework, if actually rerendering is done rather than just deleting or +# dirtying, it makes sense to move it lower down. +#------------------------------------------------------------------------------ +# m_ok "Import complete; removing lock file" +# freelock "$LOCK_FILE" + + m_ok "expiring tiles" +#------------------------------------------------------------------------------ +# When expiring tiles we need to define the style sheet if it's not "default". +# In this case it's "ajt". +# Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and +# $EXPIRY_MAXZOOM were dirtied. We currently re-render +# tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and +# delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM. +# The default path to renderd.sock is fixed. +#------------------------------------------------------------------------------ + if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then + m_info "Expiry failed" + fi + + rm "$EXPIRY_FILE.$$" + +#------------------------------------------------------------------------------ +# Only remove the lock file after expiry (if system is slow we want to delay +# the next import, not have multiple render_expired processes running) +#------------------------------------------------------------------------------ + freelock "$LOCK_FILE" + + m_ok "Done with import" + + + + +fi diff --git a/run.sh b/run.sh index d7c0d01..643226a 100755 --- a/run.sh +++ b/run.sh @@ -16,6 +16,7 @@ if [ "$#" -ne 1 ]; then echo " run: Runs Apache and renderd to serve tiles at /tile/{z}/{x}/{y}.png" echo "environment variables:" echo " THREADS: defines number of threads used for importing / tile rendering" + echo " UPDATES: consecutive updates (enabled/disabled)" exit 1 fi @@ -34,6 +35,20 @@ if [ "$1" = "import" ]; then if [ ! -f /data.osm.pbf ]; then echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..." wget -nv http://download.geofabrik.de/europe/luxembourg-latest.osm.pbf -O /data.osm.pbf + wget -nv http://download.geofabrik.de/europe/luxembourg.poly -O /data.poly + fi + + # determine and set osmosis_replication_timestamp (for consecutive updates) + osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info + osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt + REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt) + + # initial setup of osmosis workspace (for consecutive updates) + sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP + + # copy polygon file if available + if [ -f /data.poly ]; then + sudo -u renderer cp /data.poly /var/lib/mod_tile/data.poly fi # Import data @@ -52,6 +67,11 @@ if [ "$1" = "run" ]; then # Configure renderd threads sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf + # start cron job to trigger consecutive updates + if [ "$UPDATES" = "enabled" ]; then + /etc/init.d/cron start + fi + # Run sudo -u renderer renderd -f -c /usr/local/etc/renderd.conf service postgresql stop