integration of update and tile expire procedure

see https://ircama.github.io/osm-carto-tutorials/updating-data/

Implements https://github.com/Overv/openstreetmap-tile-server/issues/2

Dockerfile

@@ -6,6 +6,7 @@ FROM ubuntu:18.04
 # Set up environment
 ENV TZ=UTC
 ENV AUTOVACUUM=on
+ENV UPDATES=disabled
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 # Install dependencies
@@ -63,6 +64,10 @@ RUN echo "deb [ allow-insecure=yes ] http://apt.postgresql.org/pub/repos/apt/ bi
   unzip \
   wget \
   zlib1g-dev \
+  osmosis \
+  osmium-tool \
+  cron \
+  python-psycopg2 python-shapely python-lxml \
 && apt-get clean autoclean \
 && apt-get autoremove --yes \
 && rm -rf /var/lib/{apt,dpkg,cache,log}/
@@ -138,7 +143,22 @@ RUN chown -R postgres:postgres /var/lib/postgresql \
   && chown postgres:postgres /etc/postgresql/10/main/postgresql.custom.conf.tmpl \
   && echo "\ninclude 'postgresql.custom.conf'" >> /etc/postgresql/10/main/postgresql.conf
 
+# copy update scripts
+COPY openstreetmap-tiles-update-expire /usr/bin/
+RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \
+    && mkdir /var/log/tiles \
+    && chmod a+rw /var/log/tiles \
+    && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \
+    && echo "*  *    * * *   renderer    openstreetmap-tiles-update-expire\n" >> /etc/crontab
+
+# install trim_osc.py helper script
+USER renderer
+RUN cd ~/src \
+    && git clone https://github.com/zverik/regional \
+    && chmod u+x ~/src/regional/trim_osc.py
+
 # Start running
+USER root
 COPY run.sh /
 ENTRYPOINT ["/run.sh"]
 CMD []

README.md

@@ -9,41 +9,74 @@ First create a Docker volume to hold the PostgreSQL database that will contain t
     docker volume create openstreetmap-data
 
 Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example:
-
+```
-    docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main overv/openstreetmap-tile-server import
+    docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
-
+               -v openstreetmap-data:/var/lib/postgresql/10/main \
+               overv/openstreetmap-tile-server \
+               import
+```
 If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server.
 
+### optional step to allow consecutive updates of osm extracts
+
+if your import is a extract of planet and is based on polygon then you should download this polynom data and use follwing command for the import procedure:
+```
+    docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
+               -v /absolute/path/to/luxembourg.poly:/data.poly \
+               -v /openstreetmap-data:/var/lib/postgresql/10/main \
+               overv/openstreetmap-tile-server \
+               import
+```
 ## Running the server
 
 Run the server like this:
-
+```
-    docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run
+    docker run -p 80:80 \
-
+               -v openstreetmap-data:/var/lib/postgresql/10/main \
+               -d overv/openstreetmap-tile-server \
+               run
+```
 Your tiles will now be available at `http://localhost:80/tile/{z}/{x}/{y}.png`. The demo map in `leaflet-demo.html` will then be available on `http://localhost:80`. Note that it will initially quite a bit of time to render the larger tiles for the first time.
 
 ## Preserving rendered tiles
 
 Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To make sure that this data survives container restarts, you should create another volume for it:
-
+```
     docker volume create openstreetmap-rendered-tiles
-    docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -v openstreetmap-rendered-tiles:/var/lib/mod_tile -d overv/openstreetmap-tile-server run
+    docker run -p 80:80
+               -v openstreetmap-data:/var/lib/postgresql/10/main
+               -v openstreetmap-rendered-tiles:/var/lib/mod_tile
+               -d overv/openstreetmap-tile-server \
+               run
+```
+## Performance tuning and tweaking
 
-## Performance tuning
+### update and tile expire procedure
+
+The environment variable "UPDATES" (default value =disabled) allows you to run the container with consecutive updates.
+```
+docker run -p 80:80 \
+           -v openstreetmap-data:/var/lib/postgresql/10/main \
+           -v openstreetmap-rendered-tiles:/var/lib/mod_tile \
+           -d overv/openstreetmap-tile-server \
+           -e UPDATES=enabled \
+           run
+```
+Details for update procedure and invoked scripts can you find here [link](https://ircama.github.io/osm-carto-tutorials/updating-data/)
 
 ### THREADS
 
 The import and tile serving processes use 4 threads by default, but this number can be changed by setting the `THREADS` environment variable. For example:
-
+```
     docker run -p 80:80 -e THREADS=24 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run
-
+```
 ### AUTOVACUUM
 
 The database use the autovacuum feature by default. This behavior can be changed with `AUTOVACUUM` environment variable. For example:
 
     docker run -p 80:80 -e AUTOVACUUM=off -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server
 
-## Benchmarks
+### Benchmarks
 
 You can find an example of the import performance to expect with this image on the [OpenStreetMap wiki](https://wiki.openstreetmap.org/wiki/Osm2pgsql/benchmarks#debian_9_.2F_openstreetmap-tile-server).
 

openstreetmap-tiles-update-expire

@@ -0,0 +1,207 @@
+#!/bin/sh
+
+set -e
+
+#------------------------------------------------------------------------------
+# AJT - change directory to mod_tile directory so that we can run replag
+# and other things directly from this script when run from cron.
+# Change the actual location to wherever installed locally.
+#------------------------------------------------------------------------------
+export PATH=.:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+ACCOUNT=renderer
+cd /home/$ACCOUNT/src/mod_tile/
+
+#------------------------------------------------------------------------------
+# Extra OSM2PGSQL_OPTIONS may need setting if a tag transform script is
+# in use.  See https://github.com/SomeoneElseOSM/SomeoneElse-style and
+# http://wiki.openstreetmap.org/wiki/User:SomeoneElse/Ubuntu_1404_tileserver_load
+# The database name always needs setting.
+#------------------------------------------------------------------------------
+OSMOSIS_BIN=osmosis
+OSM2PGSQL_BIN=osm2pgsql
+TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py
+
+DBNAME=gis
+OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua -C 2048 --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style"
+
+#------------------------------------------------------------------------------
+# When using trim_osc.py we can define either a bounding box (such as this
+# example for England and Wales) or a polygon.
+# See https://github.com/zverik/regional .
+# This area will usually correspond to the data originally loaded.
+#------------------------------------------------------------------------------
+TRIM_POLY_FILE="/var/lib/mod_tile/data.poly"
+TRIM_OPTIONS="-d $DBNAME"
+#TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27"
+TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE"
+
+BASE_DIR=/var/lib/mod_tile
+LOG_DIR=/var/log/tiles/
+WORKOSM_DIR=$BASE_DIR/.osmosis
+
+LOCK_FILE=/tmp/openstreetmap-update-expire-lock.txt
+CHANGE_FILE=$BASE_DIR/changes.osc.gz
+EXPIRY_FILE=$BASE_DIR/dirty_tiles
+STOP_FILE=$BASE_DIR/stop.txt
+
+OSMOSISLOG=$LOG_DIR/osmosis.log
+PGSQLLOG=$LOG_DIR/osm2pgsql.log
+EXPIRYLOG=$LOG_DIR/expiry.log
+RUNLOG=$LOG_DIR/run.log
+
+#------------------------------------------------------------------------------
+# The tile expiry section below can re-render, delete or dirty expired tiles.
+# By default, tiles between EXPIRY_MINZOOM and EXPIRY_MAXZOOM are rerendered.
+# "render_expired" can optionally delete (and/or dirty) tiles above a certail
+# threshold rather than rendering them.
+# Here we expire (but don't immediately rerender) tiles between zoom levels
+# 13 and 18 and delete between 19 and 20.
+#------------------------------------------------------------------------------
+EXPIRY_MINZOOM=13
+EXPIRY_TOUCHFROM=13
+EXPIRY_DELETEFROM=19
+EXPIRY_MAXZOOM=20
+
+#*************************************************************************
+#*************************************************************************
+
+m_info()
+{
+        echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG"
+}
+
+m_error()
+{
+    echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ [error] $1" >> "$RUNLOG"
+
+    m_info "resetting state"
+    /bin/cp $WORKOSM_DIR/last.state.txt $WORKOSM_DIR/state.txt || true
+
+    rm "$CHANGE_FILE" || true
+    rm "$EXPIRY_FILE.$$" || true
+    rm "$LOCK_FILE"
+    exit
+}
+
+m_ok()
+{
+    echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG"
+}
+
+getlock()
+{
+    if [ -s $1 ]; then
+        if [ "$(ps -p `cat $1` | wc -l)" -gt 1 ]; then
+            return 1 #false
+        fi
+    fi
+
+    echo $$ >"$1"
+    return 0 #true
+}
+
+freelock()
+{
+    rm "$1"
+    rm "$CHANGE_FILE"
+}
+
+
+if [ $# -eq 1 ] ; then
+    m_info "Initialising Osmosis replication system to $1"
+    mkdir $WORKOSM_DIR
+    $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"
+    wget "http://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt
+    mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt
+    sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt
+else
+# make sure the lockfile is removed when we exit and then claim it
+
+    if ! getlock "$LOCK_FILE"; then
+        m_info "pid `cat $LOCK_FILE` still running"
+        exit 3
+    fi
+
+    if [ -e $STOP_FILE ]; then
+        m_info "stopped"
+        exit 2
+    fi
+
+# -----------------------------------------------------------------------------
+# Add disk space check from https://github.com/zverik/regional
+# -----------------------------------------------------------------------------
+MIN_DISK_SPACE_MB=500
+
+if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_bavail*st.f_frsize/1024/1024 > $MIN_DISK_SPACE_MB else 0)"`; then
+    m_info "there is less than $MIN_DISK_SPACE_MB MB left"
+    exit 4
+fi
+
+    seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2`
+
+    m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`"
+
+    /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
+    m_ok "downloading diff"
+
+    if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then
+        m_error "Osmosis error"
+    fi
+
+if [ -f TRIM_POLY_FILE ] ; then
+  m_ok "filtering diff"
+  /var/lib/mod_tile/data.poly
+  if ! $TRIM_BIN $TRIM_OPTIONS $TRIM_REGION_OPTIONS  -z $CHANGE_FILE $CHANGE_FILE 1>&2 2>> "$RUNLOG"; then
+      m_error "Trim_osc error"
+  fi
+else
+  m_ok "filtering diff skipped"
+fi
+    m_ok "importing diff"
+#------------------------------------------------------------------------------
+# Previously openstreetmap-tiles-update-expire tried to dirty layer
+# "$EXPIRY_MAXZOOM - 3" (which was 15) only.  Instead we write all expired
+# tiles in range to the list (note the "-" rather than ":" in the "-e"
+# parameter).
+#------------------------------------------------------------------------------
+    if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then
+        m_error "osm2pgsql error"
+    fi
+
+#------------------------------------------------------------------------------
+# The lockfile is normally removed before we expire tiles because that is
+# something thatcan be done in parallel with further processing.  In order to
+# avoid rework, if actually rerendering is done rather than just deleting or
+# dirtying, it makes sense to move it lower down.
+#------------------------------------------------------------------------------
+#    m_ok "Import complete; removing lock file"
+#    freelock "$LOCK_FILE"
+
+    m_ok "expiring tiles"
+#------------------------------------------------------------------------------
+# When expiring tiles we need to define the style sheet if it's not "default".
+# In this case it's "ajt".
+# Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and
+# $EXPIRY_MAXZOOM were dirtied.  We currently re-render
+# tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and
+# delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM.
+# The default path to renderd.sock is fixed.
+#------------------------------------------------------------------------------
+    if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
+        m_info "Expiry failed"
+    fi
+
+    rm "$EXPIRY_FILE.$$"
+
+#------------------------------------------------------------------------------
+# Only remove the lock file after expiry (if system is slow we want to delay
+# the next import, not have multiple render_expired processes running)
+#------------------------------------------------------------------------------
+   freelock "$LOCK_FILE"
+
+    m_ok "Done with import"
+
+
+
+
+fi

run.sh

@@ -16,6 +16,7 @@ if [ "$#" -ne 1 ]; then
     echo "    run: Runs Apache and renderd to serve tiles at /tile/{z}/{x}/{y}.png"
     echo "environment variables:"
     echo "    THREADS: defines number of threads used for importing / tile rendering"
+    echo "    UPDATES: consecutive updates (enabled/disabled)"
     exit 1
 fi
 
@@ -34,6 +35,20 @@ if [ "$1" = "import" ]; then
     if [ ! -f /data.osm.pbf ]; then
         echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..."
         wget -nv http://download.geofabrik.de/europe/luxembourg-latest.osm.pbf -O /data.osm.pbf
+        wget -nv http://download.geofabrik.de/europe/luxembourg.poly -O /data.poly
+    fi
+
+    # determine and set osmosis_replication_timestamp (for consecutive updates)
+    osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info
+    osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt
+    REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt)
+
+    # initial setup of osmosis workspace (for consecutive updates)
+    sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP
+
+    # copy polygon file if available
+    if [ -f /data.poly ]; then
+        sudo -u renderer cp /data.poly /var/lib/mod_tile/data.poly
     fi
 
     # Import data
@@ -52,6 +67,11 @@ if [ "$1" = "run" ]; then
     # Configure renderd threads
     sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf
 
+    # start cron job to trigger consecutive updates
+    if [ "$UPDATES" = "enabled" ]; then
+      /etc/init.d/cron start
+    fi
+
     # Run
     sudo -u renderer renderd -f -c /usr/local/etc/renderd.conf
     service postgresql stop