From 8750c004b3553c1d9ecd09cca9fcc126b9066eb8 Mon Sep 17 00:00:00 2001 From: galewis2 <62433564+galewis2@users.noreply.github.com> Date: Tue, 14 Dec 2021 08:57:23 -0500 Subject: [PATCH 1/9] Get Osmium --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b437be5..c6fd085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -139,7 +139,8 @@ RUN adduser --disabled-password --gecos "" renderer # Install python libraries RUN pip3 install \ requests \ - pyyaml + pyyaml \ + osmium # Configure Apache RUN mkdir /var/lib/mod_tile \ From 1421c963b2cf36fa3790c633539d9b9bcf340172 Mon Sep 17 00:00:00 2001 From: galewis2 <62433564+galewis2@users.noreply.github.com> Date: Tue, 14 Dec 2021 16:24:45 -0500 Subject: [PATCH 2/9] Update run.sh --- run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run.sh b/run.sh index 454239d..279e660 100755 --- a/run.sh +++ b/run.sh @@ -62,7 +62,7 @@ if [ "$1" = "import" ]; then if [ "${UPDATES:-}" = "enabled" ]; then # determine and set osmosis_replication_timestamp (for consecutive updates) osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info - osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt + osmium fileinfo --get=header.option.timestamp /data.osm.pbf > /var/lib/mod_tile/replication_timestamp.txt REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt) # initial setup of osmosis workspace (for consecutive updates) From 33ea447ba20ee7b31cbec611e4e9b827498ca84f Mon Sep 17 00:00:00 2001 From: galewis2 <62433564+galewis2@users.noreply.github.com> Date: Tue, 14 Dec 2021 16:25:23 -0500 Subject: [PATCH 3/9] Update openstreetmap-tiles-update-expire --- openstreetmap-tiles-update-expire | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire index 4a0e7e4..724cc09 100644 --- a/openstreetmap-tiles-update-expire +++ b/openstreetmap-tiles-update-expire @@ -107,9 +107,13 @@ freelock() if [ $# -eq 1 ] ; then m_info "Initialising Osmosis replication system to $1" - mkdir $WORKOSM_DIR + mkdir -p $WORKOSM_DIR $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" - wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt + + init_seq=$(pyosmium-get-changes --server https://planet.openstreetmap.org/replication/minute/ -D $1) + url_dynamicPart=$(printf %09d $init_seq | sed 's_\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)_\1/\2/\3_') + wget https://planet.openstreetmap.org/replication/minute/$url_dynamicPart.state.txt -O $WORKOSM_DIR/state.txt + mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt else @@ -167,12 +171,12 @@ fi #------------------------------------------------------------------------------ # The lockfile is normally removed before we expire tiles because that is -# something thatcan be done in parallel with further processing. In order to +# something that can be done in parallel with further processing. In order to # avoid rework, if actually rerendering is done rather than just deleting or # dirtying, it makes sense to move it lower down. #------------------------------------------------------------------------------ -# m_ok "Import complete; removing lock file" -# freelock "$LOCK_FILE" +# m_ok "Import complete; removing lock file" +# freelock "$LOCK_FILE" m_ok "expiring tiles" #------------------------------------------------------------------------------ @@ -194,11 +198,8 @@ fi # Only remove the lock file after expiry (if system is slow we want to delay # the next import, not have multiple render_expired processes running) #------------------------------------------------------------------------------ - freelock "$LOCK_FILE" + freelock "$LOCK_FILE" m_ok "Done with import" - - - fi From 630e37b4b1d42573921e5e2f7f34b66f8ef9149c Mon Sep 17 00:00:00 2001 From: galewis2 <62433564+galewis2@users.noreply.github.com> Date: Wed, 15 Dec 2021 12:54:10 -0500 Subject: [PATCH 4/9] You need the openstreetmap-rendered-tiles volume during import to ensure update process is setup correctly --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 4d6f47b..23e7f9e 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ docker run \ -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ -v /absolute/path/to/luxembourg.poly:/data.poly \ -v openstreetmap-data:/var/lib/postgresql/12/main \ + -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ overv/openstreetmap-tile-server \ import ``` From 89d238b2d0b264e0a317e76789a7b6f34ecf4f60 Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 20 Dec 2021 14:00:14 +0000 Subject: [PATCH 5/9] switch to using replication_url --- README.md | 3 ++- openstreetmap-tiles-update-expire | 11 +++++++---- run.sh | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) mode change 100644 => 100755 openstreetmap-tiles-update-expire diff --git a/README.md b/README.md index 23e7f9e..91f6963 100644 --- a/README.md +++ b/README.md @@ -29,10 +29,10 @@ If your import is an extract of the planet and has polygonal bounds associated w ``` docker run \ -e UPDATES=enabled \ + -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \ -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ -v /absolute/path/to/luxembourg.poly:/data.poly \ -v openstreetmap-data:/var/lib/postgresql/12/main \ - -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ overv/openstreetmap-tile-server \ import ``` @@ -93,6 +93,7 @@ Given that you've set up your import as described in the *Automatic updates* sec ``` docker run \ -p 8080:80 \ + -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \ -e UPDATES=enabled \ -v openstreetmap-data:/var/lib/postgresql/12/main \ -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire old mode 100644 new mode 100755 index 724cc09..1130b8c --- a/openstreetmap-tiles-update-expire +++ b/openstreetmap-tiles-update-expire @@ -110,12 +110,15 @@ if [ $# -eq 1 ] ; then mkdir -p $WORKOSM_DIR $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" - init_seq=$(pyosmium-get-changes --server https://planet.openstreetmap.org/replication/minute/ -D $1) + init_seq=$(pyosmium-get-changes --server $REPLICATION_URL -D $1) url_dynamicPart=$(printf %09d $init_seq | sed 's_\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)_\1/\2/\3_') - wget https://planet.openstreetmap.org/replication/minute/$url_dynamicPart.state.txt -O $WORKOSM_DIR/state.txt + wget $REPLICATION_URL/$url_dynamicPart.state.txt -O $WORKOSM_DIR/state.txt + + cat > /var/lib/mod_tile/.osmosis/configuration.txt <<- EOM +baseURL=$REPLICATION_URL +maxInterval=86400 +EOM - mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt - sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt else # make sure the lockfile is removed when we exit and then claim it diff --git a/run.sh b/run.sh index 279e660..6129268 100755 --- a/run.sh +++ b/run.sh @@ -67,6 +67,7 @@ if [ "$1" = "import" ]; then # initial setup of osmosis workspace (for consecutive updates) sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP + fi # copy polygon file if available From 6d2eaa23a500cf5b29661b24e8c20b0556a13cf8 Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 20 Dec 2021 17:38:41 +0000 Subject: [PATCH 6/9] update --- Dockerfile | 2 ++ README.md | 3 +++ openstreetmap-tiles-update-expire | 3 +-- run.sh | 4 +++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6fd085..1e1fbeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -161,6 +161,8 @@ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \ && mkdir /var/log/tiles \ && chmod a+rw /var/log/tiles \ && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \ +&& echo "SHELL=/bin/bash" > /etc/crontab \ +&& echo "BASH_ENV=/container.env" >> /etc/crontab \ && echo "* * * * * renderer openstreetmap-tiles-update-expire\n" >> /etc/crontab RUN mkdir /nodes \ diff --git a/README.md b/README.md index 91f6963..9551c2f 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,11 @@ If your import is an extract of the planet and has polygonal bounds associated w docker run \ -e UPDATES=enabled \ -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \ + -e MAX_INTERVAL_SECONDS=60 \ -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ -v /absolute/path/to/luxembourg.poly:/data.poly \ -v openstreetmap-data:/var/lib/postgresql/12/main \ + -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ overv/openstreetmap-tile-server \ import ``` @@ -94,6 +96,7 @@ Given that you've set up your import as described in the *Automatic updates* sec docker run \ -p 8080:80 \ -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \ + -e MAX_INTERVAL_SECONDS=60 \ -e UPDATES=enabled \ -v openstreetmap-data:/var/lib/postgresql/12/main \ -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire index 1130b8c..3ef443f 100755 --- a/openstreetmap-tiles-update-expire +++ b/openstreetmap-tiles-update-expire @@ -30,7 +30,6 @@ OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/ #------------------------------------------------------------------------------ TRIM_POLY_FILE="/var/lib/mod_tile/data.poly" TRIM_OPTIONS="-d $DBNAME" -#TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27" TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE" BASE_DIR=/var/lib/mod_tile @@ -116,7 +115,7 @@ if [ $# -eq 1 ] ; then cat > /var/lib/mod_tile/.osmosis/configuration.txt <<- EOM baseURL=$REPLICATION_URL -maxInterval=86400 +maxInterval=$MAX_INTERVAL_SECONDS EOM else diff --git a/run.sh b/run.sh index 6129268..9bb175a 100755 --- a/run.sh +++ b/run.sh @@ -2,6 +2,8 @@ set -euo pipefail +declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env + function createPostgresConfig() { cp /etc/postgresql/12/main/postgresql.custom.conf.tmpl /etc/postgresql/12/main/conf.d/postgresql.custom.conf sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/12/main/conf.d/postgresql.custom.conf @@ -66,7 +68,7 @@ if [ "$1" = "import" ]; then REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt) # initial setup of osmosis workspace (for consecutive updates) - sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP + sudo -E -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP fi From 3deaedabfac968202930f03c7472c70a010c5b67 Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 20 Dec 2021 17:50:48 +0000 Subject: [PATCH 7/9] verbose --- openstreetmap-tiles-update-expire | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire index 3ef443f..0655334 100755 --- a/openstreetmap-tiles-update-expire +++ b/openstreetmap-tiles-update-expire @@ -107,7 +107,7 @@ freelock() if [ $# -eq 1 ] ; then m_info "Initialising Osmosis replication system to $1" mkdir -p $WORKOSM_DIR - $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" + $OSMOSIS_BIN -v --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" init_seq=$(pyosmium-get-changes --server $REPLICATION_URL -D $1) url_dynamicPart=$(printf %09d $init_seq | sed 's_\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)_\1/\2/\3_') @@ -148,7 +148,7 @@ fi /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt m_ok "downloading diff" - if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then + if ! $OSMOSIS_BIN -v --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then m_error "Osmosis error" fi From 0ffff3d6333387ae92b509b7beba1a8075bb920e Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 20 Dec 2021 19:51:19 +0000 Subject: [PATCH 8/9] fix spelling --- openstreetmap-tiles-update-expire | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire index 0655334..eafd7d2 100755 --- a/openstreetmap-tiles-update-expire +++ b/openstreetmap-tiles-update-expire @@ -107,14 +107,14 @@ freelock() if [ $# -eq 1 ] ; then m_info "Initialising Osmosis replication system to $1" mkdir -p $WORKOSM_DIR - $OSMOSIS_BIN -v --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" + $OSMOSIS_BIN -v 5 --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" init_seq=$(pyosmium-get-changes --server $REPLICATION_URL -D $1) url_dynamicPart=$(printf %09d $init_seq | sed 's_\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)_\1/\2/\3_') wget $REPLICATION_URL/$url_dynamicPart.state.txt -O $WORKOSM_DIR/state.txt cat > /var/lib/mod_tile/.osmosis/configuration.txt <<- EOM -baseURL=$REPLICATION_URL +baseUrl=$REPLICATION_URL maxInterval=$MAX_INTERVAL_SECONDS EOM @@ -148,7 +148,7 @@ fi /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt m_ok "downloading diff" - if ! $OSMOSIS_BIN -v --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then + if ! $OSMOSIS_BIN -v 5 --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then m_error "Osmosis error" fi From 9936271362641b996fe4f49d4da239f5de712247 Mon Sep 17 00:00:00 2001 From: Greg Date: Mon, 20 Dec 2021 20:18:52 +0000 Subject: [PATCH 9/9] provide defaults --- openstreetmap-tiles-update-expire | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire index eafd7d2..e752850 100755 --- a/openstreetmap-tiles-update-expire +++ b/openstreetmap-tiles-update-expire @@ -59,6 +59,10 @@ EXPIRY_TOUCHFROM=13 EXPIRY_DELETEFROM=19 EXPIRY_MAXZOOM=20 + +REPLICATION_URL=${REPLICATION_URL:="https://planet.openstreetmap.org/replication/hour/"} +MAX_INTERVAL_SECONDS=${MAX_INTERVAL_SECONDS:="3600"} + #************************************************************************* #*************************************************************************