changed volume paths ; FLAT_NODES=enabled (#261)

Changes:
- /var/lib/postgresql/14/main/            => /data/database/
- /var/lib/mod_tile/                      => /data/tiles/
- /home/renderer/src/openstreetmap-carto/ => /data/style/
- /data.osm.pbf                           => /data/region.osm.pbf
- /data.poly                              => /data/region.poly
- /nodes/flat_nodes.bin                   => deprecated (use FLAT_NODES=enabled instead)

Because some files are needed later and should never be separated from the database, the import does save those files now into /data/databases/:
- /data/database/region.poly
- /data/database/flat_nodes.bin
- /data/database/planet-import-complete
- and automatic update artifacts

.github/workflows/build-and-test.yaml

@@ -26,7 +26,7 @@ jobs:
     env:
       VOLUME    : osm-db
       CONTAINER : osm-www
-      MOUNT     : /var/lib/postgresql/14/main
+      MOUNT     : /data/database/
     steps:
     -
       name: Checkout

.travis.yml

@@ -10,9 +10,9 @@ before_script:
 - docker pull overv/openstreetmap-tile-server || true
 script:
 - docker build --pull --cache-from overv/openstreetmap-tile-server --tag overv/openstreetmap-tile-server .
-- docker volume create openstreetmap-data
+- docker volume create osm-data
-- docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main overv/openstreetmap-tile-server import
+- docker run --rm -v osm-data:/data/database/ overv/openstreetmap-tile-server import
-- docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main -p 8080:80 -d overv/openstreetmap-tile-server run
+- docker run --rm -v osm-data:/data/database/ -p 8080:80 -d overv/openstreetmap-tile-server run
 - sleep 30
 - make DOCKER_IMAGE=overv/openstreetmap-tile-server stop
 after_script:

Dockerfile

@@ -97,7 +97,7 @@ RUN mkdir -p /home/renderer/src \
 
 ###########################################################################################################
 
-FROM ubuntu:20.04 AS final-base
+FROM ubuntu:20.04 AS final
 
 # Based on
 # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
@@ -164,15 +164,12 @@ RUN ln -sf /dev/stdout /var/log/apache2/access.log \
 && ln -sf /dev/stderr /var/log/apache2/error.log
 
 # Copy update scripts
-COPY openstreetmap-tiles-update-expire /usr/bin/
+COPY openstreetmap-tiles-update-expire.sh /usr/bin/
-RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \
+RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \
 && mkdir /var/log/tiles \
 && chmod a+rw /var/log/tiles \
 && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \
-&& echo "* * * * *   renderer    openstreetmap-tiles-update-expire\n" >> /etc/crontab
+&& echo "* * * * *   renderer    openstreetmap-tiles-update-expire.sh\n" >> /etc/crontab
-
-RUN mkdir /nodes \
-&& chown renderer:renderer /nodes
 
 # Configure PosgtreSQL
 COPY postgresql.custom.conf.tmpl /etc/postgresql/14/main/
@@ -181,9 +178,18 @@ RUN chown -R postgres:postgres /var/lib/postgresql \
 && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \
 && echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf
 
-###########################################################################################################
+# Create volume directories
-
+RUN   mkdir  -p  /data/database/  \
-FROM final-base AS final
+  &&  mkdir  -p  /data/style/  \
+  &&  mkdir  -p  /home/renderer/src/  \
+  &&  chown  -R  renderer:  /data/  \
+  &&  chown  -R  renderer:  /home/renderer/src/  \
+  &&  mv  /var/lib/postgresql/14/main/  /data/database/postgres/  \
+  &&  mv  /var/lib/mod_tile/            /data/tiles/     \
+  &&  ln  -s  /data/database/postgres  /var/lib/postgresql/14/main             \
+  &&  ln  -s  /data/style              /home/renderer/src/openstreetmap-carto  \
+  &&  ln  -s  /data/tiles              /var/lib/mod_tile                       \
+;
 
 # Install PostGIS
 COPY --from=compiler-postgis postgis_src/postgis-src_3.2.1-1_amd64.deb .
@@ -206,7 +212,6 @@ RUN dpkg -i renderd_1-1_amd64.deb \
 # Install mod_tile
 COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb .
 RUN dpkg -i mod-tile_1-1_amd64.deb \
- && mkdir -p /home/renderer/src/openstreetmap-carto \
  && ldconfig \
  && rm mod-tile_1-1_amd64.deb
 

Makefile

@@ -9,10 +9,10 @@ push: build
 	docker push ${DOCKER_IMAGE}:latest
 
 test: build
-	docker volume create openstreetmap-data
+	docker volume create osm-data
-	docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main ${DOCKER_IMAGE} import
+	docker run --rm -v osm-data:/data/database/ ${DOCKER_IMAGE} import
-	docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main -p 8080:80 -d ${DOCKER_IMAGE} run
+	docker run --rm -v osm-data:/data/database/ -p 8080:80 -d ${DOCKER_IMAGE} run
 
 stop:
 	docker rm -f `docker ps | grep '${DOCKER_IMAGE}' | awk '{ print $$1 }'` || true
-	docker volume rm -f openstreetmap-data
+	docker volume rm -f osm-data

README.md

@@ -9,41 +9,43 @@ This container allows you to easily set up an OpenStreetMap PNG tile server give
 
 First create a Docker volume to hold the PostgreSQL database that will contain the OpenStreetMap data:
 
-    docker volume create openstreetmap-data
+    docker volume create osm-data
 
-Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example:
+Next, download an `.osm.pbf` extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data/region.osm.pbf`. For example:
 
 ```
 docker run \
-    -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
+    -v /absolute/path/to/luxembourg.osm.pbf:/data/region.osm.pbf \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     overv/openstreetmap-tile-server \
     import
 ```
 
 If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server.
 
-Note that the import process requires an internet connection. The run process does not require an internet connection. If you want to run the openstreetmap-tile server on a computer that is isolated, you must first import on an internet connected computer, export the openstreetmap-data volume as a tarfile, and then restore the data volume on the target computer system.
+Note that the import process requires an internet connection. The run process does not require an internet connection. If you want to run the openstreetmap-tile server on a computer that is isolated, you must first import on an internet connected computer, export the `osm-data` volume as a tarfile, and then restore the data volume on the target computer system.
 
-Also when running on an isolated system, the default index.html from the container will not work, as it requires access to the web for the leaflet packages. 
+Also when running on an isolated system, the default `index.html` from the container will not work, as it requires access to the web for the leaflet packages.
 
 ### Automatic updates (optional)
 
-If your import is an extract of the planet and has polygonal bounds associated with it, like those from geofabrik.de, then it is possible to set your server up for automatic updates. Make sure to reference both the OSM file and the polygon file during the import process to facilitate this, and also include the `UPDATES=enabled` variable:
+If your import is an extract of the planet and has polygonal bounds associated with it, like those from [geofabrik.de](https://download.geofabrik.de/), then it is possible to set your server up for automatic updates. Make sure to reference both the OSM file and the polygon file during the `import` process to facilitate this, and also include the `UPDATES=enabled` variable:
 
 ```
 docker run \
     -e UPDATES=enabled \
-    -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
+    -v /absolute/path/to/luxembourg.osm.pbf:/data/region.osm.pbf \
-    -v /absolute/path/to/luxembourg.poly:/data.poly \
+    -v /absolute/path/to/luxembourg.poly:/data/region.poly \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
-    -v openstreetmap-rendered-tiles:/var/lib/mod_tile \
     overv/openstreetmap-tile-server \
     import
 ```
 
 Refer to the section *Automatic updating and tile expiry* to actually enable the updates while running the tile server.
 
+Please note: If you're not importing the whole planet, then the `.poly` file is necessary to limit automatic updates to the relevant region.
+Therefore, when you only have a `.osm.pbf` file but not a `.poly` file, you should not enable automatic updates.
+
 ### Letting the container download the file
 
 It is also possible to let the container download files for you rather than mounting them in advance by using the `DOWNLOAD_PBF` and `DOWNLOAD_POLY` parameters:
@@ -52,7 +54,7 @@ It is also possible to let the container download files for you rather than moun
 docker run \
     -e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \
     -e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     overv/openstreetmap-tile-server \
     import
 ```
@@ -69,15 +71,15 @@ docker run \
     -e NAME_STYLE=test.style
     -e NAME_MML=project.mml
     -e NAME_SQL=test.sql
-    -v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto \
+    -v /home/user/openstreetmap-carto-modified:/data/style/ \
-    -v openstreetmap-data:/var/lib/postgresql/12/main \
+    -v osm-data:/data/database/ \
     overv/openstreetmap-tile-server \
     import
 ```
 
 If you do not define the "NAME_*" variables, the script will default to those found in the openstreetmap-carto style.
 
-Be sure to mount the volume during `run` with the same `-v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto`
+Be sure to mount the volume during `run` with the same `-v /home/user/openstreetmap-carto-modified:/data/style/`
 
 If you do not see the expected style upon `run` double check your paths as the style may not have been found at the directory specified. By default, `openstreetmap-carto` will be used if a style cannot be found
 
@@ -90,7 +92,7 @@ Run the server like this:
 ```
 docker run \
     -p 8080:80 \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     -d overv/openstreetmap-tile-server \
     run
 ```
@@ -103,19 +105,19 @@ The `docker-compose.yml` file included with this repository shows how the aforem
 
 ### Preserving rendered tiles
 
-Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To make sure that this data survives container restarts, you should create another volume for it:
+Tiles that have already been rendered will be stored in `/data/tiles/`. To make sure that this data survives container restarts, you should create another volume for it:
 
 ```
-docker volume create openstreetmap-rendered-tiles
+docker volume create osm-tiles
 docker run \
     -p 8080:80 \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
-    -v openstreetmap-rendered-tiles:/var/lib/mod_tile \
+    -v osm-tiles:/data/tiles/ \
     -d overv/openstreetmap-tile-server \
     run
 ```
 
-**If you do this, then make sure to also run the import with the `openstreetmap-rendered-tiles` volume to make sure that caching works properly across updates!**
+**If you do this, then make sure to also run the import with the `osm-tiles` volume to make sure that caching works properly across updates!**
 
 ### Enabling automatic updating (optional)
 
@@ -125,8 +127,8 @@ Given that you've set up your import as described in the *Automatic updates* sec
 docker run \
     -p 8080:80 \
     -e UPDATES=enabled \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
-    -v openstreetmap-rendered-tiles:/var/lib/mod_tile \
+    -v osm-tiles:/data/tiles/ \
     -d overv/openstreetmap-tile-server \
     run
 ```
@@ -140,7 +142,7 @@ To enable the `Access-Control-Allow-Origin` header to be able to retrieve tiles
 ```
 docker run \
     -p 8080:80 \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     -e ALLOW_CORS=enabled \
     -d overv/openstreetmap-tile-server \
     run
@@ -154,7 +156,7 @@ To connect to the PostgreSQL database inside the container, make sure to expose
 docker run \
     -p 8080:80 \
     -p 5432:5432 \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     -d overv/openstreetmap-tile-server \
     run
 ```
@@ -172,7 +174,7 @@ docker run \
     -p 8080:80 \
     -p 5432:5432 \
     -e PGPASSWORD=secret \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     -d overv/openstreetmap-tile-server \
     run
 ```
@@ -188,7 +190,7 @@ The import and tile serving processes use 4 threads by default, but this number
 docker run \
     -p 8080:80 \
     -e THREADS=24 \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     -d overv/openstreetmap-tile-server \
     run
 ```
@@ -200,7 +202,7 @@ The import and tile serving processes use 800 MB RAM cache by default, but this
 docker run \
     -p 8080:80 \
     -e "OSM2PGSQL_EXTRA_ARGS=-C 4096" \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     -d overv/openstreetmap-tile-server \
     run
 ```
@@ -212,27 +214,24 @@ The database use the autovacuum feature by default. This behavior can be changed
 docker run \
     -p 8080:80 \
     -e AUTOVACUUM=off \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     -d overv/openstreetmap-tile-server \
     run
 ```
 
-### Flat nodes
+### FLAT_NODES
 
 If you are planning to import the entire planet or you are running into memory errors then you may want to enable the `--flat-nodes` option for osm2pgsql. You can then use it during the import process as follows:
 
 ```
 docker run \
-    -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
+    -v /absolute/path/to/luxembourg.osm.pbf:/data/region.osm.pbf \
-    -v openstreetmap-nodes:/nodes \
+    -v osm-data:/data/database/ \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -e "FLAT_NODES=enabled" \
-    -e "OSM2PGSQL_EXTRA_ARGS=--flat-nodes /nodes/flat_nodes.bin" \
     overv/openstreetmap-tile-server \
     import
 ```
 
->Note that if you use a folder other than `/nodes` then you must make sure that you manually set the owner to `renderer`!
-
 ### Benchmarks
 
 You can find an example of the import performance to expect with this image on the [OpenStreetMap wiki](https://wiki.openstreetmap.org/wiki/Osm2pgsql/benchmarks#debian_9_.2F_openstreetmap-tile-server).
@@ -250,7 +249,7 @@ To raise it use `--shm-size` parameter. For example:
 ```
 docker run \
     -p 8080:80 \
-    -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v osm-data:/data/database/ \
     --shm-size="192m" \
     -d overv/openstreetmap-tile-server \
     run

docker-compose.yml

@@ -4,11 +4,11 @@ services:
   map:
     image: overv/openstreetmap-tile-server
     volumes:
-      - openstreetmap-data:/var/lib/postgresql/14/main
+      - osm-data:/data/database/
     ports:
       - "8080:80"
     command: "run"
 
 volumes:
-  openstreetmap-data:
+  osm-data:
     external: true

openstreetmap-tiles-update-expire.sh

@@ -6,9 +6,6 @@ set -e
 # AJT - change directory to mod_tile directory so that we can run replag
 # and other things directly from this script when run from cron.
 # Change the actual location to wherever installed locally.
-#------------------------------------------------------------------------------
-ACCOUNT=renderer
-
 #------------------------------------------------------------------------------
 # Extra OSM2PGSQL_OPTIONS may need setting if a tag transform script is
 # in use.  See https://github.com/SomeoneElseOSM/SomeoneElse-style and
@@ -17,10 +14,15 @@ ACCOUNT=renderer
 #------------------------------------------------------------------------------
 OSMOSIS_BIN=osmosis
 OSM2PGSQL_BIN=osm2pgsql
-TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py
+TRIM_BIN=/home/renderer/src/regional/trim_osc.py
 
 DBNAME=gis
-OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} ${OSM2PGSQL_EXTRA_ARGS}"
+OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /data/style/${NAME_STYLE:-openstreetmap-carto.style} ${OSM2PGSQL_EXTRA_ARGS}"
+
+# flat-nodes
+if [ -f /data/database/flat_nodes.bin ]; then
+    OSM2PGSQL_OPTIONS="${OSM2PGSQL_OPTIONS} --flat-nodes /data/database/flat_nodes.bin"
+fi
 
 #------------------------------------------------------------------------------
 # When using trim_osc.py we can define either a bounding box (such as this
@@ -28,12 +30,12 @@ OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/
 # See https://github.com/zverik/regional .
 # This area will usually correspond to the data originally loaded.
 #------------------------------------------------------------------------------
-TRIM_POLY_FILE="/var/lib/mod_tile/data.poly"
+TRIM_POLY_FILE="/data/database/region.poly"
 TRIM_OPTIONS="-d $DBNAME"
 TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE"
 
-BASE_DIR=/var/lib/mod_tile
+BASE_DIR=/data/database
-LOG_DIR=/var/log/tiles/
+LOG_DIR=/var/log/tiles
 WORKOSM_DIR=$BASE_DIR/.osmosis
 
 LOCK_FILE=/tmp/openstreetmap-update-expire-lock.txt
@@ -64,7 +66,7 @@ EXPIRY_MAXZOOM=20
 
 m_info()
 {
-        echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG"
+    echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG"
 }
 
 m_error()
@@ -114,18 +116,19 @@ if [ $# -eq 1 ] ; then
 
     mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt
     sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt
-else
+    exit 0
+fi
+
 # make sure the lockfile is removed when we exit and then claim it
+if ! getlock "$LOCK_FILE"; then
+    m_info "pid `cat $LOCK_FILE` still running"
+    exit 3
+fi
 
-    if ! getlock "$LOCK_FILE"; then
+if [ -e $STOP_FILE ]; then
-        m_info "pid `cat $LOCK_FILE` still running"
+    m_info "stopped"
-        exit 3
+    exit 2
-    fi
+fi
-
-    if [ -e $STOP_FILE ]; then
-        m_info "stopped"
-        exit 2
-    fi
 
 # -----------------------------------------------------------------------------
 # Add disk space check from https://github.com/zverik/regional
@@ -137,16 +140,16 @@ if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_ba
     exit 4
 fi
 
-    seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2`
+seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2`
 
-    m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`"
+m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`"
 
-    /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
+/bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
-    m_ok "downloading diff"
+m_ok "downloading diff"
 
-    if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then
+if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then
-        m_error "Osmosis error"
+    m_error "Osmosis error"
-    fi
+fi
 
 if [ -f $TRIM_POLY_FILE ] ; then
   m_ok "filtering diff"
@@ -156,16 +159,17 @@ if [ -f $TRIM_POLY_FILE ] ; then
 else
   m_ok "filtering diff skipped"
 fi
-    m_ok "importing diff"
+m_ok "importing diff"
+
 #------------------------------------------------------------------------------
 # Previously openstreetmap-tiles-update-expire tried to dirty layer
 # "$EXPIRY_MAXZOOM - 3" (which was 15) only.  Instead we write all expired
 # tiles in range to the list (note the "-" rather than ":" in the "-e"
 # parameter).
 #------------------------------------------------------------------------------
-    if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then
+if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then
-        m_error "osm2pgsql error"
+    m_error "osm2pgsql error"
-    fi
+fi
 
 #------------------------------------------------------------------------------
 # The lockfile is normally removed before we expire tiles because that is
@@ -175,8 +179,8 @@ fi
 #------------------------------------------------------------------------------
 #   m_ok "Import complete; removing lock file"
 #   freelock "$LOCK_FILE"
+m_ok "expiring tiles"
 
-    m_ok "expiring tiles"
 #------------------------------------------------------------------------------
 # When expiring tiles we need to define the style sheet if it's not "default".
 # In this case it's "ajt".
@@ -186,18 +190,16 @@ fi
 # delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM.
 # The default path to renderd.sock is fixed.
 #------------------------------------------------------------------------------
-    if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
+if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
-        m_info "Expiry failed"
+    m_info "Expiry failed"
-    fi
+fi
 
-    rm "$EXPIRY_FILE.$$"
+rm "$EXPIRY_FILE.$$"
 
 #------------------------------------------------------------------------------
 # Only remove the lock file after expiry (if system is slow we want to delay
 # the next import, not have multiple render_expired processes running)
 #------------------------------------------------------------------------------
-    freelock "$LOCK_FILE"
+freelock "$LOCK_FILE"
 
-    m_ok "Done with import"
+m_ok "Done with import"
-
-fi

run.sh

@@ -15,7 +15,7 @@ function setPostgresPassword() {
 if [ "$#" -ne 1 ]; then
     echo "usage: <import|run>"
     echo "commands:"
-    echo "    import: Set up the database and import /data.osm.pbf"
+    echo "    import: Set up the database and import /data/region.osm.pbf"
     echo "    run: Runs Apache and renderd to serve tiles at /tile/{z}/{x}/{y}.png"
     echo "environment variables:"
     echo "    THREADS: defines number of threads used for importing / tile rendering"
@@ -29,24 +29,22 @@ fi
 
 set -x
 
-if [ ! "$(ls -A /home/renderer/src/openstreetmap-carto)" ]; then
+# if there is no custom style mounted, then use osm-carto
-
+if [ ! "$(ls -A /data/style/)" ]; then
-    mv /home/renderer/src/openstreetmap-carto-backup/* /home/renderer/src/openstreetmap-carto/
+    mv /home/renderer/src/openstreetmap-carto-backup/* /data/style/
-
 fi
 
-if [ ! -f /home/renderer/src/openstreetmap-carto/mapnik.xml ]; then
+# carto build
-
+if [ ! -f /data/style/mapnik.xml ]; then
-    cd /home/renderer/src/openstreetmap-carto
+    cd /data/style/
     carto ${NAME_MML:-project.mml} > mapnik.xml
-
 fi
 
-if [ "$1" = "import" ]; then
+if [ "$1" == "import" ]; then
     # Ensure that database directory is in right state
-    chown postgres:postgres -R /var/lib/postgresql
+    chown -R postgres: /var/lib/postgresql /data/database/postgres/
-    if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then
+    if [ ! -f /data/database/postgres/PG_VERSION ]; then
-        sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /var/lib/postgresql/14/main/ initdb -o "--locale C.UTF-8"
+        sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
     fi
 
     # Initialize PostgreSQL
@@ -61,61 +59,100 @@ if [ "$1" = "import" ]; then
     setPostgresPassword
 
     # Download Luxembourg as sample if no data is provided
-    if [ ! -f /data.osm.pbf ] && [ -z "${DOWNLOAD_PBF:-}" ]; then
+    if [ ! -f /data/region.osm.pbf ] && [ -z "${DOWNLOAD_PBF:-}" ]; then
-        echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..."
+        echo "WARNING: No import file at /data/region.osm.pbf, so importing Luxembourg as example..."
         DOWNLOAD_PBF="https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf"
         DOWNLOAD_POLY="https://download.geofabrik.de/europe/luxembourg.poly"
     fi
 
     if [ -n "${DOWNLOAD_PBF:-}" ]; then
         echo "INFO: Download PBF file: $DOWNLOAD_PBF"
-        wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data.osm.pbf
+        wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data/region.osm.pbf
         if [ -n "$DOWNLOAD_POLY" ]; then
             echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY"
-            wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data.poly
+            wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data/region.poly
         fi
     fi
 
-    if [ "${UPDATES:-}" = "enabled" ]; then
+    if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
         # determine and set osmosis_replication_timestamp (for consecutive updates)
-        osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info
+        REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44`
-        osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt
-        REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt)
 
         # initial setup of osmosis workspace (for consecutive updates)
-        sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP
+        sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
     fi
 
     # copy polygon file if available
-    if [ -f /data.poly ]; then
+    if [ -f /data/region.poly ]; then
-        sudo -u renderer cp /data.poly /var/lib/mod_tile/data.poly
+        cp /data/region.poly /data/database/region.poly
+        chown renderer: /data/database/region.poly
+    fi
+
+    # flat-nodes
+    if [ "${FLAT_NODES:-}" == "enabled" ] || [ "${FLAT_NODES:-}" == "1" ]; then
+        OSM2PGSQL_EXTRA_ARGS="${OSM2PGSQL_EXTRA_ARGS:-} --flat-nodes /data/database/flat_nodes.bin"
     fi
 
     # Import data
-    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS:-}
+    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore  \
+      --tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua}  \
+      --number-processes ${THREADS:-4}  \
+      -S /data/style/${NAME_STYLE:-openstreetmap-carto.style}  \
+      /data/region.osm.pbf  \
+      ${OSM2PGSQL_EXTRA_ARGS:-}  \
+    ;
+
+    # old flat-nodes dir
+    if [ -f /nodes/flat_nodes.bin ] && ! [ -f /data/database/flat_nodes.bin ]; then
+        mv /nodes/flat_nodes.bin /data/database/flat_nodes.bin
+        chown renderer: /data/database/flat_nodes.bin
+    fi
 
     # Create indexes
-    sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/${NAME_SQL:-indexes.sql}
+    if [ -f /data/style/${NAME_SQL:-indexes.sql} ]; then
+        sudo -u postgres psql -d gis -f /data/style/${NAME_SQL:-indexes.sql}
+    fi
 
     #Import external data
-    sudo chown -R renderer: /home/renderer/src
+    chown -R renderer: /home/renderer/src/ /data/style/
-
+    if [ -f /data/style/scripts/get-external-data.py ] && [ -f /data/style/external-data.yml ]; then
-    sudo -E -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data
+        sudo -E -u renderer python3 /data/style/scripts/get-external-data.py -c /data/style/external-data.yml -D /data/style/data
+    fi
 
     # Register that data has changed for mod_tile caching purposes
-    touch /var/lib/mod_tile/planet-import-complete
+    sudo -u renderer touch /data/database/planet-import-complete
 
     service postgresql stop
 
     exit 0
 fi
 
-if [ "$1" = "run" ]; then
+if [ "$1" == "run" ]; then
     # Clean /tmp
     rm -rf /tmp/*
 
+    # migrate old files
+    if [ -f /data/database/PG_VERSION ] && ! [ -d /data/database/postgres/ ]; then
+        mkdir /data/database/postgres/
+        mv /data/database/* /data/database/postgres/
+    fi
+    if [ -f /nodes/flat_nodes.bin ] && ! [ -f /data/database/flat_nodes.bin ]; then
+        mv /nodes/flat_nodes.bin /data/database/flat_nodes.bin
+    fi
+    if [ -f /data/tiles/data.poly ] && ! [ -f /data/database/region.poly ]; then
+        mv /data/tiles/data.poly /data/database/region.poly
+    fi
+
+    # sync planet-import-complete file
+    if [ -f /data/tiles/planet-import-complete ] && ! [ -f /data/database/planet-import-complete ]; then
+        cp /data/tiles/planet-import-complete /data/database/planet-import-complete
+    fi
+    if ! [ -f /data/tiles/planet-import-complete ] && [ -f /data/database/planet-import-complete ]; then
+        cp /data/database/planet-import-complete /data/tiles/planet-import-complete
+    fi
+
     # Fix postgres data privileges
-    chown postgres:postgres /var/lib/postgresql -R
+    chown -R postgres: /var/lib/postgresql/ /data/database/postgres/
 
     # Configure Apache CORS
     if [ "${ALLOW_CORS:-}" == "enabled" ] || [ "${ALLOW_CORS:-}" == "1" ]; then
@@ -132,8 +169,8 @@ if [ "$1" = "run" ]; then
     sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf
 
     # start cron job to trigger consecutive updates
-    if [ "${UPDATES:-}" = "enabled" ] || [ "${UPDATES:-}" = "1" ]; then
+    if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
-      /etc/init.d/cron start
+        /etc/init.d/cron start
     fi
 
     # Run while handling docker stop's SIGTERM