diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 93781e9..cdc9c83 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -26,7 +26,7 @@ jobs: env: VOLUME : osm-db CONTAINER : osm-www - MOUNT : /var/lib/postgresql/14/main + MOUNT : /data/database/ steps: - name: Checkout diff --git a/.travis.yml b/.travis.yml index c072dd1..85b8819 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,9 +10,9 @@ before_script: - docker pull overv/openstreetmap-tile-server || true script: - docker build --pull --cache-from overv/openstreetmap-tile-server --tag overv/openstreetmap-tile-server . -- docker volume create openstreetmap-data -- docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main overv/openstreetmap-tile-server import -- docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main -p 8080:80 -d overv/openstreetmap-tile-server run +- docker volume create osm-data +- docker run --rm -v osm-data:/data/database/ overv/openstreetmap-tile-server import +- docker run --rm -v osm-data:/data/database/ -p 8080:80 -d overv/openstreetmap-tile-server run - sleep 30 - make DOCKER_IMAGE=overv/openstreetmap-tile-server stop after_script: diff --git a/Dockerfile b/Dockerfile index 9975067..bfd60df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -97,7 +97,7 @@ RUN mkdir -p /home/renderer/src \ ########################################################################################################### -FROM ubuntu:20.04 AS final-base +FROM ubuntu:20.04 AS final # Based on # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/ @@ -164,15 +164,12 @@ RUN ln -sf /dev/stdout /var/log/apache2/access.log \ && ln -sf /dev/stderr /var/log/apache2/error.log # Copy update scripts -COPY openstreetmap-tiles-update-expire /usr/bin/ -RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \ +COPY openstreetmap-tiles-update-expire.sh /usr/bin/ +RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \ && mkdir /var/log/tiles \ && chmod a+rw /var/log/tiles \ && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \ -&& echo "* * * * * renderer openstreetmap-tiles-update-expire\n" >> /etc/crontab - -RUN mkdir /nodes \ -&& chown renderer:renderer /nodes +&& echo "* * * * * renderer openstreetmap-tiles-update-expire.sh\n" >> /etc/crontab # Configure PosgtreSQL COPY postgresql.custom.conf.tmpl /etc/postgresql/14/main/ @@ -181,9 +178,18 @@ RUN chown -R postgres:postgres /var/lib/postgresql \ && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \ && echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf -########################################################################################################### - -FROM final-base AS final +# Create volume directories +RUN mkdir -p /data/database/ \ + && mkdir -p /data/style/ \ + && mkdir -p /home/renderer/src/ \ + && chown -R renderer: /data/ \ + && chown -R renderer: /home/renderer/src/ \ + && mv /var/lib/postgresql/14/main/ /data/database/postgres/ \ + && mv /var/lib/mod_tile/ /data/tiles/ \ + && ln -s /data/database/postgres /var/lib/postgresql/14/main \ + && ln -s /data/style /home/renderer/src/openstreetmap-carto \ + && ln -s /data/tiles /var/lib/mod_tile \ +; # Install PostGIS COPY --from=compiler-postgis postgis_src/postgis-src_3.2.1-1_amd64.deb . @@ -206,7 +212,6 @@ RUN dpkg -i renderd_1-1_amd64.deb \ # Install mod_tile COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb . RUN dpkg -i mod-tile_1-1_amd64.deb \ - && mkdir -p /home/renderer/src/openstreetmap-carto \ && ldconfig \ && rm mod-tile_1-1_amd64.deb diff --git a/Makefile b/Makefile index 9be0834..1c197e3 100644 --- a/Makefile +++ b/Makefile @@ -9,10 +9,10 @@ push: build docker push ${DOCKER_IMAGE}:latest test: build - docker volume create openstreetmap-data - docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main ${DOCKER_IMAGE} import - docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main -p 8080:80 -d ${DOCKER_IMAGE} run + docker volume create osm-data + docker run --rm -v osm-data:/data/database/ ${DOCKER_IMAGE} import + docker run --rm -v osm-data:/data/database/ -p 8080:80 -d ${DOCKER_IMAGE} run stop: docker rm -f `docker ps | grep '${DOCKER_IMAGE}' | awk '{ print $$1 }'` || true - docker volume rm -f openstreetmap-data + docker volume rm -f osm-data diff --git a/README.md b/README.md index 16f9224..2d0efdc 100644 --- a/README.md +++ b/README.md @@ -9,41 +9,43 @@ This container allows you to easily set up an OpenStreetMap PNG tile server give First create a Docker volume to hold the PostgreSQL database that will contain the OpenStreetMap data: - docker volume create openstreetmap-data + docker volume create osm-data -Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example: +Next, download an `.osm.pbf` extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data/region.osm.pbf`. For example: ``` docker run \ - -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v /absolute/path/to/luxembourg.osm.pbf:/data/region.osm.pbf \ + -v osm-data:/data/database/ \ overv/openstreetmap-tile-server \ import ``` If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server. -Note that the import process requires an internet connection. The run process does not require an internet connection. If you want to run the openstreetmap-tile server on a computer that is isolated, you must first import on an internet connected computer, export the openstreetmap-data volume as a tarfile, and then restore the data volume on the target computer system. +Note that the import process requires an internet connection. The run process does not require an internet connection. If you want to run the openstreetmap-tile server on a computer that is isolated, you must first import on an internet connected computer, export the `osm-data` volume as a tarfile, and then restore the data volume on the target computer system. -Also when running on an isolated system, the default index.html from the container will not work, as it requires access to the web for the leaflet packages. +Also when running on an isolated system, the default `index.html` from the container will not work, as it requires access to the web for the leaflet packages. ### Automatic updates (optional) -If your import is an extract of the planet and has polygonal bounds associated with it, like those from geofabrik.de, then it is possible to set your server up for automatic updates. Make sure to reference both the OSM file and the polygon file during the import process to facilitate this, and also include the `UPDATES=enabled` variable: +If your import is an extract of the planet and has polygonal bounds associated with it, like those from [geofabrik.de](https://download.geofabrik.de/), then it is possible to set your server up for automatic updates. Make sure to reference both the OSM file and the polygon file during the `import` process to facilitate this, and also include the `UPDATES=enabled` variable: ``` docker run \ -e UPDATES=enabled \ - -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ - -v /absolute/path/to/luxembourg.poly:/data.poly \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ - -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ + -v /absolute/path/to/luxembourg.osm.pbf:/data/region.osm.pbf \ + -v /absolute/path/to/luxembourg.poly:/data/region.poly \ + -v osm-data:/data/database/ \ overv/openstreetmap-tile-server \ import ``` Refer to the section *Automatic updating and tile expiry* to actually enable the updates while running the tile server. +Please note: If you're not importing the whole planet, then the `.poly` file is necessary to limit automatic updates to the relevant region. +Therefore, when you only have a `.osm.pbf` file but not a `.poly` file, you should not enable automatic updates. + ### Letting the container download the file It is also possible to let the container download files for you rather than mounting them in advance by using the `DOWNLOAD_PBF` and `DOWNLOAD_POLY` parameters: @@ -52,7 +54,7 @@ It is also possible to let the container download files for you rather than moun docker run \ -e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \ -e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v osm-data:/data/database/ \ overv/openstreetmap-tile-server \ import ``` @@ -69,15 +71,15 @@ docker run \ -e NAME_STYLE=test.style -e NAME_MML=project.mml -e NAME_SQL=test.sql - -v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto \ - -v openstreetmap-data:/var/lib/postgresql/12/main \ + -v /home/user/openstreetmap-carto-modified:/data/style/ \ + -v osm-data:/data/database/ \ overv/openstreetmap-tile-server \ import ``` If you do not define the "NAME_*" variables, the script will default to those found in the openstreetmap-carto style. -Be sure to mount the volume during `run` with the same `-v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto` +Be sure to mount the volume during `run` with the same `-v /home/user/openstreetmap-carto-modified:/data/style/` If you do not see the expected style upon `run` double check your paths as the style may not have been found at the directory specified. By default, `openstreetmap-carto` will be used if a style cannot be found @@ -90,7 +92,7 @@ Run the server like this: ``` docker run \ -p 8080:80 \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v osm-data:/data/database/ \ -d overv/openstreetmap-tile-server \ run ``` @@ -103,19 +105,19 @@ The `docker-compose.yml` file included with this repository shows how the aforem ### Preserving rendered tiles -Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To make sure that this data survives container restarts, you should create another volume for it: +Tiles that have already been rendered will be stored in `/data/tiles/`. To make sure that this data survives container restarts, you should create another volume for it: ``` -docker volume create openstreetmap-rendered-tiles +docker volume create osm-tiles docker run \ -p 8080:80 \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ - -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ + -v osm-data:/data/database/ \ + -v osm-tiles:/data/tiles/ \ -d overv/openstreetmap-tile-server \ run ``` -**If you do this, then make sure to also run the import with the `openstreetmap-rendered-tiles` volume to make sure that caching works properly across updates!** +**If you do this, then make sure to also run the import with the `osm-tiles` volume to make sure that caching works properly across updates!** ### Enabling automatic updating (optional) @@ -125,8 +127,8 @@ Given that you've set up your import as described in the *Automatic updates* sec docker run \ -p 8080:80 \ -e UPDATES=enabled \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ - -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ + -v osm-data:/data/database/ \ + -v osm-tiles:/data/tiles/ \ -d overv/openstreetmap-tile-server \ run ``` @@ -140,7 +142,7 @@ To enable the `Access-Control-Allow-Origin` header to be able to retrieve tiles ``` docker run \ -p 8080:80 \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v osm-data:/data/database/ \ -e ALLOW_CORS=enabled \ -d overv/openstreetmap-tile-server \ run @@ -154,7 +156,7 @@ To connect to the PostgreSQL database inside the container, make sure to expose docker run \ -p 8080:80 \ -p 5432:5432 \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v osm-data:/data/database/ \ -d overv/openstreetmap-tile-server \ run ``` @@ -172,7 +174,7 @@ docker run \ -p 8080:80 \ -p 5432:5432 \ -e PGPASSWORD=secret \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v osm-data:/data/database/ \ -d overv/openstreetmap-tile-server \ run ``` @@ -188,7 +190,7 @@ The import and tile serving processes use 4 threads by default, but this number docker run \ -p 8080:80 \ -e THREADS=24 \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v osm-data:/data/database/ \ -d overv/openstreetmap-tile-server \ run ``` @@ -200,7 +202,7 @@ The import and tile serving processes use 800 MB RAM cache by default, but this docker run \ -p 8080:80 \ -e "OSM2PGSQL_EXTRA_ARGS=-C 4096" \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v osm-data:/data/database/ \ -d overv/openstreetmap-tile-server \ run ``` @@ -212,27 +214,24 @@ The database use the autovacuum feature by default. This behavior can be changed docker run \ -p 8080:80 \ -e AUTOVACUUM=off \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v osm-data:/data/database/ \ -d overv/openstreetmap-tile-server \ run ``` -### Flat nodes +### FLAT_NODES If you are planning to import the entire planet or you are running into memory errors then you may want to enable the `--flat-nodes` option for osm2pgsql. You can then use it during the import process as follows: ``` docker run \ - -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ - -v openstreetmap-nodes:/nodes \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ - -e "OSM2PGSQL_EXTRA_ARGS=--flat-nodes /nodes/flat_nodes.bin" \ + -v /absolute/path/to/luxembourg.osm.pbf:/data/region.osm.pbf \ + -v osm-data:/data/database/ \ + -e "FLAT_NODES=enabled" \ overv/openstreetmap-tile-server \ import ``` ->Note that if you use a folder other than `/nodes` then you must make sure that you manually set the owner to `renderer`! - ### Benchmarks You can find an example of the import performance to expect with this image on the [OpenStreetMap wiki](https://wiki.openstreetmap.org/wiki/Osm2pgsql/benchmarks#debian_9_.2F_openstreetmap-tile-server). @@ -250,7 +249,7 @@ To raise it use `--shm-size` parameter. For example: ``` docker run \ -p 8080:80 \ - -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v osm-data:/data/database/ \ --shm-size="192m" \ -d overv/openstreetmap-tile-server \ run diff --git a/docker-compose.yml b/docker-compose.yml index 51b31c9..684723c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,11 +4,11 @@ services: map: image: overv/openstreetmap-tile-server volumes: - - openstreetmap-data:/var/lib/postgresql/14/main + - osm-data:/data/database/ ports: - "8080:80" command: "run" volumes: - openstreetmap-data: + osm-data: external: true diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire.sh similarity index 77% rename from openstreetmap-tiles-update-expire rename to openstreetmap-tiles-update-expire.sh index c126570..87e73e5 100644 --- a/openstreetmap-tiles-update-expire +++ b/openstreetmap-tiles-update-expire.sh @@ -6,9 +6,6 @@ set -e # AJT - change directory to mod_tile directory so that we can run replag # and other things directly from this script when run from cron. # Change the actual location to wherever installed locally. -#------------------------------------------------------------------------------ -ACCOUNT=renderer - #------------------------------------------------------------------------------ # Extra OSM2PGSQL_OPTIONS may need setting if a tag transform script is # in use. See https://github.com/SomeoneElseOSM/SomeoneElse-style and @@ -17,10 +14,15 @@ ACCOUNT=renderer #------------------------------------------------------------------------------ OSMOSIS_BIN=osmosis OSM2PGSQL_BIN=osm2pgsql -TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py +TRIM_BIN=/home/renderer/src/regional/trim_osc.py DBNAME=gis -OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} ${OSM2PGSQL_EXTRA_ARGS}" +OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /data/style/${NAME_STYLE:-openstreetmap-carto.style} ${OSM2PGSQL_EXTRA_ARGS}" + +# flat-nodes +if [ -f /data/database/flat_nodes.bin ]; then + OSM2PGSQL_OPTIONS="${OSM2PGSQL_OPTIONS} --flat-nodes /data/database/flat_nodes.bin" +fi #------------------------------------------------------------------------------ # When using trim_osc.py we can define either a bounding box (such as this @@ -28,12 +30,12 @@ OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/ # See https://github.com/zverik/regional . # This area will usually correspond to the data originally loaded. #------------------------------------------------------------------------------ -TRIM_POLY_FILE="/var/lib/mod_tile/data.poly" +TRIM_POLY_FILE="/data/database/region.poly" TRIM_OPTIONS="-d $DBNAME" TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE" -BASE_DIR=/var/lib/mod_tile -LOG_DIR=/var/log/tiles/ +BASE_DIR=/data/database +LOG_DIR=/var/log/tiles WORKOSM_DIR=$BASE_DIR/.osmosis LOCK_FILE=/tmp/openstreetmap-update-expire-lock.txt @@ -64,7 +66,7 @@ EXPIRY_MAXZOOM=20 m_info() { - echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG" + echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG" } m_error() @@ -114,18 +116,19 @@ if [ $# -eq 1 ] ; then mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt -else + exit 0 +fi + # make sure the lockfile is removed when we exit and then claim it +if ! getlock "$LOCK_FILE"; then + m_info "pid `cat $LOCK_FILE` still running" + exit 3 +fi - if ! getlock "$LOCK_FILE"; then - m_info "pid `cat $LOCK_FILE` still running" - exit 3 - fi - - if [ -e $STOP_FILE ]; then - m_info "stopped" - exit 2 - fi +if [ -e $STOP_FILE ]; then + m_info "stopped" + exit 2 +fi # ----------------------------------------------------------------------------- # Add disk space check from https://github.com/zverik/regional @@ -137,16 +140,16 @@ if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_ba exit 4 fi - seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2` +seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2` - m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`" +m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`" - /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt - m_ok "downloading diff" +/bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt +m_ok "downloading diff" - if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then - m_error "Osmosis error" - fi +if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then + m_error "Osmosis error" +fi if [ -f $TRIM_POLY_FILE ] ; then m_ok "filtering diff" @@ -156,16 +159,17 @@ if [ -f $TRIM_POLY_FILE ] ; then else m_ok "filtering diff skipped" fi - m_ok "importing diff" +m_ok "importing diff" + #------------------------------------------------------------------------------ # Previously openstreetmap-tiles-update-expire tried to dirty layer # "$EXPIRY_MAXZOOM - 3" (which was 15) only. Instead we write all expired # tiles in range to the list (note the "-" rather than ":" in the "-e" # parameter). #------------------------------------------------------------------------------ - if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then - m_error "osm2pgsql error" - fi +if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then + m_error "osm2pgsql error" +fi #------------------------------------------------------------------------------ # The lockfile is normally removed before we expire tiles because that is @@ -175,8 +179,8 @@ fi #------------------------------------------------------------------------------ # m_ok "Import complete; removing lock file" # freelock "$LOCK_FILE" +m_ok "expiring tiles" - m_ok "expiring tiles" #------------------------------------------------------------------------------ # When expiring tiles we need to define the style sheet if it's not "default". # In this case it's "ajt". @@ -186,18 +190,16 @@ fi # delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM. # The default path to renderd.sock is fixed. #------------------------------------------------------------------------------ - if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then - m_info "Expiry failed" - fi +if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then + m_info "Expiry failed" +fi - rm "$EXPIRY_FILE.$$" +rm "$EXPIRY_FILE.$$" #------------------------------------------------------------------------------ # Only remove the lock file after expiry (if system is slow we want to delay # the next import, not have multiple render_expired processes running) #------------------------------------------------------------------------------ - freelock "$LOCK_FILE" +freelock "$LOCK_FILE" - m_ok "Done with import" - -fi +m_ok "Done with import" diff --git a/run.sh b/run.sh index ebdbe67..5552c51 100755 --- a/run.sh +++ b/run.sh @@ -15,7 +15,7 @@ function setPostgresPassword() { if [ "$#" -ne 1 ]; then echo "usage: " echo "commands:" - echo " import: Set up the database and import /data.osm.pbf" + echo " import: Set up the database and import /data/region.osm.pbf" echo " run: Runs Apache and renderd to serve tiles at /tile/{z}/{x}/{y}.png" echo "environment variables:" echo " THREADS: defines number of threads used for importing / tile rendering" @@ -29,24 +29,22 @@ fi set -x -if [ ! "$(ls -A /home/renderer/src/openstreetmap-carto)" ]; then - - mv /home/renderer/src/openstreetmap-carto-backup/* /home/renderer/src/openstreetmap-carto/ - +# if there is no custom style mounted, then use osm-carto +if [ ! "$(ls -A /data/style/)" ]; then + mv /home/renderer/src/openstreetmap-carto-backup/* /data/style/ fi -if [ ! -f /home/renderer/src/openstreetmap-carto/mapnik.xml ]; then - - cd /home/renderer/src/openstreetmap-carto +# carto build +if [ ! -f /data/style/mapnik.xml ]; then + cd /data/style/ carto ${NAME_MML:-project.mml} > mapnik.xml - fi -if [ "$1" = "import" ]; then +if [ "$1" == "import" ]; then # Ensure that database directory is in right state - chown postgres:postgres -R /var/lib/postgresql - if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then - sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /var/lib/postgresql/14/main/ initdb -o "--locale C.UTF-8" + chown -R postgres: /var/lib/postgresql /data/database/postgres/ + if [ ! -f /data/database/postgres/PG_VERSION ]; then + sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8" fi # Initialize PostgreSQL @@ -61,61 +59,100 @@ if [ "$1" = "import" ]; then setPostgresPassword # Download Luxembourg as sample if no data is provided - if [ ! -f /data.osm.pbf ] && [ -z "${DOWNLOAD_PBF:-}" ]; then - echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..." + if [ ! -f /data/region.osm.pbf ] && [ -z "${DOWNLOAD_PBF:-}" ]; then + echo "WARNING: No import file at /data/region.osm.pbf, so importing Luxembourg as example..." DOWNLOAD_PBF="https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf" DOWNLOAD_POLY="https://download.geofabrik.de/europe/luxembourg.poly" fi if [ -n "${DOWNLOAD_PBF:-}" ]; then echo "INFO: Download PBF file: $DOWNLOAD_PBF" - wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data.osm.pbf + wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data/region.osm.pbf if [ -n "$DOWNLOAD_POLY" ]; then echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY" - wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data.poly + wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data/region.poly fi fi - if [ "${UPDATES:-}" = "enabled" ]; then + if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then # determine and set osmosis_replication_timestamp (for consecutive updates) - osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info - osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt - REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt) + REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44` # initial setup of osmosis workspace (for consecutive updates) - sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP + sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP fi # copy polygon file if available - if [ -f /data.poly ]; then - sudo -u renderer cp /data.poly /var/lib/mod_tile/data.poly + if [ -f /data/region.poly ]; then + cp /data/region.poly /data/database/region.poly + chown renderer: /data/database/region.poly + fi + + # flat-nodes + if [ "${FLAT_NODES:-}" == "enabled" ] || [ "${FLAT_NODES:-}" == "1" ]; then + OSM2PGSQL_EXTRA_ARGS="${OSM2PGSQL_EXTRA_ARGS:-} --flat-nodes /data/database/flat_nodes.bin" fi # Import data - sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS:-} + sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore \ + --tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua} \ + --number-processes ${THREADS:-4} \ + -S /data/style/${NAME_STYLE:-openstreetmap-carto.style} \ + /data/region.osm.pbf \ + ${OSM2PGSQL_EXTRA_ARGS:-} \ + ; + + # old flat-nodes dir + if [ -f /nodes/flat_nodes.bin ] && ! [ -f /data/database/flat_nodes.bin ]; then + mv /nodes/flat_nodes.bin /data/database/flat_nodes.bin + chown renderer: /data/database/flat_nodes.bin + fi # Create indexes - sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/${NAME_SQL:-indexes.sql} + if [ -f /data/style/${NAME_SQL:-indexes.sql} ]; then + sudo -u postgres psql -d gis -f /data/style/${NAME_SQL:-indexes.sql} + fi #Import external data - sudo chown -R renderer: /home/renderer/src - - sudo -E -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data + chown -R renderer: /home/renderer/src/ /data/style/ + if [ -f /data/style/scripts/get-external-data.py ] && [ -f /data/style/external-data.yml ]; then + sudo -E -u renderer python3 /data/style/scripts/get-external-data.py -c /data/style/external-data.yml -D /data/style/data + fi # Register that data has changed for mod_tile caching purposes - touch /var/lib/mod_tile/planet-import-complete + sudo -u renderer touch /data/database/planet-import-complete service postgresql stop exit 0 fi -if [ "$1" = "run" ]; then +if [ "$1" == "run" ]; then # Clean /tmp rm -rf /tmp/* + # migrate old files + if [ -f /data/database/PG_VERSION ] && ! [ -d /data/database/postgres/ ]; then + mkdir /data/database/postgres/ + mv /data/database/* /data/database/postgres/ + fi + if [ -f /nodes/flat_nodes.bin ] && ! [ -f /data/database/flat_nodes.bin ]; then + mv /nodes/flat_nodes.bin /data/database/flat_nodes.bin + fi + if [ -f /data/tiles/data.poly ] && ! [ -f /data/database/region.poly ]; then + mv /data/tiles/data.poly /data/database/region.poly + fi + + # sync planet-import-complete file + if [ -f /data/tiles/planet-import-complete ] && ! [ -f /data/database/planet-import-complete ]; then + cp /data/tiles/planet-import-complete /data/database/planet-import-complete + fi + if ! [ -f /data/tiles/planet-import-complete ] && [ -f /data/database/planet-import-complete ]; then + cp /data/database/planet-import-complete /data/tiles/planet-import-complete + fi + # Fix postgres data privileges - chown postgres:postgres /var/lib/postgresql -R + chown -R postgres: /var/lib/postgresql/ /data/database/postgres/ # Configure Apache CORS if [ "${ALLOW_CORS:-}" == "enabled" ] || [ "${ALLOW_CORS:-}" == "1" ]; then @@ -132,8 +169,8 @@ if [ "$1" = "run" ]; then sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf # start cron job to trigger consecutive updates - if [ "${UPDATES:-}" = "enabled" ] || [ "${UPDATES:-}" = "1" ]; then - /etc/init.d/cron start + if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then + /etc/init.d/cron start fi # Run while handling docker stop's SIGTERM