From 0f229a0cbdd6bf00ffde1b029da57560e20aee00 Mon Sep 17 00:00:00 2001 From: galewis2 <62433564+galewis2@users.noreply.github.com> Date: Thu, 31 Mar 2022 07:44:56 -0400 Subject: [PATCH] Allow easier modification of the OpenStreetMap-carto style (#243) --- Dockerfile | 19 ++++++++++--------- README.md | 27 +++++++++++++++++++++++++++ openstreetmap-tiles-update-expire | 19 +++++++++---------- run.sh | 22 ++++++++++++++++++++-- 4 files changed, 66 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 478afd9..9975067 100644 --- a/Dockerfile +++ b/Dockerfile @@ -80,14 +80,10 @@ RUN cd ~ \ ########################################################################################################### FROM compiler-common AS compiler-stylesheet -RUN apt-get install -y --no-install-recommends \ - npm RUN cd ~ \ && git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \ && cd openstreetmap-carto \ -&& sed -ie 's#https:\/\/naciscdn.org\/naturalearth\/110m\/cultural\/ne_110m_admin_0_boundary_lines_land.zip#https:\/\/naturalearth.s3.amazonaws.com\/110m_cultural\/ne_110m_admin_0_boundary_lines_land.zip#g' external-data.yml \ -&& npm install -g carto@1.2.0 \ -&& carto project.mml > mapnik.xml +&& rm -rf .git ########################################################################################################### @@ -96,7 +92,6 @@ RUN mkdir -p /home/renderer/src \ && cd /home/renderer/src \ && git clone https://github.com/zverik/regional \ && cd regional \ -&& git checkout 889d630a1e1a1bacabdd1dad6e17b49e7d58cd4b \ && rm -rf .git \ && chmod u+x /home/renderer/src/regional/trim_osc.py @@ -124,6 +119,7 @@ RUN apt-get update \ liblua5.3-dev \ lua5.3 \ mapnik-utils \ + npm \ osmium-tool \ osmosis \ python-is-python3 \ @@ -148,7 +144,11 @@ RUN adduser --disabled-password --gecos "" renderer # Install python libraries RUN pip3 install \ requests \ + osmium \ pyyaml + +# Install carto for stylesheet +RUN npm install -g carto@0.18.2 # Configure Apache RUN mkdir /var/lib/mod_tile \ @@ -206,16 +206,17 @@ RUN dpkg -i renderd_1-1_amd64.deb \ # Install mod_tile COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb . RUN dpkg -i mod-tile_1-1_amd64.deb \ + && mkdir -p /home/renderer/src/openstreetmap-carto \ && ldconfig \ && rm mod-tile_1-1_amd64.deb -COPY --from=compiler-modtile-renderd /root/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag -# Install stylesheet -COPY --from=compiler-stylesheet /root/openstreetmap-carto /home/renderer/src/openstreetmap-carto +COPY --from=compiler-modtile-renderd /root/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag # Install helper script COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional +COPY --from=compiler-stylesheet /root/openstreetmap-carto /home/renderer/src/openstreetmap-carto-backup + # Start running COPY run.sh / ENTRYPOINT ["/run.sh"] diff --git a/README.md b/README.md index 802eb53..c969233 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ docker run \ -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \ -v /absolute/path/to/luxembourg.poly:/data.poly \ -v openstreetmap-data:/var/lib/postgresql/14/main \ + -v openstreetmap-rendered-tiles:/var/lib/mod_tile \ overv/openstreetmap-tile-server \ import ``` @@ -55,6 +56,32 @@ docker run \ import ``` +### Using an alternate style + +By default the container will use openstreetmap-carto if it is not specified. However, you can modify the style at run-time. Be aware you need the style mounted at `run` AND `import` as the Lua script needs to be run: + +``` +docker run \ + -e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \ + -e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \ + -e NAME_LUA=sample.lua + -e NAME_STYLE=test.style + -e NAME_MML=project.mml + -e NAME_SQL=test.sql + -v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto \ + -v openstreetmap-data:/var/lib/postgresql/12/main \ + overv/openstreetmap-tile-server \ + import +``` + +If you do not define the "NAME_*" variables, the script will default to those found in the openstreetmap-carto style. + +Be sure to mount the volume during `run` with the same `-v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto` + +If you do not see the expected style upon `run` double check your paths as the style may not have been found at the directory specified. By default, `openstreetmap-carto` will be used if a style cannot be found + +**Only openstreetmap-carto and styles like it, eg, ones with one lua script, one style, one mml, one SQL can be used** + ## Running the server Run the server like this: diff --git a/openstreetmap-tiles-update-expire b/openstreetmap-tiles-update-expire index 4a0e7e4..c126570 100644 --- a/openstreetmap-tiles-update-expire +++ b/openstreetmap-tiles-update-expire @@ -20,7 +20,7 @@ OSM2PGSQL_BIN=osm2pgsql TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py DBNAME=gis -OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style ${OSM2PGSQL_EXTRA_ARGS}" +OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} ${OSM2PGSQL_EXTRA_ARGS}" #------------------------------------------------------------------------------ # When using trim_osc.py we can define either a bounding box (such as this @@ -30,7 +30,6 @@ OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/ #------------------------------------------------------------------------------ TRIM_POLY_FILE="/var/lib/mod_tile/data.poly" TRIM_OPTIONS="-d $DBNAME" -#TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27" TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE" BASE_DIR=/var/lib/mod_tile @@ -107,9 +106,12 @@ freelock() if [ $# -eq 1 ] ; then m_info "Initialising Osmosis replication system to $1" - mkdir $WORKOSM_DIR + mkdir -p $WORKOSM_DIR + $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" + wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt + mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt else @@ -167,12 +169,12 @@ fi #------------------------------------------------------------------------------ # The lockfile is normally removed before we expire tiles because that is -# something thatcan be done in parallel with further processing. In order to +# something that can be done in parallel with further processing. In order to # avoid rework, if actually rerendering is done rather than just deleting or # dirtying, it makes sense to move it lower down. #------------------------------------------------------------------------------ -# m_ok "Import complete; removing lock file" -# freelock "$LOCK_FILE" +# m_ok "Import complete; removing lock file" +# freelock "$LOCK_FILE" m_ok "expiring tiles" #------------------------------------------------------------------------------ @@ -194,11 +196,8 @@ fi # Only remove the lock file after expiry (if system is slow we want to delay # the next import, not have multiple render_expired processes running) #------------------------------------------------------------------------------ - freelock "$LOCK_FILE" + freelock "$LOCK_FILE" m_ok "Done with import" - - - fi diff --git a/run.sh b/run.sh index 81dc5e4..ebdbe67 100755 --- a/run.sh +++ b/run.sh @@ -20,11 +20,28 @@ if [ "$#" -ne 1 ]; then echo "environment variables:" echo " THREADS: defines number of threads used for importing / tile rendering" echo " UPDATES: consecutive updates (enabled/disabled)" + echo " NAME_LUA: name of .lua script to run as part of the style" + echo " NAME_STYLE: name of the .style to use" + echo " NAME_MML: name of the .mml file to render to mapnik.xml" + echo " NAME_SQL: name of the .sql file to use" exit 1 fi set -x +if [ ! "$(ls -A /home/renderer/src/openstreetmap-carto)" ]; then + + mv /home/renderer/src/openstreetmap-carto-backup/* /home/renderer/src/openstreetmap-carto/ + +fi + +if [ ! -f /home/renderer/src/openstreetmap-carto/mapnik.xml ]; then + + cd /home/renderer/src/openstreetmap-carto + carto ${NAME_MML:-project.mml} > mapnik.xml + +fi + if [ "$1" = "import" ]; then # Ensure that database directory is in right state chown postgres:postgres -R /var/lib/postgresql @@ -75,13 +92,14 @@ if [ "$1" = "import" ]; then fi # Import data - sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS:-} + sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS:-} # Create indexes - sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/indexes.sql + sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/${NAME_SQL:-indexes.sql} #Import external data sudo chown -R renderer: /home/renderer/src + sudo -E -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data # Register that data has changed for mod_tile caching purposes