Allow easier modification of the OpenStreetMap-carto style (#243)

Dockerfile

@@ -80,14 +80,10 @@ RUN cd ~ \
 ###########################################################################################################
 
 FROM compiler-common AS compiler-stylesheet
-RUN apt-get install -y --no-install-recommends \
- npm
 RUN cd ~ \
 && git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
 && cd openstreetmap-carto \
-&& sed -ie 's#https:\/\/naciscdn.org\/naturalearth\/110m\/cultural\/ne_110m_admin_0_boundary_lines_land.zip#https:\/\/naturalearth.s3.amazonaws.com\/110m_cultural\/ne_110m_admin_0_boundary_lines_land.zip#g' external-data.yml \
+&& rm -rf .git
-&& npm install -g carto@1.2.0 \
-&& carto project.mml > mapnik.xml
 
 ###########################################################################################################
 
@@ -96,7 +92,6 @@ RUN mkdir -p /home/renderer/src \
 && cd /home/renderer/src \
 && git clone https://github.com/zverik/regional \
 && cd regional \
-&& git checkout 889d630a1e1a1bacabdd1dad6e17b49e7d58cd4b \
 && rm -rf .git \
 && chmod u+x /home/renderer/src/regional/trim_osc.py
 
@@ -124,6 +119,7 @@ RUN apt-get update \
  liblua5.3-dev \
  lua5.3 \
  mapnik-utils \
+ npm \
  osmium-tool \
  osmosis \
  python-is-python3 \
@@ -148,8 +144,12 @@ RUN adduser --disabled-password --gecos "" renderer
 # Install python libraries
 RUN pip3 install \
  requests \
+ osmium \
  pyyaml
  
+# Install carto for stylesheet
+RUN npm install -g carto@0.18.2
+
 # Configure Apache
 RUN mkdir /var/lib/mod_tile \
 && chown renderer /var/lib/mod_tile \
@@ -206,16 +206,17 @@ RUN dpkg -i renderd_1-1_amd64.deb \
 # Install mod_tile
 COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb .
 RUN dpkg -i mod-tile_1-1_amd64.deb \
+ && mkdir -p /home/renderer/src/openstreetmap-carto \
  && ldconfig \
  && rm mod-tile_1-1_amd64.deb
-COPY --from=compiler-modtile-renderd /root/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag
 
-# Install stylesheet
+COPY --from=compiler-modtile-renderd /root/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag
-COPY --from=compiler-stylesheet /root/openstreetmap-carto /home/renderer/src/openstreetmap-carto
 
 # Install helper script
 COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional
 
+COPY --from=compiler-stylesheet /root/openstreetmap-carto /home/renderer/src/openstreetmap-carto-backup
+
 # Start running
 COPY run.sh /
 ENTRYPOINT ["/run.sh"]

README.md

@@ -36,6 +36,7 @@ docker run \
     -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
     -v /absolute/path/to/luxembourg.poly:/data.poly \
     -v openstreetmap-data:/var/lib/postgresql/14/main \
+    -v openstreetmap-rendered-tiles:/var/lib/mod_tile \
     overv/openstreetmap-tile-server \
     import
 ```
@@ -55,6 +56,32 @@ docker run \
     import
 ```
 
+### Using an alternate style
+
+By default the container will use openstreetmap-carto if it is not specified. However, you can modify the style at run-time. Be aware you need the style mounted at `run` AND `import` as the Lua script needs to be run:
+
+```
+docker run \
+    -e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \
+    -e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \
+    -e NAME_LUA=sample.lua
+    -e NAME_STYLE=test.style
+    -e NAME_MML=project.mml
+    -e NAME_SQL=test.sql
+    -v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto \
+    -v openstreetmap-data:/var/lib/postgresql/12/main \
+    overv/openstreetmap-tile-server \
+    import
+```
+
+If you do not define the "NAME_*" variables, the script will default to those found in the openstreetmap-carto style.
+
+Be sure to mount the volume during `run` with the same `-v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto`
+
+If you do not see the expected style upon `run` double check your paths as the style may not have been found at the directory specified. By default, `openstreetmap-carto` will be used if a style cannot be found
+
+**Only openstreetmap-carto and styles like it, eg, ones with one lua script, one style, one mml, one SQL can be used**
+
 ## Running the server
 
 Run the server like this:

openstreetmap-tiles-update-expire

@@ -20,7 +20,7 @@ OSM2PGSQL_BIN=osm2pgsql
 TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py
 
 DBNAME=gis
-OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style ${OSM2PGSQL_EXTRA_ARGS}"
+OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} ${OSM2PGSQL_EXTRA_ARGS}"
 
 #------------------------------------------------------------------------------
 # When using trim_osc.py we can define either a bounding box (such as this
@@ -30,7 +30,6 @@ OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/
 #------------------------------------------------------------------------------
 TRIM_POLY_FILE="/var/lib/mod_tile/data.poly"
 TRIM_OPTIONS="-d $DBNAME"
-#TRIM_REGION_OPTIONS="-b -14.17 48.85 2.12 61.27"
 TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE"
 
 BASE_DIR=/var/lib/mod_tile
@@ -107,9 +106,12 @@ freelock()
 
 if [ $# -eq 1 ] ; then
     m_info "Initialising Osmosis replication system to $1"
-    mkdir $WORKOSM_DIR
+    mkdir -p $WORKOSM_DIR
+
     $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"
+
     wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt
+
     mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt
     sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt
 else
@@ -167,12 +169,12 @@ fi
 
 #------------------------------------------------------------------------------
 # The lockfile is normally removed before we expire tiles because that is
-# something thatcan be done in parallel with further processing.  In order to
+# something that can be done in parallel with further processing.  In order to
 # avoid rework, if actually rerendering is done rather than just deleting or
 # dirtying, it makes sense to move it lower down.
 #------------------------------------------------------------------------------
-#    m_ok "Import complete; removing lock file"
+#   m_ok "Import complete; removing lock file"
-#    freelock "$LOCK_FILE"
+#   freelock "$LOCK_FILE"
 
     m_ok "expiring tiles"
 #------------------------------------------------------------------------------
@@ -194,11 +196,8 @@ fi
 # Only remove the lock file after expiry (if system is slow we want to delay
 # the next import, not have multiple render_expired processes running)
 #------------------------------------------------------------------------------
-   freelock "$LOCK_FILE"
+    freelock "$LOCK_FILE"
 
     m_ok "Done with import"
 
-
-
-
 fi

run.sh

@@ -20,11 +20,28 @@ if [ "$#" -ne 1 ]; then
     echo "environment variables:"
     echo "    THREADS: defines number of threads used for importing / tile rendering"
     echo "    UPDATES: consecutive updates (enabled/disabled)"
+    echo "    NAME_LUA: name of .lua script to run as part of the style"
+    echo "    NAME_STYLE: name of the .style to use"
+    echo "    NAME_MML: name of the .mml file to render to mapnik.xml"
+    echo "    NAME_SQL: name of the .sql file to use"
     exit 1
 fi
 
 set -x
 
+if [ ! "$(ls -A /home/renderer/src/openstreetmap-carto)" ]; then
+
+    mv /home/renderer/src/openstreetmap-carto-backup/* /home/renderer/src/openstreetmap-carto/
+
+fi
+
+if [ ! -f /home/renderer/src/openstreetmap-carto/mapnik.xml ]; then
+
+    cd /home/renderer/src/openstreetmap-carto
+    carto ${NAME_MML:-project.mml} > mapnik.xml
+
+fi
+
 if [ "$1" = "import" ]; then
     # Ensure that database directory is in right state
     chown postgres:postgres -R /var/lib/postgresql
@@ -75,13 +92,14 @@ if [ "$1" = "import" ]; then
     fi
 
     # Import data
-    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS:-}
+    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS:-}
 
     # Create indexes
-    sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/indexes.sql
+    sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/${NAME_SQL:-indexes.sql}
 
     #Import external data
     sudo chown -R renderer: /home/renderer/src
+
     sudo -E -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data
 
     # Register that data has changed for mod_tile caching purposes