Update Dockerfile (#307)

* Use tile expiration settings from env if set, or default values as before.

* Update openstreetmap-tiles-update-expire.sh

Co-authored-by: galewis2 <62433564+galewis2@users.noreply.github.com>

.github/workflows/build-and-test.yaml

@@ -21,49 +21,65 @@ env:
 
 jobs:
 
-  test:
+  build:
+    strategy:
+      matrix:
+        include:
+        - arch    : amd64
+          mode    : build-and-test
+        - arch    : arm64
+          variant : v8
+          mode    : build-only
     runs-on: ubuntu-latest
     env:
       VOLUME    : osm-db
       CONTAINER : osm-www
       MOUNT     : /data/database/
+      PLATFORM  : linux/${{ matrix.arch }}${{ (matrix.variant != '' && format('/{0}', matrix.variant)) || '' }}
     steps:
     -
       name: Checkout
       uses: actions/checkout@v3
     -
       name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: ${{ matrix.arch }}
     -
       name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
     -
       name: Environment
       run : |
         echo  IMAGE=$(echo ${{ env.IMAGE }} | tr '[:upper:]' '[:lower:]')  >>$GITHUB_ENV
     -
       name: Docker build
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v3
       with:
         pull       : true
-        load       : true
+        load       : ${{ matrix.mode == 'build-and-test' }}
+        platforms  : ${{ env.PLATFORM }}
         context    : .
         file       : ./Dockerfile
-        tags       : ${{env.IMAGE}}:${{env.TAG}}
-        cache-from : type=gha,scope=${{ github.workflow }}
-        cache-to   : type=gha,scope=${{ github.workflow }},mode=max
+        tags       : ${{ env.IMAGE }}:${{ env.TAG }}
+        cache-from : type=gha,scope=${{ github.workflow }}:${{ env.PLATFORM }}
+        cache-to   : type=gha,scope=${{ github.workflow }}:${{ env.PLATFORM }},mode=max
     -
       name: Import Luxembourg
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
         docker  volume  create  ${VOLUME}
         docker  run  --rm  --shm-size=128M  -v ${VOLUME}:${MOUNT}  -e UPDATES=enabled  ${IMAGE}:${TAG}  import
     -
       name: Start server
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
-        docker  run  --rm  --shm-size=128M  -v ${VOLUME}:${MOUNT}  -e UPDATES=enabled  -p 80:80  -d  --name ${CONTAINER}  ${IMAGE}:${TAG}  run
+        docker  run  --shm-size=128M  -v ${VOLUME}:${MOUNT}  -e UPDATES=enabled  -p 80:80  -d  --name ${CONTAINER}  ${IMAGE}:${TAG}  run
         sleep 30
+        docker  logs  ${CONTAINER}
     -
       name: Download tiles
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
         curl  http://localhost/tile/0/0/0.png            --fail  -o 000.png
         curl  http://localhost/tile/1/0/0.png            --fail  -o 100.png
@@ -74,12 +90,14 @@ jobs:
         curl  http://localhost/tile/18/135536/89345.png  --fail  -o example.png
     -
       name: Upload tiles
+      if  : ${{ matrix.mode == 'build-and-test' }}
       uses: actions/upload-artifact@v3
       with:
         name: tiles
         path: '*.png'
     -
       name: Verify tiles
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
         sha1sum  *.png
         sha1sum  --check  <<EOF
@@ -100,6 +118,7 @@ jobs:
         done
     -
       name: Cleanup
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
         docker  rm  --force  --volumes  ${CONTAINER}
         docker  volume  rm  --force  ${VOLUME}
@@ -108,7 +127,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     needs:
-    - test
+    - build
     if: ${{ github.event_name != 'pull_request' }}
     steps:
     -
@@ -122,7 +141,7 @@ jobs:
     -
       name: Docker meta
       id: meta
-      uses: docker/metadata-action@v3
+      uses: docker/metadata-action@v4
       with:
         images: |
           ${{ env.DOCKERHUB_IMAGE }}
@@ -134,33 +153,37 @@ jobs:
           type=semver,pattern={{major}}
     -
       name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: amd64,arm64
     -
       name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
     -
       name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       if: ${{ env.DOCKERHUB_IMAGE != '' }}
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_PASSWORD }}
     -
       name: Login to GHCR
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         registry : ghcr.io
         username : ${{ github.repository_owner }}
         password : ${{ secrets.GITHUB_TOKEN }}
     -
       name: Build and push
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v3
       with:
         pull       : true
         push       : true
+        platforms  : linux/amd64,linux/arm64/v8
         context    : .
         file       : ./Dockerfile
         tags       : ${{ steps.meta.outputs.tags }}
         labels     : ${{ steps.meta.outputs.labels }}
-        cache-from : type=gha,scope=${{ github.workflow }}
-        cache-to   : type=gha,scope=${{ github.workflow }},mode=max
+        cache-from: |
+          type=gha,scope=${{ github.workflow }}:linux/amd64
+          type=gha,scope=${{ github.workflow }}:linux/arm64/v8

Dockerfile

@@ -13,6 +13,10 @@ FROM compiler-common AS compiler-stylesheet
 RUN cd ~ \
 && git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
 && cd openstreetmap-carto \
+&& sed -i 's/, "unifont Medium", "Unifont Upper Medium"//g' style/fonts.mss \
+&& sed -i 's/"Noto Sans Tibetan Regular",//g' style/fonts.mss \
+&& sed -i 's/"Noto Sans Tibetan Bold",//g' style/fonts.mss \
+&& sed -i 's/Noto Sans Syriac Eastern Regular/Noto Sans Syriac Regular/g' style/fonts.mss \
 && rm -rf .git
 
 ###########################################################################################################
@@ -34,6 +38,9 @@ FROM ubuntu:22.04 AS final
 ENV DEBIAN_FRONTEND=noninteractive
 ENV AUTOVACUUM=on
 ENV UPDATES=disabled
+ENV REPLICATION_URL=https://planet.openstreetmap.org/replication/hour/
+ENV MAX_INTERVAL_SECONDS=3600
+
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 # Get packages
@@ -42,6 +49,7 @@ RUN apt-get update \
  apache2 \
  cron \
  dateutils \
+ fonts-hanazono \
  fonts-noto-cjk \
  fonts-noto-hinted \
  fonts-noto-unhinted \
@@ -74,6 +82,12 @@ RUN apt-get update \
 
 RUN adduser --disabled-password --gecos "" renderer
 
+# Get Noto Emoji Regular font, despite it being deprecated by Google
+RUN wget https://github.com/googlefonts/noto-emoji/blob/9a5261d871451f9b5183c93483cbd68ed916b1e9/fonts/NotoEmoji-Regular.ttf?raw=true --content-disposition -P /usr/share/fonts/
+
+# For some reason this one is missing in the default packages
+RUN wget https://github.com/stamen/terrain-classic/blob/master/fonts/unifont-Medium.ttf?raw=true --content-disposition -P /usr/share/fonts/
+
 # Install python libraries
 RUN pip3 install \
  requests \
@@ -129,7 +143,8 @@ TILEDIR=/var/cache/renderd/tiles \n\
 XML=/home/renderer/src/openstreetmap-carto/mapnik.xml \n\
 HOST=localhost \n\
 TILESIZE=256 \n\
-MAXZOOM=20' >> /etc/renderd.conf
+MAXZOOM=20' >> /etc/renderd.conf \
+ && sed -i 's,/usr/share/fonts/truetype,/usr/share/fonts,g' /etc/renderd.conf
 
 # Install helper script
 COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional

README.md

@@ -137,6 +137,29 @@ docker run \
 
 This will enable a background process that automatically downloads changes from the OpenStreetMap server, filters them for the relevant region polygon you specified, updates the database and finally marks the affected tiles for rerendering.
 
+### Tile expiration (optional)
+
+Specify custom tile expiration settings to control which zoom level tiles are marked as expired when an update is performed. Tiles can be marked as expired in the cache (TOUCHFROM), but will still be served
+until a new tile has been rendered, or deleted from the cache (DELETEFROM), so nothing will be served until a new tile has been rendered.
+
+The example tile expiration values below are the default values.
+
+```
+docker run \
+    -p 8080:80 \
+    -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \
+    -e MAX_INTERVAL_SECONDS=60 \
+    -e UPDATES=enabled \
+    -e EXPIRY_MINZOOM=13 \
+    -e EXPIRY_TOUCHFROM=13 \
+    -e EXPIRY_DELETEFROM=19 \
+    -e EXPIRY_MAXZOOM=20 \
+    -v osm-data:/data/database/ \
+    -v osm-tiles:/data/tiles/ \
+    -d overv/openstreetmap-tile-server \
+    run
+```
+
 ### Cross-origin resource sharing
 
 To enable the `Access-Control-Allow-Origin` header to be able to retrieve tiles from other domains, simply set the `ALLOW_CORS` variable to `enabled`:

openstreetmap-tiles-update-expire.sh

@@ -56,14 +56,10 @@ RUNLOG=$LOG_DIR/run.log
 # Here we expire (but don't immediately rerender) tiles between zoom levels
 # 13 and 18 and delete between 19 and 20.
 #------------------------------------------------------------------------------
-EXPIRY_MINZOOM=13
-EXPIRY_TOUCHFROM=13
-EXPIRY_DELETEFROM=19
-EXPIRY_MAXZOOM=20
-
-
-REPLICATION_URL=${REPLICATION_URL:="https://planet.openstreetmap.org/replication/hour/"}
-MAX_INTERVAL_SECONDS=${MAX_INTERVAL_SECONDS:="3600"}
+EXPIRY_MINZOOM=${EXPIRY_MINZOOM:="13"}
+EXPIRY_TOUCHFROM=${EXPIRY_TOUCHFROM:="13"}
+EXPIRY_DELETEFROM=${EXPIRY_DELETEFROM:="19"}
+EXPIRY_MAXZOOM=${EXPIRY_MAXZOOM:="20"}
 
 #*************************************************************************
 #*************************************************************************

run.sh

@@ -78,10 +78,10 @@ if [ "$1" == "import" ]; then
 
     if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
         # determine and set osmosis_replication_timestamp (for consecutive updates)
-        REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -d "=" -f 2`
+        REPLICATION_TIMESTAMP=`osmium fileinfo -g header.option.osmosis_replication_timestamp /data/region.osm.pbf`
 
         # initial setup of osmosis workspace (for consecutive updates)
-        sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
+        sudo -E -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
     fi
 
     # copy polygon file if available