Add a lang parameter to the run script

* Update Dockerfile

add support to PostgreSQL 15 and alter encryption method to scram-sha-256 instead of md5

* Update run.sh

add support to PostgreSQL 15

.github/workflows/build-and-test.yaml

@@ -5,7 +5,7 @@ on:
     branches:
     - master
     tags:
-    - 'v*'
+    - 'v[0-9]+.[0-9]+.[0-9]+'
   pull_request:
     branches:
     - master
@@ -21,49 +21,65 @@ env:
 
 jobs:
 
-  test:
+  build:
+    strategy:
+      matrix:
+        include:
+        - arch    : amd64
+          mode    : build-and-test
+        - arch    : arm64
+          variant : v8
+          mode    : build-only
     runs-on: ubuntu-latest
     env:
       VOLUME    : osm-db
       CONTAINER : osm-www
       MOUNT     : /data/database/
+      PLATFORM  : linux/${{ matrix.arch }}${{ (matrix.variant != '' && format('/{0}', matrix.variant)) || '' }}
     steps:
     -
       name: Checkout
       uses: actions/checkout@v3
     -
       name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: ${{ matrix.arch }}
     -
       name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
     -
       name: Environment
       run : |
         echo  IMAGE=$(echo ${{ env.IMAGE }} | tr '[:upper:]' '[:lower:]')  >>$GITHUB_ENV
     -
       name: Docker build
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v3
       with:
         pull       : true
-        load       : true
+        load       : ${{ matrix.mode == 'build-and-test' }}
+        platforms  : ${{ env.PLATFORM }}
         context    : .
         file       : ./Dockerfile
-        tags       : ${{env.IMAGE}}:${{env.TAG}}
-        cache-from : type=gha,scope=${{ github.workflow }}
-        cache-to   : type=gha,scope=${{ github.workflow }},mode=max
+        tags       : ${{ env.IMAGE }}:${{ env.TAG }}
+        cache-from : type=gha,scope=${{ github.workflow }}:${{ env.PLATFORM }}
+        cache-to   : type=gha,scope=${{ github.workflow }}:${{ env.PLATFORM }},mode=max
     -
       name: Import Luxembourg
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
         docker  volume  create  ${VOLUME}
         docker  run  --rm  --shm-size=128M  -v ${VOLUME}:${MOUNT}  -e UPDATES=enabled  ${IMAGE}:${TAG}  import
     -
       name: Start server
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
-        docker  run  --rm  --shm-size=128M  -v ${VOLUME}:${MOUNT}  -e UPDATES=enabled  -p 80:80  -d  --name ${CONTAINER}  ${IMAGE}:${TAG}  run
+        docker  run  --shm-size=128M  -v ${VOLUME}:${MOUNT}  -e UPDATES=enabled  -p 80:80  -d  --name ${CONTAINER}  ${IMAGE}:${TAG}  run
         sleep 30
+        docker  logs  ${CONTAINER}
     -
       name: Download tiles
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
         curl  http://localhost/tile/0/0/0.png            --fail  -o 000.png
         curl  http://localhost/tile/1/0/0.png            --fail  -o 100.png
@@ -74,12 +90,14 @@ jobs:
         curl  http://localhost/tile/18/135536/89345.png  --fail  -o example.png
     -
       name: Upload tiles
+      if  : ${{ matrix.mode == 'build-and-test' }}
       uses: actions/upload-artifact@v3
       with:
         name: tiles
         path: '*.png'
     -
       name: Verify tiles
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
         sha1sum  *.png
         sha1sum  --check  <<EOF
@@ -100,6 +118,7 @@ jobs:
         done
     -
       name: Cleanup
+      if  : ${{ matrix.mode == 'build-and-test' }}
       run : |
         docker  rm  --force  --volumes  ${CONTAINER}
         docker  volume  rm  --force  ${VOLUME}
@@ -108,7 +127,7 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     needs:
-    - test
+    - build
     if: ${{ github.event_name != 'pull_request' }}
     steps:
     -
@@ -122,7 +141,7 @@ jobs:
     -
       name: Docker meta
       id: meta
-      uses: docker/metadata-action@v3
+      uses: docker/metadata-action@v4
       with:
         images: |
           ${{ env.DOCKERHUB_IMAGE }}
@@ -134,33 +153,37 @@ jobs:
           type=semver,pattern={{major}}
     -
       name: Set up QEMU
-      uses: docker/setup-qemu-action@v1
+      uses: docker/setup-qemu-action@v2
+      with:
+        platforms: amd64,arm64
     -
       name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v2
     -
       name: Login to DockerHub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       if: ${{ env.DOCKERHUB_IMAGE != '' }}
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_PASSWORD }}
     -
       name: Login to GHCR
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         registry : ghcr.io
         username : ${{ github.repository_owner }}
         password : ${{ secrets.GITHUB_TOKEN }}
     -
       name: Build and push
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v3
       with:
         pull       : true
         push       : true
+        platforms  : linux/amd64,linux/arm64/v8
         context    : .
         file       : ./Dockerfile
         tags       : ${{ steps.meta.outputs.tags }}
         labels     : ${{ steps.meta.outputs.labels }}
-        cache-from : type=gha,scope=${{ github.workflow }}
-        cache-to   : type=gha,scope=${{ github.workflow }},mode=max
+        cache-from: |
+          type=gha,scope=${{ github.workflow }}:linux/amd64
+          type=gha,scope=${{ github.workflow }}:linux/arm64/v8

.gitignore

@@ -0,0 +1,9 @@
+pbf/*.pbf
+pbf/*.poly
+!pbf/polyBEbuffer.poly
+pbf/shp/*
+pbf/*.tif
+tiles/*
+!tiles/.gitkeep
+database/*
+!database/.gitkeep

Dockerfile

@@ -1,11 +1,17 @@
 FROM ubuntu:22.04 AS compiler-common
 ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
 
 RUN apt-get update \
 && apt-get install -y --no-install-recommends \
- git-core \
- ca-certificates \
-&& apt-get update
+ ca-certificates gnupg lsb-release locales \
+ wget curl \
+ git-core unzip unrar \
+&& locale-gen $LANG && update-locale LANG=$LANG \
+&& sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
+&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+&& apt-get update && apt-get -y upgrade
 
 ###########################################################################################################
 
@@ -13,6 +19,10 @@ FROM compiler-common AS compiler-stylesheet
 RUN cd ~ \
 && git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
 && cd openstreetmap-carto \
+&& sed -i 's/, "unifont Medium", "Unifont Upper Medium"//g' style/fonts.mss \
+&& sed -i 's/"Noto Sans Tibetan Regular",//g' style/fonts.mss \
+&& sed -i 's/"Noto Sans Tibetan Bold",//g' style/fonts.mss \
+&& sed -i 's/Noto Sans Syriac Eastern Regular/Noto Sans Syriac Regular/g' style/fonts.mss \
 && rm -rf .git
 
 ###########################################################################################################
@@ -27,13 +37,17 @@ RUN mkdir -p /home/renderer/src \
 
 ###########################################################################################################
 
-FROM ubuntu:22.04 AS final
+FROM compiler-common AS final
 
 # Based on
 # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
 ENV DEBIAN_FRONTEND=noninteractive
 ENV AUTOVACUUM=on
 ENV UPDATES=disabled
+ENV REPLICATION_URL=https://planet.openstreetmap.org/replication/hour/
+ENV MAX_INTERVAL_SECONDS=3600
+ENV PG_VERSION 15
+
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 # Get packages
@@ -42,6 +56,7 @@ RUN apt-get update \
  apache2 \
  cron \
  dateutils \
+ fonts-hanazono \
  fonts-noto-cjk \
  fonts-noto-hinted \
  fonts-noto-unhinted \
@@ -55,9 +70,9 @@ RUN apt-get update \
  osm2pgsql \
  osmium-tool \
  osmosis \
- postgresql-14 \
- postgresql-14-postgis-3 \
- postgresql-14-postgis-3-scripts \
+ postgresql-$PG_VERSION \
+ postgresql-$PG_VERSION-postgis-3 \
+ postgresql-$PG_VERSION-postgis-3-scripts \
  postgis \
  python-is-python3 \
  python3-mapnik \
@@ -67,13 +82,19 @@ RUN apt-get update \
  python3-pip \
  renderd \
  sudo \
- wget \
+ vim \
 && apt-get clean autoclean \
 && apt-get autoremove --yes \
 && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 RUN adduser --disabled-password --gecos "" renderer
 
+# Get Noto Emoji Regular font, despite it being deprecated by Google
+RUN wget https://github.com/googlefonts/noto-emoji/blob/9a5261d871451f9b5183c93483cbd68ed916b1e9/fonts/NotoEmoji-Regular.ttf?raw=true --content-disposition -P /usr/share/fonts/
+
+# For some reason this one is missing in the default packages
+RUN wget https://github.com/stamen/terrain-classic/blob/master/fonts/unifont-Medium.ttf?raw=true --content-disposition -P /usr/share/fonts/
+
 # Install python libraries
 RUN pip3 install \
  requests \
@@ -88,10 +109,19 @@ RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/a
 && echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \
 && a2enconf mod_tile && a2enconf mod_headers
 COPY apache.conf /etc/apache2/sites-available/000-default.conf
-COPY leaflet-demo.html /var/www/html/index.html
 RUN ln -sf /dev/stdout /var/log/apache2/access.log \
 && ln -sf /dev/stderr /var/log/apache2/error.log
 
+# leaflet
+COPY leaflet-demo.html /var/www/html/index.html
+RUN cd /var/www/html/ \
+&& wget https://github.com/Leaflet/Leaflet/releases/download/v1.8.0/leaflet.zip \
+&& unzip leaflet.zip \
+&& rm leaflet.zip
+
+# Icon
+RUN wget -O /var/www/html/favicon.ico https://www.openstreetmap.org/favicon.ico
+
 # Copy update scripts
 COPY openstreetmap-tiles-update-expire.sh /usr/bin/
 RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \
@@ -101,11 +131,11 @@ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \
 && echo "* * * * *   renderer    openstreetmap-tiles-update-expire.sh\n" >> /etc/crontab
 
 # Configure PosgtreSQL
-COPY postgresql.custom.conf.tmpl /etc/postgresql/14/main/
+COPY postgresql.custom.conf.tmpl /etc/postgresql/$PG_VERSION/main/
 RUN chown -R postgres:postgres /var/lib/postgresql \
-&& chown postgres:postgres /etc/postgresql/14/main/postgresql.custom.conf.tmpl \
-&& echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \
-&& echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf
+&& chown postgres:postgres /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl \
+&& echo "host all all 0.0.0.0/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf \
+&& echo "host all all ::/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf
 
 # Create volume directories
 RUN mkdir -p /run/renderd/ \
@@ -115,10 +145,10 @@ RUN mkdir -p /run/renderd/ \
   &&  chown  -R  renderer:  /data/  \
   &&  chown  -R  renderer:  /home/renderer/src/  \
   &&  chown  -R  renderer:  /run/renderd  \
-  &&  mv  /var/lib/postgresql/14/main/  /data/database/postgres/  \
+  &&  mv  /var/lib/postgresql/$PG_VERSION/main/  /data/database/postgres/  \
   &&  mv  /var/cache/renderd/tiles/            /data/tiles/     \
   &&  chown  -R  renderer: /data/tiles \
-  &&  ln  -s  /data/database/postgres  /var/lib/postgresql/14/main             \
+  &&  ln  -s  /data/database/postgres  /var/lib/postgresql/$PG_VERSION/main             \
   &&  ln  -s  /data/style              /home/renderer/src/openstreetmap-carto  \
   &&  ln  -s  /data/tiles              /var/cache/renderd/tiles                \
 ;
@@ -129,7 +159,8 @@ TILEDIR=/var/cache/renderd/tiles \n\
 XML=/home/renderer/src/openstreetmap-carto/mapnik.xml \n\
 HOST=localhost \n\
 TILESIZE=256 \n\
-MAXZOOM=20' >> /etc/renderd.conf
+MAXZOOM=20' >> /etc/renderd.conf \
+ && sed -i 's,/usr/share/fonts/truetype,/usr/share/fonts,g' /etc/renderd.conf
 
 # Install helper script
 COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional

README.md

@@ -1,5 +1,110 @@
 # openstreetmap-tile-server
 
+
+## Champs-libres fork
+
+This is an adaptation of https://github.com/Overv/openstreetmap-tile-server/ for testing different cartoCss styles and generating tiles.
+
+### Use with docker compose
+
+#### Fresh install
+
+```bash
+#docker volume create osm-data
+docker compose build
+```
+
+For entering into the container, use the "terminal" argument of the run.sh script:
+
+```bash
+docker compose run --rm map terminal
+```
+
+#### Do an import
+
+Download a pbf and a poly file of your choice on https://download.geofabrik.de and put it in the `pbf` folder. Change the lines of the pbf and poly paths in the volumes in the `docker compose.yml` file
+
+Then, you can import the OSM data by doing this command:
+
+```bash
+docker compose run --rm map import
+```
+
+This will take a while, about 15 min for Luxembourg for example.
+
+You should end with :
+
+```
+...
+INFO:root:  Importing into database
+INFO:root:  Import complete
++ sudo -u renderer touch /data/database/planet-import-complete
++ service postgresql stop
+ * Stopping PostgreSQL 15 database server                                                                                                                                                       [ OK ]
++ exit 0
+```
+
+#### Run the server to produce and view tiles
+
+```bash
+docker compose run --rm map run
+```
+
+The tiles are available on http://192.168.176.2/ or something like that. Have a look at the docker container to know which is the IP address: there is a message like "Could not reliably determine the server's fully qualified domain name, using 192.168.176.2.". (should be on http://localhost:8080/ though...)
+
+#### Generate tiles from a list
+
+There's a script in utils to generate a list of tiles to be used to generate the tiles.
+
+```bash
+python3 generate-tiles-list.py > tiles.list
+```
+
+Move this `tiles.list` file in the `/tiles/` directory and then run:
+
+```bash
+docker compose run --rm map generate_tiles
+```
+
+For Belgium up to zoom 18, it might take >48 hours!
+
+NB: use `cat tiles.list | grep " 18" > tiles_18.list` to split the tiles.list per zoom level
+
+#### Transfer the tiles to tiles.champs-libres.be
+
+
+```bash
+rsync --compress --recursive ./tiles/cyclosm debian@juniperus.champs-libres.be:/var/lib/mod_tile/cyclosm
+```
+
+
+#### Things to do for the osmbe style
+
+##### Clip a europe.pbf file with osmium
+
+We want a buffer around Belgium with some generated tiles. To do so, we clip a larger pbf file like `europe.osm.pbf` with a poly file, which can be generated with http://polygons.openstreetmap.fr/get_poly.py?id=52411&params=0.150000-0.005000-0.005000
+
+Then, we clip the europe pbf with osmium:
+
+```bash
+osmium extract --strategy smart --polygon=polyBEbuffer.poly --output=belgium-buffer.osm.pbf --overwrite /mnt/tera/Projets/OSM/Data/europe-latest.osm.pbf
+```
+
+The `poly` file is kept in the pbf folder.
+
+##### Generate a tiles.list file with the buffer around Belgium
+
+This file was generated with the script `utils/generate-tiles-list-buffer.py`. The result is also kept in this repo in `utils/tiles-BE-buffer.list`.
+
+#### Troubleshooting
+
+##### Role rendered already exists
+
+While importing, if you encounter the error "createuser: error: creation of new role failed: ERROR:  role "renderer" already exists", it is because the db already exist from a previous import, just remove/rename the `postgres` folder into database.
+
+
+## Original README
+
 [![Build Status](https://travis-ci.org/Overv/openstreetmap-tile-server.svg?branch=master)](https://travis-ci.org/Overv/openstreetmap-tile-server) [![](https://images.microbadger.com/badges/image/overv/openstreetmap-tile-server.svg)](https://microbadger.com/images/overv/openstreetmap-tile-server "openstreetmap-tile-server")
 [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/overv/openstreetmap-tile-server?label=docker%20image)](https://hub.docker.com/r/overv/openstreetmap-tile-server/tags)
 
@@ -67,10 +172,10 @@ By default the container will use openstreetmap-carto if it is not specified. Ho
 docker run \
     -e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \
     -e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \
-    -e NAME_LUA=sample.lua
-    -e NAME_STYLE=test.style
-    -e NAME_MML=project.mml
-    -e NAME_SQL=test.sql
+    -e NAME_LUA=sample.lua \
+    -e NAME_STYLE=test.style \
+    -e NAME_MML=project.mml \
+    -e NAME_SQL=test.sql \
     -v /home/user/openstreetmap-carto-modified:/data/style/ \
     -v osm-data:/data/database/ \
     overv/openstreetmap-tile-server \
@@ -101,7 +206,7 @@ Your tiles will now be available at `http://localhost:8080/tile/{z}/{x}/{y}.png`
 
 ### Using Docker Compose
 
-The `docker-compose.yml` file included with this repository shows how the aforementioned command can be used with Docker Compose to run your server.
+The `docker compose.yml` file included with this repository shows how the aforementioned command can be used with Docker Compose to run your server.
 
 ### Preserving rendered tiles
 
@@ -137,6 +242,29 @@ docker run \
 
 This will enable a background process that automatically downloads changes from the OpenStreetMap server, filters them for the relevant region polygon you specified, updates the database and finally marks the affected tiles for rerendering.
 
+### Tile expiration (optional)
+
+Specify custom tile expiration settings to control which zoom level tiles are marked as expired when an update is performed. Tiles can be marked as expired in the cache (TOUCHFROM), but will still be served
+until a new tile has been rendered, or deleted from the cache (DELETEFROM), so nothing will be served until a new tile has been rendered.
+
+The example tile expiration values below are the default values.
+
+```
+docker run \
+    -p 8080:80 \
+    -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \
+    -e MAX_INTERVAL_SECONDS=60 \
+    -e UPDATES=enabled \
+    -e EXPIRY_MINZOOM=13 \
+    -e EXPIRY_TOUCHFROM=13 \
+    -e EXPIRY_DELETEFROM=19 \
+    -e EXPIRY_MAXZOOM=20 \
+    -v osm-data:/data/database/ \
+    -v osm-tiles:/data/tiles/ \
+    -d overv/openstreetmap-tile-server \
+    run
+```
+
 ### Cross-origin resource sharing
 
 To enable the `Access-Control-Allow-Origin` header to be able to retrieve tiles from other domains, simply set the `ALLOW_CORS` variable to `enabled`:

apache.conf

@@ -1,7 +1,7 @@
 <VirtualHost *:80>
     ServerAdmin webmaster@localhost
 
-    AddTileConfig /tile/ default
+    AddTileConfig /tile/ cyclosm
     LoadTileConfigFile /etc/renderd.conf
     ModTileRenderdSocketName /run/renderd/renderd.sock
     ModTileRequestTimeout 0
@@ -9,6 +9,8 @@
 
     DocumentRoot /var/www/html
 
+    ServerName  192.168.192.2
+
     ErrorLog ${APACHE_LOG_DIR}/error.log
     CustomLog ${APACHE_LOG_DIR}/access.log combined
 

docker-compose.yml

@@ -4,11 +4,19 @@ services:
   map:
     image: overv/openstreetmap-tile-server
     volumes:
-      - osm-data:/data/database/
+      #- osm-data:/data/database/
+      - ./database:/data/database/
+      - ./tiles:/data/tiles/
+      - ./osmbe/renderd.conf:/etc/renderd.conf
+      - ./run.sh:/run.sh # for dev
+      - ./pbf/belgium-buffer.osm.pbf:/data/region.osm.pbf # change here your pbf
+      - ./pbf/luxembourg.poly:/data/region.poly # and your poly
+      - ./pbf/shp/:/data/shp/
+      - ./pbf/hillshade.tif:/data/hillshade.tif
     ports:
-      - "8080:80"
+      - "9000:80"
     command: "run"
 
-volumes:
-  osm-data:
-    external: true
+# volumes:
+#   osm-data:
+#     external: true

leaflet-demo.html

@@ -6,8 +6,8 @@
         <meta charset="utf-8" />
         <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-        <link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css" integrity="sha512-xodZBNTC5n17Xt2atTPuE1HxjVMSvLVW9ocqUKLsCC5CXdbqCmblAshOMAS6/keqq/sMZMZ19scR4PsZChSR7A==" crossorigin="" />
-        <script src="https://unpkg.com/leaflet@1.7.1/dist/leaflet.js" integrity="sha512-XQoYMqMTK8LvdxXYG3nZ448hOEQiglfqkJs1NOQV44cWnUrBc8PkAOcXy20w0vlaXaVUearIOBhiXZ5V3ynxwA==" crossorigin=""></script>
+        <link rel="stylesheet" href="leaflet.css"/>
+        <script src="leaflet.js"></script>
 
         <style>
             html, body, #map {

openstreetmap-tiles-update-expire.sh

@@ -56,14 +56,10 @@ RUNLOG=$LOG_DIR/run.log
 # Here we expire (but don't immediately rerender) tiles between zoom levels
 # 13 and 18 and delete between 19 and 20.
 #------------------------------------------------------------------------------
-EXPIRY_MINZOOM=13
-EXPIRY_TOUCHFROM=13
-EXPIRY_DELETEFROM=19
-EXPIRY_MAXZOOM=20
-
-
-REPLICATION_URL=${REPLICATION_URL:="https://planet.openstreetmap.org/replication/hour/"}
-MAX_INTERVAL_SECONDS=${MAX_INTERVAL_SECONDS:="3600"}
+EXPIRY_MINZOOM=${EXPIRY_MINZOOM:="13"}
+EXPIRY_TOUCHFROM=${EXPIRY_TOUCHFROM:="13"}
+EXPIRY_DELETEFROM=${EXPIRY_DELETEFROM:="19"}
+EXPIRY_MAXZOOM=${EXPIRY_MAXZOOM:="20"}
 
 #*************************************************************************
 #*************************************************************************
@@ -195,7 +191,7 @@ m_ok "expiring tiles"
 # delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM.
 # The default path to renderd.sock is fixed.
 #------------------------------------------------------------------------------
-if ! render_expired --map=default --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
+if ! render_expired --map=default --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
     m_info "Expiry failed"
 fi
 

osmbe/renderd.conf

@@ -0,0 +1,37 @@
+; BASIC AND SIMPLE CONFIGURATION:
+
+[renderd]
+stats_file=/run/renderd/renderd.stats
+socketname=/run/renderd/renderd.sock
+num_threads=4
+tile_dir=/var/cache/renderd/tiles
+
+[mapnik]
+plugins_dir=/usr/lib/mapnik/3.1/input
+font_dir=/usr/share/fonts
+font_dir_recurse=true
+
+; ADD YOUR LAYERS:
+[osmbe]
+URI=/tile/
+TILEDIR=/var/cache/renderd/tiles
+XML=/data/style/mapnik.xml
+HOST=localhost
+TILESIZE=256
+MAXZOOM=20
+
+[osmbe_nl]
+URI=/tile_nl/
+TILEDIR=/var/cache/renderd/tiles
+XML=/data/style/mapnik-nl.xml
+HOST=localhost
+TILESIZE=256
+MAXZOOM=20
+
+[osmbe_fr]
+URI=/tile_fr/
+TILEDIR=/var/cache/renderd/tiles
+XML=/data/style/mapnik-fr.xml
+HOST=localhost
+TILESIZE=256
+MAXZOOM=20

pbf/osmium_extract.sh

@@ -0,0 +1 @@
+osmium extract --strategy smart --polygon=polyBEbuffer.poly --output=belgium-buffer.osm.pbf --overwrite europe-latest.osm.pbf

pbf/polyBEbuffer.poly

@@ -0,0 +1,196 @@
+polygon
+1
+  2.275        51.17
+  2.245        51.22
+  2.24         51.28
+  2.255        51.335
+  2.29         51.385
+  2.53         51.48
+  2.565        51.485
+  2.68         51.545
+  2.8          51.585
+  2.965        51.675
+  3.06         51.7
+  3.15         51.685
+  3.275        51.62
+  3.35         51.59
+  3.395        51.56
+  3.505        51.435
+  3.555        51.45
+  3.61         51.455
+  3.74         51.42
+  3.795        51.415
+  3.915        51.37
+  3.93         51.37
+  3.97         51.39
+  4.03         51.395
+  4.055        51.405
+  4.075        51.415
+  4.09         51.455
+  4.11         51.48
+  4.155        51.51
+  4.21         51.525
+  4.25         51.525
+  4.28         51.56
+  4.325        51.59
+  4.46         51.63
+  4.585        51.625
+  4.63         51.6
+  4.64         51.6
+  4.685        51.635
+  4.745        51.655
+  4.8          51.655
+  4.84         51.645
+  4.885        51.63
+  4.925        51.605
+  4.94         51.605
+  4.995        51.635
+  5.085        51.63
+  5.16         51.6
+  5.185        51.58
+  5.225        51.525
+  5.25         51.465
+  5.33         51.425
+  5.385        51.42
+  5.435        51.445
+  5.465        51.45
+  5.56         51.44
+  5.61         51.415
+  5.62         51.4
+  5.655        51.38
+  5.685        51.345
+  5.71         51.335
+  5.73         51.34
+  5.79         51.335
+  5.835        51.315
+  5.9          51.3
+  5.965        51.25
+  5.985        51.225
+  6.005        51.165
+  6            51.105
+  5.965        51.025
+  5.915        50.965
+  5.905        50.92
+  5.94         50.91
+  6.045        50.905
+  6.09         50.89
+  6.11         50.875
+  6.18         50.86
+  6.205        50.845
+  6.24         50.805
+  6.315        50.785
+  6.355        50.765
+  6.39         50.73
+  6.41         50.695
+  6.425        50.62
+  6.465        50.585
+  6.5          50.54
+  6.515        50.505
+  6.525        50.48
+  6.525        50.425
+  6.545        50.4
+  6.56         50.345
+  6.555        50.295
+  6.545        50.26
+  6.505        50.21
+  6.43         50.17
+  6.375        50.155
+  6.33         50.125
+  6.31         50.09
+  6.225        50.005
+  6.16         49.98
+  6.11         49.98
+  6.04         50.005
+  6.01         49.99
+  5.975        49.915
+  5.935        49.86
+  5.98         49.83
+  6.02         49.78
+  6.055        49.705
+  6.06         49.65
+  6.05         49.595
+  6.015        49.52
+  5.985        49.475
+  5.95         49.445
+  5.875        49.405
+  5.81         49.395
+  5.8          49.4
+  5.77         49.39
+  5.705        49.39
+  5.68         49.37
+  5.65         49.36
+  5.59         49.355
+  5.55         49.365
+  5.515        49.35
+  5.475        49.345
+  5.435        49.355
+  5.435        49.35
+  5.365        49.39
+  5.33         49.425
+  5.31         49.46
+  5.265        49.47
+  5.21         49.5
+  5.18         49.545
+  5.135        49.545
+  5.085        49.565
+  5.04         49.59
+  5.005        49.625
+  4.965        49.64
+  4.94         49.635
+  4.83         49.64
+  4.76         49.675
+  4.72         49.72
+  4.71         49.745
+  4.7          49.805
+  4.705        49.82
+  4.695        49.845
+  4.64         49.84
+  4.565        49.805
+  4.425        49.79
+  4.295        49.815
+  4.18         49.805
+  4.15         49.81
+  4.06         49.85
+  4.035        49.87
+  3.995        49.945
+  3.985        49.995
+  3.995        50.065
+  3.975        50.145
+  3.99         50.19
+  3.9          50.175
+  3.82         50.19
+  3.745        50.155
+  3.705        50.155
+  3.64         50.17
+  3.605        50.19
+  3.555        50.24
+  3.51         50.34
+  3.505        50.335
+  3.47         50.34
+  3.445        50.35
+  3.365        50.34
+  3.22         50.39
+  3.18         50.42
+  3.145        50.47
+  3.125        50.55
+  3.105        50.585
+  3.1          50.61
+  3.08         50.625
+  3.05         50.62
+  3.01         50.58
+  2.97         50.555
+  2.94         50.545
+  2.87         50.545
+  2.745        50.58
+  2.69         50.61
+  2.645        50.665
+  2.615        50.665
+  2.56         50.68
+  2.485        50.75
+  2.455        50.8
+  2.44         50.94
+  2.43         50.955
+  2.42         51.005
+  2.275        51.17
+END
+END

run.sh

@@ -3,15 +3,34 @@
 set -euo pipefail
 
 function createPostgresConfig() {
-  cp /etc/postgresql/14/main/postgresql.custom.conf.tmpl /etc/postgresql/14/main/conf.d/postgresql.custom.conf
-  sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/14/main/conf.d/postgresql.custom.conf
-  cat /etc/postgresql/14/main/conf.d/postgresql.custom.conf
+  cp /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
+  sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
+  cat /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
 }
 
 function setPostgresPassword() {
     sudo -u postgres psql -c "ALTER USER renderer PASSWORD '${PGPASSWORD:-renderer}'"
 }
 
+function compileStyle() {
+    local lang=$1
+    cd /data/style/
+    git clone --single-branch --branch update-2025 https://github.com/nobohan/openstreetmap-carto-be --depth 1
+    mkdir /data/style/patterns/
+    mv /data/style/openstreetmap-carto-be/patterns/* /data/style/patterns/
+    mkdir /data/style/symbols/
+    mv /data/style/openstreetmap-carto-be/symbols/* /data/style/symbols/
+    mkdir /data/style/scripts/
+    mv /data/style/openstreetmap-carto-be/scripts/* /data/style/scripts/
+    mv /data/style/openstreetmap-carto-be/external-data.yml /data/style/external-data.yml
+    if [ -z "$lang" ]; then
+        carto openstreetmap-carto-be/project.mml > mapnik.xml
+        return 1
+    fi
+    carto "openstreetmap-carto-be/project-${lang}.mml" > mapnik-${lang}.xml
+}
+
+
 if [ "$#" -ne 1 ]; then
     echo "usage: <import|run>"
     echo "commands:"
@@ -29,24 +48,20 @@ fi
 
 set -x
 
-# if there is no custom style mounted, then use osm-carto
-if [ ! "$(ls -A /data/style/)" ]; then
-    mv /home/renderer/src/openstreetmap-carto-backup/* /data/style/
-fi
-
-# carto build
-if [ ! -f /data/style/mapnik.xml ]; then
-    cd /data/style/
-    carto ${NAME_MML:-project.mml} > mapnik.xml
+if [ "$1" == "terminal" ]; then
+    bash
 fi
 
 if [ "$1" == "import" ]; then
+
+    compileStyle $2
+
     # Ensure that database directory is in right state
     mkdir -p /data/database/postgres/
     chown renderer: /data/database/
     chown -R postgres: /var/lib/postgresql /data/database/postgres/
     if [ ! -f /data/database/postgres/PG_VERSION ]; then
-        sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
+        sudo -u postgres /usr/lib/postgresql/$PG_VERSION/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
     fi
 
     # Initialize PostgreSQL
@@ -58,6 +73,7 @@ if [ "$1" == "import" ]; then
     sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore;"
     sudo -u postgres psql -d gis -c "ALTER TABLE geometry_columns OWNER TO renderer;"
     sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO renderer;"
+    sudo -u postgres psql -d gis -c "SET work_mem TO '250MB';"
     setPostgresPassword
 
     # Download Luxembourg as sample if no data is provided
@@ -78,10 +94,10 @@ if [ "$1" == "import" ]; then
 
     if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
         # determine and set osmosis_replication_timestamp (for consecutive updates)
-        REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -d "=" -f 2`
+        REPLICATION_TIMESTAMP=`osmium fileinfo -g header.option.osmosis_replication_timestamp /data/region.osm.pbf`
 
         # initial setup of osmosis workspace (for consecutive updates)
-        sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
+        sudo -E -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
     fi
 
     # copy polygon file if available
@@ -96,10 +112,7 @@ if [ "$1" == "import" ]; then
     fi
 
     # Import data
-    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore  \
-      --tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua}  \
-      --number-processes ${THREADS:-4}  \
-      -S /data/style/${NAME_STYLE:-openstreetmap-carto.style}  \
+    sudo -u renderer osm2pgsql -d gis --create --hstore -G -S openstreetmap-carto-be/openstreetmap-carto.style --tag-transform-script openstreetmap-carto-be/openstreetmap-carto.lua  \
       /data/region.osm.pbf  \
       ${OSM2PGSQL_EXTRA_ARGS:-}  \
     ;
@@ -133,6 +146,8 @@ if [ "$1" == "run" ]; then
     # Clean /tmp
     rm -rf /tmp/*
 
+    compileStyle $2
+
     # migrate old files
     if [ -f /data/database/PG_VERSION ] && ! [ -d /data/database/postgres/ ]; then
         mkdir /data/database/postgres/
@@ -168,7 +183,7 @@ if [ "$1" == "run" ]; then
     setPostgresPassword
 
     # Configure renderd threads
-    sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf
+    #sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf
 
     # start cron job to trigger consecutive updates
     if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
@@ -195,5 +210,24 @@ if [ "$1" == "run" ]; then
     exit 0
 fi
 
+if [ "$1" == "generate_tiles" ]; then
+
+    compileStyle $2
+
+    service postgresql start
+    service apache2 restart
+
+    sudo -u renderer renderd -c /etc/renderd.conf
+
+    if [ -z "$2" ]; then
+        render_list --map osmbe -n 36 --min-zoom=0 --max-zoom=7 --all
+        cat /data/tiles/osmbe/tiles.list | render_list --map osmbe -n 36
+        bash
+
+    render_list --map osmbe_${$2} -n 36 --min-zoom=0 --max-zoom=7 --all
+    cat /data/tiles/osmbe/tiles.list | render_list --map osmbe_${$2} -n 36
+    bash
+fi
+
 echo "invalid command"
 exit 1

utils/BE_buffer_polygons.osm.fr.geojson

@@ -0,0 +1 @@
+{"type":"Polygon","coordinates":[[[2.275,51.17],[2.245,51.22],[2.24,51.28],[2.255,51.335],[2.29,51.385],[2.53,51.48],[2.565,51.485],[2.68,51.545],[2.8,51.585],[2.965,51.675],[3.06,51.7],[3.15,51.685],[3.275,51.62],[3.35,51.59],[3.395,51.56],[3.505,51.435],[3.555,51.45],[3.61,51.455],[3.74,51.42],[3.795,51.415],[3.915,51.37],[3.93,51.37],[3.97,51.39],[4.03,51.395],[4.055,51.405],[4.075,51.415],[4.09,51.455],[4.11,51.48],[4.155,51.51],[4.21,51.525],[4.25,51.525],[4.28,51.56],[4.325,51.59],[4.46,51.63],[4.585,51.625],[4.63,51.6],[4.64,51.6],[4.685,51.635],[4.745,51.655],[4.8,51.655],[4.84,51.645],[4.885,51.63],[4.925,51.605],[4.94,51.605],[4.995,51.635],[5.085,51.63],[5.16,51.6],[5.185,51.58],[5.225,51.525],[5.25,51.465],[5.33,51.425],[5.385,51.42],[5.435,51.445],[5.465,51.45],[5.56,51.44],[5.61,51.415],[5.62,51.4],[5.655,51.38],[5.685,51.345],[5.71,51.335],[5.73,51.34],[5.79,51.335],[5.835,51.315],[5.9,51.3],[5.965,51.25],[5.985,51.225],[6.005,51.165],[6,51.105],[5.965,51.025],[5.915,50.965],[5.905,50.92],[5.94,50.91],[6.045,50.905],[6.09,50.89],[6.11,50.875],[6.18,50.86],[6.205,50.845],[6.24,50.805],[6.315,50.785],[6.355,50.765],[6.39,50.73],[6.41,50.695],[6.425,50.62],[6.465,50.585],[6.5,50.54],[6.515,50.505],[6.525,50.48],[6.525,50.425],[6.545,50.4],[6.56,50.345],[6.555,50.295],[6.545,50.26],[6.505,50.21],[6.43,50.17],[6.375,50.155],[6.33,50.125],[6.31,50.09],[6.225,50.005],[6.16,49.98],[6.11,49.98],[6.04,50.005],[6.01,49.99],[5.975,49.915],[5.935,49.86],[5.98,49.83],[6.02,49.78],[6.055,49.705],[6.06,49.65],[6.05,49.595],[6.015,49.52],[5.985,49.475],[5.95,49.445],[5.875,49.405],[5.81,49.395],[5.8,49.4],[5.77,49.39],[5.705,49.39],[5.68,49.37],[5.65,49.36],[5.59,49.355],[5.55,49.365],[5.515,49.35],[5.475,49.345],[5.435,49.355],[5.435,49.35],[5.365,49.39],[5.33,49.425],[5.31,49.46],[5.265,49.47],[5.21,49.5],[5.18,49.545],[5.135,49.545],[5.085,49.565],[5.04,49.59],[5.005,49.625],[4.965,49.64],[4.94,49.635],[4.83,49.64],[4.76,49.675],[4.72,49.72],[4.71,49.745],[4.7,49.805],[4.705,49.82],[4.695,49.845],[4.64,49.84],[4.565,49.805],[4.425,49.79],[4.295,49.815],[4.18,49.805],[4.15,49.81],[4.06,49.85],[4.035,49.87],[3.995,49.945],[3.985,49.995],[3.995,50.065],[3.975,50.145],[3.99,50.19],[3.9,50.175],[3.82,50.19],[3.745,50.155],[3.705,50.155],[3.64,50.17],[3.605,50.19],[3.555,50.24],[3.51,50.34],[3.505,50.335],[3.47,50.34],[3.445,50.35],[3.365,50.34],[3.22,50.39],[3.18,50.42],[3.145,50.47],[3.125,50.55],[3.105,50.585],[3.1,50.61],[3.08,50.625],[3.05,50.62],[3.01,50.58],[2.97,50.555],[2.94,50.545],[2.87,50.545],[2.745,50.58],[2.69,50.61],[2.645,50.665],[2.615,50.665],[2.56,50.68],[2.485,50.75],[2.455,50.8],[2.44,50.94],[2.43,50.955],[2.42,51.005],[2.275,51.17]]]}

utils/generate-tiles-list-buffer.py

@@ -0,0 +1,51 @@
+#!/bin/python3
+# from https://github.com/osmbe/openstreetmap-carto-be/blob/ansible/roles/render/templates/make_render_list.py
+
+import mercantile
+import requests
+from shapely.geometry import (shape, box)
+
+url = "http://polygons.openstreetmap.fr/get_geojson.py?id=52411&params=0.150000-0.005000-0.005000"
+zooms = range(0, 18+1) # we make +1 to include zoom 18
+
+# get the geojson
+r = requests.get(url)
+
+# get the shape
+geom = shape(r.json())
+bounds = geom.bounds
+north = bounds[3]
+south = bounds[1]
+west  = bounds[0]
+east  = bounds[2]
+# print(f"north: { north }, south: { south }, east: { east }, west: { west }")
+# print(zooms)
+
+# cache the processed
+processed = set()
+
+# loop over all tiles
+x = 0
+for t in mercantile.tiles(west, south, east, north, zooms):
+
+    tile_bbox = mercantile.bounds(t)
+    bbox = box(tile_bbox.west, tile_bbox.south, tile_bbox.east, tile_bbox.north)
+
+    if geom.intersects(bbox):
+        x_ul = t.x - t.x % 8
+        y_ul = t.y - t.y % 8
+        if (x_ul, y_ul, t.z) not in processed:
+            print(f"{x_ul} {y_ul} {t.z}")
+            processed.add((x_ul, y_ul, t.z))
+#    for l in f.readlines():
+#        x_str, y_str, z_str = l.split(' ')
+#        # convert to in
+#        x_int = int(x_str)
+#        y_int = int(y_str)
+#        z_int = int(z_str)
+#        x_ul  = x_int - x_int % 8
+#        y_ul  = y_int - y_int % 8
+#
+#        if (x_ul, y_ul, z_int) not in processed:
+#            print(f"{x_ul} {y_ul} {z_int}")
+#            processed.add((x_ul, y_ul, z_int))

utils/generate-tiles-list.py

@@ -0,0 +1,85 @@
+# usage: python3 generate-tiles-list.py > tiles.list
+
+import math
+
+def deg2num(lat_deg, lon_deg, zoom):
+  lat_rad = math.radians(lat_deg)
+  n = 2.0 ** zoom
+  xtile = int((lon_deg + 180.0) / 360.0 * n)
+  ytile = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n)
+
+  return (xtile, ytile)
+
+# GD lux coordinates
+max_x = 6.531
+max_y = 50.184
+min_x = 5.735
+min_y = 49.452
+
+# BE ardenne coordinates
+
+max_x = 6.40
+max_y = 50.77
+min_x = 4.13
+min_y = 49.49
+
+# BE coordinates
+
+min_x = 2.54539
+max_y = 51.50543
+min_y = 49.49
+max_x = 6.40
+
+# BE buffer coordinates
+min_x = 2.38246
+max_y = 51.64907
+min_y = 49.35438
+max_x = 6.55565
+
+#for low zoom levels BE
+min_x = 0
+max_y = 70
+min_y = 20
+max_x = 30
+
+min_z = 2
+max_z = 9
+
+for z in range(min_z, max_z+1):
+    x1_tile, y1_tile = deg2num(max_y, max_x, z)
+    x2_tile, y2_tile = deg2num(min_y, min_x, z)
+    min_x_tile = x1_tile if x1_tile < x2_tile else x2_tile
+    max_x_tile = x1_tile if x1_tile > x2_tile else x2_tile
+    min_y_tile = y1_tile if y1_tile < y2_tile else y2_tile
+    max_y_tile = y1_tile if y1_tile > y2_tile else y2_tile
+    for tx in range(min_x_tile, max_x_tile):
+        if tx % 8 != 0:
+            continue
+        for ty in range(min_y_tile, max_y_tile):
+            if ty % 8 != 0:
+                continue
+            print("{} {} {}".format(tx, ty, z))
+
+# BE buffer coordinates
+min_x = 2.38246
+max_y = 51.64907
+min_y = 49.35438
+max_x = 6.55565
+
+min_z = 10
+max_z = 18
+
+for z in range(min_z, max_z+1):
+    x1_tile, y1_tile = deg2num(max_y, max_x, z)
+    x2_tile, y2_tile = deg2num(min_y, min_x, z)
+    min_x_tile = x1_tile if x1_tile < x2_tile else x2_tile
+    max_x_tile = x1_tile if x1_tile > x2_tile else x2_tile
+    min_y_tile = y1_tile if y1_tile < y2_tile else y2_tile
+    max_y_tile = y1_tile if y1_tile > y2_tile else y2_tile
+    for tx in range(min_x_tile, max_x_tile):
+        if tx % 8 != 0:
+            continue
+        for ty in range(min_y_tile, max_y_tile):
+            if ty % 8 != 0:
+                continue
+            print("{} {} {}".format(tx, ty, z))