upd README for rsync

* Update Dockerfile

add support to PostgreSQL 15 and alter encryption method to scram-sha-256 instead of md5

* Update run.sh

add support to PostgreSQL 15

.github/workflows/build-and-test.yaml

@@ -5,7 +5,7 @@ on:
     branches:
     - master
     tags:
-    - 'v*'
+    - 'v[0-9]+.[0-9]+.[0-9]+'
   pull_request:
     branches:
     - master

.gitignore

@@ -0,0 +1,8 @@
+pbf/*.pbf
+pbf/*.poly
+pbf/shp/*
+pbf/*.tif
+tiles/*
+!tiles/.gitkeep
+database/*
+!database/.gitkeep

Dockerfile

@@ -1,11 +1,17 @@
 FROM ubuntu:22.04 AS compiler-common
 ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
 
 RUN apt-get update \
 && apt-get install -y --no-install-recommends \
- git-core \
- ca-certificates \
-&& apt-get update
+ ca-certificates gnupg lsb-release locales \
+ wget curl \
+ git-core unzip unrar \
+&& locale-gen $LANG && update-locale LANG=$LANG \
+&& sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
+&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+&& apt-get update && apt-get -y upgrade
 
 ###########################################################################################################
 
@@ -31,7 +37,7 @@ RUN mkdir -p /home/renderer/src \
 
 ###########################################################################################################
 
-FROM ubuntu:22.04 AS final
+FROM compiler-common AS final
 
 # Based on
 # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
@@ -40,6 +46,7 @@ ENV AUTOVACUUM=on
 ENV UPDATES=disabled
 ENV REPLICATION_URL=https://planet.openstreetmap.org/replication/hour/
 ENV MAX_INTERVAL_SECONDS=3600
+ENV PG_VERSION 15
 
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
@@ -63,9 +70,9 @@ RUN apt-get update \
  osm2pgsql \
  osmium-tool \
  osmosis \
- postgresql-14 \
- postgresql-14-postgis-3 \
- postgresql-14-postgis-3-scripts \
+ postgresql-$PG_VERSION \
+ postgresql-$PG_VERSION-postgis-3 \
+ postgresql-$PG_VERSION-postgis-3-scripts \
  postgis \
  python-is-python3 \
  python3-mapnik \
@@ -75,7 +82,7 @@ RUN apt-get update \
  python3-pip \
  renderd \
  sudo \
- wget \
+ vim \
 && apt-get clean autoclean \
 && apt-get autoremove --yes \
 && rm -rf /var/lib/{apt,dpkg,cache,log}/
@@ -102,10 +109,19 @@ RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/a
 && echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \
 && a2enconf mod_tile && a2enconf mod_headers
 COPY apache.conf /etc/apache2/sites-available/000-default.conf
-COPY leaflet-demo.html /var/www/html/index.html
 RUN ln -sf /dev/stdout /var/log/apache2/access.log \
 && ln -sf /dev/stderr /var/log/apache2/error.log
 
+# leaflet
+COPY leaflet-demo.html /var/www/html/index.html
+RUN cd /var/www/html/ \
+&& wget https://github.com/Leaflet/Leaflet/releases/download/v1.8.0/leaflet.zip \
+&& unzip leaflet.zip \
+&& rm leaflet.zip
+
+# Icon
+RUN wget -O /var/www/html/favicon.ico https://www.openstreetmap.org/favicon.ico
+
 # Copy update scripts
 COPY openstreetmap-tiles-update-expire.sh /usr/bin/
 RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \
@@ -115,11 +131,11 @@ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \
 && echo "* * * * *   renderer    openstreetmap-tiles-update-expire.sh\n" >> /etc/crontab
 
 # Configure PosgtreSQL
-COPY postgresql.custom.conf.tmpl /etc/postgresql/14/main/
+COPY postgresql.custom.conf.tmpl /etc/postgresql/$PG_VERSION/main/
 RUN chown -R postgres:postgres /var/lib/postgresql \
-&& chown postgres:postgres /etc/postgresql/14/main/postgresql.custom.conf.tmpl \
-&& echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \
-&& echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf
+&& chown postgres:postgres /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl \
+&& echo "host all all 0.0.0.0/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf \
+&& echo "host all all ::/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf
 
 # Create volume directories
 RUN mkdir -p /run/renderd/ \
@@ -129,10 +145,10 @@ RUN mkdir -p /run/renderd/ \
   &&  chown  -R  renderer:  /data/  \
   &&  chown  -R  renderer:  /home/renderer/src/  \
   &&  chown  -R  renderer:  /run/renderd  \
-  &&  mv  /var/lib/postgresql/14/main/  /data/database/postgres/  \
+  &&  mv  /var/lib/postgresql/$PG_VERSION/main/  /data/database/postgres/  \
   &&  mv  /var/cache/renderd/tiles/            /data/tiles/     \
   &&  chown  -R  renderer: /data/tiles \
-  &&  ln  -s  /data/database/postgres  /var/lib/postgresql/14/main             \
+  &&  ln  -s  /data/database/postgres  /var/lib/postgresql/$PG_VERSION/main             \
   &&  ln  -s  /data/style              /home/renderer/src/openstreetmap-carto  \
   &&  ln  -s  /data/tiles              /var/cache/renderd/tiles                \
 ;

README.md

@@ -1,5 +1,95 @@
 # openstreetmap-tile-server
 
+
+## Champs-libres fork
+
+This is an adaptation of https://github.com/Overv/openstreetmap-tile-server/ for testing different cartoCss styles and generating tiles.
+
+### Use with docker-compose
+
+#### Fresh install
+
+```bash
+#docker volume create osm-data
+docker-compose build
+```
+
+For entering into the container, use the "terminal" argument of the run.sh script:
+
+```bash
+docker-compose run --rm map terminal
+```
+
+#### OAM data dependencies
+
+##### Contour lines
+
+Download the geopackage from http://opendata.champs-libres.be/beautiful-contour-belgium.zip, convert it to shp and place it into the pbf/shp folder under the name "contours.shp". It will be mounted as written in the `docker-compose.yml` file.
+
+The shp is imported using `shp2pgsql` in the import step, assuming EPSG:31370 as the input shp CRS.
+
+#### Hillshade
+
+Download the hillshade at http://opendata.champs-libres.be/hillshade_belgium_EPSG3857.zip and place the tif file in the pdf folder, as mounted in the `docker-compose.yml` file.
+
+
+#### Do an import
+
+Download a pbf and a poly file of your choice on https://download.geofabrik.de and put it in the `pbf` folder. Change the lines of the pbf and poly paths in the volumes in the `docker-compose.yml` file
+
+Then, you can import the OSM data by doing this command:
+
+```bash
+docker-compose run --rm map import
+```
+
+This will take a while, about 15 min for Luxembourg for example, but up to 7 hours for Belgium with OAM!
+
+You should end with :
+
+```
+...
+INFO:root:  Importing into database
+INFO:root:  Import complete
++ sudo -u renderer touch /data/database/planet-import-complete
++ service postgresql stop
+ * Stopping PostgreSQL 15 database server                                                                                                                                                       [ OK ]
++ exit 0
+```
+
+#### Run the server to produce and view tiles
+
+```bash
+docker-compose run --rm map run
+```
+
+The tiles are available on http://192.168.176.2/ or something like that. Have a look at the docker container to know which is the IP address: there is a message like "Could not reliably determine the server's fully qualified domain name, using 192.168.176.2.". (should be on http://localhost:8080/ though...)
+
+#### Generate tiles from a list
+
+There's a script in utils to generate a list of tiles to be used to generate the tiles.
+
+```bash
+python3 generate-tiles-list.py > tiles.list
+```
+
+Move this `tiles.list` file in the `/tiles/` directory and then run:
+
+```bash
+docker-compose run --rm map generate_tiles
+```
+
+For Belgium up to zoom 18, it might take >48 hours!
+
+#### Transfer the tiles to tiles.champs-libres.be
+
+
+```bash
+rsync --compress --recursive ./tiles/cyclosm debian@juniperus.champs-libres.be:/var/lib/mod_tile/
+```
+
+## Original README
+
 [![Build Status](https://travis-ci.org/Overv/openstreetmap-tile-server.svg?branch=master)](https://travis-ci.org/Overv/openstreetmap-tile-server) [![](https://images.microbadger.com/badges/image/overv/openstreetmap-tile-server.svg)](https://microbadger.com/images/overv/openstreetmap-tile-server "openstreetmap-tile-server")
 [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/overv/openstreetmap-tile-server?label=docker%20image)](https://hub.docker.com/r/overv/openstreetmap-tile-server/tags)
 
@@ -67,10 +157,10 @@ By default the container will use openstreetmap-carto if it is not specified. Ho
 docker run \
     -e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \
     -e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \
-    -e NAME_LUA=sample.lua
-    -e NAME_STYLE=test.style
-    -e NAME_MML=project.mml
-    -e NAME_SQL=test.sql
+    -e NAME_LUA=sample.lua \
+    -e NAME_STYLE=test.style \
+    -e NAME_MML=project.mml \
+    -e NAME_SQL=test.sql \
     -v /home/user/openstreetmap-carto-modified:/data/style/ \
     -v osm-data:/data/database/ \
     overv/openstreetmap-tile-server \

apache.conf

@@ -1,7 +1,7 @@
 <VirtualHost *:80>
     ServerAdmin webmaster@localhost
 
-    AddTileConfig /tile/ default
+    AddTileConfig /tile/ cyclosm
     LoadTileConfigFile /etc/renderd.conf
     ModTileRenderdSocketName /run/renderd/renderd.sock
     ModTileRequestTimeout 0
@@ -9,6 +9,8 @@
 
     DocumentRoot /var/www/html
 
+    ServerName  192.168.192.2
+
     ErrorLog ${APACHE_LOG_DIR}/error.log
     CustomLog ${APACHE_LOG_DIR}/access.log combined
 

docker-compose.yml

@@ -4,11 +4,20 @@ services:
   map:
     image: overv/openstreetmap-tile-server
     volumes:
-      - osm-data:/data/database/
+      #- osm-data:/data/database/
+      - ./database:/data/database/
+      - ./tiles:/data/tiles/
+      - ./oam/renderd.conf:/etc/renderd.conf
+      - ./oam/alter_view_owner.sql:/data/alter_view_owner.sql
+      - ./run.sh:/run.sh # for dev
+      - ./pbf/belgium-latest.osm.pbf:/data/region.osm.pbf # change here your pbf
+      #- ./pbf/luxembourg.poly:/data/region.poly # and your poly -> not used
+      - ./pbf/shp/:/data/shp/
+      - ./pbf/hillshade.tif:/data/hillshade.tif
     ports:
-      - "8080:80"
+      - "9000:80"
     command: "run"
 
-volumes:
-  osm-data:
-    external: true
+# volumes:
+#   osm-data:
+#     external: true

leaflet-demo.html

@@ -6,8 +6,8 @@
         <meta charset="utf-8" />
         <meta name="viewport" content="width=device-width, initial-scale=1.0">
 
-        <link rel="stylesheet" href="https://unpkg.com/leaflet@1.7.1/dist/leaflet.css" integrity="sha512-xodZBNTC5n17Xt2atTPuE1HxjVMSvLVW9ocqUKLsCC5CXdbqCmblAshOMAS6/keqq/sMZMZ19scR4PsZChSR7A==" crossorigin="" />
-        <script src="https://unpkg.com/leaflet@1.7.1/dist/leaflet.js" integrity="sha512-XQoYMqMTK8LvdxXYG3nZ448hOEQiglfqkJs1NOQV44cWnUrBc8PkAOcXy20w0vlaXaVUearIOBhiXZ5V3ynxwA==" crossorigin=""></script>
+        <link rel="stylesheet" href="leaflet.css"/>
+        <script src="leaflet.js"></script>
 
         <style>
             html, body, #map {

oam/alter_view_owner.sql

@@ -0,0 +1,20 @@
+ALTER TABLE landuse OWNER TO renderer;
+ALTER TABLE train_elements OWNER TO renderer;
+ALTER TABLE places OWNER TO renderer;
+ALTER TABLE linear_features OWNER TO renderer;
+ALTER TABLE buildings OWNER TO renderer;
+ALTER TABLE cemeteries OWNER TO renderer;
+ALTER TABLE bridge_symbols OWNER TO renderer;
+ALTER TABLE landuse_overlays OWNER TO renderer;
+ALTER TABLE waters OWNER TO renderer;
+ALTER TABLE waterways OWNER TO renderer;
+ALTER TABLE boundaries OWNER TO renderer;
+ALTER TABLE tunnels OWNER TO renderer;
+ALTER TABLE bridges OWNER TO renderer;
+ALTER TABLE roads OWNER TO renderer;
+ALTER TABLE rotated_buildings OWNER TO renderer;
+ALTER TABLE trails OWNER TO renderer;
+ALTER TABLE point_features OWNER TO renderer;
+ALTER TABLE pitch OWNER TO renderer;
+ALTER TABLE landuse_label OWNER TO renderer;
+ALTER TABLE waterway_label OWNER TO renderer;

oam/renderd.conf

@@ -0,0 +1,21 @@
+; BASIC AND SIMPLE CONFIGURATION:
+
+[renderd]
+stats_file=/run/renderd/renderd.stats
+socketname=/run/renderd/renderd.sock
+num_threads=4
+tile_dir=/var/cache/renderd/tiles
+
+[mapnik]
+plugins_dir=/usr/lib/mapnik/3.1/input
+font_dir=/usr/share/fonts
+font_dir_recurse=true
+
+; ADD YOUR LAYERS:
+[oam]
+URI=/tile/
+TILEDIR=/var/cache/renderd/tiles
+XML=/data/style/mapnik.xml
+HOST=localhost
+TILESIZE=256
+MAXZOOM=20

run.sh

@@ -3,15 +3,34 @@
 set -euo pipefail
 
 function createPostgresConfig() {
-  cp /etc/postgresql/14/main/postgresql.custom.conf.tmpl /etc/postgresql/14/main/conf.d/postgresql.custom.conf
-  sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/14/main/conf.d/postgresql.custom.conf
-  cat /etc/postgresql/14/main/conf.d/postgresql.custom.conf
+  cp /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
+  sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
+  cat /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
 }
 
 function setPostgresPassword() {
     sudo -u postgres psql -c "ALTER USER renderer PASSWORD '${PGPASSWORD:-renderer}'"
 }
 
+function compileStyle() {
+    cd /data/style/
+    git clone --single-branch --branch osm-tile-server https://github.com/nobohan/OpenArdenneMap.git --depth 1
+    # mv symbols folder
+    mkdir /data/img/
+    mv /data/style/OpenArdenneMap/img/* /data/img/
+    # copy fonts
+    cp /data/style/OpenArdenneMap/fonts/* /usr/share/fonts/
+    # change dbname
+    sed -i 's/"dbname": "osmpg_db",/"dbname": "gis",/g' ./OpenArdenneMap/osm2pgsql/project.mml
+    # change hillshade
+    sed -i 's,/mnt/tera/ChampsLibres/Projets/OSM/CyclOSM/hillshade/data/dem_be_10x10_hillshade_semi_transparent_3857.tif,/data/hillshade.tif,g' ./OpenArdenneMap/osm2pgsql/project.mml
+    # change contours
+    sed -i 's,../contour/beautiful_contour_belgium.shp,/data/shp/contours_3857.shp,g' ./OpenArdenneMap/osm2pgsql/project.mml
+    # carto build
+    carto OpenArdenneMap/osm2pgsql/project.mml > mapnik.xml
+}
+
+
 if [ "$#" -ne 1 ]; then
     echo "usage: <import|run>"
     echo "commands:"
@@ -29,24 +48,20 @@ fi
 
 set -x
 
-# if there is no custom style mounted, then use osm-carto
-if [ ! "$(ls -A /data/style/)" ]; then
-    mv /home/renderer/src/openstreetmap-carto-backup/* /data/style/
-fi
-
-# carto build
-if [ ! -f /data/style/mapnik.xml ]; then
-    cd /data/style/
-    carto ${NAME_MML:-project.mml} > mapnik.xml
+if [ "$1" == "terminal" ]; then
+    bash
 fi
 
 if [ "$1" == "import" ]; then
+
+    compileStyle
+
     # Ensure that database directory is in right state
     mkdir -p /data/database/postgres/
     chown renderer: /data/database/
     chown -R postgres: /var/lib/postgresql /data/database/postgres/
     if [ ! -f /data/database/postgres/PG_VERSION ]; then
-        sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
+        sudo -u postgres /usr/lib/postgresql/$PG_VERSION/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
     fi
 
     # Initialize PostgreSQL
@@ -56,6 +71,7 @@ if [ "$1" == "import" ]; then
     sudo -u postgres createdb -E UTF8 -O renderer gis
     sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis;"
     sudo -u postgres psql -d gis -c "CREATE EXTENSION hstore;"
+    sudo -u postgres psql -d gis -c "CREATE EXTENSION postgis_sfcgal;"
     sudo -u postgres psql -d gis -c "ALTER TABLE geometry_columns OWNER TO renderer;"
     sudo -u postgres psql -d gis -c "ALTER TABLE spatial_ref_sys OWNER TO renderer;"
     setPostgresPassword
@@ -96,10 +112,7 @@ if [ "$1" == "import" ]; then
     fi
 
     # Import data
-    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore  \
-      --tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua}  \
-      --number-processes ${THREADS:-4}  \
-      -S /data/style/${NAME_STYLE:-openstreetmap-carto.style}  \
+    sudo -u renderer osm2pgsql -d gis --create --slim -G -S /data/style/OpenArdenneMap/osm2pgsql/OpenArdenneMap.style --extra-attributes  \
       /data/region.osm.pbf  \
       ${OSM2PGSQL_EXTRA_ARGS:-}  \
     ;
@@ -115,11 +128,8 @@ if [ "$1" == "import" ]; then
         sudo -u postgres psql -d gis -f /data/style/${NAME_SQL:-indexes.sql}
     fi
 
-    #Import external data
-    chown -R renderer: /home/renderer/src/ /data/style/
-    if [ -f /data/style/scripts/get-external-data.py ] && [ -f /data/style/external-data.yml ]; then
-        sudo -E -u renderer python3 /data/style/scripts/get-external-data.py -c /data/style/external-data.yml -D /data/style/data
-    fi
+    sudo -u postgres psql -d gis -f /data/style/OpenArdenneMap/create_views.sql
+    sudo -u postgres psql -d gis -f /data/alter_view_owner.sql
 
     # Register that data has changed for mod_tile caching purposes
     sudo -u renderer touch /data/database/planet-import-complete
@@ -133,6 +143,8 @@ if [ "$1" == "run" ]; then
     # Clean /tmp
     rm -rf /tmp/*
 
+    compileStyle
+
     # migrate old files
     if [ -f /data/database/PG_VERSION ] && ! [ -d /data/database/postgres/ ]; then
         mkdir /data/database/postgres/
@@ -168,7 +180,7 @@ if [ "$1" == "run" ]; then
     setPostgresPassword
 
     # Configure renderd threads
-    sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf
+    #sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf
 
     # start cron job to trigger consecutive updates
     if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
@@ -195,5 +207,19 @@ if [ "$1" == "run" ]; then
     exit 0
 fi
 
+if [ "$1" == "generate_tiles" ]; then
+
+    compileStyle
+
+    service postgresql start
+    service apache2 restart
+
+    sudo -u renderer renderd -c /etc/renderd.conf
+
+    cat /data/tiles/oam/tiles.list | render_list --map oam
+
+    bash
+fi
+
 echo "invalid command"
 exit 1

utils/generate-tiles-list.py

@@ -0,0 +1,43 @@
+# usage: python3 generate-tiles-list.py > tiles.list
+
+import math
+
+def deg2num(lat_deg, lon_deg, zoom):
+  lat_rad = math.radians(lat_deg)
+  n = 2.0 ** zoom
+  xtile = int((lon_deg + 180.0) / 360.0 * n)
+  ytile = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n)
+
+  return (xtile, ytile)
+
+# GD lux coordinates
+max_x = 6.531
+max_y = 50.184
+min_x = 5.735
+min_y = 49.452
+
+# BE ardenne coordinates
+
+max_x = 6.40
+max_y = 50.77
+min_x = 4.13
+min_y = 49.49
+
+
+min_z = 9
+max_z = 18
+
+for z in range(min_z, max_z+1):
+    x1_tile, y1_tile = deg2num(max_y, max_x, z)
+    x2_tile, y2_tile = deg2num(min_y, min_x, z)
+    min_x_tile = x1_tile if x1_tile < x2_tile else x2_tile
+    max_x_tile = x1_tile if x1_tile > x2_tile else x2_tile
+    min_y_tile = y1_tile if y1_tile < y2_tile else y2_tile
+    max_y_tile = y1_tile if y1_tile > y2_tile else y2_tile
+    for tx in range(min_x_tile, max_x_tile):
+        if tx % 8 != 0:
+            continue
+        for ty in range(min_y_tile, max_y_tile):
+            if ty % 8 != 0:
+                continue
+            print("{} {} {}".format(tx, ty, z))