drone: add step for importing

* Update Dockerfile

add support to PostgreSQL 15 and alter encryption method to scram-sha-256 instead of md5

* Update run.sh

add support to PostgreSQL 15

.drone.yml

@@ -0,0 +1,40 @@
+---
+kind: pipeline
+type: docker
+name: default
+
+trigger:
+  event:
+    - tag
+
+clone:
+  disable: true
+
+steps:
+  - name: clone
+    image: plugins/git
+    settings:
+      recursive: true
+
+  - name: download_pbf
+    image: plugins/download
+    settings:
+      source: https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf
+      #destination: /drone/src/
+
+  - name: import_pbf
+    image: plugins/docker
+    settings:
+      context: .
+      dockerfile: Dockerfile
+      pull_image: true
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      registry: h3m6q87t.gra7.container-registry.ovh.net
+      repo: h3m6q87t.gra7.container-registry.ovh.net/osm_tile_server/map
+      tag:
+        - ${DRONE_TAG:=latest}
+    commands:
+      - sh run.sh import

.gitignore

@@ -0,0 +1,8 @@
+pbf/*.pbf
+pbf/*.poly
+pbf/shp/*
+pbf/*.tif
+tiles/*
+!tiles/.gitkeep
+database/*
+!database/.gitkeep

Dockerfile

@@ -1,11 +1,17 @@
 FROM ubuntu:22.04 AS compiler-common
 ENV DEBIAN_FRONTEND=noninteractive
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
 
 RUN apt-get update \
 && apt-get install -y --no-install-recommends \
- git-core \
- ca-certificates \
-&& apt-get update
+ ca-certificates gnupg lsb-release locales \
+ wget curl \
+ git-core unzip unrar \
+&& locale-gen $LANG && update-locale LANG=$LANG \
+&& sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
+&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
+&& apt-get update && apt-get -y upgrade
 
 ###########################################################################################################
 
@@ -31,7 +37,7 @@ RUN mkdir -p /home/renderer/src \
 
 ###########################################################################################################
 
-FROM ubuntu:22.04 AS final
+FROM compiler-common AS final
 
 # Based on
 # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
@@ -40,6 +46,7 @@ ENV AUTOVACUUM=on
 ENV UPDATES=disabled
 ENV REPLICATION_URL=https://planet.openstreetmap.org/replication/hour/
 ENV MAX_INTERVAL_SECONDS=3600
+ENV PG_VERSION 15
 
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
@@ -63,9 +70,9 @@ RUN apt-get update \
  osm2pgsql \
  osmium-tool \
  osmosis \
- postgresql-14 \
- postgresql-14-postgis-3 \
- postgresql-14-postgis-3-scripts \
+ postgresql-$PG_VERSION \
+ postgresql-$PG_VERSION-postgis-3 \
+ postgresql-$PG_VERSION-postgis-3-scripts \
  postgis \
  python-is-python3 \
  python3-mapnik \
@@ -75,7 +82,7 @@ RUN apt-get update \
  python3-pip \
  renderd \
  sudo \
- wget \
+ vim \
 && apt-get clean autoclean \
 && apt-get autoremove --yes \
 && rm -rf /var/lib/{apt,dpkg,cache,log}/
@@ -124,11 +131,11 @@ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \
 && echo "* * * * *   renderer    openstreetmap-tiles-update-expire.sh\n" >> /etc/crontab
 
 # Configure PosgtreSQL
-COPY postgresql.custom.conf.tmpl /etc/postgresql/14/main/
+COPY postgresql.custom.conf.tmpl /etc/postgresql/$PG_VERSION/main/
 RUN chown -R postgres:postgres /var/lib/postgresql \
-&& chown postgres:postgres /etc/postgresql/14/main/postgresql.custom.conf.tmpl \
-&& echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \
-&& echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf
+&& chown postgres:postgres /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl \
+&& echo "host all all 0.0.0.0/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf \
+&& echo "host all all ::/0 scram-sha-256" >> /etc/postgresql/$PG_VERSION/main/pg_hba.conf
 
 # Create volume directories
 RUN mkdir -p /run/renderd/ \
@@ -138,10 +145,10 @@ RUN mkdir -p /run/renderd/ \
   &&  chown  -R  renderer:  /data/  \
   &&  chown  -R  renderer:  /home/renderer/src/  \
   &&  chown  -R  renderer:  /run/renderd  \
-  &&  mv  /var/lib/postgresql/14/main/  /data/database/postgres/  \
+  &&  mv  /var/lib/postgresql/$PG_VERSION/main/  /data/database/postgres/  \
   &&  mv  /var/cache/renderd/tiles/            /data/tiles/     \
   &&  chown  -R  renderer: /data/tiles \
-  &&  ln  -s  /data/database/postgres  /var/lib/postgresql/14/main             \
+  &&  ln  -s  /data/database/postgres  /var/lib/postgresql/$PG_VERSION/main             \
   &&  ln  -s  /data/style              /home/renderer/src/openstreetmap-carto  \
   &&  ln  -s  /data/tiles              /var/cache/renderd/tiles                \
 ;
@@ -162,6 +169,6 @@ COPY --from=compiler-stylesheet /root/openstreetmap-carto /home/renderer/src/ope
 
 # Start running
 COPY run.sh /
-ENTRYPOINT ["/run.sh"]
+#ENTRYPOINT ["/run.sh"]
 CMD []
 EXPOSE 80 5432

README.md

@@ -1,5 +1,97 @@
 # openstreetmap-tile-server
 
+
+## Champs-libres fork
+
+This is an adaptation of https://github.com/Overv/openstreetmap-tile-server/ for testing different cartoCss styles and generating tiles.
+
+### Use with docker-compose
+
+#### Fresh install
+
+```bash
+#docker volume create osm-data
+docker-compose build
+```
+
+For entering into the container, use the "terminal" argument of the run.sh script:
+
+```bash
+docker-compose run --rm map terminal
+```
+
+#### Cyclosm data dependencies
+
+##### Shp
+
+Download the 2 following zip files, unzip them, and place them in the `pbf/shp` folder. They are mounted as volume in the docker-compose.yml.
+
+- http://osmdata.openstreetmap.de/download/simplified-land-polygons-complete-3857.zip
+- http://osmdata.openstreetmap.de/download/land-polygons-split-3857.zip
+
+Also, rename them as in the zip file, so simplified-land-polygons-complete-3857.shp and land-polygons-split-3857.shp.
+
+
+##### Contour lines
+
+Download the geopackage from http://opendata.champs-libres.be/beautiful-contour-belgium.zip, convert it to shp and place it into the pbf/shp folder under the name "contours.shp". It will be mounted as written in the `docker-compose.yml` file.
+
+The shp is imported using `shp2pgsql` in the import step, assuming EPSG:31370 as the input shp CRS.
+
+#### Hillshade
+
+Download the hillshade at http://opendata.champs-libres.be/hillshade_belgium_EPSG3857.zip and place the tif file in the pdf folder, as mounted in the `docker-compose.yml` file.
+
+
+#### Do an import
+
+Download a pbf and a poly file of your choice on https://download.geofabrik.de and put it in the `pbf` folder. Change the lines of the pbf and poly paths in the volumes in the `docker-compose.yml` file
+
+Then, you can import the OSM data by doing this command:
+
+```bash
+docker-compose run --rm map import
+```
+
+This will take a while, about 15 min for Luxembourg for example.
+
+You should end with :
+
+```
+...
+INFO:root:  Importing into database
+INFO:root:  Import complete
++ sudo -u renderer touch /data/database/planet-import-complete
++ service postgresql stop
+ * Stopping PostgreSQL 15 database server                                                                                                                                                       [ OK ]
++ exit 0
+```
+
+#### Run the server to produce and view tiles
+
+```bash
+docker-compose run --rm map run
+```
+
+The tiles are available on http://192.168.176.2/ or something like that. Have a look at the docker container to know which is the IP address: there is a message like "Could not reliably determine the server's fully qualified domain name, using 192.168.176.2.". (should be on http://localhost:8080/ though...)
+
+#### Generate tiles from a list
+
+There's a script in utils to generate a list of tiles to be used to generate the tiles.
+
+```bash
+python3 generate-tiles-list.py > tiles.list
+```
+
+Move this `tiles.list` file in the `/tiles/` directory and then run:
+
+```bash
+docker-compose run --rm map generate_tiles
+```
+
+
+## Original README
+
 [![Build Status](https://travis-ci.org/Overv/openstreetmap-tile-server.svg?branch=master)](https://travis-ci.org/Overv/openstreetmap-tile-server) [![](https://images.microbadger.com/badges/image/overv/openstreetmap-tile-server.svg)](https://microbadger.com/images/overv/openstreetmap-tile-server "openstreetmap-tile-server")
 [![Docker Image Version (latest semver)](https://img.shields.io/docker/v/overv/openstreetmap-tile-server?label=docker%20image)](https://hub.docker.com/r/overv/openstreetmap-tile-server/tags)
 

apache.conf

@@ -1,7 +1,7 @@
 <VirtualHost *:80>
     ServerAdmin webmaster@localhost
 
-    AddTileConfig /tile/ default
+    AddTileConfig /tile/ cyclosm
     LoadTileConfigFile /etc/renderd.conf
     ModTileRenderdSocketName /run/renderd/renderd.sock
     ModTileRequestTimeout 0

cyclosm/renderd.conf

@@ -0,0 +1,21 @@
+; BASIC AND SIMPLE CONFIGURATION:
+
+[renderd]
+stats_file=/run/renderd/renderd.stats
+socketname=/run/renderd/renderd.sock
+num_threads=4
+tile_dir=/var/cache/renderd/tiles
+
+[mapnik]
+plugins_dir=/usr/lib/mapnik/3.1/input
+font_dir=/usr/share/fonts
+font_dir_recurse=true
+
+; ADD YOUR LAYERS:
+[cyclosm]
+URI=/tile/
+TILEDIR=/var/cache/renderd/tiles
+XML=/data/style/mapnik.xml
+HOST=localhost
+TILESIZE=256
+MAXZOOM=20

docker-compose.yml

@@ -4,11 +4,19 @@ services:
   map:
     image: overv/openstreetmap-tile-server
     volumes:
-      - osm-data:/data/database/
+      #- osm-data:/data/database/
+      - ./database:/data/database/
+      - ./tiles:/data/tiles/
+      - ./cyclosm/renderd.conf:/etc/renderd.conf
+      - ./run.sh:/run.sh # for dev
+      - ./pbf/luxembourg-latest.osm.pbf:/data/region.osm.pbf # change here your pbf
+      - ./pbf/luxembourg.poly:/data/region.poly # and your poly
+      - ./pbf/shp/:/data/shp/
+      - ./pbf/hillshade.tif:/data/hillshade.tif
     ports:
       - "8080:80"
     command: "run"
 
-volumes:
-  osm-data:
-    external: true
+# volumes:
+#   osm-data:
+#     external: true

run.sh

@@ -3,15 +3,46 @@
 set -euo pipefail
 
 function createPostgresConfig() {
-  cp /etc/postgresql/14/main/postgresql.custom.conf.tmpl /etc/postgresql/14/main/conf.d/postgresql.custom.conf
-  sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/14/main/conf.d/postgresql.custom.conf
-  cat /etc/postgresql/14/main/conf.d/postgresql.custom.conf
+  cp /etc/postgresql/$PG_VERSION/main/postgresql.custom.conf.tmpl /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
+  sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
+  cat /etc/postgresql/$PG_VERSION/main/conf.d/postgresql.custom.conf
 }
 
 function setPostgresPassword() {
     sudo -u postgres psql -c "ALTER USER renderer PASSWORD '${PGPASSWORD:-renderer}'"
 }
 
+function compileStyle() {
+    cd /data/style/
+    git clone --single-branch --branch v0.6 https://github.com/cyclosm/cyclosm-cartocss-style.git --depth 1
+    # mv symbols folder
+    mkdir /data/style/symbols/
+    mv /data/style/cyclosm-cartocss-style/symbols/* /data/style/symbols/
+    # change shp paths
+    sed -i "s,file: http://osmdata.openstreetmap.de/download,file: /data/shp,g" ./cyclosm-cartocss-style/project.mml
+    sed -i "s/.zip/.shp/g" ./cyclosm-cartocss-style/project.mml
+    # change dbname
+    sed -i 's/dbname: "osm"/dbname: "gis"/g' ./cyclosm-cartocss-style/project.mml
+    # remove some deprecated fonts
+    sed -i 's/, "unifont Medium", "Unifont Upper Medium"//g' ./cyclosm-cartocss-style/fonts.mss
+    sed -i 's/"Noto Sans Tibetan Regular",//g' ./cyclosm-cartocss-style/fonts.mss
+    sed -i 's/"Noto Sans Tibetan Bold",//g' ./cyclosm-cartocss-style/fonts.mss
+    sed -i 's/Noto Sans Syriac Eastern Regular/Noto Sans Syriac Regular/g' ./cyclosm-cartocss-style/fonts.mss
+    # change hillshade
+    sed -i 's/*extents84/*extents/g' ./cyclosm-cartocss-style/project.mml
+    sed -i 's,dem/shade.vrt,/data/hillshade.tif,g' ./cyclosm-cartocss-style/project.mml
+    sed -z 's/status: off\n- id: contours100/status: on\n- id: contours100/' -i ./cyclosm-cartocss-style/project.mml
+    # change contours
+    sed -z 's/status: off\n- id: contours50/status: on\n- id: contours50/' -i ./cyclosm-cartocss-style/project.mml
+    sed -z 's/status: off\n- id: contours20/status: on\n- id: contours20/' -i ./cyclosm-cartocss-style/project.mml
+    sed -z 's/status: off\n- id: contours10/status: on\n- id: contours10/' -i ./cyclosm-cartocss-style/project.mml
+    sed -z 's/status: off\n- id: waterway_low/status: on\n- id: waterway_low/' -i ./cyclosm-cartocss-style/project.mml
+    sed -i 's/dbname: "contours"/dbname: "gis"/g' ./cyclosm-cartocss-style/project.mml
+    # carto build
+    carto cyclosm-cartocss-style/project.mml > mapnik.xml
+}
+
+
 if [ "$#" -ne 1 ]; then
     echo "usage: <import|run>"
     echo "commands:"
@@ -29,24 +60,20 @@ fi
 
 set -x
 
-# if there is no custom style mounted, then use osm-carto
-if [ ! "$(ls -A /data/style/)" ]; then
-    mv /home/renderer/src/openstreetmap-carto-backup/* /data/style/
-fi
-
-# carto build
-if [ ! -f /data/style/mapnik.xml ]; then
-    cd /data/style/
-    carto ${NAME_MML:-project.mml} > mapnik.xml
+if [ "$1" == "terminal" ]; then
+    bash
 fi
 
 if [ "$1" == "import" ]; then
+
+    compileStyle
+
     # Ensure that database directory is in right state
     mkdir -p /data/database/postgres/
     chown renderer: /data/database/
     chown -R postgres: /var/lib/postgresql /data/database/postgres/
     if [ ! -f /data/database/postgres/PG_VERSION ]; then
-        sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
+        sudo -u postgres /usr/lib/postgresql/$PG_VERSION/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
     fi
 
     # Initialize PostgreSQL
@@ -97,9 +124,6 @@ if [ "$1" == "import" ]; then
 
     # Import data
     sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore  \
-      --tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua}  \
-      --number-processes ${THREADS:-4}  \
-      -S /data/style/${NAME_STYLE:-openstreetmap-carto.style}  \
       /data/region.osm.pbf  \
       ${OSM2PGSQL_EXTRA_ARGS:-}  \
     ;
@@ -115,11 +139,12 @@ if [ "$1" == "import" ]; then
         sudo -u postgres psql -d gis -f /data/style/${NAME_SQL:-indexes.sql}
     fi
 
-    #Import external data
-    chown -R renderer: /home/renderer/src/ /data/style/
-    if [ -f /data/style/scripts/get-external-data.py ] && [ -f /data/style/external-data.yml ]; then
-        sudo -E -u renderer python3 /data/style/scripts/get-external-data.py -c /data/style/external-data.yml -D /data/style/data
-    fi
+    # Create cyclosm special views
+    sudo -u renderer psql -d gis -f "/data/style/cyclosm-cartocss-style/views.sql"
+
+    # Import contour files
+    shp2pgsql -s 31370:3857 -c -g geometry -I /data/shp/contours.shp public.contours | sudo -u renderer psql -d gis
+    sudo -u renderer psql -d gis -c "ALTER TABLE public.contours RENAME COLUMN elev TO height;"
 
     # Register that data has changed for mod_tile caching purposes
     sudo -u renderer touch /data/database/planet-import-complete
@@ -133,6 +158,8 @@ if [ "$1" == "run" ]; then
     # Clean /tmp
     rm -rf /tmp/*
 
+    compileStyle
+
     # migrate old files
     if [ -f /data/database/PG_VERSION ] && ! [ -d /data/database/postgres/ ]; then
         mkdir /data/database/postgres/
@@ -168,7 +195,7 @@ if [ "$1" == "run" ]; then
     setPostgresPassword
 
     # Configure renderd threads
-    sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf
+    #sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf
 
     # start cron job to trigger consecutive updates
     if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
@@ -195,5 +222,19 @@ if [ "$1" == "run" ]; then
     exit 0
 fi
 
+if [ "$1" == "generate_tiles" ]; then
+
+    compileStyle
+
+    service postgresql start
+    service apache2 restart
+
+    sudo -u renderer renderd -c /etc/renderd.conf
+
+    cat /data/tiles/cyclosm/tiles.list | render_list --map cyclosm
+
+    bash
+fi
+
 echo "invalid command"
 exit 1

utils/generate-tiles-list.py

@@ -0,0 +1,35 @@
+# usage: python3 generate-tiles-list.py > tiles.list
+
+import math
+
+def deg2num(lat_deg, lon_deg, zoom):
+  lat_rad = math.radians(lat_deg)
+  n = 2.0 ** zoom
+  xtile = int((lon_deg + 180.0) / 360.0 * n)
+  ytile = int((1.0 - math.asinh(math.tan(lat_rad)) / math.pi) / 2.0 * n)
+
+  return (xtile, ytile)
+
+# GD lux coordinates
+max_x = 6.531
+max_y = 50.184
+min_x = 5.735
+min_y = 49.452
+
+min_z = 9
+max_z = 17
+
+for z in range(min_z, max_z+1):
+    x1_tile, y1_tile = deg2num(max_y, max_x, z)
+    x2_tile, y2_tile = deg2num(min_y, min_x, z)
+    min_x_tile = x1_tile if x1_tile < x2_tile else x2_tile
+    max_x_tile = x1_tile if x1_tile > x2_tile else x2_tile
+    min_y_tile = y1_tile if y1_tile < y2_tile else y2_tile
+    max_y_tile = y1_tile if y1_tile > y2_tile else y2_tile
+    for tx in range(min_x_tile, max_x_tile):
+        if tx % 8 != 0:
+            continue
+        for ty in range(min_y_tile, max_y_tile):
+            if ty % 8 != 0:
+                continue
+            print("{} {} {}".format(tx, ty, z))