Add recommended indexes for Carto style (thanks @souladm)

stevo autovacuum

Dockerfile

@@ -3,13 +3,71 @@ FROM ubuntu:18.04
 # Based on
 # https://switch2osm.org/manually-building-a-tile-server-18-04-lts/
 
-# Install dependencies
-RUN apt-get update
-RUN apt-get install -y libboost-all-dev git-core tar unzip wget bzip2 build-essential autoconf libtool libxml2-dev libgeos-dev libgeos++-dev libpq-dev libbz2-dev libproj-dev munin-node munin libprotobuf-c0-dev protobuf-c-compiler libfreetype6-dev libtiff5-dev libicu-dev libgdal-dev libcairo-dev libcairomm-1.0-dev apache2 apache2-dev libagg-dev liblua5.2-dev ttf-unifont lua5.1 liblua5.1-dev libgeotiff-epsg
-
-# Set up environment and renderer user
+# Set up environment
 ENV TZ=UTC
+ENV AUTOVACUUM=on
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# Install dependencies
+RUN echo "deb [ allow-insecure=yes ] http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \
+  && apt-get update \
+  && apt-get install -y apt-transport-https ca-certificates \
+  && apt-get install -y --no-install-recommends --allow-unauthenticated \
+  apache2 \
+  apache2-dev \
+  autoconf \
+  build-essential \
+  bzip2 \
+  cmake \
+  fonts-noto-cjk \
+  fonts-noto-hinted \
+  fonts-noto-unhinted \
+  clang \
+  gdal-bin \
+  git-core \
+  libagg-dev \
+  libboost-all-dev \
+  libbz2-dev \
+  libcairo-dev \
+  libcairomm-1.0-dev \
+  libexpat1-dev \
+  libfreetype6-dev \
+  libgdal-dev \
+  libgeos++-dev \
+  libgeos-dev \
+  libgeotiff-epsg \
+  libicu-dev \
+  liblua5.3-dev \
+  libmapnik-dev \
+  libpq-dev \
+  libproj-dev \
+  libprotobuf-c0-dev \
+  libtiff5-dev \
+  libtool \
+  libxml2-dev \
+  lua5.3 \
+  make \
+  mapnik-utils \
+  nodejs \
+  npm \
+  postgis \
+  postgresql-10 \
+  postgresql-10-postgis-2.5 \
+  postgresql-10-postgis-2.5-scripts \
+  postgresql-contrib-10 \
+  protobuf-c-compiler \
+  python-mapnik \
+  sudo \
+  tar \
+  ttf-unifont \
+  unzip \
+  wget \
+  zlib1g-dev \
+&& apt-get clean autoclean \
+&& apt-get autoremove --yes \
+&& rm -rf /var/lib/{apt,dpkg,cache,log}/
+
+# Set up renderer user
 RUN adduser --disabled-password --gecos "" renderer
 USER renderer
 
@@ -18,34 +76,28 @@ RUN mkdir /home/renderer/src
 WORKDIR /home/renderer/src
 RUN git clone https://github.com/openstreetmap/osm2pgsql.git
 WORKDIR /home/renderer/src/osm2pgsql
-USER root
-RUN apt-get install -y make cmake g++ libboost-dev libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev lua5.2 liblua5.2-dev
-USER renderer
 RUN mkdir build
 WORKDIR /home/renderer/src/osm2pgsql/build
-RUN cmake ..
-RUN make
+RUN cmake .. \
+  && make -j $(nproc)
 USER root
 RUN make install
 USER renderer
 
 # Install and test Mapnik
-USER root
-RUN apt-get -y install autoconf apache2-dev libtool libxml2-dev libbz2-dev libgeos-dev libgeos++-dev libproj-dev gdal-bin libmapnik-dev mapnik-utils python-mapnik
-USER renderer
 RUN python -c 'import mapnik'
 
 # Install mod_tile and renderd
 WORKDIR /home/renderer/src
 RUN git clone -b switch2osm https://github.com/SomeoneElseOSM/mod_tile.git
 WORKDIR /home/renderer/src/mod_tile
-RUN ./autogen.sh
-RUN ./configure
-RUN make
+RUN ./autogen.sh \
+  && ./configure \
+  && make -j $(nproc)
 USER root
-RUN make install
-RUN make install-mod_tile
-RUN ldconfig
+RUN make -j $(nproc) install \
+  && make -j $(nproc) install-mod_tile \
+  && ldconfig
 USER renderer
 
 # Configure stylesheet
@@ -53,46 +105,43 @@ WORKDIR /home/renderer/src
 RUN git clone https://github.com/gravitystorm/openstreetmap-carto.git
 WORKDIR /home/renderer/src/openstreetmap-carto
 USER root
-RUN apt-get install -y npm nodejs
 RUN npm install -g carto
 USER renderer
-RUN carto -v
 RUN carto project.mml > mapnik.xml
 
 # Load shapefiles
 WORKDIR /home/renderer/src/openstreetmap-carto
 RUN scripts/get-shapefiles.py
 
-# Install fonts
-USER root
-RUN apt-get install -y fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted ttf-unifont
-USER renderer
-
 # Configure renderd
 USER root
-RUN sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf
-RUN sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
+RUN sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \
+  && sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
 USER renderer
 
 # Configure Apache
 USER root
-RUN mkdir /var/lib/mod_tile
-RUN chown renderer /var/lib/mod_tile
-RUN mkdir /var/run/renderd
-RUN chown renderer /var/run/renderd
-RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf
-RUN a2enconf mod_tile
+RUN mkdir /var/lib/mod_tile \
+  && chown renderer /var/lib/mod_tile \
+  && mkdir /var/run/renderd \
+  && chown renderer /var/run/renderd
+RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \
+  && a2enconf mod_tile
 COPY apache.conf /etc/apache2/sites-available/000-default.conf
-USER renderer
+COPY leaflet-demo.html /var/www/html/index.html
+RUN ln -sf /proc/1/fd/1 /var/log/apache2/access.log \
+  && ln -sf /proc/1/fd/2 /var/log/apache2/error.log
 
-# Install PostgreSQL
-USER root
-RUN apt-get install -y postgresql postgresql-contrib postgis postgresql-10-postgis-2.4
-USER renderer
+# Configure PosgtreSQL
+COPY postgresql.custom.conf.tmpl /etc/postgresql/10/main/
+RUN chown -R postgres:postgres /var/lib/postgresql \
+  && chown postgres:postgres /etc/postgresql/10/main/postgresql.custom.conf.tmpl \
+  && echo "\ninclude 'postgresql.custom.conf'" >> /etc/postgresql/10/main/postgresql.conf
 
 # Start running
-USER root
-RUN apt-get install -y sudo
 COPY run.sh /
+COPY indexes.sql /
 ENTRYPOINT ["/run.sh"]
 CMD []
+
+EXPOSE 80 5432

README.md

@@ -10,7 +10,7 @@ First create a Docker volume to hold the PostgreSQL database that will contain t
 
 Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example:
 
-    docker run -v luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main overv/openstreetmap-tile-server import
+    docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main overv/openstreetmap-tile-server import
 
 If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server.
 
@@ -20,7 +20,7 @@ Run the server like this:
 
     docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run
 
-Your tiles will now be available at http://localhost:80/tile/{z}/{x}/{y}.png. If you open `leaflet-demo.html` in your browser, you should be able to see the tiles served by your own machine. Note that it will initially quite a bit of time to render the larger tiles for the first time.
+Your tiles will now be available at `http://localhost:80/tile/{z}/{x}/{y}.png`. The demo map in `leaflet-demo.html` will then be available on `http://localhost:80`. Note that it will initially quite a bit of time to render the larger tiles for the first time.
 
 ## Preserving rendered tiles
 
@@ -31,14 +31,41 @@ Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To
 
 ## Performance tuning
 
+### THREADS
+
 The import and tile serving processes use 4 threads by default, but this number can be changed by setting the `THREADS` environment variable. For example:
 
     docker run -p 80:80 -e THREADS=24 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run
 
+### AUTOVACUUM
+
+The database use the autovacuum feature by default. This behavior can be changed with `AUTOVACUUM` environment variable. For example:
+
+    docker run -p 80:80 -e AUTOVACUUM=off -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server
+
+## Benchmarks
+
+You can find an example of the import performance to expect with this image on the [OpenStreetMap wiki](https://wiki.openstreetmap.org/wiki/Osm2pgsql/benchmarks#debian_9_.2F_openstreetmap-tile-server).
+
+## Troubleshooting
+
+### ERROR: could not resize shared memory segment / No space left on device
+
+If you encounter such entries in the log, it will mean that the default shared memory limit (64 MB) is too low for the container and it should be raised:
+
+    renderd[121]: ERROR: failed to render TILE ajt 2 0-3 0-3
+    renderd[121]: reason: Postgis Plugin: ERROR: could not resize shared memory segment "/PostgreSQL.790133961" to 12615680 bytes: ### No space left on device
+
+To raise it use `--shm-size` parameter. For example:
+
+    docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main --shm-size="192m" -d overv/openstreetmap-tile-server run
+
+For too high values you may notice excessive CPU load and memory usage. It might be that you will have to experimentally find the best values for yourself.
+
 ## License
 
 ```
-Copyright 2018 Alexander Overvoorde
+Copyright 2019 Alexander Overvoorde
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

indexes.sql

@@ -0,0 +1,48 @@
+-- Extracted from https://github.com/gravitystorm/openstreetmap-carto
+-- Comes with a CC0 license
+
+-- These are optional but suggested indexes for rendering OpenStreetMap Carto
+-- with a full planet database.
+-- This file is generated with scripts/indexes.py
+
+CREATE INDEX planet_osm_roads_admin
+  ON planet_osm_roads USING GIST (way)
+  WHERE boundary = 'administrative';
+CREATE INDEX planet_osm_roads_roads_ref
+  ON planet_osm_roads USING GIST (way)
+  WHERE highway IS NOT NULL AND ref IS NOT NULL;
+CREATE INDEX planet_osm_roads_admin_low
+  ON planet_osm_roads USING GIST (way)
+  WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');
+CREATE INDEX planet_osm_line_ferry
+  ON planet_osm_line USING GIST (way)
+  WHERE route = 'ferry';
+CREATE INDEX planet_osm_line_river
+  ON planet_osm_line USING GIST (way)
+  WHERE waterway = 'river';
+CREATE INDEX planet_osm_line_name
+  ON planet_osm_line USING GIST (way)
+  WHERE name IS NOT NULL;
+CREATE INDEX planet_osm_polygon_water
+  ON planet_osm_polygon USING GIST (way)
+  WHERE waterway IN ('dock', 'riverbank', 'canal')
+    OR landuse IN ('reservoir', 'basin')
+    OR "natural" IN ('water', 'glacier');
+CREATE INDEX planet_osm_polygon_nobuilding
+  ON planet_osm_polygon USING GIST (way)
+  WHERE building IS NULL;
+CREATE INDEX planet_osm_polygon_name
+  ON planet_osm_polygon USING GIST (way)
+  WHERE name IS NOT NULL;
+CREATE INDEX planet_osm_polygon_way_area_z10
+  ON planet_osm_polygon USING GIST (way)
+  WHERE way_area > 23300;
+CREATE INDEX planet_osm_polygon_military
+  ON planet_osm_polygon USING GIST (way)
+  WHERE (landuse = 'military' OR military = 'danger_area') AND building IS NULL;
+CREATE INDEX planet_osm_polygon_way_area_z6
+  ON planet_osm_polygon USING GIST (way)
+  WHERE way_area > 5980000;
+CREATE INDEX planet_osm_point_place
+  ON planet_osm_point USING GIST (way)
+  WHERE place IS NOT NULL AND name IS NOT NULL;

leaflet-demo.html

@@ -25,7 +25,7 @@
         <script>
             var map = L.map('map').setView([0, 0], 3);
 
-            L.tileLayer('http://localhost/tile/{z}/{x}/{y}.png', {
+            L.tileLayer('/tile/{z}/{x}/{y}.png', {
                 maxZoom: 18,
                 attribution: 'Map data &copy; <a href="https://www.openstreetmap.org/">OpenStreetMap</a> contributors, <a href="https://creativecommons.org/licenses/by-sa/2.0/">CC-BY-SA</a>',
                 id: 'base'

postgresql.custom.conf.tmpl

@@ -0,0 +1,23 @@
+# Suggested minimal settings from
+# https://ircama.github.io/osm-carto-tutorials/tile-server-ubuntu/
+
+shared_buffers = 128MB
+min_wal_size = 1GB
+max_wal_size = 2GB
+maintenance_work_mem = 256MB
+
+# Suggested settings from
+# https://github.com/openstreetmap/chef/blob/master/roles/tile.rb#L38-L45
+
+max_connections = 250
+temp_buffers = 32MB
+work_mem = 128MB
+wal_buffers = 1024kB
+wal_writer_delay = 500ms
+commit_delay = 10000
+# checkpoint_segments = 60 # unrecognized in psql 10.7.1
+max_wal_size = 2880MB
+random_page_cost = 1.1
+track_activity_query_size = 16384
+autovacuum_vacuum_scale_factor = 0.05
+autovacuum_analyze_scale_factor = 0.02

run.sh

@@ -1,5 +1,14 @@
 #!/bin/bash
 
+set -x
+
+function CreatePostgressqlConfig()
+{
+  cp /etc/postgresql/10/main/postgresql.custom.conf.tmpl /etc/postgresql/10/main/postgresql.custom.conf
+  sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/10/main/postgresql.custom.conf
+  cat /etc/postgresql/10/main/postgresql.custom.conf
+}
+
 if [ "$#" -ne 1 ]; then
     echo "usage: <import|run>"
     echo "commands:"
@@ -12,6 +21,7 @@ fi
 
 if [ "$1" = "import" ]; then
     # Initialize PostgreSQL
+    CreatePostgressqlConfig
     service postgresql start
     sudo -u postgres createuser renderer
     sudo -u postgres createdb -E UTF8 -O renderer gis
@@ -29,11 +39,17 @@ if [ "$1" = "import" ]; then
     # Import data
     sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua -C 2048 --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf
 
+    # Create indexes
+    sudo -u postgres psql -d gis -f indexes.sql
+
+    service postgresql stop
+
     exit 0
 fi
 
 if [ "$1" = "run" ]; then
     # Initialize PostgreSQL and Apache
+    CreatePostgressqlConfig
     service postgresql start
     service apache2 restart
 
@@ -42,6 +58,7 @@ if [ "$1" = "run" ]; then
 
     # Run
     sudo -u renderer renderd -f -c /usr/local/etc/renderd.conf
+    service postgresql stop
 
     exit 0
 fi