Add recommended indexes for Carto style (thanks @souladm)

stevo autovacuum

Dockerfile

@@ -3,13 +3,71 @@ FROM ubuntu:18.04
 # Based on
 # https://switch2osm.org/manually-building-a-tile-server-18-04-lts/
 
-# Install dependencies
-RUN apt-get update
-RUN apt-get install -y libboost-all-dev git-core tar unzip wget bzip2 build-essential autoconf libtool libxml2-dev libgeos-dev libgeos++-dev libpq-dev libbz2-dev libproj-dev munin-node munin libprotobuf-c0-dev protobuf-c-compiler libfreetype6-dev libtiff5-dev libicu-dev libgdal-dev libcairo-dev libcairomm-1.0-dev apache2 apache2-dev libagg-dev liblua5.2-dev ttf-unifont lua5.1 liblua5.1-dev libgeotiff-epsg
-
-# Set up environment and renderer user
+# Set up environment
 ENV TZ=UTC
+ENV AUTOVACUUM=on
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# Install dependencies
+RUN echo "deb [ allow-insecure=yes ] http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main" >> /etc/apt/sources.list.d/pgdg.list \
+  && apt-get update \
+  && apt-get install -y apt-transport-https ca-certificates \
+  && apt-get install -y --no-install-recommends --allow-unauthenticated \
+  apache2 \
+  apache2-dev \
+  autoconf \
+  build-essential \
+  bzip2 \
+  cmake \
+  fonts-noto-cjk \
+  fonts-noto-hinted \
+  fonts-noto-unhinted \
+  clang \
+  gdal-bin \
+  git-core \
+  libagg-dev \
+  libboost-all-dev \
+  libbz2-dev \
+  libcairo-dev \
+  libcairomm-1.0-dev \
+  libexpat1-dev \
+  libfreetype6-dev \
+  libgdal-dev \
+  libgeos++-dev \
+  libgeos-dev \
+  libgeotiff-epsg \
+  libicu-dev \
+  liblua5.3-dev \
+  libmapnik-dev \
+  libpq-dev \
+  libproj-dev \
+  libprotobuf-c0-dev \
+  libtiff5-dev \
+  libtool \
+  libxml2-dev \
+  lua5.3 \
+  make \
+  mapnik-utils \
+  nodejs \
+  npm \
+  postgis \
+  postgresql-10 \
+  postgresql-10-postgis-2.5 \
+  postgresql-10-postgis-2.5-scripts \
+  postgresql-contrib-10 \
+  protobuf-c-compiler \
+  python-mapnik \
+  sudo \
+  tar \
+  ttf-unifont \
+  unzip \
+  wget \
+  zlib1g-dev \
+&& apt-get clean autoclean \
+&& apt-get autoremove --yes \
+&& rm -rf /var/lib/{apt,dpkg,cache,log}/
+
+# Set up renderer user
 RUN adduser --disabled-password --gecos "" renderer
 USER renderer
 
@@ -18,34 +76,28 @@ RUN mkdir /home/renderer/src
 WORKDIR /home/renderer/src
 RUN git clone https://github.com/openstreetmap/osm2pgsql.git
 WORKDIR /home/renderer/src/osm2pgsql
-USER root
-RUN apt-get install -y make cmake g++ libboost-dev libboost-system-dev libboost-filesystem-dev libexpat1-dev zlib1g-dev libbz2-dev libpq-dev libgeos-dev libgeos++-dev libproj-dev lua5.2 liblua5.2-dev
-USER renderer
 RUN mkdir build
 WORKDIR /home/renderer/src/osm2pgsql/build
-RUN cmake ..
-RUN make
+RUN cmake .. \
+  && make -j $(nproc)
 USER root
 RUN make install
 USER renderer
 
 # Install and test Mapnik
-USER root
-RUN apt-get -y install autoconf apache2-dev libtool libxml2-dev libbz2-dev libgeos-dev libgeos++-dev libproj-dev gdal-bin libmapnik-dev mapnik-utils python-mapnik
-USER renderer
 RUN python -c 'import mapnik'
 
 # Install mod_tile and renderd
 WORKDIR /home/renderer/src
 RUN git clone -b switch2osm https://github.com/SomeoneElseOSM/mod_tile.git
 WORKDIR /home/renderer/src/mod_tile
-RUN ./autogen.sh
-RUN ./configure
-RUN make
+RUN ./autogen.sh \
+  && ./configure \
+  && make -j $(nproc)
 USER root
-RUN make install
-RUN make install-mod_tile
-RUN ldconfig
+RUN make -j $(nproc) install \
+  && make -j $(nproc) install-mod_tile \
+  && ldconfig
 USER renderer
 
 # Configure stylesheet
@@ -53,46 +105,43 @@ WORKDIR /home/renderer/src
 RUN git clone https://github.com/gravitystorm/openstreetmap-carto.git
 WORKDIR /home/renderer/src/openstreetmap-carto
 USER root
-RUN apt-get install -y npm nodejs
 RUN npm install -g carto
 USER renderer
-RUN carto -v
 RUN carto project.mml > mapnik.xml
 
 # Load shapefiles
 WORKDIR /home/renderer/src/openstreetmap-carto
 RUN scripts/get-shapefiles.py
 
-# Install fonts
-USER root
-RUN apt-get install -y fonts-noto-cjk fonts-noto-hinted fonts-noto-unhinted ttf-unifont
-USER renderer
-
 # Configure renderd
 USER root
-RUN sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf
-RUN sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
+RUN sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \
+  && sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
 USER renderer
 
 # Configure Apache
 USER root
-RUN mkdir /var/lib/mod_tile
-RUN chown renderer /var/lib/mod_tile
-RUN mkdir /var/run/renderd
-RUN chown renderer /var/run/renderd
-RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf
-RUN a2enconf mod_tile
+RUN mkdir /var/lib/mod_tile \
+  && chown renderer /var/lib/mod_tile \
+  && mkdir /var/run/renderd \
+  && chown renderer /var/run/renderd
+RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \
+  && a2enconf mod_tile
 COPY apache.conf /etc/apache2/sites-available/000-default.conf
-USER renderer
+COPY leaflet-demo.html /var/www/html/index.html
+RUN ln -sf /proc/1/fd/1 /var/log/apache2/access.log \
+  && ln -sf /proc/1/fd/2 /var/log/apache2/error.log
 
-# Install PostgreSQL
-USER root
-RUN apt-get install -y postgresql postgresql-contrib postgis postgresql-10-postgis-2.4
-USER renderer
+# Configure PosgtreSQL
+COPY postgresql.custom.conf.tmpl /etc/postgresql/10/main/
+RUN chown -R postgres:postgres /var/lib/postgresql \
+  && chown postgres:postgres /etc/postgresql/10/main/postgresql.custom.conf.tmpl \
+  && echo "\ninclude 'postgresql.custom.conf'" >> /etc/postgresql/10/main/postgresql.conf
 
 # Start running
-USER root
-RUN apt-get install -y sudo
 COPY run.sh /
+COPY indexes.sql /
 ENTRYPOINT ["/run.sh"]
-CMD []
+CMD []
+
+EXPOSE 80 5432

README.md

@@ -10,7 +10,7 @@ First create a Docker volume to hold the PostgreSQL database that will contain t
 
 Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example:
 
-    docker run -v luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main overv/openstreetmap-tile-server import
+    docker run -v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf -v openstreetmap-data:/var/lib/postgresql/10/main overv/openstreetmap-tile-server import
 
 If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server.
 
@@ -20,7 +20,7 @@ Run the server like this:
 
     docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run
 
-Your tiles will now be available at http://localhost:80/tile/{z}/{x}/{y}.png. If you open `leaflet-demo.html` in your browser, you should be able to see the tiles served by your own machine. Note that it will initially quite a bit of time to render the larger tiles for the first time.
+Your tiles will now be available at `http://localhost:80/tile/{z}/{x}/{y}.png`. The demo map in `leaflet-demo.html` will then be available on `http://localhost:80`. Note that it will initially quite a bit of time to render the larger tiles for the first time.
 
 ## Preserving rendered tiles
 
@@ -31,14 +31,41 @@ Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To
 
 ## Performance tuning
 
+### THREADS
+
 The import and tile serving processes use 4 threads by default, but this number can be changed by setting the `THREADS` environment variable. For example:
 
     docker run -p 80:80 -e THREADS=24 -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server run
 
+### AUTOVACUUM
+
+The database use the autovacuum feature by default. This behavior can be changed with `AUTOVACUUM` environment variable. For example:
+
+    docker run -p 80:80 -e AUTOVACUUM=off -v openstreetmap-data:/var/lib/postgresql/10/main -d overv/openstreetmap-tile-server
+
+## Benchmarks
+
+You can find an example of the import performance to expect with this image on the [OpenStreetMap wiki](https://wiki.openstreetmap.org/wiki/Osm2pgsql/benchmarks#debian_9_.2F_openstreetmap-tile-server).
+
+## Troubleshooting
+
+### ERROR: could not resize shared memory segment / No space left on device
+
+If you encounter such entries in the log, it will mean that the default shared memory limit (64 MB) is too low for the container and it should be raised:
+
+    renderd[121]: ERROR: failed to render TILE ajt 2 0-3 0-3
+    renderd[121]: reason: Postgis Plugin: ERROR: could not resize shared memory segment "/PostgreSQL.790133961" to 12615680 bytes: ### No space left on device
+
+To raise it use `--shm-size` parameter. For example:
+
+    docker run -p 80:80 -v openstreetmap-data:/var/lib/postgresql/10/main --shm-size="192m" -d overv/openstreetmap-tile-server run
+
+For too high values you may notice excessive CPU load and memory usage. It might be that you will have to experimentally find the best values for yourself.
+
 ## License
 
 ```
-Copyright 2018 Alexander Overvoorde
+Copyright 2019 Alexander Overvoorde
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -51,4 +78,4 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-```
+```

indexes.sql

@@ -0,0 +1,48 @@
+-- Extracted from https://github.com/gravitystorm/openstreetmap-carto
+-- Comes with a CC0 license
+
+-- These are optional but suggested indexes for rendering OpenStreetMap Carto
+-- with a full planet database.
+-- This file is generated with scripts/indexes.py
+
+CREATE INDEX planet_osm_roads_admin
+  ON planet_osm_roads USING GIST (way)
+  WHERE boundary = 'administrative';
+CREATE INDEX planet_osm_roads_roads_ref
+  ON planet_osm_roads USING GIST (way)
+  WHERE highway IS NOT NULL AND ref IS NOT NULL;
+CREATE INDEX planet_osm_roads_admin_low
+  ON planet_osm_roads USING GIST (way)
+  WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');
+CREATE INDEX planet_osm_line_ferry
+  ON planet_osm_line USING GIST (way)
+  WHERE route = 'ferry';
+CREATE INDEX planet_osm_line_river
+  ON planet_osm_line USING GIST (way)
+  WHERE waterway = 'river';
+CREATE INDEX planet_osm_line_name
+  ON planet_osm_line USING GIST (way)
+  WHERE name IS NOT NULL;
+CREATE INDEX planet_osm_polygon_water
+  ON planet_osm_polygon USING GIST (way)
+  WHERE waterway IN ('dock', 'riverbank', 'canal')
+    OR landuse IN ('reservoir', 'basin')
+    OR "natural" IN ('water', 'glacier');
+CREATE INDEX planet_osm_polygon_nobuilding
+  ON planet_osm_polygon USING GIST (way)
+  WHERE building IS NULL;
+CREATE INDEX planet_osm_polygon_name
+  ON planet_osm_polygon USING GIST (way)
+  WHERE name IS NOT NULL;
+CREATE INDEX planet_osm_polygon_way_area_z10
+  ON planet_osm_polygon USING GIST (way)
+  WHERE way_area > 23300;
+CREATE INDEX planet_osm_polygon_military
+  ON planet_osm_polygon USING GIST (way)
+  WHERE (landuse = 'military' OR military = 'danger_area') AND building IS NULL;
+CREATE INDEX planet_osm_polygon_way_area_z6
+  ON planet_osm_polygon USING GIST (way)
+  WHERE way_area > 5980000;
+CREATE INDEX planet_osm_point_place
+  ON planet_osm_point USING GIST (way)
+  WHERE place IS NOT NULL AND name IS NOT NULL;

leaflet-demo.html

@@ -25,7 +25,7 @@
         <script>
             var map = L.map('map').setView([0, 0], 3);
 
-            L.tileLayer('http://localhost/tile/{z}/{x}/{y}.png', {
+            L.tileLayer('/tile/{z}/{x}/{y}.png', {
                 maxZoom: 18,
                 attribution: 'Map data &copy; <a href="https://www.openstreetmap.org/">OpenStreetMap</a> contributors, <a href="https://creativecommons.org/licenses/by-sa/2.0/">CC-BY-SA</a>',
                 id: 'base'

postgresql.custom.conf.tmpl

@@ -0,0 +1,23 @@
+# Suggested minimal settings from
+# https://ircama.github.io/osm-carto-tutorials/tile-server-ubuntu/
+
+shared_buffers = 128MB
+min_wal_size = 1GB
+max_wal_size = 2GB
+maintenance_work_mem = 256MB
+
+# Suggested settings from
+# https://github.com/openstreetmap/chef/blob/master/roles/tile.rb#L38-L45
+
+max_connections = 250
+temp_buffers = 32MB
+work_mem = 128MB
+wal_buffers = 1024kB
+wal_writer_delay = 500ms
+commit_delay = 10000
+# checkpoint_segments = 60 # unrecognized in psql 10.7.1
+max_wal_size = 2880MB
+random_page_cost = 1.1
+track_activity_query_size = 16384
+autovacuum_vacuum_scale_factor = 0.05
+autovacuum_analyze_scale_factor = 0.02

run.sh

@@ -1,5 +1,14 @@
 #!/bin/bash
 
+set -x
+
+function CreatePostgressqlConfig()
+{
+  cp /etc/postgresql/10/main/postgresql.custom.conf.tmpl /etc/postgresql/10/main/postgresql.custom.conf
+  sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/10/main/postgresql.custom.conf
+  cat /etc/postgresql/10/main/postgresql.custom.conf
+}
+
 if [ "$#" -ne 1 ]; then
     echo "usage: <import|run>"
     echo "commands:"
@@ -12,6 +21,7 @@ fi
 
 if [ "$1" = "import" ]; then
     # Initialize PostgreSQL
+    CreatePostgressqlConfig
     service postgresql start
     sudo -u postgres createuser renderer
     sudo -u postgres createdb -E UTF8 -O renderer gis
@@ -29,11 +39,17 @@ if [ "$1" = "import" ]; then
     # Import data
     sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua -C 2048 --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf
 
+    # Create indexes
+    sudo -u postgres psql -d gis -f indexes.sql
+
+    service postgresql stop
+
     exit 0
 fi
 
 if [ "$1" = "run" ]; then
     # Initialize PostgreSQL and Apache
+    CreatePostgressqlConfig
     service postgresql start
     service apache2 restart
 
@@ -42,9 +58,10 @@ if [ "$1" = "run" ]; then
 
     # Run
     sudo -u renderer renderd -f -c /usr/local/etc/renderd.conf
+    service postgresql stop
 
     exit 0
 fi
 
 echo "invalid command"
-exit 1
+exit 1