Use indexes.sql included with openstreetmap-carto instead of own copy (fixes #204)

Dockerfile

@@ -1,214 +1,184 @@
-FROM ubuntu:20.04 AS compiler-common
+FROM ubuntu:20.04
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN apt-get update \
-&& apt-get install -y --no-install-recommends \
- git-core \
- checkinstall \
- g++ \
- make \
- tar \
- wget \
- ca-certificates
-
-###########################################################################################################
-
-FROM compiler-common AS compiler-postgis
-RUN apt-get install -y --no-install-recommends \
- postgresql-server-dev-12 \
- libxml2-dev \
- libgeos-dev \
- libproj-dev
-RUN wget https://download.osgeo.org/postgis/source/postgis-3.1.1.tar.gz -O postgis.tar.gz \
-&& mkdir -p postgis_src \
-&& tar -xvzf postgis.tar.gz --strip 1 -C postgis_src \
-&& rm postgis.tar.gz \
-&& cd postgis_src \
-&& ./configure --without-protobuf --without-raster \
-&& make -j $(nproc) \
-&& checkinstall --pkgversion="3.1.1" --install=no --default make install
-
-###########################################################################################################
-
-FROM compiler-common AS compiler-osm2pgsql
-RUN apt-get install -y --no-install-recommends \
- cmake \
- libboost-dev \
- libboost-system-dev \
- libboost-filesystem-dev \
- libexpat1-dev \
- zlib1g-dev \
- libbz2-dev \
- libpq-dev \
- libproj-dev \
- lua5.3 \
- liblua5.3-dev \
- pandoc
-RUN cd ~ \
-&& git clone -b master --single-branch https://github.com/openstreetmap/osm2pgsql.git --depth 1 \
-&& cd osm2pgsql \
-&& mkdir build \
-&& cd build \
-&& cmake .. \
-&& make -j $(nproc) \
-&& checkinstall --pkgversion="1" --install=no --default make install
-
-###########################################################################################################
-
-FROM compiler-common AS compiler-modtile-renderd
-RUN apt-get install -y --no-install-recommends \
- apache2-dev \
- automake \
- autoconf \
- autotools-dev \
- libtool \
- libmapnik-dev
-RUN cd ~ \
-&& git clone -b switch2osm --single-branch https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \
-&& cd mod_tile \
-&& ./autogen.sh \
-&& ./configure \
-&& make -j $(nproc) \
-&& checkinstall --pkgversion="1" --install=no --pkgname "renderd" --default make install \
-&& checkinstall --pkgversion="1" --install=no --pkgname "mod_tile" --default make install-mod_tile
-
-###########################################################################################################
-
-FROM compiler-common AS compiler-stylesheet
-RUN apt-get install -y --no-install-recommends \
- npm
-RUN cd ~ \
-&& git clone --single-branch --branch v5.3.1 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
-&& cd openstreetmap-carto \
-&& sed -ie 's#https:\/\/naciscdn.org\/naturalearth\/110m\/cultural\/ne_110m_admin_0_boundary_lines_land.zip#https:\/\/naturalearth.s3.amazonaws.com\/110m_cultural\/ne_110m_admin_0_boundary_lines_land.zip#g' external-data.yml \
-&& npm install -g carto@0.18.2 \
-&& carto project.mml > mapnik.xml
-
-###########################################################################################################
-
-FROM compiler-common AS compiler-helper-script
-RUN mkdir -p /home/renderer/src \
-&& cd /home/renderer/src \
-&& git clone https://github.com/zverik/regional \
-&& cd regional \
-&& git checkout 889d630a1e1a1bacabdd1dad6e17b49e7d58cd4b \
-&& rm -rf .git \
-&& chmod u+x /home/renderer/src/regional/trim_osc.py
-
-###########################################################################################################
-
-FROM ubuntu:20.04 AS final-base
 
 # Based on
 # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
-ENV DEBIAN_FRONTEND=noninteractive
+
+# Set up environment
+ENV TZ=UTC
 ENV AUTOVACUUM=on
 ENV UPDATES=disabled
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
-# Get packages
+# Install dependencies
 RUN apt-get update \
-&& apt-get install -y --no-install-recommends \
+  && apt-get install -y wget gnupg2 lsb-core apt-transport-https ca-certificates curl \
- apache2 \
+  && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
- cron \
+  && echo "deb [ trusted=yes ] https://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \
- fonts-noto-cjk \
+  && wget --quiet -O - https://deb.nodesource.com/setup_14.x | bash - \
- fonts-noto-hinted \
+  && apt-get update \
- fonts-noto-unhinted \
+  && apt-get install -y nodejs
- gdal-bin \
+
- liblua5.3-dev \
+RUN apt-get install -y --no-install-recommends \
- lua5.3 \
+  apache2 \
- mapnik-utils \
+  apache2-dev \
- osmium-tool \
+  autoconf \
- osmosis \
+  build-essential \
- postgresql-12 \
+  bzip2 \
- python-is-python3 \
+  cmake \
- python3-mapnik \
+  cron \
- python3-lxml \
+  fonts-noto-cjk \
- python3-psycopg2 \
+  fonts-noto-hinted \
- python3-shapely \
+  fonts-noto-unhinted \
- python3-pip \
+  gcc \
- sudo \
+  gdal-bin \
- ttf-unifont \
+  git-core \
- wget \
+  libagg-dev \
+  libboost-filesystem-dev \
+  libboost-system-dev \
+  libbz2-dev \
+  libcairo-dev \
+  libcairomm-1.0-dev \
+  libexpat1-dev \
+  libfreetype6-dev \
+  libgdal-dev \
+  libgeos++-dev \
+  libgeos-dev \
+  libicu-dev \
+  liblua5.3-dev \
+  libmapnik-dev \
+  libpq-dev \
+  libproj-dev \
+  libprotobuf-c-dev \
+  libtiff5-dev \
+  libtool \
+  libxml2-dev \
+  lua5.3 \
+  make \
+  mapnik-utils \
+  osmium-tool \
+  osmosis \
+  postgis \
+  postgresql-12 \
+  postgresql-contrib-12 \
+  postgresql-server-dev-12 \
+  protobuf-c-compiler \
+  python-is-python3 \
+  python3-mapnik \
+  python3-lxml \
+  python3-psycopg2 \
+  python3-shapely \
+  python3-pip \
+  sudo \
+  tar \
+  ttf-unifont \
+  unzip \
+  wget \
+  zlib1g-dev \
 && apt-get clean autoclean \
 && apt-get autoremove --yes \
 && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
+# Install python libraries
+RUN pip3 install requests \
+ && pip3 install pyyaml
+
+# Set up PostGIS
+RUN wget https://download.osgeo.org/postgis/source/postgis-3.1.1.tar.gz -O postgis.tar.gz \
+ && mkdir -p postgis_src \
+ && tar -xvzf postgis.tar.gz --strip 1 -C postgis_src \
+ && rm postgis.tar.gz \
+ && cd postgis_src \
+ && ./configure \
+ && make -j $(nproc) \
+ && make -j $(nproc) install \
+ && cd .. && rm -rf postgis_src
+
+# Set up renderer user
 RUN adduser --disabled-password --gecos "" renderer
 
-# Install python libraries
+# Install latest osm2pgsql
-RUN pip3 install \
+RUN mkdir -p /home/renderer/src \
- requests \
+ && cd /home/renderer/src \
- pyyaml
+ && git clone -b master https://github.com/openstreetmap/osm2pgsql.git --depth 1 \
+ && cd /home/renderer/src/osm2pgsql \
+ && rm -rf .git \
+ && mkdir build \
+ && cd build \
+ && cmake .. \
+ && make -j $(nproc) \
+ && make -j $(nproc) install \
+ && mkdir /nodes \
+ && chown renderer:renderer /nodes \
+ && rm -rf /home/renderer/src/osm2pgsql
+
+# Install mod_tile and renderd
+RUN mkdir -p /home/renderer/src \
+ && cd /home/renderer/src \
+ && git clone -b switch2osm https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \
+ && cd mod_tile \
+ && rm -rf .git \
+ && ./autogen.sh \
+ && ./configure \
+ && make -j $(nproc) \
+ && make -j $(nproc) install \
+ && make -j $(nproc) install-mod_tile \
+ && ldconfig \
+ && cd ..
+
+# Configure stylesheet
+RUN mkdir -p /home/renderer/src \
+ && cd /home/renderer/src \
+ && git clone --single-branch --branch v5.3.1 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
+ && cd openstreetmap-carto \
+ && rm -rf .git \
+ && npm install -g carto@0.18.2 \
+ && carto project.mml > mapnik.xml
+
+# Configure renderd
+RUN sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \
+ && sed -i 's/\/truetype//g' /usr/local/etc/renderd.conf \
+ && sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
 
 # Configure Apache
 RUN mkdir /var/lib/mod_tile \
-&& chown renderer /var/lib/mod_tile \
+ && chown renderer /var/lib/mod_tile \
-&& mkdir /var/run/renderd \
+ && mkdir /var/run/renderd \
-&& chown renderer /var/run/renderd \
+ && chown renderer /var/run/renderd \
-&& echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \
+ && echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \
-&& echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \
+ && echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \
-&& a2enconf mod_tile && a2enconf mod_headers
+ && a2enconf mod_tile && a2enconf mod_headers
 COPY apache.conf /etc/apache2/sites-available/000-default.conf
 COPY leaflet-demo.html /var/www/html/index.html
 RUN ln -sf /dev/stdout /var/log/apache2/access.log \
-&& ln -sf /dev/stderr /var/log/apache2/error.log
+ && ln -sf /dev/stderr /var/log/apache2/error.log
-
-# Copy update scripts
-COPY openstreetmap-tiles-update-expire /usr/bin/
-RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \
-&& mkdir /var/log/tiles \
-&& chmod a+rw /var/log/tiles \
-&& ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \
-&& echo "* * * * *   renderer    openstreetmap-tiles-update-expire\n" >> /etc/crontab
-
-RUN mkdir /nodes \
-&& chown renderer:renderer /nodes
 
 # Configure PosgtreSQL
 COPY postgresql.custom.conf.tmpl /etc/postgresql/12/main/
 RUN chown -R postgres:postgres /var/lib/postgresql \
-&& chown postgres:postgres /etc/postgresql/12/main/postgresql.custom.conf.tmpl \
+ && chown postgres:postgres /etc/postgresql/12/main/postgresql.custom.conf.tmpl \
-&& echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/12/main/pg_hba.conf \
+ && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/12/main/pg_hba.conf \
-&& echo "host all all ::/0 md5" >> /etc/postgresql/12/main/pg_hba.conf
+ && echo "host all all ::/0 md5" >> /etc/postgresql/12/main/pg_hba.conf
 
-###########################################################################################################
+# Copy update scripts
+COPY openstreetmap-tiles-update-expire /usr/bin/
+RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \
+ && mkdir /var/log/tiles \
+ && chmod a+rw /var/log/tiles \
+ && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \
+ && echo "*  *    * * *   renderer    openstreetmap-tiles-update-expire\n" >> /etc/crontab
 
-FROM final-base AS final
+# Install trim_osc.py helper script
-
+RUN mkdir -p /home/renderer/src \
-# Install PostGIS
+ && cd /home/renderer/src \
-COPY --from=compiler-postgis postgis_src/postgis-src_3.1.1-1_amd64.deb .
+ && git clone https://github.com/zverik/regional \
-RUN dpkg -i postgis-src_3.1.1-1_amd64.deb \
+ && cd regional \
-&& rm postgis-src_3.1.1-1_amd64.deb
+ && git checkout 889d630a1e1a1bacabdd1dad6e17b49e7d58cd4b \
-
+ && rm -rf .git \
-# Install osm2pgsql
+ && chmod u+x /home/renderer/src/regional/trim_osc.py
-COPY --from=compiler-osm2pgsql /root/osm2pgsql/build/build_1-1_amd64.deb .
-RUN dpkg -i build_1-1_amd64.deb \
-&& rm build_1-1_amd64.deb
-
-# Install renderd
-COPY --from=compiler-modtile-renderd /root/mod_tile/renderd_1-1_amd64.deb .
-RUN dpkg -i renderd_1-1_amd64.deb \
-&& rm renderd_1-1_amd64.deb \
-&& sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \
-&& sed -i 's/\/truetype//g' /usr/local/etc/renderd.conf \
-&& sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
-
-# Install mod_tile
-COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb .
-RUN dpkg -i mod-tile_1-1_amd64.deb \
- && ldconfig \
- && rm mod-tile_1-1_amd64.deb
-COPY --from=compiler-modtile-renderd /root/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag
-
-# Install stylesheet
-COPY --from=compiler-stylesheet /root/openstreetmap-carto /home/renderer/src/openstreetmap-carto
-
-# Install helper script
-COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional
 
 # Start running
 COPY run.sh /
 ENTRYPOINT ["/run.sh"]
 CMD []
+
 EXPOSE 80 5432

openstreetmap-tiles-update-expire

@@ -7,7 +7,9 @@ set -e
 # and other things directly from this script when run from cron.
 # Change the actual location to wherever installed locally.
 #------------------------------------------------------------------------------
+export PATH=.:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 ACCOUNT=renderer
+cd /home/$ACCOUNT/src/mod_tile/
 
 #------------------------------------------------------------------------------
 # Extra OSM2PGSQL_OPTIONS may need setting if a tag transform script is

run.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -euo pipefail
+set -x
 
 function createPostgresConfig() {
   cp /etc/postgresql/12/main/postgresql.custom.conf.tmpl /etc/postgresql/12/main/conf.d/postgresql.custom.conf
@@ -23,8 +23,6 @@ if [ "$#" -ne 1 ]; then
     exit 1
 fi
 
-set -x
-
 if [ "$1" = "import" ]; then
     # Ensure that database directory is in right state
     chown postgres:postgres -R /var/lib/postgresql
@@ -44,22 +42,22 @@ if [ "$1" = "import" ]; then
     setPostgresPassword
 
     # Download Luxembourg as sample if no data is provided
-    if [ ! -f /data.osm.pbf ] && [ -z "${DOWNLOAD_PBF:-}" ]; then
+    if [ ! -f /data.osm.pbf ] && [ -z "$DOWNLOAD_PBF" ]; then
         echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..."
         DOWNLOAD_PBF="https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf"
         DOWNLOAD_POLY="https://download.geofabrik.de/europe/luxembourg.poly"
     fi
 
-    if [ -n "${DOWNLOAD_PBF:-}" ]; then
+    if [ -n "$DOWNLOAD_PBF" ]; then
         echo "INFO: Download PBF file: $DOWNLOAD_PBF"
-        wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data.osm.pbf
+        wget "$WGET_ARGS" "$DOWNLOAD_PBF" -O /data.osm.pbf
         if [ -n "$DOWNLOAD_POLY" ]; then
             echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY"
-            wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data.poly
+            wget "$WGET_ARGS" "$DOWNLOAD_POLY" -O /data.poly
         fi
     fi
 
-    if [ "${UPDATES:-}" = "enabled" ]; then
+    if [ "$UPDATES" = "enabled" ]; then
         # determine and set osmosis_replication_timestamp (for consecutive updates)
         osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info
         osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt
@@ -75,14 +73,14 @@ if [ "$1" = "import" ]; then
     fi
 
     # Import data
-    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS:-}
+    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS}
 
     # Create indexes
     sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/indexes.sql
 
     #Import external data
     sudo chown -R renderer: /home/renderer/src
-    sudo -E -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data
+    sudo -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data
 
     # Register that data has changed for mod_tile caching purposes
     touch /var/lib/mod_tile/planet-import-complete
@@ -100,7 +98,7 @@ if [ "$1" = "run" ]; then
     chown postgres:postgres /var/lib/postgresql -R
 
     # Configure Apache CORS
-    if [ "${ALLOW_CORS:-}" == "enabled" ] || [ "${ALLOW_CORS:-}" == "1" ]; then
+    if [ "$ALLOW_CORS" == "enabled" ] || [ "$ALLOW_CORS" == "1" ]; then
         echo "export APACHE_ARGUMENTS='-D ALLOW_CORS'" >> /etc/apache2/envvars
     fi
 
@@ -114,7 +112,7 @@ if [ "$1" = "run" ]; then
     sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf
 
     # start cron job to trigger consecutive updates
-    if [ "${UPDATES:-}" = "enabled" ] || [ "${UPDATES:-}" = "1" ]; then
+    if [ "$UPDATES" = "enabled" ] || [ "$UPDATES" = "1" ]; then
       /etc/init.d/cron start
     fi