Added E flag (#217)

Added -E flag to use the set up environment.

Dockerfile

@@ -1,139 +1,145 @@
-FROM ubuntu:20.04
+FROM ubuntu:20.04 AS compiler-common
+ENV DEBIAN_FRONTEND=noninteractive
 
-# Based on
-# https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
-
-# Set up environment
-ENV TZ=UTC
-ENV AUTOVACUUM=on
-ENV UPDATES=disabled
-RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
-
-# Install dependencies
 RUN apt-get update \
-  && apt-get install -y wget gnupg2 lsb-core apt-transport-https ca-certificates curl \
+&& apt-get install -y --no-install-recommends \
-  && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
-  && echo "deb [ trusted=yes ] https://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \
-  && wget --quiet -O - https://deb.nodesource.com/setup_10.x | bash - \
-  && apt-get update \
-  && apt-get install -y nodejs
-
-RUN apt-get install -y --no-install-recommends \
-  apache2 \
-  apache2-dev \
-  autoconf \
-  build-essential \
-  bzip2 \
-  cmake \
-  cron \
-  fonts-noto-cjk \
-  fonts-noto-hinted \
-  fonts-noto-unhinted \
-  gcc \
-  gdal-bin \
  git-core \
-  libagg-dev \
+ checkinstall \
-  libboost-filesystem-dev \
+ g++ \
-  libboost-system-dev \
-  libbz2-dev \
-  libcairo-dev \
-  libcairomm-1.0-dev \
-  libexpat1-dev \
-  libfreetype6-dev \
-  libgdal-dev \
-  libgeos++-dev \
-  libgeos-dev \
-  libicu-dev \
-  liblua5.3-dev \
-  libmapnik-dev \
-  libpq-dev \
-  libproj-dev \
-  libprotobuf-c-dev \
-  libtiff5-dev \
-  libtool \
-  libxml2-dev \
-  lua5.3 \
  make \
-  mapnik-utils \
-  osmium-tool \
-  osmosis \
-  postgis \
-  postgresql-12 \
-  postgresql-contrib-12 \
-  postgresql-server-dev-12 \
-  protobuf-c-compiler \
-  python3-mapnik \
-  python3-lxml \
-  python3-psycopg2 \
-  python3-shapely \
-  sudo \
  tar \
-  ttf-unifont \
-  unzip \
  wget \
-  zlib1g-dev \
+ ca-certificates
-&& apt-get clean autoclean \
-&& apt-get autoremove --yes \
-&& rm -rf /var/lib/{apt,dpkg,cache,log}/
 
-# Set up PostGIS
+###########################################################################################################
+
+FROM compiler-common AS compiler-postgis
+RUN apt-get install -y --no-install-recommends \
+ postgresql-server-dev-12 \
+ libxml2-dev \
+ libgeos-dev \
+ libproj-dev
 RUN wget https://download.osgeo.org/postgis/source/postgis-3.1.1.tar.gz -O postgis.tar.gz \
 && mkdir -p postgis_src \
 && tar -xvzf postgis.tar.gz --strip 1 -C postgis_src \
 && rm postgis.tar.gz \
 && cd postgis_src \
- && ./configure \
+&& ./configure --without-protobuf --without-raster \
 && make -j $(nproc) \
- && make -j $(nproc) install \
+&& checkinstall --pkgversion="3.1.1" --install=no --default make install
- && cd .. && rm -rf postgis_src
 
-# Set up renderer user
+###########################################################################################################
-RUN adduser --disabled-password --gecos "" renderer
 
-# Install latest osm2pgsql
+FROM compiler-common AS compiler-osm2pgsql
-RUN mkdir -p /home/renderer/src \
+RUN apt-get install -y --no-install-recommends \
- && cd /home/renderer/src \
+ cmake \
- && git clone -b master https://github.com/openstreetmap/osm2pgsql.git --depth 1 \
+ libboost-dev \
- && cd /home/renderer/src/osm2pgsql \
+ libboost-system-dev \
- && rm -rf .git \
+ libboost-filesystem-dev \
+ libexpat1-dev \
+ zlib1g-dev \
+ libbz2-dev \
+ libpq-dev \
+ libproj-dev \
+ lua5.3 \
+ liblua5.3-dev \
+ pandoc
+RUN cd ~ \
+&& git clone -b master --single-branch https://github.com/openstreetmap/osm2pgsql.git --depth 1 \
+&& cd osm2pgsql \
 && mkdir build \
 && cd build \
 && cmake .. \
 && make -j $(nproc) \
- && make -j $(nproc) install \
+&& checkinstall --pkgversion="1" --install=no --default make install
- && mkdir /nodes \
- && chown renderer:renderer /nodes \
- && rm -rf /home/renderer/src/osm2pgsql
 
-# Install mod_tile and renderd
+###########################################################################################################
-RUN mkdir -p /home/renderer/src \
+
- && cd /home/renderer/src \
+FROM compiler-common AS compiler-modtile-renderd
- && git clone -b switch2osm https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \
+RUN apt-get install -y --no-install-recommends \
+ apache2-dev \
+ automake \
+ autoconf \
+ autotools-dev \
+ libtool \
+ libmapnik-dev
+RUN cd ~ \
+&& git clone -b switch2osm --single-branch https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \
 && cd mod_tile \
- && rm -rf .git \
 && ./autogen.sh \
 && ./configure \
 && make -j $(nproc) \
- && make -j $(nproc) install \
+&& checkinstall --pkgversion="1" --install=no --pkgname "renderd" --default make install \
- && make -j $(nproc) install-mod_tile \
+&& checkinstall --pkgversion="1" --install=no --pkgname "mod_tile" --default make install-mod_tile
- && ldconfig \
- && cd ..
 
-# Configure stylesheet
+###########################################################################################################
+
+FROM compiler-common AS compiler-stylesheet
+RUN apt-get install -y --no-install-recommends \
+ npm
+RUN cd ~ \
+&& git clone --single-branch --branch v5.3.1 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
+&& cd openstreetmap-carto \
+&& sed -ie 's#https:\/\/naciscdn.org\/naturalearth\/110m\/cultural\/ne_110m_admin_0_boundary_lines_land.zip#https:\/\/naturalearth.s3.amazonaws.com\/110m_cultural\/ne_110m_admin_0_boundary_lines_land.zip#g' external-data.yml \
+&& npm install -g carto@0.18.2 \
+&& carto project.mml > mapnik.xml
+
+###########################################################################################################
+
+FROM compiler-common AS compiler-helper-script
 RUN mkdir -p /home/renderer/src \
 && cd /home/renderer/src \
- && git clone --single-branch --branch v5.2.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
+&& git clone https://github.com/zverik/regional \
- && cd openstreetmap-carto \
+&& cd regional \
+&& git checkout 889d630a1e1a1bacabdd1dad6e17b49e7d58cd4b \
 && rm -rf .git \
- && npm install -g carto@0.18.2 \
+&& chmod u+x /home/renderer/src/regional/trim_osc.py
- && carto project.mml > mapnik.xml \
- && scripts/get-shapefiles.py \
- && rm /home/renderer/src/openstreetmap-carto/data/*.zip
 
-# Configure renderd
+###########################################################################################################
-RUN sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \
+
- && sed -i 's/\/truetype//g' /usr/local/etc/renderd.conf \
+FROM ubuntu:20.04 AS final-base
- && sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
+
+# Based on
+# https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
+ENV DEBIAN_FRONTEND=noninteractive
+ENV AUTOVACUUM=on
+ENV UPDATES=disabled
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+# Get packages
+RUN apt-get update \
+&& apt-get install -y --no-install-recommends \
+ apache2 \
+ cron \
+ fonts-noto-cjk \
+ fonts-noto-hinted \
+ fonts-noto-unhinted \
+ gdal-bin \
+ liblua5.3-dev \
+ lua5.3 \
+ mapnik-utils \
+ osmium-tool \
+ osmosis \
+ postgresql-12 \
+ python-is-python3 \
+ python3-mapnik \
+ python3-lxml \
+ python3-psycopg2 \
+ python3-shapely \
+ python3-pip \
+ sudo \
+ ttf-unifont \
+ wget \
+&& apt-get clean autoclean \
+&& apt-get autoremove --yes \
+&& rm -rf /var/lib/{apt,dpkg,cache,log}/
+
+RUN adduser --disabled-password --gecos "" renderer
+
+# Install python libraries
+RUN pip3 install \
+ requests \
+ pyyaml
 
 # Configure Apache
 RUN mkdir /var/lib/mod_tile \
@@ -148,13 +154,6 @@ COPY leaflet-demo.html /var/www/html/index.html
 RUN ln -sf /dev/stdout /var/log/apache2/access.log \
 && ln -sf /dev/stderr /var/log/apache2/error.log
 
-# Configure PosgtreSQL
-COPY postgresql.custom.conf.tmpl /etc/postgresql/12/main/
-RUN chown -R postgres:postgres /var/lib/postgresql \
- && chown postgres:postgres /etc/postgresql/12/main/postgresql.custom.conf.tmpl \
- && echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/12/main/pg_hba.conf \
- && echo "host all all ::/0 md5" >> /etc/postgresql/12/main/pg_hba.conf
-
 # Copy update scripts
 COPY openstreetmap-tiles-update-expire /usr/bin/
 RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \
@@ -163,19 +162,52 @@ RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \
 && ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \
 && echo "* * * * *   renderer    openstreetmap-tiles-update-expire\n" >> /etc/crontab
 
-# Install trim_osc.py helper script
+RUN mkdir /nodes \
-RUN mkdir -p /home/renderer/src \
+&& chown renderer:renderer /nodes
- && cd /home/renderer/src \
+
- && git clone https://github.com/zverik/regional \
+# Configure PosgtreSQL
- && cd regional \
+COPY postgresql.custom.conf.tmpl /etc/postgresql/12/main/
- && git checkout 889d630a1e1a1bacabdd1dad6e17b49e7d58cd4b \
+RUN chown -R postgres:postgres /var/lib/postgresql \
- && rm -rf .git \
+&& chown postgres:postgres /etc/postgresql/12/main/postgresql.custom.conf.tmpl \
- && chmod u+x /home/renderer/src/regional/trim_osc.py
+&& echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/12/main/pg_hba.conf \
+&& echo "host all all ::/0 md5" >> /etc/postgresql/12/main/pg_hba.conf
+
+###########################################################################################################
+
+FROM final-base AS final
+
+# Install PostGIS
+COPY --from=compiler-postgis postgis_src/postgis-src_3.1.1-1_amd64.deb .
+RUN dpkg -i postgis-src_3.1.1-1_amd64.deb \
+&& rm postgis-src_3.1.1-1_amd64.deb
+
+# Install osm2pgsql
+COPY --from=compiler-osm2pgsql /root/osm2pgsql/build/build_1-1_amd64.deb .
+RUN dpkg -i build_1-1_amd64.deb \
+&& rm build_1-1_amd64.deb
+
+# Install renderd
+COPY --from=compiler-modtile-renderd /root/mod_tile/renderd_1-1_amd64.deb .
+RUN dpkg -i renderd_1-1_amd64.deb \
+&& rm renderd_1-1_amd64.deb \
+&& sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \
+&& sed -i 's/\/truetype//g' /usr/local/etc/renderd.conf \
+&& sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
+
+# Install mod_tile
+COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb .
+RUN dpkg -i mod-tile_1-1_amd64.deb \
+ && ldconfig \
+ && rm mod-tile_1-1_amd64.deb
+
+# Install stylesheet
+COPY --from=compiler-stylesheet /root/openstreetmap-carto /home/renderer/src/openstreetmap-carto
+
+# Install helper script
+COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional
 
 # Start running
 COPY run.sh /
-COPY indexes.sql /
 ENTRYPOINT ["/run.sh"]
 CMD []
-
 EXPOSE 80 5432

indexes.sql

@@ -1,48 +0,0 @@
--- Extracted from https://github.com/gravitystorm/openstreetmap-carto
--- Comes with a CC0 license
-
--- These are optional but suggested indexes for rendering OpenStreetMap Carto
--- with a full planet database.
--- This file is generated with scripts/indexes.py
-
-CREATE INDEX planet_osm_roads_admin
-  ON planet_osm_roads USING GIST (way)
-  WHERE boundary = 'administrative';
-CREATE INDEX planet_osm_roads_roads_ref
-  ON planet_osm_roads USING GIST (way)
-  WHERE highway IS NOT NULL AND ref IS NOT NULL;
-CREATE INDEX planet_osm_roads_admin_low
-  ON planet_osm_roads USING GIST (way)
-  WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');
-CREATE INDEX planet_osm_line_ferry
-  ON planet_osm_line USING GIST (way)
-  WHERE route = 'ferry';
-CREATE INDEX planet_osm_line_river
-  ON planet_osm_line USING GIST (way)
-  WHERE waterway = 'river';
-CREATE INDEX planet_osm_line_name
-  ON planet_osm_line USING GIST (way)
-  WHERE name IS NOT NULL;
-CREATE INDEX planet_osm_polygon_water
-  ON planet_osm_polygon USING GIST (way)
-  WHERE waterway IN ('dock', 'riverbank', 'canal')
-    OR landuse IN ('reservoir', 'basin')
-    OR "natural" IN ('water', 'glacier');
-CREATE INDEX planet_osm_polygon_nobuilding
-  ON planet_osm_polygon USING GIST (way)
-  WHERE building IS NULL;
-CREATE INDEX planet_osm_polygon_name
-  ON planet_osm_polygon USING GIST (way)
-  WHERE name IS NOT NULL;
-CREATE INDEX planet_osm_polygon_way_area_z10
-  ON planet_osm_polygon USING GIST (way)
-  WHERE way_area > 23300;
-CREATE INDEX planet_osm_polygon_military
-  ON planet_osm_polygon USING GIST (way)
-  WHERE (landuse = 'military' OR military = 'danger_area') AND building IS NULL;
-CREATE INDEX planet_osm_polygon_way_area_z6
-  ON planet_osm_polygon USING GIST (way)
-  WHERE way_area > 5980000;
-CREATE INDEX planet_osm_point_place
-  ON planet_osm_point USING GIST (way)
-  WHERE place IS NOT NULL AND name IS NOT NULL;

run.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -x
-
 function createPostgresConfig() {
   cp /etc/postgresql/12/main/postgresql.custom.conf.tmpl /etc/postgresql/12/main/conf.d/postgresql.custom.conf
   sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/12/main/conf.d/postgresql.custom.conf
@@ -23,6 +21,8 @@ if [ "$#" -ne 1 ]; then
     exit 1
 fi
 
+set -x
+
 if [ "$1" = "import" ]; then
     # Ensure that database directory is in right state
     chown postgres:postgres -R /var/lib/postgresql
@@ -50,10 +50,10 @@ if [ "$1" = "import" ]; then
 
     if [ -n "$DOWNLOAD_PBF" ]; then
         echo "INFO: Download PBF file: $DOWNLOAD_PBF"
-        wget "$WGET_ARGS" "$DOWNLOAD_PBF" -O /data.osm.pbf
+        wget $WGET_ARGS "$DOWNLOAD_PBF" -O /data.osm.pbf
         if [ -n "$DOWNLOAD_POLY" ]; then
             echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY"
-            wget "$WGET_ARGS" "$DOWNLOAD_POLY" -O /data.poly
+            wget $WGET_ARGS "$DOWNLOAD_POLY" -O /data.poly
         fi
     fi
 
@@ -76,7 +76,11 @@ if [ "$1" = "import" ]; then
     sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS}
 
     # Create indexes
-    sudo -u postgres psql -d gis -f indexes.sql
+    sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/indexes.sql
+
+    #Import external data
+    sudo chown -R renderer: /home/renderer/src
+    sudo -E -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data
 
     # Register that data has changed for mod_tile caching purposes
     touch /var/lib/mod_tile/planet-import-complete