fix naciscdn.org (#212)

* fix naciscdn.org

* fix

Co-authored-by: prih <prih@prihhost.com>

Dockerfile

@@ -1,7 +1,7 @@
-FROM ubuntu:18.04
+FROM ubuntu:20.04
 
 # Based on
-# https://switch2osm.org/manually-building-a-tile-server-18-04-lts/
+# https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
 
 # Set up environment
 ENV TZ=UTC
@@ -11,17 +11,14 @@ RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 
 # Install dependencies
 RUN apt-get update \
-  && apt-get install wget gnupg2 lsb-core -y \
+  && apt-get install -y wget gnupg2 lsb-core apt-transport-https ca-certificates curl \
   && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
-  && echo "deb [ trusted=yes ] http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \
+  && echo "deb [ trusted=yes ] https://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | tee /etc/apt/sources.list.d/pgdg.list \
+  && wget --quiet -O - https://deb.nodesource.com/setup_14.x | bash - \
   && apt-get update \
-  && apt-get install -y apt-transport-https ca-certificates 
-
-RUN apt-get install -y curl \
-  && wget --quiet -O - https://deb.nodesource.com/setup_10.x | bash - \
   && apt-get install -y nodejs
 
-RUN apt-get install -y --no-install-recommends --allow-unauthenticated \
+RUN apt-get install -y --no-install-recommends \
   apache2 \
   apache2-dev \
   autoconf \
@@ -46,20 +43,18 @@ RUN apt-get install -y --no-install-recommends --allow-unauthenticated \
   libgdal-dev \
   libgeos++-dev \
   libgeos-dev \
-  libgeotiff-epsg \
   libicu-dev \
   liblua5.3-dev \
   libmapnik-dev \
   libpq-dev \
   libproj-dev \
-  libprotobuf-c0-dev \
+  libprotobuf-c-dev \
   libtiff5-dev \
   libtool \
   libxml2-dev \
   lua5.3 \
   make \
   mapnik-utils \
-  node-gyp \
   osmium-tool \
   osmosis \
   postgis \
@@ -67,10 +62,12 @@ RUN apt-get install -y --no-install-recommends --allow-unauthenticated \
   postgresql-contrib-12 \
   postgresql-server-dev-12 \
   protobuf-c-compiler \
+  python-is-python3 \
   python3-mapnik \
   python3-lxml \
   python3-psycopg2 \
   python3-shapely \
+  python3-pip \
   sudo \
   tar \
   ttf-unifont \
@@ -81,13 +78,19 @@ RUN apt-get install -y --no-install-recommends --allow-unauthenticated \
 && apt-get autoremove --yes \
 && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
+# Install python libraries
+RUN pip3 install requests \
+ && pip3 install pyyaml
+
 # Set up PostGIS
-RUN wget http://download.osgeo.org/postgis/source/postgis-3.0.0.tar.gz -O postgis.tar.gz \
+RUN wget https://download.osgeo.org/postgis/source/postgis-3.1.1.tar.gz -O postgis.tar.gz \
  && mkdir -p postgis_src \
  && tar -xvzf postgis.tar.gz --strip 1 -C postgis_src \
  && rm postgis.tar.gz \
  && cd postgis_src \
- && ./configure && make && make install \
+ && ./configure \
+ && make -j $(nproc) \
+ && make -j $(nproc) install \
  && cd .. && rm -rf postgis_src
 
 # Set up renderer user
@@ -96,14 +99,14 @@ RUN adduser --disabled-password --gecos "" renderer
 # Install latest osm2pgsql
 RUN mkdir -p /home/renderer/src \
  && cd /home/renderer/src \
- && git clone https://github.com/openstreetmap/osm2pgsql.git \
+ && git clone -b master https://github.com/openstreetmap/osm2pgsql.git --depth 1 \
  && cd /home/renderer/src/osm2pgsql \
  && rm -rf .git \
  && mkdir build \
  && cd build \
  && cmake .. \
  && make -j $(nproc) \
- && make install \
+ && make -j $(nproc) install \
  && mkdir /nodes \
  && chown renderer:renderer /nodes \
  && rm -rf /home/renderer/src/osm2pgsql
@@ -111,8 +114,9 @@ RUN mkdir -p /home/renderer/src \
 # Install mod_tile and renderd
 RUN mkdir -p /home/renderer/src \
  && cd /home/renderer/src \
- && git clone -b switch2osm https://github.com/SomeoneElseOSM/mod_tile.git \
+ && git clone -b switch2osm https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \
  && cd mod_tile \
+ && rm -rf .git \
  && ./autogen.sh \
  && ./configure \
  && make -j $(nproc) \
@@ -124,17 +128,16 @@ RUN mkdir -p /home/renderer/src \
 # Configure stylesheet
 RUN mkdir -p /home/renderer/src \
  && cd /home/renderer/src \
- && git clone https://github.com/gravitystorm/openstreetmap-carto.git \
+ && git clone --single-branch --branch v5.3.1 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
- && git -C openstreetmap-carto checkout v4.23.0 \
  && cd openstreetmap-carto \
  && rm -rf .git \
+ && sed -ie 's#https:\/\/naciscdn.org\/naturalearth\/110m\/cultural\/ne_110m_admin_0_boundary_lines_land.zip#https:\/\/naturalearth.s3.amazonaws.com\/110m_cultural\/ne_110m_admin_0_boundary_lines_land.zip#g' external-data.yml \
  && npm install -g carto@0.18.2 \
- && carto project.mml > mapnik.xml \
+ && carto project.mml > mapnik.xml
- && scripts/get-shapefiles.py \
- && rm /home/renderer/src/openstreetmap-carto/data/*.zip
 
 # Configure renderd
 RUN sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \
+ && sed -i 's/\/truetype//g' /usr/local/etc/renderd.conf \
  && sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
 
 # Configure Apache
@@ -170,13 +173,12 @@ RUN mkdir -p /home/renderer/src \
  && cd /home/renderer/src \
  && git clone https://github.com/zverik/regional \
  && cd regional \
- && git checkout 612fe3e040d8bb70d2ab3b133f3b2cfc6c940520 \
+ && git checkout 889d630a1e1a1bacabdd1dad6e17b49e7d58cd4b \
  && rm -rf .git \
  && chmod u+x /home/renderer/src/regional/trim_osc.py
 
 # Start running
 COPY run.sh /
-COPY indexes.sql /
 ENTRYPOINT ["/run.sh"]
 CMD []
 

README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://travis-ci.org/Overv/openstreetmap-tile-server.svg?branch=master)](https://travis-ci.org/Overv/openstreetmap-tile-server) [![](https://images.microbadger.com/badges/image/overv/openstreetmap-tile-server.svg)](https://microbadger.com/images/overv/openstreetmap-tile-server "openstreetmap-tile-server")
 
-This container allows you to easily set up an OpenStreetMap PNG tile server given a `.osm.pbf` file. It is based on the [latest Ubuntu 18.04 LTS guide](https://switch2osm.org/manually-building-a-tile-server-18-04-lts/) from [switch2osm.org](https://switch2osm.org/) and therefore uses the default OpenStreetMap style.
+This container allows you to easily set up an OpenStreetMap PNG tile server given a `.osm.pbf` file. It is based on the [latest Ubuntu 18.04 LTS guide](https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/) from [switch2osm.org](https://switch2osm.org/) and therefore uses the default OpenStreetMap style.
 
 ## Setting up the server
 
@@ -38,6 +38,19 @@ docker run \
 
 Refer to the section *Automatic updating and tile expiry* to actually enable the updates while running the tile server.
 
+### Letting the container download the file
+
+It is also possible to let the container download files for you rather than mounting them in advance by using the `DOWNLOAD_PBF` and `DOWNLOAD_POLY` parameters:
+
+```
+docker run \
+    -e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \
+    -e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \
+    -v openstreetmap-data:/var/lib/postgresql/12/main \
+    overv/openstreetmap-tile-server \
+    import
+```
+
 ## Running the server
 
 Run the server like this:
@@ -225,7 +238,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+    https://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

indexes.sql

@@ -1,48 +0,0 @@
--- Extracted from https://github.com/gravitystorm/openstreetmap-carto
--- Comes with a CC0 license
-
--- These are optional but suggested indexes for rendering OpenStreetMap Carto
--- with a full planet database.
--- This file is generated with scripts/indexes.py
-
-CREATE INDEX planet_osm_roads_admin
-  ON planet_osm_roads USING GIST (way)
-  WHERE boundary = 'administrative';
-CREATE INDEX planet_osm_roads_roads_ref
-  ON planet_osm_roads USING GIST (way)
-  WHERE highway IS NOT NULL AND ref IS NOT NULL;
-CREATE INDEX planet_osm_roads_admin_low
-  ON planet_osm_roads USING GIST (way)
-  WHERE boundary = 'administrative' AND admin_level IN ('0', '1', '2', '3', '4');
-CREATE INDEX planet_osm_line_ferry
-  ON planet_osm_line USING GIST (way)
-  WHERE route = 'ferry';
-CREATE INDEX planet_osm_line_river
-  ON planet_osm_line USING GIST (way)
-  WHERE waterway = 'river';
-CREATE INDEX planet_osm_line_name
-  ON planet_osm_line USING GIST (way)
-  WHERE name IS NOT NULL;
-CREATE INDEX planet_osm_polygon_water
-  ON planet_osm_polygon USING GIST (way)
-  WHERE waterway IN ('dock', 'riverbank', 'canal')
-    OR landuse IN ('reservoir', 'basin')
-    OR "natural" IN ('water', 'glacier');
-CREATE INDEX planet_osm_polygon_nobuilding
-  ON planet_osm_polygon USING GIST (way)
-  WHERE building IS NULL;
-CREATE INDEX planet_osm_polygon_name
-  ON planet_osm_polygon USING GIST (way)
-  WHERE name IS NOT NULL;
-CREATE INDEX planet_osm_polygon_way_area_z10
-  ON planet_osm_polygon USING GIST (way)
-  WHERE way_area > 23300;
-CREATE INDEX planet_osm_polygon_military
-  ON planet_osm_polygon USING GIST (way)
-  WHERE (landuse = 'military' OR military = 'danger_area') AND building IS NULL;
-CREATE INDEX planet_osm_polygon_way_area_z6
-  ON planet_osm_polygon USING GIST (way)
-  WHERE way_area > 5980000;
-CREATE INDEX planet_osm_point_place
-  ON planet_osm_point USING GIST (way)
-  WHERE place IS NOT NULL AND name IS NOT NULL;

openstreetmap-tiles-update-expire

@@ -22,7 +22,7 @@ OSM2PGSQL_BIN=osm2pgsql
 TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py
 
 DBNAME=gis
-OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore  ${OSM2PGSQL_EXTRA_ARGS} --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style"
+OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style ${OSM2PGSQL_EXTRA_ARGS}"
 
 #------------------------------------------------------------------------------
 # When using trim_osc.py we can define either a bounding box (such as this
@@ -111,7 +111,7 @@ if [ $# -eq 1 ] ; then
     m_info "Initialising Osmosis replication system to $1"
     mkdir $WORKOSM_DIR
     $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"
-    wget "http://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt
+    wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt
     mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt
     sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt
 else

run.sh

@@ -1,7 +1,5 @@
 #!/bin/bash
 
-set -x
-
 function createPostgresConfig() {
   cp /etc/postgresql/12/main/postgresql.custom.conf.tmpl /etc/postgresql/12/main/conf.d/postgresql.custom.conf
   sudo -u postgres echo "autovacuum = $AUTOVACUUM" >> /etc/postgresql/12/main/conf.d/postgresql.custom.conf
@@ -23,6 +21,8 @@ if [ "$#" -ne 1 ]; then
     exit 1
 fi
 
+set -x
+
 if [ "$1" = "import" ]; then
     # Ensure that database directory is in right state
     chown postgres:postgres -R /var/lib/postgresql
@@ -42,10 +42,19 @@ if [ "$1" = "import" ]; then
     setPostgresPassword
 
     # Download Luxembourg as sample if no data is provided
-    if [ ! -f /data.osm.pbf ]; then
+    if [ ! -f /data.osm.pbf ] && [ -z "$DOWNLOAD_PBF" ]; then
         echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..."
-        wget -nv http://download.geofabrik.de/europe/luxembourg-latest.osm.pbf -O /data.osm.pbf
+        DOWNLOAD_PBF="https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf"
-        wget -nv http://download.geofabrik.de/europe/luxembourg.poly -O /data.poly
+        DOWNLOAD_POLY="https://download.geofabrik.de/europe/luxembourg.poly"
+    fi
+
+    if [ -n "$DOWNLOAD_PBF" ]; then
+        echo "INFO: Download PBF file: $DOWNLOAD_PBF"
+        wget $WGET_ARGS "$DOWNLOAD_PBF" -O /data.osm.pbf
+        if [ -n "$DOWNLOAD_POLY" ]; then
+            echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY"
+            wget $WGET_ARGS "$DOWNLOAD_POLY" -O /data.poly
+        fi
     fi
 
     if [ "$UPDATES" = "enabled" ]; then
@@ -64,10 +73,14 @@ if [ "$1" = "import" ]; then
     fi
 
     # Import data
-    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} ${OSM2PGSQL_EXTRA_ARGS} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf
+    sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/openstreetmap-carto.lua --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/openstreetmap-carto.style /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS}
 
     # Create indexes
-    sudo -u postgres psql -d gis -f indexes.sql
+    sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/indexes.sql
+
+    #Import external data
+    sudo chown -R renderer: /home/renderer/src
+    sudo -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data
 
     # Register that data has changed for mod_tile caching purposes
     touch /var/lib/mod_tile/planet-import-complete