Reduce dependencies for updates, switch to Ubuntu 22.04 (#234)

2022-07-05 08:23:15 -04:00 · 2022-07-05 08:23:15 -04:00 · 81c3433055
parent dc47871a5d
commit 81c3433055
5 changed files with 56 additions and 129 deletions
--- a/136
+++ b/136
@ -1,84 +1,14 @@
-FROM ubuntu:20.04 AS compiler-common
+FROM ubuntu:22.04 AS compiler-common
 ENV DEBIAN_FRONTEND=noninteractive

 RUN apt-get update \
 && apt-get install -y --no-install-recommends \
 git-core \
- checkinstall \
- g++ \
- gnupg2 \
- make \
- tar \
- wget \
 ca-certificates \
 && apt-get update

 ###########################################################################################################

-FROM compiler-common AS compiler-postgis
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
-&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
-&& apt-get update \
-&& apt-get install -y --no-install-recommends \
- postgresql-server-dev-14 \
- libxml2-dev \
- libgeos-dev \
- libproj-dev \
-&& wget https://download.osgeo.org/postgis/source/postgis-3.2.1.tar.gz -O postgis.tar.gz \
-&& mkdir -p postgis_src \
-&& tar -xvzf postgis.tar.gz --strip 1 -C postgis_src \
-&& rm postgis.tar.gz \
-&& cd postgis_src \
-&& ./configure --without-protobuf --without-raster \
-&& make -j $(nproc) \
-&& checkinstall --pkgversion="3.2.1" --install=no --default make install
-
-###########################################################################################################
-
-FROM compiler-common AS compiler-osm2pgsql
-RUN apt-get install -y --no-install-recommends \
- cmake \
- libboost-dev \
- libboost-system-dev \
- libboost-filesystem-dev \
- libexpat1-dev \
- zlib1g-dev \
- libbz2-dev \
- libpq-dev \
- libproj-dev \
- lua5.3 \
- liblua5.3-dev \
- pandoc
-RUN cd ~ \
-&& git clone -b master --single-branch https://github.com/openstreetmap/osm2pgsql.git --depth 1 \
-&& cd osm2pgsql \
-&& mkdir build \
-&& cd build \
-&& cmake .. \
-&& make -j $(nproc) \
-&& checkinstall --pkgversion="1" --install=no --default make install
-
-###########################################################################################################
-
-FROM compiler-common AS compiler-modtile-renderd
-RUN apt-get install -y --no-install-recommends \
- apache2-dev \
- automake \
- autoconf \
- autotools-dev \
- libtool \
- libmapnik-dev
-RUN cd ~ \
-&& git clone -b switch2osm --single-branch https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \
-&& cd mod_tile \
-&& ./autogen.sh \
-&& ./configure \
-&& make -j $(nproc) \
-&& checkinstall --pkgversion="1" --install=no --pkgname "renderd" --default make install \
-&& checkinstall --pkgversion="1" --install=no --pkgname "mod_tile" --default make install-mod_tile
-
-###########################################################################################################
-
 FROM compiler-common AS compiler-stylesheet
 RUN cd ~ \
 && git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
@ -97,7 +27,7 @@ RUN mkdir -p /home/renderer/src \

 ###########################################################################################################

-FROM ubuntu:20.04 AS final
+FROM ubuntu:22.04 AS final

 # Based on
 # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
@ -111,30 +41,33 @@ RUN apt-get update \
 && apt-get install -y --no-install-recommends \
 apache2 \
 cron \
+ dateutils \
 fonts-noto-cjk \
 fonts-noto-hinted \
 fonts-noto-unhinted \
+ fonts-unifont \
 gnupg2 \
 gdal-bin \
 liblua5.3-dev \
 lua5.3 \
 mapnik-utils \
 npm \
+ osm2pgsql \
 osmium-tool \
 osmosis \
+ postgresql-14 \
+ postgresql-14-postgis-3 \
+ postgresql-14-postgis-3-scripts \
+ postgis \
 python-is-python3 \
 python3-mapnik \
 python3-lxml \
 python3-psycopg2 \
 python3-shapely \
 python3-pip \
+ renderd \
 sudo \
- ttf-unifont \
 wget \
-&& echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
-&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
-&& apt-get update \
-&& apt-get install -y --no-install-recommends postgresql-14 \
 && apt-get clean autoclean \
 && apt-get autoremove --yes \
 && rm -rf /var/lib/{apt,dpkg,cache,log}/
@ -146,16 +79,12 @@ RUN pip3 install \
 requests \
 osmium \
 pyyaml
- 
+
 # Install carto for stylesheet
 RUN npm install -g carto@0.18.2

 # Configure Apache
-RUN mkdir /var/lib/mod_tile \
-&& chown renderer /var/lib/mod_tile \
-&& mkdir /var/run/renderd \
-&& chown renderer /var/run/renderd \
-&& echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \
+RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \
 && echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \
 && a2enconf mod_tile && a2enconf mod_headers
 COPY apache.conf /etc/apache2/sites-available/000-default.conf
@ -179,43 +108,28 @@ RUN chown -R postgres:postgres /var/lib/postgresql \
 && echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf

 # Create volume directories
-RUN   mkdir  -p  /data/database/  \
+RUN mkdir -p /run/renderd/ \
+  &&  mkdir  -p  /data/database/  \
  &&  mkdir  -p  /data/style/  \
  &&  mkdir  -p  /home/renderer/src/  \
  &&  chown  -R  renderer:  /data/  \
  &&  chown  -R  renderer:  /home/renderer/src/  \
+  &&  chown  -R  renderer:  /run/renderd  \
  &&  mv  /var/lib/postgresql/14/main/  /data/database/postgres/  \
-  &&  mv  /var/lib/mod_tile/            /data/tiles/     \
+  &&  mv  /var/cache/renderd/tiles/            /data/tiles/     \
+  &&  chown  -R  renderer: /data/tiles \
  &&  ln  -s  /data/database/postgres  /var/lib/postgresql/14/main             \
  &&  ln  -s  /data/style              /home/renderer/src/openstreetmap-carto  \
-  &&  ln  -s  /data/tiles              /var/lib/mod_tile                       \
+  &&  ln  -s  /data/tiles              /var/cache/renderd/tiles                \
 ;

-# Install PostGIS
-COPY --from=compiler-postgis postgis_src/postgis-src_3.2.1-1_amd64.deb .
-RUN dpkg -i postgis-src_3.2.1-1_amd64.deb \
-&& rm postgis-src_3.2.1-1_amd64.deb
-
-# Install osm2pgsql
-COPY --from=compiler-osm2pgsql /root/osm2pgsql/build/build_1-1_amd64.deb .
-RUN dpkg -i build_1-1_amd64.deb \
-&& rm build_1-1_amd64.deb
-
-# Install renderd
-COPY --from=compiler-modtile-renderd /root/mod_tile/renderd_1-1_amd64.deb .
-RUN dpkg -i renderd_1-1_amd64.deb \
-&& rm renderd_1-1_amd64.deb \
-&& sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \
-&& sed -i 's/\/truetype//g' /usr/local/etc/renderd.conf \
-&& sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
-
-# Install mod_tile
-COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb .
-RUN dpkg -i mod-tile_1-1_amd64.deb \
- && ldconfig \
- && rm mod-tile_1-1_amd64.deb
-
-COPY --from=compiler-modtile-renderd /root/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag
+RUN echo '[default] \n\
+URI=/tile/ \n\
+TILEDIR=/var/cache/renderd/tiles \n\
+XML=/home/renderer/src/openstreetmap-carto/mapnik.xml \n\
+HOST=localhost \n\
+TILESIZE=256 \n\
+MAXZOOM=20' >> /etc/renderd.conf

 # Install helper script
 COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional
--- a/README.md
+++ b/README.md
@ -126,6 +126,8 @@ Given that you've set up your import as described in the *Automatic updates* sec
 ```
 docker run \
    -p 8080:80 \
+    -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \
+    -e MAX_INTERVAL_SECONDS=60 \
    -e UPDATES=enabled \
    -v osm-data:/data/database/ \
    -v osm-tiles:/data/tiles/ \
@ -244,7 +246,7 @@ You can find an example of the import performance to expect with this image on t

 If you encounter such entries in the log, it will mean that the default shared memory limit (64 MB) is too low for the container and it should be raised:
 ```
-renderd[121]: ERROR: failed to render TILE ajt 2 0-3 0-3
+renderd[121]: ERROR: failed to render TILE default 2 0-3 0-3
 renderd[121]: reason: Postgis Plugin: ERROR: could not resize shared memory segment "/PostgreSQL.790133961" to 12615680 bytes: ### No space left on device
 ```
 To raise it use `--shm-size` parameter. For example:
--- a/apache.conf
+++ b/apache.conf
@ -1,8 +1,9 @@
 <VirtualHost *:80>
    ServerAdmin webmaster@localhost

-    LoadTileConfigFile /usr/local/etc/renderd.conf
-    ModTileRenderdSocketName /var/run/renderd/renderd.sock
+    AddTileConfig /tile/ default
+    LoadTileConfigFile /etc/renderd.conf
+    ModTileRenderdSocketName /run/renderd/renderd.sock
    ModTileRequestTimeout 0
    ModTileMissingRequestTimeout 30

--- a/openstreetmap-tiles-update-expire.sh
+++ b/openstreetmap-tiles-update-expire.sh
@ -3,7 +3,7 @@
 set -e

 #------------------------------------------------------------------------------
-# AJT - change directory to mod_tile directory so that we can run replag
+# Change directory to mod_tile directory so that we can run replag
 # and other things directly from this script when run from cron.
 # Change the actual location to wherever installed locally.
 #------------------------------------------------------------------------------
@ -61,6 +61,10 @@ EXPIRY_TOUCHFROM=13
 EXPIRY_DELETEFROM=19
 EXPIRY_MAXZOOM=20

+
+REPLICATION_URL=${REPLICATION_URL:="https://planet.openstreetmap.org/replication/hour/"}
+MAX_INTERVAL_SECONDS=${MAX_INTERVAL_SECONDS:="3600"}
+
 #*************************************************************************
 #*************************************************************************

@ -109,14 +113,16 @@ freelock()
 if [ $# -eq 1 ] ; then
    m_info "Initialising Osmosis replication system to $1"
    mkdir -p $WORKOSM_DIR
+    $OSMOSIS_BIN -v 5 --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"

-    $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"
+    init_seq=$(/usr/lib/python3-pyosmium/pyosmium-get-changes --server $REPLICATION_URL -D $1)
+    url_dynamicPart=$(printf %09d $init_seq | sed 's_\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)_\1/\2/\3_')
+    wget $REPLICATION_URL/$url_dynamicPart.state.txt -O $WORKOSM_DIR/state.txt

-    wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt
-
-    mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt
-    sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt
-    exit 0
+    cat > $WORKOSM_DIR/configuration.txt <<- EOM
+baseUrl=$REPLICATION_URL
+maxInterval=$MAX_INTERVAL_SECONDS
+EOM
 fi

 # make sure the lockfile is removed when we exit and then claim it
@ -141,8 +147,9 @@ if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_ba
 fi

 seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2`
+replag=`dateutils.ddiff $(cat $WORKOSM_DIR/state.txt | grep timestamp | cut -d "=" -f 2 | sed 's,\\\,,g') now`

-m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`"
+m_ok "start import from seq-nr $seq, replag is $replag"

 /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
 m_ok "downloading diff"
@ -182,15 +189,13 @@ fi
 m_ok "expiring tiles"

 #------------------------------------------------------------------------------
-# When expiring tiles we need to define the style sheet if it's not "default".
-# In this case it's "ajt".
 # Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and
 # $EXPIRY_MAXZOOM were dirtied.  We currently re-render
 # tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and
 # delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM.
 # The default path to renderd.sock is fixed.
 #------------------------------------------------------------------------------
-if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
+if ! render_expired --map=default --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
    m_info "Expiry failed"
 fi

--- a/run.sh
+++ b/run.sh
@ -70,7 +70,7 @@ if [ "$1" == "import" ]; then
    if [ -n "${DOWNLOAD_PBF:-}" ]; then
        echo "INFO: Download PBF file: $DOWNLOAD_PBF"
        wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data/region.osm.pbf
-        if [ -n "$DOWNLOAD_POLY" ]; then
+        if [ -n "${DOWNLOAD_POLY:-}" ]; then
            echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY"
            wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data/region.poly
        fi
@ -78,7 +78,7 @@ if [ "$1" == "import" ]; then

    if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
        # determine and set osmosis_replication_timestamp (for consecutive updates)
-        REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44`
+        REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -d "=" -f 2`

        # initial setup of osmosis workspace (for consecutive updates)
        sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
@ -168,11 +168,16 @@ if [ "$1" == "run" ]; then
    setPostgresPassword

    # Configure renderd threads
-    sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf
+    sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf

    # start cron job to trigger consecutive updates
    if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
        /etc/init.d/cron start
+        sudo -u renderer touch /var/log/tiles/run.log; tail -f /var/log/tiles/run.log >> /proc/1/fd/1 &
+        sudo -u renderer touch /var/log/tiles/osmosis.log; tail -f /var/log/tiles/osmosis.log >> /proc/1/fd/1 &
+        sudo -u renderer touch /var/log/tiles/expiry.log; tail -f /var/log/tiles/expiry.log >> /proc/1/fd/1 &
+        sudo -u renderer touch /var/log/tiles/osm2pgsql.log; tail -f /var/log/tiles/osm2pgsql.log >> /proc/1/fd/1 &
+
    fi

    # Run while handling docker stop's SIGTERM
@ -181,7 +186,7 @@ if [ "$1" == "run" ]; then
    }
    trap stop_handler SIGTERM

-    sudo -u renderer renderd -f -c /usr/local/etc/renderd.conf &
+    sudo -u renderer renderd -f -c /etc/renderd.conf &
    child=$!
    wait "$child"