From 81c343305595fb8ae976dbbcf5a7fa2efd4093c1 Mon Sep 17 00:00:00 2001 From: galewis2 <62433564+galewis2@users.noreply.github.com> Date: Tue, 5 Jul 2022 08:23:15 -0400 Subject: [PATCH] Reduce dependencies for updates, switch to Ubuntu 22.04 (#234) --- Dockerfile | 136 +++++---------------------- README.md | 4 +- apache.conf | 5 +- openstreetmap-tiles-update-expire.sh | 27 +++--- run.sh | 13 ++- 5 files changed, 56 insertions(+), 129 deletions(-) mode change 100644 => 100755 openstreetmap-tiles-update-expire.sh diff --git a/Dockerfile b/Dockerfile index bfd60df..f249f0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,84 +1,14 @@ -FROM ubuntu:20.04 AS compiler-common +FROM ubuntu:22.04 AS compiler-common ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ git-core \ - checkinstall \ - g++ \ - gnupg2 \ - make \ - tar \ - wget \ ca-certificates \ && apt-get update ########################################################################################################### -FROM compiler-common AS compiler-postgis -RUN echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ -&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ -&& apt-get update \ -&& apt-get install -y --no-install-recommends \ - postgresql-server-dev-14 \ - libxml2-dev \ - libgeos-dev \ - libproj-dev \ -&& wget https://download.osgeo.org/postgis/source/postgis-3.2.1.tar.gz -O postgis.tar.gz \ -&& mkdir -p postgis_src \ -&& tar -xvzf postgis.tar.gz --strip 1 -C postgis_src \ -&& rm postgis.tar.gz \ -&& cd postgis_src \ -&& ./configure --without-protobuf --without-raster \ -&& make -j $(nproc) \ -&& checkinstall --pkgversion="3.2.1" --install=no --default make install - -########################################################################################################### - -FROM compiler-common AS compiler-osm2pgsql -RUN apt-get install -y --no-install-recommends \ - cmake \ - libboost-dev \ - libboost-system-dev \ - libboost-filesystem-dev \ - libexpat1-dev \ - zlib1g-dev \ - libbz2-dev \ - libpq-dev \ - libproj-dev \ - lua5.3 \ - liblua5.3-dev \ - pandoc -RUN cd ~ \ -&& git clone -b master --single-branch https://github.com/openstreetmap/osm2pgsql.git --depth 1 \ -&& cd osm2pgsql \ -&& mkdir build \ -&& cd build \ -&& cmake .. \ -&& make -j $(nproc) \ -&& checkinstall --pkgversion="1" --install=no --default make install - -########################################################################################################### - -FROM compiler-common AS compiler-modtile-renderd -RUN apt-get install -y --no-install-recommends \ - apache2-dev \ - automake \ - autoconf \ - autotools-dev \ - libtool \ - libmapnik-dev -RUN cd ~ \ -&& git clone -b switch2osm --single-branch https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \ -&& cd mod_tile \ -&& ./autogen.sh \ -&& ./configure \ -&& make -j $(nproc) \ -&& checkinstall --pkgversion="1" --install=no --pkgname "renderd" --default make install \ -&& checkinstall --pkgversion="1" --install=no --pkgname "mod_tile" --default make install-mod_tile - -########################################################################################################### - FROM compiler-common AS compiler-stylesheet RUN cd ~ \ && git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \ @@ -97,7 +27,7 @@ RUN mkdir -p /home/renderer/src \ ########################################################################################################### -FROM ubuntu:20.04 AS final +FROM ubuntu:22.04 AS final # Based on # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/ @@ -111,30 +41,33 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends \ apache2 \ cron \ + dateutils \ fonts-noto-cjk \ fonts-noto-hinted \ fonts-noto-unhinted \ + fonts-unifont \ gnupg2 \ gdal-bin \ liblua5.3-dev \ lua5.3 \ mapnik-utils \ npm \ + osm2pgsql \ osmium-tool \ osmosis \ + postgresql-14 \ + postgresql-14-postgis-3 \ + postgresql-14-postgis-3-scripts \ + postgis \ python-is-python3 \ python3-mapnik \ python3-lxml \ python3-psycopg2 \ python3-shapely \ python3-pip \ + renderd \ sudo \ - ttf-unifont \ wget \ -&& echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ -&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ -&& apt-get update \ -&& apt-get install -y --no-install-recommends postgresql-14 \ && apt-get clean autoclean \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ @@ -146,16 +79,12 @@ RUN pip3 install \ requests \ osmium \ pyyaml - + # Install carto for stylesheet RUN npm install -g carto@0.18.2 # Configure Apache -RUN mkdir /var/lib/mod_tile \ -&& chown renderer /var/lib/mod_tile \ -&& mkdir /var/run/renderd \ -&& chown renderer /var/run/renderd \ -&& echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \ +RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \ && echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \ && a2enconf mod_tile && a2enconf mod_headers COPY apache.conf /etc/apache2/sites-available/000-default.conf @@ -179,43 +108,28 @@ RUN chown -R postgres:postgres /var/lib/postgresql \ && echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf # Create volume directories -RUN mkdir -p /data/database/ \ +RUN mkdir -p /run/renderd/ \ + && mkdir -p /data/database/ \ && mkdir -p /data/style/ \ && mkdir -p /home/renderer/src/ \ && chown -R renderer: /data/ \ && chown -R renderer: /home/renderer/src/ \ + && chown -R renderer: /run/renderd \ && mv /var/lib/postgresql/14/main/ /data/database/postgres/ \ - && mv /var/lib/mod_tile/ /data/tiles/ \ + && mv /var/cache/renderd/tiles/ /data/tiles/ \ + && chown -R renderer: /data/tiles \ && ln -s /data/database/postgres /var/lib/postgresql/14/main \ && ln -s /data/style /home/renderer/src/openstreetmap-carto \ - && ln -s /data/tiles /var/lib/mod_tile \ + && ln -s /data/tiles /var/cache/renderd/tiles \ ; -# Install PostGIS -COPY --from=compiler-postgis postgis_src/postgis-src_3.2.1-1_amd64.deb . -RUN dpkg -i postgis-src_3.2.1-1_amd64.deb \ -&& rm postgis-src_3.2.1-1_amd64.deb - -# Install osm2pgsql -COPY --from=compiler-osm2pgsql /root/osm2pgsql/build/build_1-1_amd64.deb . -RUN dpkg -i build_1-1_amd64.deb \ -&& rm build_1-1_amd64.deb - -# Install renderd -COPY --from=compiler-modtile-renderd /root/mod_tile/renderd_1-1_amd64.deb . -RUN dpkg -i renderd_1-1_amd64.deb \ -&& rm renderd_1-1_amd64.deb \ -&& sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \ -&& sed -i 's/\/truetype//g' /usr/local/etc/renderd.conf \ -&& sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf - -# Install mod_tile -COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb . -RUN dpkg -i mod-tile_1-1_amd64.deb \ - && ldconfig \ - && rm mod-tile_1-1_amd64.deb - -COPY --from=compiler-modtile-renderd /root/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag +RUN echo '[default] \n\ +URI=/tile/ \n\ +TILEDIR=/var/cache/renderd/tiles \n\ +XML=/home/renderer/src/openstreetmap-carto/mapnik.xml \n\ +HOST=localhost \n\ +TILESIZE=256 \n\ +MAXZOOM=20' >> /etc/renderd.conf # Install helper script COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional diff --git a/README.md b/README.md index 65409df..e3ef17b 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,8 @@ Given that you've set up your import as described in the *Automatic updates* sec ``` docker run \ -p 8080:80 \ + -e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \ + -e MAX_INTERVAL_SECONDS=60 \ -e UPDATES=enabled \ -v osm-data:/data/database/ \ -v osm-tiles:/data/tiles/ \ @@ -244,7 +246,7 @@ You can find an example of the import performance to expect with this image on t If you encounter such entries in the log, it will mean that the default shared memory limit (64 MB) is too low for the container and it should be raised: ``` -renderd[121]: ERROR: failed to render TILE ajt 2 0-3 0-3 +renderd[121]: ERROR: failed to render TILE default 2 0-3 0-3 renderd[121]: reason: Postgis Plugin: ERROR: could not resize shared memory segment "/PostgreSQL.790133961" to 12615680 bytes: ### No space left on device ``` To raise it use `--shm-size` parameter. For example: diff --git a/apache.conf b/apache.conf index 99b1615..1301252 100644 --- a/apache.conf +++ b/apache.conf @@ -1,8 +1,9 @@ ServerAdmin webmaster@localhost - LoadTileConfigFile /usr/local/etc/renderd.conf - ModTileRenderdSocketName /var/run/renderd/renderd.sock + AddTileConfig /tile/ default + LoadTileConfigFile /etc/renderd.conf + ModTileRenderdSocketName /run/renderd/renderd.sock ModTileRequestTimeout 0 ModTileMissingRequestTimeout 30 diff --git a/openstreetmap-tiles-update-expire.sh b/openstreetmap-tiles-update-expire.sh old mode 100644 new mode 100755 index 87e73e5..970a68d --- a/openstreetmap-tiles-update-expire.sh +++ b/openstreetmap-tiles-update-expire.sh @@ -3,7 +3,7 @@ set -e #------------------------------------------------------------------------------ -# AJT - change directory to mod_tile directory so that we can run replag +# Change directory to mod_tile directory so that we can run replag # and other things directly from this script when run from cron. # Change the actual location to wherever installed locally. #------------------------------------------------------------------------------ @@ -61,6 +61,10 @@ EXPIRY_TOUCHFROM=13 EXPIRY_DELETEFROM=19 EXPIRY_MAXZOOM=20 + +REPLICATION_URL=${REPLICATION_URL:="https://planet.openstreetmap.org/replication/hour/"} +MAX_INTERVAL_SECONDS=${MAX_INTERVAL_SECONDS:="3600"} + #************************************************************************* #************************************************************************* @@ -109,14 +113,16 @@ freelock() if [ $# -eq 1 ] ; then m_info "Initialising Osmosis replication system to $1" mkdir -p $WORKOSM_DIR + $OSMOSIS_BIN -v 5 --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" - $OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" + init_seq=$(/usr/lib/python3-pyosmium/pyosmium-get-changes --server $REPLICATION_URL -D $1) + url_dynamicPart=$(printf %09d $init_seq | sed 's_\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)_\1/\2/\3_') + wget $REPLICATION_URL/$url_dynamicPart.state.txt -O $WORKOSM_DIR/state.txt - wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt - - mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt - sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt - exit 0 + cat > $WORKOSM_DIR/configuration.txt <<- EOM +baseUrl=$REPLICATION_URL +maxInterval=$MAX_INTERVAL_SECONDS +EOM fi # make sure the lockfile is removed when we exit and then claim it @@ -141,8 +147,9 @@ if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_ba fi seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2` +replag=`dateutils.ddiff $(cat $WORKOSM_DIR/state.txt | grep timestamp | cut -d "=" -f 2 | sed 's,\\\,,g') now` -m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`" +m_ok "start import from seq-nr $seq, replag is $replag" /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt m_ok "downloading diff" @@ -182,15 +189,13 @@ fi m_ok "expiring tiles" #------------------------------------------------------------------------------ -# When expiring tiles we need to define the style sheet if it's not "default". -# In this case it's "ajt". # Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and # $EXPIRY_MAXZOOM were dirtied. We currently re-render # tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and # delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM. # The default path to renderd.sock is fixed. #------------------------------------------------------------------------------ -if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then +if ! render_expired --map=default --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then m_info "Expiry failed" fi diff --git a/run.sh b/run.sh index 1328dc3..40e3b65 100755 --- a/run.sh +++ b/run.sh @@ -70,7 +70,7 @@ if [ "$1" == "import" ]; then if [ -n "${DOWNLOAD_PBF:-}" ]; then echo "INFO: Download PBF file: $DOWNLOAD_PBF" wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data/region.osm.pbf - if [ -n "$DOWNLOAD_POLY" ]; then + if [ -n "${DOWNLOAD_POLY:-}" ]; then echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY" wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data/region.poly fi @@ -78,7 +78,7 @@ if [ "$1" == "import" ]; then if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then # determine and set osmosis_replication_timestamp (for consecutive updates) - REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44` + REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -d "=" -f 2` # initial setup of osmosis workspace (for consecutive updates) sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP @@ -168,11 +168,16 @@ if [ "$1" == "run" ]; then setPostgresPassword # Configure renderd threads - sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf + sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf # start cron job to trigger consecutive updates if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then /etc/init.d/cron start + sudo -u renderer touch /var/log/tiles/run.log; tail -f /var/log/tiles/run.log >> /proc/1/fd/1 & + sudo -u renderer touch /var/log/tiles/osmosis.log; tail -f /var/log/tiles/osmosis.log >> /proc/1/fd/1 & + sudo -u renderer touch /var/log/tiles/expiry.log; tail -f /var/log/tiles/expiry.log >> /proc/1/fd/1 & + sudo -u renderer touch /var/log/tiles/osm2pgsql.log; tail -f /var/log/tiles/osm2pgsql.log >> /proc/1/fd/1 & + fi # Run while handling docker stop's SIGTERM @@ -181,7 +186,7 @@ if [ "$1" == "run" ]; then } trap stop_handler SIGTERM - sudo -u renderer renderd -f -c /usr/local/etc/renderd.conf & + sudo -u renderer renderd -f -c /etc/renderd.conf & child=$! wait "$child"