Reduce dependencies for updates, switch to Ubuntu 22.04 (#234)

This commit is contained in:
galewis2 2022-07-05 08:23:15 -04:00 committed by GitHub
parent dc47871a5d
commit 81c3433055
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 56 additions and 129 deletions

View File

@ -1,84 +1,14 @@
FROM ubuntu:20.04 AS compiler-common FROM ubuntu:22.04 AS compiler-common
ENV DEBIAN_FRONTEND=noninteractive ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update \ RUN apt-get update \
&& apt-get install -y --no-install-recommends \ && apt-get install -y --no-install-recommends \
git-core \ git-core \
checkinstall \
g++ \
gnupg2 \
make \
tar \
wget \
ca-certificates \ ca-certificates \
&& apt-get update && apt-get update
########################################################################################################### ###########################################################################################################
FROM compiler-common AS compiler-postgis
RUN echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
&& apt-get update \
&& apt-get install -y --no-install-recommends \
postgresql-server-dev-14 \
libxml2-dev \
libgeos-dev \
libproj-dev \
&& wget https://download.osgeo.org/postgis/source/postgis-3.2.1.tar.gz -O postgis.tar.gz \
&& mkdir -p postgis_src \
&& tar -xvzf postgis.tar.gz --strip 1 -C postgis_src \
&& rm postgis.tar.gz \
&& cd postgis_src \
&& ./configure --without-protobuf --without-raster \
&& make -j $(nproc) \
&& checkinstall --pkgversion="3.2.1" --install=no --default make install
###########################################################################################################
FROM compiler-common AS compiler-osm2pgsql
RUN apt-get install -y --no-install-recommends \
cmake \
libboost-dev \
libboost-system-dev \
libboost-filesystem-dev \
libexpat1-dev \
zlib1g-dev \
libbz2-dev \
libpq-dev \
libproj-dev \
lua5.3 \
liblua5.3-dev \
pandoc
RUN cd ~ \
&& git clone -b master --single-branch https://github.com/openstreetmap/osm2pgsql.git --depth 1 \
&& cd osm2pgsql \
&& mkdir build \
&& cd build \
&& cmake .. \
&& make -j $(nproc) \
&& checkinstall --pkgversion="1" --install=no --default make install
###########################################################################################################
FROM compiler-common AS compiler-modtile-renderd
RUN apt-get install -y --no-install-recommends \
apache2-dev \
automake \
autoconf \
autotools-dev \
libtool \
libmapnik-dev
RUN cd ~ \
&& git clone -b switch2osm --single-branch https://github.com/SomeoneElseOSM/mod_tile.git --depth 1 \
&& cd mod_tile \
&& ./autogen.sh \
&& ./configure \
&& make -j $(nproc) \
&& checkinstall --pkgversion="1" --install=no --pkgname "renderd" --default make install \
&& checkinstall --pkgversion="1" --install=no --pkgname "mod_tile" --default make install-mod_tile
###########################################################################################################
FROM compiler-common AS compiler-stylesheet FROM compiler-common AS compiler-stylesheet
RUN cd ~ \ RUN cd ~ \
&& git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \ && git clone --single-branch --branch v5.4.0 https://github.com/gravitystorm/openstreetmap-carto.git --depth 1 \
@ -97,7 +27,7 @@ RUN mkdir -p /home/renderer/src \
########################################################################################################### ###########################################################################################################
FROM ubuntu:20.04 AS final FROM ubuntu:22.04 AS final
# Based on # Based on
# https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/ # https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
@ -111,30 +41,33 @@ RUN apt-get update \
&& apt-get install -y --no-install-recommends \ && apt-get install -y --no-install-recommends \
apache2 \ apache2 \
cron \ cron \
dateutils \
fonts-noto-cjk \ fonts-noto-cjk \
fonts-noto-hinted \ fonts-noto-hinted \
fonts-noto-unhinted \ fonts-noto-unhinted \
fonts-unifont \
gnupg2 \ gnupg2 \
gdal-bin \ gdal-bin \
liblua5.3-dev \ liblua5.3-dev \
lua5.3 \ lua5.3 \
mapnik-utils \ mapnik-utils \
npm \ npm \
osm2pgsql \
osmium-tool \ osmium-tool \
osmosis \ osmosis \
postgresql-14 \
postgresql-14-postgis-3 \
postgresql-14-postgis-3-scripts \
postgis \
python-is-python3 \ python-is-python3 \
python3-mapnik \ python3-mapnik \
python3-lxml \ python3-lxml \
python3-psycopg2 \ python3-psycopg2 \
python3-shapely \ python3-shapely \
python3-pip \ python3-pip \
renderd \
sudo \ sudo \
ttf-unifont \
wget \ wget \
&& echo "deb http://apt.postgresql.org/pub/repos/apt focal-pgdg main" > /etc/apt/sources.list.d/pgdg.list \
&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
&& apt-get update \
&& apt-get install -y --no-install-recommends postgresql-14 \
&& apt-get clean autoclean \ && apt-get clean autoclean \
&& apt-get autoremove --yes \ && apt-get autoremove --yes \
&& rm -rf /var/lib/{apt,dpkg,cache,log}/ && rm -rf /var/lib/{apt,dpkg,cache,log}/
@ -151,11 +84,7 @@ RUN pip3 install \
RUN npm install -g carto@0.18.2 RUN npm install -g carto@0.18.2
# Configure Apache # Configure Apache
RUN mkdir /var/lib/mod_tile \ RUN echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \
&& chown renderer /var/lib/mod_tile \
&& mkdir /var/run/renderd \
&& chown renderer /var/run/renderd \
&& echo "LoadModule tile_module /usr/lib/apache2/modules/mod_tile.so" >> /etc/apache2/conf-available/mod_tile.conf \
&& echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \ && echo "LoadModule headers_module /usr/lib/apache2/modules/mod_headers.so" >> /etc/apache2/conf-available/mod_headers.conf \
&& a2enconf mod_tile && a2enconf mod_headers && a2enconf mod_tile && a2enconf mod_headers
COPY apache.conf /etc/apache2/sites-available/000-default.conf COPY apache.conf /etc/apache2/sites-available/000-default.conf
@ -179,43 +108,28 @@ RUN chown -R postgres:postgres /var/lib/postgresql \
&& echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf && echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf
# Create volume directories # Create volume directories
RUN mkdir -p /data/database/ \ RUN mkdir -p /run/renderd/ \
&& mkdir -p /data/database/ \
&& mkdir -p /data/style/ \ && mkdir -p /data/style/ \
&& mkdir -p /home/renderer/src/ \ && mkdir -p /home/renderer/src/ \
&& chown -R renderer: /data/ \ && chown -R renderer: /data/ \
&& chown -R renderer: /home/renderer/src/ \ && chown -R renderer: /home/renderer/src/ \
&& chown -R renderer: /run/renderd \
&& mv /var/lib/postgresql/14/main/ /data/database/postgres/ \ && mv /var/lib/postgresql/14/main/ /data/database/postgres/ \
&& mv /var/lib/mod_tile/ /data/tiles/ \ && mv /var/cache/renderd/tiles/ /data/tiles/ \
&& chown -R renderer: /data/tiles \
&& ln -s /data/database/postgres /var/lib/postgresql/14/main \ && ln -s /data/database/postgres /var/lib/postgresql/14/main \
&& ln -s /data/style /home/renderer/src/openstreetmap-carto \ && ln -s /data/style /home/renderer/src/openstreetmap-carto \
&& ln -s /data/tiles /var/lib/mod_tile \ && ln -s /data/tiles /var/cache/renderd/tiles \
; ;
# Install PostGIS RUN echo '[default] \n\
COPY --from=compiler-postgis postgis_src/postgis-src_3.2.1-1_amd64.deb . URI=/tile/ \n\
RUN dpkg -i postgis-src_3.2.1-1_amd64.deb \ TILEDIR=/var/cache/renderd/tiles \n\
&& rm postgis-src_3.2.1-1_amd64.deb XML=/home/renderer/src/openstreetmap-carto/mapnik.xml \n\
HOST=localhost \n\
# Install osm2pgsql TILESIZE=256 \n\
COPY --from=compiler-osm2pgsql /root/osm2pgsql/build/build_1-1_amd64.deb . MAXZOOM=20' >> /etc/renderd.conf
RUN dpkg -i build_1-1_amd64.deb \
&& rm build_1-1_amd64.deb
# Install renderd
COPY --from=compiler-modtile-renderd /root/mod_tile/renderd_1-1_amd64.deb .
RUN dpkg -i renderd_1-1_amd64.deb \
&& rm renderd_1-1_amd64.deb \
&& sed -i 's/renderaccount/renderer/g' /usr/local/etc/renderd.conf \
&& sed -i 's/\/truetype//g' /usr/local/etc/renderd.conf \
&& sed -i 's/hot/tile/g' /usr/local/etc/renderd.conf
# Install mod_tile
COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb .
RUN dpkg -i mod-tile_1-1_amd64.deb \
&& ldconfig \
&& rm mod-tile_1-1_amd64.deb
COPY --from=compiler-modtile-renderd /root/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag
# Install helper script # Install helper script
COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional COPY --from=compiler-helper-script /home/renderer/src/regional /home/renderer/src/regional

View File

@ -126,6 +126,8 @@ Given that you've set up your import as described in the *Automatic updates* sec
``` ```
docker run \ docker run \
-p 8080:80 \ -p 8080:80 \
-e REPLICATION_URL=https://planet.openstreetmap.org/replication/minute/ \
-e MAX_INTERVAL_SECONDS=60 \
-e UPDATES=enabled \ -e UPDATES=enabled \
-v osm-data:/data/database/ \ -v osm-data:/data/database/ \
-v osm-tiles:/data/tiles/ \ -v osm-tiles:/data/tiles/ \
@ -244,7 +246,7 @@ You can find an example of the import performance to expect with this image on t
If you encounter such entries in the log, it will mean that the default shared memory limit (64 MB) is too low for the container and it should be raised: If you encounter such entries in the log, it will mean that the default shared memory limit (64 MB) is too low for the container and it should be raised:
``` ```
renderd[121]: ERROR: failed to render TILE ajt 2 0-3 0-3 renderd[121]: ERROR: failed to render TILE default 2 0-3 0-3
renderd[121]: reason: Postgis Plugin: ERROR: could not resize shared memory segment "/PostgreSQL.790133961" to 12615680 bytes: ### No space left on device renderd[121]: reason: Postgis Plugin: ERROR: could not resize shared memory segment "/PostgreSQL.790133961" to 12615680 bytes: ### No space left on device
``` ```
To raise it use `--shm-size` parameter. For example: To raise it use `--shm-size` parameter. For example:

View File

@ -1,8 +1,9 @@
<VirtualHost *:80> <VirtualHost *:80>
ServerAdmin webmaster@localhost ServerAdmin webmaster@localhost
LoadTileConfigFile /usr/local/etc/renderd.conf AddTileConfig /tile/ default
ModTileRenderdSocketName /var/run/renderd/renderd.sock LoadTileConfigFile /etc/renderd.conf
ModTileRenderdSocketName /run/renderd/renderd.sock
ModTileRequestTimeout 0 ModTileRequestTimeout 0
ModTileMissingRequestTimeout 30 ModTileMissingRequestTimeout 30

27
openstreetmap-tiles-update-expire.sh Normal file → Executable file
View File

@ -3,7 +3,7 @@
set -e set -e
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
# AJT - change directory to mod_tile directory so that we can run replag # Change directory to mod_tile directory so that we can run replag
# and other things directly from this script when run from cron. # and other things directly from this script when run from cron.
# Change the actual location to wherever installed locally. # Change the actual location to wherever installed locally.
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
@ -61,6 +61,10 @@ EXPIRY_TOUCHFROM=13
EXPIRY_DELETEFROM=19 EXPIRY_DELETEFROM=19
EXPIRY_MAXZOOM=20 EXPIRY_MAXZOOM=20
REPLICATION_URL=${REPLICATION_URL:="https://planet.openstreetmap.org/replication/hour/"}
MAX_INTERVAL_SECONDS=${MAX_INTERVAL_SECONDS:="3600"}
#************************************************************************* #*************************************************************************
#************************************************************************* #*************************************************************************
@ -109,14 +113,16 @@ freelock()
if [ $# -eq 1 ] ; then if [ $# -eq 1 ] ; then
m_info "Initialising Osmosis replication system to $1" m_info "Initialising Osmosis replication system to $1"
mkdir -p $WORKOSM_DIR mkdir -p $WORKOSM_DIR
$OSMOSIS_BIN -v 5 --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG"
$OSMOSIS_BIN --read-replication-interval-init workingDirectory=$WORKOSM_DIR 1>&2 2> "$OSMOSISLOG" init_seq=$(/usr/lib/python3-pyosmium/pyosmium-get-changes --server $REPLICATION_URL -D $1)
url_dynamicPart=$(printf %09d $init_seq | sed 's_\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)\([0-9][0-9][0-9]\)_\1/\2/\3_')
wget $REPLICATION_URL/$url_dynamicPart.state.txt -O $WORKOSM_DIR/state.txt
wget "https://replicate-sequences.osm.mazdermind.de/?"$1"T00:00:00Z" -O $WORKOSM_DIR/state.txt cat > $WORKOSM_DIR/configuration.txt <<- EOM
baseUrl=$REPLICATION_URL
mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt maxInterval=$MAX_INTERVAL_SECONDS
sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt EOM
exit 0
fi fi
# make sure the lockfile is removed when we exit and then claim it # make sure the lockfile is removed when we exit and then claim it
@ -141,8 +147,9 @@ if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_ba
fi fi
seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2` seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2`
replag=`dateutils.ddiff $(cat $WORKOSM_DIR/state.txt | grep timestamp | cut -d "=" -f 2 | sed 's,\\\,,g') now`
m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`" m_ok "start import from seq-nr $seq, replag is $replag"
/bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt /bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
m_ok "downloading diff" m_ok "downloading diff"
@ -182,15 +189,13 @@ fi
m_ok "expiring tiles" m_ok "expiring tiles"
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
# When expiring tiles we need to define the style sheet if it's not "default".
# In this case it's "ajt".
# Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and # Previously all tiles on the "dirty" list between $EXPIRY_MINZOOM and
# $EXPIRY_MAXZOOM were dirtied. We currently re-render # $EXPIRY_MAXZOOM were dirtied. We currently re-render
# tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and # tiles >= $EXPIRY_MINZOOM and < $EXPIRY_DELETEFROM, expiry from 14 and
# delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM. # delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM.
# The default path to renderd.sock is fixed. # The default path to renderd.sock is fixed.
#------------------------------------------------------------------------------ #------------------------------------------------------------------------------
if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then if ! render_expired --map=default --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
m_info "Expiry failed" m_info "Expiry failed"
fi fi

13
run.sh
View File

@ -70,7 +70,7 @@ if [ "$1" == "import" ]; then
if [ -n "${DOWNLOAD_PBF:-}" ]; then if [ -n "${DOWNLOAD_PBF:-}" ]; then
echo "INFO: Download PBF file: $DOWNLOAD_PBF" echo "INFO: Download PBF file: $DOWNLOAD_PBF"
wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data/region.osm.pbf wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data/region.osm.pbf
if [ -n "$DOWNLOAD_POLY" ]; then if [ -n "${DOWNLOAD_POLY:-}" ]; then
echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY" echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY"
wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data/region.poly wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data/region.poly
fi fi
@ -78,7 +78,7 @@ if [ "$1" == "import" ]; then
if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
# determine and set osmosis_replication_timestamp (for consecutive updates) # determine and set osmosis_replication_timestamp (for consecutive updates)
REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44` REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -d "=" -f 2`
# initial setup of osmosis workspace (for consecutive updates) # initial setup of osmosis workspace (for consecutive updates)
sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
@ -168,11 +168,16 @@ if [ "$1" == "run" ]; then
setPostgresPassword setPostgresPassword
# Configure renderd threads # Configure renderd threads
sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /etc/renderd.conf
# start cron job to trigger consecutive updates # start cron job to trigger consecutive updates
if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
/etc/init.d/cron start /etc/init.d/cron start
sudo -u renderer touch /var/log/tiles/run.log; tail -f /var/log/tiles/run.log >> /proc/1/fd/1 &
sudo -u renderer touch /var/log/tiles/osmosis.log; tail -f /var/log/tiles/osmosis.log >> /proc/1/fd/1 &
sudo -u renderer touch /var/log/tiles/expiry.log; tail -f /var/log/tiles/expiry.log >> /proc/1/fd/1 &
sudo -u renderer touch /var/log/tiles/osm2pgsql.log; tail -f /var/log/tiles/osm2pgsql.log >> /proc/1/fd/1 &
fi fi
# Run while handling docker stop's SIGTERM # Run while handling docker stop's SIGTERM
@ -181,7 +186,7 @@ if [ "$1" == "run" ]; then
} }
trap stop_handler SIGTERM trap stop_handler SIGTERM
sudo -u renderer renderd -f -c /usr/local/etc/renderd.conf & sudo -u renderer renderd -f -c /etc/renderd.conf &
child=$! child=$!
wait "$child" wait "$child"