champs-libres/openstreetmap-tile-server
5
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

changed volume paths ; FLAT_NODES=enabled (#261)

Browse Source 
Changes:
- /var/lib/postgresql/14/main/            => /data/database/
- /var/lib/mod_tile/                      => /data/tiles/
- /home/renderer/src/openstreetmap-carto/ => /data/style/
- /data.osm.pbf                           => /data/region.osm.pbf
- /data.poly                              => /data/region.poly
- /nodes/flat_nodes.bin                   => deprecated (use FLAT_NODES=enabled instead)

Because some files are needed later and should never be separated from the database, the import does save those files now into /data/databases/:
- /data/database/region.poly
- /data/database/flat_nodes.bin
- /data/database/planet-import-complete
- and automatic update artifacts
This commit is contained in:
Robin C. Ladiges 2022-04-09 15:00:29 +02:00 committed by GitHub
parent d0673dc72d
commit cffbd7be5b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 174 additions and 131 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/build-and-test.yaml vendored
View File

@ -26,7 +26,7 @@ jobs:
env:
VOLUME : osm-db
CONTAINER : osm-www
MOUNT : /var/lib/postgresql/14/main
MOUNT : /data/database/
steps:
-
name: Checkout

6
.travis.yml
View File

@ -10,9 +10,9 @@ before_script:
- docker pull overv/openstreetmap-tile-server || true
script:
- docker build --pull --cache-from overv/openstreetmap-tile-server --tag overv/openstreetmap-tile-server .
- docker volume create openstreetmap-data
- docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main overv/openstreetmap-tile-server import
- docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main -p 8080:80 -d overv/openstreetmap-tile-server run
- docker volume create osm-data
- docker run --rm -v osm-data:/data/database/ overv/openstreetmap-tile-server import
- docker run --rm -v osm-data:/data/database/ -p 8080:80 -d overv/openstreetmap-tile-server run
- sleep 30
- make DOCKER_IMAGE=overv/openstreetmap-tile-server stop
after_script:

27
Dockerfile
View File

@ -97,7 +97,7 @@ RUN mkdir -p /home/renderer/src \
###########################################################################################################
FROM ubuntu:20.04 AS final-base
FROM ubuntu:20.04 AS final
# Based on
# https://switch2osm.org/serving-tiles/manually-building-a-tile-server-18-04-lts/
@ -164,15 +164,12 @@ RUN ln -sf /dev/stdout /var/log/apache2/access.log \
&& ln -sf /dev/stderr /var/log/apache2/error.log
# Copy update scripts
COPY openstreetmap-tiles-update-expire /usr/bin/
RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire \
COPY openstreetmap-tiles-update-expire.sh /usr/bin/
RUN chmod +x /usr/bin/openstreetmap-tiles-update-expire.sh \
&& mkdir /var/log/tiles \
&& chmod a+rw /var/log/tiles \
&& ln -s /home/renderer/src/mod_tile/osmosis-db_replag /usr/bin/osmosis-db_replag \
&& echo "* * * * * renderer openstreetmap-tiles-update-expire\n" >> /etc/crontab
RUN mkdir /nodes \
&& chown renderer:renderer /nodes
&& echo "* * * * * renderer openstreetmap-tiles-update-expire.sh\n" >> /etc/crontab
# Configure PosgtreSQL
COPY postgresql.custom.conf.tmpl /etc/postgresql/14/main/
@ -181,9 +178,18 @@ RUN chown -R postgres:postgres /var/lib/postgresql \
&& echo "host all all 0.0.0.0/0 md5" >> /etc/postgresql/14/main/pg_hba.conf \
&& echo "host all all ::/0 md5" >> /etc/postgresql/14/main/pg_hba.conf
###########################################################################################################
FROM final-base AS final
# Create volume directories
RUN mkdir -p /data/database/ \
&& mkdir -p /data/style/ \
&& mkdir -p /home/renderer/src/ \
&& chown -R renderer: /data/ \
&& chown -R renderer: /home/renderer/src/ \
&& mv /var/lib/postgresql/14/main/ /data/database/postgres/ \
&& mv /var/lib/mod_tile/ /data/tiles/ \
&& ln -s /data/database/postgres /var/lib/postgresql/14/main \
&& ln -s /data/style /home/renderer/src/openstreetmap-carto \
&& ln -s /data/tiles /var/lib/mod_tile \
;
# Install PostGIS
COPY --from=compiler-postgis postgis_src/postgis-src_3.2.1-1_amd64.deb .
@ -206,7 +212,6 @@ RUN dpkg -i renderd_1-1_amd64.deb \
# Install mod_tile
COPY --from=compiler-modtile-renderd /root/mod_tile/mod-tile_1-1_amd64.deb .
RUN dpkg -i mod-tile_1-1_amd64.deb \
&& mkdir -p /home/renderer/src/openstreetmap-carto \
&& ldconfig \
&& rm mod-tile_1-1_amd64.deb

8
Makefile
View File

@ -9,10 +9,10 @@ push: build
docker push ${DOCKER_IMAGE}:latest
test: build
docker volume create openstreetmap-data
docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main ${DOCKER_IMAGE} import
docker run --rm -v openstreetmap-data:/var/lib/postgresql/14/main -p 8080:80 -d ${DOCKER_IMAGE} run
docker volume create osm-data
docker run --rm -v osm-data:/data/database/ ${DOCKER_IMAGE} import
docker run --rm -v osm-data:/data/database/ -p 8080:80 -d ${DOCKER_IMAGE} run
stop:
docker rm -f `docker ps | grep '${DOCKER_IMAGE}' | awk '{ print $$1 }'` || true
docker volume rm -f openstreetmap-data
docker volume rm -f osm-data

73
README.md
View File

@ -9,41 +9,43 @@ This container allows you to easily set up an OpenStreetMap PNG tile server give
First create a Docker volume to hold the PostgreSQL database that will contain the OpenStreetMap data:
docker volume create openstreetmap-data
docker volume create osm-data
Next, download an .osm.pbf extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data.osm.pbf`. For example:
Next, download an `.osm.pbf` extract from geofabrik.de for the region that you're interested in. You can then start importing it into PostgreSQL by running a container and mounting the file as `/data/region.osm.pbf`. For example:
```
docker run \
-v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v /absolute/path/to/luxembourg.osm.pbf:/data/region.osm.pbf \
-v osm-data:/data/database/ \
overv/openstreetmap-tile-server \
import
```
If the container exits without errors, then your data has been successfully imported and you are now ready to run the tile server.
Note that the import process requires an internet connection. The run process does not require an internet connection. If you want to run the openstreetmap-tile server on a computer that is isolated, you must first import on an internet connected computer, export the openstreetmap-data volume as a tarfile, and then restore the data volume on the target computer system.
Note that the import process requires an internet connection. The run process does not require an internet connection. If you want to run the openstreetmap-tile server on a computer that is isolated, you must first import on an internet connected computer, export the `osm-data` volume as a tarfile, and then restore the data volume on the target computer system.
Also when running on an isolated system, the default index.html from the container will not work, as it requires access to the web for the leaflet packages.
Also when running on an isolated system, the default `index.html` from the container will not work, as it requires access to the web for the leaflet packages.
### Automatic updates (optional)
If your import is an extract of the planet and has polygonal bounds associated with it, like those from geofabrik.de, then it is possible to set your server up for automatic updates. Make sure to reference both the OSM file and the polygon file during the import process to facilitate this, and also include the `UPDATES=enabled` variable:
If your import is an extract of the planet and has polygonal bounds associated with it, like those from [geofabrik.de](https://download.geofabrik.de/), then it is possible to set your server up for automatic updates. Make sure to reference both the OSM file and the polygon file during the `import` process to facilitate this, and also include the `UPDATES=enabled` variable:
```
docker run \
-e UPDATES=enabled \
-v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
-v /absolute/path/to/luxembourg.poly:/data.poly \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v openstreetmap-rendered-tiles:/var/lib/mod_tile \
-v /absolute/path/to/luxembourg.osm.pbf:/data/region.osm.pbf \
-v /absolute/path/to/luxembourg.poly:/data/region.poly \
-v osm-data:/data/database/ \
overv/openstreetmap-tile-server \
import
```
Refer to the section *Automatic updating and tile expiry* to actually enable the updates while running the tile server.
Please note: If you're not importing the whole planet, then the `.poly` file is necessary to limit automatic updates to the relevant region.
Therefore, when you only have a `.osm.pbf` file but not a `.poly` file, you should not enable automatic updates.
### Letting the container download the file
It is also possible to let the container download files for you rather than mounting them in advance by using the `DOWNLOAD_PBF` and `DOWNLOAD_POLY` parameters:
@ -52,7 +54,7 @@ It is also possible to let the container download files for you rather than moun
docker run \
-e DOWNLOAD_PBF=https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf \
-e DOWNLOAD_POLY=https://download.geofabrik.de/europe/luxembourg.poly \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v osm-data:/data/database/ \
overv/openstreetmap-tile-server \
import
```
@ -69,15 +71,15 @@ docker run \
-e NAME_STYLE=test.style
-e NAME_MML=project.mml
-e NAME_SQL=test.sql
-v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto \
-v openstreetmap-data:/var/lib/postgresql/12/main \
-v /home/user/openstreetmap-carto-modified:/data/style/ \
-v osm-data:/data/database/ \
overv/openstreetmap-tile-server \
import
```
If you do not define the "NAME_*" variables, the script will default to those found in the openstreetmap-carto style.
Be sure to mount the volume during `run` with the same `-v /home/user/openstreetmap-carto-modified:/home/renderer/src/openstreetmap-carto`
Be sure to mount the volume during `run` with the same `-v /home/user/openstreetmap-carto-modified:/data/style/`
If you do not see the expected style upon `run` double check your paths as the style may not have been found at the directory specified. By default, `openstreetmap-carto` will be used if a style cannot be found
@ -90,7 +92,7 @@ Run the server like this:
```
docker run \
-p 8080:80 \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v osm-data:/data/database/ \
-d overv/openstreetmap-tile-server \
run
```
@ -103,19 +105,19 @@ The `docker-compose.yml` file included with this repository shows how the aforem
### Preserving rendered tiles
Tiles that have already been rendered will be stored in `/var/lib/mod_tile`. To make sure that this data survives container restarts, you should create another volume for it:
Tiles that have already been rendered will be stored in `/data/tiles/`. To make sure that this data survives container restarts, you should create another volume for it:
```
docker volume create openstreetmap-rendered-tiles
docker volume create osm-tiles
docker run \
-p 8080:80 \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v openstreetmap-rendered-tiles:/var/lib/mod_tile \
-v osm-data:/data/database/ \
-v osm-tiles:/data/tiles/ \
-d overv/openstreetmap-tile-server \
run
```
**If you do this, then make sure to also run the import with the `openstreetmap-rendered-tiles` volume to make sure that caching works properly across updates!**
**If you do this, then make sure to also run the import with the `osm-tiles` volume to make sure that caching works properly across updates!**
### Enabling automatic updating (optional)
@ -125,8 +127,8 @@ Given that you've set up your import as described in the *Automatic updates* sec
docker run \
-p 8080:80 \
-e UPDATES=enabled \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v openstreetmap-rendered-tiles:/var/lib/mod_tile \
-v osm-data:/data/database/ \
-v osm-tiles:/data/tiles/ \
-d overv/openstreetmap-tile-server \
run
```
@ -140,7 +142,7 @@ To enable the `Access-Control-Allow-Origin` header to be able to retrieve tiles
```
docker run \
-p 8080:80 \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v osm-data:/data/database/ \
-e ALLOW_CORS=enabled \
-d overv/openstreetmap-tile-server \
run
@ -154,7 +156,7 @@ To connect to the PostgreSQL database inside the container, make sure to expose
docker run \
-p 8080:80 \
-p 5432:5432 \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v osm-data:/data/database/ \
-d overv/openstreetmap-tile-server \
run
```
@ -172,7 +174,7 @@ docker run \
-p 8080:80 \
-p 5432:5432 \
-e PGPASSWORD=secret \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v osm-data:/data/database/ \
-d overv/openstreetmap-tile-server \
run
```
@ -188,7 +190,7 @@ The import and tile serving processes use 4 threads by default, but this number
docker run \
-p 8080:80 \
-e THREADS=24 \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v osm-data:/data/database/ \
-d overv/openstreetmap-tile-server \
run
```
@ -200,7 +202,7 @@ The import and tile serving processes use 800 MB RAM cache by default, but this
docker run \
-p 8080:80 \
-e "OSM2PGSQL_EXTRA_ARGS=-C 4096" \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v osm-data:/data/database/ \
-d overv/openstreetmap-tile-server \
run
```
@ -212,27 +214,24 @@ The database use the autovacuum feature by default. This behavior can be changed
docker run \
-p 8080:80 \
-e AUTOVACUUM=off \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v osm-data:/data/database/ \
-d overv/openstreetmap-tile-server \
run
```
### Flat nodes
### FLAT_NODES
If you are planning to import the entire planet or you are running into memory errors then you may want to enable the `--flat-nodes` option for osm2pgsql. You can then use it during the import process as follows:
```
docker run \
-v /absolute/path/to/luxembourg.osm.pbf:/data.osm.pbf \
-v openstreetmap-nodes:/nodes \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-e "OSM2PGSQL_EXTRA_ARGS=--flat-nodes /nodes/flat_nodes.bin" \
-v /absolute/path/to/luxembourg.osm.pbf:/data/region.osm.pbf \
-v osm-data:/data/database/ \
-e "FLAT_NODES=enabled" \
overv/openstreetmap-tile-server \
import
```
>Note that if you use a folder other than `/nodes` then you must make sure that you manually set the owner to `renderer`!
### Benchmarks
You can find an example of the import performance to expect with this image on the [OpenStreetMap wiki](https://wiki.openstreetmap.org/wiki/Osm2pgsql/benchmarks#debian_9_.2F_openstreetmap-tile-server).
@ -250,7 +249,7 @@ To raise it use `--shm-size` parameter. For example:
```
docker run \
-p 8080:80 \
-v openstreetmap-data:/var/lib/postgresql/14/main \
-v osm-data:/data/database/ \
--shm-size="192m" \
-d overv/openstreetmap-tile-server \
run

4
docker-compose.yml
View File

@ -4,11 +4,11 @@ services:
map:
image: overv/openstreetmap-tile-server
volumes:
- openstreetmap-data:/var/lib/postgresql/14/main
- osm-data:/data/database/
ports:
- "8080:80"
command: "run"
volumes:
openstreetmap-data:
osm-data:
external: true

80
openstreetmap-tiles-update-expire → openstreetmap-tiles-update-expire.sh
View File

@ -6,9 +6,6 @@ set -e
# AJT - change directory to mod_tile directory so that we can run replag
# and other things directly from this script when run from cron.
# Change the actual location to wherever installed locally.
#------------------------------------------------------------------------------
ACCOUNT=renderer
#------------------------------------------------------------------------------
# Extra OSM2PGSQL_OPTIONS may need setting if a tag transform script is
# in use. See https://github.com/SomeoneElseOSM/SomeoneElse-style and
@ -17,10 +14,15 @@ ACCOUNT=renderer
#------------------------------------------------------------------------------
OSMOSIS_BIN=osmosis
OSM2PGSQL_BIN=osm2pgsql
TRIM_BIN=/home/$ACCOUNT/src/regional/trim_osc.py
TRIM_BIN=/home/renderer/src/regional/trim_osc.py
DBNAME=gis
OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} ${OSM2PGSQL_EXTRA_ARGS}"
OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /data/style/${NAME_STYLE:-openstreetmap-carto.style} ${OSM2PGSQL_EXTRA_ARGS}"
# flat-nodes
if [ -f /data/database/flat_nodes.bin ]; then
OSM2PGSQL_OPTIONS="${OSM2PGSQL_OPTIONS} --flat-nodes /data/database/flat_nodes.bin"
fi
#------------------------------------------------------------------------------
# When using trim_osc.py we can define either a bounding box (such as this
@ -28,12 +30,12 @@ OSM2PGSQL_OPTIONS="-d $DBNAME -G --hstore --tag-transform-script /home/renderer/
# See https://github.com/zverik/regional .
# This area will usually correspond to the data originally loaded.
#------------------------------------------------------------------------------
TRIM_POLY_FILE="/var/lib/mod_tile/data.poly"
TRIM_POLY_FILE="/data/database/region.poly"
TRIM_OPTIONS="-d $DBNAME"
TRIM_REGION_OPTIONS="-p $TRIM_POLY_FILE"
BASE_DIR=/var/lib/mod_tile
LOG_DIR=/var/log/tiles/
BASE_DIR=/data/database
LOG_DIR=/var/log/tiles
WORKOSM_DIR=$BASE_DIR/.osmosis
LOCK_FILE=/tmp/openstreetmap-update-expire-lock.txt
@ -64,7 +66,7 @@ EXPIRY_MAXZOOM=20
m_info()
{
echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG"
echo "[`date +"%Y-%m-%d %H:%M:%S"`] $$ $1" >> "$RUNLOG"
}
m_error()
@ -114,18 +116,19 @@ if [ $# -eq 1 ] ; then
mv $WORKOSM_DIR/configuration.txt $WORKOSM_DIR/configuration_orig.txt
sed "s!baseUrl=http://planet.openstreetmap.org/replication/minute!baseUrl=https://planet.openstreetmap.org/replication/minute!" $WORKOSM_DIR/configuration_orig.txt > $WORKOSM_DIR/configuration.txt
else
exit 0
fi
# make sure the lockfile is removed when we exit and then claim it
if ! getlock "$LOCK_FILE"; then
m_info "pid `cat $LOCK_FILE` still running"
exit 3
fi
if ! getlock "$LOCK_FILE"; then
m_info "pid `cat $LOCK_FILE` still running"
exit 3
fi
if [ -e $STOP_FILE ]; then
m_info "stopped"
exit 2
fi
if [ -e $STOP_FILE ]; then
m_info "stopped"
exit 2
fi
# -----------------------------------------------------------------------------
# Add disk space check from https://github.com/zverik/regional
@ -137,16 +140,16 @@ if `python -c "import os, sys; st=os.statvfs('$BASE_DIR'); sys.exit(1 if st.f_ba
exit 4
fi
seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2`
seq=`cat $WORKOSM_DIR/state.txt | grep sequenceNumber | cut -d= -f2`
m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`"
m_ok "start import from seq-nr $seq, replag is `osmosis-db_replag -h`"
/bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
m_ok "downloading diff"
/bin/cp $WORKOSM_DIR/state.txt $WORKOSM_DIR/last.state.txt
m_ok "downloading diff"
if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then
m_error "Osmosis error"
fi
if ! $OSMOSIS_BIN --read-replication-interval workingDirectory=$WORKOSM_DIR --simplify-change --write-xml-change $CHANGE_FILE 1>&2 2> "$OSMOSISLOG"; then
m_error "Osmosis error"
fi
if [ -f $TRIM_POLY_FILE ] ; then
m_ok "filtering diff"
@ -156,16 +159,17 @@ if [ -f $TRIM_POLY_FILE ] ; then
else
m_ok "filtering diff skipped"
fi
m_ok "importing diff"
m_ok "importing diff"
#------------------------------------------------------------------------------
# Previously openstreetmap-tiles-update-expire tried to dirty layer
# "$EXPIRY_MAXZOOM - 3" (which was 15) only. Instead we write all expired
# tiles in range to the list (note the "-" rather than ":" in the "-e"
# parameter).
#------------------------------------------------------------------------------
if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then
m_error "osm2pgsql error"
fi
if ! $OSM2PGSQL_BIN -a --slim -e$EXPIRY_MINZOOM-$EXPIRY_MAXZOOM $OSM2PGSQL_OPTIONS -o "$EXPIRY_FILE.$$" $CHANGE_FILE 1>&2 2> "$PGSQLLOG"; then
m_error "osm2pgsql error"
fi
#------------------------------------------------------------------------------
# The lockfile is normally removed before we expire tiles because that is
@ -175,8 +179,8 @@ fi
#------------------------------------------------------------------------------
# m_ok "Import complete; removing lock file"
# freelock "$LOCK_FILE"
m_ok "expiring tiles"
m_ok "expiring tiles"
#------------------------------------------------------------------------------
# When expiring tiles we need to define the style sheet if it's not "default".
# In this case it's "ajt".
@ -186,18 +190,16 @@ fi
# delete >= $EXPIRY_DELETEFROM and <= $EXPIRY_MAXZOOM.
# The default path to renderd.sock is fixed.
#------------------------------------------------------------------------------
if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
m_info "Expiry failed"
fi
if ! render_expired --map=ajt --min-zoom=$EXPIRY_MINZOOM --touch-from=$EXPIRY_TOUCHFROM --delete-from=$EXPIRY_DELETEFROM --max-zoom=$EXPIRY_MAXZOOM -s /var/run/renderd/renderd.sock < "$EXPIRY_FILE.$$" 2>&1 | tail -8 >> "$EXPIRYLOG"; then
m_info "Expiry failed"
fi
rm "$EXPIRY_FILE.$$"
rm "$EXPIRY_FILE.$$"
#------------------------------------------------------------------------------
# Only remove the lock file after expiry (if system is slow we want to delay
# the next import, not have multiple render_expired processes running)
#------------------------------------------------------------------------------
freelock "$LOCK_FILE"
freelock "$LOCK_FILE"
m_ok "Done with import"
fi
m_ok "Done with import"

105
run.sh
View File

@ -15,7 +15,7 @@ function setPostgresPassword() {
if [ "$#" -ne 1 ]; then
echo "usage: <import|run>"
echo "commands:"
echo " import: Set up the database and import /data.osm.pbf"
echo " import: Set up the database and import /data/region.osm.pbf"
echo " run: Runs Apache and renderd to serve tiles at /tile/{z}/{x}/{y}.png"
echo "environment variables:"
echo " THREADS: defines number of threads used for importing / tile rendering"
@ -29,24 +29,22 @@ fi
set -x
if [ ! "$(ls -A /home/renderer/src/openstreetmap-carto)" ]; then
mv /home/renderer/src/openstreetmap-carto-backup/* /home/renderer/src/openstreetmap-carto/
# if there is no custom style mounted, then use osm-carto
if [ ! "$(ls -A /data/style/)" ]; then
mv /home/renderer/src/openstreetmap-carto-backup/* /data/style/
fi
if [ ! -f /home/renderer/src/openstreetmap-carto/mapnik.xml ]; then
cd /home/renderer/src/openstreetmap-carto
# carto build
if [ ! -f /data/style/mapnik.xml ]; then
cd /data/style/
carto ${NAME_MML:-project.mml} > mapnik.xml
fi
if [ "$1" = "import" ]; then
if [ "$1" == "import" ]; then
# Ensure that database directory is in right state
chown postgres:postgres -R /var/lib/postgresql
if [ ! -f /var/lib/postgresql/14/main/PG_VERSION ]; then
sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /var/lib/postgresql/14/main/ initdb -o "--locale C.UTF-8"
chown -R postgres: /var/lib/postgresql /data/database/postgres/
if [ ! -f /data/database/postgres/PG_VERSION ]; then
sudo -u postgres /usr/lib/postgresql/14/bin/pg_ctl -D /data/database/postgres/ initdb -o "--locale C.UTF-8"
fi
# Initialize PostgreSQL
@ -61,61 +59,100 @@ if [ "$1" = "import" ]; then
setPostgresPassword
# Download Luxembourg as sample if no data is provided
if [ ! -f /data.osm.pbf ] && [ -z "${DOWNLOAD_PBF:-}" ]; then
echo "WARNING: No import file at /data.osm.pbf, so importing Luxembourg as example..."
if [ ! -f /data/region.osm.pbf ] && [ -z "${DOWNLOAD_PBF:-}" ]; then
echo "WARNING: No import file at /data/region.osm.pbf, so importing Luxembourg as example..."
DOWNLOAD_PBF="https://download.geofabrik.de/europe/luxembourg-latest.osm.pbf"
DOWNLOAD_POLY="https://download.geofabrik.de/europe/luxembourg.poly"
fi
if [ -n "${DOWNLOAD_PBF:-}" ]; then
echo "INFO: Download PBF file: $DOWNLOAD_PBF"
wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data.osm.pbf
wget ${WGET_ARGS:-} "$DOWNLOAD_PBF" -O /data/region.osm.pbf
if [ -n "$DOWNLOAD_POLY" ]; then
echo "INFO: Download PBF-POLY file: $DOWNLOAD_POLY"
wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data.poly
wget ${WGET_ARGS:-} "$DOWNLOAD_POLY" -O /data/region.poly
fi
fi
if [ "${UPDATES:-}" = "enabled" ]; then
if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
# determine and set osmosis_replication_timestamp (for consecutive updates)
osmium fileinfo /data.osm.pbf > /var/lib/mod_tile/data.osm.pbf.info
osmium fileinfo /data.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44 > /var/lib/mod_tile/replication_timestamp.txt
REPLICATION_TIMESTAMP=$(cat /var/lib/mod_tile/replication_timestamp.txt)
REPLICATION_TIMESTAMP=`osmium fileinfo /data/region.osm.pbf | grep 'osmosis_replication_timestamp=' | cut -b35-44`
# initial setup of osmosis workspace (for consecutive updates)
sudo -u renderer openstreetmap-tiles-update-expire $REPLICATION_TIMESTAMP
sudo -u renderer openstreetmap-tiles-update-expire.sh $REPLICATION_TIMESTAMP
fi
# copy polygon file if available
if [ -f /data.poly ]; then
sudo -u renderer cp /data.poly /var/lib/mod_tile/data.poly
if [ -f /data/region.poly ]; then
cp /data/region.poly /data/database/region.poly
chown renderer: /data/database/region.poly
fi
# flat-nodes
if [ "${FLAT_NODES:-}" == "enabled" ] || [ "${FLAT_NODES:-}" == "1" ]; then
OSM2PGSQL_EXTRA_ARGS="${OSM2PGSQL_EXTRA_ARGS:-} --flat-nodes /data/database/flat_nodes.bin"
fi
# Import data
sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore --tag-transform-script /home/renderer/src/openstreetmap-carto/${NAME_LUA:-openstreetmap-carto.lua} --number-processes ${THREADS:-4} -S /home/renderer/src/openstreetmap-carto/${NAME_STYLE:-openstreetmap-carto.style} /data.osm.pbf ${OSM2PGSQL_EXTRA_ARGS:-}
sudo -u renderer osm2pgsql -d gis --create --slim -G --hstore \
--tag-transform-script /data/style/${NAME_LUA:-openstreetmap-carto.lua} \
--number-processes ${THREADS:-4} \
-S /data/style/${NAME_STYLE:-openstreetmap-carto.style} \
/data/region.osm.pbf \
${OSM2PGSQL_EXTRA_ARGS:-} \
;
# old flat-nodes dir
if [ -f /nodes/flat_nodes.bin ] && ! [ -f /data/database/flat_nodes.bin ]; then
mv /nodes/flat_nodes.bin /data/database/flat_nodes.bin
chown renderer: /data/database/flat_nodes.bin
fi
# Create indexes
sudo -u postgres psql -d gis -f /home/renderer/src/openstreetmap-carto/${NAME_SQL:-indexes.sql}
if [ -f /data/style/${NAME_SQL:-indexes.sql} ]; then
sudo -u postgres psql -d gis -f /data/style/${NAME_SQL:-indexes.sql}
fi
#Import external data
sudo chown -R renderer: /home/renderer/src
sudo -E -u renderer python3 /home/renderer/src/openstreetmap-carto/scripts/get-external-data.py -c /home/renderer/src/openstreetmap-carto/external-data.yml -D /home/renderer/src/openstreetmap-carto/data
chown -R renderer: /home/renderer/src/ /data/style/
if [ -f /data/style/scripts/get-external-data.py ] && [ -f /data/style/external-data.yml ]; then
sudo -E -u renderer python3 /data/style/scripts/get-external-data.py -c /data/style/external-data.yml -D /data/style/data
fi
# Register that data has changed for mod_tile caching purposes
touch /var/lib/mod_tile/planet-import-complete
sudo -u renderer touch /data/database/planet-import-complete
service postgresql stop
exit 0
fi
if [ "$1" = "run" ]; then
if [ "$1" == "run" ]; then
# Clean /tmp
rm -rf /tmp/*
# migrate old files
if [ -f /data/database/PG_VERSION ] && ! [ -d /data/database/postgres/ ]; then
mkdir /data/database/postgres/
mv /data/database/* /data/database/postgres/
fi
if [ -f /nodes/flat_nodes.bin ] && ! [ -f /data/database/flat_nodes.bin ]; then
mv /nodes/flat_nodes.bin /data/database/flat_nodes.bin
fi
if [ -f /data/tiles/data.poly ] && ! [ -f /data/database/region.poly ]; then
mv /data/tiles/data.poly /data/database/region.poly
fi
# sync planet-import-complete file
if [ -f /data/tiles/planet-import-complete ] && ! [ -f /data/database/planet-import-complete ]; then
cp /data/tiles/planet-import-complete /data/database/planet-import-complete
fi
if ! [ -f /data/tiles/planet-import-complete ] && [ -f /data/database/planet-import-complete ]; then
cp /data/database/planet-import-complete /data/tiles/planet-import-complete
fi
# Fix postgres data privileges
chown postgres:postgres /var/lib/postgresql -R
chown -R postgres: /var/lib/postgresql/ /data/database/postgres/
# Configure Apache CORS
if [ "${ALLOW_CORS:-}" == "enabled" ] || [ "${ALLOW_CORS:-}" == "1" ]; then
@ -132,8 +169,8 @@ if [ "$1" = "run" ]; then
sed -i -E "s/num_threads=[0-9]+/num_threads=${THREADS:-4}/g" /usr/local/etc/renderd.conf
# start cron job to trigger consecutive updates
if [ "${UPDATES:-}" = "enabled" ] || [ "${UPDATES:-}" = "1" ]; then
/etc/init.d/cron start
if [ "${UPDATES:-}" == "enabled" ] || [ "${UPDATES:-}" == "1" ]; then
/etc/init.d/cron start
fi
# Run while handling docker stop's SIGTERM