diff options
author | Ilya Zverev <zverik@textual.ru> | 2015-08-06 12:58:09 +0300 |
---|---|---|
committer | Alex Zolotarev <alex@maps.me> | 2015-09-23 02:59:28 +0300 |
commit | a884f34625fb387a2ec9add8315de86ab1a46e62 (patch) | |
tree | 23cdcd7896821428639efac905f46a361b4d2766 /tools | |
parent | d639306d1d22f87ec4387cb968580a96deb972da (diff) |
[generator] More of a big refactoring
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/unix/generate_mwm.sh | 2 | ||||
-rwxr-xr-x | tools/unix/generate_planet.sh | 67 |
2 files changed, 37 insertions, 32 deletions
diff --git a/tools/unix/generate_mwm.sh b/tools/unix/generate_mwm.sh index 21be6075cd..514f8d4804 100755 --- a/tools/unix/generate_mwm.sh +++ b/tools/unix/generate_mwm.sh @@ -137,3 +137,5 @@ fi # This file is needed only for routing generation rm -f "$TARGET/$BASE_NAME.mwm.osm2ft" +# Remove temporary offsets table +[ -d "$TARGET/$BASE_NAME" ] && rm -r "$TARGET/$BASE_NAME" diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index 90dc3cdbbc..82f53d95af 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -114,8 +114,7 @@ while getopts ":couUlwrapvh" opt; do esac done -EXIT_ON_ERROR=${EXIT_ON_ERROR-1} -[ -n "$EXIT_ON_ERROR" ] && set -e # Exit when any of commands fail +set -e # Exit when any of commands fail set -o pipefail # Capture all errors in command chains set -u # Fail on undefined variables @@ -132,11 +131,11 @@ mkdir -p "$TARGET" INTDIR="${INTDIR:-$TARGET/intermediate_data}" OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}" [ ! -d "$OSMCTOOLS" ] && OSMCTOOLS="$INTDIR" -MERGE_COASTS_DELAY_MIN=40 +MERGE_INTERVAL=${MERGE_INTERVAL:-40} # set to "mem" if there is more than 64 GB of memory NODE_STORAGE=${NODE_STORAGE:-${NS:-mem}} ASYNC_PBF=${ASYNC_PBF-} -NUM_PROCESSES=${NUM_PROCESSES:-$(expr $(nproc || echo 8) - 1)} +NUM_PROCESSES=${NUM_PROCESSES:-$(($(nproc || echo 8) - 1))} KEEP_INTDIR=${KEEP_INTDIR-1} STATUS_FILE="$INTDIR/status" @@ -154,13 +153,11 @@ log "STATUS" "Start" source "$SCRIPTS_PATH/find_generator_tool.sh" # Prepare borders -[ -n "$EXIT_ON_ERROR" ] && set +e # Grep returns non-zero status mkdir -p "$TARGET/borders" -PREV_BORDERS="$(ls "$TARGET/borders" | grep \.poly)" NO_REGIONS= if [ -n "${REGIONS:-}" ]; then # If region files are specified, backup old borders and copy new - if [ -n "$PREV_BORDERS" ]; then + if [ -n "$(ls "$TARGET/borders" | grep '\.poly')" ]; then BORDERS_BACKUP_PATH="$TARGET/borders.$(date +%Y%m%d%H%M%S)" mkdir -p "$BORDERS_BACKUP_PATH" log "BORDERS" "Note: old borders from $TARGET/borders were moved to $BORDERS_BACKUP_PATH" @@ -170,15 +167,14 @@ if [ -n "${REGIONS:-}" ]; then elif [ -z "${REGIONS-1}" ]; then # A user asked specifically for no regions NO_REGIONS=1 -elif [ -z "$PREV_BORDERS" ]; then +elif [ -z "$(ls "$TARGET/borders" | grep '\.poly')" ]; then # If there are no borders, copy them from $BORDERS_PATH BORDERS_PATH="${BORDERS_PATH:-$DATA_PATH/borders}" cp "$BORDERS_PATH"/*.poly "$TARGET/borders/" fi -[ -z "$NO_REGIONS" -a -z "$(ls "$TARGET/borders" | grep \.poly)" ] && fail "No border polygons found, please use REGIONS or BORDER_PATH variables" -ULIMIT_REQ=$(expr 3 \* $(ls "$TARGET/borders" | grep \.poly | wc -l)) +[ -z "$NO_REGIONS" -a -z "$(ls "$TARGET/borders" | grep '\.poly')" ] && fail "No border polygons found, please use REGIONS or BORDER_PATH variables" +ULIMIT_REQ=$((3 * $(ls "$TARGET/borders" | { grep '\.poly' || true; } | wc -l))) [ $(ulimit -n) -lt $ULIMIT_REQ ] && fail "Ulimit is too small, you need at least $ULIMIT_REQ (e.g. ulimit -n 4000)" -[ -n "$EXIT_ON_ERROR" ] && set -e # These variables are used by external script(s), namely generate_planet_routing.sh export GENERATOR_TOOL @@ -249,26 +245,23 @@ if [ "$MODE" == "coast" ]; then # Strip coastlines from the planet to speed up the process "$OSMCTOOLS/osmfilter" "$PLANET" --keep= --keep-ways="natural=coastline" "-o=$COASTS" # Preprocess coastlines to separate intermediate directory - [ -n "$EXIT_ON_ERROR" ] && set +e # Temporary disable to read error code "$GENERATOR_TOOL" --intermediate_data_path="$INTCOASTSDIR/" --node_storage=map --osm_file_type=o5m --osm_file_name="$COASTS" \ -preprocess 2>> "$LOG_PATH/WorldCoasts.log" # Generate temporary coastlines file in the coasts intermediate dir - "$GENERATOR_TOOL" --intermediate_data_path="$INTCOASTSDIR/" --node_storage=map --osm_file_type=o5m --osm_file_name="$COASTS" \ - --user_resource_path="$DATA_PATH/" -make_coasts -fail_on_coasts 2>&1 | tee -a "$LOG_PATH/WorldCoasts.log" | grep -i 'not merged\|total' - - if [ $? != 0 ]; then + if ! "$GENERATOR_TOOL" --intermediate_data_path="$INTCOASTSDIR/" --node_storage=map --osm_file_type=o5m --osm_file_name="$COASTS" \ + --user_resource_path="$DATA_PATH/" -make_coasts -fail_on_coasts 2>&1 | tee -a "$LOG_PATH/WorldCoasts.log" | { grep -i 'not merged' || true; } + then log "STATUS" "Coastline merge failed" if [ -n "$OPT_UPDATE" ]; then - tail -n 50 "$LOG_PATH/WorldCoasts.log" | mailx -s "Generate_planet: coastline merge failed, next try in $MERGE_COASTS_DELAY_MIN minutes" "$MAIL" + [ -n "${MAIL-}" ] && tail -n 50 "$LOG_PATH/WorldCoasts.log" | mailx -s "Generate_planet: coastline merge failed, next try in $MERGE_INTERVAL minutes" "$MAIL" date -u - echo "Will try fresh coasts again in $MERGE_COASTS_DELAY_MIN minutes, or press a key..." - read -rs -n 1 -t $(expr $MERGE_COASTS_DELAY_MIN \* 60) + echo "Will try fresh coasts again in $MERGE_INTERVAL minutes, or press a key..." + read -rs -n 1 -t $(($MERGE_INTERVAL * 60)) || true TRY_AGAIN=1 else fail fi fi - [ -n "$EXIT_ON_ERROR" ] && set -e fi done # make a working copy of generated coastlines file @@ -285,9 +278,7 @@ if [ -n "$OPT_ROUTING" -a -z "$NO_REGIONS" ]; then else putmode "Step R: Starting OSRM files generation" # If *.mwm.osm2ft were moved to INTDIR, let's put them back - [ -n "$EXIT_ON_ERROR" ] && set +e # Grep returns non-zero status - [ -z "$(ls "$TARGET" | grep \.mwm\.osm2ft)" -a -n "$(ls "$INTDIR" | grep \.mwm\.osm2ft)" ] && mv "$INTDIR"/*.mwm.osm2ft "$TARGET" - [ -n "$EXIT_ON_ERROR" ] && set -e + [ -z "$(ls "$TARGET" | grep '\.mwm\.osm2ft')" -a -n "$(ls "$INTDIR" | grep '\.mwm\.osm2ft')" ] && mv "$INTDIR"/*.mwm.osm2ft "$TARGET" if [ -n "$ASYNC_PBF" ]; then ( @@ -318,7 +309,17 @@ fi if [ "$MODE" == "features" ]; then putmode "Step 4: Generating features of everything into $TARGET" # Checking for coastlines, can't build proper mwms without them - [ ! -s "$INTDIR/WorldCoasts.geom" ] && fail "Please prepare coastlines and put WorldCoasts.geom to $INTDIR" + if [ ! -s "$INTDIR/WorldCoasts.geom" ]; then + COASTS="${COASTS-WorldCoasts.geom}" + if [ -s "$COASTS" ]; then + cp "$COASTS" "$INTDIR/WorldCoasts.geom" + RAWGEOM="${COASTS%.*}.rawgeom" + [ -s "$RAWGEOM" ] && cp "$RAWGEOM" "$INTDIR/WorldCoasts.rawgeom" + else + fail "Please prepare coastlines and put WorldCoasts.geom to $INTDIR" + fi + fi + [ -n "$OPT_WORLD" -a ! -s "$INTDIR/WorldCoasts.rawgeom" ] && fail "You need WorldCoasts.rawgeom in $INTDIR to build a world file" # 2nd pass - paralleled in the code PARAMS_SPLIT="-generate_features -emit_coasts" [ -z "$NO_REGIONS" ] && PARAMS_SPLIT="$PARAMS_SPLIT -split_by_polygons" @@ -332,9 +333,7 @@ fi if [ "$MODE" == "mwm" ]; then putmode "Step 5: Building all MWMs of regions and of the whole world into $TARGET" # First, check for *.mwm.tmp - [ -n "$EXIT_ON_ERROR" ] && set +e # Grep returns non-zero status - [ -z "$NO_REGIONS" -a -z "$(ls "$INTDIR/tmp" | grep \.mwm\.tmp)" ] && fail "No .mwm.tmp files found." - [ -n "$EXIT_ON_ERROR" ] && set -e + [ -z "$NO_REGIONS" -a -z "$(ls "$INTDIR/tmp" | grep '\.mwm\.tmp')" ] && fail "No .mwm.tmp files found." # 3rd pass - do in parallel # but separate exceptions for world files to finish them earlier PARAMS="--data_path=$TARGET --intermediate_data_path=$INTDIR/ --user_resource_path=$DATA_PATH/ --node_storage=$NODE_STORAGE -generate_geometry -generate_index" @@ -400,8 +399,7 @@ if [ "$MODE" == "resources" ]; then if [ -n "$OPT_WORLD" -o -n "$OPT_COAST" ]; then # Update external resources - [ -n "$EXIT_ON_ERROR" ] && set +e # Grep returns non-zero status - [ -z "$(ls "$TARGET" | grep \.ttf)" ] && cp "$DATA_PATH"/*.ttf "$TARGET" + [ -z "$(ls "$TARGET" | grep '\.ttf')" ] && cp "$DATA_PATH"/*.ttf "$TARGET" EXT_RES="$TARGET/external_resources.txt" echo -n > "$EXT_RES" UNAME="$(uname)" @@ -415,7 +413,6 @@ if [ "$MODE" == "resources" ]; then fi done chmod 0666 "$EXT_RES" - [ -n "$EXIT_ON_ERROR" ] && set -e fi MODE=test fi @@ -430,10 +427,16 @@ if [ "$MODE" == "test" ]; then fi fi +# Clean temporary indices +if [ -n "$(ls "$TARGET" | grep '\.mwm')" ]; then + for mwm in "$TARGET"/*.mwm; do + BASENAME="${mwm%.mwm}" + [ -d "$BASENAME" ] && rm -r "$BASENAME" + done +fi # Cleaning up temporary directories -set +e rm "$STATUS_FILE" [ -f "$OSRM_FLAG" ] && rm "$OSRM_FLAG" -mv "$TARGET"/*.mwm.osm2ft "$INTDIR" +[ -n "$(ls "$TARGET" | grep '\.mwm\.osm2ft')" ] && mv "$TARGET"/*.mwm.osm2ft "$INTDIR" [ -z "$KEEP_INTDIR" ] && rm -r "$INTDIR" log "STATUS" "Done" |