[generator] Separate files for logging regions, in a dedicated dir

2 files changed, 40 insertions, 34 deletions

diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh
index 0d05c2a8ae..5d3d461934 100755
--- a/tools/unix/generate_planet.sh
+++ b/tools/unix/generate_planet.sh
@@ -5,7 +5,7 @@
 
 # Displayed when there are unknown options
 usage() {
-  echo ''
+  echo
   echo "Usage: $0 [-c] [-u] [-w] [-r]"
   echo
   echo -e "-u\tUpdate planet until coastline is not broken"
@@ -49,7 +49,7 @@ forky() {
 log() {
    local prefix="[$(date +%Y/%m/%d\ %H:%M:%S)]:"
    echo "${prefix} $@" >&2
-   echo "${prefix} $@" >> "$GENERATOR_LOG"
+   echo "${prefix} $@" >> "$PLANET_LOG"
 }
 
 # Print mode start message and store it in the status file
@@ -131,8 +131,9 @@ STATUS_FILE="$INTDIR/status"
 OSRM_FLAG="${OSRM_FLAG:-$INTDIR/osrm_done}"
 SCRIPTS_PATH="$(dirname "$0")"
 ROUTING_SCRIPT="$SCRIPTS_PATH/generate_planet_routing.sh"
-GENERATOR_LOG="$TARGET/planet_generator.log"
-ROUTING_LOG="$TARGET/planet_routing.log"
+LOG_PATH="$TARGET/logs"
+mkdir -p "$LOG_PATH"
+PLANET_LOG="$LOG_PATH/generate_planet.log"
 log "STATUS" "Start"
 
 # Run external script to find generator_tool
@@ -170,6 +171,7 @@ export PLANET
 export OSMCTOOLS
 export NUM_PROCESSES
 export KEEP_INTDIR
+export LOG_PATH
 export REGIONS= # Routing script might expect something in this variable
 export BORDERS_PATH="$TARGET/borders" # Also for the routing script
 
@@ -199,6 +201,7 @@ if [ "$MODE" == "coast" ]; then
   [ ! -x "$OSMCTOOLS/osmfilter"  ] && wget -q -O - http://m.m.i24.cc/osmfilter.c  | cc -x c - -O3 -o "$OSMCTOOLS/osmfilter"
   if [ -n "$OPT_DOWNLOAD" ]; then
     # Planet download is requested
+    log "STATUS" "Step 0: Downloading and converting the planet"
     PLANET_PBF="$(dirname "$PLANET")/planet-latest.osm.pbf"
     wget -O "$PLANET_PBF" http://planet.openstreetmap.org/pbf/planet-latest.osm.pbf
     "$OSMCTOOLS/osmconvert" "$PLANET_PBF" --drop-author --drop-version --out-o5m "-o=$PLANET"
@@ -229,11 +232,11 @@ if [ "$MODE" == "coast" ]; then
       log "TIMEMARK" "Generate coastlines intermediate"
       [ -n "$EXIT_ON_ERROR" ] && set +e # Temporary disable to read error code
       "$GENERATOR_TOOL" --intermediate_data_path="$INTCOASTSDIR/" --node_storage=map --osm_file_type=o5m --osm_file_name="$COASTS" \
-        -preprocess 2>> "$GENERATOR_LOG"
+        -preprocess 2>> "$LOG_PATH/WorldCoasts.log"
       # Generate temporary coastlines file in the coasts intermediate dir
       log "TIMEMARK" "Generate coastlines"
       "$GENERATOR_TOOL" --intermediate_data_path="$INTCOASTSDIR/" --node_storage=map --osm_file_type=o5m --osm_file_name="$COASTS" \
-        --user_resource_path="$DATA_PATH/" -make_coasts -fail_on_coasts 2>&1 | tee -a "$GENERATOR_LOG"
+        --user_resource_path="$DATA_PATH/" -make_coasts -fail_on_coasts 2>&1 | tee -a "$LOG_PATH/WorldCoasts.log"
 
       if [ $? != 0 ]; then
         log "TIMEMARK" "Coastline merge failed"
@@ -268,27 +271,27 @@ if [ -n "$OPT_ROUTING" ]; then
 
     if [ -n "$ASYNC_PBF" ]; then
       (
-        bash "$ROUTING_SCRIPT" pbf >> "$ROUTING_LOG" 2>&1
-        bash "$ROUTING_SCRIPT" prepare >> "$ROUTING_LOG" 2>&1
+        bash "$ROUTING_SCRIPT" pbf >> "$PLANET_LOG" 2>&1
+        bash "$ROUTING_SCRIPT" prepare >> "$PLANET_LOG" 2>&1
       ) &
     else
       # Osmconvert takes too much memory: it makes sense to not extract pbfs asyncronously
-      bash "$ROUTING_SCRIPT" pbf >> "$ROUTING_LOG" 2>&1
-      ( bash "$ROUTING_SCRIPT" prepare >> "$ROUTING_LOG" 2>&1 ) &
+      bash "$ROUTING_SCRIPT" pbf >> "$PLANET_LOG" 2>&1
+      ( bash "$ROUTING_SCRIPT" prepare >> "$PLANET_LOG" 2>&1 ) &
     fi
   fi
 fi
 
 if [ -n "$OPT_ONLINE_ROUTING" ]; then
   putmode "Step RO: Generating OSRM files for osrm-routed server."
-  bash "$ROUTING_SCRIPT" online >> "$ROUTING_LOG" 2>&1
+  bash "$ROUTING_SCRIPT" online >> "$PLANET_LOG" 2>&1
 fi
 
 if [ "$MODE" == "inter" ]; then
   putmode "Step 3: Generating intermediate data for all MWMs"
   # 1st pass, run in parallel - preprocess whole planet to speed up generation if all coastlines are correct
   "$GENERATOR_TOOL" --intermediate_data_path="$INTDIR/" --node_storage=$NODE_STORAGE --osm_file_type=o5m --osm_file_name="$PLANET" \
-    -preprocess 2>> "$GENERATOR_LOG"
+    -preprocess 2>> "$PLANET_LOG"
   MODE=features
 fi
 
@@ -301,7 +304,7 @@ if [ "$MODE" == "features" ]; then
   [ -n "$OPT_WORLD" ] && PARAMS_SPLIT="$PARAMS_SPLIT -generate_world"
   [ -n "$OPT_WORLD" -a "$NODE_STORAGE" == "map" ] && log "WARNING: generating world files with NODE_STORAGE=map may lead to an out of memory error. Try NODE_STORAGE=mem if it fails."
   "$GENERATOR_TOOL" --intermediate_data_path="$INTDIR/" --node_storage=$NODE_STORAGE --osm_file_type=o5m --osm_file_name="$PLANET" \
-    --data_path="$TARGET" --user_resource_path="$DATA_PATH/" $PARAMS_SPLIT 2>> "$GENERATOR_LOG"
+    --data_path="$TARGET" --user_resource_path="$DATA_PATH/" $PARAMS_SPLIT 2>> "$PLANET_LOG"
   MODE=mwm
 fi
 
@@ -310,26 +313,23 @@ if [ "$MODE" == "mwm" ]; then
   # 3rd pass - do in parallel
   # but separate exceptions for world files to finish them earlier
   PARAMS="--data_path=$TARGET --user_resource_path=$DATA_PATH/ --node_storage=$NODE_STORAGE -generate_geometry -generate_index"
-  log "TIMEMARK" "Generate final mwms"
   if [ -n "$OPT_WORLD" ]; then
-    "$GENERATOR_TOOL" $PARAMS --output=World 2>> "$GENERATOR_LOG" &
-    "$GENERATOR_TOOL" $PARAMS --output=WorldCoasts 2>> "$GENERATOR_LOG" &
+    (
+      "$GENERATOR_TOOL" $PARAMS --output=World 2>> "$LOG_PATH/World.log"
+      "$GENERATOR_TOOL" --data_path="$TARGET" --user_resource_path="$DATA_PATH/" -generate_search_index --output=World 2>> "$LOG_PATH/World.log"
+    ) &
+    "$GENERATOR_TOOL" $PARAMS --output=WorldCoasts 2>> "$LOG_PATH/WorldCoasts.log" &
   fi
 
   PARAMS_WITH_SEARCH="$PARAMS -generate_search_index"
   for file in "$TARGET"/*.mwm.tmp; do
     if [[ "$file" != *minsk-pass* && "$file" != *World* ]]; then
-      "$GENERATOR_TOOL" $PARAMS_WITH_SEARCH --output="$(basename "$file" .mwm.tmp)" 2>> "$GENERATOR_LOG" &
+      BASENAME="$(basename "$file" .mwm.tmp)"
+      "$GENERATOR_TOOL" $PARAMS_WITH_SEARCH --output="$BASENAME" 2>> "$LOG_PATH/$BASENAME.log" &
       forky
     fi
   done
 
-  if [ -n "$OPT_WORLD" ]; then
-    wait # For generator_tool --output=World
-    log "TIMEMARK" "Generate world search index"
-    "$GENERATOR_TOOL" --data_path="$TARGET" --user_resource_path="$DATA_PATH/" -generate_search_index --output=World 2>> "$GENERATOR_LOG"
-  fi
-
   if [ -n "$OPT_ROUTING" ]; then
     MODE=routing
   else
@@ -345,7 +345,7 @@ if [ "$MODE" == "routing" ]; then
   if [ ! -e "$OSRM_FLAG" ]; then
     log "OSRM files are missing, skipping routing step."
   else
-    bash "$ROUTING_SCRIPT" mwm >> "$ROUTING_LOG" 2>&1
+    bash "$ROUTING_SCRIPT" mwm >> "$PLANET_LOG" 2>&1
   fi
   MODE=resources
 fi
@@ -354,7 +354,7 @@ if [ "$MODE" == "resources" ]; then
   putmode "Step 7: Updating resource lists"
   # Update countries list
   [ ! -e "$TARGET/countries.txt" ] && cp "$DATA_PATH/countries.txt" "$TARGET/countries.txt"
-  "$GENERATOR_TOOL" --data_path="$TARGET" --user_resource_path="$DATA_PATH/" -generate_update 2>> "$GENERATOR_LOG"
+  "$GENERATOR_TOOL" --data_path="$TARGET" --user_resource_path="$DATA_PATH/" -generate_update 2>> "$PLANET_LOG"
   # We have no means of finding the resulting file, so let's assume it was magically placed in DATA_PATH
   [ -e "$DATA_PATH/countries.txt.updated" ] && mv "$DATA_PATH/countries.txt.updated" "$TARGET/countries.txt"
   # A quick fix: chmodding to a+rw all generated files
diff --git a/tools/unix/generate_planet_routing.sh b/tools/unix/generate_planet_routing.sh
index 590e9f0ee3..b6a70e72b8 100755
--- a/tools/unix/generate_planet_routing.sh
+++ b/tools/unix/generate_planet_routing.sh
@@ -30,6 +30,7 @@ mkdir -p "$INTDIR"
 NUM_PROCESSES=${NUM_PROCESSES:-8}
 KEEP_INTDIR=${KEEP_INTDIR-}
 OSRM_FLAG="${OSRM_FLAG:-$INTDIR/osrm_done}"
+LOG_PATH=${LOG_PATH:-.}
 echo "[$(date +%Y/%m/%d\ %H:%M:%S)] $0 $1"
 
 if [ "$1" == "pbf" ]; then
@@ -79,15 +80,16 @@ elif [ "$1" == "prepare" ]; then
   for PBF in "$INTDIR"/*.pbf; do
     OSRM_FILE="${PBF%.*}.osrm"
     RESTRICTIONS_FILE="$OSRM_FILE.restrictions"
+    LOG="$LOG_PATH/$(basename "$PBF" .pbf).log"
     rm -f "$OSRM_FILE"
-    "$OSRM_BUILD_PATH/osrm-extract" --config "$EXTRACT_CFG" --profile "$PROFILE" "$PBF"
-    "$OSRM_BUILD_PATH/osrm-prepare" --config "$PREPARE_CFG" --profile "$PROFILE" "$OSRM_FILE" -r "$RESTRICTIONS_FILE"
-    "$OSRM_BUILD_PATH/osrm-mapsme" -i "$OSRM_FILE"
+    "$OSRM_BUILD_PATH/osrm-extract" --config "$EXTRACT_CFG" --profile "$PROFILE" "$PBF" >> "$LOG" 2>&1
+    "$OSRM_BUILD_PATH/osrm-prepare" --config "$PREPARE_CFG" --profile "$PROFILE" "$OSRM_FILE" -r "$RESTRICTIONS_FILE" >> "$LOG" 2>&1
+    "$OSRM_BUILD_PATH/osrm-mapsme" -i "$OSRM_FILE" >> "$LOG" 2>&1
     if [ -s "$OSRM_FILE" ]; then
       [ -z "$KEEP_INTDIR" ] && rm -f "$PBF"
       ONE_OSRM_READY=1
     else
-      echo "Failed to create $OSRM_FILE"
+      echo "Failed to create $OSRM_FILE" >> "$LOG"
     fi
   done
   [ -z "${ONE_OSRM_READY-}" ] && fail "No osrm files were prepared"
@@ -104,11 +106,14 @@ elif [ "$1" == "mwm" ]; then
     cp "$BORDERS_PATH"/*.poly "$POLY_DIR/"
   fi
 
-  export GENERATOR_TOOL
+  # Xargs has 255 chars limit for exec string, so we use short variable names.
+  export G="$GENERATOR_TOOL"
+  export K="--make_routing --make_cross_section"
   export TARGET
+  export LOG_PATH
   export DATA_PATH="$OMIM_PATH/data/"
   find "$INTDIR" -name '*.osrm' -print0 | xargs -0 -P $NUM_PROCESSES -I % \
-    sh -c '"$GENERATOR_TOOL" --make_routing --make_cross_section --osrm_file_name="%" --data_path="$TARGET" --user_resource_path="$DATA_PATH" --output="$(basename "%" .osrm)"'
+    sh -c 'O="%"; B="$(basename "$O" .osrm)"; "$G" $K --osrm_file_name="$O" --data_path="$TARGET" --user_resource_path="$DATA_PATH" --output="$B" 2>> "$LOG_PATH/$B.log"'
 
   if [ -n "${POLY_DIR-}" ]; then
     # delete temporary polygons
@@ -146,13 +151,14 @@ elif [ "$1" == "online" ]; then
   export STXXLCFG="$HOME/.stxxl"
   OSRM_FILE="$INTDIR/planet.osrm"
   RESTRICTIONS_FILE="$OSRM_FILE.restrictions"
+  LOG="$LOG_PATH/planet.log"
   rm -f "$OSRM_FILE"
-  "$OSRM_BUILD_PATH/osrm-extract" --config "$EXTRACT_CFG" --profile "$PROFILE" "$PBF"
-  "$OSRM_BUILD_PATH/osrm-prepare" --config "$PREPARE_CFG" --profile "$PROFILE" "$OSRM_FILE" -r "$RESTRICTIONS_FILE"
+  "$OSRM_BUILD_PATH/osrm-extract" --config "$EXTRACT_CFG" --profile "$PROFILE" "$PBF" >> "$LOG" 2>&1
+  "$OSRM_BUILD_PATH/osrm-prepare" --config "$PREPARE_CFG" --profile "$PROFILE" "$OSRM_FILE" -r "$RESTRICTIONS_FILE" >> "$LOG" 2>&1
   if [ -s "$OSRM_FILE" ]; then
      [ -z "$KEEP_INTDIR" ] && rm -f "$PBF"
   else
-      echo "Failed to create $OSRM_FILE"
+      echo "Failed to create $OSRM_FILE" >> "$LOG"
   fi
 
 else