[generator] Add tests and fix debug run

2 files changed, 95 insertions, 4 deletions

diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh
index 5d3d461934..72e2f37259 100755
--- a/tools/unix/generate_planet.sh
+++ b/tools/unix/generate_planet.sh
@@ -67,7 +67,7 @@ OPT_UPDATE=
 OPT_DOWNLOAD=
 OPT_ROUTING=
 OPT_ONLINE_ROUTING=
-while getopts ":couUwravh" opt; do
+while getopts ":couUwrapvh" opt; do
   case $opt in
     c)
       OPT_CLEAN=1
@@ -93,6 +93,8 @@ while getopts ":couUwravh" opt; do
       OPT_UPDATE=1
       OPT_ROUTING=1
       ;;
+    p)
+      ;;
     v)
       set -x
       ;;
@@ -131,6 +133,7 @@ STATUS_FILE="$INTDIR/status"
 OSRM_FLAG="${OSRM_FLAG:-$INTDIR/osrm_done}"
 SCRIPTS_PATH="$(dirname "$0")"
 ROUTING_SCRIPT="$SCRIPTS_PATH/generate_planet_routing.sh"
+TESTING_SCRIPT="$SCRIPTS_PATH/test_planet.sh"
 LOG_PATH="$TARGET/logs"
 mkdir -p "$LOG_PATH"
 PLANET_LOG="$LOG_PATH/generate_planet.log"
@@ -149,7 +152,7 @@ if [ -n "${REGIONS:-}" ]; then
     BORDERS_BACKUP_PATH="$TARGET/borders.$(date +%Y%m%d%H%M%S)"
     mkdir -p "$BORDERS_BACKUP_PATH"
     log "BORDERS" "Note: old borders from $TARGET/borders were moved to $BORDERS_BACKUP_PATH"
-    ( cd "$TARGET/borders"; mv *.poly "$BORDERS_BACKUP_PATH/" )
+    mv "$TARGET/borders"/*.poly "$BORDERS_BACKUP_PATH"
   fi
   echo "$REGIONS" | xargs -I % cp "%" "$TARGET/borders/"
 elif [ -z "$PREV_BORDERS" ]; then
@@ -266,7 +269,7 @@ if [ -n "$OPT_ROUTING" ]; then
     putmode "Step R: Starting OSRM files generation"
     # If *.mwm.osm2ft were moved to INTDIR, let's put them back
     [ -n "$EXIT_ON_ERROR" ] && set +e # Grep returns non-zero status
-    [ -z "$(ls "$TARGET" | grep \.mwm\.osm2ft)" -a -n "$(ls "$INTDIR" | grep \.mwm\.osm2ft)" ] && mv "$INTDIR/*.mwm.osm2ft" "$TARGET"
+    [ -z "$(ls "$TARGET" | grep \.mwm\.osm2ft)" -a -n "$(ls "$INTDIR" | grep \.mwm\.osm2ft)" ] && mv "$INTDIR"/*.mwm.osm2ft "$TARGET"
     [ -n "$EXIT_ON_ERROR" ] && set -e
 
     if [ -n "$ASYNC_PBF" ]; then
@@ -356,7 +359,7 @@ if [ "$MODE" == "resources" ]; then
   [ ! -e "$TARGET/countries.txt" ] && cp "$DATA_PATH/countries.txt" "$TARGET/countries.txt"
   "$GENERATOR_TOOL" --data_path="$TARGET" --user_resource_path="$DATA_PATH/" -generate_update 2>> "$PLANET_LOG"
   # We have no means of finding the resulting file, so let's assume it was magically placed in DATA_PATH
-  [ -e "$DATA_PATH/countries.txt.updated" ] && mv "$DATA_PATH/countries.txt.updated" "$TARGET/countries.txt"
+  [ -e "$TARGET/countries.txt.updated" ] && mv "$TARGET/countries.txt.updated" "$TARGET/countries.txt"
   # A quick fix: chmodding to a+rw all generated files
   for file in "$TARGET"/*.mwm*; do
     chmod 0666 "$file"
@@ -382,6 +385,12 @@ if [ "$MODE" == "resources" ]; then
     chmod 0666 "$EXT_RES"
     [ -n "$EXIT_ON_ERROR" ] && set -e
   fi
+  MODE=test
+fi
+
+if [ "$MODE" == "test" ]; then
+  putmode "Step 8: Testing data"
+  bash "$TESTING_SCRIPT" > "$LOG_PATH/test_planet.log" "$TARGET"
 fi
 
 # Cleaning up temporary directories
diff --git a/tools/unix/test_planet.sh b/tools/unix/test_planet.sh
new file mode 100755
index 0000000000..170797f343
--- /dev/null
+++ b/tools/unix/test_planet.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+#####################################################
+# Tests a planet build made with generate_planet.sh #
+#####################################################
+
+if [ $# -eq 0 ]; then
+  echo
+  echo "This script analyzes a generate_planet.sh run and prints all issues."
+  echo "Usage: $0 <target_dir>"
+  echo
+  exit 1
+fi
+
+set -u # Fail on undefined variables
+
+OMIM_PATH="${OMIM_PATH:-$(cd "$(dirname "$0")/../.."; pwd)}"
+TARGET="${TARGET:-$1}"
+LOG_PATH="${LOG_PATH:-$TARGET/logs}"
+PLANET_LOG="$LOG_PATH/generate_planet.log"
+
+source "$(dirname "$0")/find_generator_tool.sh"
+
+# Step 1: analyze logs and find errors
+echo
+echo '### LOGS'
+grep -i 'error\|warn\|critical\|fail\|abort\|останов\|fatal' "$PLANET_LOG" | grep -v 'settings\.ini'
+for log in "$LOG_PATH"/*.log; do
+  if [ "$log" != "$PLANET_LOG" ]; then
+    CONTENT="$(grep -i 'error\|warn\|critical\|fail\|abort\|останов\|fatal' "$log" | grep -v 'settings\.ini\|language file for co\|Zero length lin\|too many tokens\|Equal choices for way\|No feature id for way\|number of threads is')"
+    if [ -n "$CONTENT" ]; then
+      echo
+      echo $log
+      echo "$CONTENT"
+    fi
+  fi
+done
+
+# Step 2: test if mwms and routing were made
+echo
+echo '### MISSING FILES'
+# Missing MWM files can be derived only from intermediate borders
+if [ -d "$TARGET/borders" ]; then
+  for border in "$TARGET/borders"/*.poly; do
+    MWM="$(basename "$border" .poly).mwm"
+    [ ! -f "$TARGET/$MWM" ] && echo "$MWM"
+  done
+fi
+# Only display missing routing files for existing MWMs
+for mwm in "$TARGET"/*.mwm; do
+  if [[ "$mwm" != *World* ]]; then
+    [ ! -f "$mwm.routing" ] && echo "$(basename "$mwm").routing"
+  fi
+done
+
+# Step 3: run calc_statistics and check for sections
+echo
+echo '### MISSING MWM SECTIONS'
+FOUND_COASTS=
+for mwm in "$TARGET"/*.mwm; do
+  BASENAME="$(basename "$mwm" .mwm)"
+  STAT="$("$GENERATOR_TOOL" --data_path="$TARGET" --user_resource_path="$OMIM_PATH/data/" --output="$BASENAME" --calc_statistics 2>/dev/null)"
+  [ -z "$FOUND_COASTS" -a -n "$(echo "$STAT" | grep 'natural|coastline|')" ] && FOUND_COASTS=1
+  SECTIONS="$(echo "$STAT" | grep 'version : 8' | wc -l |  tr -d ' ')"
+  [ -f "$mwm.routing" -a "$SECTIONS" != "2" ] && echo "$BASENAME: $SECTIONS"
+done
+
+[ -z "$FOUND_COASTS" ] && echo && echo 'WARNING: Did not find any coastlines in MWM files'
+
+# Step 4: run intergation tests
+echo
+echo '### INTEGRATION TESTS'
+# First, create a temporary directory with symlinks to all maps
+# That way, it can be easily cleaned after routing engine creates a lot of temporary directories in it
+FTARGET="$TARGET/symlinked_copy"
+mkdir -p "$FTARGET"
+TARGET_PATH="$(cd "$TARGET"; pwd)"
+for file in "$TARGET"/*.mwm*; do
+  BASENAME="$(basename "$file")"
+  ln -s "$TARGET_PATH/$BASENAME" "$FTARGET/$BASENAME"
+done
+"$(dirname "$GENERATOR_TOOL")/integration_tests" "--data_path=$FTARGET/" "--user_resource_path=$OMIM_PATH/data/" "--suppress=online_cross_tests.*" 2>&1
+rm -r "$FTARGET"