diff options
author | Ilya Zverev <zverik@textual.ru> | 2015-07-03 19:08:30 +0300 |
---|---|---|
committer | Alex Zolotarev <alex@maps.me> | 2015-09-23 02:54:39 +0300 |
commit | d22a0519445c8bd449888535a849e0a15eca3757 (patch) | |
tree | 8ae34f699a741097ebab076d460dfad43487dd4c /tools | |
parent | 008376c7a4b421b08bcfd12402e25137865c0c3e (diff) |
[generator] Add tests and fix debug run
Diffstat (limited to 'tools')
-rwxr-xr-x | tools/unix/generate_planet.sh | 17 | ||||
-rwxr-xr-x | tools/unix/test_planet.sh | 82 |
2 files changed, 95 insertions, 4 deletions
diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index 5d3d461934..72e2f37259 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -67,7 +67,7 @@ OPT_UPDATE= OPT_DOWNLOAD= OPT_ROUTING= OPT_ONLINE_ROUTING= -while getopts ":couUwravh" opt; do +while getopts ":couUwrapvh" opt; do case $opt in c) OPT_CLEAN=1 @@ -93,6 +93,8 @@ while getopts ":couUwravh" opt; do OPT_UPDATE=1 OPT_ROUTING=1 ;; + p) + ;; v) set -x ;; @@ -131,6 +133,7 @@ STATUS_FILE="$INTDIR/status" OSRM_FLAG="${OSRM_FLAG:-$INTDIR/osrm_done}" SCRIPTS_PATH="$(dirname "$0")" ROUTING_SCRIPT="$SCRIPTS_PATH/generate_planet_routing.sh" +TESTING_SCRIPT="$SCRIPTS_PATH/test_planet.sh" LOG_PATH="$TARGET/logs" mkdir -p "$LOG_PATH" PLANET_LOG="$LOG_PATH/generate_planet.log" @@ -149,7 +152,7 @@ if [ -n "${REGIONS:-}" ]; then BORDERS_BACKUP_PATH="$TARGET/borders.$(date +%Y%m%d%H%M%S)" mkdir -p "$BORDERS_BACKUP_PATH" log "BORDERS" "Note: old borders from $TARGET/borders were moved to $BORDERS_BACKUP_PATH" - ( cd "$TARGET/borders"; mv *.poly "$BORDERS_BACKUP_PATH/" ) + mv "$TARGET/borders"/*.poly "$BORDERS_BACKUP_PATH" fi echo "$REGIONS" | xargs -I % cp "%" "$TARGET/borders/" elif [ -z "$PREV_BORDERS" ]; then @@ -266,7 +269,7 @@ if [ -n "$OPT_ROUTING" ]; then putmode "Step R: Starting OSRM files generation" # If *.mwm.osm2ft were moved to INTDIR, let's put them back [ -n "$EXIT_ON_ERROR" ] && set +e # Grep returns non-zero status - [ -z "$(ls "$TARGET" | grep \.mwm\.osm2ft)" -a -n "$(ls "$INTDIR" | grep \.mwm\.osm2ft)" ] && mv "$INTDIR/*.mwm.osm2ft" "$TARGET" + [ -z "$(ls "$TARGET" | grep \.mwm\.osm2ft)" -a -n "$(ls "$INTDIR" | grep \.mwm\.osm2ft)" ] && mv "$INTDIR"/*.mwm.osm2ft "$TARGET" [ -n "$EXIT_ON_ERROR" ] && set -e if [ -n "$ASYNC_PBF" ]; then @@ -356,7 +359,7 @@ if [ "$MODE" == "resources" ]; then [ ! -e "$TARGET/countries.txt" ] && cp "$DATA_PATH/countries.txt" "$TARGET/countries.txt" "$GENERATOR_TOOL" --data_path="$TARGET" --user_resource_path="$DATA_PATH/" -generate_update 2>> "$PLANET_LOG" # We have no means of finding the resulting file, so let's assume it was magically placed in DATA_PATH - [ -e "$DATA_PATH/countries.txt.updated" ] && mv "$DATA_PATH/countries.txt.updated" "$TARGET/countries.txt" + [ -e "$TARGET/countries.txt.updated" ] && mv "$TARGET/countries.txt.updated" "$TARGET/countries.txt" # A quick fix: chmodding to a+rw all generated files for file in "$TARGET"/*.mwm*; do chmod 0666 "$file" @@ -382,6 +385,12 @@ if [ "$MODE" == "resources" ]; then chmod 0666 "$EXT_RES" [ -n "$EXIT_ON_ERROR" ] && set -e fi + MODE=test +fi + +if [ "$MODE" == "test" ]; then + putmode "Step 8: Testing data" + bash "$TESTING_SCRIPT" > "$LOG_PATH/test_planet.log" "$TARGET" fi # Cleaning up temporary directories diff --git a/tools/unix/test_planet.sh b/tools/unix/test_planet.sh new file mode 100755 index 0000000000..170797f343 --- /dev/null +++ b/tools/unix/test_planet.sh @@ -0,0 +1,82 @@ +#!/bin/bash +##################################################### +# Tests a planet build made with generate_planet.sh # +##################################################### + +if [ $# -eq 0 ]; then + echo + echo "This script analyzes a generate_planet.sh run and prints all issues." + echo "Usage: $0 <target_dir>" + echo + exit 1 +fi + +set -u # Fail on undefined variables + +OMIM_PATH="${OMIM_PATH:-$(cd "$(dirname "$0")/../.."; pwd)}" +TARGET="${TARGET:-$1}" +LOG_PATH="${LOG_PATH:-$TARGET/logs}" +PLANET_LOG="$LOG_PATH/generate_planet.log" + +source "$(dirname "$0")/find_generator_tool.sh" + +# Step 1: analyze logs and find errors +echo +echo '### LOGS' +grep -i 'error\|warn\|critical\|fail\|abort\|останов\|fatal' "$PLANET_LOG" | grep -v 'settings\.ini' +for log in "$LOG_PATH"/*.log; do + if [ "$log" != "$PLANET_LOG" ]; then + CONTENT="$(grep -i 'error\|warn\|critical\|fail\|abort\|останов\|fatal' "$log" | grep -v 'settings\.ini\|language file for co\|Zero length lin\|too many tokens\|Equal choices for way\|No feature id for way\|number of threads is')" + if [ -n "$CONTENT" ]; then + echo + echo $log + echo "$CONTENT" + fi + fi +done + +# Step 2: test if mwms and routing were made +echo +echo '### MISSING FILES' +# Missing MWM files can be derived only from intermediate borders +if [ -d "$TARGET/borders" ]; then + for border in "$TARGET/borders"/*.poly; do + MWM="$(basename "$border" .poly).mwm" + [ ! -f "$TARGET/$MWM" ] && echo "$MWM" + done +fi +# Only display missing routing files for existing MWMs +for mwm in "$TARGET"/*.mwm; do + if [[ "$mwm" != *World* ]]; then + [ ! -f "$mwm.routing" ] && echo "$(basename "$mwm").routing" + fi +done + +# Step 3: run calc_statistics and check for sections +echo +echo '### MISSING MWM SECTIONS' +FOUND_COASTS= +for mwm in "$TARGET"/*.mwm; do + BASENAME="$(basename "$mwm" .mwm)" + STAT="$("$GENERATOR_TOOL" --data_path="$TARGET" --user_resource_path="$OMIM_PATH/data/" --output="$BASENAME" --calc_statistics 2>/dev/null)" + [ -z "$FOUND_COASTS" -a -n "$(echo "$STAT" | grep 'natural|coastline|')" ] && FOUND_COASTS=1 + SECTIONS="$(echo "$STAT" | grep 'version : 8' | wc -l | tr -d ' ')" + [ -f "$mwm.routing" -a "$SECTIONS" != "2" ] && echo "$BASENAME: $SECTIONS" +done + +[ -z "$FOUND_COASTS" ] && echo && echo 'WARNING: Did not find any coastlines in MWM files' + +# Step 4: run intergation tests +echo +echo '### INTEGRATION TESTS' +# First, create a temporary directory with symlinks to all maps +# That way, it can be easily cleaned after routing engine creates a lot of temporary directories in it +FTARGET="$TARGET/symlinked_copy" +mkdir -p "$FTARGET" +TARGET_PATH="$(cd "$TARGET"; pwd)" +for file in "$TARGET"/*.mwm*; do + BASENAME="$(basename "$file")" + ln -s "$TARGET_PATH/$BASENAME" "$FTARGET/$BASENAME" +done +"$(dirname "$GENERATOR_TOOL")/integration_tests" "--data_path=$FTARGET/" "--user_resource_path=$OMIM_PATH/data/" "--suppress=online_cross_tests.*" 2>&1 +rm -r "$FTARGET" |