[generator] Removed old scripts for maps generation.

2 files changed, 0 insertions, 839 deletions

diff --git a/tools/unix/generate_mwm.sh b/tools/unix/generate_mwm.sh
deleted file mode 100755
index 178bd1e610..0000000000
--- a/tools/unix/generate_mwm.sh
+++ /dev/null
@@ -1,128 +0,0 @@
-#!/bin/bash
-###########################
-# Builds one specific MWM #
-###########################
-
-# Prerequisites:
-#
-# - The script should be placed in omim/tools/unix, or supply OMIM_PATH with path to omim repo
-# - Data path with classificators etc. should be present in $OMIM_PATH/data
-#     Inside it should be at least: categories.txt, classificator.txt, types.txt, drules_proto.bin
-# - Compiled generator_tool somewhere in omim/../build/out/whatever, or supply BUILD_PATH
-# - Target path for mwm is the same as o5m path, or supply TARGET
-# - Set TMPDIR variable if you have no space in the default temporary location.
-
-set -u
-set -e
-
-if [ $# -lt 1 ]; then
-  echo
-  echo "Usage: $0 \<file.o5m/osm/bz2/pbf\>"
-  echo
-  echo "Useful environment variables:"
-  echo
-  echo -e "BORDERS_PATH\tPath to *.poly files for cross-mwm routing"
-  echo -e "BORDER\tPath and name of a polygon file for the input"
-  echo -e "COASTS\tPath and name of WorldCoasts.geom"
-  echo -e "TARGET\tWhere to put resulting files"
-  echo -e "OSM2FT\tSet to 1 to keep the osm2ft file"
-  echo
-  exit 1
-fi
-
-fail() {
-  [ $# -gt 0 ] && echo "$@" >&2
-  exit 1
-}
-
-find_osmconvert() {
-  # just a guess
-  OSMCONVERT="${OSMCONVERT:-$HOME/osmctools/osmconvert}"
-  if [ ! -x "$OSMCONVERT" ]; then
-    OSMCONVERT="$INTDIR/osmconvert"
-    echo "compiling osmconvert"
-    if [ -e "$OMIM_PATH/tools/osmctools/osmconvert.c" ]; then
-      cc -x c -O3 "$OMIM_PATH/tools/osmctools/osmconvert.c" -o "$OSMCONVERT" -lz
-    else
-      curl -s https://raw.githubusercontent.com/mapsme/osmctools/master/osmconvert.c | cc -x c - -O3 -o $OSMCONVERT -lz
-    fi
-  fi
-}
-
-SOURCE_FILE="$1"
-SOURCE_TYPE="${1##*.}"
-BASE_NAME="$(basename "$SOURCE_FILE")"
-BASE_NAME="${BASE_NAME%%.*}"
-SCRIPT_PATH="$(dirname "$0")"
-TARGET="${TARGET:-$(dirname "$SOURCE_FILE")}"
-[ ! -d "$TARGET" ] && fail "$TARGET should be a writable folder"
-TBORDERS="$TARGET/borders"
-OMIM_PATH="${OMIM_PATH:-$(cd "$SCRIPT_PATH/../.."; pwd)}"
-DATA_PATH="${DATA_PATH:-$OMIM_PATH/data}"
-[ ! -r "$DATA_PATH/types.txt" ] && fail "Cannot find classificators in $DATA_PATH, please set correct OMIM_PATH"
-
-GENERATOR_TOOL="${GENERATOR_TOOL-$SCRIPT_PATH/bin/generator_tool}"
-[ -x "$SCRIPT_PATH/find_generator_tool.sh" ] && source "$SCRIPT_PATH/find_generator_tool.sh"
-[ ! -x "${GENERATOR_TOOL-}" ] && fail "Cannot find generator tool"
-
-if [ "$(uname)" == "Darwin" ]; then
-  INTDIR=$(mktemp -d -t mwmgen.XXXXXXXX)
-else
-  INTDIR=$(mktemp -d)
-fi
-trap "rm -rf \"${INTDIR}\"" EXIT SIGINT SIGTERM
-CITIES_BOUNDARIES_DATA="${CITIES_BOUNDARIES_DATA:-$INTDIR/cities_boundaries.bin}"
-
-# Create MWM file
-INTDIR_FLAG="--intermediate_data_path=$INTDIR/ --node_storage=map"
-GENERATE_EVERYTHING="--generate_features --generate_geometry --generate_index --generate_search_index --dump_cities_boundaries --cities_boundaries_data=$CITIES_BOUNDARIES_DATA"
-[ -n "${HOTELS-}" ] && GENERATE_EVERYTHING="$GENERATE_EVERYTHING --booking_data=$HOTELS"
-COASTS="${COASTS-WorldCoasts.geom}"
-if [ -f "$COASTS" ]; then
-  if [ ! -f "$TBORDERS/$BASE_NAME.poly" ]; then
-    BORDER="${BORDER:-${BORDERS_PATH:-.}/$BASE_NAME.poly}"
-    [ ! -f "$BORDER" ] && fail "Please specify polygon file in BORDER for coastline generation"
-    [ ! -d "$TBORDERS" ] && CLEAN_BORDERS=1 && mkdir "$TBORDERS"
-    CLEAN_POLY=1
-    cp "$BORDER" "$TBORDERS/$BASE_NAME.poly"
-  fi
-  cp "$COASTS" "$INTDIR/WorldCoasts.geom"
-  GENERATE_EVERYTHING="$GENERATE_EVERYTHING --emit_coasts=true --split_by_polygons=true"
-fi
-# Convert everything to o5m
-if [ "$SOURCE_TYPE" == "pbf" -o "$SOURCE_TYPE" == "bz2" -o "$SOURCE_TYPE" == "osm" ]; then
-  find_osmconvert
-  if [ "$SOURCE_TYPE" == "bz2" ]; then
-    bzcat "$SOURCE_FILE" | $OSMCONVERT - --out-o5m "-o=$INTDIR/$BASE_NAME.o5m" || fail
-  else
-    "$OSMCONVERT" "$SOURCE_FILE" --out-o5m "-o=$INTDIR/$BASE_NAME.o5m"
-  fi
-  SOURCE_FILE="$INTDIR/$BASE_NAME.o5m"
-  SOURCE_TYPE=o5m
-fi
-if [ "$SOURCE_TYPE" == "o5m" ]; then
-  if [ -n "${BORDERS_PATH-}" -a -f "${BORDERS_PATH-}/$BASE_NAME.poly" ]; then
-    [ ! -d "$TBORDERS" ] && CLEAN_BORDERS=1 && mkdir "$TBORDERS"
-    cp "$BORDERS_PATH"/*.poly "$TBORDERS"
-    CROSS_MWM="--make_cross_mwm"
-  fi
-
-  GENERATE_CAMERA_SECTION="--generate_cameras"
-
-  $GENERATOR_TOOL $INTDIR_FLAG --osm_file_type=o5m --osm_file_name="$SOURCE_FILE" --preprocess=true || fail "Preprocessing failed"
-  $GENERATOR_TOOL $INTDIR_FLAG --osm_file_type=o5m --osm_file_name="$SOURCE_FILE" --data_path="$TARGET" --user_resource_path="$DATA_PATH" $GENERATE_EVERYTHING --output="$BASE_NAME"
-  $GENERATOR_TOOL $INTDIR_FLAG --data_path="$TARGET" --user_resource_path="$DATA_PATH" --cities_boundaries_data="$CITIES_BOUNDARIES_DATA" --make_city_roads --output="$BASE_NAME"
-  $GENERATOR_TOOL $INTDIR_FLAG --data_path="$TARGET" --user_resource_path="$DATA_PATH" ${CROSS_MWM-} ${GENERATE_CAMERA_SECTION-} --make_routing_index --generate_traffic_keys --output="$BASE_NAME"
-else
-  echo "Unsupported source type: $SOURCE_TYPE" >&2
-fi
-
-[ -n "${CLEAN_POLY-}" ] && rm "$TBORDERS/$BASE_NAME.poly"
-[ -n "${CLEAN_BORDERS-}" ] && rm -r "$TBORDERS"
-
-# This file is needed only for routing generation
-[ -z "${OSM2FT-}" ] && rm -f "$TARGET/$BASE_NAME.mwm.osm2ft"
-# Remove temporary offsets table
-[ -d "$TARGET/$BASE_NAME" ] && rm -r "$TARGET/$BASE_NAME"
-
-exit 0 # Added so that the exit code of the script is 0, and not whatever the [ -d ] returns.
diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh
deleted file mode 100755
index 0f642deac9..0000000000
--- a/tools/unix/generate_planet.sh
+++ /dev/null
@@ -1,711 +0,0 @@
-#!/bin/bash
-###############################################
-# Builds the whole planet or selected regions #
-###############################################
-
-# Displayed when there are unknown options
-usage() {
-  echo
-  echo "Usage: $0 [-c] [-u] [-l] [-w] [-r] [-d]"
-  echo
-  echo -e "-u\tUpdate planet until coastline is not broken"
-  echo -e "-U\tDownload planet when it is missing"
-  echo -e "-l\tGenerate coastlines"
-  echo -e "-w\tGenerate a world file"
-  echo -e "-r\tGenerate routing files"
-  echo -e "-o\tGenerate online routing files"
-  echo -e "-d\tGenerate descriptions"
-  echo -e "-a\tEquivalent to -ulwrd"
-  echo -e "-p\tGenerate only countries, no world and no routing"
-  echo -e "-c\tClean last pass results if there was an error, and start anew"
-  echo -e "-v\tPrint all commands executed"
-  echo -e "-h\tThis help message"
-  echo
-  echo -e "\tIf there is unfinished job, continues it, ignoring all arguments (use -p if in doubt)."
-  echo -e "\tUse -c to clean temporary data and build everything from scratch."
-  echo
-  echo "Useful environment variables:"
-  echo
-  echo -e "PLANET\tPlanet file to use"
-  echo -e "TARGET\tWhere to put resulting files"
-  echo -e "REGIONS\tNewline-separated list of border polygons to use. Example usage:"
-  echo -e "\tREGIONS=\$(ls ../../data/A*.poly) $0"
-  echo -e "NS\tNode storage; use \"map\" when you have less than 64 GB of memory"
-  echo -e "ASYNC_PBF\tGenerate PBF files asynchronously, not in a separate step"
-  echo -e "MODE\tA mode to start with: coast, inter, routing, test, etc."
-  echo -e "MAIL\tE-mail address to send notifications"
-  echo -e "OSRM_URL\tURL of the osrm server to build world roads."
-  echo -e "SRTM_PATH\tPath to 27k zip files with SRTM data."
-  echo -e "OLD_INTDIR\tPath to an intermediate_data directory with older files."
-  echo
-}
-
-# Print an error message and exit
-fail() {
-  [ $# -gt 0 ] && echo "$@" >&2
-  exit 1
-}
-
-# Wait until there are no more processes than processors
-forky() {
-  while [ $(jobs | wc -l) -gt $NUM_PROCESSES ]; do
-    sleep 1
-  done
-}
-
-# Print timestamp and messages to stderr
-log() {
-   local prefix="[$(date +%Y/%m/%d\ %H:%M:%S)]:"
-   echo "${prefix} $@" >&2
-   echo "${prefix} $@" >> "$PLANET_LOG"
-}
-
-warn() {
-  echo -n -e "\033[0;31m" >&2
-  log 'WARNING:' $@
-  echo -n -e "\033[0m" >&2
-}
-
-# Print mode start message and store it in the status file
-putmode() {
-  [ $# -gt 0 ] && log "STATUS" "$@"
-  echo "$MFLAGS$MODE" > "$STATUS_FILE"
-}
-
-format_version() {
-  local planet_version=$1
-  local output_format=$2
-  if [ "$(uname -s)" == "Darwin" ]; then
-    echo $(date -j -u -f "%Y-%m-%dT%H:%M:%SZ" "$planet_version" "+$output_format")
-  else
-    echo $(date -d "$(echo "$planet_version" | sed -e 's/T/ /')" "+$output_format")
-  fi
-}
-
-get_python36() {
-  local pythons=('python' 'python3' 'python3.6' 'python36')
-  local best_version='0'
-  local cur_version='0'
-  local best_python=''
-  for python in ${pythons[@]}; do
-    if command -v "$python" >/dev/null 2>&1 ; then
-      cur_version=`$python -c 'import sys; print("".join(map(str, sys.version_info[:3])))'`
-      if [ "$best_version" \< "$cur_version" ]; then
-        best_version=$cur_version
-        best_python=$python
-      fi
-    fi
-  done
-  if [ "$best_version" \< "360" ]; then
-    echo "Python 3.6 minimum required, found only $best_python";
-    exit
-  fi
-
-  echo $(which $best_python)
-}
-
-get_python2() {
-  local pythons=('python' 'python2' 'python2.6' 'python2.7' 'python26' 'python27')
-  local best_version='0'
-  local cur_version='0'
-  local best_python=''
-  for python in ${pythons[@]}; do
-    if command -v "$python" >/dev/null 2>&1 ; then
-      cur_version=`$python -c 'import sys; print("".join(map(str, sys.version_info[:3])))'`
-      if [ "$best_version" \< "$cur_version" ] && [ $cur_version \< "300" ]; then
-        best_version=$cur_version
-        best_python=$python
-      fi
-    fi
-  done
-
-  if [ "$best_version" = "0" ]; then
-    echo "Can not find python with version >= 2.0";
-    exit
-  fi
-
-  echo $(which $best_python)
-}
-
-# Do not start processing when there are no arguments
-[ $# -eq 0 ] && usage && fail
-# Parse command line parameters
-OPT_CLEAN=
-OPT_COAST=
-OPT_WORLD=
-OPT_UPDATE=
-OPT_DOWNLOAD=
-OPT_ROUTING=
-OPT_ONLINE_ROUTING=
-OPT_DESCRIPTIONS=
-while getopts ":couUlwrapvhd" opt; do
-  case $opt in
-    c)
-      OPT_CLEAN=1
-      ;;
-    l)
-      OPT_COAST=1
-      ;;
-    w)
-      OPT_WORLD=1
-      ;;
-    u)
-      OPT_UPDATE=1
-      ;;
-    U)
-      OPT_DOWNLOAD=1
-      OPT_UPDATE=1
-      ;;
-    r)
-      OPT_ROUTING=1
-      ;;
-    o)
-      OPT_ONLINE_ROUTING=1
-      ;;
-    d)
-      OPT_DESCRIPTIONS=1
-      ;;
-    a)
-      OPT_COAST=1
-      OPT_WORLD=1
-      OPT_UPDATE=1
-      OPT_ROUTING=1
-      OPT_DESCRIPTIONS=1
-      ;;
-    p)
-      ;;
-    v)
-      set -x
-      ;;
-    *)
-      usage
-      fail
-      ;;
-  esac
-done
-
-set -e # Exit when any of commands fail
-set -o pipefail # Capture all errors in command chains
-set -u # Fail on undefined variables
-
-# Initialize everything. For variables with X="${X:-...}" you can override a default value
-PLANET="${PLANET:-$HOME/planet}"
-[ -d "$PLANET" ] && PLANET="$PLANET/planet-latest.o5m"
-[ ! -f "$PLANET" -a -z "$OPT_DOWNLOAD" ] && fail "Please put planet file into $PLANET, use -U, or specify correct PLANET variable"
-OMIM_PATH="${OMIM_PATH:-$(cd "$(dirname "$0")/../.."; pwd)}"
-DATA_PATH="${DATA_PATH:-$OMIM_PATH/data}"
-[ ! -r "${DATA_PATH}/types.txt" ] && fail "Cannot find classificators in $DATA_PATH, please set correct OMIM_PATH"
-if [ -n "$OPT_ROUTING" -a -n "${OSRM_URL-}" ]; then
-  curl -s "http://$OSRM_URL/way_id" | grep -q position || fail "Cannot access $OSRM_URL"
-fi
-TARGET="${TARGET:-$DATA_PATH}"
-mkdir -p "$TARGET"
-INTDIR="${INTDIR:-$TARGET/intermediate_data}"
-OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}"
-[ ! -d "$OSMCTOOLS" ] && OSMCTOOLS="$INTDIR"
-MERGE_INTERVAL=${MERGE_INTERVAL:-40}
-# set to "mem" if there is more than 64 GB of memory
-NODE_STORAGE=${NODE_STORAGE:-${NS:-mem}}
-ASYNC_PBF=${ASYNC_PBF-}
-KEEP_INTDIR=${KEEP_INTDIR-1}
-# nproc is linux-only
-if [ "$(uname -s)" == "Darwin" ]; then
-  CPUS="$(sysctl -n hw.ncpu)"
-else
-  CPUS="$(nproc || echo 8)"
-fi
-NUM_PROCESSES=${NUM_PROCESSES:-$(($CPUS - 1))}
-
-STATUS_FILE="$INTDIR/status"
-SCRIPTS_PATH="$(dirname "$0")"
-if [ -e "$SCRIPTS_PATH/hierarchy_to_countries.py" ]; then
-  # In a compact packaging python scripts may be placed along with shell scripts.
-  PYTHON_SCRIPTS_PATH="$SCRIPTS_PATH"
-else
-  PYTHON_SCRIPTS_PATH="$OMIM_PATH/tools/python"
-fi
-PYTHONPATH="${PYTHONPATH:-''}"
-PYTHONPATH="${PYTHONPATH}:$PYTHON_SCRIPTS_PATH"
-export PYTHONPATH
-ROADS_SCRIPT="$PYTHON_SCRIPTS_PATH/road_runner.py"
-HIERARCHY_SCRIPT="$PYTHON_SCRIPTS_PATH/post_generation/hierarchy_to_countries.py"
-DESCRIPTIONS_MODULE="descriptions"
-POST_GENERATION_MODULE="post_generation"
-UGC_FILE="${UGC_FILE:-$INTDIR/ugc_db.sqlite3}"
-POPULAR_PLACES_FILE="${POPULAR_PLACES_FILE:-$INTDIR/popular_places.csv}"
-WIKIDATA_FILE="${WIKIDATA_FILE:-$INTDIR/idToWikidata.csv}"
-BOOKING_MODULE="booking"
-BOOKING_FILE="${BOOKING_FILE:-$INTDIR/hotels.csv}"
-OPENTABLE_SCRIPT="$PYTHON_SCRIPTS_PATH/opentable_restaurants.py"
-OPENTABLE_FILE="${OPENTABLE_FILE:-$INTDIR/restaurants.csv}"
-CITIES_BOUNDARIES_DATA="${CITIES_BOUNDARIES_DATA:-$INTDIR/cities_boundaries.bin}"
-BRANDS_TRANSLATIONS_DATA="${BRANDS_TRANSLATIONS_DATA:-$INTDIR/translations_food.json}"
-BRANDS_DATA="${BRANDS_DATA:-$INTDIR/ids_food.json}"
-TESTING_SCRIPT="$SCRIPTS_PATH/test_planet.sh"
-PYTHON=$(get_python2)
-PYTHON36=$(get_python36)
-MWM_VERSION_FORMAT="%s"
-COUNTRIES_VERSION_FORMAT="%y%m%d"
-LOG_PATH="${LOG_PATH:-$TARGET/logs}"
-mkdir -p "$LOG_PATH"
-PLANET_LOG="$LOG_PATH/generate_planet.log"
-TEST_LOG="$LOG_PATH/test_planet.log"
-GENERATE_CAMERA_SECTION="--generate_cameras"
-[ -n "${MAIL-}" ] && trap "grep STATUS \"$PLANET_LOG\" | mailx -s \"Generate_planet: build failed at $(hostname)\" \"$MAIL\"; exit 1" SIGTERM ERR
-echo -e "\n\n----------------------------------------\n\n" >> "$PLANET_LOG"
-log "STATUS" "Start ${DESC-}"
-
-# Run external script to find generator_tool
-source "$SCRIPTS_PATH/find_generator_tool.sh"
-
-# Prepare borders
-mkdir -p "$TARGET/borders"
-if [ -n "$(ls "$TARGET/borders" | grep '\.poly')" ]; then
-  # Backup old borders
-  BORDERS_BACKUP_PATH="$TARGET/borders.$(date +%Y%m%d%H%M%S)"
-  mkdir -p "$BORDERS_BACKUP_PATH"
-  log "BORDERS" "Note: old borders from $TARGET/borders were moved to $BORDERS_BACKUP_PATH"
-  mv "$TARGET/borders"/*.poly "$BORDERS_BACKUP_PATH"
-fi
-NO_REGIONS=
-if [ -n "${REGIONS:-}" ]; then
-  echo "$REGIONS" | xargs -I % cp "%" "$TARGET/borders/"
-elif [ -z "${REGIONS-1}" ]; then
-  # A user asked specifically for no regions
-  NO_REGIONS=1
-else
-  # Copy borders from $BORDERS_PATH or omim/data/borders
-  cp "${BORDERS_PATH:-$DATA_PATH/borders}"/*.poly "$TARGET/borders/"
-fi
-[ -z "$NO_REGIONS" -a -z "$(ls "$TARGET/borders" | grep '\.poly')" ] && fail "No border polygons found, please use REGIONS or BORDER_PATH variables"
-ULIMIT_REQ=$((3 * $(ls "$TARGET/borders" | { grep '\.poly' || true; } | wc -l)))
-[ $(ulimit -n) -lt $ULIMIT_REQ ] && fail "Ulimit is too small, you need at least $ULIMIT_REQ (e.g. ulimit -n 4000)"
-
-[ -n "$OPT_CLEAN" -a -d "$INTDIR" ] && rm -r "$INTDIR"
-mkdir -p "$INTDIR"
-if [ -z "${REGIONS+1}" -a "$(df -m "$INTDIR" | tail -n 1 | awk '{ printf "%d\n", $4 / 1024 }')" -lt "400" ]; then
-  warn "You have less than 400 GB for intermediate data, that's not enough for the whole planet."
-fi
-
-# These variables are used by external script(s), namely generate_planet_routing.sh
-export GENERATOR_TOOL
-export INTDIR
-export OMIM_PATH
-export PLANET
-export OSMCTOOLS
-export NUM_PROCESSES
-export KEEP_INTDIR
-export LOG_PATH
-export REGIONS= # Routing script might expect something in this variable
-export BORDERS_PATH="$TARGET/borders" # Also for the routing script
-export LC_ALL=en_US.UTF-8
-
-# We need osmconvert both for planet/coasts and for getting the timestamp.
-[ ! -x "$OSMCTOOLS/osmconvert" ] && cc -x c -O3 "$OMIM_PATH/tools/osmctools/osmconvert.c" -o "$OSMCTOOLS/osmconvert" -lz
-
-if [ -r "$STATUS_FILE" ]; then
-  # Read all control variables from file
-  IFS=, read -r OPT_DESCRIPTIONS OPT_ROUTING OPT_UPDATE OPT_COAST OPT_WORLD NO_REGIONS MODE < "$STATUS_FILE"
-fi
-MFLAGS="$OPT_DESCRIPTIONS,$OPT_ROUTING,$OPT_UPDATE,$OPT_COAST,$OPT_WORLD,$NO_REGIONS,"
-if [ -z "${MODE:-}" ]; then
-  if [ -n "$OPT_COAST" -o -n "$OPT_UPDATE" ]; then
-    MODE=coast
-  else
-    MODE=inter
-  fi
-fi
-
-# The building process starts here
-
-# Download booking.com hotels. This takes around 3 hours, just like coastline processing.
-if [ ! -f "$BOOKING_FILE" -a -n "${BOOKING_USER-}" -a -n "${BOOKING_PASS-}" ]; then
-  log "STATUS" "Step S1: Starting background hotels downloading"
-  (
-      $PYTHON36 -m $BOOKING_MODULE --user $BOOKING_USER --password $BOOKING_PASS --output "$BOOKING_FILE" --logfile="$LOG_PATH"/booking.log || true
-      if [ -f "$BOOKING_FILE" -a "$(wc -l < "$BOOKING_FILE" || echo 0)" -gt 100 ]; then
-        echo "Hotels have been downloaded. Please ensure this line is before Step 4." >> "$PLANET_LOG"
-      else
-        if [ -n "${OLD_INTDIR-}" -a -f "${OLD_INTDIR-}/$(basename "$BOOKING_FILE")" ]; then
-          cp "$OLD_INTDIR/$(basename "$BOOKING_FILE")" "$INTDIR"
-          warn "Failed to download hotels! Using older hotels list."
-        else
-          warn "Failed to download hotels!"
-        fi
-        [ -n "${MAIL-}" ] && tail "$LOG_PATH/booking.log" | mailx -s "Failed to download hotels at $(hostname), please hurry to fix" "$MAIL"
-      fi
-  ) &
-fi
-
-# Download opentable.com restaurants. This takes around 30 minutes.
-if [ ! -f "$OPENTABLE_FILE" -a -n "${OPENTABLE_USER-}" -a -n "${OPENTABLE_PASS-}" ]; then
-  log "STATUS" "Step S2: Starting background restaurants downloading"
-  (
-      $PYTHON $OPENTABLE_SCRIPT --client $OPENTABLE_USER --secret $OPENTABLE_PASS --opentable_data "$INTDIR"/opentable.json --download --tsv "$OPENTABLE_FILE" 2>"$LOG_PATH"/opentable.log || true
-      if [ -f "$OPENTABLE_FILE" -a "$(wc -l < "$OPENTABLE_FILE" || echo 0)" -gt 100 ]; then
-        echo "Restaurants have been downloaded. Please ensure this line is before Step 4." >> "$PLANET_LOG"
-      else
-        if [ -n "${OLD_INTDIR-}" -a -f "${OLD_INTDIR-}/$(basename "$OPENTABLE_FILE")" ]; then
-          cp "$OLD_INTDIR/$(basename "$OPENTABLE_FILE")" "$INTDIR"
-          warn "Failed to download restaurants! Using older restaurants list."
-        else
-          warn "Failed to download restaurants!"
-        fi
-        [ -n "${MAIL-}" ] && tail "$LOG_PATH/opentable.log" | mailx -s "Failed to download restaurants at $(hostname), please hurry to fix" "$MAIL"
-      fi
-  ) &
-fi
-
-# Download UGC (user generated content) database.
-if [ ! -f "$UGC_FILE" -a -n "${UGC_DATABASE_URL-}" ]; then
-  putmode "Step UGC: Dowloading UGC database"
-  (
-    curl "$UGC_DATABASE_URL" --output "$UGC_FILE" --silent || true
-    if [ -f "$UGC_FILE" -a "$(wc -l < "$UGC_FILE" || echo 0)" -gt 100 ]; then
-      echo "UGC database have been downloaded. Please ensure this line is before Step 4." >> "$PLANET_LOG"
-    else
-      if [ -n "${OLD_INTDIR-}" -a -f "${OLD_INTDIR-}/$(basename "$UGC_FILE")" ]; then
-        cp "$OLD_INTDIR/$(basename "$UGC_FILE")" "$INTDIR"
-        warn "Failed to download UGC database! Using older UGC database."
-      else
-        warn "Failed to download UGC database!"
-      fi
-      [ -n "${MAIL-}" ] && tail "$LOG_PATH/ugc.log" | mailx -s "Failed to download UGC database at $(hostname), please hurry to fix" "$MAIL"
-    fi
-  ) &
-fi
-
-if [ "$MODE" == "coast" ]; then
-  putmode
-
-  [ ! -x "$OSMCTOOLS/osmupdate"  ] && cc -x c     "$OMIM_PATH/tools/osmctools/osmupdate.c"  -o "$OSMCTOOLS/osmupdate"
-  [ ! -x "$OSMCTOOLS/osmfilter"  ] && cc -x c -O3 "$OMIM_PATH/tools/osmctools/osmfilter.c"  -o "$OSMCTOOLS/osmfilter"
-  if [ -n "$OPT_DOWNLOAD" ]; then
-    # Planet download is requested
-    log "STATUS" "Step 0: Downloading and converting the planet"
-    PLANET_PBF="$(dirname "$PLANET")/planet-latest.osm.pbf"
-    curl -s -o "$PLANET_PBF" https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf
-    "$OSMCTOOLS/osmconvert" "$PLANET_PBF" --drop-author --drop-version --out-o5m "-o=$PLANET"
-    rm "$PLANET_PBF"
-  fi
-  [ ! -r "$PLANET" ] && fail "Planet file $PLANET is not found"
-  INTCOASTSDIR="$INTDIR/coasts"
-  mkdir -p "$INTCOASTSDIR"
-  COASTS_O5M="$INTCOASTSDIR/coastlines-latest.o5m"
-  TRY_AGAIN=1
-  while [ -n "$TRY_AGAIN" ]; do
-    TRY_AGAIN=
-    if [ -n "$OPT_UPDATE" ]; then
-      log "STATUS" "Step 1: Updating the planet file $PLANET"
-      PLANET_ABS="$(cd "$(dirname "$PLANET")"; pwd)/$(basename "$PLANET")"
-      [ -n "${OSC-}" ] && OSC_ABS="$(cd "$(dirname "${OSC-}")"; pwd)/$(basename "${OSC-}")"
-      (
-        cd "$OSMCTOOLS" # osmupdate requires osmconvert in a current directory
-        ./osmupdate --drop-author --drop-version --out-o5m -v "$PLANET_ABS" "$PLANET_ABS.new.o5m"
-        if [ -f "${OSC_ABS-}" ]; then
-          # Backup the planet first
-          mv "$PLANET_ABS" "$PLANET_ABS.backup"
-          log "WARNING" "Altering the planet file with $OSC, restore it from $PLANET_ABS.backup"
-          ./osmconvert "$PLANET_ABS.new.o5m" "$OSC_ABS" -o="$PLANET_ABS.merged.o5m"
-          mv -f "$PLANET_ABS.merged.o5m" "$PLANET_ABS.new.o5m"
-        fi
-      ) >> "$PLANET_LOG" 2>&1
-      mv "$PLANET.new.o5m" "$PLANET"
-    fi
-
-    if [ -n "$OPT_COAST" ]; then
-      log "STATUS" "Step 2: Creating and processing new coastline in $COASTS_O5M"
-      # Strip coastlines from the planet to speed up the process
-      "$OSMCTOOLS/osmfilter" "$PLANET" --keep= --keep-ways="natural=coastline" --keep-nodes="capital=yes place=town =city" "-o=$COASTS_O5M"
-      # Preprocess coastlines to separate intermediate directory
-      "$GENERATOR_TOOL" --intermediate_data_path="$INTCOASTSDIR/" --node_storage=map --osm_file_type=o5m --osm_file_name="$COASTS_O5M" \
-        -preprocess 2>> "$LOG_PATH/WorldCoasts.log"
-      # Generate temporary coastlines file in the coasts intermediate dir
-      if ! "$GENERATOR_TOOL" --intermediate_data_path="$INTCOASTSDIR/" --node_storage=map --osm_file_type=o5m --osm_file_name="$COASTS_O5M" \
-        --user_resource_path="$DATA_PATH/" -make_coasts -fail_on_coasts 2>&1 | tee -a "$LOG_PATH/WorldCoasts.log" | { grep -i 'not merged\|coastline polygons' || true; }
-      then
-        log "STATUS" "Coastline merge failed"
-        if [ -n "$OPT_UPDATE" ]; then
-          [ -n "${MAIL-}" ] && tail -n 50 "$LOG_PATH/WorldCoasts.log" | mailx -s "Coastline merge at $(hostname) failed, next try in $MERGE_INTERVAL minutes" "$MAIL"
-          echo "Will try fresh coasts again in $MERGE_INTERVAL minutes, or press a key..."
-          read -rs -n 1 -t $(($MERGE_INTERVAL * 60)) || true
-          TRY_AGAIN=1
-        else
-          fail
-        fi
-      fi
-    fi
-  done
-  # Make a working copy of generated coastlines file
-  if [ -n "$OPT_COAST" ]; then
-    cp "$INTCOASTSDIR"/WorldCoasts.*geom "$INTDIR"
-    cp "$INTCOASTSDIR"/*.csv "$INTDIR" || true
-  fi
-  [ -z "$KEEP_INTDIR" ] && rm -r "$INTCOASTSDIR"
-  # Exit if nothing else was requested
-  if [ -n "$OPT_ROUTING" -o -n "$OPT_WORLD" -o -z "$NO_REGIONS" ]; then
-    MODE=roads
-  else
-    log "STATUS" "Nothing but coastline temporary files were requested, finishing"
-    MODE=last
-  fi
-fi
-
-# This mode is started only after updating or processing a planet file
-if [ "$MODE" == "roads" ]; then
-  if [ -z "${OSRM_URL-}" ]; then
-    if [ -n "${OLD_INTDIR-}" -a -f "${OLD_INTDIR-}/ways.csv" ]; then
-      cp "$OLD_INTDIR/ways.csv" "$INTDIR"
-      warn "OSRM_URL variable is not set. Using older world roads file."
-    else
-      warn "OSRM_URL variable is not set. World roads will not be calculated."
-    fi
-  else
-    putmode "Step 2a: Generating road networks for the World map"
-    $PYTHON "$ROADS_SCRIPT" "$INTDIR" "$OSRM_URL" >>"$LOG_PATH"/road_runner.log
-  fi
-  MODE=inter
-fi
-
-if [ "$MODE" == "inter" ]; then
-  putmode "Step 3: Generating intermediate data for all MWMs"
-  # 1st pass, run in parallel - preprocess whole planet to speed up generation if all coastlines are correct
-  "$GENERATOR_TOOL" --intermediate_data_path="$INTDIR/" --node_storage=$NODE_STORAGE --osm_file_type=o5m --osm_file_name="$PLANET" \
-    -preprocess 2>> "$PLANET_LOG"
-  MODE=features
-fi
-
-# Wait until booking and restaurants lists are downloaded.
-wait
-
-if [ "$MODE" == "features" ]; then
-  putmode "Step 4: Generating features of everything into $TARGET"
-  # Checking for coastlines, can't build proper mwms without them
-  if [ ! -s "$INTDIR/WorldCoasts.geom" ]; then
-    COASTS="${COASTS-WorldCoasts.geom}"
-    if [ ! -s "$COASTS" -a -n "${OLD_INTDIR-}" -a -s "${OLD_INTDIR-}/WorldCoasts.geom" ]; then
-      log "Using older coastlines."
-      COASTS="$OLD_INTDIR/WorldCoasts.geom"
-    fi
-    if [ -s "$COASTS" ]; then
-      cp "$COASTS" "$INTDIR/WorldCoasts.geom"
-      RAWGEOM="${COASTS%.*}.rawgeom"
-      [ -s "$RAWGEOM" ] && cp "$RAWGEOM" "$INTDIR/WorldCoasts.rawgeom"
-    else
-      fail "Please prepare coastlines and put WorldCoasts.geom to $INTDIR"
-    fi
-  fi
-  [ -n "$OPT_WORLD" -a ! -s "$INTDIR/WorldCoasts.rawgeom" ] && fail "You need WorldCoasts.rawgeom in $INTDIR to build a world file"
-  # 2nd pass - paralleled in the code
-  PARAMS_SPLIT="-generate_features -emit_coasts"
-  [ -z "$NO_REGIONS" ] && PARAMS_SPLIT="$PARAMS_SPLIT -split_by_polygons"
-  [ -n "$OPT_WORLD" ] && PARAMS_SPLIT="$PARAMS_SPLIT -generate_world"
-  [ -n "$OPT_WORLD" -a "$NODE_STORAGE" == "map" ] && warn "generating world files with NODE_STORAGE=map may lead to an out of memory error. Try NODE_STORAGE=mem if it fails."
-  [ -f "$BOOKING_FILE" ] && PARAMS_SPLIT="$PARAMS_SPLIT --booking_data=$BOOKING_FILE"
-  [ -f "$OPENTABLE_FILE" ] && PARAMS_SPLIT="$PARAMS_SPLIT --opentable_data=$OPENTABLE_FILE"
-  [ -f "$POPULAR_PLACES_FILE" ] && PARAMS_SPLIT="$PARAMS_SPLIT --popular_places_data=$POPULAR_PLACES_FILE"
-  [ -n "$OPT_DESCRIPTIONS" ] && PARAMS_SPLIT="$PARAMS_SPLIT  --idToWikidata=$WIKIDATA_FILE"
-  [ -f "$BRANDS_TRANSLATIONS_DATA" ] && PARAMS_SPLIT="$PARAMS_SPLIT --brands_translations_data=$BRANDS_TRANSLATIONS_DATA"
-  [ -f "$BRANDS_DATA" ] && PARAMS_SPLIT="$PARAMS_SPLIT --brands_data=$BRANDS_DATA"
-
-  "$GENERATOR_TOOL" --intermediate_data_path="$INTDIR/" \
-                    --node_storage=$NODE_STORAGE \
-                    --osm_file_type=o5m \
-                    --osm_file_name="$PLANET" \
-                    --data_path="$TARGET" \
-                    --user_resource_path="$DATA_PATH/" \
-                    --dump_cities_boundaries \
-                    --cities_boundaries_data="$CITIES_BOUNDARIES_DATA" \
-                    $PARAMS_SPLIT 2>> "$PLANET_LOG"
-  MODE=mwm
-fi
-
-# Get version from the planet file
-if [ -z "${VERSION-}" ]; then
-  PLANET_VERSION="$("$OSMCTOOLS/osmconvert" --out-timestamp "$PLANET")"
-  if [[ $PLANET_VERSION == *nvalid* ]]; then
-    MWM_VERSION="$(date "+$MWM_VERSION_FORMAT")"
-    COUNTRIES_VERSION="$(date "+$COUNTRIES_VERSION_FORMAT")"
-  else
-    MWM_VERSION="$(format_version "$PLANET_VERSION" "$MWM_VERSION_FORMAT")"
-    COUNTRIES_VERSION="$(format_version "$PLANET_VERSION" "$COUNTRIES_VERSION_FORMAT")"
-  fi
-fi
-
-if [ "$MODE" == "mwm" ]; then
-  putmode "Step 5: Building all MWMs of regions and of the whole world into $TARGET"
-  # First, check for *.mwm.tmp
-  [ -z "$NO_REGIONS" -a -z "$(ls "$INTDIR/tmp" | grep '\.mwm\.tmp')" ] && fail "No .mwm.tmp files found."
-  # 3rd pass - do in parallel
-  # but separate exceptions for world files to finish them earlier
-  PARAMS="--data_path=$TARGET --intermediate_data_path=$INTDIR/ --user_resource_path=$DATA_PATH/ --node_storage=$NODE_STORAGE --planet_version=$MWM_VERSION -generate_geometry -generate_index"
-  if [ -n "$OPT_WORLD" ]; then
-    (
-      "$GENERATOR_TOOL" $PARAMS --output=World 2>> "$LOG_PATH/World.log"
-      "$GENERATOR_TOOL" --data_path="$TARGET" \
-                        --user_resource_path="$DATA_PATH/" \
-                        --generate_search_index \
-                        --generate_cities_boundaries \
-                        --cities_boundaries_data="$CITIES_BOUNDARIES_DATA" \
-                        --output=World 2>> "$LOG_PATH/World.log"
-    ) &
-    "$GENERATOR_TOOL" $PARAMS --output=WorldCoasts 2>> "$LOG_PATH/WorldCoasts.log" &
-  fi
-
-  if [ -z "$NO_REGIONS" ]; then
-    PARAMS_WITH_SEARCH="$PARAMS --generate_search_index --cities_boundaries_data=$CITIES_BOUNDARIES_DATA --make_city_roads --generate_maxspeed"
-    [ -n "${SRTM_PATH-}" -a -d "${SRTM_PATH-}" ] && PARAMS_WITH_SEARCH="$PARAMS_WITH_SEARCH --srtm_path=$SRTM_PATH"
-    [ -f "$UGC_FILE" ] && PARAMS_WITH_SEARCH="$PARAMS_WITH_SEARCH --ugc_data=$UGC_FILE"
-    [ -f "$POPULAR_PLACES_FILE" ] && PARAMS_WITH_SEARCH="$PARAMS_WITH_SEARCH --popular_places_data=$POPULAR_PLACES_FILE --generate_popular_places"
-    for file in "$INTDIR"/tmp/*.mwm.tmp; do
-      if [[ "$file" != *minsk-pass* && "$file" != *World* ]]; then
-        BASENAME="$(basename "$file" .mwm.tmp)"
-        "$GENERATOR_TOOL" $PARAMS_WITH_SEARCH --output="$BASENAME" 2>> "$LOG_PATH/$BASENAME.log" &
-        forky
-      fi
-    done
-    wait
-  fi
-
-  if [ -n "$OPT_ROUTING" -a -z "$NO_REGIONS" ]; then
-    MODE=routing
-  elif [ -n "$OPT_DESCRIPTIONS" -a -z "$NO_REGIONS" ]; then
-    MODE=descriptions
-  else
-    MODE=resources
-  fi
-fi
-
-if [ "$MODE" == "routing" ]; then
-  putmode "Step 6: Using freshly generated *.mwm to create routing files"
-  # If *.mwm.osm2ft were moved to INTDIR, let's put them back
-  [ -z "$(ls "$TARGET" | grep '\.mwm\.osm2ft')" -a -n "$(ls "$INTDIR" | grep '\.mwm\.osm2ft')" ] && mv "$INTDIR"/*.mwm.osm2ft "$TARGET"
-
-  for file in "$TARGET"/*.mwm; do
-    if [[ "$file" != *minsk-pass* && "$file" != *World* ]]; then
-      BASENAME="$(basename "$file" .mwm)"
-      (
-        "$GENERATOR_TOOL" --data_path="$TARGET" --intermediate_data_path="$INTDIR/" --user_resource_path="$DATA_PATH/" \
-        --make_cross_mwm --disable_cross_mwm_progress  ${GENERATE_CAMERA_SECTION-} --make_routing_index --generate_traffic_keys --output="$BASENAME" 2>> "$LOG_PATH/$BASENAME.log"
-        "$GENERATOR_TOOL" --data_path="$TARGET" --intermediate_data_path="$INTDIR/" --user_resource_path="$DATA_PATH/" \
-        --make_transit_cross_mwm --transit_path="$DATA_PATH" --output="$BASENAME" 2>> "$LOG_PATH/$BASENAME.log"
-        ) &
-      forky
-    fi
-  done
-  wait
-
-  if [ -n "$OPT_DESCRIPTIONS" -a -z "$NO_REGIONS" ]; then
-    MODE=descriptions
-  else
-    MODE=resources
-  fi
-fi
-
-if [ "$MODE" == "descriptions" ]; then
-  putmode "Step 7: Using freshly generated *.mwm to create descriptions files"
-
-  URLS_PATH="$INTDIR/wiki_urls.txt"
-  WIKI_PAGES_PATH="$INTDIR/descriptions"
-  LOG="$LOG_PATH/descriptions.log"
-  LANGS="en ru es"
-
-  "$GENERATOR_TOOL" --intermediate_data_path="$INTDIR/" --user_resource_path="$DATA_PATH/" \
-  --dump_wikipedia_urls="$URLS_PATH" --idToWikidata="$WIKIDATA_FILE" 2>> $LOG
-
-  PARAMS="--wikipedia $URLS_PATH --wikidata $WIKIDATA_FILE --output_dir $WIKI_PAGES_PATH"
-  [ -f "$POPULAR_PLACES_FILE" ] && PARAMS="$PARAMS --popularity=$POPULAR_PLACES_FILE"
-  $PYTHON36 -m $DESCRIPTIONS_MODULE $PARAMS --langs $LANGS 2>> $LOG
-
-  for file in "$TARGET"/*.mwm; do
-    if [[ "$file" != *minsk-pass* && "$file" != *World* ]]; then
-      BASENAME="$(basename "$file" .mwm)"
-      "$GENERATOR_TOOL" --wikipedia_pages="$WIKI_PAGES_PATH/" --idToWikidata="$WIKIDATA_FILE" \
-      --data_path="$TARGET" --user_resource_path="$DATA_PATH/" --output="$BASENAME" 2>> "$LOG_PATH/$BASENAME.log" &
-      forky
-    fi
-  done
-  wait
-  MODE=resources
-fi
-
-# Clean up temporary routing files
-[ -n "$(ls "$TARGET" | grep '\.mwm\.osm2ft')" ] && mv "$TARGET"/*.mwm.osm2ft "$INTDIR"
-
-if [ "$MODE" == "resources" ]; then
-  putmode "Step 8: Updating resource lists"
-  # Update countries list
-  $PYTHON36 -m $POST_GENERATION_MODULE hierarchy_to_countries --target "$TARGET" --hierarchy "$DATA_PATH/hierarchy.txt" --mwm_version "$COUNTRIES_VERSION" \
-    --old "$DATA_PATH/old_vs_new.csv" --osm "$DATA_PATH/borders_vs_osm.csv" --countries_synonyms "$DATA_PATH/countries_synonyms.csv" --output "$TARGET/countries.txt" >> "$PLANET_LOG" 2>&1
-
-  # A quick fix: chmodding to a+rw all generated files
-  for file in "$TARGET"/*.mwm*; do
-    chmod 0666 "$file"
-  done
-  chmod 0666 "$TARGET"/countries*.txt
-
-  # Move skipped nodes list from the intermediate directory
-  [ -e "$INTDIR/skipped_elements.lst" ] && mv "$INTDIR/skipped_elements.lst" "$TARGET/skipped_elements.lst"
-
-  if [ -n "$OPT_WORLD" ]; then
-    # Update external resources
-    [ -z "$(ls "$TARGET" | grep '\.ttf')" ] && cp "$DATA_PATH"/*.ttf "$TARGET"
-    cp "$DATA_PATH/WorldCoasts_obsolete.mwm" "$TARGET"
-    EXT_RES="$TARGET/external_resources.txt"
-    echo -n > "$EXT_RES"
-    UNAME="$(uname)"
-    for file in "$TARGET"/World*.mwm "$TARGET"/*.ttf; do
-      if [[ "$file" != *roboto_reg* ]]; then
-        if [ "$UNAME" == "Darwin" ]; then
-          stat -f "%N %z" "$file" | sed 's#^.*/##' >> "$EXT_RES"
-        else
-          stat -c "%n %s" "$file" | sed 's#^.*/##' >> "$EXT_RES"
-        fi
-      fi
-    done
-    chmod 0666 "$EXT_RES"
-  fi
-  MODE=test
-fi
-
-if [ -n "${LOCALADS-}" ]; then
-  putmode "Step AD: Generating CSV for local ads database"
-  (
-    LOCALADS_LOG="$LOG_PATH/localads.log"
-    LOCALADS_PATH="$INTDIR/localads"
-    mkdir -p "$LOCALADS_PATH"
-    $PYTHON36 -m "$POST_GENERATION_MODULE" localads_mwm_to_csv "$TARGET" --osm2ft "$INTDIR" --mwm_version "$COUNTRIES_VERSION" --types "$DATA_PATH/types.txt" --output "$LOCALADS_PATH" >> "$LOCALADS_LOG" 2>&1
-    LOCALADS_ARCHIVE="localads_$COUNTRIES_VERSION.tgz"
-    cd "$LOCALADS_PATH"
-    tar -czf "$LOCALADS_ARCHIVE" *.csv >> "$LOCALADS_LOG" 2>&1
-    AD_URL="$(curl -s --upload-file "$LOCALADS_ARCHIVE" "https://t.bk.ru/$(basename "$LOCALADS_ARCHIVE")")" >> "$LOCALADS_LOG" 2>&1
-    log STATUS "Uploaded localads file: $AD_URL"
-  ) &
-fi
-
-if [ "$MODE" == "test" -a -z "${SKIP_TESTS-}" ]; then
-  putmode "Step 9: Testing data"
-  bash "$TESTING_SCRIPT" "$TARGET" "${DELTA_WITH-}" > "$TEST_LOG"
-else
-  echo "Tests were skipped" > "$TEST_LOG"
-fi
-
-wait
-
-# Send both log files via e-mail
-if [ -n "${MAIL-}" ]; then
-  cat <(grep STATUS "$PLANET_LOG") <(echo ---------------) "$TEST_LOG" | mailx -s "Generate_planet: build completed at $(hostname)" "$MAIL"
-fi
-
-# Clean temporary indices
-if [ -n "$(ls "$TARGET" | grep '\.mwm$')" ]; then
-  for mwm in "$TARGET"/*.mwm; do
-    BASENAME="${mwm%.mwm}"
-    [ -d "$BASENAME" ] && rm -r "$BASENAME"
-  done
-fi
-# Cleaning up temporary directories
-rm "$STATUS_FILE"
-[ -z "$KEEP_INTDIR" ] && rm -r "$INTDIR"
-trap - SIGTERM ERR
-log "STATUS" "Done"