diff options
author | Arsentiy Milchakov <milcars@mapswithme.com> | 2017-06-15 18:01:10 +0300 |
---|---|---|
committer | Ilya Zverev <ilya@zverev.info> | 2017-06-28 19:41:05 +0300 |
commit | 2174fd1800da931b31a676966be70f692c3d6dad (patch) | |
tree | d4521191f894b79d15ebad61b2193572e35d94f2 /tools | |
parent | f14b8d06174ea6bc54409e39f465944bd2d57f97 (diff) |
[generator] viator
Diffstat (limited to 'tools')
-rw-r--r-- | tools/python/viator_cities.py | 72 | ||||
-rwxr-xr-x | tools/unix/generate_planet.sh | 22 |
2 files changed, 94 insertions, 0 deletions
diff --git a/tools/python/viator_cities.py b/tools/python/viator_cities.py new file mode 100644 index 0000000000..36c4423b11 --- /dev/null +++ b/tools/python/viator_cities.py @@ -0,0 +1,72 @@ +from __future__ import print_function + +import argparse +import json +import logging +import os +import urllib2 + +logging.basicConfig(level=logging.DEBUG, format='[%(asctime)s] %(levelname)s: %(message)s') + + +class ViatorApi(object): + def __init__(self, apikey): + self.apikey = apikey + + def get_locations(self): + url = 'http://viatorapi.viator.com/service/taxonomy/locations?apiKey=' + self.apikey + request = urllib2.Request(url) + stream = urllib2.urlopen(request) + payload = stream.read() + locations = json.loads(payload) + return locations + + +def check_errors(locations): + if not locations['success']: + raise Exception('Viator error, error codes:{} error text:{}' + .format(locations['errorCodes'], locations['errorMessageText'])) + + +def save_cities(locations, output_file_name): + with open(output_file_name, 'w') as output_file: + for l in locations['data']: + if l['destinationType'] == 'CITY' and l['destinationId'] and \ + l['destinationName'] and l['latitude'] and l['longitude']: + city = '\t'.join([ + str(l['destinationId']), + l['destinationName'], + str(l['latitude']), + str(l['longitude']) + ]) + print(city.encode('utf-8'), file=output_file) + + +def run(options): + try: + api = ViatorApi(options.apikey) + locations = api.get_locations() + check_errors(locations) + save_cities(locations, options.output) + except Exception as e: + logging.exception(e) + + +def process_options(): + parser = argparse.ArgumentParser(description='Download and process viator cities.') + + parser.add_argument('--apikey', dest='apikey', help='Viator apikey', required=True) + parser.add_argument('--output', dest='output', help='Destination file', required=True) + + options = parser.parse_args() + + return options + + +def main(): + options = process_options() + run(options) + + +if __name__ == '__main__': + main() diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index 4d0211b95a..debdb2e4c0 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -181,6 +181,8 @@ BOOKING_SCRIPT="$PYTHON_SCRIPTS_PATH/booking_hotels.py" BOOKING_FILE="${BOOKING_FILE:-$INTDIR/hotels.csv}" OPENTABLE_SCRIPT="$PYTHON_SCRIPTS_PATH/opentable_restaurants.py" OPENTABLE_FILE="${OPENTABLE_FILE:-$INTDIR/restaurants.csv}" +VIATOR_SCRIPT="$PYTHON_SCRIPTS_PATH/viator_cities.py" +VIATOR_FILE="${VIATOR_FILE:-$INTDIR/viator.csv}" TESTING_SCRIPT="$SCRIPTS_PATH/test_planet.sh" PYTHON="$(which python2.7)" MWM_VERSION_FORMAT="%s" @@ -295,6 +297,25 @@ if [ ! -f "$OPENTABLE_FILE" -a -n "${OPENTABLE_USER-}" -a -n "${OPENTABLE_PASS-} ) & fi +# Download viator.com cities. This takes around 3 seconds. +if [ ! -f "$VIATOR_FILE" -a -n "${VIATOR_KEY-}" ]; then + log "STATUS" "Step S3: Starting background viator cities downloading" + ( + $PYTHON $VIATOR_SCRIPT --apikey $VIATOR_KEY --output "$VIATOR_FILE" 2>"$LOG_PATH"/viator.log || true + if [ -f "$VIATOR_FILE" -a "$(wc -l < "$VIATOR_FILE" || echo 0)" -gt 100 ]; then + echo "Viator cities have been downloaded. Please ensure this line is before Step 4." >> "$PLANET_LOG" + else + if [ -n "${OLD_INTDIR-}" -a -f "${OLD_INTDIR-}/$(basename "$VIATOR_FILE")" ]; then + cp "$OLD_INTDIR/$(basename "$VIATOR_FILE")" "$INTDIR" + warn "Failed to download viator cities! Using older viator cities list." + else + warn "Failed to download viator cities!" + fi + [ -n "${MAIL-}" ] && tail "$LOG_PATH/viator.log" | mailx -s "Failed to download viator cities at $(hostname), please hurry to fix" "$MAIL" + fi + ) & +fi + if [ "$MODE" == "coast" ]; then putmode @@ -419,6 +440,7 @@ if [ "$MODE" == "features" ]; then [ -n "$OPT_WORLD" -a "$NODE_STORAGE" == "map" ] && warn "generating world files with NODE_STORAGE=map may lead to an out of memory error. Try NODE_STORAGE=mem if it fails." [ -f "$BOOKING_FILE" ] && PARAMS_SPLIT="$PARAMS_SPLIT --booking_data=$BOOKING_FILE" [ -f "$OPENTABLE_FILE" ] && PARAMS_SPLIT="$PARAMS_SPLIT --opentable_data=$OPENTABLE_FILE" + [ -f "$VIATOR_FILE" ] && PARAMS_SPLIT="$PARAMS_SPLIT --viator_data=$VIATOR_FILE" "$GENERATOR_TOOL" --intermediate_data_path="$INTDIR/" --node_storage=$NODE_STORAGE --osm_file_type=o5m --osm_file_name="$PLANET" \ --data_path="$TARGET" --user_resource_path="$DATA_PATH/" $PARAMS_SPLIT 2>> "$PLANET_LOG" MODE=mwm |