diff options
author | Maksim Andrianov <maksimandrianov1@gmail.com> | 2019-04-19 18:20:19 +0300 |
---|---|---|
committer | Maksim Andrianov <maksimandrianov1@gmail.com> | 2019-05-06 18:42:27 +0300 |
commit | 15cec734e2f5ca73fae3627233044cbd0f98ceef (patch) | |
tree | 391d472c0c3def7adb7236939b5526316dd58bbb /tools | |
parent | bc56c4166e795954f9a145c8265e187a4b484c6e (diff) |
[python] Refactoring: removed unused options and code, move to python3, code divided into functions, added executable module.
Diffstat (limited to 'tools')
-rw-r--r-- | tools/python/post_generation/__main__.py | 86 | ||||
-rwxr-xr-x | tools/python/post_generation/hierarchy_to_countries.py | 241 | ||||
-rwxr-xr-x | tools/python/post_generation/localads_mwm_to_csv.py | 106 | ||||
-rwxr-xr-x | tools/unix/generate_planet.sh | 6 |
4 files changed, 213 insertions, 226 deletions
diff --git a/tools/python/post_generation/__main__.py b/tools/python/post_generation/__main__.py new file mode 100644 index 0000000000..6f7d4cf111 --- /dev/null +++ b/tools/python/post_generation/__main__.py @@ -0,0 +1,86 @@ +import argparse +import os +import sys + +from .hierarchy_to_countries import hierarchy_to_countries as hierarchy_to_countries_ +from .localads_mwm_to_csv import create_csv + + +class PostGeneration: + def __init__(self): + parser = argparse.ArgumentParser( + description="Post generation instruments", + usage="""post_generation <command> [<args>] +The most commonly used post_generation commands are: + localads_mwm_to_csv Prepares CSV files for uploading to localads database from mwm files. + hierarchy_to_countries Produces countries.txt from hierarchy.txt. + """) + parser.add_argument("command", help="Subcommand to run") + args = parser.parse_args(sys.argv[1:2]) + if not hasattr(self, args.command): + print(f"Unrecognized command {args.command}") + parser.print_help() + exit(1) + getattr(self, args.command)() + + @staticmethod + def localads_mwm_to_csv(): + parser = argparse.ArgumentParser( + description="Prepares CSV files for uploading to localads database " + "from mwm files.") + parser.add_argument("mwm", help="path to mwm files") + parser.add_argument( + "--osm2ft", + help="path to osm2ft files (default is the same as mwm)") + parser.add_argument("--output", + default=".", + help="path to generated files ('.' by default)") + types_default = os.path.join(os.path.dirname(__file__), "..", "..", + "..", "data", "types.txt") + parser.add_argument("--types", + default=types_default, + help="path to omim/data/types.txt") + parser.add_argument("--threads", + type=int, + help="number of threads to process files") + parser.add_argument("--version", type=int, help="override mwm version") + parser.add_argument("--debug", + action="store_true", + help="debug parse_mwm call") + args = parser.parse_args(sys.argv[2:]) + if not args.osm2ft: + args.osm2ft = args.mwm + + create_csv(args.output, args.mwm, args.osm2ft, args.types, + args.version, args.threads, args.debug) + + @staticmethod + def hierarchy_to_countries(): + parser = argparse.ArgumentParser( + description="Produces countries.txt from hierarchy.txt.") + parser.add_argument("--target", required=True, + help="Path to mwm files") + parser.add_argument("--hierarchy", required=True, + default="hierarchy.txt", + help="Hierarchy file") + parser.add_argument("--old", required=True, + help="old_vs_new.csv file") + parser.add_argument("--osm", required=True, + help="borders_vs_osm.csv file") + parser.add_argument("--version", type=int, default=151231, + help="Version") + parser.add_argument("-o", "--output", required=True, + help="Output countries.txt file (default is stdout)") + args = parser.parse_args(sys.argv[2:]) + countries_json = hierarchy_to_countries_(args.old, args.osm, + args.hierarchy, + args.target, + args.version) + if args.output: + with open(args.output, "w") as f: + f.write(countries_json) + else: + print(countries_json) + + +PostGeneration() diff --git a/tools/python/post_generation/hierarchy_to_countries.py b/tools/python/post_generation/hierarchy_to_countries.py index 0e9958669b..66dddbb92b 100755 --- a/tools/python/post_generation/hierarchy_to_countries.py +++ b/tools/python/post_generation/hierarchy_to_countries.py @@ -1,12 +1,10 @@ -#!/usr/bin/python -# -# Produces countries.txt (either legacy or new format) from hierarchy.txt +# Produces countries.txt from hierarchy.txt # # Hierarchy.txt format: # # Sample lines: # Iran;Q794;ir;fa -# Iran_South;Q794-South +# Iran_South;Q794-South # # Number of leading spaces mean hierarchy depth. In above case, Iran_South is inside Iran. # Then follows a semicolon-separated list: @@ -14,21 +12,17 @@ # 2. Region name template using wikidata Qxxx codes and predefined strings # 3. Country ISO code (used for flags in the legacy format) # 4. Comma-separated list of language ISO codes for the region -# import base64 -import codecs import hashlib import json import os.path import re -import sys -from optparse import OptionParser class CountryDict(dict): def __init__(self, *args, **kwargs): dict.__init__(self, *args, **kwargs) - self.order = ['id', 'n', 'f', 'v', 'c', 's', 'sha1_base64', 'rs', 'g'] + self.order = ["id", "n", "f", "v", "c", "s", "sha1_base64", "rs", "g"] def __iter__(self): for key in self.order: @@ -44,183 +38,124 @@ class CountryDict(dict): def get_hash(path, name): - if path == '0': - return '' - filename = os.path.join(path, '{0}.mwm'.format(name)) + filename = os.path.join(path, f"{name}.mwm") h = hashlib.sha1() - with open(filename, 'rb') as f: + with open(filename, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): h.update(chunk) - return base64.b64encode(h.digest()) + return str(base64.b64encode(h.digest()), "utf-8") def get_size(path, name): - if path == '0': - return 0 - filename = os.path.join(path, '{0}.mwm'.format(name)) - try: - return os.path.getsize(filename) - except OSError: - sys.stderr.write('Missing file: {0}\n'.format(filename)) - return -1 + filename = os.path.join(path, f"{name}.mwm") + return os.path.getsize(filename) def collapse_single(root): - for i in range(len(root['g'])): - if 'g' in root['g'][i]: - if len(root['g'][i]['g']) == 1: + for i in range(len(root["g"])): + if "g" in root["g"][i]: + if len(root["g"][i]["g"]) == 1: # replace group by a leaf - if 'c' in root['g'][i]: - root['g'][i]['g'][0]['c'] = root['g'][i]['c'] - root['g'][i] = root['g'][i]['g'][0] + if "c" in root["g"][i]: + root["g"][i]["g"][0]["c"] = root["g"][i]["c"] + root["g"][i] = root["g"][i]["g"][0] else: - collapse_single(root['g'][i]) + collapse_single(root["g"][i]) def get_name(leaf): - if 'n' in leaf: - return leaf['n'].lower() + if "n" in leaf: + return leaf["n"].lower() else: - return leaf['id'].lower() + return leaf["id"].lower() def sort_tree(root): - root['g'].sort(key=get_name) - for leaf in root['g']: - if 'g' in leaf: + root["g"].sort(key=get_name) + for leaf in root["g"]: + if "g" in leaf: sort_tree(leaf) -parser = OptionParser(add_help_option=False) -parser.add_option('-t', '--target', help='Path to mwm files') -parser.add_option('-h', '--hierarchy', default='hierarchy.txt', - help='Hierarchy file') -parser.add_option('--old', help='old_vs_new.csv file') -parser.add_option('--osm', help='borders_vs_osm.csv file') -parser.add_option('-v', '--version', type='int', default=151231, help='Version') -parser.add_option('-o', '--output', - help='Output countries.txt file (default is stdout)') -parser.add_option('-m', '--help', action='store_true', help='Display this help') -parser.add_option('--flag', action='store_true', - help='Add flags ("c") to countries') -parser.add_option('--lang', action='store_true', - help='Add languages ("lang") to countries') -parser.add_option('-l', '--legacy', action='store_true', - help='Produce a legacy format file') -parser.add_option('-n', '--names', - help='Translations for file names (for legacy format)') -parser.add_option('-s', '--sort', action='store_true', - help='Sort leaves by name (useful for legacy)') -(options, args) = parser.parse_args() - -if options.help: - parser.print_help() - sys.exit(0) - -if not os.path.isfile(options.hierarchy): - parser.error('Hierarchy file is required.') - -oldvs = {} -if options.old: - with open(options.old, 'r') as f: +def parse_old_vs_new(old_vs_new_csv_path): + oldvs = {} + if not old_vs_new_csv_path: + return oldvs + + with open(old_vs_new_csv_path, "r") as f: for line in f: - m = re.match(r'(.+?)\t(.+)', line.strip()) + m = re.match(r"(.+?)\t(.+)", line.strip()) if m: if m.group(2) in oldvs: oldvs[m.group(2)].append(m.group(1)) else: oldvs[m.group(2)] = [m.group(1)] + return oldvs + -vsosm = {} -if options.osm: - with codecs.open(options.osm, 'r', 'utf-8') as f: +def parse_borders_vs_osm(borders_vs_osm_csv_path): + vsosm = {} + if not borders_vs_osm_csv_path: + return vsosm + + with open(borders_vs_osm_csv_path, "r") as f: for line in f: - m = re.match(r'^(.+?)\t(\d)\t(.+?)$', line.strip()) + m = re.match(r"^(.+?)\t(\d)\t(.+?)$", line.strip()) if m: if m.group(1) in vsosm: vsosm[m.group(1)].append(m.group(3)) else: vsosm[m.group(1)] = [m.group(3)] + return vsosm + -names = {} -if options.names: - with codecs.open(options.names, 'r', 'utf-8') as f: +def hierarchy_to_countries(old_vs_new_csv_path, borders_vs_osm_csv_path, + hierarchy_path, target_path, version): + oldvs = parse_old_vs_new(old_vs_new_csv_path) + vsosm = parse_borders_vs_osm(borders_vs_osm_csv_path) + stack = [CountryDict(v=version, nameattr="Countries", g=[])] + last = None + with open(hierarchy_path, "r") as f: for line in f: - pair = [x.strip() for x in line.split('=', 1)] - if len(pair) == 2 and pair[0] != pair[1]: - try: - names[pair[0]] = pair[1] - except (KeyError, IndexError): - sys.stderr.write( - 'Could not read translation for {0}\n'.format(pair[0])) - -nameattr = 'n' if options.legacy else 'id' -mwmpath = '0' if not options.target else options.target -stack = [CountryDict( - {"v": options.version, nameattr: "World" if options.legacy else "Countries", - "g": []})] -last = None -with open(options.hierarchy, 'r') as f: - for line in f: - m = re.match('( *)(.+?)\n', line) - if m: - depth = len(m.group(1)) - if last is not None: - lastd = last['d'] - del last['d'] - if lastd < depth: - # last is a group - last['g'] = [] - if options.legacy and 'f' in last: - del last['f'] - stack.append(last) - else: - name = last['f' if 'f' in last else nameattr] - last['s'] = get_size(mwmpath, name) - last['sha1_base64'] = get_hash(mwmpath, name) - if options.legacy: - last['rs'] = 0 - if last['s'] >= 0: - stack[-1]['g'].append(last) - while depth < len(stack) - 1: - # group ended, add it to higher group - g = stack.pop() - if len(g['g']) > 0: - stack[-1]['g'].append(g) - items = m.group(2).split(';') - last = CountryDict({nameattr: items[0], "d": depth}) - if not options.legacy and items[0] in oldvs: - last['old'] = oldvs[items[0]] - if not options.legacy and items[0] in vsosm: - last['affiliations'] = vsosm[items[0]] - if (options.legacy or options.flag) and len(items) > 2 and len( - items[2]) > 0: - last['c'] = items[2] - if options.lang and len(items) > 3 and len(items[3]) > 0: - last['lang'] = items[3].split(',') - if options.legacy and items[0] in names: - last['f'] = last[nameattr] - last[nameattr] = names[items[0]] - -# the last line is always a file -del last['d'] -name = last['f' if 'f' in last else nameattr] -last['s'] = get_size(mwmpath, name) -last['sha1_base64'] = get_hash(mwmpath, name) -if options.legacy: - last['rs'] = 0 -if last['s'] >= 0: - stack[-1]['g'].append(last) -while len(stack) > 1: - g = stack.pop() - if len(g['g']) > 0: - stack[-1]['g'].append(g) - -collapse_single(stack[-1]) -if options.sort: - sort_tree(stack[-1]) -if options.output: - with codecs.open(options.output, 'w', 'utf-8') as f: - json.dump(stack[-1], f, ensure_ascii=True, indent=1) -else: - print(json.dumps(stack[-1], ensure_ascii=True, indent=1)) + m = re.match("( *)(.+?)\n", line) + if m: + depth = len(m.group(1)) + if last is not None: + lastd = last["d"] + del last["d"] + if lastd < depth: + # last is a group + last["g"] = [] + stack.append(last) + else: + name = last["f" if "f" in last else "id"] + last["s"] = get_size(target_path, name) + last["sha1_base64"] = get_hash(target_path, name) + if last["s"] >= 0: + stack[-1]["g"].append(last) + while depth < len(stack) - 1: + # group ended, add it to higher group + g = stack.pop() + if len(g["g"]) > 0: + stack[-1]["g"].append(g) + items = m.group(2).split(";") + last = CountryDict({"id": items[0], "d": depth}) + if items[0] in oldvs: + last["old"] = oldvs[items[0]] + if items[0] in vsosm: + last["affiliations"] = vsosm[items[0]] + + # the last line is always a file + del last["d"] + name = last["f" if "f" in last else "id"] + last["s"] = get_size(target_path, name) + last["sha1_base64"] = get_hash(target_path, name) + if last["s"] >= 0: + stack[-1]["g"].append(last) + while len(stack) > 1: + g = stack.pop() + if len(g["g"]) > 0: + stack[-1]["g"].append(g) + + collapse_single(stack[-1]) + return json.dumps(stack[-1], ensure_ascii=True, indent=1) diff --git a/tools/python/post_generation/localads_mwm_to_csv.py b/tools/python/post_generation/localads_mwm_to_csv.py index 8ab19ab215..caa0b77d7a 100755 --- a/tools/python/post_generation/localads_mwm_to_csv.py +++ b/tools/python/post_generation/localads_mwm_to_csv.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python2.7 -import argparse import csv import ctypes import logging @@ -8,12 +6,12 @@ import sys from multiprocessing import Pool, Queue, Process from zlib import adler32 -from ..mwm import mwm +from mwm import mwm HEADERS = { - 'mapping': 'osmid fid mwm_id mwm_version source_type'.split(), - 'sponsored': 'sid fid mwm_id mwm_version source_type'.split(), - 'mwm': 'mwm_id name mwm_version'.split(), + "mapping": "osmid fid mwm_id mwm_version source_type".split(), + "sponsored": "sid fid mwm_id mwm_version source_type".split(), + "mwm": "mwm_id name mwm_version".split(), } QUEUES = {name: Queue() for name in HEADERS} GOOD_TYPES = ("amenity", "shop", "tourism", "leisure", "sport", @@ -22,7 +20,7 @@ GOOD_TYPES = ("amenity", "shop", "tourism", "leisure", "sport", "natural-spring", "natural-cave_entrance", "waterway-waterfall", "place-island", "railway-station", "railway-halt", "aerialway-station", "building-train_station") -SOURCE_TYPES = {'osm': 0, 'booking': 1} +SOURCE_TYPES = {"osm": 0, "booking": 1} # Big enough to never intersect with a feature id (there are below 3 mln usually). FAKE_FEATURE_ID = 100111000 @@ -35,47 +33,45 @@ def generate_id_from_name_and_version(name, version): def parse_mwm(mwm_name, osm2ft_name, override_version, types_name): region_name = os.path.splitext(os.path.basename(mwm_name))[0] logging.info(region_name) - with open(osm2ft_name, 'rb') as f: + with open(osm2ft_name, "rb") as f: ft2osm = mwm.read_osm2ft(f, ft2osm=True, tuples=False) - with open(mwm_name, 'rb') as f: + with open(mwm_name, "rb") as f: mwm_file = mwm.MWM(f) - version = override_version or mwm_file.read_version()['version'] + version = override_version or mwm_file.read_version()["version"] mwm_id = generate_id_from_name_and_version(region_name, version) - QUEUES['mwm'].put((mwm_id, region_name, version)) + QUEUES["mwm"].put((mwm_id, region_name, version)) mwm_file.read_header() mwm_file.read_types(types_name) for feature in mwm_file.iter_features(metadata=True): - osm_id = ft2osm.get(feature['id'], None) + osm_id = ft2osm.get(feature["id"], None) if osm_id is None: - if 'metadata' in feature and 'ref:sponsored' in feature[ - 'metadata']: - for t in feature['header']['types']: - if t.startswith('sponsored-'): - QUEUES['sponsored'].put( - (feature['metadata']['ref:sponsored'], - feature['id'], - mwm_id, - version, - SOURCE_TYPES[t[t.find('-') + 1:]])) + if "metadata" in feature and "ref:sponsored" in feature["metadata"]: + for t in feature["header"]["types"]: + if t.startswith("sponsored-"): + QUEUES["sponsored"].put((feature["metadata"]["ref:sponsored"], + feature["id"], + mwm_id, + version, + SOURCE_TYPES[t[t.find("-") + 1:]])) break else: - for t in feature['header']['types']: + for t in feature["header"]["types"]: if t.startswith(GOOD_TYPES): - QUEUES['mapping'].put((ctypes.c_long(osm_id).value, - feature['id'], + QUEUES["mapping"].put((ctypes.c_long(osm_id).value, + feature["id"], mwm_id, version, - SOURCE_TYPES['osm'])) + SOURCE_TYPES["osm"])) break - QUEUES['mapping'].put((ctypes.c_long(FAKE_FEATURE_ID).value, + QUEUES["mapping"].put((ctypes.c_long(FAKE_FEATURE_ID).value, FAKE_FEATURE_ID, mwm_id, version, - SOURCE_TYPES['osm'])) + SOURCE_TYPES["osm"])) def write_csv(output_dir, qtype): - with open(os.path.join(output_dir, qtype + '.csv'), 'w') as f: + with open(os.path.join(output_dir, qtype + ".csv"), "w") as f: mapping = QUEUES[qtype].get() w = csv.writer(f) w.writerow(HEADERS[qtype]) @@ -84,50 +80,24 @@ def write_csv(output_dir, qtype): mapping = QUEUES[qtype].get() -def main(): - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s', - datefmt='%H:%M:%S') - parser = argparse.ArgumentParser( - description='Prepares CSV files for uploading to localads database from mwm files.') - parser.add_argument('mwm', help='path to mwm files') - parser.add_argument('--osm2ft', - help='path to osm2ft files (default is the same as mwm)') - parser.add_argument('--output', default='.', - help='path to generated files ("." by default)') - types_default = os.path.join(os.path.dirname(sys.argv[0]), - '..', '..', '..', 'data', 'types.txt') - parser.add_argument('--types', default=types_default, - help='path to omim/data/types.txt') - parser.add_argument('--threads', type=int, - help='number of threads to process files') - parser.add_argument('--version', type=int, help='override mwm version') - parser.add_argument('--debug', action='store_true', - help='debug parse_mwm call') - args = parser.parse_args() - if not args.osm2ft: - args.osm2ft = args.mwm - - if not os.path.isdir(args.output): - os.mkdir(args.output) +def create_csv(output, mwm_path, osm2ft_path, types, version, threads, debug=False): + if not os.path.isdir(output): + os.mkdir(output) # Create CSV writer processes for each queue and a pool of MWM readers. - writers = [Process(target=write_csv, args=(args.output, qtype)) for qtype in - QUEUES] + writers = [Process(target=write_csv, args=(output, qtype)) for qtype in QUEUES] for w in writers: w.start() - pool = Pool(processes=args.threads) - for mwm_name in os.listdir(args.mwm): - if 'World' in mwm_name or 'minsk_pass' in mwm_name or not mwm_name.endswith( - '.mwm'): + pool = Pool(processes=threads) + for mwm_name in os.listdir(mwm_path): + if "World" in mwm_name or "minsk_pass" in mwm_name or not mwm_name.endswith(".mwm"): continue - osm2ft_name = os.path.join(args.osm2ft, - os.path.basename(mwm_name) + '.osm2ft') + osm2ft_name = os.path.join(osm2ft_path, os.path.basename(mwm_name) + ".osm2ft") if not os.path.exists(osm2ft_name): - logging.error('Cannot find %s', osm2ft_name) + logging.error("Cannot find %s", osm2ft_name) sys.exit(2) - parse_mwm_args = ( - os.path.join(args.mwm, mwm_name), osm2ft_name, args.version, args.types) - if args.debug: + parse_mwm_args = (os.path.join(mwm_path, mwm_name), osm2ft_name, version, types) + if debug: parse_mwm(*parse_mwm_args) else: pool.apply_async(parse_mwm, parse_mwm_args) @@ -137,7 +107,3 @@ def main(): queue.put(None) for w in writers: w.join() - - -if __name__ == '__main__': - main() diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index cb743a03c2..0d388b98de 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -230,7 +230,7 @@ export PYTHONPATH ROADS_SCRIPT="$PYTHON_SCRIPTS_PATH/road_runner.py" HIERARCHY_SCRIPT="$PYTHON_SCRIPTS_PATH/post_generation/hierarchy_to_countries.py" DESCRIPTIONS_MODULE="descriptions" -LOCALADS_SCRIPT="$PYTHON_SCRIPTS_PATH/post_generation/localads_mwm_to_csv.py" +POST_GENERATION_MODULE="post_generation" UGC_FILE="${UGC_FILE:-$INTDIR/ugc_db.sqlite3}" POPULAR_PLACES_FILE="${POPULAR_PLACES_FILE:-$INTDIR/popular_places.csv}" WIKIDATA_FILE="${WIKIDATA_FILE:-$INTDIR/idToWikidata.csv}" @@ -635,7 +635,7 @@ fi if [ "$MODE" == "resources" ]; then putmode "Step 8: Updating resource lists" # Update countries list - $PYTHON $HIERARCHY_SCRIPT --target "$TARGET" --hierarchy "$DATA_PATH/hierarchy.txt" --version "$COUNTRIES_VERSION" \ + $PYTHON36 -m $POST_GENERATION_MODULE hierarchy_to_countries --target "$TARGET" --hierarchy "$DATA_PATH/hierarchy.txt" --version "$COUNTRIES_VERSION" \ --old "$DATA_PATH/old_vs_new.csv" --osm "$DATA_PATH/borders_vs_osm.csv" --output "$TARGET/countries.txt" >> "$PLANET_LOG" 2>&1 # A quick fix: chmodding to a+rw all generated files @@ -674,7 +674,7 @@ if [ -n "${LOCALADS-}" ]; then LOCALADS_LOG="$LOG_PATH/localads.log" LOCALADS_PATH="$INTDIR/localads" mkdir -p "$LOCALADS_PATH" - $PYTHON "$LOCALADS_SCRIPT" "$TARGET" --osm2ft "$INTDIR" --version "$COUNTRIES_VERSION" --types "$DATA_PATH/types.txt" --output "$LOCALADS_PATH" >> "$LOCALADS_LOG" 2>&1 + $PYTHON36 -m "$POST_GENERATION_MODULE" localads_mwm_to_csv "$TARGET" --osm2ft "$INTDIR" --version "$COUNTRIES_VERSION" --types "$DATA_PATH/types.txt" --output "$LOCALADS_PATH" >> "$LOCALADS_LOG" 2>&1 LOCALADS_ARCHIVE="localads_$COUNTRIES_VERSION.tgz" cd "$LOCALADS_PATH" tar -czf "$LOCALADS_ARCHIVE" *.csv >> "$LOCALADS_LOG" 2>&1 |