diff options
author | tatiana-yan <tatiana.kondakova@gmail.com> | 2019-05-24 15:19:15 +0300 |
---|---|---|
committer | mpimenov <mpimenov@users.noreply.github.com> | 2019-05-27 17:08:42 +0300 |
commit | e08a9d988ea63fd625bdcdaac646bfac41060c43 (patch) | |
tree | 14cbf7d7df2e3f3f599294b1d1ff04f48ec6da6b /tools | |
parent | 3cd6936c9e04bc1a0a886fd4644cce3f0fb40eea (diff) |
[storage] Add optional country_name_synonyms parameter to countries.txt to find corresponding CountryId by country name without countries.txt keys modification.
Diffstat (limited to 'tools')
-rw-r--r-- | tools/python/maps_generator/generator/env.py | 4 | ||||
-rw-r--r-- | tools/python/maps_generator/maps_generator.py | 1 | ||||
-rw-r--r-- | tools/python/post_generation/__main__.py | 3 | ||||
-rwxr-xr-x | tools/python/post_generation/hierarchy_to_countries.py | 20 | ||||
-rwxr-xr-x | tools/unix/generate_planet.sh | 2 |
5 files changed, 28 insertions, 2 deletions
diff --git a/tools/python/maps_generator/generator/env.py b/tools/python/maps_generator/generator/env.py index 7459293c3d..a087a783eb 100644 --- a/tools/python/maps_generator/generator/env.py +++ b/tools/python/maps_generator/generator/env.py @@ -209,6 +209,10 @@ class Env: return os.path.join(self.user_resource_path, "borders_vs_osm.csv") @property + def countries_synonyms_path(self): + return os.path.join(self.user_resource_path, "countries_synonyms.csv") + + @property def counties_txt_path(self): return os.path.join(self.mwm_path, "countries.txt") diff --git a/tools/python/maps_generator/maps_generator.py b/tools/python/maps_generator/maps_generator.py index 5e4ef8c215..4018a56863 100644 --- a/tools/python/maps_generator/maps_generator.py +++ b/tools/python/maps_generator/maps_generator.py @@ -223,6 +223,7 @@ def stage_descriptions(env): def stage_countries_txt(env): countries = hierarchy_to_countries(env.old_to_new_path, env.borders_to_osm_path, + env.country_synonyms_path, env.hierarchy_path, env.mwm_path, env.mwm_version) with open(env.counties_txt_path, "w") as f: diff --git a/tools/python/post_generation/__main__.py b/tools/python/post_generation/__main__.py index c004c7deb9..deb526186c 100644 --- a/tools/python/post_generation/__main__.py +++ b/tools/python/post_generation/__main__.py @@ -66,12 +66,15 @@ The post_generation commands are: help="old_vs_new.csv file") parser.add_argument("--osm", required=True, help="borders_vs_osm.csv file") + parser.add_argument("--countries_synonyms", required=True, + help="countries_synonyms.csv file") parser.add_argument("--mwm_version", type=int, required=True, help="Mwm version") parser.add_argument("-o", "--output", required=True, help="Output countries.txt file (default is stdout)") args = parser.parse_args(sys.argv[2:]) countries_json = hierarchy_to_countries_(args.old, args.osm, + args.countries_synonyms, args.hierarchy, args.target, args.mwm_version) diff --git a/tools/python/post_generation/hierarchy_to_countries.py b/tools/python/post_generation/hierarchy_to_countries.py index e63f33cca2..b2c518e758 100755 --- a/tools/python/post_generation/hierarchy_to_countries.py +++ b/tools/python/post_generation/hierarchy_to_countries.py @@ -109,9 +109,24 @@ def parse_borders_vs_osm(borders_vs_osm_csv_path): vsosm[m.group(1)] = [m.group(3)] return vsosm +def parse_countries_synonyms(countries_synonyms_csv_path): + countries_synonyms = {} + if not countries_synonyms_csv_path: + return countries_synonyms + + with open(countries_synonyms_csv_path) as f: + for line in f: + m = re.match(r"(.+)\t(.+)", line.strip()) + assert m + if m.group(1) in countries_synonyms: + countries_synonyms[m.group(1)].append(m.group(2)) + else: + countries_synonyms[m.group(1)] = [m.group(2)] + return countries_synonyms def hierarchy_to_countries(old_vs_new_csv_path, borders_vs_osm_csv_path, - hierarchy_path, target_path, version): + countries_synonyms_csv_path, hierarchy_path, + target_path, version): def fill_last(last, stack): name = last["id"] @@ -124,6 +139,7 @@ def hierarchy_to_countries(old_vs_new_csv_path, borders_vs_osm_csv_path, oldvs = parse_old_vs_new(old_vs_new_csv_path) vsosm = parse_borders_vs_osm(borders_vs_osm_csv_path) + countries_synonyms = parse_countries_synonyms(countries_synonyms_csv_path) stack = [CountryDict(v=version, nameattr="Countries", g=[])] last = None with open(hierarchy_path) as f: @@ -151,6 +167,8 @@ def hierarchy_to_countries(old_vs_new_csv_path, borders_vs_osm_csv_path, last["old"] = oldvs[items[0]] if items[0] in vsosm: last["affiliations"] = vsosm[items[0]] + if items[0] in countries_synonyms: + last["country_name_synonyms"] = countries_synonyms[items[0]] # the last line is always a file del last["d"] diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index 1f5e867bf7..0f642deac9 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -636,7 +636,7 @@ if [ "$MODE" == "resources" ]; then putmode "Step 8: Updating resource lists" # Update countries list $PYTHON36 -m $POST_GENERATION_MODULE hierarchy_to_countries --target "$TARGET" --hierarchy "$DATA_PATH/hierarchy.txt" --mwm_version "$COUNTRIES_VERSION" \ - --old "$DATA_PATH/old_vs_new.csv" --osm "$DATA_PATH/borders_vs_osm.csv" --output "$TARGET/countries.txt" >> "$PLANET_LOG" 2>&1 + --old "$DATA_PATH/old_vs_new.csv" --osm "$DATA_PATH/borders_vs_osm.csv" --countries_synonyms "$DATA_PATH/countries_synonyms.csv" --output "$TARGET/countries.txt" >> "$PLANET_LOG" 2>&1 # A quick fix: chmodding to a+rw all generated files for file in "$TARGET"/*.mwm*; do |