diff options
author | Maksim Andrianov <maksimandrianov1@gmail.com> | 2019-05-21 16:27:21 +0300 |
---|---|---|
committer | gmoryes <gmoryes@gmail.com> | 2019-05-21 19:20:07 +0300 |
commit | ca4426daef29e6cd3478f908df2039df3c9f6904 (patch) | |
tree | e22a88385c46709816422326c610059df16b732e /tools | |
parent | e78943ca12e55fbfb2ec03efce4feb53f5c20496 (diff) |
[python] Supported download coasts files.
Diffstat (limited to 'tools')
-rw-r--r-- | tools/python/maps_generator/README.md | 4 | ||||
-rw-r--r-- | tools/python/maps_generator/__main__.py | 14 | ||||
-rw-r--r-- | tools/python/maps_generator/generator/coastline.py | 8 | ||||
-rw-r--r-- | tools/python/maps_generator/generator/settings.py | 4 | ||||
-rw-r--r-- | tools/python/maps_generator/maps_generator.py | 32 | ||||
-rw-r--r-- | tools/python/maps_generator/var/etc/map_generator.ini.default | 1 |
6 files changed, 46 insertions, 17 deletions
diff --git a/tools/python/maps_generator/README.md b/tools/python/maps_generator/README.md index ee165d0a41..88595c775e 100644 --- a/tools/python/maps_generator/README.md +++ b/tools/python/maps_generator/README.md @@ -63,7 +63,8 @@ OSM_TOOLS_PATH: ~/osmctools # PLANET_URL: # The url to the file with md5 sum of the planet. # PLANET_MD5_URL: - +# The url to WorldCoasts.geom and WorldCoasts.rawgeom(without file name). +# PLANET_COASTS_URL: # The url to the subway file. # SUBWAY_URL: @@ -116,6 +117,7 @@ optional arguments: preprocess, features, mwm, descriptions, countries_txt, cleanup, index, ugc, popularity, routing, routing_transit. + --only_coasts Build WorldCoasts.raw and WorldCoasts.rawgeom files --production Build production maps. In another case, 'osm only maps' are built - maps without additional data and advertising. diff --git a/tools/python/maps_generator/__main__.py b/tools/python/maps_generator/__main__.py index 7b7fe83722..a7161f5c44 100644 --- a/tools/python/maps_generator/__main__.py +++ b/tools/python/maps_generator/__main__.py @@ -5,8 +5,8 @@ from argparse import ArgumentParser from .generator import settings from .generator.env import Env, find_last_build_dir, WORLDS_NAMES from .generator.exceptions import ContinueError, SkipError, ValidationError -from .maps_generator import (start, reset_to_stage, ALL_STAGES, - stage_download_production_external, +from .maps_generator import (generate_maps, generate_coasts, reset_to_stage, + ALL_STAGES, stage_download_production_external, stage_descriptions, stage_ugc, stage_popularity, stages_as_string) from .utils.collections import unique @@ -49,6 +49,11 @@ def parse_options(): help=f"Stage from which maps will be rebuild. Available stages: " f"{', '.join([s.replace('stage_', '') for s in ALL_STAGES])}.") parser.add_argument( + "--only_coasts", + default=False, + action="store_true", + help="Build WorldCoasts.raw and WorldCoasts.rawgeom files") + parser.add_argument( "--production", default=False, action="store_true", @@ -136,7 +141,10 @@ def main(): env = Env(options) if env.from_stage: reset_to_stage(env.from_stage, env) - start(env) + if env.only_coasts: + generate_coasts(env) + else: + generate_maps(env) env.finish() diff --git a/tools/python/maps_generator/generator/coastline.py b/tools/python/maps_generator/generator/coastline.py index 7c341a90cd..9508b096f6 100644 --- a/tools/python/maps_generator/generator/coastline.py +++ b/tools/python/maps_generator/generator/coastline.py @@ -38,11 +38,3 @@ def make_coastline(env): user_resource_path=env.user_resource_path, make_coasts=True, fail_on_coasts=True) - - prefix = "WorldCoasts" - coastline_files = [] - for f in os.listdir(env.coastline_path): - path = os.path.join(env.coastline_path, f) - if os.path.isfile(path) and f.startswith(prefix): - coastline_files.append(path) - return coastline_files diff --git a/tools/python/maps_generator/generator/settings.py b/tools/python/maps_generator/generator/settings.py index 1d809f450d..9d671b6bbf 100644 --- a/tools/python/maps_generator/generator/settings.py +++ b/tools/python/maps_generator/generator/settings.py @@ -42,6 +42,7 @@ VERSION_FILE_NAME = "version.txt" # External resources PLANET_URL = "https://planet.openstreetmap.org/pbf/planet-latest.osm.pbf" PLANET_MD5_URL = PLANET_URL + ".md5" +PLANET_COASTS_URL = "" UGC_URL = "" HOTELS_URL = "" POPULARITY_URL= "" @@ -110,6 +111,7 @@ os.makedirs(os.path.dirname(os.path.abspath(LOG_FILE_PATH)), exist_ok=True) PLANET_URL = _get_opt_path(config, "External", "PLANET_URL", PLANET_URL) PLANET_MD5_URL = _get_opt_path(config, "External", "PLANET_MD5_URL", PLANET_MD5_URL) +PLANET_COASTS_URL = _get_opt_path(config, "External", "PLANET_COASTS_URL", PLANET_COASTS_URL) UGC_URL = _get_opt_path(config, "External", "UGC_URL", UGC_URL) HOTELS_URL = _get_opt_path(config, "External", "HOTELS_URL", HOTELS_URL) POPULARITY_URL = _get_opt_path(config, "External", "POPULARITY_URL", POPULARITY_URL) @@ -120,6 +122,8 @@ FOOD_TRANSLATIONS_URL = _get_opt(config, "External", "FOOD_TRANSLATIONS_URL", PLANET_O5M = os.path.join(MAIN_OUT_PATH, PLANET + ".o5m") PLANET_PBF = os.path.join(MAIN_OUT_PATH, PLANET + ".osm.pbf") +PLANET_COASTS_GEOM_URL = os.path.join(PLANET_COASTS_URL, "latest.geom") +PLANET_COASTS_RAWGEOM_URL = os.path.join(PLANET_COASTS_URL, "latest.rawgeom") if DEBUG: PLANET_URL = "http://osmz.ru/mwm/islands/islands.o5m" diff --git a/tools/python/maps_generator/maps_generator.py b/tools/python/maps_generator/maps_generator.py index a2d37da15d..d38ea41f44 100644 --- a/tools/python/maps_generator/maps_generator.py +++ b/tools/python/maps_generator/maps_generator.py @@ -17,7 +17,7 @@ from .generator import settings from .generator.decorators import stage, country_stage, country_stage_log from .generator.env import (planet_lock_file, build_lock_file, WORLD_COASTS_NAME, WORLD_NAME, WORLDS_NAMES) -from .generator.exceptions import (ContinueError, +from .generator.exceptions import (ContinueError, BadExitStatusError, wait_and_raise_if_fail) from .generator.gen_tool import run_gen_tool from .utils.file import is_verified, download_file @@ -103,9 +103,22 @@ def stage_features(env): @stage def stage_coastline(env): - coastline_files = coastline.make_coastline(env) - for file in coastline_files: - shutil.copy2(file, env.intermediate_path) + coasts_geom = "WorldCoasts.geom" + coasts_rawgeom = "WorldCoasts.rawgeom" + try: + coastline.make_coastline(env) + except BadExitStatusError: + logger.info("Build costs failed. Try to download the costs...") + download_external({ + settings.PLANET_COASTS_GEOM_URL: + os.path.join(env.coastline_path, coasts_geom), + settings.PLANET_COASTS_RAWGEOM_URL: + os.path.join(env.coastline_path, coasts_rawgeom) + }) + + for f in [coasts_geom, coasts_rawgeom]: + path = os.path.join(env.coastline_path, f) + shutil.copy2(path, env.intermediate_path) @country_stage @@ -281,7 +294,7 @@ def reset_to_stage(stage_name, env): f.write(main_status) -def start(env): +def generate_maps(env): stage_download_external(env) stage_download_production_external(env) with FileLock(planet_lock_file(), timeout=1) as planet_lock: @@ -296,3 +309,12 @@ def start(env): stage_descriptions(env) stage_countries_txt(env) stage_cleanup(env) + + +def generate_coasts(env): + with FileLock(planet_lock_file(), timeout=1) as planet_lock: + stage_update_planet(env) + with FileLock(build_lock_file(env.out_path), timeout=1): + stage_coastline(env) + planet_lock.release() + stage_cleanup(env) diff --git a/tools/python/maps_generator/var/etc/map_generator.ini.default b/tools/python/maps_generator/var/etc/map_generator.ini.default index cabd1c9644..2fc56d6361 100644 --- a/tools/python/maps_generator/var/etc/map_generator.ini.default +++ b/tools/python/maps_generator/var/etc/map_generator.ini.default @@ -26,6 +26,7 @@ OSM_TOOLS_PATH: ~/osmctools [External] # PLANET_URL: # PLANET_MD5_URL: +# PLANET_COASTS_URL: # UGC_URL: # HOTELS_URL: # POPULARITY_URL: |