diff options
author | Maksim Andrianov <maksimandrianov1@gmail.com> | 2019-05-23 18:53:47 +0300 |
---|---|---|
committer | gmoryes <gmoryes@gmail.com> | 2019-05-23 20:09:54 +0300 |
commit | 5a8b6509b279153664703f6e01c9cee9f315b637 (patch) | |
tree | a5b84726182af7bdda698fc2af89bf6e82d3d0fb /tools | |
parent | 6431847809f13a4c4142dee7cd4792870b862a3a (diff) |
[python] Added stage_localads.
Diffstat (limited to 'tools')
-rw-r--r-- | tools/python/maps_generator/__main__.py | 5 | ||||
-rw-r--r-- | tools/python/maps_generator/generator/env.py | 15 | ||||
-rw-r--r-- | tools/python/maps_generator/maps_generator.py | 14 | ||||
-rw-r--r-- | tools/python/maps_generator/utils/file.py | 6 |
4 files changed, 33 insertions, 7 deletions
diff --git a/tools/python/maps_generator/__main__.py b/tools/python/maps_generator/__main__.py index 6053b7135d..ead17107d2 100644 --- a/tools/python/maps_generator/__main__.py +++ b/tools/python/maps_generator/__main__.py @@ -8,7 +8,7 @@ from .generator.exceptions import ContinueError, SkipError, ValidationError from .maps_generator import (generate_maps, generate_coasts, reset_to_stage, ALL_STAGES, stage_download_production_external, stage_descriptions, stage_ugc, stage_popularity, - stages_as_string) + stage_localads, stages_as_string) from .utils.collections import unique logger = logging.getLogger("maps_generator") @@ -133,7 +133,8 @@ def main(): if not options["production"]: options["skip"] += stages_as_string(stage_download_production_external, stage_ugc, stage_popularity, - stage_descriptions) + stage_descriptions, + stage_localads) if not all(s in ALL_STAGES for s in options["skip"]): raise SkipError(f"Stages {set(options['skip']) - set(ALL_STAGES)} " f"not found.") diff --git a/tools/python/maps_generator/generator/env.py b/tools/python/maps_generator/generator/env.py index cf2b0cf1d1..7459293c3d 100644 --- a/tools/python/maps_generator/generator/env.py +++ b/tools/python/maps_generator/generator/env.py @@ -142,6 +142,16 @@ class Env: return stage_name not in self._skipped_stages @property + def localads_path(self): + path = os.path.join(self.out_path, f"localads_{self.mwm_version}") + self._create_if_not_exist(path) + return path + + @property + def types_path(self): + return os.path.join(self.user_resource_path, "types.txt") + + @property def external_resources_path(self): return os.path.join(self.mwm_path, "external_resources.txt") @@ -226,9 +236,8 @@ class Env: @staticmethod def _logging_setup(): def exception_handler(type, value, tb): - logger.exception( - f"Uncaught exception: {str(value)}", - exc_info=(type, value, tb)) + logger.exception(f"Uncaught exception: {str(value)}", + exc_info=(type, value, tb)) logging.config.dictConfig(settings.LOGGING) sys.excepthook = exception_handler diff --git a/tools/python/maps_generator/maps_generator.py b/tools/python/maps_generator/maps_generator.py index fd3f7574c2..f24d0cdac2 100644 --- a/tools/python/maps_generator/maps_generator.py +++ b/tools/python/maps_generator/maps_generator.py @@ -3,12 +3,14 @@ import os import shutil from functools import partial from multiprocessing.pool import ThreadPool +import multiprocessing from descriptions.descriptions_downloader import (check_and_get_checker, download_from_wikipedia_tags, download_from_wikidata_tags) from filelock import FileLock from post_generation.hierarchy_to_countries import hierarchy_to_countries +from post_generation.localads_mwm_to_csv import create_csv from .generator import basic_stages from .generator import coastline @@ -20,7 +22,7 @@ from .generator.env import (planet_lock_file, build_lock_file, from .generator.exceptions import (ContinueError, BadExitStatusError, wait_and_raise_if_fail) from .generator.gen_tool import run_gen_tool -from .utils.file import is_verified, download_file +from .utils.file import is_verified, download_file, make_tarfile logger = logging.getLogger("maps_generator") @@ -250,6 +252,13 @@ def stage_external_resources(env): @stage +def stage_localads(env): + create_csv(env.localads_path, env.mwm_path, env.mwm_path, env.types_path, + env.mwm_version, multiprocessing.cpu_count()) + make_tarfile(f"{env.localads_path}.tar.gz", env.localads_path) + + +@stage def stage_cleanup(env): osm2ft_path = os.path.join(env.out_path, "osm2ft") os.makedirs(osm2ft_path, exist_ok=True) @@ -273,7 +282,7 @@ STAGES = [s.__name__ for s in stage_download_and_convert_planet, stage_update_planet, stage_coastline, stage_preprocess, stage_features, stage_mwm, stage_descriptions, stage_countries_txt, stage_external_resources, - stage_cleanup)] + stage_localads, stage_cleanup)] ALL_STAGES = STAGES + COUNTRIES_STAGES @@ -332,6 +341,7 @@ def generate_maps(env): stage_descriptions(env) stage_countries_txt(env) stage_external_resources(env) + stage_localads(env) stage_cleanup(env) diff --git a/tools/python/maps_generator/utils/file.py b/tools/python/maps_generator/utils/file.py index ee79eac43d..fe7575af20 100644 --- a/tools/python/maps_generator/utils/file.py +++ b/tools/python/maps_generator/utils/file.py @@ -4,6 +4,7 @@ import glob import os import shutil import subprocess +import tarfile from .md5 import md5, check_md5 @@ -55,3 +56,8 @@ def symlink_force(target, link_name): os.symlink(target, link_name) else: raise e + + +def make_tarfile(output_filename, source_dir): + with tarfile.open(output_filename, "w:gz") as tar: + tar.add(source_dir, arcname=os.path.basename(source_dir)) |