diff options
| author | Maksim Andrianov <maksimandrianov1@gmail.com> | 2019-04-19 12:49:30 +0300 |
|---|---|---|
| committer | mpimenov <mpimenov@users.noreply.github.com> | 2019-04-19 13:05:15 +0300 |
| commit | 66b1805bab660f275a9129b37f2234e98bc83e6f (patch) | |
| tree | aae5f9ae59d98636a08e5789ac05ea5a99c77c9f /tools | |
| parent | 19437eebb53a5cd7a2eb41d2eada89b0d747d7c8 (diff) | |
[generator] Created descriptions module.
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/python/descriptions/__init__.py | 0 | ||||
| -rw-r--r-- | tools/python/descriptions/descriptions_downloader.py (renamed from tools/python/descriptions_downloader.py) | 19 | ||||
| -rw-r--r-- | tools/python/descriptions/exceptions.py | 10 | ||||
| -rw-r--r-- | tools/python/descriptions/requirements.txt | 5 | ||||
| -rwxr-xr-x | tools/unix/generate_planet.sh | 2 |
5 files changed, 19 insertions, 17 deletions
diff --git a/tools/python/descriptions/__init__.py b/tools/python/descriptions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/tools/python/descriptions/__init__.py diff --git a/tools/python/descriptions_downloader.py b/tools/python/descriptions/descriptions_downloader.py index d7391c731c..9471608066 100644 --- a/tools/python/descriptions_downloader.py +++ b/tools/python/descriptions/descriptions_downloader.py @@ -17,6 +17,8 @@ import wikipediaapi from bs4 import BeautifulSoup from wikidata.client import Client +from .exceptions import GettingError, ParseError + """ This script downloads Wikipedia pages for different languages. """ @@ -35,22 +37,6 @@ BAD_SECTIONS = { } -class MyException(Exception): - def __init__(self, value): - self.value = value - - def __str__(self): - return repr(self.value) - - -class ParseError(MyException): - pass - - -class GettingError(MyException): - pass - - def try_get(obj, prop, *args, **kwargs): attempts = REQUEST_ATTEMPTS while attempts != 0: @@ -333,5 +319,6 @@ def main(): else: log.warning(f"Wikidata ({wikidata_file}) file not set.") + if __name__ == "__main__": main() diff --git a/tools/python/descriptions/exceptions.py b/tools/python/descriptions/exceptions.py new file mode 100644 index 0000000000..0db47bee7f --- /dev/null +++ b/tools/python/descriptions/exceptions.py @@ -0,0 +1,10 @@ +class DescriptionError(Exception): + pass + + +class ParseError(DescriptionError): + pass + + +class GettingError(DescriptionError): + pass diff --git a/tools/python/descriptions/requirements.txt b/tools/python/descriptions/requirements.txt new file mode 100644 index 0000000000..4c5abf335f --- /dev/null +++ b/tools/python/descriptions/requirements.txt @@ -0,0 +1,5 @@ +htmlmin +requests +bs4 +wikidata +wikipedia-api
\ No newline at end of file diff --git a/tools/unix/generate_planet.sh b/tools/unix/generate_planet.sh index 4b61212543..5bfbd77680 100755 --- a/tools/unix/generate_planet.sh +++ b/tools/unix/generate_planet.sh @@ -226,7 +226,7 @@ else fi ROADS_SCRIPT="$PYTHON_SCRIPTS_PATH/road_runner.py" HIERARCHY_SCRIPT="$PYTHON_SCRIPTS_PATH/hierarchy_to_countries.py" -DESCRIPTIONS_DOWNLOADER="$PYTHON_SCRIPTS_PATH/descriptions_downloader.py" +DESCRIPTIONS_DOWNLOADER="$PYTHON_SCRIPTS_PATH/descriptions/descriptions_downloader.py" LOCALADS_SCRIPT="$PYTHON_SCRIPTS_PATH/local_ads/mwm_to_csv_4localads.py" UGC_FILE="${UGC_FILE:-$INTDIR/ugc_db.sqlite3}" POPULAR_PLACES_FILE="${POPULAR_PLACES_FILE:-$INTDIR/popular_places.csv}" |