diff options
| author | Yury Melnichek <melnichek@gmail.com> | 2012-09-17 14:10:34 +0400 |
|---|---|---|
| committer | Alex Zolotarev <alex@maps.me> | 2015-09-23 01:43:33 +0300 |
| commit | c97750449bb772135dacf5bafeec28f05d9a8a45 (patch) | |
| tree | 1043f2d1d3f3e9dcdc5ec90fb3f4ba53ff268bef /crawler | |
| parent | 626981b8f28d3f297dafc84c0fc76205165eda5d (diff) | |
[crawler] Catch all exceptions when downloading pages. Decrease delay between downloads to 0.3s.
Diffstat (limited to 'crawler')
| -rwxr-xr-x | crawler/wikitravel-download-pages.py | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/crawler/wikitravel-download-pages.py b/crawler/wikitravel-download-pages.py index 9634ae594e..7a658bc6d6 100755 --- a/crawler/wikitravel-download-pages.py +++ b/crawler/wikitravel-download-pages.py @@ -25,7 +25,7 @@ for i, line in enumerate(sys.stdin): finally: remoteFile.close() break - except IOError as error: + except Exception as error: sys.stderr.write('Try {0}, error: {1}\n'.format(tryCount, error)) if tryCount >= 5: raise @@ -36,4 +36,4 @@ for i, line in enumerate(sys.stdin): localFile.write(data) localFile.close() - time.sleep(1) + time.sleep(0.3) |