1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
|
#!/opt/local/bin/python
import scrapelib
import argparse
import json
import os.path
import sys
import time
import urllib2
argParser = argparse.ArgumentParser(description = 'Download Wikipedia for a given locale.')
argParser.add_argument("--locale", required = True)
argParser.add_argument("--minlat", type=float, default=-1000)
argParser.add_argument("--maxlat", type=float, default=1000)
argParser.add_argument("--minlon", type=float, default=-1000)
argParser.add_argument("--maxlon", type=float, default=1000)
ARGS = argParser.parse_args()
for i, line in enumerate(sys.stdin):
(itemId, lat, lon, itemType, title) = json.loads(line)
if lat < ARGS.minlat or lat > ARGS.maxlat or lon < ARGS.minlon or lon > ARGS.maxlon:
continue
if itemType == 'country' and (int(lat) == lat or int(lon) == lon):
sys.stderr.write('Ignoring country {0} {1} - probably parallel or meridian\n')
continue
fileName = urllib2.quote(title.encode("utf-8"), " ()") + ".html"
url = "http://{0}.wikipedia.org/w/index.php?curid={1}&useformat=mobile".format(ARGS.locale, itemId)
if title.find('_') != -1:
sys.stderr.write('WARNING! Title contains "_". It will not be found!\n')
scrapelib.ScrapeUrl(url, fileName, 1, i)
|
