Add scripts to geocode wikitravel.

3 files changed, 84 insertions, 0 deletions

diff --git a/crawler/wikitravel-crawler.sh b/crawler/wikitravel-crawler.sh
index c5dbeca43e..d6e8406bd3 100755
--- a/crawler/wikitravel-crawler.sh
+++ b/crawler/wikitravel-crawler.sh
@@ -18,6 +18,10 @@ wc -l wikitravel-pages.json
 
 cat wikitravel-pages.json | python $MY_PATH/wikitravel-download-pages.py
 
+cat wikitravel-pages.json | python $MY_PATH/wikitravel-geocode-yahoo.py
+
+cat wikitravel-pages.json | python $MY_PATH/wikitravel-geocode-google.py
+
 cat wikitravel-pages.json | python $MY_PATH/wikitravel-process-articles.py
 
 cat wikitravel-pages.json | python $MY_PATH/wikitravel-optimize-articles.py
diff --git a/crawler/wikitravel-geocode-google.py b/crawler/wikitravel-geocode-google.py
new file mode 100755
index 0000000000..079c4a0ae5
--- /dev/null
+++ b/crawler/wikitravel-geocode-google.py
@@ -0,0 +1,40 @@
+#!/opt/local/bin/python
+import json
+import os.path
+import sys
+import time
+import urllib2
+
+for i, line in enumerate(sys.stdin):
+  (_, title, fileName) = json.loads(line)
+
+  url = "http://maps.googleapis.com/maps/api/geocode/json?sensor=false&address=" + urllib2.quote(title.encode("utf-8"), "")
+  fileName = fileName + ".google_geocoded"
+
+  if os.path.exists(fileName):
+    sys.stderr.write('Skipping existing {0} {1}\n'.format(i, fileName))
+  else:
+    sys.stderr.write('Downloading {0} {1}\n'.format(i, fileName))
+
+    tryCount = 0
+    while True:
+      try:
+        tryCount = tryCount + 1
+        remoteFile = urllib2.urlopen(url)
+        try:
+          data = remoteFile.read();
+        finally:
+          remoteFile.close()
+        break
+      except IOError as error:
+        sys.stderr.write('Try {0}, error: {1}\n'.format(tryCount, error))
+        if tryCount >= 5:
+          raise
+        else:
+          time.sleep(2)
+
+    localFile = open(fileName, 'w')
+    localFile.write(data)
+    localFile.close()
+
+    time.sleep(36)
diff --git a/crawler/wikitravel-geocode-yahoo.py b/crawler/wikitravel-geocode-yahoo.py
new file mode 100755
index 0000000000..a060e0562d
--- /dev/null
+++ b/crawler/wikitravel-geocode-yahoo.py
@@ -0,0 +1,40 @@
+#!/opt/local/bin/python
+import json
+import os.path
+import sys
+import time
+import urllib2
+
+for i, line in enumerate(sys.stdin):
+  (_, title, fileName) = json.loads(line)
+
+  url = "http://where.yahooapis.com/geocode?flags=GRTXJ&q=" + urllib2.quote(title.encode("utf-8"), "")
+  fileName = fileName + ".yahoo_geocoded"
+
+  if os.path.exists(fileName):
+    sys.stderr.write('Skipping existing {0} {1}\n'.format(i, fileName))
+  else:
+    sys.stderr.write('Downloading {0} {1}\n'.format(i, fileName))
+
+    tryCount = 0
+    while True:
+      try:
+        tryCount = tryCount + 1
+        remoteFile = urllib2.urlopen(url)
+        try:
+          data = remoteFile.read();
+        finally:
+          remoteFile.close()
+        break
+      except IOError as error:
+        sys.stderr.write('Try {0}, error: {1}\n'.format(tryCount, error))
+        if tryCount >= 5:
+          raise
+        else:
+          time.sleep(2)
+
+    localFile = open(fileName, 'w')
+    localFile.write(data)
+    localFile.close()
+
+    time.sleep(1)