blob: 7a658bc6d66189099172d78c053c2d526367d27d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
|
#!/opt/local/bin/python
import json
import os.path
import sys
import time
import urllib2
for i, line in enumerate(sys.stdin):
(url, title, fileName) = json.loads(line)
if os.path.exists(fileName):
sys.stderr.write('Skipping existing {0} {1}\n'.format(i, fileName))
else:
sys.stderr.write('Downloading {0} {1}\n'.format(i, fileName))
if title.find('_') != -1:
sys.stderr.write('WARNING! Title contains "_". It will not be found!\n')
tryCount = 0
while True:
try:
tryCount = tryCount + 1
remoteFile = urllib2.urlopen(url)
try:
data = remoteFile.read();
finally:
remoteFile.close()
break
except Exception as error:
sys.stderr.write('Try {0}, error: {1}\n'.format(tryCount, error))
if tryCount >= 5:
raise
else:
time.sleep(120)
localFile = open(fileName, 'w')
localFile.write(data)
localFile.close()
time.sleep(0.3)
|
