1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
/**
* Fetch an HTTP url to a local file.
**/
var http = require("http")
, url = require("url")
, sys = require("sys")
, fs = require("./graceful-fs")
, get = require("./get")
, set = require("./set")
, log = require("./log")
, npm = require("../../npm")
, Buffer = require("buffer").Buffer
module.exports = fetch
function fetch (remote, local, headers, cb) {
if (!cb) cb = headers, headers = {}
log.info(remote, "fetch")
log.verbose(local, "fetch to")
headers.host = url.parse(remote).hostname
fs.open(local, process.O_CREAT | process.O_WRONLY | process.O_TRUNC, 0755,
function (er, fd) {
if (er) return cb(new Error(
"Failed to create "+local+": "+er.message), fd)
fetchAndWrite(remote, fd, headers, cb)
}
)
}
function fetchAndWrite (remote, fd, headers, maxRedirects, redirects, cb) {
if (!cb) cb = redirects, redirects = 0
if (!cb) cb = maxRedirects, maxRedirects = 10
if (!cb) throw new Error("No callback provided")
remote = url.parse(remote)
set(headers, "host", remote.hostname)
remote.path = remote.pathname+(remote.search||"")+(remote.hash||"")
var proxyConfig = npm.config.get("proxy")
, proxy = (proxyConfig) ? url.parse(proxyConfig) : null
, https = (proxy || remote).protocol === "https:"
, port = (proxy || remote).port || (https ? 443 : 80)
, hostname = (proxy || remote).hostname
, path = proxy ? remote.href : (remote.pathname||"/")+(remote.search||"")+(remote.hash||"")
http
.createClient(port, hostname, https)
.request( "GET", path, headers)
.on("response", function (response) {
// handle redirects.
var loc = get(response.headers, "location")
if (Math.floor(response.statusCode / 100) === 3
&& loc && loc !== remote.href && redirects < maxRedirects) {
// This is a laughably naïve way to handle this situation.
// @TODO: Really need a full curl or wget style module that would
// do all this kind of stuff for us.
var cookie = get(response.headers, "Set-Cookie")
if (cookie) {
cookie = (cookie + "").split(";").shift()
set(headers, "Cookie", cookie)
}
log.verbose(response.statusCode+" "+loc, "fetch")
return fetchAndWrite(loc, fd, headers, maxRedirects, redirects + 1, cb)
}
if (response.statusCode !== 200) {
return cb(new Error(response.statusCode + " " +
(sys.inspect(response.headers).replace(/\s+/, ' '))))
}
var written = 0
, len = response.headers["content-length"]
, pct = 0
, lastPct = 0
log.verbose(len || "unknown", "content-length")
response.on("data", function (chunk) {
// write the chunk...
written += chunk.length
if (len) {
pct = written / len * 100
if (lastPct === 0 || (pct - lastPct > 10)) {
log.verbose(Math.round(pct)+"%", "downloaded")
lastPct = pct
}
}
try {
fs.writeSync(fd, chunk, 0, chunk.length, null)
} catch (er) {
return cb(er)
}
})
response.on("error", cb)
response.on("end", function () {
log.verbose(remote.href+" - "+written+" bytes", "fetched")
fs.close(fd, cb)
})
})
.end()
}
if (module !== process.mainModule) return
log("testing", "fetch")
var exec = require("./exec")
, urls =
[ "http://registry.npmjs.org/npm/-/npm-0.1.16.tgz"
, "http://registry.npmjs.org/less/-/less-1.0.5.tgz"
]
, path = require("path")
, assert = require("assert")
urls.forEach(function (url) {
var fetchFile = path.basename(url, ".tgz")+"-fetch.tgz"
, wgetFile = path.basename(url, ".tgz")+"-wget.tgz"
fetch(url, fetchFile, function (e) {
if (e) { log.error(e, fetchFile) ; throw e }
exec( "wget"
, ["-O", wgetFile, url]
, function (er, code, stdout, stderr) {
if (e) { log.error(e, wgetFile) ; throw e }
exec("md5", ["-q", wgetFile], function (er, _, wghash, __) {
if (e) { log.error(e, "md5 "+wgetFile) ; throw e }
exec("md5", ["-q", fetchFile], function (er, _, fhash, __) {
if (e) { log.error(e, "md5 "+fetchFile) ; throw e }
assert.equal(wghash, fhash, fetchFile + " == " + wgetFile)
})
})
}
)
})
})
|
