1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
|
var mkdir = require("mkdirp")
, assert = require("assert")
, log = require("npmlog")
, path = require("path")
, sha = require("sha")
, retry = require("retry")
, createWriteStream = require("fs-write-stream-atomic")
, npm = require("../npm.js")
, inflight = require("inflight")
, addLocalTarball = require("./add-local-tarball.js")
, cacheFile = require("npm-cache-filename")
, rimraf = require("rimraf")
, pulseTillDone = require("../utils/pulse-till-done.js")
module.exports = addRemoteTarball
function addRemoteTarball (u, pkgData, shasum, auth, cb_) {
assert(typeof u === "string", "must have module URL")
assert(typeof cb_ === "function", "must have callback")
function cb (er, data) {
if (data) {
data._from = u
data._resolved = u
data._shasum = data._shasum || shasum
}
cb_(er, data)
}
cb_ = inflight(u, cb_)
if (!cb_) return log.verbose("addRemoteTarball", u, "already in flight; waiting")
log.verbose("addRemoteTarball", u, "not in flight; adding")
// XXX Fetch direct to cache location, store tarballs under
// ${cache}/registry.npmjs.org/pkg/-/pkg-1.2.3.tgz
var tmp = cacheFile(npm.tmp, u)
function next (er, resp, shasum) {
if (er) return cb(er)
addLocalTarball(tmp, pkgData, shasum, cleanup)
}
function cleanup (er, data) {
if (er) return cb(er)
rimraf(tmp, function () {
cb(er, data)
})
}
log.verbose("addRemoteTarball", [u, shasum])
mkdir(path.dirname(tmp), function (er) {
if (er) return cb(er)
addRemoteTarball_(u, tmp, shasum, auth, next)
})
}
function addRemoteTarball_ (u, tmp, shasum, auth, cb) {
// Tuned to spread 3 attempts over about a minute.
// See formula at <https://github.com/tim-kos/node-retry>.
var operation = retry.operation({
retries: npm.config.get("fetch-retries")
, factor: npm.config.get("fetch-retry-factor")
, minTimeout: npm.config.get("fetch-retry-mintimeout")
, maxTimeout: npm.config.get("fetch-retry-maxtimeout")
})
operation.attempt(function (currentAttempt) {
log.info("retry", "fetch attempt " + currentAttempt
+ " at " + (new Date()).toLocaleTimeString())
fetchAndShaCheck(u, tmp, shasum, auth, function (er, response, shasum) {
// Only retry on 408, 5xx or no `response`.
var sc = response && response.statusCode
var statusRetry = !sc || (sc === 408 || sc >= 500)
if (er && statusRetry && operation.retry(er)) {
log.warn("retry", "will retry, error on last attempt: " + er)
return
}
cb(er, response, shasum)
})
})
}
function fetchAndShaCheck (u, tmp, shasum, auth, cb) {
cb = pulseTillDone("fetchTarball", cb)
npm.registry.fetch(u, { auth : auth }, function (er, response) {
if (er) {
log.error("fetch failed", u)
return cb(er, response)
}
var tarball = createWriteStream(tmp, { mode : npm.modes.file })
tarball.on("error", function (er) {
cb(er)
tarball.destroy()
})
tarball.on("finish", function () {
if (!shasum) {
// Well, we weren't given a shasum, so at least sha what we have
// in case we want to compare it to something else later
return sha.get(tmp, function (er, shasum) {
log.silly("fetchAndShaCheck", "shasum", shasum)
cb(er, response, shasum)
})
}
// validate that the url we just downloaded matches the expected shasum.
log.silly("fetchAndShaCheck", "shasum", shasum)
sha.check(tmp, shasum, function (er) {
if (er && er.message) {
// add original filename for better debuggability
er.message = er.message + "\n" + "From: " + u
}
return cb(er, response, shasum)
})
})
response.pipe(tarball)
})
}
|