var mkdir = require("mkdirp") , assert = require("assert") , log = require("npmlog") , path = require("path") , sha = require("sha") , retry = require("retry") , createWriteStream = require("fs-write-stream-atomic") , npm = require("../npm.js") , inflight = require("inflight") , addLocalTarball = require("./add-local-tarball.js") , cacheFile = require("npm-cache-filename") , rimraf = require("rimraf") , pulseTillDone = require("../utils/pulse-till-done.js") module.exports = addRemoteTarball function addRemoteTarball (u, pkgData, shasum, auth, cb_) { assert(typeof u === "string", "must have module URL") assert(typeof cb_ === "function", "must have callback") function cb (er, data) { if (data) { data._from = u data._resolved = u data._shasum = data._shasum || shasum } cb_(er, data) } cb_ = inflight(u, cb_) if (!cb_) return log.verbose("addRemoteTarball", u, "already in flight; waiting") log.verbose("addRemoteTarball", u, "not in flight; adding") // XXX Fetch direct to cache location, store tarballs under // ${cache}/registry.npmjs.org/pkg/-/pkg-1.2.3.tgz var tmp = cacheFile(npm.tmp, u) function next (er, resp, shasum) { if (er) return cb(er) addLocalTarball(tmp, pkgData, shasum, cleanup) } function cleanup (er, data) { if (er) return cb(er) rimraf(tmp, function () { cb(er, data) }) } log.verbose("addRemoteTarball", [u, shasum]) mkdir(path.dirname(tmp), function (er) { if (er) return cb(er) addRemoteTarball_(u, tmp, shasum, auth, next) }) } function addRemoteTarball_ (u, tmp, shasum, auth, cb) { // Tuned to spread 3 attempts over about a minute. // See formula at . var operation = retry.operation({ retries: npm.config.get("fetch-retries") , factor: npm.config.get("fetch-retry-factor") , minTimeout: npm.config.get("fetch-retry-mintimeout") , maxTimeout: npm.config.get("fetch-retry-maxtimeout") }) operation.attempt(function (currentAttempt) { log.info("retry", "fetch attempt " + currentAttempt + " at " + (new Date()).toLocaleTimeString()) fetchAndShaCheck(u, tmp, shasum, auth, function (er, response, shasum) { // Only retry on 408, 5xx or no `response`. var sc = response && response.statusCode var statusRetry = !sc || (sc === 408 || sc >= 500) if (er && statusRetry && operation.retry(er)) { log.warn("retry", "will retry, error on last attempt: " + er) return } cb(er, response, shasum) }) }) } function fetchAndShaCheck (u, tmp, shasum, auth, cb) { cb = pulseTillDone("fetchTarball", cb) npm.registry.fetch(u, { auth : auth }, function (er, response) { if (er) { log.error("fetch failed", u) return cb(er, response) } var tarball = createWriteStream(tmp, { mode : npm.modes.file }) tarball.on("error", function (er) { cb(er) tarball.destroy() }) tarball.on("finish", function () { if (!shasum) { // Well, we weren't given a shasum, so at least sha what we have // in case we want to compare it to something else later return sha.get(tmp, function (er, shasum) { log.silly("fetchAndShaCheck", "shasum", shasum) cb(er, response, shasum) }) } // validate that the url we just downloaded matches the expected shasum. log.silly("fetchAndShaCheck", "shasum", shasum) sha.check(tmp, shasum, function (er) { if (er && er.message) { // add original filename for better debuggability er.message = er.message + "\n" + "From: " + u } return cb(er, response, shasum) }) }) response.pipe(tarball) }) }