diff options
author | Forrest L Norvell <forrest@npmjs.com> | 2015-03-05 03:38:15 +0300 |
---|---|---|
committer | Forrest L Norvell <forrest@npmjs.com> | 2015-03-05 09:55:00 +0300 |
commit | 6823807bba6c00228a724e1205ae90d67df0adad (patch) | |
tree | 76bfe9bf0e72c2e50c1167da0d84ee3eaad100c8 /lib | |
parent | 1b8ba7426393cbae2c76ad2c35953782d4401871 (diff) |
restore order to caching git remotes
Also, update code coments to be accurate, and extirpate single-letter
variable names. And introduce a params object because those argument
lists were completely ridiculous.
Diffstat (limited to 'lib')
-rw-r--r-- | lib/cache/add-remote-git.js | 532 | ||||
-rw-r--r-- | lib/cache/add-remote-tarball.js | 2 |
2 files changed, 307 insertions, 227 deletions
diff --git a/lib/cache/add-remote-git.js b/lib/cache/add-remote-git.js index 85a4c722d..9eaf6b18a 100644 --- a/lib/cache/add-remote-git.js +++ b/lib/cache/add-remote-git.js @@ -1,269 +1,330 @@ -var mkdir = require("mkdirp") - , assert = require("assert") - , git = require("../utils/git.js") - , fs = require("graceful-fs") - , log = require("npmlog") - , path = require("path") - , url = require("url") - , chownr = require("chownr") - , crypto = require("crypto") - , npm = require("../npm.js") - , rm = require("../utils/gently-rm.js") - , inflight = require("inflight") - , getCacheStat = require("./get-stat.js") - , addLocal = require("./add-local.js") - , realizePackageSpecifier = require("realize-package-specifier") - , normalizeGitUrl = require("normalize-git-url") - , randomBytes = require("crypto").pseudoRandomBytes // only need uniqueness - -var remotes = path.resolve(npm.config.get("cache"), "_git-remotes") -var templates = path.join(remotes, "_templates") +var mkdir = require('mkdirp') +var assert = require('assert') +var git = require('../utils/git.js') +var fs = require('graceful-fs') +var log = require('npmlog') +var path = require('path') +var url = require('url') +var chownr = require('chownr') +var crypto = require('crypto') +var npm = require('../npm.js') +var rm = require('../utils/gently-rm.js') +var inflight = require('inflight') +var getCacheStat = require('./get-stat.js') +var addLocal = require('./add-local.js') +var realizePackageSpecifier = require('realize-package-specifier') +var normalizeGitUrl = require('normalize-git-url') +var randomBytes = require('crypto').pseudoRandomBytes // only need uniqueness + +var remotes = path.resolve(npm.config.get('cache'), '_git-remotes') +var templates = path.join(remotes, '_templates') var VALID_VARIABLES = [ - "GIT_SSH", - "GIT_SSL_NO_VERIFY", - "GIT_PROXY_COMMAND", - "GIT_SSL_CAINFO" + 'GIT_SSH', + 'GIT_SSL_NO_VERIFY', + 'GIT_PROXY_COMMAND', + 'GIT_SSL_CAINFO' ] -// 1. cacheDir = path.join(cache,'_git-remotes',sha1(u)) -// 2. checkGitDir(cacheDir) ? 4. : 3. (rm cacheDir if necessary) -// 3. git clone --mirror u cacheDir -// 4. cd cacheDir && git fetch -a origin -// 5. git archive /tmp/random.tgz -// 6. addLocalTarball(/tmp/random.tgz) <gitref> --format=tar --prefix=package/ -// silent flag is used if this should error quietly -module.exports = function addRemoteGit (u, silent, cb) { - assert(typeof u === "string", "must have git URL") - assert(typeof cb === "function", "must have callback") - - log.verbose("addRemoteGit", "u=%j silent=%j", u, silent) - var normalized = normalizeGitUrl(u) - log.silly("addRemoteGit", "normalized", normalized) - - var v = crypto.createHash("sha1").update(normalized.url).digest("hex").slice(0, 8) - v = normalized.url.replace(/[^a-zA-Z0-9]+/g, "-")+"-"+v - log.silly("addRemoteGit", "v", v) - - var p = path.join(remotes, v) - cb = inflight(p, cb) - if (!cb) return log.verbose("addRemoteGit", p, "already in flight; waiting") - log.verbose("addRemoteGit", p, "not in flight; cloning") +module.exports = function addRemoteGit (uri, silent, cb) { + assert(typeof uri === 'string', 'must have git URL') + assert(typeof cb === 'function', 'must have callback') + + // reconstruct the URL as it was passed in – realizePackageSpecifier + // strips off `git+` and `maybeGithub` doesn't include it. + var originalURL + if (!/^git[+:]/.test(uri)) { + originalURL = 'git+' + uri + } else { + originalURL = uri + } - getGitDir(function (er) { - if (er) return cb(er) - checkGitDir(p, normalized.url, normalized.branch, u, silent, function (er, data) { - if (er) return cb(er, data) + // break apart the origin URL and the branch / tag / commitish + var normalized = normalizeGitUrl(uri) + var gitURL = normalized.url + var treeish = normalized.branch - addModeRecursive(p, npm.modes.file, function (er) { - return cb(er, data) - }) - }) - }) -} + // ensure that similarly-named remotes don't collide + var repoID = gitURL.replace(/[^a-zA-Z0-9]+/g, '-') + '-' + + crypto.createHash('sha1').update(gitURL).digest('hex').slice(0, 8) + var cachedRemote = path.join(remotes, repoID) -function getGitDir (cb) { - getCacheStat(function (er, st) { - if (er) return cb(er) + // set later, as the callback flow proceeds + var resolvedURL + var resolvedTreeish + var tmpdir - // We don't need global templates when cloning. Use an empty directory for - // the templates, creating it (and setting its permissions) if necessary. - mkdir(templates, function (er) { - if (er) return cb(er) + cb = inflight(repoID, cb) + if (!cb) { + return log.verbose('addRemoteGit', repoID, 'already in flight; waiting') + } + log.verbose('addRemoteGit', repoID, 'not in flight; caching') - // Ensure that both the template and remotes directories have the correct - // permissions. - fs.chown(templates, st.uid, st.gid, function (er) { - if (er) return cb(er) + // initialize the remotes cache with the correct perms + getGitDir(function (er) { + if (er) return cb(er) + fs.stat(cachedRemote, function (er, s) { + if (er) return mirrorRemote(finish) + if (!s.isDirectory()) return resetRemote(finish) - fs.chown(remotes, st.uid, st.gid, function (er) { - cb(er, st) - }) - }) + validateExistingRemote(finish) }) + + // always set permissions on the cached remote + function finish (er, data) { + if (er) return cb(er, data) + addModeRecursive(cachedRemote, npm.modes.file, function (er) { + return cb(er, data) + }) + } }) -} -function checkGitDir (p, u, co, origUrl, silent, cb) { - fs.stat(p, function (er, s) { - if (er) return cloneGitRemote(p, u, co, origUrl, silent, cb) - if (!s.isDirectory()) return rm(p, function (er) { + // don't try too hard to hold on to a remote + function resetRemote (cb) { + log.info('addRemoteGit', 'resetting', cachedRemote) + rm(cachedRemote, function (er) { if (er) return cb(er) - cloneGitRemote(p, u, co, origUrl, silent, cb) + mirrorRemote(cb) }) + } + // reuse a cached remote when possible, but nuke it if it's in an + // inconsistent state + function validateExistingRemote (cb) { git.whichAndExec( - [ "config", "--get", "remote.origin.url" ], - { cwd : p, env : gitEnv() }, + ['config', '--get', 'remote.origin.url'], + { cwd: cachedRemote, env: gitEnv() }, function (er, stdout, stderr) { - var stdoutTrimmed = (stdout + "\n" + stderr).trim() - if (er || u !== stdout.trim()) { - log.warn( "`git config --get remote.origin.url` returned " - + "wrong result ("+u+")", stdoutTrimmed ) - return rm(p, function (er){ - if (er) return cb(er) - cloneGitRemote(p, u, co, origUrl, silent, cb) - }) + var originURL = stdout.trim() + stderr = stderr.trim() + log.verbose('addRemoteGit', 'remote.origin.url:', originURL) + + if (stderr || er) { + log.warn('addRemoteGit', 'resetting remote', cachedRemote, 'because of error:', stderr || er) + return resetRemote(cb) + } else if (gitURL !== originURL) { + log.warn( + 'addRemoteGit', + 'pre-existing cached repo', cachedRemote, 'points to', originURL, 'and not', gitURL + ) + return resetRemote(cb) } - log.verbose("git remote.origin.url", stdoutTrimmed) - fetchRemote(p, u, co, origUrl, cb) + + log.verbose('addRemoteGit', 'updating existing cached remote', cachedRemote) + updateRemote(cb) } ) - }) -} + } -function cloneGitRemote (p, u, co, origUrl, silent, cb) { - mkdir(p, function (er) { - if (er) return cb(er) + // make a complete bare mirror of the remote repo + // NOTE: npm uses a blank template directory to prevent weird inconsistencies + // https://github.com/npm/npm/issues/5867 + function mirrorRemote (cb) { + mkdir(cachedRemote, function (er) { + if (er) return cb(er) - git.whichAndExec( - [ "clone", "--template=" + templates, "--mirror", u, p ], - { cwd : p, env : gitEnv() }, - function (er, stdout, stderr) { - stdout = (stdout + "\n" + stderr).trim() - if (er) { - if (silent) { - log.verbose("git clone " + u, stdout) - } else { - log.error("git clone " + u, stdout) + var args = [ + 'clone', + '--template=' + templates, + '--mirror', + gitURL, cachedRemote + ] + git.whichAndExec( + ['clone', '--template=' + templates, '--mirror', gitURL, cachedRemote], + { cwd: cachedRemote, env: gitEnv() }, + function (er, stdout, stderr) { + if (er) { + var combined = (stdout + '\n' + stderr).trim() + var command = 'git ' + args.join(' ') + ':' + if (silent) { + log.verbose(command, combined) + } else { + log.error(command, combined) + } + return cb(er) } - return cb(er) + log.verbose('addRemoteGit', 'git clone ' + gitURL, stdout.trim()) + setPermissions(cb) } - log.verbose("git clone " + u, stdout) - fetchRemote(p, u, co, origUrl, cb) - } - ) - }) -} + ) + }) + } -function fetchRemote (p, u, co, origUrl, cb) { - git.whichAndExec( - [ "fetch", "-a", "origin" ], - { cwd : p, env : gitEnv() }, - function (er, stdout, stderr) { - stdout = (stdout + "\n" + stderr).trim() - if (er) { - log.error("git fetch -a origin ("+u+")", stdout) - return cb(er) - } - log.verbose("git fetch -a origin ("+u+")", stdout) + function setPermissions (cb) { + if (process.platform === 'win32') { + log.verbose('addRemoteGit', 'skipping chownr on Windows') + resolveHead(cb) + } else { + getGitDir(function (er, cs) { + if (er) { + log.error('addRemoteGit', 'could not get cache stat') + return cb(er) + } - if (process.platform === "win32") { - log.silly("verifyOwnership", "skipping for windows") - resolveHead(p, u, co, origUrl, cb) - } - else { - getGitDir(function (er, cs) { + chownr(cachedRemote, cs.uid, cs.gid, function (er) { if (er) { - log.error("Could not get cache stat") + log.error( + 'addRemoteGit', + 'Failed to change folder ownership under npm cache for', + cachedRemote + ) return cb(er) } - chownr(p, cs.uid, cs.gid, function (er) { - if (er) { - log.error("Failed to change folder ownership under npm cache for %s", p) - return cb(er) - } - - resolveHead(p, u, co, origUrl, cb) - }) + log.verbose('addRemoteGit', 'set permissions on', cachedRemote) + resolveHead(cb) }) - } + }) } - ) -} - -function resolveHead (p, u, co, origUrl, cb) { - git.whichAndExec( - [ "rev-list", "-n1", co ], - { cwd : p, env : gitEnv() }, - function (er, stdout, stderr) { - stdout = (stdout + "\n" + stderr).trim() - if (er) { - log.error("Failed resolving git HEAD (" + u + ")", stderr) - return cb(er) - } - log.verbose("git rev-list -n1 " + co, stdout) - var parsed = url.parse(origUrl) - parsed.hash = stdout - var resolved = url.format(parsed) + } - if (!/^git[+:]/.test(parsed.protocol)) { - resolved = "git+" + resolved - } + // always fetch the origin, even right after mirroring, because this way + // permissions will get set correctly + function updateRemote (cb) { + git.whichAndExec( + ['fetch', '-a', 'origin'], + { cwd: cachedRemote, env: gitEnv() }, + function (er, stdout, stderr) { + if (er) { + var combined = (stdout + '\n' + stderr).trim() + log.error('git fetch -a origin (' + gitURL + ')', combined) + return cb(er) + } + log.verbose('addRemoteGit', 'git fetch -a origin (' + gitURL + ')', stdout.trim()) - // https://github.com/npm/npm/issues/3224 - // node incorrectly sticks a / at the start of the path We know that the - // host won't change, so split and detect this - var spo = origUrl.split(parsed.host) - var spr = resolved.split(parsed.host) - if (spo[1].charAt(0) === ":" && spr[1].charAt(0) === "/") { - spr[1] = spr[1].slice(1) + setPermissions(cb) } - resolved = spr.join(parsed.host) + ) + } - log.verbose("resolved git url", resolved) - cache(p, u, stdout, resolved, cb) - } - ) -} + // branches and tags are both symbolic labels that can be attached to different + // commits, so resolve the commitish to the current actual treeish the label + // corresponds to + // + // important for shrinkwrap + function resolveHead (cb) { + log.verbose('addRemoteGit', 'original treeish:', treeish) + var args = ['rev-list', '-n1', treeish] + git.whichAndExec( + args, + { cwd: cachedRemote, env: gitEnv() }, + function (er, stdout, stderr) { + if (er) { + log.error('git ' + args.join(' ') + ':', stderr) + return cb(er) + } -/** - * Make an actual clone from the bare (mirrored) cache. There is no safe way to - * do a one-step clone to a treeish that isn't guaranteed to be a branch, so - * this has to be two steps. - */ -function cache (p, u, treeish, resolved, cb) { - // generate a unique filename - randomBytes(6, function (er, random) { - if (er) return cb(er) + resolvedTreeish = stdout.trim() + log.silly('addRemoteGit', 'resolved treeish:', resolvedTreeish) - var tmp = path.join( - npm.tmp, - "git-cache-"+random.toString("hex"), - treeish - ) + resolvedURL = getResolved(originalURL, resolvedTreeish) + log.verbose('addRemoteGit', 'resolved Git URL:', resolvedURL) - mkdir(tmp, function (er) { - if (er) return cb(er) + // generate a unique filename + tmpdir = path.join( + npm.tmp, + 'git-cache-' + randomBytes(6).toString('hex'), + resolvedTreeish + ) + log.silly('addRemoteGit', 'Git working directory:', tmpdir) - git.whichAndExec(["clone", p, tmp], { cwd : p, env : gitEnv() }, clone) - }) + mkdir(tmpdir, function (er) { + if (er) return cb(er) - function clone (er, stdout, stderr) { - stdout = (stdout + "\n" + stderr).trim() - if (er) { - log.error("Failed to clone "+resolved+" from "+u, stderr) - return cb(er) + cloneResolved(cb) + }) } - log.verbose("git clone", "from", p) - log.verbose("git clone", stdout) + ) + } - git.whichAndExec(["checkout", treeish], { cwd : tmp, env : gitEnv() }, checkout) - } + // make a clone from the mirrored cache so we have a temporary directory in + // which we can check out the resolved treeish + function cloneResolved (cb) { + var args = ['clone', cachedRemote, tmpdir] + git.whichAndExec( + args, + { cwd: cachedRemote, env: gitEnv() }, + function (er, stdout, stderr) { + stdout = (stdout + '\n' + stderr).trim() + if (er) { + log.error('git ' + args.join(' ') + ':', stderr) + return cb(er) + } + log.verbose('addRemoteGit', 'clone', stdout) - function checkout (er, stdout, stderr) { - stdout = (stdout + "\n" + stderr).trim() - if (er) { - log.error("Failed to check out "+treeish, stderr) - return cb(er) + checkoutTreeish(cb) } - log.verbose("git checkout", stdout) + ) + } - realizePackageSpecifier(tmp, function (er, spec) { + // there is no safe way to do a one-step clone to a treeish that isn't + // guaranteed to be a branch, so explicitly check out the treeish once it's + // cloned + function checkoutTreeish (cb) { + var args = ['checkout', resolvedTreeish] + git.whichAndExec( + args, + { cwd: tmpdir, env: gitEnv() }, + function (er, stdout, stderr) { + stdout = (stdout + '\n' + stderr).trim() if (er) { - log.error("Failed to map", tmp, "to a package specifier") + log.error('git ' + args.join(' ') + ':', stderr) return cb(er) } + log.verbose('addRemoteGit', 'checkout', stdout) - // https://github.com/npm/npm/issues/6400 - // ensure pack logic is applied - addLocal(spec, null, function (er, data) { - if (data) data._resolved = resolved - cb(er, data) + // convince addLocal that the checkout is a local dependency + realizePackageSpecifier(tmpdir, function (er, spec) { + if (er) { + log.error('addRemoteGit', 'Failed to map', tmpdir, 'to a package specifier') + return cb(er) + } + + // ensure pack logic is applied + // https://github.com/npm/npm/issues/6400 + addLocal(spec, null, function (er, data) { + if (data) { + log.verbose('addRemoteGit', 'data._resolved:', resolvedURL) + data._resolved = resolvedURL + + // the spec passed to addLocal is not what the user originally requested, + // so remap + // https://github.com/npm/npm/issues/7121 + if (!data._fromGitHub) { + log.silly('addRemoteGit', 'data._from:', originalURL) + data._from = originalURL + } else { + log.silly('addRemoteGit', 'data._from:', data._from, '(GitHub)') + } + } + + cb(er, data) + }) + }) + } + ) + } +} + +function getGitDir (cb) { + getCacheStat(function (er, stats) { + if (er) return cb(er) + + // We don't need global templates when cloning. Use an empty directory for + // the templates, creating it (and setting its permissions) if necessary. + mkdir(templates, function (er) { + if (er) return cb(er) + + // Ensure that both the template and remotes directories have the correct + // permissions. + fs.chown(templates, stats.uid, stats.gid, function (er) { + if (er) return cb(er) + + fs.chown(remotes, stats.uid, stats.gid, function (er) { + cb(er, stats) }) }) - } + }) }) } @@ -280,41 +341,60 @@ function gitEnv () { return gitEnv_ } +function getResolved (uri, treeish) { + var parsed = url.parse(uri) + parsed.hash = treeish + if (!/^git[+:]/.test(parsed.protocol)) { + parsed.protocol = 'git+' + parsed.protocol + } + var resolved = url.format(parsed) + + // node incorrectly sticks a / at the start of the path We know that the host + // won't change, so split and detect this + // https://github.com/npm/npm/issues/3224 + var spo = uri.split(parsed.host) + var spr = resolved.split(parsed.host) + if (spo[1].charAt(0) === ':' && spr[1].charAt(0) === '/') { + spr[1] = spr[1].slice(1) + } + return spr.join(parsed.host) +} + // similar to chmodr except it add permissions rather than overwriting them // adapted from https://github.com/isaacs/chmodr/blob/master/chmodr.js -function addModeRecursive(p, mode, cb) { - fs.readdir(p, function (er, children) { +function addModeRecursive (cachedRemote, mode, cb) { + fs.readdir(cachedRemote, function (er, children) { // Any error other than ENOTDIR means it's not readable, or doesn't exist. // Give up. - if (er && er.code !== "ENOTDIR") return cb(er) - if (er || !children.length) return addMode(p, mode, cb) + if (er && er.code !== 'ENOTDIR') return cb(er) + if (er || !children.length) return addMode(cachedRemote, mode, cb) var len = children.length var errState = null children.forEach(function (child) { - addModeRecursive(path.resolve(p, child), mode, then) + addModeRecursive(path.resolve(cachedRemote, child), mode, then) }) function then (er) { if (errState) return undefined if (er) return cb(errState = er) - if (--len === 0) return addMode(p, dirMode(mode), cb) + if (--len === 0) return addMode(cachedRemote, dirMode(mode), cb) } }) } -function addMode(p, mode, cb) { - fs.stat(p, function (er, stats) { +function addMode (cachedRemote, mode, cb) { + fs.stat(cachedRemote, function (er, stats) { if (er) return cb(er) mode = stats.mode | mode - fs.chmod(p, mode, cb) + fs.chmod(cachedRemote, mode, cb) }) } // taken from https://github.com/isaacs/chmodr/blob/master/chmodr.js -function dirMode(mode) { - if (mode & parseInt("0400", 8)) mode |= parseInt("0100", 8) - if (mode & parseInt( "040", 8)) mode |= parseInt( "010", 8) - if (mode & parseInt( "04", 8)) mode |= parseInt( "01", 8) +function dirMode (mode) { + if (mode & parseInt('0400', 8)) mode |= parseInt('0100', 8) + if (mode & parseInt('040', 8)) mode |= parseInt('010', 8) + if (mode & parseInt('04', 8)) mode |= parseInt('01', 8) return mode } diff --git a/lib/cache/add-remote-tarball.js b/lib/cache/add-remote-tarball.js index e87ac54bb..66d220096 100644 --- a/lib/cache/add-remote-tarball.js +++ b/lib/cache/add-remote-tarball.js @@ -19,8 +19,8 @@ function addRemoteTarball (u, pkgData, shasum, auth, cb_) { function cb (er, data) { if (data) { data._from = u - data._shasum = data._shasum || shasum data._resolved = u + data._shasum = data._shasum || shasum } cb_(er, data) } |