diff options
author | isaacs <i@izs.me> | 2020-05-05 02:17:58 +0300 |
---|---|---|
committer | isaacs <i@izs.me> | 2020-05-08 04:19:20 +0300 |
commit | 59d937387d510d3f5b09390d601d53ab66a63d37 (patch) | |
tree | 1b8903aef7b82e678ba3397799220f28d37a832b | |
parent | 873665e2c7f400c5de1962135f2510acc304b599 (diff) |
Consistent output for most reify() commands
This adds a 'reify-output.js' util, which can be passed any Arborist
object after it reifies a tree. Consistent output is printed in all
cases, showing the number of packages added/removed/changed, packages
needing funding, and a minimal (but always actionable and relevant)
audit summary.
The only code using the Installer class now is in lib/outdated.js, which
is has a pending update coming soon.
Prune and dedupe commands are awaiting top-level Arborist methods, so
that they can be similarly tightened up. (For now, this commit just has
them fail with a 'coming soon' message.)
The last piece holding the 'install/*.js' code in this repo is that it
is used in 'ls', 'fund', 'shrinkwrap', and the error-message util.
-rw-r--r-- | lib/audit.js | 312 | ||||
-rw-r--r-- | lib/ci.js | 53 | ||||
-rw-r--r-- | lib/dedupe.js | 169 | ||||
-rw-r--r-- | lib/install.js | 161 | ||||
-rw-r--r-- | lib/link.js | 213 | ||||
-rw-r--r-- | lib/npm.js | 2 | ||||
-rw-r--r-- | lib/prune.js | 75 | ||||
-rw-r--r-- | lib/uninstall.js | 99 | ||||
-rw-r--r-- | lib/update.js | 28 | ||||
-rw-r--r-- | lib/utils/reify-output.js | 136 |
10 files changed, 363 insertions, 885 deletions
diff --git a/lib/audit.js b/lib/audit.js index e34f14f9f..0ad2df0d4 100644 --- a/lib/audit.js +++ b/lib/audit.js @@ -1,50 +1,41 @@ -'use strict' - -const Bluebird = require('bluebird') - -const audit = require('./install/audit.js') -const figgyPudding = require('figgy-pudding') -const fs = require('graceful-fs') -const Installer = require('./install.js').Installer -const lockVerify = require('lock-verify') -const log = require('npmlog') -const npa = require('npm-package-arg') +const Arborist = require('@npmcli/arborist') +const auditReport = require('npm-audit-report') const npm = require('./npm.js') -const npmConfig = require('./config/figgy-config.js') const output = require('./utils/output.js') -const parseJson = require('json-parse-better-errors') +const reifyOutput = require('./utils/reify-output.js') -const readFile = Bluebird.promisify(fs.readFile) - -const AuditConfig = figgyPudding({ - also: {}, - 'audit-level': {}, - deepArgs: 'deep-args', - 'deep-args': {}, - dev: {}, - force: {}, - 'dry-run': {}, - global: {}, - json: {}, - only: {}, - parseable: {}, - prod: {}, - production: {}, - registry: {}, - runId: {} -}) +const audit = async args => { + const arb = new Arborist({ + ...npm.flatOptions, + audit: true, + path: npm.prefix + }) + const fix = args[0] === 'fix' + const result = await arb.audit({ fix }) + if (fix) { + reifyOutput(arb) + } else { + const reporter = npm.flatOptions.json ? 'json' : 'detail' + const result = auditReport(arb.auditReport, { + ...npm.flatOptions, + reporter + }) + process.exitCode = process.exitCode || result.exitCode + output(result.report) + } +} -module.exports = auditCmd +const cmd = (args, cb) => audit(args).then(() => cb()).catch(cb) -const usage = require('./utils/usage') -auditCmd.usage = usage( +const usageUtil = require('./utils/usage') +const usage = usageUtil( 'audit', '\nnpm audit [--json] [--production]' + '\nnpm audit fix ' + '[--force|--package-lock-only|--dry-run|--production|--only=(dev|prod)]' ) -auditCmd.completion = function (opts, cb) { +const completion = (opts, cb) => { const argv = opts.conf.argv.remain switch (argv[2]) { @@ -55,251 +46,4 @@ auditCmd.completion = function (opts, cb) { } } -class Auditor extends (class {}) { - constructor (where, dryrun, args, opts) { - super(where, dryrun, args, opts) - this.deepArgs = (opts && opts.deepArgs) || [] - this.runId = opts.runId || '' - this.audit = false - } - - loadAllDepsIntoIdealTree (cb) { - Bluebird.fromNode(cb => super.loadAllDepsIntoIdealTree(cb)).then(() => { - if (this.deepArgs && this.deepArgs.length) { - this.deepArgs.forEach(arg => { - arg.reduce((acc, child, ii) => { - if (!acc) { - // We might not always be able to find `target` through the given - // path. If we can't we'll just ignore it. - return - } - const spec = npa(child) - const target = ( - acc.requires.find(n => n.package.name === spec.name) || - acc.requires.find( - n => audit.scrub(n.package.name, this.runId) === spec.name - ) - ) - if (target && ii === arg.length - 1) { - target.loaded = false - // This kills `hasModernMeta()` and forces a re-fetch - target.package = { - name: spec.name, - version: spec.fetchSpec, - _requested: target.package._requested - } - delete target.fakeChild - let parent = target.parent - while (parent) { - parent.loaded = false - parent = parent.parent - } - target.requiredBy.forEach(par => { - par.loaded = false - delete par.fakeChild - }) - } - return target - }, this.idealTree) - }) - return Bluebird.fromNode(cb => super.loadAllDepsIntoIdealTree(cb)) - } - }).nodeify(cb) - } - - // no top level lifecycles on audit - runPreinstallTopLevelLifecycles (cb) { cb() } - runPostinstallTopLevelLifecycles (cb) { cb() } -} - -function maybeReadFile (name) { - const file = `${npm.prefix}/${name}` - return readFile(file) - .then((data) => { - try { - return parseJson(data) - } catch (ex) { - ex.code = 'EJSONPARSE' - throw ex - } - }) - .catch({code: 'ENOENT'}, () => null) - .catch((ex) => { - ex.file = file - throw ex - }) -} - -function filterEnv (action, opts) { - const includeDev = opts.dev || - (!/^prod(uction)?$/.test(opts.only) && !opts.production) || - /^dev(elopment)?$/.test(opts.only) || - /^dev(elopment)?$/.test(opts.also) - const includeProd = !/^dev(elopment)?$/.test(opts.only) - const resolves = action.resolves.filter(({dev}) => { - return (dev && includeDev) || (!dev && includeProd) - }) - if (resolves.length) { - return Object.assign({}, action, {resolves}) - } -} - -function auditCmd (args, cb) { - const opts = AuditConfig(npmConfig()) - if (opts.global) { - const err = new Error('`npm audit` does not support testing globals') - err.code = 'EAUDITGLOBAL' - throw err - } - if (args.length && args[0] !== 'fix') { - return cb(new Error('Invalid audit subcommand: `' + args[0] + '`\n\nUsage:\n' + auditCmd.usage)) - } - return Bluebird.all([ - maybeReadFile('npm-shrinkwrap.json'), - maybeReadFile('package-lock.json'), - maybeReadFile('package.json') - ]).spread((shrinkwrap, lockfile, pkgJson) => { - const sw = shrinkwrap || lockfile - if (!pkgJson) { - const err = new Error('No package.json found: Cannot audit a project without a package.json') - err.code = 'EAUDITNOPJSON' - throw err - } - if (!sw) { - const err = new Error('Neither npm-shrinkwrap.json nor package-lock.json found: Cannot audit a project without a lockfile') - err.code = 'EAUDITNOLOCK' - throw err - } else if (shrinkwrap && lockfile) { - log.warn('audit', 'Both npm-shrinkwrap.json and package-lock.json exist, using npm-shrinkwrap.json.') - } - const requires = Object.assign( - {}, - (pkgJson && pkgJson.dependencies) || {}, - (!opts.production && pkgJson && pkgJson.devDependencies) || {} - ) - return lockVerify(npm.prefix).then((result) => { - if (result.status) return audit.generate(sw, requires) - - const lockFile = shrinkwrap ? 'npm-shrinkwrap.json' : 'package-lock.json' - const err = new Error(`Errors were found in your ${lockFile}, run npm install to fix them.\n ` + - result.errors.join('\n ')) - err.code = 'ELOCKVERIFY' - throw err - }) - }).then((auditReport) => { - return audit.submitForFullReport(auditReport) - }).catch((err) => { - if (err.statusCode >= 400) { - let msg - if (err.statusCode === 401) { - msg = `Either your login credentials are invalid or your registry (${opts.registry}) does not support audit.` - } else if (err.statusCode === 404) { - msg = `Your configured registry (${opts.registry}) does not support audit requests.` - } else { - msg = `Your configured registry (${opts.registry}) may not support audit requests, or the audit endpoint may be temporarily unavailable.` - } - if (err.body.length) { - msg += '\nThe server said: ' + err.body - } - const ne = new Error(msg) - ne.code = 'ENOAUDIT' - ne.wrapped = err - throw ne - } - throw err - }).then((auditResult) => { - if (args[0] === 'fix') { - const actions = (auditResult.actions || []).reduce((acc, action) => { - action = filterEnv(action, opts) - if (!action) { return acc } - if (action.isMajor) { - acc.major.add(`${action.module}@${action.target}`) - action.resolves.forEach(({id, path}) => acc.majorFixes.add(`${id}::${path}`)) - } else if (action.action === 'install') { - acc.install.add(`${action.module}@${action.target}`) - action.resolves.forEach(({id, path}) => acc.installFixes.add(`${id}::${path}`)) - } else if (action.action === 'update') { - const name = action.module - const version = action.target - action.resolves.forEach(vuln => { - acc.updateFixes.add(`${vuln.id}::${vuln.path}`) - const modPath = vuln.path.split('>') - const newPath = modPath.slice( - 0, modPath.indexOf(name) - ).concat(`${name}@${version}`) - if (newPath.length === 1) { - acc.install.add(newPath[0]) - } else { - acc.update.add(newPath.join('>')) - } - }) - } else if (action.action === 'review') { - action.resolves.forEach(({id, path}) => acc.review.add(`${id}::${path}`)) - } - return acc - }, { - install: new Set(), - installFixes: new Set(), - update: new Set(), - updateFixes: new Set(), - major: new Set(), - majorFixes: new Set(), - review: new Set() - }) - return Bluebird.try(() => { - const installMajor = opts.force - const installCount = actions.install.size + (installMajor ? actions.major.size : 0) + actions.update.size - const vulnFixCount = new Set([...actions.installFixes, ...actions.updateFixes, ...(installMajor ? actions.majorFixes : [])]).size - const metavuln = auditResult.metadata.vulnerabilities - const total = Object.keys(metavuln).reduce((acc, key) => acc + metavuln[key], 0) - if (installCount) { - log.verbose( - 'audit', - 'installing', - [...actions.install, ...(installMajor ? actions.major : []), ...actions.update] - ) - } - return Bluebird.fromNode(cb => { - new Auditor( - npm.prefix, - !!opts['dry-run'], - [...actions.install, ...(installMajor ? actions.major : [])], - opts.concat({ - runId: auditResult.runId, - deepArgs: [...actions.update].map(u => u.split('>')) - }).toJSON() - ).run(cb) - }).then(() => { - const numScanned = auditResult.metadata.totalDependencies - if (!opts.json && !opts.parseable) { - output(`fixed ${vulnFixCount} of ${total} vulnerabilit${total === 1 ? 'y' : 'ies'} in ${numScanned} scanned package${numScanned === 1 ? '' : 's'}`) - if (actions.review.size) { - output(` ${actions.review.size} vulnerabilit${actions.review.size === 1 ? 'y' : 'ies'} required manual review and could not be updated`) - } - if (actions.major.size) { - output(` ${actions.major.size} package update${actions.major.size === 1 ? '' : 's'} for ${actions.majorFixes.size} vulnerabilit${actions.majorFixes.size === 1 ? 'y' : 'ies'} involved breaking changes`) - if (installMajor) { - output(' (installed due to `--force` option)') - } else { - output(' (use `npm audit fix --force` to install breaking changes;' + - ' or refer to `npm audit` for steps to fix these manually)') - } - } - } - }) - }) - } else { - const levels = ['low', 'moderate', 'high', 'critical'] - const minLevel = levels.indexOf(opts['audit-level']) - const vulns = levels.reduce((count, level, i) => { - return i < minLevel ? count : count + (auditResult.metadata.vulnerabilities[level] || 0) - }, 0) - if (vulns > 0) process.exitCode = 1 - if (opts.parseable) { - return audit.printParseableReport(auditResult) - } else { - return audit.printFullReport(auditResult) - } - } - }).asCallback(cb) -} +module.exports = Object.assign(cmd, { usage, completion }) @@ -1,24 +1,20 @@ -'use strict' - const util = require('util') const Arborist = require('@npmcli/arborist') const rimraf = util.promisify(require('rimraf')) +const reifyOutput = require('./utils/reify-output.js') +const log = require('npmlog') const npm = require('./npm.js') const output = require('./utils/output.js') +const usageUtil = require('./utils/usage.js') -cmd.usage = 'npm ci' +const usage = usageUtil('ci', 'npm ci') -cmd.completion = (cb) => cb(null, []) +const completion = (cb) => cb(null, []) -module.exports = cmd -function cmd(cb) { - ci() - .then(() => cb()) - .catch(cb) -} +const cmd = (args, cb) => ci().then(() => cb()).catch(cb) -async function ci () { +const ci = async () => { if (npm.flatOptions.global) { const err = new Error('`npm ci` does not work for global packages') err.code = 'ECIGLOBAL' @@ -28,26 +24,19 @@ async function ci () { const where = npm.prefix const arb = new Arborist({ ...npm.flatOptions, path: where }) - try { - await arb.loadVirtual() - const start = Date.now() - await rimraf(`${where}/node_modules/`) - await arb.reify() - const stop = Date.now() - - const time = (stop - start) / 1000 - const pkgCount = arb.diff.children.length - const added = `added ${pkgCount}` - output(`${added} packages in ${time}s`) - - } catch (err) { - if (err.message.match(/shrinkwrap/)) { - const msg = 'The \`npm ci\` command can only install packages with an existing ' + - 'package-lock.json or npm-shrinkwrap.json with lockfileVersion >= 1. Run an install ' + - 'with npm@5 or later to generate a package-lock.json file, then try again.' + await Promise.all([ + arb.loadVirtual().catch(er => { + log.verbose('loadVirtual', er.stack) + const msg = + 'The `npm ci` command can only install with an existing package-lock.json or\n' + + 'npm-shrinkwrap.json with lockfileVersion >= 1. Run an install with npm@5 or\n' + + 'later to generate a package-lock.json file, then try again.' throw new Error(msg) - } else { - throw err - } - } + }), + rimraf(`${where}/node_modules/`) + ]) + await arb.reify() + reifyOutput(arb) } + +module.exports = Object.assign(cmd, { completion, usage }) diff --git a/lib/dedupe.js b/lib/dedupe.js index 5174013a3..6e696e5ca 100644 --- a/lib/dedupe.js +++ b/lib/dedupe.js @@ -1,162 +1,19 @@ -// XXX replace this with @npmcli/arborist +// dedupe duplicated packages, or find them in the tree +const util = require('util') +const Arborist = require('@npmcli/arborist') +const rimraf = util.promisify(require('rimraf')) +const reifyOutput = require('./utils/reify-output.js') +const usageUtil = require('./utils/usage.js') -var util = require('util') -var path = require('path') -var validate = require('aproba') -var without = require('lodash.without') -var asyncMap = require('slide').asyncMap -var chain = require('slide').chain -var npa = require('npm-package-arg') -var log = require('npmlog') -var npm = require('./npm.js') -var Installer = require('./install.js').Installer -var findRequirement = require('./install/deps.js').findRequirement -var earliestInstallable = require('./install/deps.js').earliestInstallable -var checkPermissions = require('./install/check-permissions.js') -var decomposeActions = require('./install/decompose-actions.js') -var loadExtraneous = require('./install/deps.js').loadExtraneous -var computeMetadata = require('./install/deps.js').computeMetadata -var sortActions = require('./install/diff-trees.js').sortActions -var moduleName = require('./utils/module-name.js') -var packageId = require('./utils/package-id.js') -var childPath = require('./utils/child-path.js') -var usage = require('./utils/usage') -var getRequested = require('./install/get-requested.js') +const usage = usageUtil('dedupe', 'npm dedupe') -module.exports = dedupe -module.exports.Deduper = Deduper +const completion = (cb) => cb(null, []) -dedupe.usage = usage( - 'dedupe', - 'npm dedupe' -) +const cmd = (args, cb) => dedupe(args).then(() => cb()).catch(cb) -function dedupe (args, cb) { - validate('AF', arguments) - // the /path/to/node_modules/.. - var where = path.resolve(npm.dir, '..') - var dryrun = false - if (npm.command.match(/^find/)) dryrun = true - if (npm.config.get('dry-run')) dryrun = true - if (dryrun && !npm.config.get('json')) npm.config.set('parseable', true) - - new Deduper(where, dryrun).run(cb) -} - -function Deduper (where, dryrun) { - validate('SB', arguments) - Installer.call(this, where, dryrun, []) - this.noPackageJsonOk = true - this.topLevelLifecycles = false -} -util.inherits(Deduper, class {}) // Installer) - -Deduper.prototype.loadIdealTree = function (cb) { - validate('F', arguments) - log.silly('install', 'loadIdealTree') - - var self = this - chain([ - [this.newTracker(this.progress.loadIdealTree, 'cloneCurrentTree')], - [this, this.cloneCurrentTreeToIdealTree], - [this, this.finishTracker, 'cloneCurrentTree'], - - [this.newTracker(this.progress.loadIdealTree, 'loadAllDepsIntoIdealTree', 10)], - [ function (next) { - loadExtraneous(self.idealTree, self.progress.loadAllDepsIntoIdealTree, next) - } ], - [this, this.finishTracker, 'loadAllDepsIntoIdealTree'], - - [this, andComputeMetadata(this.idealTree)] - ], cb) -} - -function andComputeMetadata (tree) { - return function (next) { - next(null, computeMetadata(tree)) - } -} - -Deduper.prototype.generateActionsToTake = function (cb) { - validate('F', arguments) - log.silly('dedupe', 'generateActionsToTake') - chain([ - [this.newTracker(log, 'hoist', 1)], - [hoistChildren, this.idealTree, this.differences], - [this, this.finishTracker, 'hoist'], - [this.newTracker(log, 'sort-actions', 1)], - [this, function (next) { - this.differences = sortActions(this.differences) - next() - }], - [this, this.finishTracker, 'sort-actions'], - [checkPermissions, this.differences], - [decomposeActions, this.differences, this.todo] - ], cb) +const dedupe = async args => { + require('npmlog').warn('coming soon!') + throw new Error('not yet implemented') } -function move (node, hoistTo, diff) { - node.parent.children = without(node.parent.children, node) - hoistTo.children.push(node) - node.fromPath = node.path - node.path = childPath(hoistTo.path, node) - node.parent = hoistTo - if (!diff.filter(function (action) { return action[0] === 'move' && action[1] === node }).length) { - diff.push(['move', node]) - } -} - -function moveRemainingChildren (node, diff) { - node.children.forEach(function (child) { - move(child, node, diff) - moveRemainingChildren(child, diff) - }) -} - -function remove (child, diff, done) { - remove_(child, diff, new Set(), done) -} - -function remove_ (child, diff, seen, done) { - if (seen.has(child)) return done() - seen.add(child) - diff.push(['remove', child]) - child.parent.children = without(child.parent.children, child) - asyncMap(child.children, function (child, next) { - remove_(child, diff, seen, next) - }, done) -} - -function hoistChildren (tree, diff, next) { - hoistChildren_(tree, diff, new Set(), next) -} - -function hoistChildren_ (tree, diff, seen, next) { - validate('OAOF', arguments) - if (seen.has(tree)) return next() - seen.add(tree) - asyncMap(tree.children, function (child, done) { - if (!tree.parent || child.fromBundle || child.package._inBundle) return hoistChildren_(child, diff, seen, done) - var better = findRequirement(tree.parent, moduleName(child), getRequested(child) || npa(packageId(child))) - if (better) { - return chain([ - [remove, child, diff], - [andComputeMetadata(tree)] - ], done) - } - var hoistTo = earliestInstallable(tree, tree.parent, child.package, log) - if (hoistTo) { - move(child, hoistTo, diff) - chain([ - [andComputeMetadata(hoistTo)], - [hoistChildren_, child, diff, seen], - [ function (next) { - moveRemainingChildren(child, diff) - next() - } ] - ], done) - } else { - done() - } - }, next) -} +module.exports = Object.assign(cmd, { usage, completion }) diff --git a/lib/install.js b/lib/install.js index eb60c794d..e43f07ca4 100644 --- a/lib/install.js +++ b/lib/install.js @@ -1,24 +1,56 @@ 'use strict' /* eslint-disable camelcase */ /* eslint-disable standard/no-callback-literal */ -// npm install <pkg> <pkg> <pkg> -// -// See doc/cli/npm-install.md for more description -// -// Managing contexts... -// there's a lot of state associated with an "install" operation, including -// packages that are already installed, parent packages, current shrinkwrap, and -// so on. We maintain this state in a "context" object that gets passed around. -// every time we dive into a deeper node_modules folder, the "family" list that -// gets passed along uses the previous "family" list as its __proto__. Any -// "resolved precise dependency" things that aren't already on this object get -// added, and then that's passed to the next generation of installation. - -module.exports = install - -var usage = require('./utils/usage') - -install.usage = usage( + +const npm = require('./npm.js') +const usageUtil = require('./utils/usage.js') +const reifyOutput = require('./utils/reify-output.js') +const log = require('npmlog') +const { resolve, join } = require('path') +const Arborist = require('@npmcli/arborist') + +// XXX remove anything relying on this "where" argument, then remove it +const install = async (where, args, cb) => { + // the /path/to/node_modules/.. + const globalTop = resolve(npm.globalDir, '..') + const { dryRun, global: isGlobalInstall } = npm.flatOptions + if (typeof args === 'function') { + cb = args + args = where + where = isGlobalInstall ? globalTop : npm.prefix + } + + // don't try to install the prefix into itself + args = args.filter(a => resolve(a) !== npm.prefix) + + // `npm i -g` => "install this package globally" + if (where === globalTop && !args.length) { + args = ['.'] + } + + // TODO: Add warnings for other deprecated flags? or remove this one? + if (npm.config.get('dev')) { + log.warn('install', 'Usage of the `--dev` option is deprecated. Use `--include=dev` instead.') + } + + const arb = new Arborist({ + ...npm.flatOptions, + path: where + }) + + try { + const tree = await arb.reify({ + ...npm.flatOptions, + add: args, + }) + reifyOutput(arb) + cb() + } catch (er) { + cb(er) + } +} + +const usage = usageUtil( 'install', '\nnpm install (with no args, in package dir)' + '\nnpm install [<@scope>/]<pkg>' + @@ -34,10 +66,7 @@ install.usage = usage( '[--save-prod|--save-dev|--save-optional] [--save-exact] [--no-save]' ) -const npa = require('npm-package-arg') - -install.completion = function (opts, cb) { - validate('OF', arguments) +const completion = (opts, cb) => { // install can complete to a folder with a package.json, or any package. // if it has a slash, then it's gotta be a folder // if it starts with https?://, then just give up, because it's a url @@ -51,13 +80,12 @@ install.completion = function (opts, cb) { // is a folder containing a package.json file. If that is not the // case we return 0 matches, which will trigger the default bash // complete. - var lastSlashIdx = opts.partialWord.lastIndexOf('/') - var partialName = opts.partialWord.slice(lastSlashIdx + 1) - var partialPath = opts.partialWord.slice(0, lastSlashIdx) - if (partialPath === '') partialPath = '/' + const lastSlashIdx = opts.partialWord.lastIndexOf('/') + const partialName = opts.partialWord.slice(lastSlashIdx + 1) + const partialPath = opts.partialWord.slice(0, lastSlashIdx) || '/' - var annotatePackageDirMatch = function (sibling, cb) { - var fullPath = path.join(partialPath, sibling) + const annotatePackageDirMatch = (sibling, cb) => { + const fullPath = join(partialPath, sibling) if (sibling.slice(0, partialName.length) !== partialName) { return cb(null, null) // not name match } @@ -67,20 +95,20 @@ install.completion = function (opts, cb) { cb( null, { - fullPath: fullPath, + fullPath, isPackage: contents.indexOf('package.json') !== -1 } ) }) } - return fs.readdir(partialPath, function (err, siblings) { + return fs.readdir(partialPath, (err, siblings) => { if (err) return cb(null, []) // invalid dir: no matching - asyncMap(siblings, annotatePackageDirMatch, function (err, matches) { + asyncMap(siblings, annotatePackageDirMatch, (err, matches) => { if (err) return cb(err) - var cleaned = matches.filter(function (x) { return x !== null }) + const cleaned = matches.filter(x => x !== null) if (cleaned.length !== 1) return cb(null, []) if (!cleaned[0].isPackage) return cb(null, []) @@ -90,74 +118,9 @@ install.completion = function (opts, cb) { }) } - // FIXME: there used to be registry completion here, but it stopped making + // Note: there used to be registry completion here, but it stopped making // sense somewhere around 50,000 packages on the registry cb() } -const Arborist = require('@npmcli/arborist') - -// dependencies -var log = require('npmlog') -// const sillyLogTree = require('./util/silly-log-tree.js') - -// npm internal utils -var npm = require('./npm.js') -var output = require('./utils/output.js') -var saveMetrics = require('./utils/metrics.js').save - -// install specific libraries -var audit = require('./install/audit.js') -var { - getPrintFundingReport, - getPrintFundingReportJSON -} = require('./install/fund.js') -var errorMessage = require('./utils/error-message.js') - -const path = require('path') - -function install (where, args, cb) { - if (!cb) { - cb = args - args = where - where = null - } - // the /path/to/node_modules/.. - const globalTop = path.resolve(npm.globalDir, '..') - if (!where) { - where = npm.flatOptions.global - ? globalTop - : npm.prefix - } - const {dryRun} = npm.flatOptions - - // TODO: Add warnings for other deprecated flags - if (npm.config.get('dev')) { - log.warn('install', 'Usage of the `--dev` option is deprecated. Use `--include=dev` instead.') - } - - if (where === globalTop && !args.length) { - args = ['.'] - } - args = args.filter(a => path.resolve(a) !== npm.prefix) - - const arb = new Arborist({ - ...this.flatOptions, - path: where, - }) - - // TODO: - // - audit - // - funding - // - more logging (archy-ize the tree for silly logging) - // - global installs in Arborist - - const opt = { - ...this.flatOptions, - add: args, - } - arb[dryRun ? 'buildIdealTree' : 'reify'](opt).then(tree => { - output('TREEEEEEEE', tree) - cb() - }, er => cb(er)) -} +module.exports = Object.assign(install, { usage, completion }) diff --git a/lib/link.js b/lib/link.js index e05526c40..ddf3e6da7 100644 --- a/lib/link.js +++ b/lib/link.js @@ -1,47 +1,28 @@ // link with no args: symlink the folder to the global location // link with package arg: symlink the global to the local -var npm = require('./npm.js') -var symlink = require('./utils/link.js') -var fs = require('graceful-fs') -var log = require('npmlog') -var asyncMap = require('slide').asyncMap -var chain = require('slide').chain -var path = require('path') -var build = require('./build.js') -var npa = require('npm-package-arg') -var usage = require('./utils/usage') -var output = require('./utils/output.js') - -module.exports = link +const npm = require('./npm.js') +const usageUtil = require('./utils/usage.js') +const reifyOutput = require('./utils/reify-output.js') +const log = require('npmlog') +const { resolve } = require('path') +const Arborist = require('@npmcli/arborist') + +const completion = (opts, cb) => { + const { readdir } = require('fs') + const dir = npm.globalDir + readdir(dir, (er, files) => cb(er, files.filter(f => !/^[._-]/.test(f)))) +} -link.usage = usage( +const usage = usageUtil( 'link', 'npm link (in package dir)' + '\nnpm link [<@scope>/]<pkg>[@<version>]' ) -link.completion = function (opts, cb) { - var dir = npm.globalDir - fs.readdir(dir, function (er, files) { - cb(er, files.filter(function (f) { - return !f.match(/^[._-]/) - })) - }) -} - -function link (args, cb) { - if (process.platform === 'win32') { - var semver = require('semver') - if (!semver.gte(process.version, '0.7.9')) { - var msg = 'npm link not supported on windows prior to node 0.7.9' - var e = new Error(msg) - e.code = 'ENOTSUP' - e.errno = require('constants').ENOTSUP // eslint-disable-line node/no-deprecated-api - return cb(e) - } - } +const cmd = (args, cb) => link(args).then(() => cb()).catch(cb) +const link = async args => { if (npm.config.get('global')) { return cb(new Error( 'link should never be --global.\n' + @@ -49,149 +30,39 @@ function link (args, cb) { )) } - if (args.length === 1 && args[0] === '.') args = [] - if (args.length) return linkInstall(args, cb) - linkPkg(npm.prefix, cb) + args = args.filter(a => resolve(a) !== npm.prefix) + return args.length ? linkInstall(args) : linkPkg() } -function parentFolder (id, folder) { - if (id[0] === '@') { - return path.resolve(folder, '..', '..') - } else { - return path.resolve(folder, '..') - } -} - -function linkInstall (pkgs, cb) { - asyncMap(pkgs, function (pkg, cb) { - var t = path.resolve(npm.globalDir, '..') - var pp = path.resolve(npm.globalDir, pkg) - var rp = null - var target = path.resolve(npm.dir, pkg) - - function n (er, data) { - if (er) return cb(er, data) - // we want the ONE thing that was installed into the global dir - var installed = data.filter(function (info) { - var id = info[0] - var folder = info[1] - return parentFolder(id, folder) === npm.globalDir - }) - var id = installed[0][0] - pp = installed[0][1] - var what = npa(id) - pkg = what.name - target = path.resolve(npm.dir, pkg) - next() - } - - // if it's a folder, a random not-installed thing, or not a scoped package, - // then link or install it first - if (pkg[0] !== '@' && (pkg.indexOf('/') !== -1 || pkg.indexOf('\\') !== -1)) { - return fs.lstat(path.resolve(pkg), function (er, st) { - if (er || !st.isDirectory()) { - npm.commands.install(t, pkg, n) - } else { - rp = path.resolve(pkg) - linkPkg(rp, n) - } - }) - } - - fs.lstat(pp, function (er, st) { - if (er) { - rp = pp - return npm.commands.install(t, [pkg], n) - } else if (!st.isSymbolicLink()) { - rp = pp - next() - } else { - return fs.realpath(pp, function (er, real) { - if (er) log.warn('invalid symbolic link', pkg) - else rp = real - next() - }) - } - }) - - function next () { - if (npm.config.get('dry-run')) return resultPrinter(pkg, pp, target, rp, cb) - chain( - [ - [ function (cb) { - log.verbose('link', 'symlinking %s to %s', pp, target) - cb() - } ], - [symlink, pp, target, false, false], - // do not run any scripts - rp && [build, [target], npm.config.get('global'), build._noLC, true], - [resultPrinter, pkg, pp, target, rp] - ], - cb - ) - } - }, cb) -} - -function linkPkg (folder, cb_) { - var me = folder || npm.prefix - var readJson = require('read-package-json') +const linkInstall = async args => { + // add all the args as global installs, and then add symlink installs locally + // to the packages in the global space. + const globalArb = new Arborist({ + ...npm.flatOptions, + path: resolve(npm.globalDir, '..'), + global: true + }) - log.verbose('linkPkg', folder) + const globals = await globalArb.reify({ add: args }) - readJson(path.resolve(me, 'package.json'), function (er, d) { - function cb (er) { - return cb_(er, [[d && d._id, target, null, null]]) - } - if (er) return cb(er) - if (!d.name) { - er = new Error('Package must have a name field to be linked') - return cb(er) - } - var target = path.resolve(npm.globalDir, d.name) - if (npm.config.get('dry-run')) return resultPrinter(path.basename(me), me, target, cb) - symlink(me, target, false, true, function (er) { - if (er) return cb(er) - log.verbose('link', 'build target', target) - // also install missing dependencies. - npm.commands.install(me, [], function (er) { - if (er) return cb(er) - // build the global stuff. Don't run *any* scripts, because - // install command already will have done that. - build([target], true, build._noLC, true, function (er) { - if (er) return cb(er) - resultPrinter(path.basename(me), me, target, cb) - }) - }) - }) + const links = globals.edgesOut.keys() + const localArb = new Arborist({ + ...npm.flatOptions, + path: npm.prefix + }) + await localArb.reify({ + add: links.map(l => `file:${resolve(globalTop, 'node_modules', l)}`) }) -} -function resultPrinter (pkg, src, dest, rp, cb) { - if (typeof cb !== 'function') { - cb = rp - rp = null - } - var where = dest - rp = (rp || '').trim() - src = (src || '').trim() - // XXX If --json is set, then look up the data from the package.json - if (npm.config.get('parseable')) { - return parseableOutput(dest, rp || src, cb) - } - if (rp === src) rp = null - output(where + ' -> ' + src + (rp ? ' -> ' + rp : '')) - cb() + reifyOutput(localArb) } -function parseableOutput (dest, rp, cb) { - // XXX this should match ls --parseable and install --parseable - // look up the data from package.json, format it the same way. - // - // link is always effectively 'long', since it doesn't help much to - // *just* print the target folder. - // However, we don't actually ever read the version number, so - // the second field is always blank. - output(dest + '::' + rp) - cb() +const linkPkg = async () => { + const arb = new Arborist({ + ...npm.flatOptions, + path: resolve(npm.globalDir, '..'), + global: true + }) + await arb.reify({ add: [`file:${npm.prefix}`] }) + reifyOutput(arb) } diff --git a/lib/npm.js b/lib/npm.js index b16ee36ff..3f4a06c5e 100644 --- a/lib/npm.js +++ b/lib/npm.js @@ -22,6 +22,7 @@ var EventEmitter = require('events').EventEmitter var npm = module.exports = new EventEmitter() + npm.started = Date.now() var npmconf = require('./config/core.js') var log = require('npmlog') var inspect = require('util').inspect @@ -263,6 +264,7 @@ ua = ua.replace(/\{platform\}/gi, process.platform) ua = ua.replace(/\{arch\}/gi, process.arch) + // XXX replace with @npmcli/detect-ci module // continuous integration platforms const ciName = process.env.GERRIT_PROJECT ? 'gerrit' : process.env.GITLAB_CI ? 'gitlab' diff --git a/lib/prune.js b/lib/prune.js index 62c9a9225..6e1cd9318 100644 --- a/lib/prune.js +++ b/lib/prune.js @@ -1,69 +1,20 @@ -// XXX replace this with @npmcli/arborist // prune extraneous packages. +const util = require('util') +const Arborist = require('@npmcli/arborist') +const rimraf = util.promisify(require('rimraf')) +const reifyOutput = require('./utils/reify-output.js') +const usageUtil = require('./utils/usage.js') -module.exports = prune -module.exports.Pruner = Pruner +const usage = usageUtil('prune', + 'npm prune [[<@scope>/]<pkg>...] [--production]') -prune.usage = 'npm prune [[<@scope>/]<pkg>...] [--production]' +const completion = require('./utils/completion/installed-deep.js') -var npm = require('./npm.js') -var log = require('npmlog') -var util = require('util') -var moduleName = require('./utils/module-name.js') -var Installer = require('./install.js').Installer -var isExtraneous = require('./install/is-extraneous.js') -var isOnlyDev = require('./install/is-only-dev.js') -var removeDeps = require('./install/deps.js').removeDeps -var loadExtraneous = require('./install/deps.js').loadExtraneous -var chain = require('slide').chain -var computeMetadata = require('./install/deps.js').computeMetadata +const cmd = (args, cb) => prune(args).then(() => cb()).catch(cb) -prune.completion = require('./utils/completion/installed-deep.js') - -function prune (args, cb) { - var dryrun = !!npm.config.get('dry-run') - new Pruner('.', dryrun, args).run(cb) -} - -function Pruner (where, dryrun, args) { - Installer.call(this, where, dryrun, args) - this.autoPrune = true -} -util.inherits(Pruner, class {}) // Installer) - -Pruner.prototype.loadAllDepsIntoIdealTree = function (cb) { - log.silly('uninstall', 'loadAllDepsIntoIdealTree') - - var cg = this.progress['loadIdealTree:loadAllDepsIntoIdealTree'] - var steps = [] - - computeMetadata(this.idealTree) - var self = this - var excludeDev = npm.config.get('production') || /^prod(uction)?$/.test(npm.config.get('only')) - function shouldPrune (child) { - if (isExtraneous(child)) return true - if (!excludeDev) return false - return isOnlyDev(child) - } - function getModuleName (child) { - // wrapping because moduleName doesn't like extra args and we're called - // from map. - return moduleName(child) - } - function matchesArg (name) { - return self.args.length === 0 || self.args.indexOf(name) !== -1 - } - function nameObj (name) { - return {name: name} - } - var toPrune = this.idealTree.children.filter(shouldPrune).map(getModuleName).filter(matchesArg).map(nameObj) - - steps.push( - [removeDeps, toPrune, this.idealTree, null], - [loadExtraneous, this.idealTree, cg.newGroup('loadExtraneous')]) - chain(steps, cb) +const prune = async args => { + require('npmlog').warn('coming soon!') + throw new Error('not yet implemented') } -Pruner.prototype.runPreinstallTopLevelLifecycles = function (cb) { cb() } -Pruner.prototype.runPostinstallTopLevelLifecycles = function (cb) { cb() } -Pruner.prototype.saveToDependencies = function (cb) { cb() } +module.exports = Object.assign(cmd, { usage, completion }) diff --git a/lib/uninstall.js b/lib/uninstall.js index 1c7a89d32..c4275de49 100644 --- a/lib/uninstall.js +++ b/lib/uninstall.js @@ -1,80 +1,45 @@ -'use strict' -// XXX replace this with @npmcli/arborist // remove a package. -module.exports = uninstall - -const path = require('path') -const validate = require('aproba') -const readJson = require('read-package-json') -const iferr = require('iferr') +const Arborist = require('@npmcli/arborist') const npm = require('./npm.js') -const Installer = require('./install.js').Installer -const getSaveType = require('./install/save.js').getSaveType -const removeDeps = require('./install/deps.js').removeDeps -const log = require('npmlog') -const usage = require('./utils/usage') - -uninstall.usage = usage( - 'uninstall', - 'npm uninstall [<@scope>/]<pkg>[@<version>]... [--save-prod|--save-dev|--save-optional] [--no-save]' -) +const rpj = require('read-package-json-fast') +const { resolve } = require('path') +const usageUtil = require('./utils/usage.js') +const reifyOutput = require('./utils/reify-output.js') -uninstall.completion = require('./utils/completion/installed-shallow.js') +const cmd = (args, cb) => rm(args).then(() => cb()).catch(cb) -function uninstall (args, cb) { - validate('AF', arguments) +const rm = async args => { // the /path/to/node_modules/.. - const dryrun = !!npm.config.get('dry-run') - - if (args.length === 1 && args[0] === '.') args = [] + const { dryRun, global, prefix } = npm.flatOptions + const path = global ? resolve(npm.globalDir, '..') : prefix + + if (!args.length) { + if (!global) { + throw new Error('must provide a package name to remove') + } else { + const pkg = await rpj(resolve(npm.localPrefix, 'package.json')) + .catch(er => { + throw er.code !== 'ENOENT' && er.code !== 'ENOTDIR' ? er : usage() + }) + args.push(pkg.name) + } + } - const where = npm.config.get('global') || !args.length - ? path.resolve(npm.globalDir, '..') - : npm.prefix + const arb = new Arborist({ ...npm.flatOptions, path }) - args = args.filter(function (a) { - return path.resolve(a) !== where + const tree = await arb.reify({ + ...npm.flatOptions, + rm: args, }) - - if (args.length) { - new Uninstaller(where, dryrun, args).run(cb) - } else { - // remove this package from the global space, if it's installed there - readJson(path.resolve(npm.localPrefix, 'package.json'), function (er, pkg) { - if (er && er.code !== 'ENOENT' && er.code !== 'ENOTDIR') return cb(er) - if (er) return cb(uninstall.usage) - new Uninstaller(where, dryrun, [pkg.name]).run(cb) - }) - } + reifyOutput(arb) } -class Uninstaller extends (class {}) { - constructor (where, dryrun, args) { - super(where, dryrun, args) - this.remove = [] - } - - loadArgMetadata (next) { - this.args = this.args.map(function (arg) { return {name: arg} }) - next() - } - - loadAllDepsIntoIdealTree (cb) { - validate('F', arguments) - this.remove = this.args - this.args = [] - log.silly('uninstall', 'loadAllDepsIntoIdealTree') - const saveDeps = getSaveType() - - super.loadAllDepsIntoIdealTree(iferr(cb, () => { - removeDeps(this.remove, this.idealTree, saveDeps, cb) - })) - } +const usage = usageUtil( + 'uninstall', + 'npm uninstall [<@scope>/]<pkg>[@<version>]... [--save-prod|--save-dev|--save-optional] [--no-save]' +) - // no top level lifecycles on rm - runPreinstallTopLevelLifecycles (cb) { cb() } - runPostinstallTopLevelLifecycles (cb) { cb() } -} +const completion = require('./utils/completion/installed-shallow.js') -module.exports.Uninstaller = Uninstaller +module.exports = Object.assign(cmd, { usage, completion }) diff --git a/lib/update.js b/lib/update.js index aa8318bfd..63df7b85a 100644 --- a/lib/update.js +++ b/lib/update.js @@ -2,9 +2,10 @@ const Arborist = require('@npmcli/arborist') +const log = require('npmlog') const npm = require('./npm.js') const usage = require('./utils/usage.js') -const output = require('./utils/output.js') +const reifyOutput = require('./utils/reify-output.js') cmd.usage = usage( 'update', @@ -18,23 +19,22 @@ function cmd(args, cb) { .catch(cb) } -async function update (args) { +const update = async args => { const update = args.length === 0 ? true : args const where = npm.flatOptions.global ? globalTop : npm.prefix - - const arb = new Arborist({ - ...npm.flatOptions, - path: where + + if (npm.flatOptions.depth !== Infinity) { + log.warn('update', 'The --depth option no longer has any effect. See RFC0019.\n' + + 'https://github.com/npm/rfcs/blob/latest/accepted/0019-remove-update-depth-option.md') + } + + const arb = new Arborist({ + ...npm.flatOptions, + path: where }) - - const start = Date.now() - await arb.reify({ update }) - const stop = Date.now() - const time = (stop - start) / 1000 - const pkgCount = arb.diff.children.length - const added = `updated ${pkgCount}` - output(`${added} packages in ${time}s`) + await arb.reify({ update }) + reifyOutput(arb) } diff --git a/lib/utils/reify-output.js b/lib/utils/reify-output.js new file mode 100644 index 000000000..5fd948662 --- /dev/null +++ b/lib/utils/reify-output.js @@ -0,0 +1,136 @@ +// pass in an arborist object, and it'll output the data about what +// was done, what was audited, etc. +// +// added 351 packages, removed 132 packages, and audited 13388 packages in 19.157s +// +// 1 package is looking for funding +// run `npm fund` for details +// +// found 37 vulnerabilities (5 low, 7 moderate, 25 high) +// run `npm audit fix` to fix them, or `npm audit` for details + +const npm = require('../npm.js') +const log = require('npmlog') +const output = log.level === 'silent' ? () => {} : require('./output.js') +const { depth } = require('treeverse') +const ms = require('ms') +const auditReport = require('npm-audit-report') + +// TODO: output JSON if flatOptions.json is true +const reifyOutput = arb => { + const {diff, auditReport, actualTree} = arb + + const summary = { + added: 0, + removed: 0, + changed: 0, + audited: auditReport ? actualTree.inventory.size : 0, + fund: 0 + } + + depth({ + tree: diff, + visit: d => { + switch (d.action) { + case 'REMOVE': + summary.removed ++ + break + case 'ADD': + summary.added ++ + break + case 'CHANGE': + summary.changed ++ + break + default: + return + } + const node = d.actual || d.ideal + log.silly(d.action, node.location) + }, + getChildren: d => d.children + }) + + for (const node of actualTree.inventory.filter(n => n.package.funding)) { + summary.fund ++ + } + + if (npm.flatOptions.json) { + if (arb.auditReport) { + summary.audit = npm.command === 'audit' ? arb.auditReport + : arb.auditReport.toJSON().metadata + } + output(JSON.stringify(summary, 0, 2)) + } else { + packagesChangedMessage(summary) + packagesFundingMessage(summary) + printAuditReport(arb) + } +} + +// if we're running `npm audit fix`, then we print the full audit report +// at the end if there's still stuff, because it's silly for `npm audit` +// to tell you to run `npm audit` for details. otherwise, use the summary +// report. if we get here, we know it's not quiet or json. +const printAuditReport = arb => { + const reporter = npm.command !== 'audit' ? 'install' : 'detail' + const res = auditReport(arb.auditReport, { + reporter, + ...npm.flatOptions + }) + process.exitCode = process.exitCode || res.exitCode + output('\n' + res.report) +} + +const packagesChangedMessage = ({ added, removed, changed, audited }) => { + const msg = ['\n'] + if (added === 0 && removed === 0 && changed === 0) { + msg.push('up to date') + if (audited) { + msg.push(', ') + } + } else { + if (added) { + msg.push(`added ${added} package${ added === 1 ? '' : 's' }`) + } + if (removed) { + if (added) { + msg.push(', ') + } + if (!audited && !changed) { + msg.push('and ') + } + msg.push(`removed ${removed} package${ removed === 1 ? '' : 's' }`) + } + if (changed) { + if (added || removed) { + msg.push(', ') + } + if (!audited) { + msg.push('and ') + } + msg.push(`changed ${changed} package${ changed === 1 ? '' : 's' }`) + } + if (audited) { + msg.push(', and ') + } + } + if (audited) { + msg.push(`audited ${audited} package${ audited === 1 ? '' : 's' }`) + } + msg.push(` in ${ms(Date.now() - npm.started)}`) + output(msg.join('')) +} + +const packagesFundingMessage = ({ fund }) => { + if (!fund) { + return + } + + output('') + const pkg = fund === 1 ? 'package' : 'packages' + const is = fund === 1 ? 'is' : 'are' + output(`${fund} ${pkg} ${is} looking for funding`) + output(' run `npm fund` for details') +} + +module.exports = reifyOutput |