Bring in all libnpm modules + arborist as workspaces (#4166)

Added libnpm workspaces and arborist

14 files changed, 516 insertions, 0 deletions

diff --git a/workspaces/arborist/bin/actual.js b/workspaces/arborist/bin/actual.js
new file mode 100644
index 000000000..eb0495997
--- /dev/null
+++ b/workspaces/arborist/bin/actual.js
@@ -0,0 +1,23 @@
+const Arborist = require('../')
+const print = require('./lib/print-tree.js')
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const start = process.hrtime()
+new Arborist(options).loadActual(options).then(tree => {
+  const end = process.hrtime(start)
+  if (!process.argv.includes('--quiet')) {
+    print(tree)
+  }
+
+  console.error(`read ${tree.inventory.size} deps in ${end[0] * 1000 + end[1] / 1e6}ms`)
+  if (options.save) {
+    tree.meta.save()
+  }
+  if (options.saveHidden) {
+    tree.meta.hiddenLockfile = true
+    tree.meta.filename = options.path + '/node_modules/.package-lock.json'
+    tree.meta.save()
+  }
+}).catch(er => console.error(er))
diff --git a/workspaces/arborist/bin/audit.js b/workspaces/arborist/bin/audit.js
new file mode 100644
index 000000000..d9ac532d3
--- /dev/null
+++ b/workspaces/arborist/bin/audit.js
@@ -0,0 +1,54 @@
+const Arborist = require('../')
+
+const print = require('./lib/print-tree.js')
+const options = require('./lib/options.js')
+require('./lib/timers.js')
+require('./lib/logging.js')
+
+const Vuln = require('../lib/vuln.js')
+const printReport = report => {
+  for (const vuln of report.values()) {
+    console.log(printVuln(vuln))
+  }
+  if (report.topVulns.size) {
+    console.log('\n# top-level vulnerabilities')
+    for (const vuln of report.topVulns.values()) {
+      console.log(printVuln(vuln))
+    }
+  }
+}
+
+const printVuln = vuln => {
+  return {
+    __proto__: { constructor: Vuln },
+    name: vuln.name,
+    issues: [...vuln.advisories].map(a => printAdvisory(a)),
+    range: vuln.simpleRange,
+    nodes: [...vuln.nodes].map(node => `${node.name} ${node.location || '#ROOT'}`),
+    ...(vuln.topNodes.size === 0 ? {} : {
+      topNodes: [...vuln.topNodes].map(node => `${node.location || '#ROOT'}`),
+    }),
+  }
+}
+
+const printAdvisory = a => `${a.title}${a.url ? ' ' + a.url : ''}`
+
+const start = process.hrtime()
+process.emit('time', 'audit script')
+const arb = new Arborist(options)
+arb.audit(options).then(tree => {
+  process.emit('timeEnd', 'audit script')
+  const end = process.hrtime(start)
+  if (options.fix) {
+    print(tree)
+  }
+  if (!options.quiet) {
+    printReport(arb.auditReport)
+  }
+  if (options.fix) {
+    console.error(`resolved ${tree.inventory.size} deps in ${end[0] + end[1] / 1e9}s`)
+  }
+  if (tree.meta && options.save) {
+    tree.meta.save()
+  }
+}).catch(er => console.error(er))
diff --git a/workspaces/arborist/bin/funding.js b/workspaces/arborist/bin/funding.js
new file mode 100644
index 000000000..d0f4f3165
--- /dev/null
+++ b/workspaces/arborist/bin/funding.js
@@ -0,0 +1,34 @@
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const Arborist = require('../')
+const a = new Arborist(options)
+const query = options._.shift()
+const start = process.hrtime()
+a.loadVirtual().then(tree => {
+  // only load the actual tree if the virtual one doesn't have modern metadata
+  if (!tree.meta || !(tree.meta.originalLockfileVersion >= 2)) {
+    console.error('old metadata, load actual')
+    throw 'load actual'
+  } else {
+    console.error('meta ok, return virtual tree')
+    return tree
+  }
+}).catch(() => a.loadActual()).then(tree => {
+  const end = process.hrtime(start)
+  if (!query) {
+    for (const node of tree.inventory.values()) {
+      if (node.package.funding) {
+        console.log(node.name, node.location, node.package.funding)
+      }
+    }
+  } else {
+    for (const node of tree.inventory.query('name', query)) {
+      if (node.package.funding) {
+        console.log(node.name, node.location, node.package.funding)
+      }
+    }
+  }
+  console.error(`read ${tree.inventory.size} deps in ${end[0] * 1000 + end[1] / 1e6}ms`)
+})
diff --git a/workspaces/arborist/bin/ideal.js b/workspaces/arborist/bin/ideal.js
new file mode 100644
index 000000000..5d1ed0dcd
--- /dev/null
+++ b/workspaces/arborist/bin/ideal.js
@@ -0,0 +1,21 @@
+const Arborist = require('../')
+
+const { inspect } = require('util')
+const options = require('./lib/options.js')
+const print = require('./lib/print-tree.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const start = process.hrtime()
+new Arborist(options).buildIdealTree(options).then(tree => {
+  const end = process.hrtime(start)
+  print(tree)
+  console.error(`resolved ${tree.inventory.size} deps in ${end[0] + end[1] / 10e9}s`)
+  if (tree.meta && options.save) {
+    tree.meta.save()
+  }
+}).catch(er => {
+  const opt = { depth: Infinity, color: true }
+  console.error(er.code === 'ERESOLVE' ? inspect(er, opt) : er)
+  process.exitCode = 1
+})
diff --git a/workspaces/arborist/bin/index.js b/workspaces/arborist/bin/index.js
new file mode 100755
index 000000000..5449a50e6
--- /dev/null
+++ b/workspaces/arborist/bin/index.js
@@ -0,0 +1,81 @@
+#!/usr/bin/env node
+const [cmd] = process.argv.splice(2, 1)
+
+const usage = () => `Arborist - the npm tree doctor
+
+Version: ${require('../package.json').version}
+
+# USAGE
+  arborist <cmd> [path] [options...]
+
+# COMMANDS
+
+* reify: reify ideal tree to node_modules (install, update, rm, ...)
+* prune: prune the ideal tree and reify (like npm prune)
+* ideal: generate and print the ideal tree
+* actual: read and print the actual tree in node_modules
+* virtual: read and print the virtual tree in the local shrinkwrap file
+* shrinkwrap: load a local shrinkwrap and print its data
+* audit: perform a security audit on project dependencies
+* funding: query funding information in the local package tree.  A second
+  positional argument after the path name can limit to a package name.
+* license: query license information in the local package tree.  A second
+  positional argument after the path name can limit to a license type.
+* help: print this text
+
+# OPTIONS
+
+Most npm options are supported, but in camelCase rather than css-case.  For
+example, instead of '--dry-run', use '--dryRun'.
+
+Additionally:
+
+* --quiet will supppress the printing of package trees
+* Instead of 'npm install <pkg>', use 'arborist reify --add=<pkg>'.
+  The '--add=<pkg>' option can be specified multiple times.
+* Instead of 'npm rm <pkg>', use 'arborist reify --rm=<pkg>'.
+  The '--rm=<pkg>' option can be specified multiple times.
+* Instead of 'npm update', use 'arborist reify --update-all'.
+* 'npm audit fix' is 'arborist audit --fix'
+`
+
+const help = () => console.log(usage())
+
+switch (cmd) {
+  case 'actual':
+    require('./actual.js')
+    break
+  case 'virtual':
+    require('./virtual.js')
+    break
+  case 'ideal':
+    require('./ideal.js')
+    break
+  case 'prune':
+    require('./prune.js')
+    break
+  case 'reify':
+    require('./reify.js')
+    break
+  case 'audit':
+    require('./audit.js')
+    break
+  case 'funding':
+    require('./funding.js')
+    break
+  case 'license':
+    require('./license.js')
+    break
+  case 'shrinkwrap':
+    require('./shrinkwrap.js')
+    break
+  case 'help':
+  case '-h':
+  case '--help':
+    help()
+    break
+  default:
+    process.exitCode = 1
+    console.error(usage())
+    break
+}
diff --git a/workspaces/arborist/bin/lib/logging.js b/workspaces/arborist/bin/lib/logging.js
new file mode 100644
index 000000000..8183ece1f
--- /dev/null
+++ b/workspaces/arborist/bin/lib/logging.js
@@ -0,0 +1,42 @@
+const options = require('./options.js')
+const { quiet = false } = options
+const { loglevel = quiet ? 'warn' : 'silly' } = options
+
+const levels = [
+  'silly',
+  'verbose',
+  'info',
+  'timing',
+  'http',
+  'notice',
+  'warn',
+  'error',
+  'silent',
+]
+
+const levelMap = new Map(levels.reduce((set, level, index) => {
+  set.push([level, index], [index, level])
+  return set
+}, []))
+
+const { inspect, format } = require('util')
+const colors = process.stderr.isTTY
+const magenta = colors ? msg => `\x1B[35m${msg}\x1B[39m` : m => m
+if (loglevel !== 'silent') {
+  process.on('log', (level, ...args) => {
+    if (levelMap.get(level) < levelMap.get(loglevel)) {
+      return
+    }
+    const pref = `${process.pid} ${magenta(level)} `
+    if (level === 'warn' && args[0] === 'ERESOLVE') {
+      args[2] = inspect(args[2], { depth: 10, colors })
+    } else {
+      args = args.map(a => {
+        return typeof a === 'string' ? a
+          : inspect(a, { depth: 10, colors })
+      })
+    }
+    const msg = pref + format(...args).trim().split('\n').join(`\n${pref}`)
+    console.error(msg)
+  })
+}
diff --git a/workspaces/arborist/bin/lib/options.js b/workspaces/arborist/bin/lib/options.js
new file mode 100644
index 000000000..23e89ddce
--- /dev/null
+++ b/workspaces/arborist/bin/lib/options.js
@@ -0,0 +1,59 @@
+const options = module.exports = {
+  path: undefined,
+  cache: `${process.env.HOME}/.npm/_cacache`,
+  _: [],
+}
+
+for (const arg of process.argv.slice(2)) {
+  if (/^--add=/.test(arg)) {
+    options.add = options.add || []
+    options.add.push(arg.substr('--add='.length))
+  } else if (/^--rm=/.test(arg)) {
+    options.rm = options.rm || []
+    options.rm.push(arg.substr('--rm='.length))
+  } else if (arg === '--global') {
+    options.global = true
+  } else if (arg === '--global-style') {
+    options.globalStyle = true
+  } else if (arg === '--prefer-dedupe') {
+    options.preferDedupe = true
+  } else if (arg === '--legacy-peer-deps') {
+    options.legacyPeerDeps = true
+  } else if (arg === '--force') {
+    options.force = true
+  } else if (arg === '--update-all') {
+    options.update = options.update || {}
+    options.update.all = true
+  } else if (/^--update=/.test(arg)) {
+    options.update = options.update || {}
+    options.update.names = options.update.names || []
+    options.update.names.push(arg.substr('--update='.length))
+  } else if (/^--omit=/.test(arg)) {
+    options.omit = options.omit || []
+    options.omit.push(arg.substr('--omit='.length))
+  } else if (/^--before=/.test(arg)) {
+    options.before = new Date(arg.substr('--before='.length))
+  } else if (/^-w.+/.test(arg)) {
+    options.workspaces = options.workspaces || []
+    options.workspaces.push(arg.replace(/^-w/, ''))
+  } else if (/^--workspace=/.test(arg)) {
+    options.workspaces = options.workspaces || []
+    options.workspaces.push(arg.replace(/^--workspace=/, ''))
+  } else if (/^--[^=]+=/.test(arg)) {
+    const [key, ...v] = arg.replace(/^--/, '').split('=')
+    const val = v.join('=')
+    options[key] = val === 'false' ? false : val === 'true' ? true : val
+  } else if (/^--.+/.test(arg)) {
+    options[arg.replace(/^--/, '')] = true
+  } else if (options.path === undefined) {
+    options.path = arg
+  } else {
+    options._.push(arg)
+  }
+}
+
+if (options.path === undefined) {
+  options.path = '.'
+}
+
+console.error(options)
diff --git a/workspaces/arborist/bin/lib/print-tree.js b/workspaces/arborist/bin/lib/print-tree.js
new file mode 100644
index 000000000..1ea2a7218
--- /dev/null
+++ b/workspaces/arborist/bin/lib/print-tree.js
@@ -0,0 +1,5 @@
+const { inspect } = require('util')
+const { quiet } = require('./options.js')
+
+module.exports = quiet ? () => {}
+  : tree => console.log(inspect(tree.toJSON(), { depth: Infinity }))
diff --git a/workspaces/arborist/bin/lib/timers.js b/workspaces/arborist/bin/lib/timers.js
new file mode 100644
index 000000000..242431980
--- /dev/null
+++ b/workspaces/arborist/bin/lib/timers.js
@@ -0,0 +1,31 @@
+const timers = Object.create(null)
+const { format } = require('util')
+const options = require('./options.js')
+
+process.on('time', name => {
+  if (timers[name]) {
+    throw new Error('conflicting timer! ' + name)
+  }
+  timers[name] = process.hrtime()
+})
+
+const dim = process.stderr.isTTY ? msg => `\x1B[2m${msg}\x1B[22m` : m => m
+const red = process.stderr.isTTY ? msg => `\x1B[31m${msg}\x1B[39m` : m => m
+process.on('timeEnd', name => {
+  if (!timers[name]) {
+    throw new Error('timer not started! ' + name)
+  }
+  const res = process.hrtime(timers[name])
+  delete timers[name]
+  const msg = format(`${process.pid} ${name}`, res[0] * 1e3 + res[1] / 1e6)
+  if (options.timers !== false) {
+    console.error(dim(msg))
+  }
+})
+
+process.on('exit', () => {
+  for (const name of Object.keys(timers)) {
+    console.error(red('Dangling timer:'), name)
+    process.exitCode = 1
+  }
+})
diff --git a/workspaces/arborist/bin/license.js b/workspaces/arborist/bin/license.js
new file mode 100644
index 000000000..7fc08dd83
--- /dev/null
+++ b/workspaces/arborist/bin/license.js
@@ -0,0 +1,38 @@
+const Arborist = require('../')
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const a = new Arborist(options)
+const query = options._.shift()
+
+a.loadVirtual().then(tree => {
+  // only load the actual tree if the virtual one doesn't have modern metadata
+  if (!tree.meta || !(tree.meta.originalLockfileVersion >= 2)) {
+    throw 'load actual'
+  } else {
+    return tree
+  }
+}).catch((er) => {
+  console.error('loading actual tree', er)
+  return a.loadActual()
+}).then(tree => {
+  if (!query) {
+    const set = []
+    for (const license of tree.inventory.query('license')) {
+      set.push([tree.inventory.query('license', license).size, license])
+    }
+
+    for (const [count, license] of set.sort((a, b) =>
+      a[1] && b[1] ? b[0] - a[0] || a[1].localeCompare(b[1], 'en')
+      : a[1] ? -1
+      : b[1] ? 1
+      : 0)) {
+      console.log(count, license)
+    }
+  } else {
+    for (const node of tree.inventory.query('license', query === 'undefined' ? undefined : query)) {
+      console.log(`${node.name} ${node.location} ${node.package.description || ''}`)
+    }
+  }
+})
diff --git a/workspaces/arborist/bin/prune.js b/workspaces/arborist/bin/prune.js
new file mode 100644
index 000000000..e11858c20
--- /dev/null
+++ b/workspaces/arborist/bin/prune.js
@@ -0,0 +1,49 @@
+const Arborist = require('../')
+
+const options = require('./lib/options.js')
+const print = require('./lib/print-tree.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const printDiff = diff => {
+  const { depth } = require('treeverse')
+  depth({
+    tree: diff,
+    visit: d => {
+      if (d.location === '') {
+        return
+      }
+      switch (d.action) {
+        case 'REMOVE':
+          console.error('REMOVE', d.actual.location)
+          break
+        case 'ADD':
+          console.error('ADD', d.ideal.location, d.ideal.resolved)
+          break
+        case 'CHANGE':
+          console.error('CHANGE', d.actual.location, {
+            from: d.actual.resolved,
+            to: d.ideal.resolved,
+          })
+          break
+      }
+    },
+    getChildren: d => d.children,
+  })
+}
+
+const start = process.hrtime()
+process.emit('time', 'install')
+const arb = new Arborist(options)
+arb.prune(options).then(tree => {
+  process.emit('timeEnd', 'install')
+  const end = process.hrtime(start)
+  print(tree)
+  if (options.dryRun) {
+    printDiff(arb.diff)
+  }
+  console.error(`resolved ${tree.inventory.size} deps in ${end[0] + end[1] / 1e9}s`)
+  if (tree.meta && options.save) {
+    tree.meta.save()
+  }
+}).catch(er => console.error(require('util').inspect(er, { depth: Infinity })))
diff --git a/workspaces/arborist/bin/reify.js b/workspaces/arborist/bin/reify.js
new file mode 100644
index 000000000..9dc26d2b9
--- /dev/null
+++ b/workspaces/arborist/bin/reify.js
@@ -0,0 +1,49 @@
+const Arborist = require('../')
+
+const options = require('./lib/options.js')
+const print = require('./lib/print-tree.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const printDiff = diff => {
+  const { depth } = require('treeverse')
+  depth({
+    tree: diff,
+    visit: d => {
+      if (d.location === '') {
+        return
+      }
+      switch (d.action) {
+        case 'REMOVE':
+          console.error('REMOVE', d.actual.location)
+          break
+        case 'ADD':
+          console.error('ADD', d.ideal.location, d.ideal.resolved)
+          break
+        case 'CHANGE':
+          console.error('CHANGE', d.actual.location, {
+            from: d.actual.resolved,
+            to: d.ideal.resolved,
+          })
+          break
+      }
+    },
+    getChildren: d => d.children,
+  })
+}
+
+const start = process.hrtime()
+process.emit('time', 'install')
+const arb = new Arborist(options)
+arb.reify(options).then(tree => {
+  process.emit('timeEnd', 'install')
+  const end = process.hrtime(start)
+  print(tree)
+  if (options.dryRun) {
+    printDiff(arb.diff)
+  }
+  console.error(`resolved ${tree.inventory.size} deps in ${end[0] + end[1] / 1e9}s`)
+  if (tree.meta && options.save) {
+    tree.meta.save()
+  }
+}).catch(er => console.error(require('util').inspect(er, { depth: Infinity })))
diff --git a/workspaces/arborist/bin/shrinkwrap.js b/workspaces/arborist/bin/shrinkwrap.js
new file mode 100644
index 000000000..b40416b7b
--- /dev/null
+++ b/workspaces/arborist/bin/shrinkwrap.js
@@ -0,0 +1,12 @@
+const Shrinkwrap = require('../lib/shrinkwrap.js')
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const { quiet } = options
+Shrinkwrap.load(options)
+  .then(s => quiet || console.log(JSON.stringify(s.commit(), 0, 2)))
+  .catch(er => {
+    console.error('shrinkwrap load failure', er)
+    process.exit(1)
+  })
diff --git a/workspaces/arborist/bin/virtual.js b/workspaces/arborist/bin/virtual.js
new file mode 100644
index 000000000..457c945e7
--- /dev/null
+++ b/workspaces/arborist/bin/virtual.js
@@ -0,0 +1,18 @@
+const Arborist = require('../')
+
+const print = require('./lib/print-tree.js')
+const options = require('./lib/options.js')
+require('./lib/logging.js')
+require('./lib/timers.js')
+
+const start = process.hrtime()
+new Arborist(options).loadVirtual().then(tree => {
+  const end = process.hrtime(start)
+  if (!options.quiet) {
+    print(tree)
+  }
+  if (options.save) {
+    tree.meta.save()
+  }
+  console.error(`read ${tree.inventory.size} deps in ${end[0] * 1000 + end[1] / 1e6}ms`)
+}).catch(er => console.error(er))