deps: Fix git & tagged dependency matching with shrinkwraps

When you depend on a tag, eg `foo@latest`, your `npm-shrinkwrap.json` will
record this in its `_from` field. Similarly when you depend on a git dependency
with something like `github:foo/bar` it saves this to your `_from`.

When you install under the above circumstances the installed module will be
installed from the associated tarball URL for tagged dependencies or git URL
with commit hash for git dependencies.

As a result the `_requested` section of the `package.json` will be based on
the tarball URL or githash and cann't be matched up to the `package.json`.

It was _supposed_ to fallback to comparing the `package.json` `_from` field
with but it was doing this by comparing `rawSpec` to `_from`, which _can't_
work consistently.

Fixes: #13496
Fixes: #11736
PR-URL: https://github.com/npm/npm/pull/13941
Credit: @iarna
Reviewed-By: @othiym23

2 files changed, 178 insertions, 1 deletions

diff --git a/lib/install/deps.js b/lib/install/deps.js
index 762f39675..96b88f84c 100644
--- a/lib/install/deps.js
+++ b/lib/install/deps.js
@@ -64,11 +64,26 @@ function doesChildVersionMatch (child, requested, requestor) {
   if (requested.spec === '*') return true
 
   var childReq = child.package._requested
+  if (!childReq) childReq = npa(moduleName(child) + '@' + child.package._from)
   if (childReq) {
     if (childReq.rawSpec === requested.rawSpec) return true
     if (childReq.type === requested.type && childReq.spec === requested.spec) return true
   }
-  if (!registryTypes[requested.type]) return requested.rawSpec === child.package._from
+  // If _requested didn't exist OR if it didn't match then we'll try using
+  // _from. We pass it through npa to normalize the specifier.
+  // This can happen when installing from an `npm-shrinkwrap.json` where `_requested` will
+  // be the tarball URL from `resolved` and thus can't match what's in the `package.json`.
+  // In those cases _from, will be preserved and we can compare that to ensure that they
+  // really came from the same sources.
+  // You'll see this scenario happen with at least tags and git dependencies.
+  if (!registryTypes[requested.type]) {
+    if (child.package._from) {
+      var fromReq = npa(child.package._from)
+      if (fromReq.rawSpec === requested.rawSpec) return true
+      if (fromReq.type === requested.type && fromReq.spec === requested.spec) return true
+    }
+    return false
+  }
   return semver.satisfies(child.package.version, requested.spec)
 }
 
diff --git a/test/tap/tagged-version-matching.js b/test/tap/tagged-version-matching.js
new file mode 100644
index 000000000..f7d51d90b
--- /dev/null
+++ b/test/tap/tagged-version-matching.js
@@ -0,0 +1,162 @@
+'use strict'
+var path = require('path')
+var test = require('tap').test
+var Tacks = require('tacks')
+var File = Tacks.File
+var Dir = Tacks.Dir
+var extend = Object.assign || require('util')._extend
+var common = require('../common-tap.js')
+
+var basedir = path.join(__dirname, path.basename(__filename, '.js'))
+var testdir = path.join(basedir, 'testdir')
+var cachedir = path.join(basedir, 'cache')
+var globaldir = path.join(basedir, 'global')
+var tmpdir = path.join(basedir, 'tmp')
+
+var conf = {
+  cwd: testdir,
+  env: extend({
+    npm_config_cache: cachedir,
+    npm_config_tmp: tmpdir,
+    npm_config_prefix: globaldir,
+    npm_config_registry: common.registry,
+    npm_config_loglevel: 'warn'
+  }, process.env)
+}
+
+var fixture = new Tacks(Dir({
+  cache: Dir(),
+  global: Dir(),
+  tmp: Dir(),
+  testdir: Dir({
+    node_modules: Dir({
+      example: Dir({
+        'package.json': File({
+          _from: 'example',
+          _id: 'example@1.0.0',
+          _requested: {
+            raw: 'example@file:example',
+            scope: null,
+            escapedName: 'example',
+            name: 'example',
+            rawSpec: 'file:example',
+            type: 'directory'
+          },
+          dependencies: {
+            tagdep: 'latest',
+            gitdep: 'npm/example-gitdep'
+          },
+          name: 'example',
+          version: '1.0.0'
+        })
+      }),
+      gitdep: Dir({
+        'package.json': File({
+          _from: 'npm/example-gitdep',
+          _id: 'gitdep@1.0.0',
+          _requested: {
+            raw: 'gitdep@git://github.com/npm/example-gitdep.git#da39a3ee5e6b4b0d3255bfef95601890afd80709',
+            scope: null,
+            escapedName: 'gitdep',
+            name: 'gitdep',
+            rawSpec: 'git://github.com/npm/example-gitdep.git#da39a3ee5e6b4b0d3255bfef95601890afd80709',
+            spec: 'git://github.com/npm/example-gitdep.git#da39a3ee5e6b4b0d3255bfef95601890afd80709',
+            type: 'hosted',
+            hosted: {
+              type: 'github',
+              ssh: 'git@github.com:npm/example-gitdep.git#da39a3ee5e6b4b0d3255bfef95601890afd80709',
+              sshUrl: 'git+ssh://git@github.com/npm/example-gitdep.git#da39a3ee5e6b4b0d3255bfef95601890afd80709',
+              httpsUrl: 'git+https://github.com/npm/example-gitdep.git#da39a3ee5e6b4b0d3255bfef95601890afd80709',
+              gitUrl: 'git://github.com/npm/example-gitdep.git#da39a3ee5e6b4b0d3255bfef95601890afd80709',
+              shortcut: 'github:npm/example-gitdep#da39a3ee5e6b4b0d3255bfef95601890afd80709',
+              directUrl: 'https://raw.githubusercontent.com/npm/example-gitdep/da39a3ee5e6b4b0d3255bfef95601890afd80709/package.json'
+            }
+          },
+          name: 'gitdep',
+          version: '1.0.0'
+        })
+      }),
+      tagdep: Dir({
+        'package.json': File({
+          _from: 'tagdep@latest',
+          _id: 'tagdep@1.0.0',
+          _requested: {
+            raw: 'tagdep@https://registry.example.com/tagdep/-/tagdep-1.0.0.tgz',
+            scope: null,
+            escapedName: 'tagdep',
+            name: 'tagdep',
+            rawSpec: 'https://registry.example.com/tagdep/-/tagdep-1.0.0.tgz',
+            spec: 'https://registry.example.com/tagdep/-/tagdep-1.0.0.tgz',
+            type: 'remote'
+          },
+          name: 'tagdep',
+          version: '1.0.0'
+        })
+      })
+    }),
+    'npm-shrinkwrap.json': File({
+      name: 'tagged-version-matching',
+      version: '1.0.0',
+      dependencies: {
+        tagdep: {
+          version: '1.0.0',
+          from: 'tagdep@latest',
+          resolved: 'https://registry.example.com/tagdep/-/tagdep-1.0.0.tgz'
+        },
+        example: {
+          version: '1.0.0',
+          from: 'example'
+        },
+        gitdep: {
+          version: '1.0.0',
+          from: 'npm/example-gitdep',
+          resolved: 'git://github.com/npm/example-gitdep.git#da39a3ee5e6b4b0d3255bfef95601890afd80709'
+        }
+      }
+    }),
+    'package.json': File({
+      name: 'tagged-version-matching',
+      version: '1.0.0',
+      dependencies: {
+        example: 'file:example',
+        gitdep: 'npm/example-gitdep'
+      }
+    })
+  })
+}))
+
+function setup () {
+  cleanup()
+  fixture.create(basedir)
+}
+
+function cleanup () {
+  fixture.remove(basedir)
+}
+
+test('setup', function (t) {
+  setup()
+  t.done()
+})
+
+test('tagged-version-matching', function (t) {
+  common.npm(['ls', '--json'], conf, function (err, code, stdout, stderr) {
+    if (err) throw err
+    t.is(code, 0, 'command ran ok')
+    if (stderr.trim()) t.comment(stderr.trim())
+    var result = JSON.parse(stdout.trim())
+    var problems = result.problems || []
+    // Original PR: https://github.com/npm/npm/pull/13941
+    // Original issue: https://github.com/npm/npm/issues/13496
+    // Original issue: https://github.com/npm/npm/issues/11736
+    t.is(problems.length, 0, 'no problems')
+    t.ok(!problems.some(function (err) { return /missing: tagdep/.test(err) }), 'tagged dependency matched ok')
+    t.ok(!problems.some(function (err) { return /missing: gitdep/.test(err) }), 'git dependency matched ok')
+    t.done()
+  })
+})
+
+test('cleanup', function (t) {
+  cleanup()
+  t.done()
+})