1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
|
const Fetcher = require('./fetcher.js')
const FileFetcher = require('./file.js')
const RemoteFetcher = require('./remote.js')
const DirFetcher = require('./dir.js')
const hashre = /^[a-f0-9]{40}$/
const git = require('@npmcli/git')
const pickManifest = require('npm-pick-manifest')
const npa = require('npm-package-arg')
const url = require('url')
const Minipass = require('minipass')
const cacache = require('cacache')
const { promisify } = require('util')
const readPackageJson = require('read-package-json-fast')
const npm = require('./util/npm.js')
const _resolvedFromRepo = Symbol('_resolvedFromRepo')
const _resolvedFromHosted = Symbol('_resolvedFromHosted')
const _resolvedFromClone = Symbol('_resolvedFromClone')
const _tarballFromResolved = Symbol.for('pacote.Fetcher._tarballFromResolved')
const _addGitSha = Symbol('_addGitSha')
const addGitSha = require('./util/add-git-sha.js')
const _clone = Symbol('_clone')
const _cloneHosted = Symbol('_cloneHosted')
const _cloneRepo = Symbol('_cloneRepo')
const _setResolvedWithSha = Symbol('_setResolvedWithSha')
const _prepareDir = Symbol('_prepareDir')
// get the repository url.
// prefer https if there's auth, since ssh will drop that.
// otherwise, prefer ssh if available (more secure).
// We have to add the git+ back because npa suppresses it.
const repoUrl = (h, opts) =>
h.sshurl && !(h.https && h.auth) && addGitPlus(h.sshurl(opts)) ||
h.https && addGitPlus(h.https(opts))
// add git+ to the url, but only one time.
const addGitPlus = url => url && `git+${url}`.replace(/^(git\+)+/, 'git+')
class GitFetcher extends Fetcher {
constructor (spec, opts) {
super(spec, opts)
this.resolvedRef = null
if (this.spec.hosted)
this.from = this.spec.hosted.shortcut({ noCommittish: false })
// shortcut: avoid full clone when we can go straight to the tgz
// if we have the full sha and it's a hosted git platform
if (this.spec.gitCommittish && hashre.test(this.spec.gitCommittish)) {
this.resolvedSha = this.spec.gitCommittish
// use hosted.tarball() when we shell to RemoteFetcher later
this.resolved = this.spec.hosted
? repoUrl(this.spec.hosted, { noCommittish: false })
: this.spec.rawSpec
} else
this.resolvedSha = ''
}
// just exposed to make it easier to test all the combinations
static repoUrl (hosted, opts) {
return repoUrl(hosted, opts)
}
get types () {
return ['git']
}
resolve () {
// likely a hosted git repo with a sha, so get the tarball url
// but in general, no reason to resolve() more than necessary!
if (this.resolved)
return super.resolve()
// fetch the git repo and then look at the current hash
const h = this.spec.hosted
// try to use ssh, fall back to git.
return h ? this[_resolvedFromHosted](h)
: this[_resolvedFromRepo](this.spec.fetchSpec)
}
// first try https, since that's faster and passphrase-less for
// public repos, and supports private repos when auth is provided.
// Fall back to SSH to support private repos
// NB: we always store the https url in resolved field if auth
// is present, otherwise ssh if the hosted type provides it
[_resolvedFromHosted] (hosted) {
return this[_resolvedFromRepo](hosted.https && hosted.https())
.catch(er => {
// Throw early since we know pathspec errors will fail again if retried
if (er instanceof git.errors.GitPathspecError)
throw er
const ssh = hosted.sshurl && hosted.sshurl()
// no fallthrough if we can't fall through or have https auth
if (!ssh || hosted.auth)
throw er
return this[_resolvedFromRepo](ssh)
})
}
[_resolvedFromRepo] (gitRemote) {
// XXX make this a custom error class
if (!gitRemote)
return Promise.reject(new Error(`No git url for ${this.spec}`))
const gitRange = this.spec.gitRange
const name = this.spec.name
return git.revs(gitRemote, this.opts).then(remoteRefs => {
return gitRange ? pickManifest({
versions: remoteRefs.versions,
'dist-tags': remoteRefs['dist-tags'],
name,
}, gitRange, this.opts)
: this.spec.gitCommittish ?
remoteRefs.refs[this.spec.gitCommittish] ||
remoteRefs.refs[remoteRefs.shas[this.spec.gitCommittish]]
: remoteRefs.refs.HEAD // no git committish, get default head
}).then(revDoc => {
// the committish provided isn't in the rev list
// things like HEAD~3 or @yesterday can land here.
if (!revDoc || !revDoc.sha)
return this[_resolvedFromClone]()
this.resolvedRef = revDoc
this.resolvedSha = revDoc.sha
this[_addGitSha](revDoc.sha)
return this.resolved
})
}
[_setResolvedWithSha] (withSha) {
// we haven't cloned, so a tgz download is still faster
// of course, if it's not a known host, we can't do that.
this.resolved = !this.spec.hosted ? withSha
: repoUrl(npa(withSha).hosted, { noCommittish: false })
}
// when we get the git sha, we affix it to our spec to build up
// either a git url with a hash, or a tarball download URL
[_addGitSha] (sha) {
this[_setResolvedWithSha](addGitSha(this.spec, sha))
}
[_resolvedFromClone] () {
// do a full or shallow clone, then look at the HEAD
// kind of wasteful, but no other option, really
return this[_clone](dir => this.resolved)
}
[_prepareDir] (dir) {
return readPackageJson(dir + '/package.json').then(mani => {
// no need if we aren't going to do any preparation.
const scripts = mani.scripts
if (!scripts || !(
scripts.postinstall ||
scripts.build ||
scripts.preinstall ||
scripts.install ||
scripts.prepare))
return
// to avoid cases where we have an cycle of git deps that depend
// on one another, we only ever do preparation for one instance
// of a given git dep along the chain of installations.
// Note that this does mean that a dependency MAY in theory end up
// trying to run its prepare script using a dependency that has not
// been properly prepared itself, but that edge case is smaller
// and less hazardous than a fork bomb of npm and git commands.
const noPrepare = !process.env._PACOTE_NO_PREPARE_ ? []
: process.env._PACOTE_NO_PREPARE_.split('\n')
if (noPrepare.includes(this.resolved)) {
this.log.info('prepare', 'skip prepare, already seen', this.resolved)
return
}
noPrepare.push(this.resolved)
// the DirFetcher will do its own preparation to run the prepare scripts
// All we have to do is put the deps in place so that it can succeed.
return npm(
this.npmBin,
[].concat(this.npmInstallCmd).concat(this.npmCliConfig),
dir,
{ ...process.env, _PACOTE_NO_PREPARE_: noPrepare.join('\n') },
{ message: 'git dep preparation failed' }
)
})
}
[_tarballFromResolved] () {
const stream = new Minipass()
stream.resolved = this.resolved
stream.integrity = this.integrity
stream.from = this.from
// check it out and then shell out to the DirFetcher tarball packer
this[_clone](dir => this[_prepareDir](dir)
.then(() => new Promise((res, rej) => {
const df = new DirFetcher(`file:${dir}`, {
...this.opts,
resolved: null,
integrity: null,
})
const dirStream = df[_tarballFromResolved]()
dirStream.on('error', rej)
dirStream.on('end', res)
dirStream.pipe(stream)
}))).catch(
/* istanbul ignore next: very unlikely and hard to test */
er => stream.emit('error', er)
)
return stream
}
// clone a git repo into a temp folder (or fetch and unpack if possible)
// handler accepts a directory, and returns a promise that resolves
// when we're done with it, at which point, cacache deletes it
//
// TODO: after cloning, create a tarball of the folder, and add to the cache
// with cacache.put.stream(), using a key that's deterministic based on the
// spec and repo, so that we don't ever clone the same thing multiple times.
[_clone] (handler, tarballOk = true) {
const o = { tmpPrefix: 'git-clone' }
const ref = this.resolvedSha || this.spec.gitCommittish
const h = this.spec.hosted
const resolved = this.resolved
// can be set manually to false to fall back to actual git clone
tarballOk = tarballOk &&
h && resolved === repoUrl(h, { noCommittish: false }) && h.tarball
return cacache.tmp.withTmp(this.cache, o, tmp => {
// if we're resolved, and have a tarball url, shell out to RemoteFetcher
if (tarballOk) {
const nameat = this.spec.name ? `${this.spec.name}@` : ''
return new RemoteFetcher(h.tarball({ noCommittish: false }), {
...this.opts,
allowGitIgnore: true,
pkgid: `git:${nameat}${this.resolved}`,
resolved: this.resolved,
integrity: null, // it'll always be different, if we have one
}).extract(tmp).then(() => handler(tmp), er => {
// fall back to ssh download if tarball fails
if (er.constructor.name.match(/^Http/))
return this[_clone](handler, false)
else
throw er
})
}
return (
h ? this[_cloneHosted](ref, tmp)
: this[_cloneRepo](this.spec.fetchSpec, ref, tmp)
).then(sha => {
this.resolvedSha = sha
if (!this.resolved)
this[_addGitSha](sha)
})
.then(() => handler(tmp))
})
}
// first try https, since that's faster and passphrase-less for
// public repos, and supports private repos when auth is provided.
// Fall back to SSH to support private repos
// NB: we always store the https url in resolved field if auth
// is present, otherwise ssh if the hosted type provides it
[_cloneHosted] (ref, tmp) {
const hosted = this.spec.hosted
return this[_cloneRepo](hosted.https({ noCommittish: true }), ref, tmp)
.catch(er => {
// Throw early since we know pathspec errors will fail again if retried
if (er instanceof git.errors.GitPathspecError)
throw er
const ssh = hosted.sshurl && hosted.sshurl({ noCommittish: true })
// no fallthrough if we can't fall through or have https auth
if (!ssh || hosted.auth)
throw er
return this[_cloneRepo](ssh, ref, tmp)
})
}
[_cloneRepo] (repo, ref, tmp) {
const { opts, spec } = this
return git.clone(repo, ref, tmp, { ...opts, spec })
}
manifest () {
if (this.package)
return Promise.resolve(this.package)
return this.spec.hosted && this.resolved
? FileFetcher.prototype.manifest.apply(this)
: this[_clone](dir =>
readPackageJson(dir + '/package.json')
.then(mani => this.package = {
...mani,
_integrity: this.integrity && String(this.integrity),
_resolved: this.resolved,
_from: this.from,
}))
}
packument () {
return FileFetcher.prototype.packument.apply(this)
}
}
module.exports = GitFetcher
|