source: trip-planner-front/node_modules/pacote/lib/fetcher.js@ 76712b2

Last change on this file since 76712b2 was 6a3a178, checked in by Ema <ema_spirova@…>, 3 years ago

initial commit

  • Property mode set to 100644
File size: 17.4 KB
Line 
1// This is the base class that the other fetcher types in lib
2// all descend from.
3// It handles the unpacking and retry logic that is shared among
4// all of the other Fetcher types.
5
6const npa = require('npm-package-arg')
7const ssri = require('ssri')
8const { promisify } = require('util')
9const { basename, dirname } = require('path')
10const rimraf = promisify(require('rimraf'))
11const tar = require('tar')
12const procLog = require('./util/proc-log.js')
13const retry = require('promise-retry')
14const fsm = require('fs-minipass')
15const cacache = require('cacache')
16const isPackageBin = require('./util/is-package-bin.js')
17const getContents = require('@npmcli/installed-package-contents')
18
19// we only change ownership on unix platforms, and only if uid is 0
20const selfOwner = process.getuid && process.getuid() === 0 ? {
21 uid: 0,
22 gid: process.getgid(),
23} : null
24const chownr = selfOwner ? promisify(require('chownr')) : null
25const inferOwner = selfOwner ? require('infer-owner') : null
26const mkdirp = require('mkdirp')
27const cacheDir = require('./util/cache-dir.js')
28
29// Private methods.
30// Child classes should not have to override these.
31// Users should never call them.
32const _chown = Symbol('_chown')
33const _extract = Symbol('_extract')
34const _mkdir = Symbol('_mkdir')
35const _empty = Symbol('_empty')
36const _toFile = Symbol('_toFile')
37const _tarxOptions = Symbol('_tarxOptions')
38const _entryMode = Symbol('_entryMode')
39const _istream = Symbol('_istream')
40const _assertType = Symbol('_assertType')
41const _tarballFromCache = Symbol('_tarballFromCache')
42const _tarballFromResolved = Symbol.for('pacote.Fetcher._tarballFromResolved')
43const _cacheFetches = Symbol.for('pacote.Fetcher._cacheFetches')
44
45class FetcherBase {
46 constructor (spec, opts) {
47 if (!opts || typeof opts !== 'object')
48 throw new TypeError('options object is required')
49 this.spec = npa(spec, opts.where)
50
51 this.allowGitIgnore = !!opts.allowGitIgnore
52
53 // a bit redundant because presumably the caller already knows this,
54 // but it makes it easier to not have to keep track of the requested
55 // spec when we're dispatching thousands of these at once, and normalizing
56 // is nice. saveSpec is preferred if set, because it turns stuff like
57 // x/y#committish into github:x/y#committish. use name@rawSpec for
58 // registry deps so that we turn xyz and xyz@ -> xyz@
59 this.from = this.spec.registry
60 ? `${this.spec.name}@${this.spec.rawSpec}` : this.spec.saveSpec
61
62 this[_assertType]()
63 // clone the opts object so that others aren't upset when we mutate it
64 // by adding/modifying the integrity value.
65 this.opts = {...opts}
66
67 this.cache = opts.cache || cacheDir()
68 this.resolved = opts.resolved || null
69
70 // default to caching/verifying with sha512, that's what we usually have
71 // need to change this default, or start overriding it, when sha512
72 // is no longer strong enough.
73 this.defaultIntegrityAlgorithm = opts.defaultIntegrityAlgorithm || 'sha512'
74
75 if (typeof opts.integrity === 'string')
76 this.opts.integrity = ssri.parse(opts.integrity)
77
78 this.package = null
79 this.type = this.constructor.name
80 this.fmode = opts.fmode || 0o666
81 this.dmode = opts.dmode || 0o777
82 // we don't need a default umask, because we don't chmod files coming
83 // out of package tarballs. they're forced to have a mode that is
84 // valid, regardless of what's in the tarball entry, and then we let
85 // the process's umask setting do its job. but if configured, we do
86 // respect it.
87 this.umask = opts.umask || 0
88 this.log = opts.log || procLog
89
90 this.preferOnline = !!opts.preferOnline
91 this.preferOffline = !!opts.preferOffline
92 this.offline = !!opts.offline
93
94 this.before = opts.before
95 this.fullMetadata = this.before ? true : !!opts.fullMetadata
96
97 this.defaultTag = opts.defaultTag || 'latest'
98 this.registry = (opts.registry || 'https://registry.npmjs.org')
99 .replace(/\/+$/, '')
100
101 // command to run 'prepare' scripts on directories and git dirs
102 // To use pacote with yarn, for example, set npmBin to 'yarn'
103 // and npmCliConfig with yarn's equivalents.
104 this.npmBin = opts.npmBin || 'npm'
105
106 // command to install deps for preparing
107 this.npmInstallCmd = opts.npmInstallCmd || [ 'install', '--force' ]
108
109 // XXX fill more of this in based on what we know from this.opts
110 // we explicitly DO NOT fill in --tag, though, since we are often
111 // going to be packing in the context of a publish, which may set
112 // a dist-tag, but certainly wants to keep defaulting to latest.
113 this.npmCliConfig = opts.npmCliConfig || [
114 `--cache=${dirname(this.cache)}`,
115 `--prefer-offline=${!!this.preferOffline}`,
116 `--prefer-online=${!!this.preferOnline}`,
117 `--offline=${!!this.offline}`,
118 ...(this.before ? [`--before=${this.before.toISOString()}`] : []),
119 '--no-progress',
120 '--no-save',
121 '--no-audit',
122 // override any omit settings from the environment
123 '--include=dev',
124 '--include=peer',
125 '--include=optional',
126 // we need the actual things, not just the lockfile
127 '--no-package-lock-only',
128 '--no-dry-run',
129 ]
130 }
131
132 get integrity () {
133 return this.opts.integrity || null
134 }
135 set integrity (i) {
136 if (!i)
137 return
138
139 i = ssri.parse(i)
140 const current = this.opts.integrity
141
142 // do not ever update an existing hash value, but do
143 // merge in NEW algos and hashes that we don't already have.
144 if (current)
145 current.merge(i)
146 else
147 this.opts.integrity = i
148 }
149
150 get notImplementedError () {
151 return new Error('not implemented in this fetcher type: ' + this.type)
152 }
153
154 // override in child classes
155 // Returns a Promise that resolves to this.resolved string value
156 resolve () {
157 return this.resolved ? Promise.resolve(this.resolved)
158 : Promise.reject(this.notImplementedError)
159 }
160
161 packument () {
162 return Promise.reject(this.notImplementedError)
163 }
164
165 // override in child class
166 // returns a manifest containing:
167 // - name
168 // - version
169 // - _resolved
170 // - _integrity
171 // - plus whatever else was in there (corgi, full metadata, or pj file)
172 manifest () {
173 return Promise.reject(this.notImplementedError)
174 }
175
176 // private, should be overridden.
177 // Note that they should *not* calculate or check integrity or cache,
178 // but *just* return the raw tarball data stream.
179 [_tarballFromResolved] () {
180 throw this.notImplementedError
181 }
182
183 // public, should not be overridden
184 tarball () {
185 return this.tarballStream(stream => stream.concat().then(data => {
186 data.integrity = this.integrity && String(this.integrity)
187 data.resolved = this.resolved
188 data.from = this.from
189 return data
190 }))
191 }
192
193 // private
194 // Note: cacache will raise a EINTEGRITY error if the integrity doesn't match
195 [_tarballFromCache] () {
196 return cacache.get.stream.byDigest(this.cache, this.integrity, this.opts)
197 }
198
199 get [_cacheFetches] () {
200 return true
201 }
202
203 [_istream] (stream) {
204 // everyone will need one of these, either for verifying or calculating
205 // We always set it, because we have might only have a weak legacy hex
206 // sha1 in the packument, and this MAY upgrade it to a stronger algo.
207 // If we had an integrity, and it doesn't match, then this does not
208 // override that error; the istream will raise the error before it
209 // gets to the point of re-setting the integrity.
210 const istream = ssri.integrityStream(this.opts)
211 istream.on('integrity', i => this.integrity = i)
212 stream.on('error', er => istream.emit('error', er))
213
214 // if not caching this, just pipe through to the istream and return it
215 if (!this.opts.cache || !this[_cacheFetches])
216 return stream.pipe(istream)
217
218 // we have to return a stream that gets ALL the data, and proxies errors,
219 // but then pipe from the original tarball stream into the cache as well.
220 // To do this without losing any data, and since the cacache put stream
221 // is not a passthrough, we have to pipe from the original stream into
222 // the cache AFTER we pipe into the istream. Since the cache stream
223 // has an asynchronous flush to write its contents to disk, we need to
224 // defer the istream end until the cache stream ends.
225 stream.pipe(istream, { end: false })
226 const cstream = cacache.put.stream(
227 this.opts.cache,
228 `pacote:tarball:${this.from}`,
229 this.opts
230 )
231 stream.pipe(cstream)
232 // defer istream end until after cstream
233 // cache write errors should not crash the fetch, this is best-effort.
234 cstream.promise().catch(() => {}).then(() => istream.end())
235
236 return istream
237 }
238
239 pickIntegrityAlgorithm () {
240 return this.integrity ? this.integrity.pickAlgorithm(this.opts)
241 : this.defaultIntegrityAlgorithm
242 }
243
244 // TODO: check error class, once those are rolled out to our deps
245 isDataCorruptionError (er) {
246 return er.code === 'EINTEGRITY' || er.code === 'Z_DATA_ERROR'
247 }
248
249 // override the types getter
250 get types () {}
251 [_assertType] () {
252 if (this.types && !this.types.includes(this.spec.type)) {
253 throw new TypeError(`Wrong spec type (${
254 this.spec.type
255 }) for ${
256 this.constructor.name
257 }. Supported types: ${this.types.join(', ')}`)
258 }
259 }
260
261 // We allow ENOENTs from cacache, but not anywhere else.
262 // An ENOENT trying to read a tgz file, for example, is Right Out.
263 isRetriableError (er) {
264 // TODO: check error class, once those are rolled out to our deps
265 return this.isDataCorruptionError(er) ||
266 er.code === 'ENOENT' ||
267 er.code === 'EISDIR'
268 }
269
270 // Mostly internal, but has some uses
271 // Pass in a function which returns a promise
272 // Function will be called 1 or more times with streams that may fail.
273 // Retries:
274 // Function MUST handle errors on the stream by rejecting the promise,
275 // so that retry logic can pick it up and either retry or fail whatever
276 // promise it was making (ie, failing extraction, etc.)
277 //
278 // The return value of this method is a Promise that resolves the same
279 // as whatever the streamHandler resolves to.
280 //
281 // This should never be overridden by child classes, but it is public.
282 tarballStream (streamHandler) {
283 // Only short-circuit via cache if we have everything else we'll need,
284 // and the user has not expressed a preference for checking online.
285
286 const fromCache = (
287 !this.preferOnline &&
288 this.integrity &&
289 this.resolved
290 ) ? streamHandler(this[_tarballFromCache]()).catch(er => {
291 if (this.isDataCorruptionError(er)) {
292 this.log.warn('tarball', `cached data for ${
293 this.spec
294 } (${this.integrity}) seems to be corrupted. Refreshing cache.`)
295 return this.cleanupCached().then(() => { throw er })
296 } else {
297 throw er
298 }
299 }) : null
300
301 const fromResolved = er => {
302 if (er) {
303 if (!this.isRetriableError(er))
304 throw er
305 this.log.silly('tarball', `no local data for ${
306 this.spec
307 }. Extracting by manifest.`)
308 }
309 return this.resolve().then(() => retry(tryAgain =>
310 streamHandler(this[_istream](this[_tarballFromResolved]()))
311 .catch(er => {
312 // Most likely data integrity. A cache ENOENT error is unlikely
313 // here, since we're definitely not reading from the cache, but it
314 // IS possible that the fetch subsystem accessed the cache, and the
315 // entry got blown away or something. Try one more time to be sure.
316 if (this.isRetriableError(er)) {
317 this.log.warn('tarball', `tarball data for ${
318 this.spec
319 } (${this.integrity}) seems to be corrupted. Trying again.`)
320 return this.cleanupCached().then(() => tryAgain(er))
321 }
322 throw er
323 }), { retries: 1, minTimeout: 0, maxTimeout: 0 }))
324 }
325
326 return fromCache ? fromCache.catch(fromResolved) : fromResolved()
327 }
328
329 cleanupCached () {
330 return cacache.rm.content(this.cache, this.integrity, this.opts)
331 }
332
333 async [_chown] (path, uid, gid) {
334 return selfOwner && (selfOwner.gid !== gid || selfOwner.uid !== uid)
335 ? chownr(path, uid, gid)
336 : /* istanbul ignore next - we don't test in root-owned folders */ null
337 }
338
339 [_empty] (path) {
340 return getContents({path, depth: 1}).then(contents => Promise.all(
341 contents.map(entry => rimraf(entry))))
342 }
343
344 [_mkdir] (dest) {
345 // if we're bothering to do owner inference, then do it.
346 // otherwise just make the dir, and return an empty object.
347 // always empty the dir dir to start with, but do so
348 // _after_ inferring the owner, in case there's an existing folder
349 // there that we would want to preserve which differs from the
350 // parent folder (rare, but probably happens sometimes).
351 return !inferOwner
352 ? this[_empty](dest).then(() => mkdirp(dest)).then(() => ({}))
353 : inferOwner(dest).then(({uid, gid}) =>
354 this[_empty](dest)
355 .then(() => mkdirp(dest))
356 .then(made => {
357 // ignore the || dest part in coverage. It's there to handle
358 // race conditions where the dir may be made by someone else
359 // after being removed by us.
360 const dir = made || /* istanbul ignore next */ dest
361 return this[_chown](dir, uid, gid)
362 })
363 .then(() => ({uid, gid})))
364 }
365
366 // extraction is always the same. the only difference is where
367 // the tarball comes from.
368 extract (dest) {
369 return this[_mkdir](dest).then(({uid, gid}) =>
370 this.tarballStream(tarball => this[_extract](dest, tarball, uid, gid)))
371 }
372
373 [_toFile] (dest) {
374 return this.tarballStream(str => new Promise((res, rej) => {
375 const writer = new fsm.WriteStream(dest)
376 str.on('error', er => writer.emit('error', er))
377 writer.on('error', er => rej(er))
378 writer.on('close', () => res({
379 integrity: this.integrity && String(this.integrity),
380 resolved: this.resolved,
381 from: this.from,
382 }))
383 str.pipe(writer)
384 }))
385 }
386
387 // don't use this[_mkdir] because we don't want to rimraf anything
388 tarballFile (dest) {
389 const dir = dirname(dest)
390 return !inferOwner
391 ? mkdirp(dir).then(() => this[_toFile](dest))
392 : inferOwner(dest).then(({uid, gid}) =>
393 mkdirp(dir).then(made => this[_toFile](dest)
394 .then(res => this[_chown](made || dir, uid, gid)
395 .then(() => res))))
396 }
397
398 [_extract] (dest, tarball, uid, gid) {
399 const extractor = tar.x(this[_tarxOptions]({ cwd: dest, uid, gid }))
400 const p = new Promise((resolve, reject) => {
401 extractor.on('end', () => {
402 resolve({
403 resolved: this.resolved,
404 integrity: this.integrity && String(this.integrity),
405 from: this.from,
406 })
407 })
408
409 extractor.on('error', er => {
410 this.log.warn('tar', er.message)
411 this.log.silly('tar', er)
412 reject(er)
413 })
414
415 tarball.on('error', er => reject(er))
416 })
417
418 tarball.pipe(extractor)
419 return p
420 }
421
422 // always ensure that entries are at least as permissive as our configured
423 // dmode/fmode, but never more permissive than the umask allows.
424 [_entryMode] (path, mode, type) {
425 const m = /Directory|GNUDumpDir/.test(type) ? this.dmode
426 : /File$/.test(type) ? this.fmode
427 : /* istanbul ignore next - should never happen in a pkg */ 0
428
429 // make sure package bins are executable
430 const exe = isPackageBin(this.package, path) ? 0o111 : 0
431 // always ensure that files are read/writable by the owner
432 return ((mode | m) & ~this.umask) | exe | 0o600
433 }
434
435 [_tarxOptions] ({ cwd, uid, gid }) {
436 const sawIgnores = new Set()
437 return {
438 cwd,
439 noChmod: true,
440 noMtime: true,
441 filter: (name, entry) => {
442 if (/Link$/.test(entry.type))
443 return false
444 entry.mode = this[_entryMode](entry.path, entry.mode, entry.type)
445 // this replicates the npm pack behavior where .gitignore files
446 // are treated like .npmignore files, but only if a .npmignore
447 // file is not present.
448 if (/File$/.test(entry.type)) {
449 const base = basename(entry.path)
450 if (base === '.npmignore')
451 sawIgnores.add(entry.path)
452 else if (base === '.gitignore' && !this.allowGitIgnore) {
453 // rename, but only if there's not already a .npmignore
454 const ni = entry.path.replace(/\.gitignore$/, '.npmignore')
455 if (sawIgnores.has(ni))
456 return false
457 entry.path = ni
458 }
459 return true
460 }
461 },
462 strip: 1,
463 onwarn: /* istanbul ignore next - we can trust that tar logs */
464 (code, msg, data) => {
465 this.log.warn('tar', code, msg)
466 this.log.silly('tar', code, msg, data)
467 },
468 uid,
469 gid,
470 umask: this.umask,
471 }
472 }
473}
474
475module.exports = FetcherBase
476
477// Child classes
478const GitFetcher = require('./git.js')
479const RegistryFetcher = require('./registry.js')
480const FileFetcher = require('./file.js')
481const DirFetcher = require('./dir.js')
482const RemoteFetcher = require('./remote.js')
483
484// Get an appropriate fetcher object from a spec and options
485FetcherBase.get = (rawSpec, opts = {}) => {
486 const spec = npa(rawSpec, opts.where)
487 switch (spec.type) {
488 case 'git':
489 return new GitFetcher(spec, opts)
490
491 case 'remote':
492 return new RemoteFetcher(spec, opts)
493
494 case 'version':
495 case 'range':
496 case 'tag':
497 case 'alias':
498 return new RegistryFetcher(spec.subSpec || spec, opts)
499
500 case 'file':
501 return new FileFetcher(spec, opts)
502
503 case 'directory':
504 return new DirFetcher(spec, opts)
505
506 default:
507 throw new TypeError('Unknown spec type: ' + spec.type)
508 }
509}
Note: See TracBrowser for help on using the repository browser.