source: trip-planner-front/node_modules/npm-packlist/index.js@ 1ad8e64

Last change on this file since 1ad8e64 was 6a3a178, checked in by Ema <ema_spirova@…>, 3 years ago

initial commit

  • Property mode set to 100644
File size: 14.6 KB
Line 
1'use strict'
2
3// Do a two-pass walk, first to get the list of packages that need to be
4// bundled, then again to get the actual files and folders.
5// Keep a cache of node_modules content and package.json data, so that the
6// second walk doesn't have to re-do all the same work.
7
8const bundleWalk = require('npm-bundled')
9const BundleWalker = bundleWalk.BundleWalker
10const BundleWalkerSync = bundleWalk.BundleWalkerSync
11
12const ignoreWalk = require('ignore-walk')
13const IgnoreWalker = ignoreWalk.Walker
14const IgnoreWalkerSync = ignoreWalk.WalkerSync
15
16const rootBuiltinRules = Symbol('root-builtin-rules')
17const packageNecessaryRules = Symbol('package-necessary-rules')
18const path = require('path')
19
20const normalizePackageBin = require('npm-normalize-package-bin')
21
22// Weird side-effect of this: a readme (etc) file will be included
23// if it exists anywhere within a folder with a package.json file.
24// The original intent was only to include these files in the root,
25// but now users in the wild are dependent on that behavior for
26// localized documentation and other use cases. Adding a `/` to
27// these rules, while tempting and arguably more "correct", is a
28// significant change that will break existing use cases.
29const packageMustHaveFileNames = 'readme|copying|license|licence'
30
31const packageMustHaves = `@(${packageMustHaveFileNames}){,.*[^~$]}`
32const packageMustHavesRE = new RegExp(`^(${packageMustHaveFileNames})(\\..*[^~$])?$`, 'i')
33
34const fs = require('fs')
35const glob = require('glob')
36
37const defaultRules = [
38 '.npmignore',
39 '.gitignore',
40 '**/.git',
41 '**/.svn',
42 '**/.hg',
43 '**/CVS',
44 '**/.git/**',
45 '**/.svn/**',
46 '**/.hg/**',
47 '**/CVS/**',
48 '/.lock-wscript',
49 '/.wafpickle-*',
50 '/build/config.gypi',
51 'npm-debug.log',
52 '**/.npmrc',
53 '.*.swp',
54 '.DS_Store',
55 '**/.DS_Store/**',
56 '._*',
57 '**/._*/**',
58 '*.orig',
59 '/package-lock.json',
60 '/yarn.lock',
61 '/archived-packages/**',
62]
63
64// There may be others, but :?|<> are handled by node-tar
65const nameIsBadForWindows = file => /\*/.test(file)
66
67// a decorator that applies our custom rules to an ignore walker
68const npmWalker = Class => class Walker extends Class {
69 constructor (opt) {
70 opt = opt || {}
71
72 // the order in which rules are applied.
73 opt.ignoreFiles = [
74 rootBuiltinRules,
75 'package.json',
76 '.npmignore',
77 '.gitignore',
78 packageNecessaryRules,
79 ]
80
81 opt.includeEmpty = false
82 opt.path = opt.path || process.cwd()
83
84 // only follow links in the root node_modules folder, because if those
85 // folders are included, it's because they're bundled, and bundles
86 // should include the contents, not the symlinks themselves.
87 // This regexp tests to see that we're either a node_modules folder,
88 // or a @scope within a node_modules folder, in the root's node_modules
89 // hierarchy (ie, not in test/foo/node_modules/ or something).
90 const followRe = /^(?:\/node_modules\/(?:@[^/]+\/[^/]+|[^/]+)\/)*\/node_modules(?:\/@[^/]+)?$/
91 const rootPath = opt.parent ? opt.parent.root : opt.path
92 const followTestPath = opt.path.replace(/\\/g, '/').substr(rootPath.length)
93 opt.follow = followRe.test(followTestPath)
94
95 super(opt)
96
97 // ignore a bunch of things by default at the root level.
98 // also ignore anything in the main project node_modules hierarchy,
99 // except bundled dependencies
100 if (!this.parent) {
101 this.bundled = opt.bundled || []
102 this.bundledScopes = Array.from(new Set(
103 this.bundled.filter(f => /^@/.test(f))
104 .map(f => f.split('/')[0])))
105 const rules = defaultRules.join('\n') + '\n'
106 this.packageJsonCache = opt.packageJsonCache || new Map()
107 super.onReadIgnoreFile(rootBuiltinRules, rules, _ => _)
108 } else {
109 this.bundled = []
110 this.bundledScopes = []
111 this.packageJsonCache = this.parent.packageJsonCache
112 }
113 }
114
115 onReaddir (entries) {
116 if (!this.parent) {
117 entries = entries.filter(e =>
118 e !== '.git' &&
119 !(e === 'node_modules' && this.bundled.length === 0)
120 )
121 }
122
123 // if we have a package.json, then look in it for 'files'
124 // we _only_ do this in the root project, not bundled deps
125 // or other random folders. Bundled deps are always assumed
126 // to be in the state the user wants to include them, and
127 // a package.json somewhere else might be a template or
128 // test or something else entirely.
129 if (this.parent || !entries.includes('package.json'))
130 return super.onReaddir(entries)
131
132 // when the cache has been seeded with the root manifest,
133 // we must respect that (it may differ from the filesystem)
134 const ig = path.resolve(this.path, 'package.json')
135
136 if (this.packageJsonCache.has(ig)) {
137 const pkg = this.packageJsonCache.get(ig)
138
139 // fall back to filesystem when seeded manifest is invalid
140 if (!pkg || typeof pkg !== 'object')
141 return this.readPackageJson(entries)
142
143 // feels wonky, but this ensures package bin is _always_
144 // normalized, as well as guarding against invalid JSON
145 return this.getPackageFiles(entries, JSON.stringify(pkg))
146 }
147
148 this.readPackageJson(entries)
149 }
150
151 onReadPackageJson (entries, er, pkg) {
152 if (er)
153 this.emit('error', er)
154 else
155 this.getPackageFiles(entries, pkg)
156 }
157
158 mustHaveFilesFromPackage (pkg) {
159 const files = []
160 if (pkg.browser)
161 files.push('/' + pkg.browser)
162 if (pkg.main)
163 files.push('/' + pkg.main)
164 if (pkg.bin) {
165 // always an object because normalized already
166 for (const key in pkg.bin)
167 files.push('/' + pkg.bin[key])
168 }
169 files.push(
170 '/package.json',
171 '/npm-shrinkwrap.json',
172 '!/package-lock.json',
173 packageMustHaves
174 )
175 return files
176 }
177
178 getPackageFiles (entries, pkg) {
179 try {
180 // XXX this could be changed to use read-package-json-fast
181 // which handles the normalizing of bins for us, and simplifies
182 // the test for bundleDependencies and bundledDependencies later.
183 // HOWEVER if we do this, we need to be sure that we're careful
184 // about what we write back out since rpj-fast removes some fields
185 // that the user likely wants to keep. it also would add a second
186 // file read that we would want to optimize away.
187 pkg = normalizePackageBin(JSON.parse(pkg.toString()))
188 } catch (er) {
189 // not actually a valid package.json
190 return super.onReaddir(entries)
191 }
192
193 const ig = path.resolve(this.path, 'package.json')
194 this.packageJsonCache.set(ig, pkg)
195
196 // no files list, just return the normal readdir() result
197 if (!Array.isArray(pkg.files))
198 return super.onReaddir(entries)
199
200 pkg.files.push(...this.mustHaveFilesFromPackage(pkg))
201
202 // If the package has a files list, then it's unlikely to include
203 // node_modules, because why would you do that? but since we use
204 // the files list as the effective readdir result, that means it
205 // looks like we don't have a node_modules folder at all unless we
206 // include it here.
207 if ((pkg.bundleDependencies || pkg.bundledDependencies) && entries.includes('node_modules'))
208 pkg.files.push('node_modules')
209
210 const patterns = Array.from(new Set(pkg.files)).reduce((set, pattern) => {
211 const excl = pattern.match(/^!+/)
212 if (excl)
213 pattern = pattern.substr(excl[0].length)
214 // strip off any / from the start of the pattern. /foo => foo
215 pattern = pattern.replace(/^\/+/, '')
216 // an odd number of ! means a negated pattern. !!foo ==> foo
217 const negate = excl && excl[0].length % 2 === 1
218 set.push({ pattern, negate })
219 return set
220 }, [])
221
222 let n = patterns.length
223 const set = new Set()
224 const negates = new Set()
225 const results = []
226 const then = (pattern, negate, er, fileList, i) => {
227 if (er)
228 return this.emit('error', er)
229
230 results[i] = { negate, fileList }
231 if (--n === 0)
232 processResults(results)
233 }
234 const processResults = results => {
235 for (const {negate, fileList} of results) {
236 if (negate) {
237 fileList.forEach(f => {
238 f = f.replace(/\/+$/, '')
239 set.delete(f)
240 negates.add(f)
241 })
242 } else {
243 fileList.forEach(f => {
244 f = f.replace(/\/+$/, '')
245 set.add(f)
246 negates.delete(f)
247 })
248 }
249 }
250
251 const list = Array.from(set)
252 // replace the files array with our computed explicit set
253 pkg.files = list.concat(Array.from(negates).map(f => '!' + f))
254 const rdResult = Array.from(new Set(
255 list.map(f => f.replace(/^\/+/, ''))
256 ))
257 super.onReaddir(rdResult)
258 }
259
260 // maintain the index so that we process them in-order only once all
261 // are completed, otherwise the parallelism messes things up, since a
262 // glob like **/*.js will always be slower than a subsequent !foo.js
263 patterns.forEach(({pattern, negate}, i) =>
264 this.globFiles(pattern, (er, res) => then(pattern, negate, er, res, i)))
265 }
266
267 filterEntry (entry, partial) {
268 // get the partial path from the root of the walk
269 const p = this.path.substr(this.root.length + 1)
270 const pkgre = /^node_modules\/(@[^/]+\/?[^/]+|[^/]+)(\/.*)?$/
271 const isRoot = !this.parent
272 const pkg = isRoot && pkgre.test(entry) ?
273 entry.replace(pkgre, '$1') : null
274 const rootNM = isRoot && entry === 'node_modules'
275 const rootPJ = isRoot && entry === 'package.json'
276
277 return (
278 // if we're in a bundled package, check with the parent.
279 /^node_modules($|\/)/i.test(p) ? this.parent.filterEntry(
280 this.basename + '/' + entry, partial)
281
282 // if package is bundled, all files included
283 // also include @scope dirs for bundled scoped deps
284 // they'll be ignored if no files end up in them.
285 // However, this only matters if we're in the root.
286 // node_modules folders elsewhere, like lib/node_modules,
287 // should be included normally unless ignored.
288 : pkg ? this.bundled.indexOf(pkg) !== -1 ||
289 this.bundledScopes.indexOf(pkg) !== -1
290
291 // only walk top node_modules if we want to bundle something
292 : rootNM ? !!this.bundled.length
293
294 // always include package.json at the root.
295 : rootPJ ? true
296
297 // always include readmes etc in any included dir
298 : packageMustHavesRE.test(entry) ? true
299
300 // npm-shrinkwrap and package.json always included in the root pkg
301 : isRoot && (entry === 'npm-shrinkwrap.json' || entry === 'package.json')
302 ? true
303
304 // package-lock never included
305 : isRoot && entry === 'package-lock.json' ? false
306
307 // otherwise, follow ignore-walk's logic
308 : super.filterEntry(entry, partial)
309 )
310 }
311
312 filterEntries () {
313 if (this.ignoreRules['.npmignore'])
314 this.ignoreRules['.gitignore'] = null
315 this.filterEntries = super.filterEntries
316 super.filterEntries()
317 }
318
319 addIgnoreFile (file, then) {
320 const ig = path.resolve(this.path, file)
321 if (file === 'package.json' && this.parent)
322 then()
323 else if (this.packageJsonCache.has(ig))
324 this.onPackageJson(ig, this.packageJsonCache.get(ig), then)
325 else
326 super.addIgnoreFile(file, then)
327 }
328
329 onPackageJson (ig, pkg, then) {
330 this.packageJsonCache.set(ig, pkg)
331
332 if (Array.isArray(pkg.files)) {
333 // in this case we already included all the must-haves
334 super.onReadIgnoreFile('package.json', pkg.files.map(
335 f => '!' + f
336 ).join('\n') + '\n', then)
337 } else {
338 // if there's a bin, browser or main, make sure we don't ignore it
339 // also, don't ignore the package.json itself, or any files that
340 // must be included in the package.
341 const rules = this.mustHaveFilesFromPackage(pkg).map(f => `!${f}`)
342 const data = rules.join('\n') + '\n'
343 super.onReadIgnoreFile(packageNecessaryRules, data, then)
344 }
345 }
346
347 // override parent stat function to completely skip any filenames
348 // that will break windows entirely.
349 // XXX(isaacs) Next major version should make this an error instead.
350 stat (entry, file, dir, then) {
351 if (nameIsBadForWindows(entry))
352 then()
353 else
354 super.stat(entry, file, dir, then)
355 }
356
357 // override parent onstat function to nix all symlinks
358 onstat (st, entry, file, dir, then) {
359 if (st.isSymbolicLink())
360 then()
361 else
362 super.onstat(st, entry, file, dir, then)
363 }
364
365 onReadIgnoreFile (file, data, then) {
366 if (file === 'package.json') {
367 try {
368 const ig = path.resolve(this.path, file)
369 this.onPackageJson(ig, JSON.parse(data), then)
370 } catch (er) {
371 // ignore package.json files that are not json
372 then()
373 }
374 } else
375 super.onReadIgnoreFile(file, data, then)
376 }
377
378 sort (a, b) {
379 return sort(a, b)
380 }
381}
382
383class Walker extends npmWalker(IgnoreWalker) {
384 globFiles (pattern, cb) {
385 glob(pattern, { dot: true, cwd: this.path, nocase: true }, cb)
386 }
387
388 readPackageJson (entries) {
389 fs.readFile(this.path + '/package.json', (er, pkg) =>
390 this.onReadPackageJson(entries, er, pkg))
391 }
392
393 walker (entry, then) {
394 new Walker(this.walkerOpt(entry)).on('done', then).start()
395 }
396}
397
398class WalkerSync extends npmWalker(IgnoreWalkerSync) {
399 globFiles (pattern, cb) {
400 cb(null, glob.sync(pattern, { dot: true, cwd: this.path, nocase: true }))
401 }
402
403 readPackageJson (entries) {
404 const p = this.path + '/package.json'
405 try {
406 this.onReadPackageJson(entries, null, fs.readFileSync(p))
407 } catch (er) {
408 this.onReadPackageJson(entries, er)
409 }
410 }
411
412 walker (entry, then) {
413 new WalkerSync(this.walkerOpt(entry)).start()
414 then()
415 }
416}
417
418const walk = (options, callback) => {
419 options = options || {}
420 const p = new Promise((resolve, reject) => {
421 const bw = new BundleWalker(options)
422 bw.on('done', bundled => {
423 options.bundled = bundled
424 options.packageJsonCache = bw.packageJsonCache
425 new Walker(options).on('done', resolve).on('error', reject).start()
426 })
427 bw.start()
428 })
429 return callback ? p.then(res => callback(null, res), callback) : p
430}
431
432const walkSync = options => {
433 options = options || {}
434 const bw = new BundleWalkerSync(options).start()
435 options.bundled = bw.result
436 options.packageJsonCache = bw.packageJsonCache
437 const walker = new WalkerSync(options)
438 walker.start()
439 return walker.result
440}
441
442// optimize for compressibility
443// extname, then basename, then locale alphabetically
444// https://twitter.com/isntitvacant/status/1131094910923231232
445const sort = (a, b) => {
446 const exta = path.extname(a).toLowerCase()
447 const extb = path.extname(b).toLowerCase()
448 const basea = path.basename(a).toLowerCase()
449 const baseb = path.basename(b).toLowerCase()
450
451 return exta.localeCompare(extb, 'en') ||
452 basea.localeCompare(baseb, 'en') ||
453 a.localeCompare(b, 'en')
454}
455
456module.exports = walk
457walk.sync = walkSync
458walk.Walker = Walker
459walk.WalkerSync = WalkerSync
Note: See TracBrowser for help on using the repository browser.