1 | /*
|
---|
2 | MIT License http://www.opensource.org/licenses/mit-license.php
|
---|
3 | Author Tobias Koppers @sokra
|
---|
4 | */
|
---|
5 |
|
---|
6 | "use strict";
|
---|
7 |
|
---|
8 | const { resolve, extname, dirname } = require("path");
|
---|
9 | const { URL } = require("url");
|
---|
10 | const { createGunzip, createBrotliDecompress, createInflate } = require("zlib");
|
---|
11 | const NormalModule = require("../NormalModule");
|
---|
12 | const createHash = require("../util/createHash");
|
---|
13 | const { mkdirp } = require("../util/fs");
|
---|
14 | const memoize = require("../util/memoize");
|
---|
15 |
|
---|
16 | /** @typedef {import("../../declarations/plugins/schemes/HttpUriPlugin").HttpUriPluginOptions} HttpUriPluginOptions */
|
---|
17 | /** @typedef {import("../Compiler")} Compiler */
|
---|
18 |
|
---|
19 | const getHttp = memoize(() => require("http"));
|
---|
20 | const getHttps = memoize(() => require("https"));
|
---|
21 |
|
---|
22 | const toSafePath = str =>
|
---|
23 | str
|
---|
24 | .replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, "")
|
---|
25 | .replace(/[^a-zA-Z0-9._-]+/g, "_");
|
---|
26 |
|
---|
27 | const computeIntegrity = content => {
|
---|
28 | const hash = createHash("sha512");
|
---|
29 | hash.update(content);
|
---|
30 | const integrity = "sha512-" + hash.digest("base64");
|
---|
31 | return integrity;
|
---|
32 | };
|
---|
33 |
|
---|
34 | const verifyIntegrity = (content, integrity) => {
|
---|
35 | if (integrity === "ignore") return true;
|
---|
36 | return computeIntegrity(content) === integrity;
|
---|
37 | };
|
---|
38 |
|
---|
39 | /**
|
---|
40 | * @param {string} str input
|
---|
41 | * @returns {Record<string, string>} parsed
|
---|
42 | */
|
---|
43 | const parseKeyValuePairs = str => {
|
---|
44 | /** @type {Record<string, string>} */
|
---|
45 | const result = {};
|
---|
46 | for (const item of str.split(",")) {
|
---|
47 | const i = item.indexOf("=");
|
---|
48 | if (i >= 0) {
|
---|
49 | const key = item.slice(0, i).trim();
|
---|
50 | const value = item.slice(i + 1).trim();
|
---|
51 | result[key] = value;
|
---|
52 | } else {
|
---|
53 | const key = item.trim();
|
---|
54 | if (!key) continue;
|
---|
55 | result[key] = key;
|
---|
56 | }
|
---|
57 | }
|
---|
58 | return result;
|
---|
59 | };
|
---|
60 |
|
---|
61 | const parseCacheControl = (cacheControl, requestTime) => {
|
---|
62 | // When false resource is not stored in cache
|
---|
63 | let storeCache = true;
|
---|
64 | // When false resource is not stored in lockfile cache
|
---|
65 | let storeLock = true;
|
---|
66 | // Resource is only revalidated, after that timestamp and when upgrade is chosen
|
---|
67 | let validUntil = 0;
|
---|
68 | if (cacheControl) {
|
---|
69 | const parsed = parseKeyValuePairs(cacheControl);
|
---|
70 | if (parsed["no-cache"]) storeCache = storeLock = false;
|
---|
71 | if (parsed["max-age"] && !isNaN(+parsed["max-age"])) {
|
---|
72 | validUntil = requestTime + +parsed["max-age"] * 1000;
|
---|
73 | }
|
---|
74 | if (parsed["must-revalidate"]) validUntil = 0;
|
---|
75 | }
|
---|
76 | return {
|
---|
77 | storeLock,
|
---|
78 | storeCache,
|
---|
79 | validUntil
|
---|
80 | };
|
---|
81 | };
|
---|
82 |
|
---|
83 | /**
|
---|
84 | * @typedef {Object} LockfileEntry
|
---|
85 | * @property {string} resolved
|
---|
86 | * @property {string} integrity
|
---|
87 | * @property {string} contentType
|
---|
88 | */
|
---|
89 |
|
---|
90 | const areLockfileEntriesEqual = (a, b) => {
|
---|
91 | return (
|
---|
92 | a.resolved === b.resolved &&
|
---|
93 | a.integrity === b.integrity &&
|
---|
94 | a.contentType === b.contentType
|
---|
95 | );
|
---|
96 | };
|
---|
97 |
|
---|
98 | const entryToString = entry => {
|
---|
99 | return `resolved: ${entry.resolved}, integrity: ${entry.integrity}, contentType: ${entry.contentType}`;
|
---|
100 | };
|
---|
101 |
|
---|
102 | class Lockfile {
|
---|
103 | constructor() {
|
---|
104 | this.version = 1;
|
---|
105 | /** @type {Map<string, LockfileEntry | "ignore" | "no-cache">} */
|
---|
106 | this.entries = new Map();
|
---|
107 | this.outdated = false;
|
---|
108 | }
|
---|
109 |
|
---|
110 | static parse(content) {
|
---|
111 | // TODO handle merge conflicts
|
---|
112 | const data = JSON.parse(content);
|
---|
113 | if (data.version !== 1)
|
---|
114 | throw new Error(`Unsupported lockfile version ${data.version}`);
|
---|
115 | const lockfile = new Lockfile();
|
---|
116 | for (const key of Object.keys(data)) {
|
---|
117 | if (key === "version") continue;
|
---|
118 | const entry = data[key];
|
---|
119 | lockfile.entries.set(
|
---|
120 | key,
|
---|
121 | typeof entry === "string"
|
---|
122 | ? entry
|
---|
123 | : {
|
---|
124 | resolved: key,
|
---|
125 | ...entry
|
---|
126 | }
|
---|
127 | );
|
---|
128 | }
|
---|
129 | return lockfile;
|
---|
130 | }
|
---|
131 |
|
---|
132 | toString() {
|
---|
133 | let str = "{\n";
|
---|
134 | const entries = Array.from(this.entries).sort(([a], [b]) =>
|
---|
135 | a < b ? -1 : 1
|
---|
136 | );
|
---|
137 | for (const [key, entry] of entries) {
|
---|
138 | if (typeof entry === "string") {
|
---|
139 | str += ` ${JSON.stringify(key)}: ${JSON.stringify(entry)},\n`;
|
---|
140 | } else {
|
---|
141 | str += ` ${JSON.stringify(key)}: { `;
|
---|
142 | if (entry.resolved !== key)
|
---|
143 | str += `"resolved": ${JSON.stringify(entry.resolved)}, `;
|
---|
144 | str += `"integrity": ${JSON.stringify(
|
---|
145 | entry.integrity
|
---|
146 | )}, "contentType": ${JSON.stringify(entry.contentType)} },\n`;
|
---|
147 | }
|
---|
148 | }
|
---|
149 | str += ` "version": ${this.version}\n}\n`;
|
---|
150 | return str;
|
---|
151 | }
|
---|
152 | }
|
---|
153 |
|
---|
154 | /**
|
---|
155 | * @template R
|
---|
156 | * @param {function(function(Error=, R=): void): void} fn function
|
---|
157 | * @returns {function(function(Error=, R=): void): void} cached function
|
---|
158 | */
|
---|
159 | const cachedWithoutKey = fn => {
|
---|
160 | let inFlight = false;
|
---|
161 | /** @type {Error | undefined} */
|
---|
162 | let cachedError = undefined;
|
---|
163 | /** @type {R | undefined} */
|
---|
164 | let cachedResult = undefined;
|
---|
165 | /** @type {(function(Error=, R=): void)[] | undefined} */
|
---|
166 | let cachedCallbacks = undefined;
|
---|
167 | return callback => {
|
---|
168 | if (inFlight) {
|
---|
169 | if (cachedResult !== undefined) return callback(null, cachedResult);
|
---|
170 | if (cachedError !== undefined) return callback(cachedError);
|
---|
171 | if (cachedCallbacks === undefined) cachedCallbacks = [callback];
|
---|
172 | else cachedCallbacks.push(callback);
|
---|
173 | return;
|
---|
174 | }
|
---|
175 | inFlight = true;
|
---|
176 | fn((err, result) => {
|
---|
177 | if (err) cachedError = err;
|
---|
178 | else cachedResult = result;
|
---|
179 | const callbacks = cachedCallbacks;
|
---|
180 | cachedCallbacks = undefined;
|
---|
181 | callback(err, result);
|
---|
182 | if (callbacks !== undefined) for (const cb of callbacks) cb(err, result);
|
---|
183 | });
|
---|
184 | };
|
---|
185 | };
|
---|
186 |
|
---|
187 | /**
|
---|
188 | * @template T
|
---|
189 | * @template R
|
---|
190 | * @param {function(T, function(Error=, R=): void): void} fn function
|
---|
191 | * @param {function(T, function(Error=, R=): void): void=} forceFn function for the second try
|
---|
192 | * @returns {(function(T, function(Error=, R=): void): void) & { force: function(T, function(Error=, R=): void): void }} cached function
|
---|
193 | */
|
---|
194 | const cachedWithKey = (fn, forceFn = fn) => {
|
---|
195 | /** @typedef {{ result?: R, error?: Error, callbacks?: (function(Error=, R=): void)[], force?: true }} CacheEntry */
|
---|
196 | /** @type {Map<T, CacheEntry>} */
|
---|
197 | const cache = new Map();
|
---|
198 | const resultFn = (arg, callback) => {
|
---|
199 | const cacheEntry = cache.get(arg);
|
---|
200 | if (cacheEntry !== undefined) {
|
---|
201 | if (cacheEntry.result !== undefined)
|
---|
202 | return callback(null, cacheEntry.result);
|
---|
203 | if (cacheEntry.error !== undefined) return callback(cacheEntry.error);
|
---|
204 | if (cacheEntry.callbacks === undefined) cacheEntry.callbacks = [callback];
|
---|
205 | else cacheEntry.callbacks.push(callback);
|
---|
206 | return;
|
---|
207 | }
|
---|
208 | /** @type {CacheEntry} */
|
---|
209 | const newCacheEntry = {
|
---|
210 | result: undefined,
|
---|
211 | error: undefined,
|
---|
212 | callbacks: undefined
|
---|
213 | };
|
---|
214 | cache.set(arg, newCacheEntry);
|
---|
215 | fn(arg, (err, result) => {
|
---|
216 | if (err) newCacheEntry.error = err;
|
---|
217 | else newCacheEntry.result = result;
|
---|
218 | const callbacks = newCacheEntry.callbacks;
|
---|
219 | newCacheEntry.callbacks = undefined;
|
---|
220 | callback(err, result);
|
---|
221 | if (callbacks !== undefined) for (const cb of callbacks) cb(err, result);
|
---|
222 | });
|
---|
223 | };
|
---|
224 | resultFn.force = (arg, callback) => {
|
---|
225 | const cacheEntry = cache.get(arg);
|
---|
226 | if (cacheEntry !== undefined && cacheEntry.force) {
|
---|
227 | if (cacheEntry.result !== undefined)
|
---|
228 | return callback(null, cacheEntry.result);
|
---|
229 | if (cacheEntry.error !== undefined) return callback(cacheEntry.error);
|
---|
230 | if (cacheEntry.callbacks === undefined) cacheEntry.callbacks = [callback];
|
---|
231 | else cacheEntry.callbacks.push(callback);
|
---|
232 | return;
|
---|
233 | }
|
---|
234 | /** @type {CacheEntry} */
|
---|
235 | const newCacheEntry = {
|
---|
236 | result: undefined,
|
---|
237 | error: undefined,
|
---|
238 | callbacks: undefined,
|
---|
239 | force: true
|
---|
240 | };
|
---|
241 | cache.set(arg, newCacheEntry);
|
---|
242 | forceFn(arg, (err, result) => {
|
---|
243 | if (err) newCacheEntry.error = err;
|
---|
244 | else newCacheEntry.result = result;
|
---|
245 | const callbacks = newCacheEntry.callbacks;
|
---|
246 | newCacheEntry.callbacks = undefined;
|
---|
247 | callback(err, result);
|
---|
248 | if (callbacks !== undefined) for (const cb of callbacks) cb(err, result);
|
---|
249 | });
|
---|
250 | };
|
---|
251 | return resultFn;
|
---|
252 | };
|
---|
253 |
|
---|
254 | /**
|
---|
255 | * @typedef {Object} HttpUriPluginAdvancedOptions
|
---|
256 | * @property {string | typeof import("../util/Hash")=} hashFunction
|
---|
257 | * @property {string=} hashDigest
|
---|
258 | * @property {number=} hashDigestLength
|
---|
259 | */
|
---|
260 |
|
---|
261 | class HttpUriPlugin {
|
---|
262 | /**
|
---|
263 | * @param {HttpUriPluginOptions & HttpUriPluginAdvancedOptions} options options
|
---|
264 | */
|
---|
265 | constructor(options = {}) {
|
---|
266 | this._lockfileLocation = options.lockfileLocation;
|
---|
267 | this._cacheLocation = options.cacheLocation;
|
---|
268 | this._upgrade = options.upgrade;
|
---|
269 | this._frozen = options.frozen;
|
---|
270 | this._hashFunction = options.hashFunction;
|
---|
271 | this._hashDigest = options.hashDigest;
|
---|
272 | this._hashDigestLength = options.hashDigestLength;
|
---|
273 | }
|
---|
274 |
|
---|
275 | /**
|
---|
276 | * Apply the plugin
|
---|
277 | * @param {Compiler} compiler the compiler instance
|
---|
278 | * @returns {void}
|
---|
279 | */
|
---|
280 | apply(compiler) {
|
---|
281 | const schemes = [
|
---|
282 | {
|
---|
283 | scheme: "http",
|
---|
284 | fetch: (url, options, callback) => getHttp().get(url, options, callback)
|
---|
285 | },
|
---|
286 | {
|
---|
287 | scheme: "https",
|
---|
288 | fetch: (url, options, callback) =>
|
---|
289 | getHttps().get(url, options, callback)
|
---|
290 | }
|
---|
291 | ];
|
---|
292 | let lockfileCache;
|
---|
293 | compiler.hooks.compilation.tap(
|
---|
294 | "HttpUriPlugin",
|
---|
295 | (compilation, { normalModuleFactory }) => {
|
---|
296 | const intermediateFs = compiler.intermediateFileSystem;
|
---|
297 | const fs = compilation.inputFileSystem;
|
---|
298 | const cache = compilation.getCache("webpack.HttpUriPlugin");
|
---|
299 | const logger = compilation.getLogger("webpack.HttpUriPlugin");
|
---|
300 | const lockfileLocation =
|
---|
301 | this._lockfileLocation ||
|
---|
302 | resolve(
|
---|
303 | compiler.context,
|
---|
304 | compiler.name
|
---|
305 | ? `${toSafePath(compiler.name)}.webpack.lock`
|
---|
306 | : "webpack.lock"
|
---|
307 | );
|
---|
308 | const cacheLocation =
|
---|
309 | this._cacheLocation !== undefined
|
---|
310 | ? this._cacheLocation
|
---|
311 | : lockfileLocation + ".data";
|
---|
312 | const upgrade = this._upgrade || false;
|
---|
313 | const frozen = this._frozen || false;
|
---|
314 | const hashFunction =
|
---|
315 | this._hashFunction || compilation.outputOptions.hashFunction;
|
---|
316 | const hashDigest =
|
---|
317 | this._hashDigest || compilation.outputOptions.hashDigest;
|
---|
318 | const hashDigestLength =
|
---|
319 | this._hashDigestLength || compilation.outputOptions.hashDigestLength;
|
---|
320 |
|
---|
321 | let warnedAboutEol = false;
|
---|
322 |
|
---|
323 | const cacheKeyCache = new Map();
|
---|
324 | /**
|
---|
325 | * @param {string} url the url
|
---|
326 | * @returns {string} the key
|
---|
327 | */
|
---|
328 | const getCacheKey = url => {
|
---|
329 | const cachedResult = cacheKeyCache.get(url);
|
---|
330 | if (cachedResult !== undefined) return cachedResult;
|
---|
331 | const result = _getCacheKey(url);
|
---|
332 | cacheKeyCache.set(url, result);
|
---|
333 | return result;
|
---|
334 | };
|
---|
335 |
|
---|
336 | /**
|
---|
337 | * @param {string} url the url
|
---|
338 | * @returns {string} the key
|
---|
339 | */
|
---|
340 | const _getCacheKey = url => {
|
---|
341 | const parsedUrl = new URL(url);
|
---|
342 | const folder = toSafePath(parsedUrl.origin);
|
---|
343 | const name = toSafePath(parsedUrl.pathname);
|
---|
344 | const query = toSafePath(parsedUrl.search);
|
---|
345 | let ext = extname(name);
|
---|
346 | if (ext.length > 20) ext = "";
|
---|
347 | const basename = ext ? name.slice(0, -ext.length) : name;
|
---|
348 | const hash = createHash(hashFunction);
|
---|
349 | hash.update(url);
|
---|
350 | const digest = hash.digest(hashDigest).slice(0, hashDigestLength);
|
---|
351 | return `${folder.slice(-50)}/${`${basename}${
|
---|
352 | query ? `_${query}` : ""
|
---|
353 | }`.slice(0, 150)}_${digest}${ext}`;
|
---|
354 | };
|
---|
355 |
|
---|
356 | const getLockfile = cachedWithoutKey(
|
---|
357 | /**
|
---|
358 | * @param {function(Error=, Lockfile=): void} callback callback
|
---|
359 | * @returns {void}
|
---|
360 | */
|
---|
361 | callback => {
|
---|
362 | const readLockfile = () => {
|
---|
363 | intermediateFs.readFile(lockfileLocation, (err, buffer) => {
|
---|
364 | if (err && err.code !== "ENOENT") {
|
---|
365 | compilation.missingDependencies.add(lockfileLocation);
|
---|
366 | return callback(err);
|
---|
367 | }
|
---|
368 | compilation.fileDependencies.add(lockfileLocation);
|
---|
369 | compilation.fileSystemInfo.createSnapshot(
|
---|
370 | compiler.fsStartTime,
|
---|
371 | buffer ? [lockfileLocation] : [],
|
---|
372 | [],
|
---|
373 | buffer ? [] : [lockfileLocation],
|
---|
374 | { timestamp: true },
|
---|
375 | (err, snapshot) => {
|
---|
376 | if (err) return callback(err);
|
---|
377 | const lockfile = buffer
|
---|
378 | ? Lockfile.parse(buffer.toString("utf-8"))
|
---|
379 | : new Lockfile();
|
---|
380 | lockfileCache = {
|
---|
381 | lockfile,
|
---|
382 | snapshot
|
---|
383 | };
|
---|
384 | callback(null, lockfile);
|
---|
385 | }
|
---|
386 | );
|
---|
387 | });
|
---|
388 | };
|
---|
389 | if (lockfileCache) {
|
---|
390 | compilation.fileSystemInfo.checkSnapshotValid(
|
---|
391 | lockfileCache.snapshot,
|
---|
392 | (err, valid) => {
|
---|
393 | if (err) return callback(err);
|
---|
394 | if (!valid) return readLockfile();
|
---|
395 | callback(null, lockfileCache.lockfile);
|
---|
396 | }
|
---|
397 | );
|
---|
398 | } else {
|
---|
399 | readLockfile();
|
---|
400 | }
|
---|
401 | }
|
---|
402 | );
|
---|
403 |
|
---|
404 | let outdatedLockfile = undefined;
|
---|
405 | const storeLockEntry = (lockfile, url, entry) => {
|
---|
406 | const oldEntry = lockfile.entries.get(url);
|
---|
407 | lockfile.entries.set(url, entry);
|
---|
408 | outdatedLockfile = lockfile;
|
---|
409 | if (!oldEntry) {
|
---|
410 | logger.log(`${url} added to lockfile`);
|
---|
411 | } else if (typeof oldEntry === "string") {
|
---|
412 | if (typeof entry === "string") {
|
---|
413 | logger.log(`${url} updated in lockfile: ${oldEntry} -> ${entry}`);
|
---|
414 | } else {
|
---|
415 | logger.log(
|
---|
416 | `${url} updated in lockfile: ${oldEntry} -> ${entry.resolved}`
|
---|
417 | );
|
---|
418 | }
|
---|
419 | } else if (typeof entry === "string") {
|
---|
420 | logger.log(
|
---|
421 | `${url} updated in lockfile: ${oldEntry.resolved} -> ${entry}`
|
---|
422 | );
|
---|
423 | } else if (oldEntry.resolved !== entry.resolved) {
|
---|
424 | logger.log(
|
---|
425 | `${url} updated in lockfile: ${oldEntry.resolved} -> ${entry.resolved}`
|
---|
426 | );
|
---|
427 | } else if (oldEntry.integrity !== entry.integrity) {
|
---|
428 | logger.log(`${url} updated in lockfile: content changed`);
|
---|
429 | } else if (oldEntry.contentType !== entry.contentType) {
|
---|
430 | logger.log(
|
---|
431 | `${url} updated in lockfile: ${oldEntry.contentType} -> ${entry.contentType}`
|
---|
432 | );
|
---|
433 | } else {
|
---|
434 | logger.log(`${url} updated in lockfile`);
|
---|
435 | }
|
---|
436 | };
|
---|
437 |
|
---|
438 | const storeResult = (lockfile, url, result, callback) => {
|
---|
439 | if (result.storeLock) {
|
---|
440 | storeLockEntry(lockfile, url, result.entry);
|
---|
441 | if (!cacheLocation || !result.content)
|
---|
442 | return callback(null, result);
|
---|
443 | const key = getCacheKey(result.entry.resolved);
|
---|
444 | const filePath = resolve(cacheLocation, key);
|
---|
445 | mkdirp(intermediateFs, dirname(filePath), err => {
|
---|
446 | if (err) return callback(err);
|
---|
447 | intermediateFs.writeFile(filePath, result.content, err => {
|
---|
448 | if (err) return callback(err);
|
---|
449 | callback(null, result);
|
---|
450 | });
|
---|
451 | });
|
---|
452 | } else {
|
---|
453 | storeLockEntry(lockfile, url, "no-cache");
|
---|
454 | callback(null, result);
|
---|
455 | }
|
---|
456 | };
|
---|
457 |
|
---|
458 | for (const { scheme, fetch } of schemes) {
|
---|
459 | /**
|
---|
460 | *
|
---|
461 | * @param {string} url URL
|
---|
462 | * @param {string} integrity integrity
|
---|
463 | * @param {function(Error=, { entry: LockfileEntry, content: Buffer, storeLock: boolean }=): void} callback callback
|
---|
464 | */
|
---|
465 | const resolveContent = (url, integrity, callback) => {
|
---|
466 | const handleResult = (err, result) => {
|
---|
467 | if (err) return callback(err);
|
---|
468 | if ("location" in result) {
|
---|
469 | return resolveContent(
|
---|
470 | result.location,
|
---|
471 | integrity,
|
---|
472 | (err, innerResult) => {
|
---|
473 | if (err) return callback(err);
|
---|
474 | callback(null, {
|
---|
475 | entry: innerResult.entry,
|
---|
476 | content: innerResult.content,
|
---|
477 | storeLock: innerResult.storeLock && result.storeLock
|
---|
478 | });
|
---|
479 | }
|
---|
480 | );
|
---|
481 | } else {
|
---|
482 | if (
|
---|
483 | !result.fresh &&
|
---|
484 | integrity &&
|
---|
485 | result.entry.integrity !== integrity &&
|
---|
486 | !verifyIntegrity(result.content, integrity)
|
---|
487 | ) {
|
---|
488 | return fetchContent.force(url, handleResult);
|
---|
489 | }
|
---|
490 | return callback(null, {
|
---|
491 | entry: result.entry,
|
---|
492 | content: result.content,
|
---|
493 | storeLock: result.storeLock
|
---|
494 | });
|
---|
495 | }
|
---|
496 | };
|
---|
497 | fetchContent(url, handleResult);
|
---|
498 | };
|
---|
499 |
|
---|
500 | /** @typedef {{ storeCache: boolean, storeLock: boolean, validUntil: number, etag: string | undefined, fresh: boolean }} FetchResultMeta */
|
---|
501 | /** @typedef {FetchResultMeta & { location: string }} RedirectFetchResult */
|
---|
502 | /** @typedef {FetchResultMeta & { entry: LockfileEntry, content: Buffer }} ContentFetchResult */
|
---|
503 | /** @typedef {RedirectFetchResult | ContentFetchResult} FetchResult */
|
---|
504 |
|
---|
505 | /**
|
---|
506 | * @param {string} url URL
|
---|
507 | * @param {FetchResult} cachedResult result from cache
|
---|
508 | * @param {function(Error=, FetchResult=): void} callback callback
|
---|
509 | * @returns {void}
|
---|
510 | */
|
---|
511 | const fetchContentRaw = (url, cachedResult, callback) => {
|
---|
512 | const requestTime = Date.now();
|
---|
513 | fetch(
|
---|
514 | new URL(url),
|
---|
515 | {
|
---|
516 | headers: {
|
---|
517 | "accept-encoding": "gzip, deflate, br",
|
---|
518 | "user-agent": "webpack",
|
---|
519 | "if-none-match": cachedResult
|
---|
520 | ? cachedResult.etag || null
|
---|
521 | : null
|
---|
522 | }
|
---|
523 | },
|
---|
524 | res => {
|
---|
525 | const etag = res.headers["etag"];
|
---|
526 | const location = res.headers["location"];
|
---|
527 | const cacheControl = res.headers["cache-control"];
|
---|
528 | const { storeLock, storeCache, validUntil } = parseCacheControl(
|
---|
529 | cacheControl,
|
---|
530 | requestTime
|
---|
531 | );
|
---|
532 | /**
|
---|
533 | * @param {Partial<Pick<FetchResultMeta, "fresh">> & (Pick<RedirectFetchResult, "location"> | Pick<ContentFetchResult, "content" | "entry">)} partialResult result
|
---|
534 | * @returns {void}
|
---|
535 | */
|
---|
536 | const finishWith = partialResult => {
|
---|
537 | if ("location" in partialResult) {
|
---|
538 | logger.debug(
|
---|
539 | `GET ${url} [${res.statusCode}] -> ${partialResult.location}`
|
---|
540 | );
|
---|
541 | } else {
|
---|
542 | logger.debug(
|
---|
543 | `GET ${url} [${res.statusCode}] ${Math.ceil(
|
---|
544 | partialResult.content.length / 1024
|
---|
545 | )} kB${!storeLock ? " no-cache" : ""}`
|
---|
546 | );
|
---|
547 | }
|
---|
548 | const result = {
|
---|
549 | ...partialResult,
|
---|
550 | fresh: true,
|
---|
551 | storeLock,
|
---|
552 | storeCache,
|
---|
553 | validUntil,
|
---|
554 | etag
|
---|
555 | };
|
---|
556 | if (!storeCache) {
|
---|
557 | logger.log(
|
---|
558 | `${url} can't be stored in cache, due to Cache-Control header: ${cacheControl}`
|
---|
559 | );
|
---|
560 | return callback(null, result);
|
---|
561 | }
|
---|
562 | cache.store(
|
---|
563 | url,
|
---|
564 | null,
|
---|
565 | {
|
---|
566 | ...result,
|
---|
567 | fresh: false
|
---|
568 | },
|
---|
569 | err => {
|
---|
570 | if (err) {
|
---|
571 | logger.warn(
|
---|
572 | `${url} can't be stored in cache: ${err.message}`
|
---|
573 | );
|
---|
574 | logger.debug(err.stack);
|
---|
575 | }
|
---|
576 | callback(null, result);
|
---|
577 | }
|
---|
578 | );
|
---|
579 | };
|
---|
580 | if (res.statusCode === 304) {
|
---|
581 | if (
|
---|
582 | cachedResult.validUntil < validUntil ||
|
---|
583 | cachedResult.storeLock !== storeLock ||
|
---|
584 | cachedResult.storeCache !== storeCache ||
|
---|
585 | cachedResult.etag !== etag
|
---|
586 | ) {
|
---|
587 | return finishWith(cachedResult);
|
---|
588 | } else {
|
---|
589 | logger.debug(`GET ${url} [${res.statusCode}] (unchanged)`);
|
---|
590 | return callback(null, {
|
---|
591 | ...cachedResult,
|
---|
592 | fresh: true
|
---|
593 | });
|
---|
594 | }
|
---|
595 | }
|
---|
596 | if (
|
---|
597 | location &&
|
---|
598 | res.statusCode >= 301 &&
|
---|
599 | res.statusCode <= 308
|
---|
600 | ) {
|
---|
601 | return finishWith({
|
---|
602 | location: new URL(location, url).href
|
---|
603 | });
|
---|
604 | }
|
---|
605 | const contentType = res.headers["content-type"] || "";
|
---|
606 | const bufferArr = [];
|
---|
607 |
|
---|
608 | const contentEncoding = res.headers["content-encoding"];
|
---|
609 | let stream = res;
|
---|
610 | if (contentEncoding === "gzip") {
|
---|
611 | stream = stream.pipe(createGunzip());
|
---|
612 | } else if (contentEncoding === "br") {
|
---|
613 | stream = stream.pipe(createBrotliDecompress());
|
---|
614 | } else if (contentEncoding === "deflate") {
|
---|
615 | stream = stream.pipe(createInflate());
|
---|
616 | }
|
---|
617 |
|
---|
618 | stream.on("data", chunk => {
|
---|
619 | bufferArr.push(chunk);
|
---|
620 | });
|
---|
621 |
|
---|
622 | stream.on("end", () => {
|
---|
623 | if (!res.complete) {
|
---|
624 | logger.log(`GET ${url} [${res.statusCode}] (terminated)`);
|
---|
625 | return callback(new Error(`${url} request was terminated`));
|
---|
626 | }
|
---|
627 |
|
---|
628 | const content = Buffer.concat(bufferArr);
|
---|
629 |
|
---|
630 | if (res.statusCode !== 200) {
|
---|
631 | logger.log(`GET ${url} [${res.statusCode}]`);
|
---|
632 | return callback(
|
---|
633 | new Error(
|
---|
634 | `${url} request status code = ${
|
---|
635 | res.statusCode
|
---|
636 | }\n${content.toString("utf-8")}`
|
---|
637 | )
|
---|
638 | );
|
---|
639 | }
|
---|
640 |
|
---|
641 | const integrity = computeIntegrity(content);
|
---|
642 | const entry = { resolved: url, integrity, contentType };
|
---|
643 |
|
---|
644 | finishWith({
|
---|
645 | entry,
|
---|
646 | content
|
---|
647 | });
|
---|
648 | });
|
---|
649 | }
|
---|
650 | ).on("error", err => {
|
---|
651 | logger.log(`GET ${url} (error)`);
|
---|
652 | err.message += `\nwhile fetching ${url}`;
|
---|
653 | callback(err);
|
---|
654 | });
|
---|
655 | };
|
---|
656 |
|
---|
657 | const fetchContent = cachedWithKey(
|
---|
658 | /**
|
---|
659 | * @param {string} url URL
|
---|
660 | * @param {function(Error=, { validUntil: number, etag?: string, entry: LockfileEntry, content: Buffer, fresh: boolean } | { validUntil: number, etag?: string, location: string, fresh: boolean }=): void} callback callback
|
---|
661 | * @returns {void}
|
---|
662 | */ (url, callback) => {
|
---|
663 | cache.get(url, null, (err, cachedResult) => {
|
---|
664 | if (err) return callback(err);
|
---|
665 | if (cachedResult) {
|
---|
666 | const isValid = cachedResult.validUntil >= Date.now();
|
---|
667 | if (isValid) return callback(null, cachedResult);
|
---|
668 | }
|
---|
669 | fetchContentRaw(url, cachedResult, callback);
|
---|
670 | });
|
---|
671 | },
|
---|
672 | (url, callback) => fetchContentRaw(url, undefined, callback)
|
---|
673 | );
|
---|
674 |
|
---|
675 | const getInfo = cachedWithKey(
|
---|
676 | /**
|
---|
677 | * @param {string} url the url
|
---|
678 | * @param {function(Error=, { entry: LockfileEntry, content: Buffer }=): void} callback callback
|
---|
679 | * @returns {void}
|
---|
680 | */
|
---|
681 | (url, callback) => {
|
---|
682 | getLockfile((err, lockfile) => {
|
---|
683 | if (err) return callback(err);
|
---|
684 | const entryOrString = lockfile.entries.get(url);
|
---|
685 | if (!entryOrString) {
|
---|
686 | if (frozen) {
|
---|
687 | return callback(
|
---|
688 | new Error(
|
---|
689 | `${url} has no lockfile entry and lockfile is frozen`
|
---|
690 | )
|
---|
691 | );
|
---|
692 | }
|
---|
693 | resolveContent(url, null, (err, result) => {
|
---|
694 | if (err) return callback(err);
|
---|
695 | storeResult(lockfile, url, result, callback);
|
---|
696 | });
|
---|
697 | return;
|
---|
698 | }
|
---|
699 | if (typeof entryOrString === "string") {
|
---|
700 | const entryTag = entryOrString;
|
---|
701 | resolveContent(url, null, (err, result) => {
|
---|
702 | if (err) return callback(err);
|
---|
703 | if (!result.storeLock || entryTag === "ignore")
|
---|
704 | return callback(null, result);
|
---|
705 | if (frozen) {
|
---|
706 | return callback(
|
---|
707 | new Error(
|
---|
708 | `${url} used to have ${entryTag} lockfile entry and has content now, but lockfile is frozen`
|
---|
709 | )
|
---|
710 | );
|
---|
711 | }
|
---|
712 | if (!upgrade) {
|
---|
713 | return callback(
|
---|
714 | new Error(
|
---|
715 | `${url} used to have ${entryTag} lockfile entry and has content now.
|
---|
716 | This should be reflected in the lockfile, so this lockfile entry must be upgraded, but upgrading is not enabled.
|
---|
717 | Remove this line from the lockfile to force upgrading.`
|
---|
718 | )
|
---|
719 | );
|
---|
720 | }
|
---|
721 | storeResult(lockfile, url, result, callback);
|
---|
722 | });
|
---|
723 | return;
|
---|
724 | }
|
---|
725 | let entry = entryOrString;
|
---|
726 | const doFetch = lockedContent => {
|
---|
727 | resolveContent(url, entry.integrity, (err, result) => {
|
---|
728 | if (err) {
|
---|
729 | if (lockedContent) {
|
---|
730 | logger.warn(
|
---|
731 | `Upgrade request to ${url} failed: ${err.message}`
|
---|
732 | );
|
---|
733 | logger.debug(err.stack);
|
---|
734 | return callback(null, {
|
---|
735 | entry,
|
---|
736 | content: lockedContent
|
---|
737 | });
|
---|
738 | }
|
---|
739 | return callback(err);
|
---|
740 | }
|
---|
741 | if (!result.storeLock) {
|
---|
742 | // When the lockfile entry should be no-cache
|
---|
743 | // we need to update the lockfile
|
---|
744 | if (frozen) {
|
---|
745 | return callback(
|
---|
746 | new Error(
|
---|
747 | `${url} has a lockfile entry and is no-cache now, but lockfile is frozen\nLockfile: ${entryToString(
|
---|
748 | entry
|
---|
749 | )}`
|
---|
750 | )
|
---|
751 | );
|
---|
752 | }
|
---|
753 | storeResult(lockfile, url, result, callback);
|
---|
754 | return;
|
---|
755 | }
|
---|
756 | if (!areLockfileEntriesEqual(result.entry, entry)) {
|
---|
757 | // When the lockfile entry is outdated
|
---|
758 | // we need to update the lockfile
|
---|
759 | if (frozen) {
|
---|
760 | return callback(
|
---|
761 | new Error(
|
---|
762 | `${url} has an outdated lockfile entry, but lockfile is frozen\nLockfile: ${entryToString(
|
---|
763 | entry
|
---|
764 | )}\nExpected: ${entryToString(result.entry)}`
|
---|
765 | )
|
---|
766 | );
|
---|
767 | }
|
---|
768 | storeResult(lockfile, url, result, callback);
|
---|
769 | return;
|
---|
770 | }
|
---|
771 | if (!lockedContent && cacheLocation) {
|
---|
772 | // When the lockfile cache content is missing
|
---|
773 | // we need to update the lockfile
|
---|
774 | if (frozen) {
|
---|
775 | return callback(
|
---|
776 | new Error(
|
---|
777 | `${url} is missing content in the lockfile cache, but lockfile is frozen\nLockfile: ${entryToString(
|
---|
778 | entry
|
---|
779 | )}`
|
---|
780 | )
|
---|
781 | );
|
---|
782 | }
|
---|
783 | storeResult(lockfile, url, result, callback);
|
---|
784 | return;
|
---|
785 | }
|
---|
786 | return callback(null, result);
|
---|
787 | });
|
---|
788 | };
|
---|
789 | if (cacheLocation) {
|
---|
790 | // When there is a lockfile cache
|
---|
791 | // we read the content from there
|
---|
792 | const key = getCacheKey(entry.resolved);
|
---|
793 | const filePath = resolve(cacheLocation, key);
|
---|
794 | fs.readFile(filePath, (err, result) => {
|
---|
795 | const content = /** @type {Buffer} */ (result);
|
---|
796 | if (err) {
|
---|
797 | if (err.code === "ENOENT") return doFetch();
|
---|
798 | return callback(err);
|
---|
799 | }
|
---|
800 | const continueWithCachedContent = result => {
|
---|
801 | if (!upgrade) {
|
---|
802 | // When not in upgrade mode, we accept the result from the lockfile cache
|
---|
803 | return callback(null, { entry, content });
|
---|
804 | }
|
---|
805 | return doFetch(content);
|
---|
806 | };
|
---|
807 | if (!verifyIntegrity(content, entry.integrity)) {
|
---|
808 | let contentWithChangedEol;
|
---|
809 | let isEolChanged = false;
|
---|
810 | try {
|
---|
811 | contentWithChangedEol = Buffer.from(
|
---|
812 | content.toString("utf-8").replace(/\r\n/g, "\n")
|
---|
813 | );
|
---|
814 | isEolChanged = verifyIntegrity(
|
---|
815 | contentWithChangedEol,
|
---|
816 | entry.integrity
|
---|
817 | );
|
---|
818 | } catch (e) {
|
---|
819 | // ignore
|
---|
820 | }
|
---|
821 | if (isEolChanged) {
|
---|
822 | if (!warnedAboutEol) {
|
---|
823 | const explainer = `Incorrect end of line sequence was detected in the lockfile cache.
|
---|
824 | The lockfile cache is protected by integrity checks, so any external modification will lead to a corrupted lockfile cache.
|
---|
825 | When using git make sure to configure .gitattributes correctly for the lockfile cache:
|
---|
826 | **/*webpack.lock.data/** -text
|
---|
827 | This will avoid that the end of line sequence is changed by git on Windows.`;
|
---|
828 | if (frozen) {
|
---|
829 | logger.error(explainer);
|
---|
830 | } else {
|
---|
831 | logger.warn(explainer);
|
---|
832 | logger.info(
|
---|
833 | "Lockfile cache will be automatically fixed now, but when lockfile is frozen this would result in an error."
|
---|
834 | );
|
---|
835 | }
|
---|
836 | warnedAboutEol = true;
|
---|
837 | }
|
---|
838 | if (!frozen) {
|
---|
839 | // "fix" the end of line sequence of the lockfile content
|
---|
840 | logger.log(
|
---|
841 | `${filePath} fixed end of line sequence (\\r\\n instead of \\n).`
|
---|
842 | );
|
---|
843 | intermediateFs.writeFile(
|
---|
844 | filePath,
|
---|
845 | contentWithChangedEol,
|
---|
846 | err => {
|
---|
847 | if (err) return callback(err);
|
---|
848 | continueWithCachedContent(contentWithChangedEol);
|
---|
849 | }
|
---|
850 | );
|
---|
851 | return;
|
---|
852 | }
|
---|
853 | }
|
---|
854 | if (frozen) {
|
---|
855 | return callback(
|
---|
856 | new Error(
|
---|
857 | `${
|
---|
858 | entry.resolved
|
---|
859 | } integrity mismatch, expected content with integrity ${
|
---|
860 | entry.integrity
|
---|
861 | } but got ${computeIntegrity(content)}.
|
---|
862 | Lockfile corrupted (${
|
---|
863 | isEolChanged
|
---|
864 | ? "end of line sequence was unexpectedly changed"
|
---|
865 | : "incorrectly merged? changed by other tools?"
|
---|
866 | }).
|
---|
867 | Run build with un-frozen lockfile to automatically fix lockfile.`
|
---|
868 | )
|
---|
869 | );
|
---|
870 | } else {
|
---|
871 | // "fix" the lockfile entry to the correct integrity
|
---|
872 | // the content has priority over the integrity value
|
---|
873 | entry = {
|
---|
874 | ...entry,
|
---|
875 | integrity: computeIntegrity(content)
|
---|
876 | };
|
---|
877 | storeLockEntry(lockfile, url, entry);
|
---|
878 | }
|
---|
879 | }
|
---|
880 | continueWithCachedContent(result);
|
---|
881 | });
|
---|
882 | } else {
|
---|
883 | doFetch();
|
---|
884 | }
|
---|
885 | });
|
---|
886 | }
|
---|
887 | );
|
---|
888 |
|
---|
889 | const respondWithUrlModule = (url, resourceData, callback) => {
|
---|
890 | getInfo(url.href, (err, result) => {
|
---|
891 | if (err) return callback(err);
|
---|
892 | resourceData.resource = url.href;
|
---|
893 | resourceData.path = url.origin + url.pathname;
|
---|
894 | resourceData.query = url.search;
|
---|
895 | resourceData.fragment = url.hash;
|
---|
896 | resourceData.context = new URL(
|
---|
897 | ".",
|
---|
898 | result.entry.resolved
|
---|
899 | ).href.slice(0, -1);
|
---|
900 | resourceData.data.mimetype = result.entry.contentType;
|
---|
901 | callback(null, true);
|
---|
902 | });
|
---|
903 | };
|
---|
904 | normalModuleFactory.hooks.resolveForScheme
|
---|
905 | .for(scheme)
|
---|
906 | .tapAsync(
|
---|
907 | "HttpUriPlugin",
|
---|
908 | (resourceData, resolveData, callback) => {
|
---|
909 | respondWithUrlModule(
|
---|
910 | new URL(resourceData.resource),
|
---|
911 | resourceData,
|
---|
912 | callback
|
---|
913 | );
|
---|
914 | }
|
---|
915 | );
|
---|
916 | normalModuleFactory.hooks.resolveInScheme
|
---|
917 | .for(scheme)
|
---|
918 | .tapAsync("HttpUriPlugin", (resourceData, data, callback) => {
|
---|
919 | // Only handle relative urls (./xxx, ../xxx, /xxx, //xxx)
|
---|
920 | if (
|
---|
921 | data.dependencyType !== "url" &&
|
---|
922 | !/^\.{0,2}\//.test(resourceData.resource)
|
---|
923 | ) {
|
---|
924 | return callback();
|
---|
925 | }
|
---|
926 | respondWithUrlModule(
|
---|
927 | new URL(resourceData.resource, data.context + "/"),
|
---|
928 | resourceData,
|
---|
929 | callback
|
---|
930 | );
|
---|
931 | });
|
---|
932 | const hooks = NormalModule.getCompilationHooks(compilation);
|
---|
933 | hooks.readResourceForScheme
|
---|
934 | .for(scheme)
|
---|
935 | .tapAsync("HttpUriPlugin", (resource, module, callback) => {
|
---|
936 | return getInfo(resource, (err, result) => {
|
---|
937 | if (err) return callback(err);
|
---|
938 | callback(null, result.content);
|
---|
939 | });
|
---|
940 | });
|
---|
941 | hooks.needBuild.tapAsync(
|
---|
942 | "HttpUriPlugin",
|
---|
943 | (module, context, callback) => {
|
---|
944 | if (
|
---|
945 | module.resource &&
|
---|
946 | module.resource.startsWith(`${scheme}://`)
|
---|
947 | ) {
|
---|
948 | getInfo(module.resource, (err, result) => {
|
---|
949 | if (err) return callback(err);
|
---|
950 | if (
|
---|
951 | result.entry.integrity !==
|
---|
952 | module.buildInfo.resourceIntegrity
|
---|
953 | ) {
|
---|
954 | return callback(null, true);
|
---|
955 | }
|
---|
956 | callback();
|
---|
957 | });
|
---|
958 | } else {
|
---|
959 | return callback();
|
---|
960 | }
|
---|
961 | }
|
---|
962 | );
|
---|
963 | }
|
---|
964 | compilation.hooks.finishModules.tapAsync(
|
---|
965 | "HttpUriPlugin",
|
---|
966 | (modules, callback) => {
|
---|
967 | if (!outdatedLockfile) return callback();
|
---|
968 | intermediateFs.writeFile(
|
---|
969 | lockfileLocation,
|
---|
970 | outdatedLockfile.toString(),
|
---|
971 | callback
|
---|
972 | );
|
---|
973 | }
|
---|
974 | );
|
---|
975 | }
|
---|
976 | );
|
---|
977 | }
|
---|
978 | }
|
---|
979 |
|
---|
980 | module.exports = HttpUriPlugin;
|
---|