source: imaps-frontend/node_modules/webpack/lib/schemes/HttpUriPlugin.js@ 79a0317

main
Last change on this file since 79a0317 was 79a0317, checked in by stefan toskovski <stefantoska84@…>, 4 days ago

F4 Finalna Verzija
  • Property mode set to 100644
File size: 38.9 KB
Line 
1/*
2 MIT License http://www.opensource.org/licenses/mit-license.php
3 Author Tobias Koppers @sokra
4*/
5
6"use strict";
7
8const EventEmitter = require("events");
9const { extname, basename } = require("path");
10const { URL } = require("url");
11const { createGunzip, createBrotliDecompress, createInflate } = require("zlib");
12const NormalModule = require("../NormalModule");
13const createSchemaValidation = require("../util/create-schema-validation");
14const createHash = require("../util/createHash");
15const { mkdirp, dirname, join } = require("../util/fs");
16const memoize = require("../util/memoize");
17
18/** @typedef {import("http").IncomingMessage} IncomingMessage */
19/** @typedef {import("http").RequestOptions} RequestOptions */
20/** @typedef {import("net").Socket} Socket */
21/** @typedef {import("stream").Readable} Readable */
22/** @typedef {import("../../declarations/plugins/schemes/HttpUriPlugin").HttpUriPluginOptions} HttpUriPluginOptions */
23/** @typedef {import("../Compiler")} Compiler */
24/** @typedef {import("../FileSystemInfo").Snapshot} Snapshot */
25/** @typedef {import("../Module").BuildInfo} BuildInfo */
26/** @typedef {import("../NormalModuleFactory").ResourceDataWithData} ResourceDataWithData */
27/** @typedef {import("../util/fs").IntermediateFileSystem} IntermediateFileSystem */
28
29const getHttp = memoize(() => require("http"));
30const getHttps = memoize(() => require("https"));
31
32/**
33 * @param {typeof import("http") | typeof import("https")} request request
34 * @param {string | { toString: () => string } | undefined} proxy proxy
35 * @returns {function(URL, RequestOptions, function(IncomingMessage): void): EventEmitter} fn
36 */
37const proxyFetch = (request, proxy) => (url, options, callback) => {
38 const eventEmitter = new EventEmitter();
39
40 /**
41 * @param {Socket=} socket socket
42 * @returns {void}
43 */
44 const doRequest = socket => {
45 request
46 .get(url, { ...options, ...(socket && { socket }) }, callback)
47 .on("error", eventEmitter.emit.bind(eventEmitter, "error"));
48 };
49
50 if (proxy) {
51 const { hostname: host, port } = new URL(proxy);
52
53 getHttp()
54 .request({
55 host, // IP address of proxy server
56 port, // port of proxy server
57 method: "CONNECT",
58 path: url.host
59 })
60 .on("connect", (res, socket) => {
61 if (res.statusCode === 200) {
62 // connected to proxy server
63 doRequest(socket);
64 }
65 })
66 .on("error", err => {
67 eventEmitter.emit(
68 "error",
69 new Error(
70 `Failed to connect to proxy server "${proxy}": ${err.message}`
71 )
72 );
73 })
74 .end();
75 } else {
76 doRequest();
77 }
78
79 return eventEmitter;
80};
81
82/** @typedef {() => void} InProgressWriteItem */
83/** @type {InProgressWriteItem[] | undefined} */
84let inProgressWrite;
85
86const validate = createSchemaValidation(
87 require("../../schemas/plugins/schemes/HttpUriPlugin.check.js"),
88 () => require("../../schemas/plugins/schemes/HttpUriPlugin.json"),
89 {
90 name: "Http Uri Plugin",
91 baseDataPath: "options"
92 }
93);
94
95/**
96 * @param {string} str path
97 * @returns {string} safe path
98 */
99const toSafePath = str =>
100 str
101 .replace(/^[^a-zA-Z0-9]+|[^a-zA-Z0-9]+$/g, "")
102 .replace(/[^a-zA-Z0-9._-]+/g, "_");
103
104/**
105 * @param {Buffer} content content
106 * @returns {string} integrity
107 */
108const computeIntegrity = content => {
109 const hash = createHash("sha512");
110 hash.update(content);
111 const integrity = `sha512-${hash.digest("base64")}`;
112 return integrity;
113};
114
115/**
116 * @param {Buffer} content content
117 * @param {string} integrity integrity
118 * @returns {boolean} true, if integrity matches
119 */
120const verifyIntegrity = (content, integrity) => {
121 if (integrity === "ignore") return true;
122 return computeIntegrity(content) === integrity;
123};
124
125/**
126 * @param {string} str input
127 * @returns {Record<string, string>} parsed
128 */
129const parseKeyValuePairs = str => {
130 /** @type {Record<string, string>} */
131 const result = {};
132 for (const item of str.split(",")) {
133 const i = item.indexOf("=");
134 if (i >= 0) {
135 const key = item.slice(0, i).trim();
136 const value = item.slice(i + 1).trim();
137 result[key] = value;
138 } else {
139 const key = item.trim();
140 if (!key) continue;
141 result[key] = key;
142 }
143 }
144 return result;
145};
146
147/**
148 * @param {string | undefined} cacheControl Cache-Control header
149 * @param {number} requestTime timestamp of request
150 * @returns {{storeCache: boolean, storeLock: boolean, validUntil: number}} Logic for storing in cache and lockfile cache
151 */
152const parseCacheControl = (cacheControl, requestTime) => {
153 // When false resource is not stored in cache
154 let storeCache = true;
155 // When false resource is not stored in lockfile cache
156 let storeLock = true;
157 // Resource is only revalidated, after that timestamp and when upgrade is chosen
158 let validUntil = 0;
159 if (cacheControl) {
160 const parsed = parseKeyValuePairs(cacheControl);
161 if (parsed["no-cache"]) storeCache = storeLock = false;
162 if (parsed["max-age"] && !Number.isNaN(Number(parsed["max-age"]))) {
163 validUntil = requestTime + Number(parsed["max-age"]) * 1000;
164 }
165 if (parsed["must-revalidate"]) validUntil = 0;
166 }
167 return {
168 storeLock,
169 storeCache,
170 validUntil
171 };
172};
173
174/**
175 * @typedef {object} LockfileEntry
176 * @property {string} resolved
177 * @property {string} integrity
178 * @property {string} contentType
179 */
180
181/**
182 * @param {LockfileEntry} a first lockfile entry
183 * @param {LockfileEntry} b second lockfile entry
184 * @returns {boolean} true when equal, otherwise false
185 */
186const areLockfileEntriesEqual = (a, b) =>
187 a.resolved === b.resolved &&
188 a.integrity === b.integrity &&
189 a.contentType === b.contentType;
190
191/**
192 * @param {LockfileEntry} entry lockfile entry
193 * @returns {`resolved: ${string}, integrity: ${string}, contentType: ${*}`} stringified entry
194 */
195const entryToString = entry =>
196 `resolved: ${entry.resolved}, integrity: ${entry.integrity}, contentType: ${entry.contentType}`;
197
198class Lockfile {
199 constructor() {
200 this.version = 1;
201 /** @type {Map<string, LockfileEntry | "ignore" | "no-cache">} */
202 this.entries = new Map();
203 }
204
205 /**
206 * @param {string} content content of the lockfile
207 * @returns {Lockfile} lockfile
208 */
209 static parse(content) {
210 // TODO handle merge conflicts
211 const data = JSON.parse(content);
212 if (data.version !== 1)
213 throw new Error(`Unsupported lockfile version ${data.version}`);
214 const lockfile = new Lockfile();
215 for (const key of Object.keys(data)) {
216 if (key === "version") continue;
217 const entry = data[key];
218 lockfile.entries.set(
219 key,
220 typeof entry === "string"
221 ? entry
222 : {
223 resolved: key,
224 ...entry
225 }
226 );
227 }
228 return lockfile;
229 }
230
231 /**
232 * @returns {string} stringified lockfile
233 */
234 toString() {
235 let str = "{\n";
236 const entries = Array.from(this.entries).sort(([a], [b]) =>
237 a < b ? -1 : 1
238 );
239 for (const [key, entry] of entries) {
240 if (typeof entry === "string") {
241 str += ` ${JSON.stringify(key)}: ${JSON.stringify(entry)},\n`;
242 } else {
243 str += ` ${JSON.stringify(key)}: { `;
244 if (entry.resolved !== key)
245 str += `"resolved": ${JSON.stringify(entry.resolved)}, `;
246 str += `"integrity": ${JSON.stringify(
247 entry.integrity
248 )}, "contentType": ${JSON.stringify(entry.contentType)} },\n`;
249 }
250 }
251 str += ` "version": ${this.version}\n}\n`;
252 return str;
253 }
254}
255
256/**
257 * @template R
258 * @param {function(function(Error | null, R=): void): void} fn function
259 * @returns {function(function(Error | null, R=): void): void} cached function
260 */
261const cachedWithoutKey = fn => {
262 let inFlight = false;
263 /** @type {Error | undefined} */
264 let cachedError;
265 /** @type {R | undefined} */
266 let cachedResult;
267 /** @type {(function(Error| null, R=): void)[] | undefined} */
268 let cachedCallbacks;
269 return callback => {
270 if (inFlight) {
271 if (cachedResult !== undefined) return callback(null, cachedResult);
272 if (cachedError !== undefined) return callback(cachedError);
273 if (cachedCallbacks === undefined) cachedCallbacks = [callback];
274 else cachedCallbacks.push(callback);
275 return;
276 }
277 inFlight = true;
278 fn((err, result) => {
279 if (err) cachedError = err;
280 else cachedResult = result;
281 const callbacks = cachedCallbacks;
282 cachedCallbacks = undefined;
283 callback(err, result);
284 if (callbacks !== undefined) for (const cb of callbacks) cb(err, result);
285 });
286 };
287};
288
289/**
290 * @template T
291 * @template R
292 * @param {function(T, function(Error | null, R=): void): void} fn function
293 * @param {function(T, function(Error | null, R=): void): void=} forceFn function for the second try
294 * @returns {(function(T, function(Error | null, R=): void): void) & { force: function(T, function(Error | null, R=): void): void }} cached function
295 */
296const cachedWithKey = (fn, forceFn = fn) => {
297 /**
298 * @template R
299 * @typedef {{ result?: R, error?: Error, callbacks?: (function(Error | null, R=): void)[], force?: true }} CacheEntry
300 */
301 /** @type {Map<T, CacheEntry<R>>} */
302 const cache = new Map();
303 /**
304 * @param {T} arg arg
305 * @param {function(Error | null, R=): void} callback callback
306 * @returns {void}
307 */
308 const resultFn = (arg, callback) => {
309 const cacheEntry = cache.get(arg);
310 if (cacheEntry !== undefined) {
311 if (cacheEntry.result !== undefined)
312 return callback(null, cacheEntry.result);
313 if (cacheEntry.error !== undefined) return callback(cacheEntry.error);
314 if (cacheEntry.callbacks === undefined) cacheEntry.callbacks = [callback];
315 else cacheEntry.callbacks.push(callback);
316 return;
317 }
318 /** @type {CacheEntry<R>} */
319 const newCacheEntry = {
320 result: undefined,
321 error: undefined,
322 callbacks: undefined
323 };
324 cache.set(arg, newCacheEntry);
325 fn(arg, (err, result) => {
326 if (err) newCacheEntry.error = err;
327 else newCacheEntry.result = result;
328 const callbacks = newCacheEntry.callbacks;
329 newCacheEntry.callbacks = undefined;
330 callback(err, result);
331 if (callbacks !== undefined) for (const cb of callbacks) cb(err, result);
332 });
333 };
334 /**
335 * @param {T} arg arg
336 * @param {function(Error | null, R=): void} callback callback
337 * @returns {void}
338 */
339 resultFn.force = (arg, callback) => {
340 const cacheEntry = cache.get(arg);
341 if (cacheEntry !== undefined && cacheEntry.force) {
342 if (cacheEntry.result !== undefined)
343 return callback(null, cacheEntry.result);
344 if (cacheEntry.error !== undefined) return callback(cacheEntry.error);
345 if (cacheEntry.callbacks === undefined) cacheEntry.callbacks = [callback];
346 else cacheEntry.callbacks.push(callback);
347 return;
348 }
349 /** @type {CacheEntry<R>} */
350 const newCacheEntry = {
351 result: undefined,
352 error: undefined,
353 callbacks: undefined,
354 force: true
355 };
356 cache.set(arg, newCacheEntry);
357 forceFn(arg, (err, result) => {
358 if (err) newCacheEntry.error = err;
359 else newCacheEntry.result = result;
360 const callbacks = newCacheEntry.callbacks;
361 newCacheEntry.callbacks = undefined;
362 callback(err, result);
363 if (callbacks !== undefined) for (const cb of callbacks) cb(err, result);
364 });
365 };
366 return resultFn;
367};
368
369/**
370 * @typedef {object} LockfileCache
371 * @property {Lockfile} lockfile lockfile
372 * @property {Snapshot} snapshot snapshot
373 */
374
375/**
376 * @typedef {object} ResolveContentResult
377 * @property {LockfileEntry} entry lockfile entry
378 * @property {Buffer} content content
379 * @property {boolean} storeLock need store lockfile
380 */
381
382/** @typedef {{ storeCache: boolean, storeLock: boolean, validUntil: number, etag: string | undefined, fresh: boolean }} FetchResultMeta */
383/** @typedef {FetchResultMeta & { location: string }} RedirectFetchResult */
384/** @typedef {FetchResultMeta & { entry: LockfileEntry, content: Buffer }} ContentFetchResult */
385/** @typedef {RedirectFetchResult | ContentFetchResult} FetchResult */
386
387class HttpUriPlugin {
388 /**
389 * @param {HttpUriPluginOptions} options options
390 */
391 constructor(options) {
392 validate(options);
393 this._lockfileLocation = options.lockfileLocation;
394 this._cacheLocation = options.cacheLocation;
395 this._upgrade = options.upgrade;
396 this._frozen = options.frozen;
397 this._allowedUris = options.allowedUris;
398 this._proxy = options.proxy;
399 }
400
401 /**
402 * Apply the plugin
403 * @param {Compiler} compiler the compiler instance
404 * @returns {void}
405 */
406 apply(compiler) {
407 const proxy =
408 this._proxy || process.env.http_proxy || process.env.HTTP_PROXY;
409 const schemes = [
410 {
411 scheme: "http",
412 fetch: proxyFetch(getHttp(), proxy)
413 },
414 {
415 scheme: "https",
416 fetch: proxyFetch(getHttps(), proxy)
417 }
418 ];
419 /** @type {LockfileCache} */
420 let lockfileCache;
421 compiler.hooks.compilation.tap(
422 "HttpUriPlugin",
423 (compilation, { normalModuleFactory }) => {
424 const intermediateFs =
425 /** @type {IntermediateFileSystem} */
426 (compiler.intermediateFileSystem);
427 const fs = compilation.inputFileSystem;
428 const cache = compilation.getCache("webpack.HttpUriPlugin");
429 const logger = compilation.getLogger("webpack.HttpUriPlugin");
430 /** @type {string} */
431 const lockfileLocation =
432 this._lockfileLocation ||
433 join(
434 intermediateFs,
435 compiler.context,
436 compiler.name
437 ? `${toSafePath(compiler.name)}.webpack.lock`
438 : "webpack.lock"
439 );
440 /** @type {string | false} */
441 const cacheLocation =
442 this._cacheLocation !== undefined
443 ? this._cacheLocation
444 : `${lockfileLocation}.data`;
445 const upgrade = this._upgrade || false;
446 const frozen = this._frozen || false;
447 const hashFunction = "sha512";
448 const hashDigest = "hex";
449 const hashDigestLength = 20;
450 const allowedUris = this._allowedUris;
451
452 let warnedAboutEol = false;
453
454 /** @type {Map<string, string>} */
455 const cacheKeyCache = new Map();
456 /**
457 * @param {string} url the url
458 * @returns {string} the key
459 */
460 const getCacheKey = url => {
461 const cachedResult = cacheKeyCache.get(url);
462 if (cachedResult !== undefined) return cachedResult;
463 const result = _getCacheKey(url);
464 cacheKeyCache.set(url, result);
465 return result;
466 };
467
468 /**
469 * @param {string} url the url
470 * @returns {string} the key
471 */
472 const _getCacheKey = url => {
473 const parsedUrl = new URL(url);
474 const folder = toSafePath(parsedUrl.origin);
475 const name = toSafePath(parsedUrl.pathname);
476 const query = toSafePath(parsedUrl.search);
477 let ext = extname(name);
478 if (ext.length > 20) ext = "";
479 const basename = ext ? name.slice(0, -ext.length) : name;
480 const hash = createHash(hashFunction);
481 hash.update(url);
482 const digest = hash.digest(hashDigest).slice(0, hashDigestLength);
483 return `${folder.slice(-50)}/${`${basename}${
484 query ? `_${query}` : ""
485 }`.slice(0, 150)}_${digest}${ext}`;
486 };
487
488 const getLockfile = cachedWithoutKey(
489 /**
490 * @param {function(Error | null, Lockfile=): void} callback callback
491 * @returns {void}
492 */
493 callback => {
494 const readLockfile = () => {
495 intermediateFs.readFile(lockfileLocation, (err, buffer) => {
496 if (err && err.code !== "ENOENT") {
497 compilation.missingDependencies.add(lockfileLocation);
498 return callback(err);
499 }
500 compilation.fileDependencies.add(lockfileLocation);
501 compilation.fileSystemInfo.createSnapshot(
502 compiler.fsStartTime,
503 buffer ? [lockfileLocation] : [],
504 [],
505 buffer ? [] : [lockfileLocation],
506 { timestamp: true },
507 (err, s) => {
508 if (err) return callback(err);
509 const lockfile = buffer
510 ? Lockfile.parse(buffer.toString("utf-8"))
511 : new Lockfile();
512 lockfileCache = {
513 lockfile,
514 snapshot: /** @type {Snapshot} */ (s)
515 };
516 callback(null, lockfile);
517 }
518 );
519 });
520 };
521 if (lockfileCache) {
522 compilation.fileSystemInfo.checkSnapshotValid(
523 lockfileCache.snapshot,
524 (err, valid) => {
525 if (err) return callback(err);
526 if (!valid) return readLockfile();
527 callback(null, lockfileCache.lockfile);
528 }
529 );
530 } else {
531 readLockfile();
532 }
533 }
534 );
535
536 /** @typedef {Map<string, LockfileEntry | "ignore" | "no-cache">} LockfileUpdates */
537
538 /** @type {LockfileUpdates | undefined} */
539 let lockfileUpdates;
540
541 /**
542 * @param {Lockfile} lockfile lockfile instance
543 * @param {string} url url to store
544 * @param {LockfileEntry | "ignore" | "no-cache"} entry lockfile entry
545 */
546 const storeLockEntry = (lockfile, url, entry) => {
547 const oldEntry = lockfile.entries.get(url);
548 if (lockfileUpdates === undefined) lockfileUpdates = new Map();
549 lockfileUpdates.set(url, entry);
550 lockfile.entries.set(url, entry);
551 if (!oldEntry) {
552 logger.log(`${url} added to lockfile`);
553 } else if (typeof oldEntry === "string") {
554 if (typeof entry === "string") {
555 logger.log(`${url} updated in lockfile: ${oldEntry} -> ${entry}`);
556 } else {
557 logger.log(
558 `${url} updated in lockfile: ${oldEntry} -> ${entry.resolved}`
559 );
560 }
561 } else if (typeof entry === "string") {
562 logger.log(
563 `${url} updated in lockfile: ${oldEntry.resolved} -> ${entry}`
564 );
565 } else if (oldEntry.resolved !== entry.resolved) {
566 logger.log(
567 `${url} updated in lockfile: ${oldEntry.resolved} -> ${entry.resolved}`
568 );
569 } else if (oldEntry.integrity !== entry.integrity) {
570 logger.log(`${url} updated in lockfile: content changed`);
571 } else if (oldEntry.contentType !== entry.contentType) {
572 logger.log(
573 `${url} updated in lockfile: ${oldEntry.contentType} -> ${entry.contentType}`
574 );
575 } else {
576 logger.log(`${url} updated in lockfile`);
577 }
578 };
579
580 /**
581 * @param {Lockfile} lockfile lockfile
582 * @param {string} url url
583 * @param {ResolveContentResult} result result
584 * @param {function(Error | null, ResolveContentResult=): void} callback callback
585 * @returns {void}
586 */
587 const storeResult = (lockfile, url, result, callback) => {
588 if (result.storeLock) {
589 storeLockEntry(lockfile, url, result.entry);
590 if (!cacheLocation || !result.content)
591 return callback(null, result);
592 const key = getCacheKey(result.entry.resolved);
593 const filePath = join(intermediateFs, cacheLocation, key);
594 mkdirp(intermediateFs, dirname(intermediateFs, filePath), err => {
595 if (err) return callback(err);
596 intermediateFs.writeFile(filePath, result.content, err => {
597 if (err) return callback(err);
598 callback(null, result);
599 });
600 });
601 } else {
602 storeLockEntry(lockfile, url, "no-cache");
603 callback(null, result);
604 }
605 };
606
607 for (const { scheme, fetch } of schemes) {
608 /**
609 * @param {string} url URL
610 * @param {string | null} integrity integrity
611 * @param {function(Error | null, ResolveContentResult=): void} callback callback
612 */
613 const resolveContent = (url, integrity, callback) => {
614 /**
615 * @param {Error | null} err error
616 * @param {TODO} result result result
617 * @returns {void}
618 */
619 const handleResult = (err, result) => {
620 if (err) return callback(err);
621 if ("location" in result) {
622 return resolveContent(
623 result.location,
624 integrity,
625 (err, innerResult) => {
626 if (err) return callback(err);
627 const { entry, content, storeLock } =
628 /** @type {ResolveContentResult} */ (innerResult);
629 callback(null, {
630 entry,
631 content,
632 storeLock: storeLock && result.storeLock
633 });
634 }
635 );
636 }
637 if (
638 !result.fresh &&
639 integrity &&
640 result.entry.integrity !== integrity &&
641 !verifyIntegrity(result.content, integrity)
642 ) {
643 return fetchContent.force(url, handleResult);
644 }
645 return callback(null, {
646 entry: result.entry,
647 content: result.content,
648 storeLock: result.storeLock
649 });
650 };
651 fetchContent(url, handleResult);
652 };
653
654 /**
655 * @param {string} url URL
656 * @param {FetchResult | RedirectFetchResult | undefined} cachedResult result from cache
657 * @param {function(Error | null, FetchResult=): void} callback callback
658 * @returns {void}
659 */
660 const fetchContentRaw = (url, cachedResult, callback) => {
661 const requestTime = Date.now();
662 fetch(
663 new URL(url),
664 {
665 headers: {
666 "accept-encoding": "gzip, deflate, br",
667 "user-agent": "webpack",
668 "if-none-match": /** @type {TODO} */ (
669 cachedResult ? cachedResult.etag || null : null
670 )
671 }
672 },
673 res => {
674 const etag = res.headers.etag;
675 const location = res.headers.location;
676 const cacheControl = res.headers["cache-control"];
677 const { storeLock, storeCache, validUntil } = parseCacheControl(
678 cacheControl,
679 requestTime
680 );
681 /**
682 * @param {Partial<Pick<FetchResultMeta, "fresh">> & (Pick<RedirectFetchResult, "location"> | Pick<ContentFetchResult, "content" | "entry">)} partialResult result
683 * @returns {void}
684 */
685 const finishWith = partialResult => {
686 if ("location" in partialResult) {
687 logger.debug(
688 `GET ${url} [${res.statusCode}] -> ${partialResult.location}`
689 );
690 } else {
691 logger.debug(
692 `GET ${url} [${res.statusCode}] ${Math.ceil(
693 partialResult.content.length / 1024
694 )} kB${!storeLock ? " no-cache" : ""}`
695 );
696 }
697 const result = {
698 ...partialResult,
699 fresh: true,
700 storeLock,
701 storeCache,
702 validUntil,
703 etag
704 };
705 if (!storeCache) {
706 logger.log(
707 `${url} can't be stored in cache, due to Cache-Control header: ${cacheControl}`
708 );
709 return callback(null, result);
710 }
711 cache.store(
712 url,
713 null,
714 {
715 ...result,
716 fresh: false
717 },
718 err => {
719 if (err) {
720 logger.warn(
721 `${url} can't be stored in cache: ${err.message}`
722 );
723 logger.debug(err.stack);
724 }
725 callback(null, result);
726 }
727 );
728 };
729 if (res.statusCode === 304) {
730 const result = /** @type {FetchResult} */ (cachedResult);
731 if (
732 result.validUntil < validUntil ||
733 result.storeLock !== storeLock ||
734 result.storeCache !== storeCache ||
735 result.etag !== etag
736 ) {
737 return finishWith(result);
738 }
739 logger.debug(`GET ${url} [${res.statusCode}] (unchanged)`);
740 return callback(null, { ...result, fresh: true });
741 }
742 if (
743 location &&
744 res.statusCode &&
745 res.statusCode >= 301 &&
746 res.statusCode <= 308
747 ) {
748 const result = {
749 location: new URL(location, url).href
750 };
751 if (
752 !cachedResult ||
753 !("location" in cachedResult) ||
754 cachedResult.location !== result.location ||
755 cachedResult.validUntil < validUntil ||
756 cachedResult.storeLock !== storeLock ||
757 cachedResult.storeCache !== storeCache ||
758 cachedResult.etag !== etag
759 ) {
760 return finishWith(result);
761 }
762 logger.debug(`GET ${url} [${res.statusCode}] (unchanged)`);
763 return callback(null, {
764 ...result,
765 fresh: true,
766 storeLock,
767 storeCache,
768 validUntil,
769 etag
770 });
771 }
772 const contentType = res.headers["content-type"] || "";
773 /** @type {Buffer[]} */
774 const bufferArr = [];
775
776 const contentEncoding = res.headers["content-encoding"];
777 /** @type {Readable} */
778 let stream = res;
779 if (contentEncoding === "gzip") {
780 stream = stream.pipe(createGunzip());
781 } else if (contentEncoding === "br") {
782 stream = stream.pipe(createBrotliDecompress());
783 } else if (contentEncoding === "deflate") {
784 stream = stream.pipe(createInflate());
785 }
786
787 stream.on("data", chunk => {
788 bufferArr.push(chunk);
789 });
790
791 stream.on("end", () => {
792 if (!res.complete) {
793 logger.log(`GET ${url} [${res.statusCode}] (terminated)`);
794 return callback(new Error(`${url} request was terminated`));
795 }
796
797 const content = Buffer.concat(bufferArr);
798
799 if (res.statusCode !== 200) {
800 logger.log(`GET ${url} [${res.statusCode}]`);
801 return callback(
802 new Error(
803 `${url} request status code = ${
804 res.statusCode
805 }\n${content.toString("utf-8")}`
806 )
807 );
808 }
809
810 const integrity = computeIntegrity(content);
811 const entry = { resolved: url, integrity, contentType };
812
813 finishWith({
814 entry,
815 content
816 });
817 });
818 }
819 ).on("error", err => {
820 logger.log(`GET ${url} (error)`);
821 err.message += `\nwhile fetching ${url}`;
822 callback(err);
823 });
824 };
825
826 const fetchContent = cachedWithKey(
827 /**
828 * @param {string} url URL
829 * @param {function(Error | null, { validUntil: number, etag?: string, entry: LockfileEntry, content: Buffer, fresh: boolean } | { validUntil: number, etag?: string, location: string, fresh: boolean }=): void} callback callback
830 * @returns {void}
831 */
832 (url, callback) => {
833 cache.get(url, null, (err, cachedResult) => {
834 if (err) return callback(err);
835 if (cachedResult) {
836 const isValid = cachedResult.validUntil >= Date.now();
837 if (isValid) return callback(null, cachedResult);
838 }
839 fetchContentRaw(url, cachedResult, callback);
840 });
841 },
842 (url, callback) => fetchContentRaw(url, undefined, callback)
843 );
844
845 /**
846 * @param {string} uri uri
847 * @returns {boolean} true when allowed, otherwise false
848 */
849 const isAllowed = uri => {
850 for (const allowed of allowedUris) {
851 if (typeof allowed === "string") {
852 if (uri.startsWith(allowed)) return true;
853 } else if (typeof allowed === "function") {
854 if (allowed(uri)) return true;
855 } else if (allowed.test(uri)) {
856 return true;
857 }
858 }
859 return false;
860 };
861
862 /** @typedef {{ entry: LockfileEntry, content: Buffer }} Info */
863
864 const getInfo = cachedWithKey(
865 /**
866 * @param {string} url the url
867 * @param {function(Error | null, Info=): void} callback callback
868 * @returns {void}
869 */
870 // eslint-disable-next-line no-loop-func
871 (url, callback) => {
872 if (!isAllowed(url)) {
873 return callback(
874 new Error(
875 `${url} doesn't match the allowedUris policy. These URIs are allowed:\n${allowedUris
876 .map(uri => ` - ${uri}`)
877 .join("\n")}`
878 )
879 );
880 }
881 getLockfile((err, _lockfile) => {
882 if (err) return callback(err);
883 const lockfile = /** @type {Lockfile} */ (_lockfile);
884 const entryOrString = lockfile.entries.get(url);
885 if (!entryOrString) {
886 if (frozen) {
887 return callback(
888 new Error(
889 `${url} has no lockfile entry and lockfile is frozen`
890 )
891 );
892 }
893 resolveContent(url, null, (err, result) => {
894 if (err) return callback(err);
895 storeResult(
896 /** @type {Lockfile} */
897 (lockfile),
898 url,
899 /** @type {ResolveContentResult} */
900 (result),
901 callback
902 );
903 });
904 return;
905 }
906 if (typeof entryOrString === "string") {
907 const entryTag = entryOrString;
908 resolveContent(url, null, (err, _result) => {
909 if (err) return callback(err);
910 const result =
911 /** @type {ResolveContentResult} */
912 (_result);
913 if (!result.storeLock || entryTag === "ignore")
914 return callback(null, result);
915 if (frozen) {
916 return callback(
917 new Error(
918 `${url} used to have ${entryTag} lockfile entry and has content now, but lockfile is frozen`
919 )
920 );
921 }
922 if (!upgrade) {
923 return callback(
924 new Error(
925 `${url} used to have ${entryTag} lockfile entry and has content now.
926This should be reflected in the lockfile, so this lockfile entry must be upgraded, but upgrading is not enabled.
927Remove this line from the lockfile to force upgrading.`
928 )
929 );
930 }
931 storeResult(lockfile, url, result, callback);
932 });
933 return;
934 }
935 let entry = entryOrString;
936 /**
937 * @param {Buffer=} lockedContent locked content
938 */
939 const doFetch = lockedContent => {
940 resolveContent(url, entry.integrity, (err, _result) => {
941 if (err) {
942 if (lockedContent) {
943 logger.warn(
944 `Upgrade request to ${url} failed: ${err.message}`
945 );
946 logger.debug(err.stack);
947 return callback(null, {
948 entry,
949 content: lockedContent
950 });
951 }
952 return callback(err);
953 }
954 const result =
955 /** @type {ResolveContentResult} */
956 (_result);
957 if (!result.storeLock) {
958 // When the lockfile entry should be no-cache
959 // we need to update the lockfile
960 if (frozen) {
961 return callback(
962 new Error(
963 `${url} has a lockfile entry and is no-cache now, but lockfile is frozen\nLockfile: ${entryToString(
964 entry
965 )}`
966 )
967 );
968 }
969 storeResult(lockfile, url, result, callback);
970 return;
971 }
972 if (!areLockfileEntriesEqual(result.entry, entry)) {
973 // When the lockfile entry is outdated
974 // we need to update the lockfile
975 if (frozen) {
976 return callback(
977 new Error(
978 `${url} has an outdated lockfile entry, but lockfile is frozen\nLockfile: ${entryToString(
979 entry
980 )}\nExpected: ${entryToString(result.entry)}`
981 )
982 );
983 }
984 storeResult(lockfile, url, result, callback);
985 return;
986 }
987 if (!lockedContent && cacheLocation) {
988 // When the lockfile cache content is missing
989 // we need to update the lockfile
990 if (frozen) {
991 return callback(
992 new Error(
993 `${url} is missing content in the lockfile cache, but lockfile is frozen\nLockfile: ${entryToString(
994 entry
995 )}`
996 )
997 );
998 }
999 storeResult(lockfile, url, result, callback);
1000 return;
1001 }
1002 return callback(null, result);
1003 });
1004 };
1005 if (cacheLocation) {
1006 // When there is a lockfile cache
1007 // we read the content from there
1008 const key = getCacheKey(entry.resolved);
1009 const filePath = join(intermediateFs, cacheLocation, key);
1010 fs.readFile(filePath, (err, result) => {
1011 if (err) {
1012 if (err.code === "ENOENT") return doFetch();
1013 return callback(err);
1014 }
1015 const content = /** @type {Buffer} */ (result);
1016 /**
1017 * @param {Buffer | undefined} _result result
1018 * @returns {void}
1019 */
1020 const continueWithCachedContent = _result => {
1021 if (!upgrade) {
1022 // When not in upgrade mode, we accept the result from the lockfile cache
1023 return callback(null, { entry, content });
1024 }
1025 return doFetch(content);
1026 };
1027 if (!verifyIntegrity(content, entry.integrity)) {
1028 /** @type {Buffer | undefined} */
1029 let contentWithChangedEol;
1030 let isEolChanged = false;
1031 try {
1032 contentWithChangedEol = Buffer.from(
1033 content.toString("utf-8").replace(/\r\n/g, "\n")
1034 );
1035 isEolChanged = verifyIntegrity(
1036 contentWithChangedEol,
1037 entry.integrity
1038 );
1039 } catch (_err) {
1040 // ignore
1041 }
1042 if (isEolChanged) {
1043 if (!warnedAboutEol) {
1044 const explainer = `Incorrect end of line sequence was detected in the lockfile cache.
1045The lockfile cache is protected by integrity checks, so any external modification will lead to a corrupted lockfile cache.
1046When using git make sure to configure .gitattributes correctly for the lockfile cache:
1047 **/*webpack.lock.data/** -text
1048This will avoid that the end of line sequence is changed by git on Windows.`;
1049 if (frozen) {
1050 logger.error(explainer);
1051 } else {
1052 logger.warn(explainer);
1053 logger.info(
1054 "Lockfile cache will be automatically fixed now, but when lockfile is frozen this would result in an error."
1055 );
1056 }
1057 warnedAboutEol = true;
1058 }
1059 if (!frozen) {
1060 // "fix" the end of line sequence of the lockfile content
1061 logger.log(
1062 `${filePath} fixed end of line sequence (\\r\\n instead of \\n).`
1063 );
1064 intermediateFs.writeFile(
1065 filePath,
1066 /** @type {Buffer} */
1067 (contentWithChangedEol),
1068 err => {
1069 if (err) return callback(err);
1070 continueWithCachedContent(
1071 /** @type {Buffer} */
1072 (contentWithChangedEol)
1073 );
1074 }
1075 );
1076 return;
1077 }
1078 }
1079 if (frozen) {
1080 return callback(
1081 new Error(
1082 `${
1083 entry.resolved
1084 } integrity mismatch, expected content with integrity ${
1085 entry.integrity
1086 } but got ${computeIntegrity(content)}.
1087Lockfile corrupted (${
1088 isEolChanged
1089 ? "end of line sequence was unexpectedly changed"
1090 : "incorrectly merged? changed by other tools?"
1091 }).
1092Run build with un-frozen lockfile to automatically fix lockfile.`
1093 )
1094 );
1095 }
1096 // "fix" the lockfile entry to the correct integrity
1097 // the content has priority over the integrity value
1098 entry = {
1099 ...entry,
1100 integrity: computeIntegrity(content)
1101 };
1102 storeLockEntry(lockfile, url, entry);
1103 }
1104 continueWithCachedContent(result);
1105 });
1106 } else {
1107 doFetch();
1108 }
1109 });
1110 }
1111 );
1112
1113 /**
1114 * @param {URL} url url
1115 * @param {ResourceDataWithData} resourceData resource data
1116 * @param {function(Error | null, true | void): void} callback callback
1117 */
1118 const respondWithUrlModule = (url, resourceData, callback) => {
1119 getInfo(url.href, (err, _result) => {
1120 if (err) return callback(err);
1121 const result = /** @type {Info} */ (_result);
1122 resourceData.resource = url.href;
1123 resourceData.path = url.origin + url.pathname;
1124 resourceData.query = url.search;
1125 resourceData.fragment = url.hash;
1126 resourceData.context = new URL(
1127 ".",
1128 result.entry.resolved
1129 ).href.slice(0, -1);
1130 resourceData.data.mimetype = result.entry.contentType;
1131 callback(null, true);
1132 });
1133 };
1134 normalModuleFactory.hooks.resolveForScheme
1135 .for(scheme)
1136 .tapAsync(
1137 "HttpUriPlugin",
1138 (resourceData, resolveData, callback) => {
1139 respondWithUrlModule(
1140 new URL(resourceData.resource),
1141 resourceData,
1142 callback
1143 );
1144 }
1145 );
1146 normalModuleFactory.hooks.resolveInScheme
1147 .for(scheme)
1148 .tapAsync("HttpUriPlugin", (resourceData, data, callback) => {
1149 // Only handle relative urls (./xxx, ../xxx, /xxx, //xxx)
1150 if (
1151 data.dependencyType !== "url" &&
1152 !/^\.{0,2}\//.test(resourceData.resource)
1153 ) {
1154 return callback();
1155 }
1156 respondWithUrlModule(
1157 new URL(resourceData.resource, `${data.context}/`),
1158 resourceData,
1159 callback
1160 );
1161 });
1162 const hooks = NormalModule.getCompilationHooks(compilation);
1163 hooks.readResourceForScheme
1164 .for(scheme)
1165 .tapAsync("HttpUriPlugin", (resource, module, callback) =>
1166 getInfo(resource, (err, _result) => {
1167 if (err) return callback(err);
1168 const result = /** @type {Info} */ (_result);
1169 /** @type {BuildInfo} */
1170 (module.buildInfo).resourceIntegrity = result.entry.integrity;
1171 callback(null, result.content);
1172 })
1173 );
1174 hooks.needBuild.tapAsync(
1175 "HttpUriPlugin",
1176 (module, context, callback) => {
1177 if (
1178 module.resource &&
1179 module.resource.startsWith(`${scheme}://`)
1180 ) {
1181 getInfo(module.resource, (err, _result) => {
1182 if (err) return callback(err);
1183 const result = /** @type {Info} */ (_result);
1184 if (
1185 result.entry.integrity !==
1186 /** @type {BuildInfo} */
1187 (module.buildInfo).resourceIntegrity
1188 ) {
1189 return callback(null, true);
1190 }
1191 callback();
1192 });
1193 } else {
1194 return callback();
1195 }
1196 }
1197 );
1198 }
1199 compilation.hooks.finishModules.tapAsync(
1200 "HttpUriPlugin",
1201 (modules, callback) => {
1202 if (!lockfileUpdates) return callback();
1203 const ext = extname(lockfileLocation);
1204 const tempFile = join(
1205 intermediateFs,
1206 dirname(intermediateFs, lockfileLocation),
1207 `.${basename(lockfileLocation, ext)}.${
1208 (Math.random() * 10000) | 0
1209 }${ext}`
1210 );
1211
1212 const writeDone = () => {
1213 const nextOperation =
1214 /** @type {InProgressWriteItem[]} */
1215 (inProgressWrite).shift();
1216 if (nextOperation) {
1217 nextOperation();
1218 } else {
1219 inProgressWrite = undefined;
1220 }
1221 };
1222 const runWrite = () => {
1223 intermediateFs.readFile(lockfileLocation, (err, buffer) => {
1224 if (err && err.code !== "ENOENT") {
1225 writeDone();
1226 return callback(err);
1227 }
1228 const lockfile = buffer
1229 ? Lockfile.parse(buffer.toString("utf-8"))
1230 : new Lockfile();
1231 for (const [key, value] of /** @type {LockfileUpdates} */ (
1232 lockfileUpdates
1233 )) {
1234 lockfile.entries.set(key, value);
1235 }
1236 intermediateFs.writeFile(tempFile, lockfile.toString(), err => {
1237 if (err) {
1238 writeDone();
1239 return (
1240 /** @type {NonNullable<IntermediateFileSystem["unlink"]>} */
1241 (intermediateFs.unlink)(tempFile, () => callback(err))
1242 );
1243 }
1244 intermediateFs.rename(tempFile, lockfileLocation, err => {
1245 if (err) {
1246 writeDone();
1247 return (
1248 /** @type {NonNullable<IntermediateFileSystem["unlink"]>} */
1249 (intermediateFs.unlink)(tempFile, () => callback(err))
1250 );
1251 }
1252 writeDone();
1253 callback();
1254 });
1255 });
1256 });
1257 };
1258 if (inProgressWrite) {
1259 inProgressWrite.push(runWrite);
1260 } else {
1261 inProgressWrite = [];
1262 runWrite();
1263 }
1264 }
1265 );
1266 }
1267 );
1268 }
1269}
1270
1271module.exports = HttpUriPlugin;
Note: See TracBrowser for help on using the repository browser.