source: imaps-frontend/node_modules/webpack/lib/cache/PackFileCacheStrategy.js

main
Last change on this file was 79a0317, checked in by stefan toskovski <stefantoska84@…>, 3 days ago

F4 Finalna Verzija
  • Property mode set to 100644
File size: 43.2 KB
Line 
1/*
2 MIT License http://www.opensource.org/licenses/mit-license.php
3 Author Tobias Koppers @sokra
4*/
5
6"use strict";
7
8const FileSystemInfo = require("../FileSystemInfo");
9const ProgressPlugin = require("../ProgressPlugin");
10const { formatSize } = require("../SizeFormatHelpers");
11const SerializerMiddleware = require("../serialization/SerializerMiddleware");
12const LazySet = require("../util/LazySet");
13const makeSerializable = require("../util/makeSerializable");
14const memoize = require("../util/memoize");
15const {
16 createFileSerializer,
17 NOT_SERIALIZABLE
18} = require("../util/serialization");
19
20/** @typedef {import("../../declarations/WebpackOptions").SnapshotOptions} SnapshotOptions */
21/** @typedef {import("../Cache").Etag} Etag */
22/** @typedef {import("../Compiler")} Compiler */
23/** @typedef {import("../FileSystemInfo").ResolveBuildDependenciesResult} ResolveBuildDependenciesResult */
24/** @typedef {import("../FileSystemInfo").Snapshot} Snapshot */
25/** @typedef {import("../logging/Logger").Logger} Logger */
26/** @typedef {import("../serialization/ObjectMiddleware").ObjectDeserializerContext} ObjectDeserializerContext */
27/** @typedef {import("../serialization/ObjectMiddleware").ObjectSerializerContext} ObjectSerializerContext */
28/** @typedef {import("../util/fs").IntermediateFileSystem} IntermediateFileSystem */
29
30/** @typedef {Map<string, string | false>} ResolveResults */
31/** @typedef {Set<string>} Items */
32/** @typedef {Set<string>} BuildDependencies */
33/** @typedef {Map<string, PackItemInfo>} ItemInfo */
34
35class PackContainer {
36 /**
37 * @param {object} data stored data
38 * @param {string} version version identifier
39 * @param {Snapshot} buildSnapshot snapshot of all build dependencies
40 * @param {BuildDependencies} buildDependencies list of all unresolved build dependencies captured
41 * @param {ResolveResults} resolveResults result of the resolved build dependencies
42 * @param {Snapshot} resolveBuildDependenciesSnapshot snapshot of the dependencies of the build dependencies resolving
43 */
44 constructor(
45 data,
46 version,
47 buildSnapshot,
48 buildDependencies,
49 resolveResults,
50 resolveBuildDependenciesSnapshot
51 ) {
52 this.data = data;
53 this.version = version;
54 this.buildSnapshot = buildSnapshot;
55 this.buildDependencies = buildDependencies;
56 this.resolveResults = resolveResults;
57 this.resolveBuildDependenciesSnapshot = resolveBuildDependenciesSnapshot;
58 }
59
60 /**
61 * @param {ObjectSerializerContext} context context
62 */
63 serialize({ write, writeLazy }) {
64 write(this.version);
65 write(this.buildSnapshot);
66 write(this.buildDependencies);
67 write(this.resolveResults);
68 write(this.resolveBuildDependenciesSnapshot);
69 /** @type {NonNullable<ObjectSerializerContext["writeLazy"]>} */
70 (writeLazy)(this.data);
71 }
72
73 /**
74 * @param {ObjectDeserializerContext} context context
75 */
76 deserialize({ read }) {
77 this.version = read();
78 this.buildSnapshot = read();
79 this.buildDependencies = read();
80 this.resolveResults = read();
81 this.resolveBuildDependenciesSnapshot = read();
82 this.data = read();
83 }
84}
85
86makeSerializable(
87 PackContainer,
88 "webpack/lib/cache/PackFileCacheStrategy",
89 "PackContainer"
90);
91
92const MIN_CONTENT_SIZE = 1024 * 1024; // 1 MB
93const CONTENT_COUNT_TO_MERGE = 10;
94const MIN_ITEMS_IN_FRESH_PACK = 100;
95const MAX_ITEMS_IN_FRESH_PACK = 50000;
96const MAX_TIME_IN_FRESH_PACK = 1 * 60 * 1000; // 1 min
97
98class PackItemInfo {
99 /**
100 * @param {string} identifier identifier of item
101 * @param {string | null | undefined} etag etag of item
102 * @param {any} value fresh value of item
103 */
104 constructor(identifier, etag, value) {
105 this.identifier = identifier;
106 this.etag = etag;
107 this.location = -1;
108 this.lastAccess = Date.now();
109 this.freshValue = value;
110 }
111}
112
113class Pack {
114 /**
115 * @param {Logger} logger a logger
116 * @param {number} maxAge max age of cache items
117 */
118 constructor(logger, maxAge) {
119 /** @type {ItemInfo} */
120 this.itemInfo = new Map();
121 /** @type {(string | undefined)[]} */
122 this.requests = [];
123 this.requestsTimeout = undefined;
124 /** @type {ItemInfo} */
125 this.freshContent = new Map();
126 /** @type {(undefined | PackContent)[]} */
127 this.content = [];
128 this.invalid = false;
129 this.logger = logger;
130 this.maxAge = maxAge;
131 }
132
133 /**
134 * @param {string} identifier identifier
135 */
136 _addRequest(identifier) {
137 this.requests.push(identifier);
138 if (this.requestsTimeout === undefined) {
139 this.requestsTimeout = setTimeout(() => {
140 this.requests.push(undefined);
141 this.requestsTimeout = undefined;
142 }, MAX_TIME_IN_FRESH_PACK);
143 if (this.requestsTimeout.unref) this.requestsTimeout.unref();
144 }
145 }
146
147 stopCapturingRequests() {
148 if (this.requestsTimeout !== undefined) {
149 clearTimeout(this.requestsTimeout);
150 this.requestsTimeout = undefined;
151 }
152 }
153
154 /**
155 * @param {string} identifier unique name for the resource
156 * @param {string | null} etag etag of the resource
157 * @returns {any} cached content
158 */
159 get(identifier, etag) {
160 const info = this.itemInfo.get(identifier);
161 this._addRequest(identifier);
162 if (info === undefined) {
163 return;
164 }
165 if (info.etag !== etag) return null;
166 info.lastAccess = Date.now();
167 const loc = info.location;
168 if (loc === -1) {
169 return info.freshValue;
170 }
171 if (!this.content[loc]) {
172 return;
173 }
174 return /** @type {PackContent} */ (this.content[loc]).get(identifier);
175 }
176
177 /**
178 * @param {string} identifier unique name for the resource
179 * @param {string | null} etag etag of the resource
180 * @param {any} data cached content
181 * @returns {void}
182 */
183 set(identifier, etag, data) {
184 if (!this.invalid) {
185 this.invalid = true;
186 this.logger.log(`Pack got invalid because of write to: ${identifier}`);
187 }
188 const info = this.itemInfo.get(identifier);
189 if (info === undefined) {
190 const newInfo = new PackItemInfo(identifier, etag, data);
191 this.itemInfo.set(identifier, newInfo);
192 this._addRequest(identifier);
193 this.freshContent.set(identifier, newInfo);
194 } else {
195 const loc = info.location;
196 if (loc >= 0) {
197 this._addRequest(identifier);
198 this.freshContent.set(identifier, info);
199 const content = /** @type {PackContent} */ (this.content[loc]);
200 content.delete(identifier);
201 if (content.items.size === 0) {
202 this.content[loc] = undefined;
203 this.logger.debug("Pack %d got empty and is removed", loc);
204 }
205 }
206 info.freshValue = data;
207 info.lastAccess = Date.now();
208 info.etag = etag;
209 info.location = -1;
210 }
211 }
212
213 getContentStats() {
214 let count = 0;
215 let size = 0;
216 for (const content of this.content) {
217 if (content !== undefined) {
218 count++;
219 const s = content.getSize();
220 if (s > 0) {
221 size += s;
222 }
223 }
224 }
225 return { count, size };
226 }
227
228 /**
229 * @returns {number} new location of data entries
230 */
231 _findLocation() {
232 let i;
233 for (i = 0; i < this.content.length && this.content[i] !== undefined; i++);
234 return i;
235 }
236
237 /**
238 * @private
239 * @param {Items} items items
240 * @param {Items} usedItems used items
241 * @param {number} newLoc new location
242 */
243 _gcAndUpdateLocation(items, usedItems, newLoc) {
244 let count = 0;
245 let lastGC;
246 const now = Date.now();
247 for (const identifier of items) {
248 const info = /** @type {PackItemInfo} */ (this.itemInfo.get(identifier));
249 if (now - info.lastAccess > this.maxAge) {
250 this.itemInfo.delete(identifier);
251 items.delete(identifier);
252 usedItems.delete(identifier);
253 count++;
254 lastGC = identifier;
255 } else {
256 info.location = newLoc;
257 }
258 }
259 if (count > 0) {
260 this.logger.log(
261 "Garbage Collected %d old items at pack %d (%d items remaining) e. g. %s",
262 count,
263 newLoc,
264 items.size,
265 lastGC
266 );
267 }
268 }
269
270 _persistFreshContent() {
271 /** @typedef {{ items: Items, map: Map<string, any>, loc: number }} PackItem */
272 const itemsCount = this.freshContent.size;
273 if (itemsCount > 0) {
274 const packCount = Math.ceil(itemsCount / MAX_ITEMS_IN_FRESH_PACK);
275 const itemsPerPack = Math.ceil(itemsCount / packCount);
276 /** @type {PackItem[]} */
277 const packs = [];
278 let i = 0;
279 let ignoreNextTimeTick = false;
280 const createNextPack = () => {
281 const loc = this._findLocation();
282 this.content[loc] = /** @type {EXPECTED_ANY} */ (null); // reserve
283 /** @type {PackItem} */
284 const pack = {
285 items: new Set(),
286 map: new Map(),
287 loc
288 };
289 packs.push(pack);
290 return pack;
291 };
292 let pack = createNextPack();
293 if (this.requestsTimeout !== undefined)
294 clearTimeout(this.requestsTimeout);
295 for (const identifier of this.requests) {
296 if (identifier === undefined) {
297 if (ignoreNextTimeTick) {
298 ignoreNextTimeTick = false;
299 } else if (pack.items.size >= MIN_ITEMS_IN_FRESH_PACK) {
300 i = 0;
301 pack = createNextPack();
302 }
303 continue;
304 }
305 const info = this.freshContent.get(identifier);
306 if (info === undefined) continue;
307 pack.items.add(identifier);
308 pack.map.set(identifier, info.freshValue);
309 info.location = pack.loc;
310 info.freshValue = undefined;
311 this.freshContent.delete(identifier);
312 if (++i > itemsPerPack) {
313 i = 0;
314 pack = createNextPack();
315 ignoreNextTimeTick = true;
316 }
317 }
318 this.requests.length = 0;
319 for (const pack of packs) {
320 this.content[pack.loc] = new PackContent(
321 pack.items,
322 new Set(pack.items),
323 new PackContentItems(pack.map)
324 );
325 }
326 this.logger.log(
327 `${itemsCount} fresh items in cache put into pack ${
328 packs.length > 1
329 ? packs
330 .map(pack => `${pack.loc} (${pack.items.size} items)`)
331 .join(", ")
332 : packs[0].loc
333 }`
334 );
335 }
336 }
337
338 /**
339 * Merges small content files to a single content file
340 */
341 _optimizeSmallContent() {
342 // 1. Find all small content files
343 // Treat unused content files separately to avoid
344 // a merge-split cycle
345 /** @type {number[]} */
346 const smallUsedContents = [];
347 /** @type {number} */
348 let smallUsedContentSize = 0;
349 /** @type {number[]} */
350 const smallUnusedContents = [];
351 /** @type {number} */
352 let smallUnusedContentSize = 0;
353 for (let i = 0; i < this.content.length; i++) {
354 const content = this.content[i];
355 if (content === undefined) continue;
356 if (content.outdated) continue;
357 const size = content.getSize();
358 if (size < 0 || size > MIN_CONTENT_SIZE) continue;
359 if (content.used.size > 0) {
360 smallUsedContents.push(i);
361 smallUsedContentSize += size;
362 } else {
363 smallUnusedContents.push(i);
364 smallUnusedContentSize += size;
365 }
366 }
367
368 // 2. Check if minimum number is reached
369 let mergedIndices;
370 if (
371 smallUsedContents.length >= CONTENT_COUNT_TO_MERGE ||
372 smallUsedContentSize > MIN_CONTENT_SIZE
373 ) {
374 mergedIndices = smallUsedContents;
375 } else if (
376 smallUnusedContents.length >= CONTENT_COUNT_TO_MERGE ||
377 smallUnusedContentSize > MIN_CONTENT_SIZE
378 ) {
379 mergedIndices = smallUnusedContents;
380 } else return;
381
382 /** @type {PackContent[] } */
383 const mergedContent = [];
384
385 // 3. Remove old content entries
386 for (const i of mergedIndices) {
387 mergedContent.push(/** @type {PackContent} */ (this.content[i]));
388 this.content[i] = undefined;
389 }
390
391 // 4. Determine merged items
392 /** @type {Items} */
393 const mergedItems = new Set();
394 /** @type {Items} */
395 const mergedUsedItems = new Set();
396 /** @type {(function(Map<string, any>): Promise<void>)[]} */
397 const addToMergedMap = [];
398 for (const content of mergedContent) {
399 for (const identifier of content.items) {
400 mergedItems.add(identifier);
401 }
402 for (const identifier of content.used) {
403 mergedUsedItems.add(identifier);
404 }
405 addToMergedMap.push(async map => {
406 // unpack existing content
407 // after that values are accessible in .content
408 await content.unpack(
409 "it should be merged with other small pack contents"
410 );
411 for (const [identifier, value] of /** @type {Content} */ (
412 content.content
413 )) {
414 map.set(identifier, value);
415 }
416 });
417 }
418
419 // 5. GC and update location of merged items
420 const newLoc = this._findLocation();
421 this._gcAndUpdateLocation(mergedItems, mergedUsedItems, newLoc);
422
423 // 6. If not empty, store content somewhere
424 if (mergedItems.size > 0) {
425 this.content[newLoc] = new PackContent(
426 mergedItems,
427 mergedUsedItems,
428 memoize(async () => {
429 /** @type {Content} */
430 const map = new Map();
431 await Promise.all(addToMergedMap.map(fn => fn(map)));
432 return new PackContentItems(map);
433 })
434 );
435 this.logger.log(
436 "Merged %d small files with %d cache items into pack %d",
437 mergedContent.length,
438 mergedItems.size,
439 newLoc
440 );
441 }
442 }
443
444 /**
445 * Split large content files with used and unused items
446 * into two parts to separate used from unused items
447 */
448 _optimizeUnusedContent() {
449 // 1. Find a large content file with used and unused items
450 for (let i = 0; i < this.content.length; i++) {
451 const content = this.content[i];
452 if (content === undefined) continue;
453 const size = content.getSize();
454 if (size < MIN_CONTENT_SIZE) continue;
455 const used = content.used.size;
456 const total = content.items.size;
457 if (used > 0 && used < total) {
458 // 2. Remove this content
459 this.content[i] = undefined;
460
461 // 3. Determine items for the used content file
462 const usedItems = new Set(content.used);
463 const newLoc = this._findLocation();
464 this._gcAndUpdateLocation(usedItems, usedItems, newLoc);
465
466 // 4. Create content file for used items
467 if (usedItems.size > 0) {
468 this.content[newLoc] = new PackContent(
469 usedItems,
470 new Set(usedItems),
471 async () => {
472 await content.unpack(
473 "it should be splitted into used and unused items"
474 );
475 const map = new Map();
476 for (const identifier of usedItems) {
477 map.set(
478 identifier,
479 /** @type {Content} */
480 (content.content).get(identifier)
481 );
482 }
483 return new PackContentItems(map);
484 }
485 );
486 }
487
488 // 5. Determine items for the unused content file
489 const unusedItems = new Set(content.items);
490 const usedOfUnusedItems = new Set();
491 for (const identifier of usedItems) {
492 unusedItems.delete(identifier);
493 }
494 const newUnusedLoc = this._findLocation();
495 this._gcAndUpdateLocation(unusedItems, usedOfUnusedItems, newUnusedLoc);
496
497 // 6. Create content file for unused items
498 if (unusedItems.size > 0) {
499 this.content[newUnusedLoc] = new PackContent(
500 unusedItems,
501 usedOfUnusedItems,
502 async () => {
503 await content.unpack(
504 "it should be splitted into used and unused items"
505 );
506 const map = new Map();
507 for (const identifier of unusedItems) {
508 map.set(
509 identifier,
510 /** @type {Content} */
511 (content.content).get(identifier)
512 );
513 }
514 return new PackContentItems(map);
515 }
516 );
517 }
518
519 this.logger.log(
520 "Split pack %d into pack %d with %d used items and pack %d with %d unused items",
521 i,
522 newLoc,
523 usedItems.size,
524 newUnusedLoc,
525 unusedItems.size
526 );
527
528 // optimizing only one of them is good enough and
529 // reduces the amount of serialization needed
530 return;
531 }
532 }
533 }
534
535 /**
536 * Find the content with the oldest item and run GC on that.
537 * Only runs for one content to avoid large invalidation.
538 */
539 _gcOldestContent() {
540 /** @type {PackItemInfo | undefined} */
541 let oldest;
542 for (const info of this.itemInfo.values()) {
543 if (oldest === undefined || info.lastAccess < oldest.lastAccess) {
544 oldest = info;
545 }
546 }
547 if (
548 Date.now() - /** @type {PackItemInfo} */ (oldest).lastAccess >
549 this.maxAge
550 ) {
551 const loc = /** @type {PackItemInfo} */ (oldest).location;
552 if (loc < 0) return;
553 const content = /** @type {PackContent} */ (this.content[loc]);
554 const items = new Set(content.items);
555 const usedItems = new Set(content.used);
556 this._gcAndUpdateLocation(items, usedItems, loc);
557
558 this.content[loc] =
559 items.size > 0
560 ? new PackContent(items, usedItems, async () => {
561 await content.unpack(
562 "it contains old items that should be garbage collected"
563 );
564 const map = new Map();
565 for (const identifier of items) {
566 map.set(
567 identifier,
568 /** @type {Content} */
569 (content.content).get(identifier)
570 );
571 }
572 return new PackContentItems(map);
573 })
574 : undefined;
575 }
576 }
577
578 /**
579 * @param {ObjectSerializerContext} context context
580 */
581 serialize({ write, writeSeparate }) {
582 this._persistFreshContent();
583 this._optimizeSmallContent();
584 this._optimizeUnusedContent();
585 this._gcOldestContent();
586 for (const identifier of this.itemInfo.keys()) {
587 write(identifier);
588 }
589 write(null); // null as marker of the end of keys
590 for (const info of this.itemInfo.values()) {
591 write(info.etag);
592 }
593 for (const info of this.itemInfo.values()) {
594 write(info.lastAccess);
595 }
596 for (let i = 0; i < this.content.length; i++) {
597 const content = this.content[i];
598 if (content !== undefined) {
599 write(content.items);
600 content.writeLazy(lazy => writeSeparate(lazy, { name: `${i}` }));
601 } else {
602 write(undefined); // undefined marks an empty content slot
603 }
604 }
605 write(null); // null as marker of the end of items
606 }
607
608 /**
609 * @param {ObjectDeserializerContext & { logger: Logger }} context context
610 */
611 deserialize({ read, logger }) {
612 this.logger = logger;
613 {
614 const items = [];
615 let item = read();
616 while (item !== null) {
617 items.push(item);
618 item = read();
619 }
620 this.itemInfo.clear();
621 const infoItems = items.map(identifier => {
622 const info = new PackItemInfo(identifier, undefined, undefined);
623 this.itemInfo.set(identifier, info);
624 return info;
625 });
626 for (const info of infoItems) {
627 info.etag = read();
628 }
629 for (const info of infoItems) {
630 info.lastAccess = read();
631 }
632 }
633 this.content.length = 0;
634 let items = read();
635 while (items !== null) {
636 if (items === undefined) {
637 this.content.push(items);
638 } else {
639 const idx = this.content.length;
640 const lazy = read();
641 this.content.push(
642 new PackContent(
643 items,
644 new Set(),
645 lazy,
646 logger,
647 `${this.content.length}`
648 )
649 );
650 for (const identifier of items) {
651 /** @type {PackItemInfo} */
652 (this.itemInfo.get(identifier)).location = idx;
653 }
654 }
655 items = read();
656 }
657 }
658}
659
660makeSerializable(Pack, "webpack/lib/cache/PackFileCacheStrategy", "Pack");
661
662/** @typedef {Map<string, any>} Content */
663
664class PackContentItems {
665 /**
666 * @param {Content} map items
667 */
668 constructor(map) {
669 this.map = map;
670 }
671
672 /**
673 * @param {ObjectSerializerContext & { snapshot: TODO, rollback: TODO, logger: Logger, profile: boolean | undefined }} context context
674 */
675 serialize({ write, snapshot, rollback, logger, profile }) {
676 if (profile) {
677 write(false);
678 for (const [key, value] of this.map) {
679 const s = snapshot();
680 try {
681 write(key);
682 const start = process.hrtime();
683 write(value);
684 const durationHr = process.hrtime(start);
685 const duration = durationHr[0] * 1000 + durationHr[1] / 1e6;
686 if (duration > 1) {
687 if (duration > 500)
688 logger.error(`Serialization of '${key}': ${duration} ms`);
689 else if (duration > 50)
690 logger.warn(`Serialization of '${key}': ${duration} ms`);
691 else if (duration > 10)
692 logger.info(`Serialization of '${key}': ${duration} ms`);
693 else if (duration > 5)
694 logger.log(`Serialization of '${key}': ${duration} ms`);
695 else logger.debug(`Serialization of '${key}': ${duration} ms`);
696 }
697 } catch (err) {
698 rollback(s);
699 if (err === NOT_SERIALIZABLE) continue;
700 const msg = "Skipped not serializable cache item";
701 const notSerializableErr = /** @type {Error} */ (err);
702 if (notSerializableErr.message.includes("ModuleBuildError")) {
703 logger.log(
704 `${msg} (in build error): ${notSerializableErr.message}`
705 );
706 logger.debug(
707 `${msg} '${key}' (in build error): ${notSerializableErr.stack}`
708 );
709 } else {
710 logger.warn(`${msg}: ${notSerializableErr.message}`);
711 logger.debug(`${msg} '${key}': ${notSerializableErr.stack}`);
712 }
713 }
714 }
715 write(null);
716 return;
717 }
718 // Try to serialize all at once
719 const s = snapshot();
720 try {
721 write(true);
722 write(this.map);
723 } catch (_err) {
724 rollback(s);
725
726 // Try to serialize each item on it's own
727 write(false);
728 for (const [key, value] of this.map) {
729 const s = snapshot();
730 try {
731 write(key);
732 write(value);
733 } catch (err) {
734 rollback(s);
735 if (err === NOT_SERIALIZABLE) continue;
736 const notSerializableErr = /** @type {Error} */ (err);
737 logger.warn(
738 `Skipped not serializable cache item '${key}': ${notSerializableErr.message}`
739 );
740 logger.debug(notSerializableErr.stack);
741 }
742 }
743 write(null);
744 }
745 }
746
747 /**
748 * @param {ObjectDeserializerContext & { logger: Logger, profile: boolean | undefined }} context context
749 */
750 deserialize({ read, logger, profile }) {
751 if (read()) {
752 this.map = read();
753 } else if (profile) {
754 const map = new Map();
755 let key = read();
756 while (key !== null) {
757 const start = process.hrtime();
758 const value = read();
759 const durationHr = process.hrtime(start);
760 const duration = durationHr[0] * 1000 + durationHr[1] / 1e6;
761 if (duration > 1) {
762 if (duration > 100)
763 logger.error(`Deserialization of '${key}': ${duration} ms`);
764 else if (duration > 20)
765 logger.warn(`Deserialization of '${key}': ${duration} ms`);
766 else if (duration > 5)
767 logger.info(`Deserialization of '${key}': ${duration} ms`);
768 else if (duration > 2)
769 logger.log(`Deserialization of '${key}': ${duration} ms`);
770 else logger.debug(`Deserialization of '${key}': ${duration} ms`);
771 }
772 map.set(key, value);
773 key = read();
774 }
775 this.map = map;
776 } else {
777 const map = new Map();
778 let key = read();
779 while (key !== null) {
780 map.set(key, read());
781 key = read();
782 }
783 this.map = map;
784 }
785 }
786}
787
788makeSerializable(
789 PackContentItems,
790 "webpack/lib/cache/PackFileCacheStrategy",
791 "PackContentItems"
792);
793
794/** @typedef {(function(): Promise<PackContentItems> | PackContentItems)} LazyFn */
795
796class PackContent {
797 /*
798 This class can be in these states:
799 | this.lazy | this.content | this.outdated | state
800 A1 | undefined | Map | false | fresh content
801 A2 | undefined | Map | true | (will not happen)
802 B1 | lazy () => {} | undefined | false | not deserialized
803 B2 | lazy () => {} | undefined | true | not deserialized, but some items has been removed
804 C1 | lazy* () => {} | Map | false | deserialized
805 C2 | lazy* () => {} | Map | true | deserialized, and some items has been removed
806
807 this.used is a subset of this.items.
808 this.items is a subset of this.content.keys() resp. this.lazy().map.keys()
809 When this.outdated === false, this.items === this.content.keys() resp. this.lazy().map.keys()
810 When this.outdated === true, this.items should be used to recreated this.lazy/this.content.
811 When this.lazy and this.content is set, they contain the same data.
812 this.get must only be called with a valid item from this.items.
813 In state C this.lazy is unMemoized
814 */
815
816 /**
817 * @param {Items} items keys
818 * @param {Items} usedItems used keys
819 * @param {PackContentItems | function(): Promise<PackContentItems>} dataOrFn sync or async content
820 * @param {Logger=} logger logger for logging
821 * @param {string=} lazyName name of dataOrFn for logging
822 */
823 constructor(items, usedItems, dataOrFn, logger, lazyName) {
824 this.items = items;
825 /** @type {LazyFn | undefined} */
826 this.lazy = typeof dataOrFn === "function" ? dataOrFn : undefined;
827 /** @type {Content | undefined} */
828 this.content = typeof dataOrFn === "function" ? undefined : dataOrFn.map;
829 this.outdated = false;
830 this.used = usedItems;
831 this.logger = logger;
832 this.lazyName = lazyName;
833 }
834
835 /**
836 * @param {string} identifier identifier
837 * @returns {string | Promise<string>} result
838 */
839 get(identifier) {
840 this.used.add(identifier);
841 if (this.content) {
842 return this.content.get(identifier);
843 }
844
845 const logger = /** @type {Logger} */ (this.logger);
846 // We are in state B
847 const { lazyName } = this;
848 /** @type {string | undefined} */
849 let timeMessage;
850 if (lazyName) {
851 // only log once
852 this.lazyName = undefined;
853 timeMessage = `restore cache content ${lazyName} (${formatSize(
854 this.getSize()
855 )})`;
856 logger.log(
857 `starting to restore cache content ${lazyName} (${formatSize(
858 this.getSize()
859 )}) because of request to: ${identifier}`
860 );
861 logger.time(timeMessage);
862 }
863 const value = /** @type {LazyFn} */ (this.lazy)();
864 if ("then" in value) {
865 return value.then(data => {
866 const map = data.map;
867 if (timeMessage) {
868 logger.timeEnd(timeMessage);
869 }
870 // Move to state C
871 this.content = map;
872 this.lazy = SerializerMiddleware.unMemoizeLazy(
873 /** @type {LazyFn} */
874 (this.lazy)
875 );
876 return map.get(identifier);
877 });
878 }
879
880 const map = value.map;
881 if (timeMessage) {
882 logger.timeEnd(timeMessage);
883 }
884 // Move to state C
885 this.content = map;
886 this.lazy = SerializerMiddleware.unMemoizeLazy(
887 /** @type {LazyFn} */
888 (this.lazy)
889 );
890 return map.get(identifier);
891 }
892
893 /**
894 * @param {string} reason explanation why unpack is necessary
895 * @returns {void | Promise<void>} maybe a promise if lazy
896 */
897 unpack(reason) {
898 if (this.content) return;
899
900 const logger = /** @type {Logger} */ (this.logger);
901 // Move from state B to C
902 if (this.lazy) {
903 const { lazyName } = this;
904 /** @type {string | undefined} */
905 let timeMessage;
906 if (lazyName) {
907 // only log once
908 this.lazyName = undefined;
909 timeMessage = `unpack cache content ${lazyName} (${formatSize(
910 this.getSize()
911 )})`;
912 logger.log(
913 `starting to unpack cache content ${lazyName} (${formatSize(
914 this.getSize()
915 )}) because ${reason}`
916 );
917 logger.time(timeMessage);
918 }
919 const value = this.lazy();
920 if ("then" in value) {
921 return value.then(data => {
922 if (timeMessage) {
923 logger.timeEnd(timeMessage);
924 }
925 this.content = data.map;
926 });
927 }
928 if (timeMessage) {
929 logger.timeEnd(timeMessage);
930 }
931 this.content = value.map;
932 }
933 }
934
935 /**
936 * @returns {number} size of the content or -1 if not known
937 */
938 getSize() {
939 if (!this.lazy) return -1;
940 const options = /** @type {any} */ (this.lazy).options;
941 if (!options) return -1;
942 const size = options.size;
943 if (typeof size !== "number") return -1;
944 return size;
945 }
946
947 /**
948 * @param {string} identifier identifier
949 */
950 delete(identifier) {
951 this.items.delete(identifier);
952 this.used.delete(identifier);
953 this.outdated = true;
954 }
955
956 /**
957 * @template T
958 * @param {function(any): function(): Promise<PackContentItems> | PackContentItems} write write function
959 * @returns {void}
960 */
961 writeLazy(write) {
962 if (!this.outdated && this.lazy) {
963 // State B1 or C1
964 // this.lazy is still the valid deserialized version
965 write(this.lazy);
966 return;
967 }
968 if (!this.outdated && this.content) {
969 // State A1
970 const map = new Map(this.content);
971 // Move to state C1
972 this.lazy = SerializerMiddleware.unMemoizeLazy(
973 write(() => new PackContentItems(map))
974 );
975 return;
976 }
977 if (this.content) {
978 // State A2 or C2
979 /** @type {Content} */
980 const map = new Map();
981 for (const item of this.items) {
982 map.set(item, this.content.get(item));
983 }
984 // Move to state C1
985 this.outdated = false;
986 this.content = map;
987 this.lazy = SerializerMiddleware.unMemoizeLazy(
988 write(() => new PackContentItems(map))
989 );
990 return;
991 }
992 const logger = /** @type {Logger} */ (this.logger);
993 // State B2
994 const { lazyName } = this;
995 /** @type {string | undefined} */
996 let timeMessage;
997 if (lazyName) {
998 // only log once
999 this.lazyName = undefined;
1000 timeMessage = `unpack cache content ${lazyName} (${formatSize(
1001 this.getSize()
1002 )})`;
1003 logger.log(
1004 `starting to unpack cache content ${lazyName} (${formatSize(
1005 this.getSize()
1006 )}) because it's outdated and need to be serialized`
1007 );
1008 logger.time(timeMessage);
1009 }
1010 const value = /** @type {LazyFn} */ (this.lazy)();
1011 this.outdated = false;
1012 if ("then" in value) {
1013 // Move to state B1
1014 this.lazy = write(() =>
1015 value.then(data => {
1016 if (timeMessage) {
1017 logger.timeEnd(timeMessage);
1018 }
1019 const oldMap = data.map;
1020 /** @type {Content} */
1021 const map = new Map();
1022 for (const item of this.items) {
1023 map.set(item, oldMap.get(item));
1024 }
1025 // Move to state C1 (or maybe C2)
1026 this.content = map;
1027 this.lazy = SerializerMiddleware.unMemoizeLazy(
1028 /** @type {LazyFn} */
1029 (this.lazy)
1030 );
1031
1032 return new PackContentItems(map);
1033 })
1034 );
1035 } else {
1036 // Move to state C1
1037 if (timeMessage) {
1038 logger.timeEnd(timeMessage);
1039 }
1040 const oldMap = value.map;
1041 /** @type {Content} */
1042 const map = new Map();
1043 for (const item of this.items) {
1044 map.set(item, oldMap.get(item));
1045 }
1046 this.content = map;
1047 this.lazy = write(() => new PackContentItems(map));
1048 }
1049 }
1050}
1051
1052/**
1053 * @param {Buffer} buf buffer
1054 * @returns {Buffer} buffer that can be collected
1055 */
1056const allowCollectingMemory = buf => {
1057 const wasted = buf.buffer.byteLength - buf.byteLength;
1058 if (wasted > 8192 && (wasted > 1048576 || wasted > buf.byteLength)) {
1059 return Buffer.from(buf);
1060 }
1061 return buf;
1062};
1063
1064class PackFileCacheStrategy {
1065 /**
1066 * @param {object} options options
1067 * @param {Compiler} options.compiler the compiler
1068 * @param {IntermediateFileSystem} options.fs the filesystem
1069 * @param {string} options.context the context directory
1070 * @param {string} options.cacheLocation the location of the cache data
1071 * @param {string} options.version version identifier
1072 * @param {Logger} options.logger a logger
1073 * @param {SnapshotOptions} options.snapshot options regarding snapshotting
1074 * @param {number} options.maxAge max age of cache items
1075 * @param {boolean | undefined} options.profile track and log detailed timing information for individual cache items
1076 * @param {boolean | undefined} options.allowCollectingMemory allow to collect unused memory created during deserialization
1077 * @param {false | "gzip" | "brotli" | undefined} options.compression compression used
1078 * @param {boolean | undefined} options.readonly disable storing cache into filesystem
1079 */
1080 constructor({
1081 compiler,
1082 fs,
1083 context,
1084 cacheLocation,
1085 version,
1086 logger,
1087 snapshot,
1088 maxAge,
1089 profile,
1090 allowCollectingMemory,
1091 compression,
1092 readonly
1093 }) {
1094 this.fileSerializer = createFileSerializer(
1095 fs,
1096 compiler.options.output.hashFunction
1097 );
1098 this.fileSystemInfo = new FileSystemInfo(fs, {
1099 managedPaths: snapshot.managedPaths,
1100 immutablePaths: snapshot.immutablePaths,
1101 logger: logger.getChildLogger("webpack.FileSystemInfo"),
1102 hashFunction: compiler.options.output.hashFunction
1103 });
1104 this.compiler = compiler;
1105 this.context = context;
1106 this.cacheLocation = cacheLocation;
1107 this.version = version;
1108 this.logger = logger;
1109 this.maxAge = maxAge;
1110 this.profile = profile;
1111 this.readonly = readonly;
1112 this.allowCollectingMemory = allowCollectingMemory;
1113 this.compression = compression;
1114 this._extension =
1115 compression === "brotli"
1116 ? ".pack.br"
1117 : compression === "gzip"
1118 ? ".pack.gz"
1119 : ".pack";
1120 this.snapshot = snapshot;
1121 /** @type {BuildDependencies} */
1122 this.buildDependencies = new Set();
1123 /** @type {LazySet<string>} */
1124 this.newBuildDependencies = new LazySet();
1125 /** @type {Snapshot | undefined} */
1126 this.resolveBuildDependenciesSnapshot = undefined;
1127 /** @type {ResolveResults | undefined} */
1128 this.resolveResults = undefined;
1129 /** @type {Snapshot | undefined} */
1130 this.buildSnapshot = undefined;
1131 /** @type {Promise<Pack> | undefined} */
1132 this.packPromise = this._openPack();
1133 this.storePromise = Promise.resolve();
1134 }
1135
1136 /**
1137 * @returns {Promise<Pack>} pack
1138 */
1139 _getPack() {
1140 if (this.packPromise === undefined) {
1141 this.packPromise = this.storePromise.then(() => this._openPack());
1142 }
1143 return this.packPromise;
1144 }
1145
1146 /**
1147 * @returns {Promise<Pack>} the pack
1148 */
1149 _openPack() {
1150 const { logger, profile, cacheLocation, version } = this;
1151 /** @type {Snapshot} */
1152 let buildSnapshot;
1153 /** @type {BuildDependencies} */
1154 let buildDependencies;
1155 /** @type {BuildDependencies} */
1156 let newBuildDependencies;
1157 /** @type {Snapshot} */
1158 let resolveBuildDependenciesSnapshot;
1159 /** @type {ResolveResults | undefined} */
1160 let resolveResults;
1161 logger.time("restore cache container");
1162 return this.fileSerializer
1163 .deserialize(null, {
1164 filename: `${cacheLocation}/index${this._extension}`,
1165 extension: `${this._extension}`,
1166 logger,
1167 profile,
1168 retainedBuffer: this.allowCollectingMemory
1169 ? allowCollectingMemory
1170 : undefined
1171 })
1172 .catch(err => {
1173 if (err.code !== "ENOENT") {
1174 logger.warn(
1175 `Restoring pack failed from ${cacheLocation}${this._extension}: ${err}`
1176 );
1177 logger.debug(err.stack);
1178 } else {
1179 logger.debug(
1180 `No pack exists at ${cacheLocation}${this._extension}: ${err}`
1181 );
1182 }
1183 return undefined;
1184 })
1185 .then(packContainer => {
1186 logger.timeEnd("restore cache container");
1187 if (!packContainer) return;
1188 if (!(packContainer instanceof PackContainer)) {
1189 logger.warn(
1190 `Restored pack from ${cacheLocation}${this._extension}, but contained content is unexpected.`,
1191 packContainer
1192 );
1193 return;
1194 }
1195 if (packContainer.version !== version) {
1196 logger.log(
1197 `Restored pack from ${cacheLocation}${this._extension}, but version doesn't match.`
1198 );
1199 return;
1200 }
1201 logger.time("check build dependencies");
1202 return Promise.all([
1203 new Promise((resolve, reject) => {
1204 this.fileSystemInfo.checkSnapshotValid(
1205 packContainer.buildSnapshot,
1206 (err, valid) => {
1207 if (err) {
1208 logger.log(
1209 `Restored pack from ${cacheLocation}${this._extension}, but checking snapshot of build dependencies errored: ${err}.`
1210 );
1211 logger.debug(err.stack);
1212 return resolve(false);
1213 }
1214 if (!valid) {
1215 logger.log(
1216 `Restored pack from ${cacheLocation}${this._extension}, but build dependencies have changed.`
1217 );
1218 return resolve(false);
1219 }
1220 buildSnapshot = packContainer.buildSnapshot;
1221 return resolve(true);
1222 }
1223 );
1224 }),
1225 new Promise((resolve, reject) => {
1226 this.fileSystemInfo.checkSnapshotValid(
1227 packContainer.resolveBuildDependenciesSnapshot,
1228 (err, valid) => {
1229 if (err) {
1230 logger.log(
1231 `Restored pack from ${cacheLocation}${this._extension}, but checking snapshot of resolving of build dependencies errored: ${err}.`
1232 );
1233 logger.debug(err.stack);
1234 return resolve(false);
1235 }
1236 if (valid) {
1237 resolveBuildDependenciesSnapshot =
1238 packContainer.resolveBuildDependenciesSnapshot;
1239 buildDependencies = packContainer.buildDependencies;
1240 resolveResults = packContainer.resolveResults;
1241 return resolve(true);
1242 }
1243 logger.log(
1244 "resolving of build dependencies is invalid, will re-resolve build dependencies"
1245 );
1246 this.fileSystemInfo.checkResolveResultsValid(
1247 packContainer.resolveResults,
1248 (err, valid) => {
1249 if (err) {
1250 logger.log(
1251 `Restored pack from ${cacheLocation}${this._extension}, but resolving of build dependencies errored: ${err}.`
1252 );
1253 logger.debug(err.stack);
1254 return resolve(false);
1255 }
1256 if (valid) {
1257 newBuildDependencies = packContainer.buildDependencies;
1258 resolveResults = packContainer.resolveResults;
1259 return resolve(true);
1260 }
1261 logger.log(
1262 `Restored pack from ${cacheLocation}${this._extension}, but build dependencies resolve to different locations.`
1263 );
1264 return resolve(false);
1265 }
1266 );
1267 }
1268 );
1269 })
1270 ])
1271 .catch(err => {
1272 logger.timeEnd("check build dependencies");
1273 throw err;
1274 })
1275 .then(([buildSnapshotValid, resolveValid]) => {
1276 logger.timeEnd("check build dependencies");
1277 if (buildSnapshotValid && resolveValid) {
1278 logger.time("restore cache content metadata");
1279 const d = packContainer.data();
1280 logger.timeEnd("restore cache content metadata");
1281 return d;
1282 }
1283 return undefined;
1284 });
1285 })
1286 .then(pack => {
1287 if (pack) {
1288 pack.maxAge = this.maxAge;
1289 this.buildSnapshot = buildSnapshot;
1290 if (buildDependencies) this.buildDependencies = buildDependencies;
1291 if (newBuildDependencies)
1292 this.newBuildDependencies.addAll(newBuildDependencies);
1293 this.resolveResults = resolveResults;
1294 this.resolveBuildDependenciesSnapshot =
1295 resolveBuildDependenciesSnapshot;
1296 return pack;
1297 }
1298 return new Pack(logger, this.maxAge);
1299 })
1300 .catch(err => {
1301 this.logger.warn(
1302 `Restoring pack from ${cacheLocation}${this._extension} failed: ${err}`
1303 );
1304 this.logger.debug(err.stack);
1305 return new Pack(logger, this.maxAge);
1306 });
1307 }
1308
1309 /**
1310 * @param {string} identifier unique name for the resource
1311 * @param {Etag | null} etag etag of the resource
1312 * @param {any} data cached content
1313 * @returns {Promise<void>} promise
1314 */
1315 store(identifier, etag, data) {
1316 if (this.readonly) return Promise.resolve();
1317
1318 return this._getPack().then(pack => {
1319 pack.set(identifier, etag === null ? null : etag.toString(), data);
1320 });
1321 }
1322
1323 /**
1324 * @param {string} identifier unique name for the resource
1325 * @param {Etag | null} etag etag of the resource
1326 * @returns {Promise<any>} promise to the cached content
1327 */
1328 restore(identifier, etag) {
1329 return this._getPack()
1330 .then(pack =>
1331 pack.get(identifier, etag === null ? null : etag.toString())
1332 )
1333 .catch(err => {
1334 if (err && err.code !== "ENOENT") {
1335 this.logger.warn(
1336 `Restoring failed for ${identifier} from pack: ${err}`
1337 );
1338 this.logger.debug(err.stack);
1339 }
1340 });
1341 }
1342
1343 /**
1344 * @param {LazySet<string> | Iterable<string>} dependencies dependencies to store
1345 */
1346 storeBuildDependencies(dependencies) {
1347 if (this.readonly) return;
1348 this.newBuildDependencies.addAll(dependencies);
1349 }
1350
1351 afterAllStored() {
1352 const packPromise = this.packPromise;
1353 if (packPromise === undefined) return Promise.resolve();
1354 const reportProgress = ProgressPlugin.getReporter(this.compiler);
1355 return (this.storePromise = packPromise
1356 .then(pack => {
1357 pack.stopCapturingRequests();
1358 if (!pack.invalid) return;
1359 this.packPromise = undefined;
1360 this.logger.log("Storing pack...");
1361 let promise;
1362 const newBuildDependencies = new Set();
1363 for (const dep of this.newBuildDependencies) {
1364 if (!this.buildDependencies.has(dep)) {
1365 newBuildDependencies.add(dep);
1366 }
1367 }
1368 if (newBuildDependencies.size > 0 || !this.buildSnapshot) {
1369 if (reportProgress) reportProgress(0.5, "resolve build dependencies");
1370 this.logger.debug(
1371 `Capturing build dependencies... (${Array.from(
1372 newBuildDependencies
1373 ).join(", ")})`
1374 );
1375 promise = new Promise((resolve, reject) => {
1376 this.logger.time("resolve build dependencies");
1377 this.fileSystemInfo.resolveBuildDependencies(
1378 this.context,
1379 newBuildDependencies,
1380 (err, result) => {
1381 this.logger.timeEnd("resolve build dependencies");
1382 if (err) return reject(err);
1383
1384 this.logger.time("snapshot build dependencies");
1385 const {
1386 files,
1387 directories,
1388 missing,
1389 resolveResults,
1390 resolveDependencies
1391 } = /** @type {ResolveBuildDependenciesResult} */ (result);
1392 if (this.resolveResults) {
1393 for (const [key, value] of resolveResults) {
1394 this.resolveResults.set(key, value);
1395 }
1396 } else {
1397 this.resolveResults = resolveResults;
1398 }
1399 if (reportProgress) {
1400 reportProgress(
1401 0.6,
1402 "snapshot build dependencies",
1403 "resolving"
1404 );
1405 }
1406 this.fileSystemInfo.createSnapshot(
1407 undefined,
1408 resolveDependencies.files,
1409 resolveDependencies.directories,
1410 resolveDependencies.missing,
1411 this.snapshot.resolveBuildDependencies,
1412 (err, snapshot) => {
1413 if (err) {
1414 this.logger.timeEnd("snapshot build dependencies");
1415 return reject(err);
1416 }
1417 if (!snapshot) {
1418 this.logger.timeEnd("snapshot build dependencies");
1419 return reject(
1420 new Error("Unable to snapshot resolve dependencies")
1421 );
1422 }
1423 if (this.resolveBuildDependenciesSnapshot) {
1424 this.resolveBuildDependenciesSnapshot =
1425 this.fileSystemInfo.mergeSnapshots(
1426 this.resolveBuildDependenciesSnapshot,
1427 snapshot
1428 );
1429 } else {
1430 this.resolveBuildDependenciesSnapshot = snapshot;
1431 }
1432 if (reportProgress) {
1433 reportProgress(
1434 0.7,
1435 "snapshot build dependencies",
1436 "modules"
1437 );
1438 }
1439 this.fileSystemInfo.createSnapshot(
1440 undefined,
1441 files,
1442 directories,
1443 missing,
1444 this.snapshot.buildDependencies,
1445 (err, snapshot) => {
1446 this.logger.timeEnd("snapshot build dependencies");
1447 if (err) return reject(err);
1448 if (!snapshot) {
1449 return reject(
1450 new Error("Unable to snapshot build dependencies")
1451 );
1452 }
1453 this.logger.debug("Captured build dependencies");
1454
1455 if (this.buildSnapshot) {
1456 this.buildSnapshot =
1457 this.fileSystemInfo.mergeSnapshots(
1458 this.buildSnapshot,
1459 snapshot
1460 );
1461 } else {
1462 this.buildSnapshot = snapshot;
1463 }
1464
1465 resolve();
1466 }
1467 );
1468 }
1469 );
1470 }
1471 );
1472 });
1473 } else {
1474 promise = Promise.resolve();
1475 }
1476 return promise.then(() => {
1477 if (reportProgress) reportProgress(0.8, "serialize pack");
1478 this.logger.time("store pack");
1479 const updatedBuildDependencies = new Set(this.buildDependencies);
1480 for (const dep of newBuildDependencies) {
1481 updatedBuildDependencies.add(dep);
1482 }
1483 const content = new PackContainer(
1484 pack,
1485 this.version,
1486 /** @type {Snapshot} */
1487 (this.buildSnapshot),
1488 updatedBuildDependencies,
1489 /** @type {ResolveResults} */
1490 (this.resolveResults),
1491 /** @type {Snapshot} */
1492 (this.resolveBuildDependenciesSnapshot)
1493 );
1494 return this.fileSerializer
1495 .serialize(content, {
1496 filename: `${this.cacheLocation}/index${this._extension}`,
1497 extension: `${this._extension}`,
1498 logger: this.logger,
1499 profile: this.profile
1500 })
1501 .then(() => {
1502 for (const dep of newBuildDependencies) {
1503 this.buildDependencies.add(dep);
1504 }
1505 this.newBuildDependencies.clear();
1506 this.logger.timeEnd("store pack");
1507 const stats = pack.getContentStats();
1508 this.logger.log(
1509 "Stored pack (%d items, %d files, %d MiB)",
1510 pack.itemInfo.size,
1511 stats.count,
1512 Math.round(stats.size / 1024 / 1024)
1513 );
1514 })
1515 .catch(err => {
1516 this.logger.timeEnd("store pack");
1517 this.logger.warn(`Caching failed for pack: ${err}`);
1518 this.logger.debug(err.stack);
1519 });
1520 });
1521 })
1522 .catch(err => {
1523 this.logger.warn(`Caching failed for pack: ${err}`);
1524 this.logger.debug(err.stack);
1525 }));
1526 }
1527
1528 clear() {
1529 this.fileSystemInfo.clear();
1530 this.buildDependencies.clear();
1531 this.newBuildDependencies.clear();
1532 this.resolveBuildDependenciesSnapshot = undefined;
1533 this.resolveResults = undefined;
1534 this.buildSnapshot = undefined;
1535 this.packPromise = undefined;
1536 }
1537}
1538
1539module.exports = PackFileCacheStrategy;
Note: See TracBrowser for help on using the repository browser.