source: trip-planner-front/node_modules/webpack/lib/cache/PackFileCacheStrategy.js@ 6a3a178

Last change on this file since 6a3a178 was 6a3a178, checked in by Ema <ema_spirova@…>, 3 years ago

initial commit
  • Property mode set to 100644
File size: 35.7 KB
RevLine 
[6a3a178]1/*
2 MIT License http://www.opensource.org/licenses/mit-license.php
3 Author Tobias Koppers @sokra
4*/
5
6"use strict";
7
8const FileSystemInfo = require("../FileSystemInfo");
9const ProgressPlugin = require("../ProgressPlugin");
10const { formatSize } = require("../SizeFormatHelpers");
11const SerializerMiddleware = require("../serialization/SerializerMiddleware");
12const LazySet = require("../util/LazySet");
13const makeSerializable = require("../util/makeSerializable");
14const memoize = require("../util/memoize");
15const {
16 createFileSerializer,
17 NOT_SERIALIZABLE
18} = require("../util/serialization");
19
20/** @typedef {import("../../declarations/WebpackOptions").SnapshotOptions} SnapshotOptions */
21/** @typedef {import("../Cache").Etag} Etag */
22/** @typedef {import("../Compiler")} Compiler */
23/** @typedef {import("../FileSystemInfo").Snapshot} Snapshot */
24/** @typedef {import("../logging/Logger").Logger} Logger */
25/** @typedef {import("../util/fs").IntermediateFileSystem} IntermediateFileSystem */
26
27class PackContainer {
28 /**
29 * @param {Object} data stored data
30 * @param {string} version version identifier
31 * @param {Snapshot} buildSnapshot snapshot of all build dependencies
32 * @param {Set<string>} buildDependencies list of all unresolved build dependencies captured
33 * @param {Map<string, string | false>} resolveResults result of the resolved build dependencies
34 * @param {Snapshot} resolveBuildDependenciesSnapshot snapshot of the dependencies of the build dependencies resolving
35 */
36 constructor(
37 data,
38 version,
39 buildSnapshot,
40 buildDependencies,
41 resolveResults,
42 resolveBuildDependenciesSnapshot
43 ) {
44 this.data = data;
45 this.version = version;
46 this.buildSnapshot = buildSnapshot;
47 this.buildDependencies = buildDependencies;
48 this.resolveResults = resolveResults;
49 this.resolveBuildDependenciesSnapshot = resolveBuildDependenciesSnapshot;
50 }
51
52 serialize({ write, writeLazy }) {
53 write(this.version);
54 write(this.buildSnapshot);
55 write(this.buildDependencies);
56 write(this.resolveResults);
57 write(this.resolveBuildDependenciesSnapshot);
58 writeLazy(this.data);
59 }
60
61 deserialize({ read }) {
62 this.version = read();
63 this.buildSnapshot = read();
64 this.buildDependencies = read();
65 this.resolveResults = read();
66 this.resolveBuildDependenciesSnapshot = read();
67 this.data = read();
68 }
69}
70
71makeSerializable(
72 PackContainer,
73 "webpack/lib/cache/PackFileCacheStrategy",
74 "PackContainer"
75);
76
77const MIN_CONTENT_SIZE = 1024 * 1024; // 1 MB
78const CONTENT_COUNT_TO_MERGE = 10;
79const MAX_ITEMS_IN_FRESH_PACK = 50000;
80
81class PackItemInfo {
82 /**
83 * @param {string} identifier identifier of item
84 * @param {string | null} etag etag of item
85 * @param {any} value fresh value of item
86 */
87 constructor(identifier, etag, value) {
88 this.identifier = identifier;
89 this.etag = etag;
90 this.location = -1;
91 this.lastAccess = Date.now();
92 this.freshValue = value;
93 }
94}
95
96class Pack {
97 constructor(logger, maxAge) {
98 /** @type {Map<string, PackItemInfo>} */
99 this.itemInfo = new Map();
100 /** @type {string[]} */
101 this.requests = [];
102 /** @type {Map<string, PackItemInfo>} */
103 this.freshContent = new Map();
104 /** @type {(undefined | PackContent)[]} */
105 this.content = [];
106 this.invalid = false;
107 this.logger = logger;
108 this.maxAge = maxAge;
109 }
110
111 /**
112 * @param {string} identifier unique name for the resource
113 * @param {string | null} etag etag of the resource
114 * @returns {any} cached content
115 */
116 get(identifier, etag) {
117 const info = this.itemInfo.get(identifier);
118 this.requests.push(identifier);
119 if (info === undefined) {
120 return undefined;
121 }
122 if (info.etag !== etag) return null;
123 info.lastAccess = Date.now();
124 const loc = info.location;
125 if (loc === -1) {
126 return info.freshValue;
127 } else {
128 if (!this.content[loc]) {
129 return undefined;
130 }
131 return this.content[loc].get(identifier);
132 }
133 }
134
135 /**
136 * @param {string} identifier unique name for the resource
137 * @param {string | null} etag etag of the resource
138 * @param {any} data cached content
139 * @returns {void}
140 */
141 set(identifier, etag, data) {
142 if (!this.invalid) {
143 this.invalid = true;
144 this.logger.log(`Pack got invalid because of write to: ${identifier}`);
145 }
146 const info = this.itemInfo.get(identifier);
147 if (info === undefined) {
148 const newInfo = new PackItemInfo(identifier, etag, data);
149 this.itemInfo.set(identifier, newInfo);
150 this.requests.push(identifier);
151 this.freshContent.set(identifier, newInfo);
152 } else {
153 const loc = info.location;
154 if (loc >= 0) {
155 this.requests.push(identifier);
156 this.freshContent.set(identifier, info);
157 const content = this.content[loc];
158 content.delete(identifier);
159 if (content.items.size === 0) {
160 this.content[loc] = undefined;
161 this.logger.debug("Pack %d got empty and is removed", loc);
162 }
163 }
164 info.freshValue = data;
165 info.lastAccess = Date.now();
166 info.etag = etag;
167 info.location = -1;
168 }
169 }
170
171 getContentStats() {
172 let count = 0;
173 let size = 0;
174 for (const content of this.content) {
175 if (content !== undefined) {
176 count++;
177 const s = content.getSize();
178 if (s > 0) {
179 size += s;
180 }
181 }
182 }
183 return { count, size };
184 }
185
186 /**
187 * @returns {number} new location of data entries
188 */
189 _findLocation() {
190 let i;
191 for (i = 0; i < this.content.length && this.content[i] !== undefined; i++);
192 return i;
193 }
194
195 _gcAndUpdateLocation(items, usedItems, newLoc) {
196 let count = 0;
197 let lastGC;
198 const now = Date.now();
199 for (const identifier of items) {
200 const info = this.itemInfo.get(identifier);
201 if (now - info.lastAccess > this.maxAge) {
202 this.itemInfo.delete(identifier);
203 items.delete(identifier);
204 usedItems.delete(identifier);
205 count++;
206 lastGC = identifier;
207 } else {
208 info.location = newLoc;
209 }
210 }
211 if (count > 0) {
212 this.logger.log(
213 "Garbage Collected %d old items at pack %d (%d items remaining) e. g. %s",
214 count,
215 newLoc,
216 items.size,
217 lastGC
218 );
219 }
220 }
221
222 _persistFreshContent() {
223 if (this.freshContent.size > 0) {
224 const packCount = Math.ceil(
225 this.freshContent.size / MAX_ITEMS_IN_FRESH_PACK
226 );
227 const itemsPerPack = Math.ceil(this.freshContent.size / packCount);
228 this.logger.log(`${this.freshContent.size} fresh items in cache`);
229 const packs = Array.from({ length: packCount }, () => {
230 const loc = this._findLocation();
231 this.content[loc] = null; // reserve
232 return {
233 /** @type {Set<string>} */
234 items: new Set(),
235 /** @type {Map<string, any>} */
236 map: new Map(),
237 loc
238 };
239 });
240 let i = 0;
241 let pack = packs[0];
242 let packIndex = 0;
243 for (const identifier of this.requests) {
244 const info = this.freshContent.get(identifier);
245 if (info === undefined) continue;
246 pack.items.add(identifier);
247 pack.map.set(identifier, info.freshValue);
248 info.location = pack.loc;
249 info.freshValue = undefined;
250 this.freshContent.delete(identifier);
251 if (++i > itemsPerPack) {
252 i = 0;
253 pack = packs[++packIndex];
254 }
255 }
256 for (const pack of packs) {
257 this.content[pack.loc] = new PackContent(
258 pack.items,
259 new Set(pack.items),
260 new PackContentItems(pack.map)
261 );
262 }
263 }
264 }
265
266 /**
267 * Merges small content files to a single content file
268 */
269 _optimizeSmallContent() {
270 // 1. Find all small content files
271 // Treat unused content files separately to avoid
272 // a merge-split cycle
273 /** @type {number[]} */
274 const smallUsedContents = [];
275 /** @type {number} */
276 let smallUsedContentSize = 0;
277 /** @type {number[]} */
278 const smallUnusedContents = [];
279 /** @type {number} */
280 let smallUnusedContentSize = 0;
281 for (let i = 0; i < this.content.length; i++) {
282 const content = this.content[i];
283 if (content === undefined) continue;
284 if (content.outdated) continue;
285 const size = content.getSize();
286 if (size < 0 || size > MIN_CONTENT_SIZE) continue;
287 if (content.used.size > 0) {
288 smallUsedContents.push(i);
289 smallUsedContentSize += size;
290 } else {
291 smallUnusedContents.push(i);
292 smallUnusedContentSize += size;
293 }
294 }
295
296 // 2. Check if minimum number is reached
297 let mergedIndices;
298 if (
299 smallUsedContents.length >= CONTENT_COUNT_TO_MERGE ||
300 smallUsedContentSize > MIN_CONTENT_SIZE
301 ) {
302 mergedIndices = smallUsedContents;
303 } else if (
304 smallUnusedContents.length >= CONTENT_COUNT_TO_MERGE ||
305 smallUnusedContentSize > MIN_CONTENT_SIZE
306 ) {
307 mergedIndices = smallUnusedContents;
308 } else return;
309
310 const mergedContent = [];
311
312 // 3. Remove old content entries
313 for (const i of mergedIndices) {
314 mergedContent.push(this.content[i]);
315 this.content[i] = undefined;
316 }
317
318 // 4. Determine merged items
319 /** @type {Set<string>} */
320 const mergedItems = new Set();
321 /** @type {Set<string>} */
322 const mergedUsedItems = new Set();
323 /** @type {(function(Map<string, any>): Promise)[]} */
324 const addToMergedMap = [];
325 for (const content of mergedContent) {
326 for (const identifier of content.items) {
327 mergedItems.add(identifier);
328 }
329 for (const identifer of content.used) {
330 mergedUsedItems.add(identifer);
331 }
332 addToMergedMap.push(async map => {
333 // unpack existing content
334 // after that values are accessible in .content
335 await content.unpack();
336 for (const [identifier, value] of content.content) {
337 map.set(identifier, value);
338 }
339 });
340 }
341
342 // 5. GC and update location of merged items
343 const newLoc = this._findLocation();
344 this._gcAndUpdateLocation(mergedItems, mergedUsedItems, newLoc);
345
346 // 6. If not empty, store content somewhere
347 if (mergedItems.size > 0) {
348 this.content[newLoc] = new PackContent(
349 mergedItems,
350 mergedUsedItems,
351 memoize(async () => {
352 /** @type {Map<string, any>} */
353 const map = new Map();
354 await Promise.all(addToMergedMap.map(fn => fn(map)));
355 return new PackContentItems(map);
356 })
357 );
358 this.logger.log(
359 "Merged %d small files with %d cache items into pack %d",
360 mergedContent.length,
361 mergedItems.size,
362 newLoc
363 );
364 }
365 }
366
367 /**
368 * Split large content files with used and unused items
369 * into two parts to separate used from unused items
370 */
371 _optimizeUnusedContent() {
372 // 1. Find a large content file with used and unused items
373 for (let i = 0; i < this.content.length; i++) {
374 const content = this.content[i];
375 if (content === undefined) continue;
376 const size = content.getSize();
377 if (size < MIN_CONTENT_SIZE) continue;
378 const used = content.used.size;
379 const total = content.items.size;
380 if (used > 0 && used < total) {
381 // 2. Remove this content
382 this.content[i] = undefined;
383
384 // 3. Determine items for the used content file
385 const usedItems = new Set(content.used);
386 const newLoc = this._findLocation();
387 this._gcAndUpdateLocation(usedItems, usedItems, newLoc);
388
389 // 4. Create content file for used items
390 if (usedItems.size > 0) {
391 this.content[newLoc] = new PackContent(
392 usedItems,
393 new Set(usedItems),
394 async () => {
395 await content.unpack();
396 const map = new Map();
397 for (const identifier of usedItems) {
398 map.set(identifier, content.content.get(identifier));
399 }
400 return new PackContentItems(map);
401 }
402 );
403 }
404
405 // 5. Determine items for the unused content file
406 const unusedItems = new Set(content.items);
407 const usedOfUnusedItems = new Set();
408 for (const identifier of usedItems) {
409 unusedItems.delete(identifier);
410 }
411 const newUnusedLoc = this._findLocation();
412 this._gcAndUpdateLocation(unusedItems, usedOfUnusedItems, newUnusedLoc);
413
414 // 6. Create content file for unused items
415 if (unusedItems.size > 0) {
416 this.content[newUnusedLoc] = new PackContent(
417 unusedItems,
418 usedOfUnusedItems,
419 async () => {
420 await content.unpack();
421 const map = new Map();
422 for (const identifier of unusedItems) {
423 map.set(identifier, content.content.get(identifier));
424 }
425 return new PackContentItems(map);
426 }
427 );
428 }
429
430 this.logger.log(
431 "Split pack %d into pack %d with %d used items and pack %d with %d unused items",
432 i,
433 newLoc,
434 usedItems.size,
435 newUnusedLoc,
436 unusedItems.size
437 );
438
439 // optimizing only one of them is good enough and
440 // reduces the amount of serialization needed
441 return;
442 }
443 }
444 }
445
446 /**
447 * Find the content with the oldest item and run GC on that.
448 * Only runs for one content to avoid large invalidation.
449 */
450 _gcOldestContent() {
451 /** @type {PackItemInfo} */
452 let oldest = undefined;
453 for (const info of this.itemInfo.values()) {
454 if (oldest === undefined || info.lastAccess < oldest.lastAccess) {
455 oldest = info;
456 }
457 }
458 if (Date.now() - oldest.lastAccess > this.maxAge) {
459 const loc = oldest.location;
460 if (loc < 0) return;
461 const content = this.content[loc];
462 const items = new Set(content.items);
463 const usedItems = new Set(content.used);
464 this._gcAndUpdateLocation(items, usedItems, loc);
465
466 this.content[loc] =
467 items.size > 0
468 ? new PackContent(items, usedItems, async () => {
469 await content.unpack();
470 const map = new Map();
471 for (const identifier of items) {
472 map.set(identifier, content.content.get(identifier));
473 }
474 return new PackContentItems(map);
475 })
476 : undefined;
477 }
478 }
479
480 serialize({ write, writeSeparate }) {
481 this._persistFreshContent();
482 this._optimizeSmallContent();
483 this._optimizeUnusedContent();
484 this._gcOldestContent();
485 for (const identifier of this.itemInfo.keys()) {
486 write(identifier);
487 }
488 write(null); // null as marker of the end of keys
489 for (const info of this.itemInfo.values()) {
490 write(info.etag);
491 }
492 for (const info of this.itemInfo.values()) {
493 write(info.lastAccess);
494 }
495 for (let i = 0; i < this.content.length; i++) {
496 const content = this.content[i];
497 if (content !== undefined) {
498 write(content.items);
499 writeSeparate(content.getLazyContentItems(), { name: `${i}` });
500 } else {
501 write(undefined); // undefined marks an empty content slot
502 }
503 }
504 write(null); // null as marker of the end of items
505 }
506
507 deserialize({ read, logger }) {
508 this.logger = logger;
509 {
510 const items = [];
511 let item = read();
512 while (item !== null) {
513 items.push(item);
514 item = read();
515 }
516 this.itemInfo.clear();
517 const infoItems = items.map(identifier => {
518 const info = new PackItemInfo(identifier, undefined, undefined);
519 this.itemInfo.set(identifier, info);
520 return info;
521 });
522 for (const info of infoItems) {
523 info.etag = read();
524 }
525 for (const info of infoItems) {
526 info.lastAccess = read();
527 }
528 }
529 this.content.length = 0;
530 let items = read();
531 while (items !== null) {
532 if (items === undefined) {
533 this.content.push(items);
534 } else {
535 const idx = this.content.length;
536 const lazy = read();
537 this.content.push(
538 new PackContent(
539 items,
540 new Set(),
541 lazy,
542 logger,
543 `${this.content.length}`
544 )
545 );
546 for (const identifier of items) {
547 this.itemInfo.get(identifier).location = idx;
548 }
549 }
550 items = read();
551 }
552 }
553}
554
555makeSerializable(Pack, "webpack/lib/cache/PackFileCacheStrategy", "Pack");
556
557class PackContentItems {
558 /**
559 * @param {Map<string, any>} map items
560 */
561 constructor(map) {
562 this.map = map;
563 }
564
565 serialize({ write, snapshot, rollback, logger, profile }) {
566 if (profile) {
567 write(false);
568 for (const [key, value] of this.map) {
569 const s = snapshot();
570 try {
571 write(key);
572 const start = process.hrtime();
573 write(value);
574 const durationHr = process.hrtime(start);
575 const duration = durationHr[0] * 1000 + durationHr[1] / 1e6;
576 if (duration > 1) {
577 if (duration > 500)
578 logger.error(`Serialization of '${key}': ${duration} ms`);
579 else if (duration > 50)
580 logger.warn(`Serialization of '${key}': ${duration} ms`);
581 else if (duration > 10)
582 logger.info(`Serialization of '${key}': ${duration} ms`);
583 else if (duration > 5)
584 logger.log(`Serialization of '${key}': ${duration} ms`);
585 else logger.debug(`Serialization of '${key}': ${duration} ms`);
586 }
587 } catch (e) {
588 rollback(s);
589 if (e === NOT_SERIALIZABLE) continue;
590 logger.warn(
591 `Skipped not serializable cache item '${key}': ${e.message}`
592 );
593 logger.debug(e.stack);
594 }
595 }
596 write(null);
597 return;
598 }
599 // Try to serialize all at once
600 const s = snapshot();
601 try {
602 write(true);
603 write(this.map);
604 } catch (e) {
605 rollback(s);
606
607 // Try to serialize each item on it's own
608 write(false);
609 for (const [key, value] of this.map) {
610 const s = snapshot();
611 try {
612 write(key);
613 write(value);
614 } catch (e) {
615 rollback(s);
616 if (e === NOT_SERIALIZABLE) continue;
617 logger.warn(
618 `Skipped not serializable cache item '${key}': ${e.message}`
619 );
620 logger.debug(e.stack);
621 }
622 }
623 write(null);
624 }
625 }
626
627 deserialize({ read, logger, profile }) {
628 if (read()) {
629 this.map = read();
630 } else if (profile) {
631 const map = new Map();
632 let key = read();
633 while (key !== null) {
634 const start = process.hrtime();
635 const value = read();
636 const durationHr = process.hrtime(start);
637 const duration = durationHr[0] * 1000 + durationHr[1] / 1e6;
638 if (duration > 1) {
639 if (duration > 100)
640 logger.error(`Deserialization of '${key}': ${duration} ms`);
641 else if (duration > 20)
642 logger.warn(`Deserialization of '${key}': ${duration} ms`);
643 else if (duration > 5)
644 logger.info(`Deserialization of '${key}': ${duration} ms`);
645 else if (duration > 2)
646 logger.log(`Deserialization of '${key}': ${duration} ms`);
647 else logger.debug(`Deserialization of '${key}': ${duration} ms`);
648 }
649 map.set(key, value);
650 key = read();
651 }
652 this.map = map;
653 } else {
654 const map = new Map();
655 let key = read();
656 while (key !== null) {
657 map.set(key, read());
658 key = read();
659 }
660 this.map = map;
661 }
662 }
663}
664
665makeSerializable(
666 PackContentItems,
667 "webpack/lib/cache/PackFileCacheStrategy",
668 "PackContentItems"
669);
670
671class PackContent {
672 /**
673 * @param {Set<string>} items keys
674 * @param {Set<string>} usedItems used keys
675 * @param {PackContentItems | function(): Promise<PackContentItems>} dataOrFn sync or async content
676 * @param {Logger=} logger logger for logging
677 * @param {string=} lazyName name of dataOrFn for logging
678 */
679 constructor(items, usedItems, dataOrFn, logger, lazyName) {
680 this.items = items;
681 /** @type {function(): Promise<PackContentItems> | PackContentItems } */
682 this.lazy = typeof dataOrFn === "function" ? dataOrFn : undefined;
683 /** @type {Map<string, any>} */
684 this.content = typeof dataOrFn === "function" ? undefined : dataOrFn.map;
685 this.outdated = false;
686 this.used = usedItems;
687 this.logger = logger;
688 this.lazyName = lazyName;
689 }
690
691 get(identifier) {
692 this.used.add(identifier);
693 if (this.content) {
694 return this.content.get(identifier);
695 }
696 const { lazyName } = this;
697 let timeMessage;
698 if (lazyName) {
699 // only log once
700 this.lazyName = undefined;
701 timeMessage = `restore cache content ${lazyName} (${formatSize(
702 this.getSize()
703 )})`;
704 this.logger.log(
705 `starting to restore cache content ${lazyName} (${formatSize(
706 this.getSize()
707 )}) because of request to: ${identifier}`
708 );
709 this.logger.time(timeMessage);
710 }
711 const value = this.lazy();
712 if (value instanceof Promise) {
713 return value.then(data => {
714 const map = data.map;
715 if (timeMessage) {
716 this.logger.timeEnd(timeMessage);
717 }
718 this.content = map;
719 this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
720 return map.get(identifier);
721 });
722 } else {
723 const map = value.map;
724 if (timeMessage) {
725 this.logger.timeEnd(timeMessage);
726 }
727 this.content = map;
728 this.lazy = SerializerMiddleware.unMemoizeLazy(this.lazy);
729 return map.get(identifier);
730 }
731 }
732
733 /**
734 * @returns {void | Promise} maybe a promise if lazy
735 */
736 unpack() {
737 if (this.content) return;
738 if (this.lazy) {
739 const { lazyName } = this;
740 let timeMessage;
741 if (lazyName) {
742 // only log once
743 this.lazyName = undefined;
744 timeMessage = `unpack cache content ${lazyName} (${formatSize(
745 this.getSize()
746 )})`;
747 this.logger.time(timeMessage);
748 }
749 const value = this.lazy();
750 if (value instanceof Promise) {
751 return value.then(data => {
752 if (timeMessage) {
753 this.logger.timeEnd(timeMessage);
754 }
755 this.content = data.map;
756 });
757 } else {
758 if (timeMessage) {
759 this.logger.timeEnd(timeMessage);
760 }
761 this.content = value.map;
762 }
763 }
764 }
765
766 /**
767 * @returns {number} size of the content or -1 if not known
768 */
769 getSize() {
770 if (!this.lazy) return -1;
771 const options = /** @type {any} */ (this.lazy).options;
772 if (!options) return -1;
773 const size = options.size;
774 if (typeof size !== "number") return -1;
775 return size;
776 }
777
778 delete(identifier) {
779 this.items.delete(identifier);
780 this.used.delete(identifier);
781 this.outdated = true;
782 }
783
784 /**
785 * @returns {function(): PackContentItems | Promise<PackContentItems>} lazy content items
786 */
787 getLazyContentItems() {
788 if (!this.outdated && this.lazy) return this.lazy;
789 if (!this.outdated && this.content) {
790 const map = new Map(this.content);
791 return (this.lazy = memoize(() => new PackContentItems(map)));
792 }
793 this.outdated = false;
794 if (this.content) {
795 return (this.lazy = memoize(() => {
796 /** @type {Map<string, any>} */
797 const map = new Map();
798 for (const item of this.items) {
799 map.set(item, this.content.get(item));
800 }
801 return new PackContentItems(map);
802 }));
803 }
804 const lazy = this.lazy;
805 return (this.lazy = () => {
806 const value = lazy();
807 if (value instanceof Promise) {
808 return value.then(data => {
809 const oldMap = data.map;
810 /** @type {Map<string, any>} */
811 const map = new Map();
812 for (const item of this.items) {
813 map.set(item, oldMap.get(item));
814 }
815 return new PackContentItems(map);
816 });
817 } else {
818 const oldMap = value.map;
819 /** @type {Map<string, any>} */
820 const map = new Map();
821 for (const item of this.items) {
822 map.set(item, oldMap.get(item));
823 }
824 return new PackContentItems(map);
825 }
826 });
827 }
828}
829
830const allowCollectingMemory = buf => {
831 const wasted = buf.buffer.byteLength - buf.byteLength;
832 if (wasted > 8192 && (wasted > 1048576 || wasted > buf.byteLength)) {
833 return Buffer.from(buf);
834 }
835 return buf;
836};
837
838class PackFileCacheStrategy {
839 /**
840 * @param {Object} options options
841 * @param {Compiler} options.compiler the compiler
842 * @param {IntermediateFileSystem} options.fs the filesystem
843 * @param {string} options.context the context directory
844 * @param {string} options.cacheLocation the location of the cache data
845 * @param {string} options.version version identifier
846 * @param {Logger} options.logger a logger
847 * @param {SnapshotOptions} options.snapshot options regarding snapshotting
848 * @param {number} options.maxAge max age of cache items
849 * @param {boolean} options.profile track and log detailed timing information for individual cache items
850 * @param {boolean} options.allowCollectingMemory allow to collect unused memory created during deserialization
851 * @param {false | "gzip" | "brotli"} options.compression compression used
852 */
853 constructor({
854 compiler,
855 fs,
856 context,
857 cacheLocation,
858 version,
859 logger,
860 snapshot,
861 maxAge,
862 profile,
863 allowCollectingMemory,
864 compression
865 }) {
866 this.fileSerializer = createFileSerializer(fs);
867 this.fileSystemInfo = new FileSystemInfo(fs, {
868 managedPaths: snapshot.managedPaths,
869 immutablePaths: snapshot.immutablePaths,
870 logger: logger.getChildLogger("webpack.FileSystemInfo")
871 });
872 this.compiler = compiler;
873 this.context = context;
874 this.cacheLocation = cacheLocation;
875 this.version = version;
876 this.logger = logger;
877 this.maxAge = maxAge;
878 this.profile = profile;
879 this.allowCollectingMemory = allowCollectingMemory;
880 this.compression = compression;
881 this._extension =
882 compression === "brotli"
883 ? ".pack.br"
884 : compression === "gzip"
885 ? ".pack.gz"
886 : ".pack";
887 this.snapshot = snapshot;
888 /** @type {Set<string>} */
889 this.buildDependencies = new Set();
890 /** @type {LazySet<string>} */
891 this.newBuildDependencies = new LazySet();
892 /** @type {Snapshot} */
893 this.resolveBuildDependenciesSnapshot = undefined;
894 /** @type {Map<string, string | false>} */
895 this.resolveResults = undefined;
896 /** @type {Snapshot} */
897 this.buildSnapshot = undefined;
898 /** @type {Promise<Pack>} */
899 this.packPromise = this._openPack();
900 this.storePromise = Promise.resolve();
901 }
902
903 _getPack() {
904 if (this.packPromise === undefined) {
905 this.packPromise = this.storePromise.then(() => this._openPack());
906 }
907 return this.packPromise;
908 }
909
910 /**
911 * @returns {Promise<Pack>} the pack
912 */
913 _openPack() {
914 const { logger, profile, cacheLocation, version } = this;
915 /** @type {Snapshot} */
916 let buildSnapshot;
917 /** @type {Set<string>} */
918 let buildDependencies;
919 /** @type {Set<string>} */
920 let newBuildDependencies;
921 /** @type {Snapshot} */
922 let resolveBuildDependenciesSnapshot;
923 /** @type {Map<string, string | false>} */
924 let resolveResults;
925 logger.time("restore cache container");
926 return this.fileSerializer
927 .deserialize(null, {
928 filename: `${cacheLocation}/index${this._extension}`,
929 extension: `${this._extension}`,
930 logger,
931 profile,
932 retainedBuffer: this.allowCollectingMemory
933 ? allowCollectingMemory
934 : undefined
935 })
936 .catch(err => {
937 if (err.code !== "ENOENT") {
938 logger.warn(
939 `Restoring pack failed from ${cacheLocation}${this._extension}: ${err}`
940 );
941 logger.debug(err.stack);
942 } else {
943 logger.debug(
944 `No pack exists at ${cacheLocation}${this._extension}: ${err}`
945 );
946 }
947 return undefined;
948 })
949 .then(packContainer => {
950 logger.timeEnd("restore cache container");
951 if (!packContainer) return undefined;
952 if (!(packContainer instanceof PackContainer)) {
953 logger.warn(
954 `Restored pack from ${cacheLocation}${this._extension}, but contained content is unexpected.`,
955 packContainer
956 );
957 return undefined;
958 }
959 if (packContainer.version !== version) {
960 logger.log(
961 `Restored pack from ${cacheLocation}${this._extension}, but version doesn't match.`
962 );
963 return undefined;
964 }
965 logger.time("check build dependencies");
966 return Promise.all([
967 new Promise((resolve, reject) => {
968 this.fileSystemInfo.checkSnapshotValid(
969 packContainer.buildSnapshot,
970 (err, valid) => {
971 if (err) {
972 logger.log(
973 `Restored pack from ${cacheLocation}${this._extension}, but checking snapshot of build dependencies errored: ${err}.`
974 );
975 logger.debug(err.stack);
976 return resolve(false);
977 }
978 if (!valid) {
979 logger.log(
980 `Restored pack from ${cacheLocation}${this._extension}, but build dependencies have changed.`
981 );
982 return resolve(false);
983 }
984 buildSnapshot = packContainer.buildSnapshot;
985 return resolve(true);
986 }
987 );
988 }),
989 new Promise((resolve, reject) => {
990 this.fileSystemInfo.checkSnapshotValid(
991 packContainer.resolveBuildDependenciesSnapshot,
992 (err, valid) => {
993 if (err) {
994 logger.log(
995 `Restored pack from ${cacheLocation}${this._extension}, but checking snapshot of resolving of build dependencies errored: ${err}.`
996 );
997 logger.debug(err.stack);
998 return resolve(false);
999 }
1000 if (valid) {
1001 resolveBuildDependenciesSnapshot =
1002 packContainer.resolveBuildDependenciesSnapshot;
1003 buildDependencies = packContainer.buildDependencies;
1004 resolveResults = packContainer.resolveResults;
1005 return resolve(true);
1006 }
1007 logger.log(
1008 "resolving of build dependencies is invalid, will re-resolve build dependencies"
1009 );
1010 this.fileSystemInfo.checkResolveResultsValid(
1011 packContainer.resolveResults,
1012 (err, valid) => {
1013 if (err) {
1014 logger.log(
1015 `Restored pack from ${cacheLocation}${this._extension}, but resolving of build dependencies errored: ${err}.`
1016 );
1017 logger.debug(err.stack);
1018 return resolve(false);
1019 }
1020 if (valid) {
1021 newBuildDependencies = packContainer.buildDependencies;
1022 resolveResults = packContainer.resolveResults;
1023 return resolve(true);
1024 }
1025 logger.log(
1026 `Restored pack from ${cacheLocation}${this._extension}, but build dependencies resolve to different locations.`
1027 );
1028 return resolve(false);
1029 }
1030 );
1031 }
1032 );
1033 })
1034 ])
1035 .catch(err => {
1036 logger.timeEnd("check build dependencies");
1037 throw err;
1038 })
1039 .then(([buildSnapshotValid, resolveValid]) => {
1040 logger.timeEnd("check build dependencies");
1041 if (buildSnapshotValid && resolveValid) {
1042 logger.time("restore cache content metadata");
1043 const d = packContainer.data();
1044 logger.timeEnd("restore cache content metadata");
1045 return d;
1046 }
1047 return undefined;
1048 });
1049 })
1050 .then(pack => {
1051 if (pack) {
1052 pack.maxAge = this.maxAge;
1053 this.buildSnapshot = buildSnapshot;
1054 if (buildDependencies) this.buildDependencies = buildDependencies;
1055 if (newBuildDependencies)
1056 this.newBuildDependencies.addAll(newBuildDependencies);
1057 this.resolveResults = resolveResults;
1058 this.resolveBuildDependenciesSnapshot =
1059 resolveBuildDependenciesSnapshot;
1060 return pack;
1061 }
1062 return new Pack(logger, this.maxAge);
1063 })
1064 .catch(err => {
1065 this.logger.warn(
1066 `Restoring pack from ${cacheLocation}${this._extension} failed: ${err}`
1067 );
1068 this.logger.debug(err.stack);
1069 return new Pack(logger, this.maxAge);
1070 });
1071 }
1072
1073 /**
1074 * @param {string} identifier unique name for the resource
1075 * @param {Etag | null} etag etag of the resource
1076 * @param {any} data cached content
1077 * @returns {Promise<void>} promise
1078 */
1079 store(identifier, etag, data) {
1080 return this._getPack().then(pack => {
1081 pack.set(identifier, etag === null ? null : etag.toString(), data);
1082 });
1083 }
1084
1085 /**
1086 * @param {string} identifier unique name for the resource
1087 * @param {Etag | null} etag etag of the resource
1088 * @returns {Promise<any>} promise to the cached content
1089 */
1090 restore(identifier, etag) {
1091 return this._getPack()
1092 .then(pack =>
1093 pack.get(identifier, etag === null ? null : etag.toString())
1094 )
1095 .catch(err => {
1096 if (err && err.code !== "ENOENT") {
1097 this.logger.warn(
1098 `Restoring failed for ${identifier} from pack: ${err}`
1099 );
1100 this.logger.debug(err.stack);
1101 }
1102 });
1103 }
1104
1105 storeBuildDependencies(dependencies) {
1106 this.newBuildDependencies.addAll(dependencies);
1107 }
1108
1109 afterAllStored() {
1110 const packPromise = this.packPromise;
1111 if (packPromise === undefined) return Promise.resolve();
1112 const reportProgress = ProgressPlugin.getReporter(this.compiler);
1113 this.packPromise = undefined;
1114 return (this.storePromise = packPromise
1115 .then(pack => {
1116 if (!pack.invalid) return;
1117 this.logger.log(`Storing pack...`);
1118 let promise;
1119 const newBuildDependencies = new Set();
1120 for (const dep of this.newBuildDependencies) {
1121 if (!this.buildDependencies.has(dep)) {
1122 newBuildDependencies.add(dep);
1123 }
1124 }
1125 if (newBuildDependencies.size > 0 || !this.buildSnapshot) {
1126 if (reportProgress) reportProgress(0.5, "resolve build dependencies");
1127 this.logger.debug(
1128 `Capturing build dependencies... (${Array.from(
1129 newBuildDependencies
1130 ).join(", ")})`
1131 );
1132 promise = new Promise((resolve, reject) => {
1133 this.logger.time("resolve build dependencies");
1134 this.fileSystemInfo.resolveBuildDependencies(
1135 this.context,
1136 newBuildDependencies,
1137 (err, result) => {
1138 this.logger.timeEnd("resolve build dependencies");
1139 if (err) return reject(err);
1140
1141 this.logger.time("snapshot build dependencies");
1142 const {
1143 files,
1144 directories,
1145 missing,
1146 resolveResults,
1147 resolveDependencies
1148 } = result;
1149 if (this.resolveResults) {
1150 for (const [key, value] of resolveResults) {
1151 this.resolveResults.set(key, value);
1152 }
1153 } else {
1154 this.resolveResults = resolveResults;
1155 }
1156 if (reportProgress) {
1157 reportProgress(
1158 0.6,
1159 "snapshot build dependencies",
1160 "resolving"
1161 );
1162 }
1163 this.fileSystemInfo.createSnapshot(
1164 undefined,
1165 resolveDependencies.files,
1166 resolveDependencies.directories,
1167 resolveDependencies.missing,
1168 this.snapshot.resolveBuildDependencies,
1169 (err, snapshot) => {
1170 if (err) {
1171 this.logger.timeEnd("snapshot build dependencies");
1172 return reject(err);
1173 }
1174 if (!snapshot) {
1175 this.logger.timeEnd("snapshot build dependencies");
1176 return reject(
1177 new Error("Unable to snapshot resolve dependencies")
1178 );
1179 }
1180 if (this.resolveBuildDependenciesSnapshot) {
1181 this.resolveBuildDependenciesSnapshot =
1182 this.fileSystemInfo.mergeSnapshots(
1183 this.resolveBuildDependenciesSnapshot,
1184 snapshot
1185 );
1186 } else {
1187 this.resolveBuildDependenciesSnapshot = snapshot;
1188 }
1189 if (reportProgress) {
1190 reportProgress(
1191 0.7,
1192 "snapshot build dependencies",
1193 "modules"
1194 );
1195 }
1196 this.fileSystemInfo.createSnapshot(
1197 undefined,
1198 files,
1199 directories,
1200 missing,
1201 this.snapshot.buildDependencies,
1202 (err, snapshot) => {
1203 this.logger.timeEnd("snapshot build dependencies");
1204 if (err) return reject(err);
1205 if (!snapshot) {
1206 return reject(
1207 new Error("Unable to snapshot build dependencies")
1208 );
1209 }
1210 this.logger.debug("Captured build dependencies");
1211
1212 if (this.buildSnapshot) {
1213 this.buildSnapshot =
1214 this.fileSystemInfo.mergeSnapshots(
1215 this.buildSnapshot,
1216 snapshot
1217 );
1218 } else {
1219 this.buildSnapshot = snapshot;
1220 }
1221
1222 resolve();
1223 }
1224 );
1225 }
1226 );
1227 }
1228 );
1229 });
1230 } else {
1231 promise = Promise.resolve();
1232 }
1233 return promise.then(() => {
1234 if (reportProgress) reportProgress(0.8, "serialize pack");
1235 this.logger.time(`store pack`);
1236 const updatedBuildDependencies = new Set(this.buildDependencies);
1237 for (const dep of newBuildDependencies) {
1238 updatedBuildDependencies.add(dep);
1239 }
1240 const content = new PackContainer(
1241 pack,
1242 this.version,
1243 this.buildSnapshot,
1244 updatedBuildDependencies,
1245 this.resolveResults,
1246 this.resolveBuildDependenciesSnapshot
1247 );
1248 return this.fileSerializer
1249 .serialize(content, {
1250 filename: `${this.cacheLocation}/index${this._extension}`,
1251 extension: `${this._extension}`,
1252 logger: this.logger,
1253 profile: this.profile
1254 })
1255 .then(() => {
1256 for (const dep of newBuildDependencies) {
1257 this.buildDependencies.add(dep);
1258 }
1259 this.newBuildDependencies.clear();
1260 this.logger.timeEnd(`store pack`);
1261 const stats = pack.getContentStats();
1262 this.logger.log(
1263 "Stored pack (%d items, %d files, %d MiB)",
1264 pack.itemInfo.size,
1265 stats.count,
1266 Math.round(stats.size / 1024 / 1024)
1267 );
1268 })
1269 .catch(err => {
1270 this.logger.timeEnd(`store pack`);
1271 this.logger.warn(`Caching failed for pack: ${err}`);
1272 this.logger.debug(err.stack);
1273 });
1274 });
1275 })
1276 .catch(err => {
1277 this.logger.warn(`Caching failed for pack: ${err}`);
1278 this.logger.debug(err.stack);
1279 }));
1280 }
1281
1282 clear() {
1283 this.fileSystemInfo.clear();
1284 this.buildDependencies.clear();
1285 this.newBuildDependencies.clear();
1286 this.resolveBuildDependenciesSnapshot = undefined;
1287 this.resolveResults = undefined;
1288 this.buildSnapshot = undefined;
1289 this.packPromise = undefined;
1290 }
1291}
1292
1293module.exports = PackFileCacheStrategy;
Note: See TracBrowser for help on using the repository browser.