source: trip-planner-front/node_modules/webpack/lib/MultiCompiler.js

Last change on this file was 6a3a178, checked in by Ema <ema_spirova@…>, 3 years ago

initial commit
  • Property mode set to 100644
File size: 15.7 KB
Line 
1/*
2 MIT License http://www.opensource.org/licenses/mit-license.php
3 Author Tobias Koppers @sokra
4*/
5
6"use strict";
7
8const asyncLib = require("neo-async");
9const { SyncHook, MultiHook } = require("tapable");
10
11const ConcurrentCompilationError = require("./ConcurrentCompilationError");
12const MultiStats = require("./MultiStats");
13const MultiWatching = require("./MultiWatching");
14const ArrayQueue = require("./util/ArrayQueue");
15
16/** @template T @typedef {import("tapable").AsyncSeriesHook<T>} AsyncSeriesHook<T> */
17/** @template T @template R @typedef {import("tapable").SyncBailHook<T, R>} SyncBailHook<T, R> */
18/** @typedef {import("../declarations/WebpackOptions").WatchOptions} WatchOptions */
19/** @typedef {import("./Compiler")} Compiler */
20/** @typedef {import("./Stats")} Stats */
21/** @typedef {import("./Watching")} Watching */
22/** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
23/** @typedef {import("./util/fs").IntermediateFileSystem} IntermediateFileSystem */
24/** @typedef {import("./util/fs").OutputFileSystem} OutputFileSystem */
25/** @typedef {import("./util/fs").WatchFileSystem} WatchFileSystem */
26
27/**
28 * @template T
29 * @callback Callback
30 * @param {Error=} err
31 * @param {T=} result
32 */
33
34/**
35 * @callback RunWithDependenciesHandler
36 * @param {Compiler} compiler
37 * @param {Callback<MultiStats>} callback
38 */
39
40/**
41 * @typedef {Object} MultiCompilerOptions
42 * @property {number=} parallelism how many Compilers are allows to run at the same time in parallel
43 */
44
45module.exports = class MultiCompiler {
46 /**
47 * @param {Compiler[] | Record<string, Compiler>} compilers child compilers
48 * @param {MultiCompilerOptions} options options
49 */
50 constructor(compilers, options) {
51 if (!Array.isArray(compilers)) {
52 compilers = Object.keys(compilers).map(name => {
53 compilers[name].name = name;
54 return compilers[name];
55 });
56 }
57
58 this.hooks = Object.freeze({
59 /** @type {SyncHook<[MultiStats]>} */
60 done: new SyncHook(["stats"]),
61 /** @type {MultiHook<SyncHook<[string | null, number]>>} */
62 invalid: new MultiHook(compilers.map(c => c.hooks.invalid)),
63 /** @type {MultiHook<AsyncSeriesHook<[Compiler]>>} */
64 run: new MultiHook(compilers.map(c => c.hooks.run)),
65 /** @type {SyncHook<[]>} */
66 watchClose: new SyncHook([]),
67 /** @type {MultiHook<AsyncSeriesHook<[Compiler]>>} */
68 watchRun: new MultiHook(compilers.map(c => c.hooks.watchRun)),
69 /** @type {MultiHook<SyncBailHook<[string, string, any[]], true>>} */
70 infrastructureLog: new MultiHook(
71 compilers.map(c => c.hooks.infrastructureLog)
72 )
73 });
74 this.compilers = compilers;
75 /** @type {MultiCompilerOptions} */
76 this._options = {
77 parallelism: options.parallelism || Infinity
78 };
79 /** @type {WeakMap<Compiler, string[]>} */
80 this.dependencies = new WeakMap();
81 this.running = false;
82
83 /** @type {Stats[]} */
84 const compilerStats = this.compilers.map(() => null);
85 let doneCompilers = 0;
86 for (let index = 0; index < this.compilers.length; index++) {
87 const compiler = this.compilers[index];
88 const compilerIndex = index;
89 let compilerDone = false;
90 compiler.hooks.done.tap("MultiCompiler", stats => {
91 if (!compilerDone) {
92 compilerDone = true;
93 doneCompilers++;
94 }
95 compilerStats[compilerIndex] = stats;
96 if (doneCompilers === this.compilers.length) {
97 this.hooks.done.call(new MultiStats(compilerStats));
98 }
99 });
100 compiler.hooks.invalid.tap("MultiCompiler", () => {
101 if (compilerDone) {
102 compilerDone = false;
103 doneCompilers--;
104 }
105 });
106 }
107 }
108
109 get options() {
110 return Object.assign(
111 this.compilers.map(c => c.options),
112 this._options
113 );
114 }
115
116 get outputPath() {
117 let commonPath = this.compilers[0].outputPath;
118 for (const compiler of this.compilers) {
119 while (
120 compiler.outputPath.indexOf(commonPath) !== 0 &&
121 /[/\\]/.test(commonPath)
122 ) {
123 commonPath = commonPath.replace(/[/\\][^/\\]*$/, "");
124 }
125 }
126
127 if (!commonPath && this.compilers[0].outputPath[0] === "/") return "/";
128 return commonPath;
129 }
130
131 get inputFileSystem() {
132 throw new Error("Cannot read inputFileSystem of a MultiCompiler");
133 }
134
135 get outputFileSystem() {
136 throw new Error("Cannot read outputFileSystem of a MultiCompiler");
137 }
138
139 get watchFileSystem() {
140 throw new Error("Cannot read watchFileSystem of a MultiCompiler");
141 }
142
143 get intermediateFileSystem() {
144 throw new Error("Cannot read outputFileSystem of a MultiCompiler");
145 }
146
147 /**
148 * @param {InputFileSystem} value the new input file system
149 */
150 set inputFileSystem(value) {
151 for (const compiler of this.compilers) {
152 compiler.inputFileSystem = value;
153 }
154 }
155
156 /**
157 * @param {OutputFileSystem} value the new output file system
158 */
159 set outputFileSystem(value) {
160 for (const compiler of this.compilers) {
161 compiler.outputFileSystem = value;
162 }
163 }
164
165 /**
166 * @param {WatchFileSystem} value the new watch file system
167 */
168 set watchFileSystem(value) {
169 for (const compiler of this.compilers) {
170 compiler.watchFileSystem = value;
171 }
172 }
173
174 /**
175 * @param {IntermediateFileSystem} value the new intermediate file system
176 */
177 set intermediateFileSystem(value) {
178 for (const compiler of this.compilers) {
179 compiler.intermediateFileSystem = value;
180 }
181 }
182
183 getInfrastructureLogger(name) {
184 return this.compilers[0].getInfrastructureLogger(name);
185 }
186
187 /**
188 * @param {Compiler} compiler the child compiler
189 * @param {string[]} dependencies its dependencies
190 * @returns {void}
191 */
192 setDependencies(compiler, dependencies) {
193 this.dependencies.set(compiler, dependencies);
194 }
195
196 /**
197 * @param {Callback<MultiStats>} callback signals when the validation is complete
198 * @returns {boolean} true if the dependencies are valid
199 */
200 validateDependencies(callback) {
201 /** @type {Set<{source: Compiler, target: Compiler}>} */
202 const edges = new Set();
203 /** @type {string[]} */
204 const missing = [];
205 const targetFound = compiler => {
206 for (const edge of edges) {
207 if (edge.target === compiler) {
208 return true;
209 }
210 }
211 return false;
212 };
213 const sortEdges = (e1, e2) => {
214 return (
215 e1.source.name.localeCompare(e2.source.name) ||
216 e1.target.name.localeCompare(e2.target.name)
217 );
218 };
219 for (const source of this.compilers) {
220 const dependencies = this.dependencies.get(source);
221 if (dependencies) {
222 for (const dep of dependencies) {
223 const target = this.compilers.find(c => c.name === dep);
224 if (!target) {
225 missing.push(dep);
226 } else {
227 edges.add({
228 source,
229 target
230 });
231 }
232 }
233 }
234 }
235 /** @type {string[]} */
236 const errors = missing.map(m => `Compiler dependency \`${m}\` not found.`);
237 const stack = this.compilers.filter(c => !targetFound(c));
238 while (stack.length > 0) {
239 const current = stack.pop();
240 for (const edge of edges) {
241 if (edge.source === current) {
242 edges.delete(edge);
243 const target = edge.target;
244 if (!targetFound(target)) {
245 stack.push(target);
246 }
247 }
248 }
249 }
250 if (edges.size > 0) {
251 /** @type {string[]} */
252 const lines = Array.from(edges)
253 .sort(sortEdges)
254 .map(edge => `${edge.source.name} -> ${edge.target.name}`);
255 lines.unshift("Circular dependency found in compiler dependencies.");
256 errors.unshift(lines.join("\n"));
257 }
258 if (errors.length > 0) {
259 const message = errors.join("\n");
260 callback(new Error(message));
261 return false;
262 }
263 return true;
264 }
265
266 // TODO webpack 6 remove
267 /**
268 * @deprecated This method should have been private
269 * @param {Compiler[]} compilers the child compilers
270 * @param {RunWithDependenciesHandler} fn a handler to run for each compiler
271 * @param {Callback<MultiStats>} callback the compiler's handler
272 * @returns {void}
273 */
274 runWithDependencies(compilers, fn, callback) {
275 const fulfilledNames = new Set();
276 let remainingCompilers = compilers;
277 const isDependencyFulfilled = d => fulfilledNames.has(d);
278 const getReadyCompilers = () => {
279 let readyCompilers = [];
280 let list = remainingCompilers;
281 remainingCompilers = [];
282 for (const c of list) {
283 const dependencies = this.dependencies.get(c);
284 const ready =
285 !dependencies || dependencies.every(isDependencyFulfilled);
286 if (ready) {
287 readyCompilers.push(c);
288 } else {
289 remainingCompilers.push(c);
290 }
291 }
292 return readyCompilers;
293 };
294 const runCompilers = callback => {
295 if (remainingCompilers.length === 0) return callback();
296 asyncLib.map(
297 getReadyCompilers(),
298 (compiler, callback) => {
299 fn(compiler, err => {
300 if (err) return callback(err);
301 fulfilledNames.add(compiler.name);
302 runCompilers(callback);
303 });
304 },
305 callback
306 );
307 };
308 runCompilers(callback);
309 }
310
311 /**
312 * @template SetupResult
313 * @param {function(Compiler, number, Callback<Stats>, function(): boolean, function(): void, function(): void): SetupResult} setup setup a single compiler
314 * @param {function(Compiler, SetupResult, Callback<Stats>): void} run run/continue a single compiler
315 * @param {Callback<MultiStats>} callback callback when all compilers are done, result includes Stats of all changed compilers
316 * @returns {SetupResult[]} result of setup
317 */
318 _runGraph(setup, run, callback) {
319 /** @typedef {{ compiler: Compiler, setupResult: SetupResult, result: Stats, state: "pending" | "blocked" | "queued" | "starting" | "running" | "running-outdated" | "done", children: Node[], parents: Node[] }} Node */
320
321 // State transitions for nodes:
322 // -> blocked (initial)
323 // blocked -> starting [running++] (when all parents done)
324 // queued -> starting [running++] (when processing the queue)
325 // starting -> running (when run has been called)
326 // running -> done [running--] (when compilation is done)
327 // done -> pending (when invalidated from file change)
328 // pending -> blocked [add to queue] (when invalidated from aggregated changes)
329 // done -> blocked [add to queue] (when invalidated, from parent invalidation)
330 // running -> running-outdated (when invalidated, either from change or parent invalidation)
331 // running-outdated -> blocked [running--] (when compilation is done)
332
333 /** @type {Node[]} */
334 const nodes = this.compilers.map(compiler => ({
335 compiler,
336 setupResult: undefined,
337 result: undefined,
338 state: "blocked",
339 children: [],
340 parents: []
341 }));
342 /** @type {Map<string, Node>} */
343 const compilerToNode = new Map();
344 for (const node of nodes) compilerToNode.set(node.compiler.name, node);
345 for (const node of nodes) {
346 const dependencies = this.dependencies.get(node.compiler);
347 if (!dependencies) continue;
348 for (const dep of dependencies) {
349 const parent = compilerToNode.get(dep);
350 node.parents.push(parent);
351 parent.children.push(node);
352 }
353 }
354 /** @type {ArrayQueue<Node>} */
355 const queue = new ArrayQueue();
356 for (const node of nodes) {
357 if (node.parents.length === 0) {
358 node.state = "queued";
359 queue.enqueue(node);
360 }
361 }
362 let errored = false;
363 let running = 0;
364 const parallelism = this._options.parallelism;
365 /**
366 * @param {Node} node node
367 * @param {Error=} err error
368 * @param {Stats=} stats result
369 * @returns {void}
370 */
371 const nodeDone = (node, err, stats) => {
372 if (errored) return;
373 if (err) {
374 errored = true;
375 return asyncLib.each(
376 nodes,
377 (node, callback) => {
378 if (node.compiler.watching) {
379 node.compiler.watching.close(callback);
380 } else {
381 callback();
382 }
383 },
384 () => callback(err)
385 );
386 }
387 node.result = stats;
388 running--;
389 if (node.state === "running") {
390 node.state = "done";
391 for (const child of node.children) {
392 if (child.state === "blocked") queue.enqueue(child);
393 }
394 } else if (node.state === "running-outdated") {
395 node.state = "blocked";
396 queue.enqueue(node);
397 }
398 processQueue();
399 };
400 /**
401 * @param {Node} node node
402 * @returns {void}
403 */
404 const nodeInvalidFromParent = node => {
405 if (node.state === "done") {
406 node.state = "blocked";
407 } else if (node.state === "running") {
408 node.state = "running-outdated";
409 }
410 for (const child of node.children) {
411 nodeInvalidFromParent(child);
412 }
413 };
414 /**
415 * @param {Node} node node
416 * @returns {void}
417 */
418 const nodeInvalid = node => {
419 if (node.state === "done") {
420 node.state = "pending";
421 } else if (node.state === "running") {
422 node.state = "running-outdated";
423 }
424 for (const child of node.children) {
425 nodeInvalidFromParent(child);
426 }
427 };
428 /**
429 * @param {Node} node node
430 * @returns {void}
431 */
432 const nodeChange = node => {
433 nodeInvalid(node);
434 if (node.state === "pending") {
435 node.state = "blocked";
436 }
437 if (node.state === "blocked") {
438 queue.enqueue(node);
439 processQueue();
440 }
441 };
442
443 const setupResults = [];
444 nodes.forEach((node, i) => {
445 setupResults.push(
446 (node.setupResult = setup(
447 node.compiler,
448 i,
449 nodeDone.bind(null, node),
450 () => node.state !== "starting" && node.state !== "running",
451 () => nodeChange(node),
452 () => nodeInvalid(node)
453 ))
454 );
455 });
456 let processing = true;
457 const processQueue = () => {
458 if (processing) return;
459 processing = true;
460 process.nextTick(processQueueWorker);
461 };
462 const processQueueWorker = () => {
463 while (running < parallelism && queue.length > 0 && !errored) {
464 const node = queue.dequeue();
465 if (
466 node.state === "queued" ||
467 (node.state === "blocked" &&
468 node.parents.every(p => p.state === "done"))
469 ) {
470 running++;
471 node.state = "starting";
472 run(node.compiler, node.setupResult, nodeDone.bind(null, node));
473 node.state = "running";
474 }
475 }
476 processing = false;
477 if (
478 !errored &&
479 running === 0 &&
480 nodes.every(node => node.state === "done")
481 ) {
482 const stats = [];
483 for (const node of nodes) {
484 const result = node.result;
485 if (result) {
486 node.result = undefined;
487 stats.push(result);
488 }
489 }
490 if (stats.length > 0) {
491 callback(null, new MultiStats(stats));
492 }
493 }
494 };
495 processQueueWorker();
496 return setupResults;
497 }
498
499 /**
500 * @param {WatchOptions|WatchOptions[]} watchOptions the watcher's options
501 * @param {Callback<MultiStats>} handler signals when the call finishes
502 * @returns {MultiWatching} a compiler watcher
503 */
504 watch(watchOptions, handler) {
505 if (this.running) {
506 return handler(new ConcurrentCompilationError());
507 }
508 this.running = true;
509
510 if (this.validateDependencies(handler)) {
511 const watchings = this._runGraph(
512 (compiler, idx, callback, isBlocked, setChanged, setInvalid) => {
513 const watching = compiler.watch(
514 Array.isArray(watchOptions) ? watchOptions[idx] : watchOptions,
515 callback
516 );
517 if (watching) {
518 watching._onInvalid = setInvalid;
519 watching._onChange = setChanged;
520 watching._isBlocked = isBlocked;
521 }
522 return watching;
523 },
524 (compiler, watching, callback) => {
525 if (compiler.watching !== watching) return;
526 if (!watching.running) watching.invalidate();
527 },
528 handler
529 );
530 return new MultiWatching(watchings, this);
531 }
532
533 return new MultiWatching([], this);
534 }
535
536 /**
537 * @param {Callback<MultiStats>} callback signals when the call finishes
538 * @returns {void}
539 */
540 run(callback) {
541 if (this.running) {
542 return callback(new ConcurrentCompilationError());
543 }
544 this.running = true;
545
546 if (this.validateDependencies(callback)) {
547 this._runGraph(
548 () => {},
549 (compiler, setupResult, callback) => compiler.run(callback),
550 (err, stats) => {
551 this.running = false;
552
553 if (callback !== undefined) {
554 return callback(err, stats);
555 }
556 }
557 );
558 }
559 }
560
561 purgeInputFileSystem() {
562 for (const compiler of this.compilers) {
563 if (compiler.inputFileSystem && compiler.inputFileSystem.purge) {
564 compiler.inputFileSystem.purge();
565 }
566 }
567 }
568
569 /**
570 * @param {Callback<void>} callback signals when the compiler closes
571 * @returns {void}
572 */
573 close(callback) {
574 asyncLib.each(
575 this.compilers,
576 (compiler, callback) => {
577 compiler.close(callback);
578 },
579 callback
580 );
581 }
582};
Note: See TracBrowser for help on using the repository browser.