| [79a0317] | 1 | /*
|
|---|
| 2 | MIT License http://www.opensource.org/licenses/mit-license.php
|
|---|
| 3 | Author Tobias Koppers @sokra
|
|---|
| 4 | */
|
|---|
| 5 |
|
|---|
| 6 | "use strict";
|
|---|
| 7 |
|
|---|
| 8 | const Cache = require("../Cache");
|
|---|
| 9 | const ProgressPlugin = require("../ProgressPlugin");
|
|---|
| 10 |
|
|---|
| 11 | /** @typedef {import("../Compiler")} Compiler */
|
|---|
| 12 | /** @typedef {import("./PackFileCacheStrategy")} PackFileCacheStrategy */
|
|---|
| 13 |
|
|---|
| 14 | const BUILD_DEPENDENCIES_KEY = Symbol("build dependencies key");
|
|---|
| 15 |
|
|---|
| 16 | class IdleFileCachePlugin {
|
|---|
| 17 | /**
|
|---|
| 18 | * @param {PackFileCacheStrategy} strategy cache strategy
|
|---|
| 19 | * @param {number} idleTimeout timeout
|
|---|
| 20 | * @param {number} idleTimeoutForInitialStore initial timeout
|
|---|
| 21 | * @param {number} idleTimeoutAfterLargeChanges timeout after changes
|
|---|
| 22 | */
|
|---|
| 23 | constructor(
|
|---|
| 24 | strategy,
|
|---|
| 25 | idleTimeout,
|
|---|
| 26 | idleTimeoutForInitialStore,
|
|---|
| 27 | idleTimeoutAfterLargeChanges
|
|---|
| 28 | ) {
|
|---|
| 29 | this.strategy = strategy;
|
|---|
| 30 | this.idleTimeout = idleTimeout;
|
|---|
| 31 | this.idleTimeoutForInitialStore = idleTimeoutForInitialStore;
|
|---|
| 32 | this.idleTimeoutAfterLargeChanges = idleTimeoutAfterLargeChanges;
|
|---|
| 33 | }
|
|---|
| 34 |
|
|---|
| 35 | /**
|
|---|
| 36 | * Apply the plugin
|
|---|
| 37 | * @param {Compiler} compiler the compiler instance
|
|---|
| 38 | * @returns {void}
|
|---|
| 39 | */
|
|---|
| 40 | apply(compiler) {
|
|---|
| 41 | const strategy = this.strategy;
|
|---|
| 42 | const idleTimeout = this.idleTimeout;
|
|---|
| 43 | const idleTimeoutForInitialStore = Math.min(
|
|---|
| 44 | idleTimeout,
|
|---|
| 45 | this.idleTimeoutForInitialStore
|
|---|
| 46 | );
|
|---|
| 47 | const idleTimeoutAfterLargeChanges = this.idleTimeoutAfterLargeChanges;
|
|---|
| 48 | const resolvedPromise = Promise.resolve();
|
|---|
| 49 |
|
|---|
| 50 | let timeSpendInBuild = 0;
|
|---|
| 51 | let timeSpendInStore = 0;
|
|---|
| 52 | let avgTimeSpendInStore = 0;
|
|---|
| 53 |
|
|---|
| 54 | /** @type {Map<string | typeof BUILD_DEPENDENCIES_KEY, () => Promise<void>>} */
|
|---|
| 55 | const pendingIdleTasks = new Map();
|
|---|
| 56 |
|
|---|
| 57 | compiler.cache.hooks.store.tap(
|
|---|
| 58 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 59 | (identifier, etag, data) => {
|
|---|
| 60 | pendingIdleTasks.set(identifier, () =>
|
|---|
| 61 | strategy.store(identifier, etag, data)
|
|---|
| 62 | );
|
|---|
| 63 | }
|
|---|
| 64 | );
|
|---|
| 65 |
|
|---|
| 66 | compiler.cache.hooks.get.tapPromise(
|
|---|
| 67 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 68 | (identifier, etag, gotHandlers) => {
|
|---|
| 69 | const restore = () =>
|
|---|
| 70 | strategy.restore(identifier, etag).then(cacheEntry => {
|
|---|
| 71 | if (cacheEntry === undefined) {
|
|---|
| 72 | gotHandlers.push((result, callback) => {
|
|---|
| 73 | if (result !== undefined) {
|
|---|
| 74 | pendingIdleTasks.set(identifier, () =>
|
|---|
| 75 | strategy.store(identifier, etag, result)
|
|---|
| 76 | );
|
|---|
| 77 | }
|
|---|
| 78 | callback();
|
|---|
| 79 | });
|
|---|
| 80 | } else {
|
|---|
| 81 | return cacheEntry;
|
|---|
| 82 | }
|
|---|
| 83 | });
|
|---|
| 84 | const pendingTask = pendingIdleTasks.get(identifier);
|
|---|
| 85 | if (pendingTask !== undefined) {
|
|---|
| 86 | pendingIdleTasks.delete(identifier);
|
|---|
| 87 | return pendingTask().then(restore);
|
|---|
| 88 | }
|
|---|
| 89 | return restore();
|
|---|
| 90 | }
|
|---|
| 91 | );
|
|---|
| 92 |
|
|---|
| 93 | compiler.cache.hooks.storeBuildDependencies.tap(
|
|---|
| 94 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 95 | dependencies => {
|
|---|
| 96 | pendingIdleTasks.set(BUILD_DEPENDENCIES_KEY, () =>
|
|---|
| 97 | Promise.resolve().then(() =>
|
|---|
| 98 | strategy.storeBuildDependencies(dependencies)
|
|---|
| 99 | )
|
|---|
| 100 | );
|
|---|
| 101 | }
|
|---|
| 102 | );
|
|---|
| 103 |
|
|---|
| 104 | compiler.cache.hooks.shutdown.tapPromise(
|
|---|
| 105 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 106 | () => {
|
|---|
| 107 | if (idleTimer) {
|
|---|
| 108 | clearTimeout(idleTimer);
|
|---|
| 109 | idleTimer = undefined;
|
|---|
| 110 | }
|
|---|
| 111 | isIdle = false;
|
|---|
| 112 | const reportProgress = ProgressPlugin.getReporter(compiler);
|
|---|
| 113 | const jobs = Array.from(pendingIdleTasks.values());
|
|---|
| 114 | if (reportProgress) reportProgress(0, "process pending cache items");
|
|---|
| 115 | const promises = jobs.map(fn => fn());
|
|---|
| 116 | pendingIdleTasks.clear();
|
|---|
| 117 | promises.push(currentIdlePromise);
|
|---|
| 118 | const promise = Promise.all(promises);
|
|---|
| 119 | currentIdlePromise = promise.then(() => strategy.afterAllStored());
|
|---|
| 120 | if (reportProgress) {
|
|---|
| 121 | currentIdlePromise = currentIdlePromise.then(() => {
|
|---|
| 122 | reportProgress(1, "stored");
|
|---|
| 123 | });
|
|---|
| 124 | }
|
|---|
| 125 | return currentIdlePromise.then(() => {
|
|---|
| 126 | // Reset strategy
|
|---|
| 127 | if (strategy.clear) strategy.clear();
|
|---|
| 128 | });
|
|---|
| 129 | }
|
|---|
| 130 | );
|
|---|
| 131 |
|
|---|
| 132 | /** @type {Promise<any>} */
|
|---|
| 133 | let currentIdlePromise = resolvedPromise;
|
|---|
| 134 | let isIdle = false;
|
|---|
| 135 | let isInitialStore = true;
|
|---|
| 136 | const processIdleTasks = () => {
|
|---|
| 137 | if (isIdle) {
|
|---|
| 138 | const startTime = Date.now();
|
|---|
| 139 | if (pendingIdleTasks.size > 0) {
|
|---|
| 140 | const promises = [currentIdlePromise];
|
|---|
| 141 | const maxTime = startTime + 100;
|
|---|
| 142 | let maxCount = 100;
|
|---|
| 143 | for (const [filename, factory] of pendingIdleTasks) {
|
|---|
| 144 | pendingIdleTasks.delete(filename);
|
|---|
| 145 | promises.push(factory());
|
|---|
| 146 | if (maxCount-- <= 0 || Date.now() > maxTime) break;
|
|---|
| 147 | }
|
|---|
| 148 | currentIdlePromise = Promise.all(promises);
|
|---|
| 149 | currentIdlePromise.then(() => {
|
|---|
| 150 | timeSpendInStore += Date.now() - startTime;
|
|---|
| 151 | // Allow to exit the process between
|
|---|
| 152 | idleTimer = setTimeout(processIdleTasks, 0);
|
|---|
| 153 | idleTimer.unref();
|
|---|
| 154 | });
|
|---|
| 155 | return;
|
|---|
| 156 | }
|
|---|
| 157 | currentIdlePromise = currentIdlePromise
|
|---|
| 158 | .then(async () => {
|
|---|
| 159 | await strategy.afterAllStored();
|
|---|
| 160 | timeSpendInStore += Date.now() - startTime;
|
|---|
| 161 | avgTimeSpendInStore =
|
|---|
| 162 | Math.max(avgTimeSpendInStore, timeSpendInStore) * 0.9 +
|
|---|
| 163 | timeSpendInStore * 0.1;
|
|---|
| 164 | timeSpendInStore = 0;
|
|---|
| 165 | timeSpendInBuild = 0;
|
|---|
| 166 | })
|
|---|
| 167 | .catch(err => {
|
|---|
| 168 | const logger = compiler.getInfrastructureLogger(
|
|---|
| 169 | "IdleFileCachePlugin"
|
|---|
| 170 | );
|
|---|
| 171 | logger.warn(`Background tasks during idle failed: ${err.message}`);
|
|---|
| 172 | logger.debug(err.stack);
|
|---|
| 173 | });
|
|---|
| 174 | isInitialStore = false;
|
|---|
| 175 | }
|
|---|
| 176 | };
|
|---|
| 177 | /** @type {ReturnType<typeof setTimeout> | undefined} */
|
|---|
| 178 | let idleTimer;
|
|---|
| 179 | compiler.cache.hooks.beginIdle.tap(
|
|---|
| 180 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 181 | () => {
|
|---|
| 182 | const isLargeChange = timeSpendInBuild > avgTimeSpendInStore * 2;
|
|---|
| 183 | if (isInitialStore && idleTimeoutForInitialStore < idleTimeout) {
|
|---|
| 184 | compiler
|
|---|
| 185 | .getInfrastructureLogger("IdleFileCachePlugin")
|
|---|
| 186 | .log(
|
|---|
| 187 | `Initial cache was generated and cache will be persisted in ${
|
|---|
| 188 | idleTimeoutForInitialStore / 1000
|
|---|
| 189 | }s.`
|
|---|
| 190 | );
|
|---|
| 191 | } else if (
|
|---|
| 192 | isLargeChange &&
|
|---|
| 193 | idleTimeoutAfterLargeChanges < idleTimeout
|
|---|
| 194 | ) {
|
|---|
| 195 | compiler
|
|---|
| 196 | .getInfrastructureLogger("IdleFileCachePlugin")
|
|---|
| 197 | .log(
|
|---|
| 198 | `Spend ${Math.round(timeSpendInBuild) / 1000}s in build and ${
|
|---|
| 199 | Math.round(avgTimeSpendInStore) / 1000
|
|---|
| 200 | }s in average in cache store. This is considered as large change and cache will be persisted in ${
|
|---|
| 201 | idleTimeoutAfterLargeChanges / 1000
|
|---|
| 202 | }s.`
|
|---|
| 203 | );
|
|---|
| 204 | }
|
|---|
| 205 | idleTimer = setTimeout(
|
|---|
| 206 | () => {
|
|---|
| 207 | idleTimer = undefined;
|
|---|
| 208 | isIdle = true;
|
|---|
| 209 | resolvedPromise.then(processIdleTasks);
|
|---|
| 210 | },
|
|---|
| 211 | Math.min(
|
|---|
| 212 | isInitialStore ? idleTimeoutForInitialStore : Infinity,
|
|---|
| 213 | isLargeChange ? idleTimeoutAfterLargeChanges : Infinity,
|
|---|
| 214 | idleTimeout
|
|---|
| 215 | )
|
|---|
| 216 | );
|
|---|
| 217 | idleTimer.unref();
|
|---|
| 218 | }
|
|---|
| 219 | );
|
|---|
| 220 | compiler.cache.hooks.endIdle.tap(
|
|---|
| 221 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 222 | () => {
|
|---|
| 223 | if (idleTimer) {
|
|---|
| 224 | clearTimeout(idleTimer);
|
|---|
| 225 | idleTimer = undefined;
|
|---|
| 226 | }
|
|---|
| 227 | isIdle = false;
|
|---|
| 228 | }
|
|---|
| 229 | );
|
|---|
| 230 | compiler.hooks.done.tap("IdleFileCachePlugin", stats => {
|
|---|
| 231 | // 10% build overhead is ignored, as it's not cacheable
|
|---|
| 232 | timeSpendInBuild *= 0.9;
|
|---|
| 233 | timeSpendInBuild +=
|
|---|
| 234 | /** @type {number} */ (stats.endTime) -
|
|---|
| 235 | /** @type {number} */ (stats.startTime);
|
|---|
| 236 | });
|
|---|
| 237 | }
|
|---|
| 238 | }
|
|---|
| 239 |
|
|---|
| 240 | module.exports = IdleFileCachePlugin;
|
|---|
