| 1 | /*
|
|---|
| 2 | MIT License http://www.opensource.org/licenses/mit-license.php
|
|---|
| 3 | Author Tobias Koppers @sokra
|
|---|
| 4 | */
|
|---|
| 5 |
|
|---|
| 6 | "use strict";
|
|---|
| 7 |
|
|---|
| 8 | const Cache = require("../Cache");
|
|---|
| 9 | const ProgressPlugin = require("../ProgressPlugin");
|
|---|
| 10 |
|
|---|
| 11 | /** @typedef {import("../Compiler")} Compiler */
|
|---|
| 12 |
|
|---|
| 13 | const BUILD_DEPENDENCIES_KEY = Symbol();
|
|---|
| 14 |
|
|---|
| 15 | class IdleFileCachePlugin {
|
|---|
| 16 | /**
|
|---|
| 17 | * @param {TODO} strategy cache strategy
|
|---|
| 18 | * @param {number} idleTimeout timeout
|
|---|
| 19 | * @param {number} idleTimeoutForInitialStore initial timeout
|
|---|
| 20 | * @param {number} idleTimeoutAfterLargeChanges timeout after changes
|
|---|
| 21 | */
|
|---|
| 22 | constructor(
|
|---|
| 23 | strategy,
|
|---|
| 24 | idleTimeout,
|
|---|
| 25 | idleTimeoutForInitialStore,
|
|---|
| 26 | idleTimeoutAfterLargeChanges
|
|---|
| 27 | ) {
|
|---|
| 28 | this.strategy = strategy;
|
|---|
| 29 | this.idleTimeout = idleTimeout;
|
|---|
| 30 | this.idleTimeoutForInitialStore = idleTimeoutForInitialStore;
|
|---|
| 31 | this.idleTimeoutAfterLargeChanges = idleTimeoutAfterLargeChanges;
|
|---|
| 32 | }
|
|---|
| 33 |
|
|---|
| 34 | /**
|
|---|
| 35 | * Apply the plugin
|
|---|
| 36 | * @param {Compiler} compiler the compiler instance
|
|---|
| 37 | * @returns {void}
|
|---|
| 38 | */
|
|---|
| 39 | apply(compiler) {
|
|---|
| 40 | let strategy = this.strategy;
|
|---|
| 41 | const idleTimeout = this.idleTimeout;
|
|---|
| 42 | const idleTimeoutForInitialStore = Math.min(
|
|---|
| 43 | idleTimeout,
|
|---|
| 44 | this.idleTimeoutForInitialStore
|
|---|
| 45 | );
|
|---|
| 46 | const idleTimeoutAfterLargeChanges = this.idleTimeoutAfterLargeChanges;
|
|---|
| 47 | const resolvedPromise = Promise.resolve();
|
|---|
| 48 |
|
|---|
| 49 | let timeSpendInBuild = 0;
|
|---|
| 50 | let timeSpendInStore = 0;
|
|---|
| 51 | let avgTimeSpendInStore = 0;
|
|---|
| 52 |
|
|---|
| 53 | /** @type {Map<string | typeof BUILD_DEPENDENCIES_KEY, () => Promise>} */
|
|---|
| 54 | const pendingIdleTasks = new Map();
|
|---|
| 55 |
|
|---|
| 56 | compiler.cache.hooks.store.tap(
|
|---|
| 57 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 58 | (identifier, etag, data) => {
|
|---|
| 59 | pendingIdleTasks.set(identifier, () =>
|
|---|
| 60 | strategy.store(identifier, etag, data)
|
|---|
| 61 | );
|
|---|
| 62 | }
|
|---|
| 63 | );
|
|---|
| 64 |
|
|---|
| 65 | compiler.cache.hooks.get.tapPromise(
|
|---|
| 66 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 67 | (identifier, etag, gotHandlers) => {
|
|---|
| 68 | const restore = () =>
|
|---|
| 69 | strategy.restore(identifier, etag).then(cacheEntry => {
|
|---|
| 70 | if (cacheEntry === undefined) {
|
|---|
| 71 | gotHandlers.push((result, callback) => {
|
|---|
| 72 | if (result !== undefined) {
|
|---|
| 73 | pendingIdleTasks.set(identifier, () =>
|
|---|
| 74 | strategy.store(identifier, etag, result)
|
|---|
| 75 | );
|
|---|
| 76 | }
|
|---|
| 77 | callback();
|
|---|
| 78 | });
|
|---|
| 79 | } else {
|
|---|
| 80 | return cacheEntry;
|
|---|
| 81 | }
|
|---|
| 82 | });
|
|---|
| 83 | const pendingTask = pendingIdleTasks.get(identifier);
|
|---|
| 84 | if (pendingTask !== undefined) {
|
|---|
| 85 | pendingIdleTasks.delete(identifier);
|
|---|
| 86 | return pendingTask().then(restore);
|
|---|
| 87 | }
|
|---|
| 88 | return restore();
|
|---|
| 89 | }
|
|---|
| 90 | );
|
|---|
| 91 |
|
|---|
| 92 | compiler.cache.hooks.storeBuildDependencies.tap(
|
|---|
| 93 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 94 | dependencies => {
|
|---|
| 95 | pendingIdleTasks.set(BUILD_DEPENDENCIES_KEY, () =>
|
|---|
| 96 | strategy.storeBuildDependencies(dependencies)
|
|---|
| 97 | );
|
|---|
| 98 | }
|
|---|
| 99 | );
|
|---|
| 100 |
|
|---|
| 101 | compiler.cache.hooks.shutdown.tapPromise(
|
|---|
| 102 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 103 | () => {
|
|---|
| 104 | if (idleTimer) {
|
|---|
| 105 | clearTimeout(idleTimer);
|
|---|
| 106 | idleTimer = undefined;
|
|---|
| 107 | }
|
|---|
| 108 | isIdle = false;
|
|---|
| 109 | const reportProgress = ProgressPlugin.getReporter(compiler);
|
|---|
| 110 | const jobs = Array.from(pendingIdleTasks.values());
|
|---|
| 111 | if (reportProgress) reportProgress(0, "process pending cache items");
|
|---|
| 112 | const promises = jobs.map(fn => fn());
|
|---|
| 113 | pendingIdleTasks.clear();
|
|---|
| 114 | promises.push(currentIdlePromise);
|
|---|
| 115 | const promise = Promise.all(promises);
|
|---|
| 116 | currentIdlePromise = promise.then(() => strategy.afterAllStored());
|
|---|
| 117 | if (reportProgress) {
|
|---|
| 118 | currentIdlePromise = currentIdlePromise.then(() => {
|
|---|
| 119 | reportProgress(1, `stored`);
|
|---|
| 120 | });
|
|---|
| 121 | }
|
|---|
| 122 | return currentIdlePromise.then(() => {
|
|---|
| 123 | // Reset strategy
|
|---|
| 124 | if (strategy.clear) strategy.clear();
|
|---|
| 125 | });
|
|---|
| 126 | }
|
|---|
| 127 | );
|
|---|
| 128 |
|
|---|
| 129 | /** @type {Promise<any>} */
|
|---|
| 130 | let currentIdlePromise = resolvedPromise;
|
|---|
| 131 | let isIdle = false;
|
|---|
| 132 | let isInitialStore = true;
|
|---|
| 133 | const processIdleTasks = () => {
|
|---|
| 134 | if (isIdle) {
|
|---|
| 135 | const startTime = Date.now();
|
|---|
| 136 | if (pendingIdleTasks.size > 0) {
|
|---|
| 137 | const promises = [currentIdlePromise];
|
|---|
| 138 | const maxTime = startTime + 100;
|
|---|
| 139 | let maxCount = 100;
|
|---|
| 140 | for (const [filename, factory] of pendingIdleTasks) {
|
|---|
| 141 | pendingIdleTasks.delete(filename);
|
|---|
| 142 | promises.push(factory());
|
|---|
| 143 | if (maxCount-- <= 0 || Date.now() > maxTime) break;
|
|---|
| 144 | }
|
|---|
| 145 | currentIdlePromise = Promise.all(promises);
|
|---|
| 146 | currentIdlePromise.then(() => {
|
|---|
| 147 | timeSpendInStore += Date.now() - startTime;
|
|---|
| 148 | // Allow to exit the process between
|
|---|
| 149 | idleTimer = setTimeout(processIdleTasks, 0);
|
|---|
| 150 | idleTimer.unref();
|
|---|
| 151 | });
|
|---|
| 152 | return;
|
|---|
| 153 | }
|
|---|
| 154 | currentIdlePromise = currentIdlePromise
|
|---|
| 155 | .then(async () => {
|
|---|
| 156 | await strategy.afterAllStored();
|
|---|
| 157 | timeSpendInStore += Date.now() - startTime;
|
|---|
| 158 | avgTimeSpendInStore =
|
|---|
| 159 | Math.max(avgTimeSpendInStore, timeSpendInStore) * 0.9 +
|
|---|
| 160 | timeSpendInStore * 0.1;
|
|---|
| 161 | timeSpendInStore = 0;
|
|---|
| 162 | timeSpendInBuild = 0;
|
|---|
| 163 | })
|
|---|
| 164 | .catch(err => {
|
|---|
| 165 | const logger = compiler.getInfrastructureLogger(
|
|---|
| 166 | "IdleFileCachePlugin"
|
|---|
| 167 | );
|
|---|
| 168 | logger.warn(`Background tasks during idle failed: ${err.message}`);
|
|---|
| 169 | logger.debug(err.stack);
|
|---|
| 170 | });
|
|---|
| 171 | isInitialStore = false;
|
|---|
| 172 | }
|
|---|
| 173 | };
|
|---|
| 174 | let idleTimer = undefined;
|
|---|
| 175 | compiler.cache.hooks.beginIdle.tap(
|
|---|
| 176 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 177 | () => {
|
|---|
| 178 | const isLargeChange = timeSpendInBuild > avgTimeSpendInStore * 2;
|
|---|
| 179 | if (isInitialStore && idleTimeoutForInitialStore < idleTimeout) {
|
|---|
| 180 | compiler
|
|---|
| 181 | .getInfrastructureLogger("IdleFileCachePlugin")
|
|---|
| 182 | .log(
|
|---|
| 183 | `Initial cache was generated and cache will be persisted in ${
|
|---|
| 184 | idleTimeoutForInitialStore / 1000
|
|---|
| 185 | }s.`
|
|---|
| 186 | );
|
|---|
| 187 | } else if (
|
|---|
| 188 | isLargeChange &&
|
|---|
| 189 | idleTimeoutAfterLargeChanges < idleTimeout
|
|---|
| 190 | ) {
|
|---|
| 191 | compiler
|
|---|
| 192 | .getInfrastructureLogger("IdleFileCachePlugin")
|
|---|
| 193 | .log(
|
|---|
| 194 | `Spend ${Math.round(timeSpendInBuild) / 1000}s in build and ${
|
|---|
| 195 | Math.round(avgTimeSpendInStore) / 1000
|
|---|
| 196 | }s in average in cache store. This is considered as large change and cache will be persisted in ${
|
|---|
| 197 | idleTimeoutAfterLargeChanges / 1000
|
|---|
| 198 | }s.`
|
|---|
| 199 | );
|
|---|
| 200 | }
|
|---|
| 201 | idleTimer = setTimeout(() => {
|
|---|
| 202 | idleTimer = undefined;
|
|---|
| 203 | isIdle = true;
|
|---|
| 204 | resolvedPromise.then(processIdleTasks);
|
|---|
| 205 | }, Math.min(isInitialStore ? idleTimeoutForInitialStore : Infinity, isLargeChange ? idleTimeoutAfterLargeChanges : Infinity, idleTimeout));
|
|---|
| 206 | idleTimer.unref();
|
|---|
| 207 | }
|
|---|
| 208 | );
|
|---|
| 209 | compiler.cache.hooks.endIdle.tap(
|
|---|
| 210 | { name: "IdleFileCachePlugin", stage: Cache.STAGE_DISK },
|
|---|
| 211 | () => {
|
|---|
| 212 | if (idleTimer) {
|
|---|
| 213 | clearTimeout(idleTimer);
|
|---|
| 214 | idleTimer = undefined;
|
|---|
| 215 | }
|
|---|
| 216 | isIdle = false;
|
|---|
| 217 | }
|
|---|
| 218 | );
|
|---|
| 219 | compiler.hooks.done.tap("IdleFileCachePlugin", stats => {
|
|---|
| 220 | // 10% build overhead is ignored, as it's not cacheable
|
|---|
| 221 | timeSpendInBuild *= 0.9;
|
|---|
| 222 | timeSpendInBuild += stats.endTime - stats.startTime;
|
|---|
| 223 | });
|
|---|
| 224 | }
|
|---|
| 225 | }
|
|---|
| 226 |
|
|---|
| 227 | module.exports = IdleFileCachePlugin;
|
|---|
