IdleFileCachePlugin.js 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const Cache = require("../Cache");
  7. const ProgressPlugin = require("../ProgressPlugin");
  8. /** @typedef {import("../Compiler")} Compiler */
  9. /** @typedef {import("./PackFileCacheStrategy")} PackFileCacheStrategy */
  10. const BUILD_DEPENDENCIES_KEY = Symbol("build dependencies key");
  11. const PLUGIN_NAME = "IdleFileCachePlugin";
  12. class IdleFileCachePlugin {
  13. /**
  14. * @param {PackFileCacheStrategy} strategy cache strategy
  15. * @param {number} idleTimeout timeout
  16. * @param {number} idleTimeoutForInitialStore initial timeout
  17. * @param {number} idleTimeoutAfterLargeChanges timeout after changes
  18. */
  19. constructor(
  20. strategy,
  21. idleTimeout,
  22. idleTimeoutForInitialStore,
  23. idleTimeoutAfterLargeChanges
  24. ) {
  25. this.strategy = strategy;
  26. this.idleTimeout = idleTimeout;
  27. this.idleTimeoutForInitialStore = idleTimeoutForInitialStore;
  28. this.idleTimeoutAfterLargeChanges = idleTimeoutAfterLargeChanges;
  29. }
  30. /**
  31. * Apply the plugin
  32. * @param {Compiler} compiler the compiler instance
  33. * @returns {void}
  34. */
  35. apply(compiler) {
  36. const strategy = this.strategy;
  37. const idleTimeout = this.idleTimeout;
  38. const idleTimeoutForInitialStore = Math.min(
  39. idleTimeout,
  40. this.idleTimeoutForInitialStore
  41. );
  42. const idleTimeoutAfterLargeChanges = this.idleTimeoutAfterLargeChanges;
  43. const resolvedPromise = Promise.resolve();
  44. let timeSpendInBuild = 0;
  45. let timeSpendInStore = 0;
  46. let avgTimeSpendInStore = 0;
  47. /** @type {Map<string | typeof BUILD_DEPENDENCIES_KEY, () => Promise<void>>} */
  48. const pendingIdleTasks = new Map();
  49. compiler.cache.hooks.store.tap(
  50. { name: PLUGIN_NAME, stage: Cache.STAGE_DISK },
  51. (identifier, etag, data) => {
  52. pendingIdleTasks.set(identifier, () =>
  53. strategy.store(identifier, etag, data)
  54. );
  55. }
  56. );
  57. compiler.cache.hooks.get.tapPromise(
  58. { name: PLUGIN_NAME, stage: Cache.STAGE_DISK },
  59. (identifier, etag, gotHandlers) => {
  60. const restore = () =>
  61. strategy.restore(identifier, etag).then(cacheEntry => {
  62. if (cacheEntry === undefined) {
  63. gotHandlers.push((result, callback) => {
  64. if (result !== undefined) {
  65. pendingIdleTasks.set(identifier, () =>
  66. strategy.store(identifier, etag, result)
  67. );
  68. }
  69. callback();
  70. });
  71. } else {
  72. return cacheEntry;
  73. }
  74. });
  75. const pendingTask = pendingIdleTasks.get(identifier);
  76. if (pendingTask !== undefined) {
  77. pendingIdleTasks.delete(identifier);
  78. return pendingTask().then(restore);
  79. }
  80. return restore();
  81. }
  82. );
  83. compiler.cache.hooks.storeBuildDependencies.tap(
  84. { name: PLUGIN_NAME, stage: Cache.STAGE_DISK },
  85. dependencies => {
  86. pendingIdleTasks.set(BUILD_DEPENDENCIES_KEY, () =>
  87. Promise.resolve().then(() =>
  88. strategy.storeBuildDependencies(dependencies)
  89. )
  90. );
  91. }
  92. );
  93. compiler.cache.hooks.shutdown.tapPromise(
  94. { name: PLUGIN_NAME, stage: Cache.STAGE_DISK },
  95. () => {
  96. if (idleTimer) {
  97. clearTimeout(idleTimer);
  98. idleTimer = undefined;
  99. }
  100. isIdle = false;
  101. const reportProgress = ProgressPlugin.getReporter(compiler);
  102. const jobs = Array.from(pendingIdleTasks.values());
  103. if (reportProgress) reportProgress(0, "process pending cache items");
  104. const promises = jobs.map(fn => fn());
  105. pendingIdleTasks.clear();
  106. promises.push(currentIdlePromise);
  107. const promise = Promise.all(promises);
  108. currentIdlePromise = promise.then(() => strategy.afterAllStored());
  109. if (reportProgress) {
  110. currentIdlePromise = currentIdlePromise.then(() => {
  111. reportProgress(1, "stored");
  112. });
  113. }
  114. return currentIdlePromise.then(() => {
  115. // Reset strategy
  116. if (strategy.clear) strategy.clear();
  117. });
  118. }
  119. );
  120. /** @type {Promise<TODO>} */
  121. let currentIdlePromise = resolvedPromise;
  122. let isIdle = false;
  123. let isInitialStore = true;
  124. const processIdleTasks = () => {
  125. if (isIdle) {
  126. const startTime = Date.now();
  127. if (pendingIdleTasks.size > 0) {
  128. const promises = [currentIdlePromise];
  129. const maxTime = startTime + 100;
  130. let maxCount = 100;
  131. for (const [filename, factory] of pendingIdleTasks) {
  132. pendingIdleTasks.delete(filename);
  133. promises.push(factory());
  134. if (maxCount-- <= 0 || Date.now() > maxTime) break;
  135. }
  136. currentIdlePromise = Promise.all(promises);
  137. currentIdlePromise.then(() => {
  138. timeSpendInStore += Date.now() - startTime;
  139. // Allow to exit the process between
  140. idleTimer = setTimeout(processIdleTasks, 0);
  141. idleTimer.unref();
  142. });
  143. return;
  144. }
  145. currentIdlePromise = currentIdlePromise
  146. .then(async () => {
  147. await strategy.afterAllStored();
  148. timeSpendInStore += Date.now() - startTime;
  149. avgTimeSpendInStore =
  150. Math.max(avgTimeSpendInStore, timeSpendInStore) * 0.9 +
  151. timeSpendInStore * 0.1;
  152. timeSpendInStore = 0;
  153. timeSpendInBuild = 0;
  154. })
  155. .catch(err => {
  156. const logger = compiler.getInfrastructureLogger(PLUGIN_NAME);
  157. logger.warn(`Background tasks during idle failed: ${err.message}`);
  158. logger.debug(err.stack);
  159. });
  160. isInitialStore = false;
  161. }
  162. };
  163. /** @type {ReturnType<typeof setTimeout> | undefined} */
  164. let idleTimer;
  165. compiler.cache.hooks.beginIdle.tap(
  166. { name: PLUGIN_NAME, stage: Cache.STAGE_DISK },
  167. () => {
  168. const isLargeChange = timeSpendInBuild > avgTimeSpendInStore * 2;
  169. if (isInitialStore && idleTimeoutForInitialStore < idleTimeout) {
  170. compiler
  171. .getInfrastructureLogger(PLUGIN_NAME)
  172. .log(
  173. `Initial cache was generated and cache will be persisted in ${
  174. idleTimeoutForInitialStore / 1000
  175. }s.`
  176. );
  177. } else if (
  178. isLargeChange &&
  179. idleTimeoutAfterLargeChanges < idleTimeout
  180. ) {
  181. compiler
  182. .getInfrastructureLogger(PLUGIN_NAME)
  183. .log(
  184. `Spend ${Math.round(timeSpendInBuild) / 1000}s in build and ${
  185. Math.round(avgTimeSpendInStore) / 1000
  186. }s in average in cache store. This is considered as large change and cache will be persisted in ${
  187. idleTimeoutAfterLargeChanges / 1000
  188. }s.`
  189. );
  190. }
  191. idleTimer = setTimeout(
  192. () => {
  193. idleTimer = undefined;
  194. isIdle = true;
  195. resolvedPromise.then(processIdleTasks);
  196. },
  197. Math.min(
  198. isInitialStore ? idleTimeoutForInitialStore : Infinity,
  199. isLargeChange ? idleTimeoutAfterLargeChanges : Infinity,
  200. idleTimeout
  201. )
  202. );
  203. idleTimer.unref();
  204. }
  205. );
  206. compiler.cache.hooks.endIdle.tap(
  207. { name: PLUGIN_NAME, stage: Cache.STAGE_DISK },
  208. () => {
  209. if (idleTimer) {
  210. clearTimeout(idleTimer);
  211. idleTimer = undefined;
  212. }
  213. isIdle = false;
  214. }
  215. );
  216. compiler.hooks.done.tap(PLUGIN_NAME, stats => {
  217. // 10% build overhead is ignored, as it's not cacheable
  218. timeSpendInBuild *= 0.9;
  219. timeSpendInBuild +=
  220. /** @type {number} */ (stats.endTime) -
  221. /** @type {number} */ (stats.startTime);
  222. });
  223. }
  224. }
  225. module.exports = IdleFileCachePlugin;