RealContentHashPlugin.js 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const { SyncBailHook } = require("tapable");
  7. const { RawSource, CachedSource, CompatSource } = require("webpack-sources");
  8. const Compilation = require("../Compilation");
  9. const WebpackError = require("../WebpackError");
  10. const { compareSelect, compareStrings } = require("../util/comparators");
  11. const createHash = require("../util/createHash");
  12. /** @typedef {import("webpack-sources").Source} Source */
  13. /** @typedef {import("../Cache").Etag} Etag */
  14. /** @typedef {import("../Compilation").AssetInfo} AssetInfo */
  15. /** @typedef {import("../Compiler")} Compiler */
  16. /** @typedef {typeof import("../util/Hash")} Hash */
  17. const EMPTY_SET = new Set();
  18. /**
  19. * @template T
  20. * @param {T | T[]} itemOrItems item or items
  21. * @param {Set<T>} list list
  22. */
  23. const addToList = (itemOrItems, list) => {
  24. if (Array.isArray(itemOrItems)) {
  25. for (const item of itemOrItems) {
  26. list.add(item);
  27. }
  28. } else if (itemOrItems) {
  29. list.add(itemOrItems);
  30. }
  31. };
  32. /**
  33. * @template T
  34. * @param {T[]} input list
  35. * @param {(item: T) => Buffer} fn map function
  36. * @returns {Buffer[]} buffers without duplicates
  37. */
  38. const mapAndDeduplicateBuffers = (input, fn) => {
  39. // Buffer.equals compares size first so this should be efficient enough
  40. // If it becomes a performance problem we can use a map and group by size
  41. // instead of looping over all assets.
  42. const result = [];
  43. outer: for (const value of input) {
  44. const buf = fn(value);
  45. for (const other of result) {
  46. if (buf.equals(other)) continue outer;
  47. }
  48. result.push(buf);
  49. }
  50. return result;
  51. };
  52. /**
  53. * Escapes regular expression metacharacters
  54. * @param {string} str String to quote
  55. * @returns {string} Escaped string
  56. */
  57. const quoteMeta = str => str.replace(/[-[\]\\/{}()*+?.^$|]/g, "\\$&");
  58. const cachedSourceMap = new WeakMap();
  59. /**
  60. * @param {Source} source source
  61. * @returns {CachedSource} cached source
  62. */
  63. const toCachedSource = source => {
  64. if (source instanceof CachedSource) {
  65. return source;
  66. }
  67. const entry = cachedSourceMap.get(source);
  68. if (entry !== undefined) return entry;
  69. const newSource = new CachedSource(CompatSource.from(source));
  70. cachedSourceMap.set(source, newSource);
  71. return newSource;
  72. };
  73. /** @typedef {Set<string>} OwnHashes */
  74. /** @typedef {Set<string>} ReferencedHashes */
  75. /** @typedef {Set<string>} Hashes */
  76. /**
  77. * @typedef {object} AssetInfoForRealContentHash
  78. * @property {string} name
  79. * @property {AssetInfo} info
  80. * @property {Source} source
  81. * @property {RawSource | undefined} newSource
  82. * @property {RawSource | undefined} newSourceWithoutOwn
  83. * @property {string} content
  84. * @property {OwnHashes | undefined} ownHashes
  85. * @property {Promise<void> | undefined} contentComputePromise
  86. * @property {Promise<void> | undefined} contentComputeWithoutOwnPromise
  87. * @property {ReferencedHashes | undefined} referencedHashes
  88. * @property {Hashes} hashes
  89. */
  90. /**
  91. * @typedef {object} CompilationHooks
  92. * @property {SyncBailHook<[Buffer[], string], string | void>} updateHash
  93. */
  94. /** @type {WeakMap<Compilation, CompilationHooks>} */
  95. const compilationHooksMap = new WeakMap();
  96. /**
  97. * @typedef {object} RealContentHashPluginOptions
  98. * @property {string | Hash} hashFunction the hash function to use
  99. * @property {string=} hashDigest the hash digest to use
  100. */
  101. const PLUGIN_NAME = "RealContentHashPlugin";
  102. class RealContentHashPlugin {
  103. /**
  104. * @param {Compilation} compilation the compilation
  105. * @returns {CompilationHooks} the attached hooks
  106. */
  107. static getCompilationHooks(compilation) {
  108. if (!(compilation instanceof Compilation)) {
  109. throw new TypeError(
  110. "The 'compilation' argument must be an instance of Compilation"
  111. );
  112. }
  113. let hooks = compilationHooksMap.get(compilation);
  114. if (hooks === undefined) {
  115. hooks = {
  116. updateHash: new SyncBailHook(["content", "oldHash"])
  117. };
  118. compilationHooksMap.set(compilation, hooks);
  119. }
  120. return hooks;
  121. }
  122. /**
  123. * @param {RealContentHashPluginOptions} options options
  124. */
  125. constructor({ hashFunction, hashDigest }) {
  126. this._hashFunction = hashFunction;
  127. this._hashDigest = hashDigest;
  128. }
  129. /**
  130. * Apply the plugin
  131. * @param {Compiler} compiler the compiler instance
  132. * @returns {void}
  133. */
  134. apply(compiler) {
  135. compiler.hooks.compilation.tap(PLUGIN_NAME, compilation => {
  136. const cacheAnalyse = compilation.getCache(
  137. "RealContentHashPlugin|analyse"
  138. );
  139. const cacheGenerate = compilation.getCache(
  140. "RealContentHashPlugin|generate"
  141. );
  142. const hooks = RealContentHashPlugin.getCompilationHooks(compilation);
  143. compilation.hooks.processAssets.tapPromise(
  144. {
  145. name: PLUGIN_NAME,
  146. stage: Compilation.PROCESS_ASSETS_STAGE_OPTIMIZE_HASH
  147. },
  148. async () => {
  149. const assets = compilation.getAssets();
  150. /** @type {AssetInfoForRealContentHash[]} */
  151. const assetsWithInfo = [];
  152. /** @type {Map<string, [AssetInfoForRealContentHash]>} */
  153. const hashToAssets = new Map();
  154. for (const { source, info, name } of assets) {
  155. const cachedSource = toCachedSource(source);
  156. const content = /** @type {string} */ (cachedSource.source());
  157. /** @type {Hashes} */
  158. const hashes = new Set();
  159. addToList(info.contenthash, hashes);
  160. /** @type {AssetInfoForRealContentHash} */
  161. const data = {
  162. name,
  163. info,
  164. source: cachedSource,
  165. newSource: undefined,
  166. newSourceWithoutOwn: undefined,
  167. content,
  168. ownHashes: undefined,
  169. contentComputePromise: undefined,
  170. contentComputeWithoutOwnPromise: undefined,
  171. referencedHashes: undefined,
  172. hashes
  173. };
  174. assetsWithInfo.push(data);
  175. for (const hash of hashes) {
  176. const list = hashToAssets.get(hash);
  177. if (list === undefined) {
  178. hashToAssets.set(hash, [data]);
  179. } else {
  180. list.push(data);
  181. }
  182. }
  183. }
  184. if (hashToAssets.size === 0) return;
  185. const hashRegExp = new RegExp(
  186. Array.from(hashToAssets.keys(), quoteMeta).join("|"),
  187. "g"
  188. );
  189. await Promise.all(
  190. assetsWithInfo.map(async asset => {
  191. const { name, source, content, hashes } = asset;
  192. if (Buffer.isBuffer(content)) {
  193. asset.referencedHashes = EMPTY_SET;
  194. asset.ownHashes = EMPTY_SET;
  195. return;
  196. }
  197. const etag = cacheAnalyse.mergeEtags(
  198. cacheAnalyse.getLazyHashedEtag(source),
  199. Array.from(hashes).join("|")
  200. );
  201. [asset.referencedHashes, asset.ownHashes] =
  202. await cacheAnalyse.providePromise(name, etag, () => {
  203. const referencedHashes = new Set();
  204. const ownHashes = new Set();
  205. const inContent = content.match(hashRegExp);
  206. if (inContent) {
  207. for (const hash of inContent) {
  208. if (hashes.has(hash)) {
  209. ownHashes.add(hash);
  210. continue;
  211. }
  212. referencedHashes.add(hash);
  213. }
  214. }
  215. return [referencedHashes, ownHashes];
  216. });
  217. })
  218. );
  219. /**
  220. * @param {string} hash the hash
  221. * @returns {undefined | ReferencedHashes} the referenced hashes
  222. */
  223. const getDependencies = hash => {
  224. const assets = hashToAssets.get(hash);
  225. if (!assets) {
  226. const referencingAssets = assetsWithInfo.filter(asset =>
  227. /** @type {ReferencedHashes} */ (asset.referencedHashes).has(
  228. hash
  229. )
  230. );
  231. const err = new WebpackError(`RealContentHashPlugin
  232. Some kind of unexpected caching problem occurred.
  233. An asset was cached with a reference to another asset (${hash}) that's not in the compilation anymore.
  234. Either the asset was incorrectly cached, or the referenced asset should also be restored from cache.
  235. Referenced by:
  236. ${referencingAssets
  237. .map(a => {
  238. const match = new RegExp(`.{0,20}${quoteMeta(hash)}.{0,20}`).exec(
  239. a.content
  240. );
  241. return ` - ${a.name}: ...${match ? match[0] : "???"}...`;
  242. })
  243. .join("\n")}`);
  244. compilation.errors.push(err);
  245. return;
  246. }
  247. const hashes = new Set();
  248. for (const { referencedHashes, ownHashes } of assets) {
  249. if (!(/** @type {OwnHashes} */ (ownHashes).has(hash))) {
  250. for (const hash of /** @type {OwnHashes} */ (ownHashes)) {
  251. hashes.add(hash);
  252. }
  253. }
  254. for (const hash of /** @type {ReferencedHashes} */ (
  255. referencedHashes
  256. )) {
  257. hashes.add(hash);
  258. }
  259. }
  260. return hashes;
  261. };
  262. /**
  263. * @param {string} hash the hash
  264. * @returns {string} the hash info
  265. */
  266. const hashInfo = hash => {
  267. const assets = hashToAssets.get(hash);
  268. return `${hash} (${Array.from(
  269. /** @type {AssetInfoForRealContentHash[]} */ (assets),
  270. a => a.name
  271. )})`;
  272. };
  273. const hashesInOrder = new Set();
  274. for (const hash of hashToAssets.keys()) {
  275. /**
  276. * @param {string} hash the hash
  277. * @param {Set<string>} stack stack of hashes
  278. */
  279. const add = (hash, stack) => {
  280. const deps = getDependencies(hash);
  281. if (!deps) return;
  282. stack.add(hash);
  283. for (const dep of deps) {
  284. if (hashesInOrder.has(dep)) continue;
  285. if (stack.has(dep)) {
  286. throw new Error(
  287. `Circular hash dependency ${Array.from(
  288. stack,
  289. hashInfo
  290. ).join(" -> ")} -> ${hashInfo(dep)}`
  291. );
  292. }
  293. add(dep, stack);
  294. }
  295. hashesInOrder.add(hash);
  296. stack.delete(hash);
  297. };
  298. if (hashesInOrder.has(hash)) continue;
  299. add(hash, new Set());
  300. }
  301. const hashToNewHash = new Map();
  302. /**
  303. * @param {AssetInfoForRealContentHash} asset asset info
  304. * @returns {Etag} etag
  305. */
  306. const getEtag = asset =>
  307. cacheGenerate.mergeEtags(
  308. cacheGenerate.getLazyHashedEtag(asset.source),
  309. Array.from(
  310. /** @type {ReferencedHashes} */ (asset.referencedHashes),
  311. hash => hashToNewHash.get(hash)
  312. ).join("|")
  313. );
  314. /**
  315. * @param {AssetInfoForRealContentHash} asset asset info
  316. * @returns {Promise<void>}
  317. */
  318. const computeNewContent = asset => {
  319. if (asset.contentComputePromise) return asset.contentComputePromise;
  320. return (asset.contentComputePromise = (async () => {
  321. if (
  322. /** @type {OwnHashes} */ (asset.ownHashes).size > 0 ||
  323. Array.from(
  324. /** @type {ReferencedHashes} */
  325. (asset.referencedHashes)
  326. ).some(hash => hashToNewHash.get(hash) !== hash)
  327. ) {
  328. const identifier = asset.name;
  329. const etag = getEtag(asset);
  330. asset.newSource = await cacheGenerate.providePromise(
  331. identifier,
  332. etag,
  333. () => {
  334. const newContent = asset.content.replace(hashRegExp, hash =>
  335. hashToNewHash.get(hash)
  336. );
  337. return new RawSource(newContent);
  338. }
  339. );
  340. }
  341. })());
  342. };
  343. /**
  344. * @param {AssetInfoForRealContentHash} asset asset info
  345. * @returns {Promise<void>}
  346. */
  347. const computeNewContentWithoutOwn = asset => {
  348. if (asset.contentComputeWithoutOwnPromise)
  349. return asset.contentComputeWithoutOwnPromise;
  350. return (asset.contentComputeWithoutOwnPromise = (async () => {
  351. if (
  352. /** @type {OwnHashes} */ (asset.ownHashes).size > 0 ||
  353. Array.from(
  354. /** @type {ReferencedHashes} */
  355. (asset.referencedHashes)
  356. ).some(hash => hashToNewHash.get(hash) !== hash)
  357. ) {
  358. const identifier = `${asset.name}|without-own`;
  359. const etag = getEtag(asset);
  360. asset.newSourceWithoutOwn = await cacheGenerate.providePromise(
  361. identifier,
  362. etag,
  363. () => {
  364. const newContent = asset.content.replace(
  365. hashRegExp,
  366. hash => {
  367. if (
  368. /** @type {OwnHashes} */ (asset.ownHashes).has(hash)
  369. ) {
  370. return "";
  371. }
  372. return hashToNewHash.get(hash);
  373. }
  374. );
  375. return new RawSource(newContent);
  376. }
  377. );
  378. }
  379. })());
  380. };
  381. const comparator = compareSelect(a => a.name, compareStrings);
  382. for (const oldHash of hashesInOrder) {
  383. const assets =
  384. /** @type {AssetInfoForRealContentHash[]} */
  385. (hashToAssets.get(oldHash));
  386. assets.sort(comparator);
  387. await Promise.all(
  388. assets.map(asset =>
  389. /** @type {OwnHashes} */ (asset.ownHashes).has(oldHash)
  390. ? computeNewContentWithoutOwn(asset)
  391. : computeNewContent(asset)
  392. )
  393. );
  394. const assetsContent = mapAndDeduplicateBuffers(assets, asset => {
  395. if (/** @type {OwnHashes} */ (asset.ownHashes).has(oldHash)) {
  396. return asset.newSourceWithoutOwn
  397. ? asset.newSourceWithoutOwn.buffer()
  398. : asset.source.buffer();
  399. }
  400. return asset.newSource
  401. ? asset.newSource.buffer()
  402. : asset.source.buffer();
  403. });
  404. let newHash = hooks.updateHash.call(assetsContent, oldHash);
  405. if (!newHash) {
  406. const hash = createHash(this._hashFunction);
  407. if (compilation.outputOptions.hashSalt) {
  408. hash.update(compilation.outputOptions.hashSalt);
  409. }
  410. for (const content of assetsContent) {
  411. hash.update(content);
  412. }
  413. const digest = hash.digest(this._hashDigest);
  414. newHash = /** @type {string} */ (digest.slice(0, oldHash.length));
  415. }
  416. hashToNewHash.set(oldHash, newHash);
  417. }
  418. await Promise.all(
  419. assetsWithInfo.map(async asset => {
  420. await computeNewContent(asset);
  421. const newName = asset.name.replace(hashRegExp, hash =>
  422. hashToNewHash.get(hash)
  423. );
  424. const infoUpdate = {};
  425. const hash = asset.info.contenthash;
  426. infoUpdate.contenthash = Array.isArray(hash)
  427. ? hash.map(hash => hashToNewHash.get(hash))
  428. : hashToNewHash.get(hash);
  429. if (asset.newSource !== undefined) {
  430. compilation.updateAsset(
  431. asset.name,
  432. asset.newSource,
  433. infoUpdate
  434. );
  435. } else {
  436. compilation.updateAsset(asset.name, asset.source, infoUpdate);
  437. }
  438. if (asset.name !== newName) {
  439. compilation.renameAsset(asset.name, newName);
  440. }
  441. })
  442. );
  443. }
  444. );
  445. });
  446. }
  447. }
  448. module.exports = RealContentHashPlugin;