/** * @license * Copyright 2020 Google LLC * SPDX-License-Identifier: Apache-2.0 */ import {makeComputedArtifact} from './computed-artifact.js'; import {JSBundles} from './js-bundles.js'; const RELATIVE_SIZE_THRESHOLD = 0.1; const ABSOLUTE_SIZE_THRESHOLD_BYTES = 1024 * 0.5; class ModuleDuplication { /** * @param {string} source */ static normalizeSource(source) { // Trim trailing question mark - b/c webpack. source = source.replace(/\?$/, ''); // Normalize paths for dependencies by only keeping everything after the last `node_modules`. const lastNodeModulesIndex = source.lastIndexOf('node_modules'); if (lastNodeModulesIndex !== -1) { source = source.substring(lastNodeModulesIndex); } return source; } /** * @param {string} source */ static _shouldIgnoreSource(source) { // Ignore bundle overhead. if (source.includes('webpack/bootstrap')) return true; if (source.includes('(webpack)/buildin')) return true; // Ignore webpack module shims, i.e. aliases of the form `module.exports = window.jQuery` if (source.includes('external ')) return true; return false; } /** * @param {Map>} moduleNameToSourceData */ static _normalizeAggregatedData(moduleNameToSourceData) { for (const [key, originalSourceData] of moduleNameToSourceData.entries()) { let sourceData = originalSourceData; // Sort by resource size. sourceData.sort((a, b) => b.resourceSize - a.resourceSize); // Remove modules smaller than a % size of largest. if (sourceData.length > 1) { const largestResourceSize = sourceData[0].resourceSize; sourceData = sourceData.filter(data => { const percentSize = data.resourceSize / largestResourceSize; return percentSize >= RELATIVE_SIZE_THRESHOLD; }); } // Remove modules smaller than an absolute theshold. sourceData = sourceData.filter(data => data.resourceSize >= ABSOLUTE_SIZE_THRESHOLD_BYTES); // Delete source datas with only one value (no duplicates). if (sourceData.length > 1) { moduleNameToSourceData.set(key, sourceData); } else { moduleNameToSourceData.delete(key); } } } /** * @param {Pick} artifacts * @param {LH.Artifacts.ComputedContext} context */ static async compute_(artifacts, context) { const bundles = await JSBundles.request(artifacts, context); /** * @typedef SourceData * @property {string} source * @property {number} resourceSize */ /** @type {Map} */ const sourceDatasMap = new Map(); // Determine size of each `sources` entry. for (const {rawMap, sizes} of bundles) { if ('errorMessage' in sizes) continue; /** @type {SourceData[]} */ const sourceDataArray = []; sourceDatasMap.set(rawMap, sourceDataArray); for (let i = 0; i < rawMap.sources.length; i++) { if (this._shouldIgnoreSource(rawMap.sources[i])) continue; const sourceKey = (rawMap.sourceRoot || '') + rawMap.sources[i]; const sourceSize = sizes.files[sourceKey]; sourceDataArray.push({ source: ModuleDuplication.normalizeSource(rawMap.sources[i]), resourceSize: sourceSize, }); } } /** @type {Map>} */ const moduleNameToSourceData = new Map(); for (const {rawMap, script} of bundles) { const sourceDataArray = sourceDatasMap.get(rawMap); if (!sourceDataArray) continue; for (const sourceData of sourceDataArray) { let data = moduleNameToSourceData.get(sourceData.source); if (!data) { data = []; moduleNameToSourceData.set(sourceData.source, data); } data.push({ scriptId: script.scriptId, scriptUrl: script.url, resourceSize: sourceData.resourceSize, }); } } this._normalizeAggregatedData(moduleNameToSourceData); return moduleNameToSourceData; } } const ModuleDuplicationComputed = makeComputedArtifact(ModuleDuplication, ['Scripts', 'SourceMaps']); export {ModuleDuplicationComputed as ModuleDuplication};