// @ts-check import { writeFileSync, copyFileSync, readFileSync, existsSync, globSync, } from "fs"; import { readFile, rm, writeFile } from "fs/promises"; import { join, basename, resolve, dirname } from "path"; import { exec, execSync } from "child_process"; import { createRequire } from "module"; import { finished } from "stream/promises"; import gulp from "gulp"; import open from "open"; import karma from "karma"; import yargs from "yargs"; import { build as esbuild } from "esbuild"; import { createInstrumenter } from "istanbul-lib-instrument"; import { buildCore } from "./packages/core/scripts/build.js"; import { buildGltf } from "./packages/gltf/scripts/build.js"; import { buildEngine } from "./packages/engine/scripts/build.js"; import { buildWidgets } from "./packages/widgets/scripts/build.js"; import { bundleWorkers, createCoverageFilter, defaultESBuildOptions, getWorkspaces, glslToJavaScript, shaderFiles, } from "./scripts/build-utilities.js"; import { buildCesium, bundleTestWorkers, createCombinedSpecList, } from "./scripts/build.js"; import { readNeedsShimList } from "./scripts/readNeedsShimList.js"; /** @import { BuildContext, Plugin } from "esbuild"; */ /** @import { FilePattern } from "karma"; */ /** @import { Workspace } from "./scripts/build-utilities.js"; */ // Determines the scope of the workspace packages. If the scope is set to cesium, the workspaces should be @cesium/engine. // This should match the scope of the dependencies of the root level package.json. const scope = "cesium"; const require = createRequire(import.meta.url); const packageJson = require("./package.json"); let version = packageJson.version; if (/\.0$/.test(version)) { version = version.substring(0, version.length - 2); } const karmaConfigFile = resolve("./Specs/karma.conf.cjs"); /** * Strips the "@cesium/" scope and/or "packages/" prefix from a user-supplied --workspace CLI argument. * @param {string} workspace e.g. "@cesium/engine", "packages/engine", or "engine". * @returns {string} The bare workspace directory name, e.g. "engine". */ function normalizeWorkspaceArg(workspace) { return workspace.replace(`@${scope}/`, "").replace(`packages/`, ""); } /** * Loads the karma runtime-asset file patterns a workspace declares it needs at * test time (e.g. widgets needs engine's Workers/Assets/ThirdParty/CSS). * Workspaces that don't need any simply don't export this. * @param {string} workspace The workspace directory name, e.g. "engine" or "widgets". * @returns {Promise} */ async function getRuntimeTestAssetFiles(workspace) { const { runtimeTestAssetFiles } = await import( `./packages/${workspace}/scripts/build.js` ); return runtimeTestAssetFiles ?? []; } const devDeployUrl = process.env.DEPLOYED_URL; // Command line arguments read by the gulp tasks in this file. const argv = yargs(process.argv) .options({ all: { type: "boolean" }, browsers: { type: "string", description: "Comma-separated browser names" }, buildVersion: { type: "string" }, debug: { type: "boolean" }, debugCanvasHeight: { type: "number" }, debugCanvasWidth: { type: "number" }, exclude: { type: "string" }, failTaskOnError: { type: "boolean" }, grep: { type: "string" }, include: { type: "string" }, includeName: { type: "string" }, message: { type: "string" }, minify: { type: "boolean" }, node: { type: "boolean" }, pragmas: { type: "boolean" }, private: { type: "boolean" }, production: { type: "boolean" }, release: { type: "boolean" }, removePragmas: { type: "boolean" }, sourcemap: { type: "boolean" }, status: { type: "string" }, suppressPassed: { type: "boolean" }, verbose: { type: "boolean" }, webglStub: { type: "boolean" }, webglValidation: { type: "boolean" }, workspace: { type: "string" }, }) .parseSync(); const verbose = argv.verbose; const sourceFiles = [ "packages/engine/Source/**/*.js", "!packages/engine/Source/*.js", "packages/widgets/Source/**/*.js", "!packages/widgets/Source/*.js", "packages/core/Source/*.js", "packages/gltf/Source/*.js", "!packages/engine/Source/Shaders/**", "!packages/engine/Source/ThirdParty/Workers/**", "!packages/engine/Source/ThirdParty/google-earth-dbroot-parser.js", "!packages/engine/Source/ThirdParty/_*", ]; const watchedSpecFiles = [ "packages/engine/Specs/**/*Spec.js", "!packages/engine/Specs/SpecList.js", "packages/widgets/Specs/**/*Spec.js", "!packages/widgets/Specs/SpecList.js", "packages/core/Specs/*Spec.js", "packages/gltf/Specs/*Spec.js", "Specs/*.js", "!Specs/SpecList.js", "Specs/TestWorkers/*.js", ]; /** * TypeScript projects (directories containing tsconfig.json) that are NOT * already listed as workspaces. Included when running `npm run tsc`. */ const nonWorkspaceTsProjects = ["packages/sandcastle/gallery"]; export async function build() { // Configure build options from command line arguments. const minify = argv.minify ?? false; const removePragmas = argv.removePragmas ?? false; const sourcemap = argv.sourcemap ?? true; const node = argv.node ?? true; const buildOptions = { iife: true, minify: minify, removePragmas: removePragmas, sourcemap: sourcemap, node: node, }; const packageBuildOptions = { minify: minify }; // Configure build target. const workspace = argv.workspace ? argv.workspace : undefined; if (workspace === `@${scope}/core`) { return buildCore(packageBuildOptions); } else if (workspace === `@${scope}/gltf`) { return buildGltf(packageBuildOptions); } else if (workspace === `@${scope}/engine`) { return buildEngine(packageBuildOptions); } else if (workspace === `@${scope}/widgets`) { return buildWidgets(packageBuildOptions); } await buildCore(packageBuildOptions); await buildGltf(packageBuildOptions); await buildEngine(packageBuildOptions); await buildWidgets(packageBuildOptions); await buildCesium(buildOptions); } export default build; export async function buildWatch() { await build(); const minify = argv.minify ? argv.minify : false; const removePragmas = argv.pragmas ? argv.pragmas : false; const sourcemap = argv.sourcemap ? argv.sourcemap : true; const outputDirectory = join("Build", `Cesium${!minify ? "Unminified" : ""}`); const bundles = await buildCesium({ minify: minify, outputDirectory: outputDirectory, removePragmas: removePragmas, sourcemap: sourcemap, incremental: true, }); // Incremental builds return contexts that can be rebuilt and disposed. const esm = /** @type {BuildContext} */ (bundles.esm); const cjs = /** @type {BuildContext|undefined} */ (bundles.node); const iife = /** @type {BuildContext|undefined} */ (bundles.iife); const specs = /** @type {BuildContext} */ (bundles.specs); gulp.watch(shaderFiles, async () => { glslToJavaScript(minify, "Build/minifyShaders.state", "engine"); await esm.rebuild(); if (iife) { await iife.rebuild(); } if (cjs) { await cjs.rebuild(); } }); gulp.watch( [ ...sourceFiles, // Shader results are generated in the previous watch task; no need to rebuild twice "!Source/Shaders/**", ], async () => { await esm.rebuild(); if (iife) { await iife.rebuild(); } if (cjs) { await cjs.rebuild(); } await bundleWorkers({ minify: minify, path: outputDirectory, removePragmas: removePragmas, sourcemap: sourcemap, }); }, ); gulp.watch( watchedSpecFiles, { events: ["add", "unlink"], }, async () => { createCombinedSpecList(); await specs.rebuild(); }, ); gulp.watch( watchedSpecFiles, { events: ["change"], }, async () => { await specs.rebuild(); }, ); process.on("SIGINT", () => { // Free up resources esm.dispose(); if (iife) { iife.dispose(); } if (cjs) { cjs.dispose(); } specs.dispose(); process.exit(0); }); } export async function buildTs() { /** @type {string[]} */ let workspaces; if (Array.isArray(argv.workspace)) { workspaces = argv.workspace; } else if (argv.workspace) { workspaces = [argv.workspace]; } else { workspaces = getWorkspaces(true); } // Earlier workspaces that aren't regenerated in this run are imported using their existing declarations. /** @type {Object>} */ const dependencyExports = {}; const requested = workspaces.map(normalizeWorkspaceArg); const allWorkspaces = getWorkspaces(true); const lastIndex = Math.max( 0, ...requested.map((workspace) => allWorkspaces.indexOf(workspace)), ); for (const dependency of allWorkspaces.slice(0, lastIndex)) { if (!requested.includes(dependency)) { dependencyExports[dependency] = readWorkspaceExports(dependency); } } // Generate types for passed packages in order. for (const workspace of workspaces) { const directory = normalizeWorkspaceArg(workspace); const workspaceExports = await generateWorkspaceTypeScriptDefinitions( directory, `packages/${directory}/index.d.ts`, `packages/${directory}/tsd-conf.json`, // The engine package needs its own Viewer circular-dependency workaround. directory === "engine" ? processEngineSource : undefined, // Add Math, which parseWorkspaceExports misses in core. directory === "core" ? addMathExport : undefined, dependencyExports, ); dependencyExports[directory] = workspaceExports; } if (argv.workspace) { return; } // Generate types for CesiumJS. await generateCesiumTypeScriptDefinitions(); } export async function tsc() { let projects; if (argv.workspace && !Array.isArray(argv.workspace)) { projects = [argv.workspace]; } else if (argv.workspace) { projects = argv.workspace; } else { execSync( `npm exec --package=typescript --offline -- tsc --project tsconfig.json`, { stdio: "inherit" }, ); projects = getWorkspaces(true); projects.push(...nonWorkspaceTsProjects); } for (const project of projects) { const directory = normalizeWorkspaceArg(project); const tsconfigPath = `packages/${directory}/tsconfig.json`; if (existsSync(tsconfigPath)) { execSync( `npm exec --package=typescript --offline -- tsc --project ${tsconfigPath}`, { stdio: "inherit" }, ); } } } const filesToClean = [ "Source/Cesium.js", "Source/Shaders/**/*.js", "Source/ThirdParty/Shaders/*.js", "Source/**/*.d.ts", "Specs/SpecList.js", "Specs/jasmine/**", "Cesium-*.zip", "cesium-*.tgz", "packages/**/*.tgz", ]; export async function clean() { await rm("Build", { recursive: true, force: true }); const files = globSync(filesToClean); return Promise.all( files.map((file) => rm(file, { recursive: true, force: true })), ); } async function clocSource() { let cmdLine; //Run cloc on primary Source files only const source = new Promise(function (resolve, reject) { cmdLine = "npx cloc" + " --quiet --progress-rate=0" + " packages/engine/Source/ packages/widgets/Source packages/core/Source packages/gltf/Source --exclude-dir=Assets,ThirdParty,Workers"; exec(cmdLine, function (error, stdout, stderr) { if (error) { console.log(stderr); return reject(error); } console.log("Source:"); console.log(stdout); resolve(undefined); }); }); //If running cloc on source succeeded, also run it on the tests. await source; return new Promise(function (resolve, reject) { cmdLine = "npx cloc" + " --quiet --progress-rate=0" + " Specs/ packages/engine/Specs packages/widgets/Specs packages/core/Specs packages/gltf/Specs --exclude-dir=Data --not-match-f=SpecList.js --not-match-f=eslint.config.js"; exec(cmdLine, function (error, stdout, stderr) { if (error) { console.log(stderr); return reject(error); } console.log("Specs:"); console.log(stdout); resolve(undefined); }); }); } export async function prepare() { // Copy Draco3D files from node_modules into Source copyFileSync( "node_modules/draco3d/draco_decoder.wasm", "packages/engine/Source/ThirdParty/draco_decoder.wasm", ); // Copy Gaussian Splatting utilities into Source copyFileSync( "node_modules/@cesium/wasm-splats/wasm_splats_bg.wasm", "packages/engine/Source/ThirdParty/wasm_splats_bg.wasm", ); // Copy zip.js worker and wasm files to Source/ThirdParty copyFileSync( "node_modules/@zip.js/zip.js/dist/zip-web-worker.js", "packages/engine/Source/ThirdParty/Workers/zip-web-worker.js", ); copyFileSync( "node_modules/@zip.js/zip.js/dist/zip-module.wasm", "packages/engine/Source/ThirdParty/zip-module.wasm", ); // Copy prism.js and prism.css files into Tools copyFileSync( "node_modules/prismjs/prism.js", "Tools/jsdoc/cesium_template/static/javascript/prism.js", ); copyFileSync( "node_modules/prismjs/themes/prism.min.css", "Tools/jsdoc/cesium_template/static/styles/prism.css", ); // Copy jasmine runner files into Specs const files = globSync(["node_modules/jasmine-core/lib/jasmine-core/**"], { exclude: ["node_modules/jasmine-core/lib/jasmine-core/example/**"], }); const stream = gulp.src(files).pipe(gulp.dest("Specs/jasmine")); await finished(stream); return stream; } export async function cloc() { await clean(); await clocSource(); } //Builds the documentation export async function buildDocs() { const generatePrivateDocumentation = argv.private ? "--private" : ""; const env = Object.assign({}, process.env, { CESIUM_VERSION: version, CESIUM_PACKAGES: getWorkspaces(true), }); // --private already makes jsdoc reveal @private symbols; skip hiding // @internal ones too in that case, so --private shows everything. if (!argv.private) { env.CESIUM_HIDE_INTERNAL = "true"; } execSync( `npx jsdoc --configure Tools/jsdoc/conf.json --pedantic ${generatePrivateDocumentation}`, { stdio: "inherit", env, }, ); const stream = gulp .src(["Documentation/Images/**"], { encoding: false }) .pipe(gulp.dest("Build/Documentation/Images")); await finished(stream); return stream; } export async function buildDocsWatch() { await buildDocs(); console.log("Listening for changes in documentation..."); return gulp.watch(sourceFiles, buildDocs); } export async function websiteRelease() { await buildCore({}); await buildGltf({}); await buildEngine({}); await buildWidgets({}); // Generate Build/CesiumUnminified await buildCesium({ minify: false, removePragmas: false, node: false, }); // Generate Build/Cesium await buildCesium({ minify: true, removePragmas: true, node: false, }); // Generate Build/Sandcastle/CesiumUnminified await buildCesium({ minify: false, removePragmas: false, node: false, outputDirectory: join("Build", "Sandcastle", "CesiumUnminified"), }); await buildDocs(); } export async function buildRelease() { await buildCore({}); await buildGltf({}); await buildEngine({}); await buildWidgets({}); // Generate Build/CesiumUnminified await buildCesium({ minify: false, removePragmas: false, node: true, sourcemap: false, }); // Generate Build/Cesium await buildCesium({ minify: true, removePragmas: true, node: true, sourcemap: false, }); } export async function release() { await buildRelease(); await Promise.all([buildTs(), buildDocs()]); } export const postversion = async function () { const workspace = argv.workspace; if (!workspace) { return; } const directory = normalizeWorkspaceArg(workspace); const workspacePackageJson = require(`./packages/${directory}/package.json`); const version = workspacePackageJson.version; // Iterate through all package JSONs that may depend on the updated package and // update the version of the updated workspace. const packageJsons = globSync([ "./package.json", "./packages/*/package.json", ]); const promises = packageJsons.map(async (packageJsonPath) => { const fullPackageJsonPath = resolve(packageJsonPath); // Ensure that we don't check the updated workspace itself. if (basename(dirname(packageJsonPath)) === directory) { return; } // Ensure that we only update workspaces where the dependency to the updated workspace already exists. const packageJson = JSON.parse(readFileSync(fullPackageJsonPath, "utf-8")); if (!Object.hasOwn(packageJson.dependencies, workspace)) { console.log( `Skipping update for ${workspace} as it is not a dependency.`, ); return; } // Update the version for the updated workspace. packageJson.dependencies[workspace] = `^${version}`; await writeFile( fullPackageJsonPath, `${JSON.stringify(packageJson, undefined, 2)}\n`, ); }); return Promise.all(promises); }; export async function deploySetVersion() { const buildVersion = argv.buildVersion; if (buildVersion) { // NPM versions can only contain alphanumeric and hyphen characters packageJson.version += `-${buildVersion.replace(/[^[0-9A-Za-z-]/g, "")}`; return writeFile( "package.json", `${JSON.stringify(packageJson, undefined, 2)}\n`, ); } } export async function deployStatus() { const status = argv.status; const message = argv.message; const deployUrl = `${devDeployUrl}`; const zipUrl = `${deployUrl}Cesium-${version}.zip`; const npmUrl = `${deployUrl}cesium-${version}.tgz`; const coverageUrl = `${deployUrl}Build/Coverage/index.html`; return Promise.all([ setStatus(status, deployUrl, message, "deploy / artifact: deployment"), setStatus(status, zipUrl, message, "deploy / artifact: zip file"), setStatus(status, npmUrl, message, "deploy / artifact: npm package"), setStatus( status, coverageUrl, message, "deploy / artifact: coverage results", ), ]); } /** * Sets a commit status on GitHub for the current deployment. * * @param {string|undefined} state The status state, e.g. "success". * @param {string} targetUrl The URL linked from the status. * @param {string|undefined} description A short description of the status. * @param {string} context The label that identifies the status. * @returns {Promise} The GitHub API response, or undefined if no GITHUB_TOKEN is set. */ async function setStatus(state, targetUrl, description, context) { // skip if the environment does not have the token if (!process.env.GITHUB_TOKEN) { return; } const body = { state: state, target_url: targetUrl, description: description, context: context, }; const response = await fetch( `https://api.github.com/repos/${process.env.GITHUB_REPO}/statuses/${process.env.GITHUB_SHA}`, { method: "POST", body: JSON.stringify(body), headers: { "Content-Type": "application/json", Authorization: `token ${process.env.GITHUB_TOKEN}`, "User-Agent": "Cesium", }, }, ); const result = await response.json(); return result; } /** * Generates coverage report. * * @param {object} options An object with the following properties: * @param {string} options.outputDirectory The output directory for the generated build artifacts. * @param {string} options.coverageDirectory The path where the coverage reports should be saved to. * @param {string} options.specList The path to the spec list for the package. * @param {RegExp} options.filter The filter for finding which files should be instrumented. * @param {boolean} [options.webglStub=false] True if WebGL stub should be used when running tests. * @param {boolean} [options.suppressPassed=false] True if output should be suppressed for tests that pass. * @param {boolean} [options.failTaskOnError=false] True if the gulp task should fail on errors in the tests. * @param {string} [options.workspace] The name of the workspace, if any. */ export async function runCoverage(options) { const webglStub = options.webglStub ?? false; const suppressPassed = options.suppressPassed ?? false; const failTaskOnError = options.failTaskOnError ?? false; const workspace = options.workspace; /** @type {string[]} */ const folders = []; let browsers = ["Chrome"]; if (argv.browsers) { browsers = argv.browsers.split(","); } const instrumenter = createInstrumenter({ esModules: true, }); // Setup plugin to use instrumenter on source files. /** @type {Plugin} */ const instrumentPlugin = { name: "instrument", setup: (build) => { build.onLoad( { filter: options.filter, }, async (args) => { const source = await readFile(args.path, { encoding: "utf8" }); try { const generatedCode = instrumenter.instrumentSync( source, args.path, ); return { contents: generatedCode }; } catch (e) { return { errors: [ { text: /** @type {Error} */ (e).message, }, ], }; } }, ); }, }; const karmaBundle = join(options.outputDirectory, "karma-main.js"); await esbuild({ ...defaultESBuildOptions(), entryPoints: ["Specs/karma-main.js"], sourcemap: true, format: "esm", outfile: karmaBundle, logLevel: "error", // print errors immediately, and collect warnings so we can filter out known ones }); // Generate instrumented bundle for Specs. const specListBundle = join(options.outputDirectory, "SpecList.js"); await esbuild({ ...defaultESBuildOptions(), entryPoints: [options.specList], sourcemap: true, format: "esm", outfile: specListBundle, plugins: [instrumentPlugin], logLevel: "error", // print errors immediately, and collect warnings so we can filter out known ones }); /** @type {FilePattern[]} */ let files = [ { pattern: karmaBundle, included: true, type: "module", }, { pattern: specListBundle, included: true, type: "module", }, // Static assets are always served from the shared/combined folders. { pattern: "Specs/Data/**", included: false }, { pattern: "Specs/TestWorkers/**/*.wasm", included: false }, { pattern: "Build/CesiumUnminified/**", included: false }, { pattern: "Build/Specs/TestWorkers/**.js", included: false }, ]; if (workspace) { // Include the workspace's declared runtime assets, since some packages // (e.g. widgets) depend on another package's assets at runtime. const assetFiles = await getRuntimeTestAssetFiles(workspace); files = [ { pattern: karmaBundle, included: true, type: "module", }, { pattern: specListBundle, included: true, type: "module", }, { pattern: "Specs/Data/**", included: false }, { pattern: "Specs/TestWorkers/**/*.wasm", included: false }, ...assetFiles, { pattern: "Build/Specs/TestWorkers/**.js", included: false }, ]; } // Setup Karma config. // @ts-expect-error karma's types declare client.args as string[], but karma passes any values through to the specs. const config = await karma.config.parseConfig( karmaConfigFile, { configFile: karmaConfigFile, browsers: browsers, specReporter: { suppressErrorSummary: false, suppressFailed: false, suppressPassed: suppressPassed, suppressSkipped: true, }, files: files, reporters: ["spec", "coverage"], coverageReporter: { dir: options.coverageDirectory, subdir: function (/** @type {string} */ browserName) { folders.push(browserName); return browserName; }, includeAllSources: true, }, client: { captureConsole: false, args: [ undefined, undefined, undefined, webglStub, undefined, undefined, undefined, ], }, }, { promiseConfig: true, throwErrors: true }, ); return new Promise((resolve, reject) => { const server = new karma.Server(config, function doneCallback(e) { let html = "
    "; folders.forEach(function (folder) { html += `
  • ${folder}
  • `; }); html += "
"; writeFileSync(join(options.coverageDirectory, "index.html"), html); if (!process.env.CI) { folders.forEach(function (dir) { open(join(options.coverageDirectory, `${dir}/index.html`)); }); } if (failTaskOnError && e) { reject(e); return; } resolve(undefined); }); server.start(); }); } export async function coverage() { let workspace = argv.workspace; if (workspace) { workspace = normalizeWorkspaceArg(workspace); } if (workspace) { return runCoverage({ outputDirectory: `packages/${workspace}/Build/Instrumented`, coverageDirectory: `packages/${workspace}/Build/Coverage`, specList: `packages/${workspace}/Specs/SpecList.js`, filter: createCoverageFilter(/** @type {Workspace} */ (workspace)), webglStub: argv.webglStub, suppressPassed: argv.suppressPassed, failTaskOnError: argv.failTaskOnError, workspace: workspace, }); } const directories = getWorkspaces(true); return runCoverage({ outputDirectory: "Build/Instrumented", coverageDirectory: "Build/Coverage", specList: "Specs/SpecList.js", filter: new RegExp( String.raw`packages(\\|\/)(${directories.join("|")})(\\|\/)Source((\\|\/)\w+)+\.js$`, ), webglStub: argv.webglStub, suppressPassed: argv.suppressPassed, failTaskOnError: argv.failTaskOnError, }); } // Cache contexts for successive calls to test export async function test() { const enableAllBrowsers = argv.all ? true : false; const includeCategory = argv.include ? argv.include : ""; const excludeCategory = argv.exclude ? argv.exclude : ""; const webglValidation = argv.webglValidation ? argv.webglValidation : false; const webglStub = argv.webglStub ? argv.webglStub : false; const release = argv.release ? argv.release : false; const failTaskOnError = argv.failTaskOnError ? argv.failTaskOnError : false; const suppressPassed = argv.suppressPassed ? argv.suppressPassed : false; const debug = argv.debug ? true : false; const debugCanvasWidth = argv.debugCanvasWidth; const debugCanvasHeight = argv.debugCanvasHeight; const isProduction = argv.production; let grep = argv.includeName ?? argv.grep ?? ""; grep = grep.replaceAll(/(Spec)?(\.js)?$/g, ""); let workspace = argv.workspace; if (workspace) { workspace = normalizeWorkspaceArg(workspace); } // --release always tests the combined build; --workspace is not supported alongside it. if (!isProduction && !release) { console.log("Building specs..."); if (workspace === "core") { await buildCore({}); } else if (workspace === "gltf") { await buildGltf({}); } else if (workspace === "engine") { await buildEngine({}); // Engine's TaskProcessor specs need these workers. TODO: why not do this inside buildEngine? await bundleTestWorkers(); } else if (workspace === "widgets") { await buildWidgets({}); } else { await buildCesium({ iife: true }); } } let browsers = debug ? ["ChromeDebugging"] : ["Chrome"]; if (argv.browsers) { browsers = argv.browsers.split(","); } /** @type {FilePattern[]} */ let files = [ { pattern: "Specs/Data/**", included: false }, { pattern: "Specs/TestWorkers/**/*.wasm", included: false }, { pattern: "Build/CesiumUnminified/Cesium.js", included: true }, { pattern: "Build/CesiumUnminified/Cesium.js.map", included: false }, { pattern: "Build/CesiumUnminified/**", included: false }, { pattern: "Build/Specs/karma-main.js", included: true, type: "module" }, { pattern: "Build/Specs/SpecList.js", included: true, type: "module" }, { pattern: "Build/Specs/TestWorkers/**.js", included: false }, ]; if (workspace) { // Include the workspace's declared runtime assets, since some packages // (e.g. widgets) depend on another package's assets at runtime. const assetFiles = await getRuntimeTestAssetFiles(workspace); files = [ { pattern: `packages/${workspace}/Build/Specs/karma-main.js`, included: true, type: "module", }, { pattern: `packages/${workspace}/Build/Specs/SpecList.js`, included: true, type: "module", }, { pattern: "Specs/Data/**", included: false }, { pattern: "Specs/TestWorkers/**/*.wasm", included: false }, ...assetFiles, { pattern: "Build/Specs/TestWorkers/**.js", included: false }, ]; } if (release) { files = [ { pattern: "Specs/Data/**", included: false }, { pattern: "Specs/TestWorkers/**/*.wasm", included: false }, { pattern: "Specs/ThirdParty/**", included: false, type: "module" }, { pattern: "Build/Cesium/Cesium.js", included: true }, { pattern: "Build/Cesium/Cesium.js.map", included: false }, { pattern: "Build/Cesium/**", included: false }, { pattern: "Build/Specs/karma-main.js", included: true }, { pattern: "Build/Specs/SpecList.js", included: true, type: "module" }, { pattern: "Build/Specs/TestWorkers/**.js", included: false }, ]; } // @ts-expect-error karma's types declare client.args as string[], but karma passes any values through to the specs. const config = await karma.config.parseConfig( karmaConfigFile, { port: 9876, singleRun: !debug, browsers: browsers, specReporter: { suppressErrorSummary: false, suppressFailed: false, suppressPassed: suppressPassed, suppressSkipped: true, }, detectBrowsers: { enabled: enableAllBrowsers, }, logLevel: verbose ? karma.constants.LOG_INFO : karma.constants.LOG_ERROR, files: files, client: { captureConsole: verbose, args: [ includeCategory, excludeCategory, webglValidation, webglStub, release, debugCanvasWidth, debugCanvasHeight, "--grep", grep, ], }, }, { promiseConfig: true, throwErrors: true }, ); return new Promise((resolve, reject) => { const server = new karma.Server(config, function doneCallback(exitCode) { if (failTaskOnError && exitCode) { reject(exitCode); return; } resolve(undefined); }); server.start(); }); } /** * Fix the typescript definitions output to match what we need. * * - Change declare => export since we are wrapping everything in a namespace * - Change CesiumMath => Math to help avoid conflicts with the native Math class * - Fix up the WebGLConstants aliasing by simply unquoting the strings. * @param {string} source * @param {boolean} includeKnockout Whether to append the knockout type definition * @returns The modified source */ function fixTypeScriptDefinitionsSource(source, includeKnockout) { let fixedSource = source .replace(/^declare /gm, "export ") .replace(/module "Math"/gm, "namespace Math") .replace(/CesiumMath/gm, "Math") .replace( /= "WebGLConstants\.(.+)"/gm, // eslint-disable-next-line no-unused-vars (match, p1) => `= WebGLConstants.${p1}`, ) // Strip const enums which can cause errors - https://www.typescriptlang.org/docs/handbook/enums.html#const-enum-pitfalls .replace(/^(\s*)(export )?const enum (\S+) {(\s*)$/gm, "$1$2enum $3 {$4") // Quote enum keys that start with a digit, which tsd-jsdoc writes unquoted .replace(/^(\s+)(\d\w*) = /gm, '$1"$2" = ') // Replace JSDoc generation version of defined with an improved version using TS type predicates .replace( /\n?export function defined\(value: any\): boolean;/gm, `\n${readFileSync("./packages/core/Source/defined.d.ts") .toString() .replace(/\n*\/\*.*?\*\/\n*/gms, "") .replace("export default", "export")}`, ) // Replace JSDoc generation version of Check with one that asserts the type of variables after called .replace( /\/\*\*[\*\s\w]*?\*\/\nexport const Check: any;/m, `\n${readFileSync("./packages/core/Source/Check.d.ts") .toString() .replace(/export default.*\n?/, "") .replace("const Check", "export const Check")}`, ) // Fix https://github.com/CesiumGS/cesium/issues/10498 so we can use the rest parameter expand tuple .replace( "raiseEvent(...arguments: Parameters[]): void;", "raiseEvent(...arguments: Parameters): void;", ); if (includeKnockout) { fixedSource += readFileSync("./packages/widgets/Source/knockout.d.ts") .toString() .replace(/^\/\/.*\n/gm, "") .replace(/export default knockout;\n?/, ""); } return fixedSource; } /** * Generates the TypeScript definitions file (.d.ts) for a workspace. * * @param {string} workspaceName The workspace directory name, e.g. "core". * @param {string} definitionsPath The path of the .d.ts file to generate. * @param {string} configurationPath The path of the JSDoc configuration file that runs tsd-jsdoc for this workspace. * @param {function(string): string} [processSourceFunc] Transforms the generated source before its exports are collected. Receives the source and returns the new source. * @param {function(Set): Set} [processExportsFunc] Adjusts the collected export names, and returns the adjusted set. * @param {Object>} [dependencyExports] The export names of earlier workspaces, keyed by workspace directory name, used to generate this workspace's imports from them. * @returns {Promise>} The names this workspace exports. */ function generateWorkspaceTypeScriptDefinitions( workspaceName, definitionsPath, configurationPath, processSourceFunc, processExportsFunc, dependencyExports, ) { // Run JSDoc with tsd-jsdoc to generate an initial definition file. execSync(`npx jsdoc --configure ${configurationPath}`, { stdio: `inherit`, }); let source = readFileSync(definitionsPath).toString(); if (processSourceFunc) { source = processSourceFunc(source); } // Collect the names this workspace exports, so that later workspaces can import them. let workspaceExports = parseWorkspaceExports(source); if (processExportsFunc) { workspaceExports = processExportsFunc(workspaceExports); } source = fixTypeScriptDefinitionsSource(source, workspaceName === "widgets"); if (dependencyExports) { let imports = ""; Object.keys(dependencyExports).forEach((workspace) => { const usedExports = Array.from(dependencyExports[workspace]).filter( (exportName) => source.indexOf(exportName) !== -1, ); if (usedExports.length === 0) { return; } imports += `import { ${usedExports.join( ",\n", )} } from "@${scope}/${workspace}";\n`; }); source = imports + source; } // Wrap the source to actually be inside of a declared cesium module // and add any workaround and private utility types. source = `declare module "@${scope}/${workspaceName}" { ${source} } `; // Write the final source file back out writeFileSync(definitionsPath, source); // Use tsc to compile it and make sure it is valid execSync(`npx tsc -p packages/${workspaceName}/tsconfig.types.json`, { stdio: "inherit", }); return Promise.resolve(workspaceExports); } /** * Parses the names exported by a workspace's TypeScript definitions source. * * @param {string} source The definitions source. * @returns {Set} The exported names. */ function parseWorkspaceExports(source) { const regex = /^(?:declare|export) (?:const )?(function|class|namespace|enum|interface|type) (.+)/gm; const exportNames = new Set(); let match; while ((match = regex.exec(source))) { const nameMatch = match[2].match(/([^<\s|\(]+)/); if (nameMatch) { exportNames.add(nameMatch[1]); } } return exportNames; } /** * Reads the names exported by a workspace's previously generated definitions file. * * @param {string} workspaceName The workspace directory name, e.g. "core". * @returns {Set} The exported names. */ function readWorkspaceExports(workspaceName) { const definitionsPath = `packages/${workspaceName}/index.d.ts`; if (!existsSync(definitionsPath)) { throw new Error( `${definitionsPath} does not exist, but is needed to import @${scope}/${workspaceName} types. Generate it first with: npm run build-ts --workspace @${scope}/${workspaceName}`, ); } return parseWorkspaceExports(readFileSync(definitionsPath).toString()); } /** * @param {Set} exportNames The names exported by the core workspace. * @returns {Set} The same set, with "Math" added. */ function addMathExport(exportNames) { // Math.js documents CesiumMath with @exports and @alias Math, so tsd-jsdoc emits it // as `declare module "Math"`, which parseWorkspaceExports doesn't match. Add the // name directly so later workspaces can import it. exportNames.add("Math"); return exportNames; } /** * @param {string} source The engine definitions source. * @returns {string} The source, with the Viewer and Transforms workarounds added. */ function processEngineSource(source) { let newSource = source; // Manually add a type definition from Viewer to avoid circular dependency // with the widgets package. This will no longer be needed past Cesium 1.100. newSource += ` /** * @property scene - The scene in the widget. */ export type Viewer = { scene: Scene; }; `; // The @borrows tags in Transforms.js can't resolve FixedFrameTransforms from another package. // Declare Transforms as the combined type. const transformsNamespace = /(\/\*\*(?:[^*]|\*(?!\/))*\*\/\s*)?declare namespace Transforms \{/; const transformsDoc = newSource.match(transformsNamespace)?.[1] ?? ""; newSource = newSource.replace( transformsNamespace, "declare namespace _CelestialFrameTransforms {", ); newSource += ` ${transformsDoc}declare const Transforms: typeof FixedFrameTransforms & typeof _CelestialFrameTransforms; `; // Each name gets its own @deprecated tag; a plain re-export wouldn't show it in hover. // Renaming on export avoids conflicting with a plain import of the same name elsewhere in engine's declarations. const deprecatedExports = readNeedsShimList( "scripts/core-reexport-shim-candidates.txt", ) .map((name) => { const comment = `/** * @deprecated ${name} has been moved from cesium/engine to cesium/core in CesiumJS 1.146, and will be removed from cesium/engine in 1.150. Import from cesium/core instead. */`; // MapProjection is the only moved name that's an interface with no runtime value. const declaration = name === "MapProjection" ? `type _${name}Shim = _${name};` : `declare const _${name}Shim: typeof _${name};`; return `import { ${name} as _${name} } from "@cesium/core";\n${comment}\n${declaration}\nexport { _${name}Shim as ${name} };`; }) .join("\n"); newSource = `${deprecatedExports}\n\n${newSource}`; return newSource; } function generateCesiumTypeScriptDefinitions() { // Run jsdoc with tsd-jsdoc to generate an initial Cesium.d.ts file. // @internal symbols must stay out of the combined "cesium" package's API. execSync("npx jsdoc --configure Tools/jsdoc/ts-conf.json", { stdio: "inherit", env: { ...process.env, CESIUM_HIDE_INTERNAL: "true" }, }); let source = readFileSync("Source/Cesium.d.ts").toString(); source = fixTypeScriptDefinitionsSource(source, true); // Wrap the source to actually be inside of a declared cesium module // and add any workaround and private utility types. source = `declare module "cesium" { ${source} } `; // Write the final source file back out writeFileSync("Source/Cesium.d.ts", source); // Use tsc to compile it and make sure it is valid execSync("npx tsc -p Tools/jsdoc/tsconfig.json", { stdio: "inherit", }); // Also compile our smokescreen to make sure interfaces work as expected. execSync("npx tsc -p Specs/TypeScript/tsconfig.json", { stdio: "inherit", }); return Promise.resolve(); } /** * License information for a third-party dependency, as listed in ThirdParty.json. * * @typedef {object} LicenseData * @property {string} name * @property {string[]} license * @property {string} version * @property {string} url * @property {string} [notes] */ /** * Reads `ThirdParty.extra.json` file * @param {string} path Path to `ThirdParty.extra.json` * @param {string[]} discoveredDependencies List of previously discovered modules * @returns {Promise>} A promise to the license data for each entry, or undefined for entries already discovered */ async function getLicenseDataFromThirdPartyExtra(path, discoveredDependencies) { if (!existsSync(path)) { return Promise.reject(`${path} does not exist`); } const contents = await readFile(path, "utf8"); /** @type {LicenseData[]} */ const thirdPartyExtra = JSON.parse(contents); return Promise.all( thirdPartyExtra.map(async function (module) { if (!discoveredDependencies.includes(module.name)) { let result = await getLicenseDataFromPackage( packageJson, module.name, discoveredDependencies, module.license, module.notes, ); if (result) { return result; } // Recursively check the workspaces for (const workspace of getWorkspaces(true)) { const workspacePackageJson = require( `./packages/${workspace}/package.json`, ); result = await getLicenseDataFromPackage( workspacePackageJson, module.name, discoveredDependencies, module.license, module.notes, ); if (result) { return result; } } // If this is not a npm module, return existing info discoveredDependencies.push(module.name); return module; } }), ); } /** * Extracts name, license, and url from `package.json` file. * * @param {{dependencies: Object, devDependencies?: Object}} packageJson The package.json that may list the package as a dependency. * @param {string} packageName Name of package * @param {string[]} discoveredDependencies List of previously discovered modules * @param {string[]} [licenseOverride] If specified, override info fetched from package.json. Useful in the case where there are multiple licenses and we might chose a single one. * @param {string} [notes] Notes to include in the license data. * @returns {Promise} A promise to the license data, or undefined if the package is not a dependency or was already discovered */ async function getLicenseDataFromPackage( packageJson, packageName, discoveredDependencies, licenseOverride, notes, ) { if ( !packageJson.dependencies[packageName] && (!packageJson.devDependencies || !packageJson.devDependencies[packageName]) ) { return; } if (discoveredDependencies.includes(packageName)) { return; } discoveredDependencies.push(packageName); const packagePath = join("node_modules", packageName, "package.json"); let contents; if (existsSync(packagePath)) { // Package exists at top-level, so use it. contents = await readFile(packagePath, "utf8"); } if (!contents) { return Promise.reject( new Error(`Unable to read ${packageName} license information`), ); } const packageData = JSON.parse(contents); // Check for license let licenseField = licenseOverride; if (!licenseField) { licenseField = [packageData.license]; } if (!licenseField && packageData.licenses) { licenseField = packageData.licenses; } if (!licenseField) { console.log(`No license found for ${packageName}`); licenseField = ["NONE"]; } let packageVersion = packageData.version; if (!packageData.version) { console.log(`No version information found for ${packageName}`); packageVersion = "NONE"; } return { name: packageName, license: licenseField, version: packageVersion, url: `https://www.npmjs.com/package/${packageName}`, notes: notes, }; } export async function buildThirdParty() { /** @type {LicenseData[]} */ let licenseJson = []; /** @type {string[]} */ const discoveredDependencies = []; // Generate ThirdParty.json from ThirdParty.extra.json and package.json const licenseInfo = await getLicenseDataFromThirdPartyExtra( "ThirdParty.extra.json", discoveredDependencies, ); licenseJson = licenseJson.concat(/** @type {LicenseData[]} */ (licenseInfo)); licenseJson.sort(function (a, b) { const nameA = a.name.toLowerCase(); const nameB = b.name.toLowerCase(); if (nameA < nameB) { return -1; } if (nameA > nameB) { return 1; } return 0; }); return writeFile("ThirdParty.json", JSON.stringify(licenseJson, null, 2)); }