import type { ChildProcess } from 'child_process' import fs from 'fs' import globby from 'globby' import minimist from 'minimist' import { createServer } from 'net' import path from 'path' import shelljs from 'shelljs' import slash from 'slash' import { fileURLToPath } from 'url' import type { TestServerProcess } from './__helpers/shared/devServer.js' import { spawnTestServer } from './__helpers/shared/devServer.js' const __filename = fileURLToPath(import.meta.url) const dirname = path.dirname(__filename) shelljs.env.DISABLE_LOGGING = 'true' // --prod-server boots a real production server (next build / vite build) per suite // against the packed dist packages. Without it, the dev server runs against source. const prodServer = process.argv.includes('--prod-server') if (prodServer) { process.env.PAYLOAD_TEST_PROD = 'true' shelljs.env.PAYLOAD_TEST_PROD = 'true' } const turbo = process.argv.includes('--no-turbo') ? false : true process.argv = process.argv.filter((arg) => arg !== '--prod-server' && arg !== '--no-turbo') const playwrightBin = path.resolve(dirname, '../node_modules/.bin/playwright') const testRunCodes: { code: number; suiteName: string }[] = [] const { _: args, audit, bail, 'fully-parallel': fullyParallel, grep, 'grep-invert': grepInvert, headed, part, 'screen-reader': screenReader, shard, 'update-snapshots': updateSnapshots, workers, } = minimist(process.argv.slice(2), { alias: { g: 'grep' } }) const suiteName = args[0] ?? (audit ? 'a11y' : undefined) let configFile = bail ? 'playwright.bail.config.ts' : 'playwright.config.ts' let testFilePattern = '*e2e.spec.ts' if (audit && screenReader) { throw new Error('Choose either --audit or --screen-reader.') } if (audit) { if (suiteName !== 'a11y') { throw new Error('Audit validation is only available for the a11y suite.') } configFile = 'playwright.audit.config.ts' testFilePattern = 'WCAG.audit.spec.ts' } else if (screenReader) { configFile = 'playwright.screen-reader.config.ts' testFilePattern = '*screen-reader.spec.ts' } // `@visual` screenshot comparisons only run against a real production build (see // `expectScreenshot`) and are opted into explicitly, via `--grep @visual`, by the dedicated // visual-regression CI job and `pnpm test:visual`. Exclude them by default so the // plain dev-server run (`pnpm test:e2e` / `pnpm test`) doesn't hit `expectScreenshot`'s // production-build check. const effectiveGrepInvert = grepInvert ?? (grep === '@visual' ? undefined : '@visual') // Run all if (!suiteName) { let files = await globby(`${path.resolve(dirname).replace(/\\/g, '/')}/**/${testFilePattern}`) // Accessibility has its own gated CI job; exclude it from aggregate runs. if (process.env.CI === 'true' && !screenReader) { files = files.filter((file) => path.relative(dirname, file).split(path.sep)[0] !== 'a11y') } if (grep) { // Booting a server per suite is slow, so skip spec files that can't match. This only works // for greps that appear literally in the spec file, e.g. `@also-without-cache-components`. files = files.filter((file) => fs.readFileSync(file, 'utf8').includes(grep)) } const totalFiles = files.length if (part) { if (!part.includes('/')) { throw new Error('part must be in the format of "1/2"') } const [partToRun, totalParts] = part.split('/').map((n: string) => parseInt(n)) if (partToRun > totalParts) { throw new Error('part cannot be greater than totalParts') } const partSize = Math.ceil(files.length / totalParts) const start = (partToRun - 1) * partSize const end = start + partSize files = files.slice(start, end) } if (files.length !== totalFiles) { console.log(`\n\nExecuting part ${part}: ${files.length} of ${totalFiles} E2E tests...\n\n`) } else { console.log(`\n\nExecuting all ${files.length} E2E tests...\n\n`) } console.log(`${files.join('\n')}\n`) for (const file of files) { clearWebpackCache() const baseTestFolder = file?.split('/test/')?.[1]?.split('/')?.[0] if (!baseTestFolder) { throw new Error(`No base test folder found for ${file}`) } await executePlaywright({ bail, baseTestFolder, grepArg: grep, grepInvertArg: effectiveGrepInvert, headedArg: headed, suitePaths: file, updateSnapshotsArg: updateSnapshots, }) } } else { let inputSuitePath: string | undefined = suiteName let suiteConfigPath: string | undefined = 'config.ts' if (suiteName.includes('#')) { ;[inputSuitePath, suiteConfigPath] = suiteName.split('#') } if (!inputSuitePath) { throw new Error(`No test suite found for ${suiteName}`) } // Run specific suite clearWebpackCache() const suiteFolderPath: string | undefined = path .resolve(dirname, inputSuitePath) .replaceAll('__', '/') const allSuitesInFolder = await globby( `${suiteFolderPath.replace(/\\/g, '/')}/${testFilePattern}`, ) const baseTestFolder = inputSuitePath.split('__')[0] if (!baseTestFolder || !allSuitesInFolder?.length) { throw new Error(`No test suite found for ${suiteName}`) } console.log(`\n\nExecuting all ${allSuitesInFolder.length} E2E tests...\n\n`) console.log(`${allSuitesInFolder.join('\n')}\n`) // Run all spec files in the folder with a single dev server and playwright invocation // This avoids port conflicts when multiple spec files exist in the same folder await executePlaywright({ bail, baseTestFolder, fullyParallelArg: fullyParallel, grepArg: grep, grepInvertArg: effectiveGrepInvert, headedArg: headed, shardArg: shard, suiteConfigPath, suitePaths: allSuitesInFolder, updateSnapshotsArg: updateSnapshots, workersArg: workers, }) } console.log('\nRESULTS:') testRunCodes.forEach((tr) => { console.log(`\tSuite: ${tr.suiteName}, Success: ${tr.code === 0}`) }) console.log('\n') // baseTestFolder is the most top level folder of the test suite, that contains the payload config. // We need this because pnpm dev for a given test suite will always be run from the top level test folder, // not from a nested suite folder. async function executePlaywright({ bail = false, baseTestFolder, fullyParallelArg, grepArg, grepInvertArg, headedArg, shardArg, suiteConfigPath, suitePaths, updateSnapshotsArg, workersArg, }: { bail?: boolean baseTestFolder: string fullyParallelArg?: boolean grepArg?: string grepInvertArg?: string headedArg?: boolean shardArg?: string suiteConfigPath?: string suitePaths: string | string[] updateSnapshotsArg?: boolean workersArg?: number }) { const paths = Array.isArray(suitePaths) ? suitePaths : [suitePaths] console.log(`Executing ${paths.join(', ')}...`) const playwrightCfg = path.resolve(dirname, configFile) const spawnDevArgs: string[] = [ 'dev', suiteConfigPath ? `${baseTestFolder}#${suiteConfigPath}` : baseTestFolder, ] if (prodServer) { spawnDevArgs.push('--prod-server') } if (!turbo) { spawnDevArgs.push('--no-turbo') } spawnDevArgs.push('--no-seed') process.env.START_MEMORY_DB = 'true' const e2ePort = process.env.PORT ? Number(process.env.PORT) : 3000 const portInUse = await new Promise((resolve) => { const server = createServer() server.once('error', () => resolve(true)) server.once('listening', () => server.close(() => resolve(false))) server.listen(e2ePort) }) let server: TestServerProcess | undefined if (portInUse) { console.log(`Port ${e2ePort} is already in use — reusing existing dev server.`) } else { server = spawnTestServer({ args: spawnDevArgs }) } // A prod server only starts listening after the build/init completes, which outlasts Playwright's navigation timeout. // Wait for it before running tests. // (The dev server compiles routes lazily, so it needs no upfront wait.) if (prodServer && !portInUse) { await waitForServer({ child: server?.child, port: e2ePort }) } await resetServer({ child: server?.child, port: e2ePort }) const auditBailFlag = audit && bail && !process.env.CI ? ' --max-failures=1' : '' const shardFlag = shardArg ? ` --shard=${shardArg}` : '' const fullyParallelFlag = fullyParallelArg ? ' --fully-parallel' : '' const workersFlag = workersArg !== undefined ? ` --workers=${workersArg}` : '' const grepFlag = grepArg ? ` --grep="${grepArg}"` : '' const grepInvertFlag = grepInvertArg ? ` --grep-invert="${grepInvertArg}"` : '' const headedFlag = headedArg ? ' --headed' : '' const updateSnapshotsFlag = updateSnapshotsArg ? ' --update-snapshots' : '' const cmd = slash( `${playwrightBin} test ${paths.join(' ')} -c ${playwrightCfg}${shardFlag}${fullyParallelFlag}${workersFlag}${auditBailFlag}${grepFlag}${grepInvertFlag}${headedFlag}${updateSnapshotsFlag}`, ) console.log('\n', cmd) const { code, stdout } = shelljs.exec(cmd, { cwd: path.resolve(dirname, '..'), }) const suite = path.basename(path.dirname(paths[0]!)) const results = { code, suiteName: suite } if (code) { if (bail) { console.error(`TEST FAILURE DURING ${suite} suite.`) } await server?.stop() process.exit(1) } else { await server?.stop() } testRunCodes.push(results) return stdout } function clearWebpackCache() { const webpackCachePath = path.resolve(dirname, '../node_modules/.cache/webpack') shelljs.rm('-rf', webpackCachePath) } /** * Poll a port until the server responds, so Playwright doesn't start against a prod server that is still building. * Resolves on any HTTP response (the server only binds after the build/init finishes); * rejects if the server process exits or it never comes up. */ async function waitForServer({ child, port, timeoutMs = 8 * 60 * 1000, }: { /** The server process started by this run, if any. Polling stops as soon as it exits. */ child?: ChildProcess port: number timeoutMs?: number }): Promise { const url = `http://localhost:${port}/` const start = Date.now() console.log(`Waiting for prod server on ${url} …`) while (Date.now() - start < timeoutMs) { assertServerIsRunning({ child }) try { await fetch(url) console.log(`Prod server ready after ${Math.round((Date.now() - start) / 1000)}s`) return } catch { await new Promise((resolve) => setTimeout(resolve, 1000)) } } throw new Error(`Prod server did not start within ${timeoutMs / 1000}s`) } async function resetServer({ child, port, timeoutMs = 8 * 60 * 1000, }: { /** The server process started by this run, if any. Polling stops as soon as it exits. */ child?: ChildProcess port: number timeoutMs?: number }): Promise { const url = `http://localhost:${port}/api/re-initialize` const start = Date.now() let lastConnectionError: unknown console.log(`Waiting to reset test data at ${url} …`) while (Date.now() - start < timeoutMs) { assertServerIsRunning({ child }) let response: Response try { response = await fetch(url, { method: 'POST' }) } catch (error) { lastConnectionError = error await new Promise((resolve) => setTimeout(resolve, 1000)) continue } if (response.ok || response.status === 404) { return } throw new Error(`Failed to reset test data: ${response.status} ${await response.text()}`) } const connectionError = lastConnectionError instanceof Error ? `: ${lastConnectionError.message}` : '' throw new Error(`Timed out waiting to reset test data at ${url}${connectionError}`) } /** * Without this check, a server that crashed on startup (e.g. a failed `next build`) * would leave the caller polling a dead port until its timeout. */ function assertServerIsRunning({ child }: { child?: ChildProcess }): void { if (child && (child.exitCode !== null || child.signalCode !== null)) { throw new Error( `Test server exited with ${child.signalCode ?? `code ${child.exitCode}`} before it was ready`, ) } }