// Compares coverage between two runs and fails when a metric falls under
// THRESHOLD.
//
// Both sides are measured in the same CI job — the PR head and its merge-base
// with main — so there is no baseline file to keep in sync, no per-platform
// skew, and packages can be added or removed without anyone updating a
// checked-in number.
//
// Usage:
// node scripts/coverage-check.mjs --collect
[--expect pkg,pkg]
// node scripts/coverage-check.mjs --base --head [--report ]
import {
appendFileSync,
mkdirSync,
readFileSync,
readdirSync,
writeFileSync,
} from 'node:fs'
import { join } from 'node:path'
const METRICS = ['statements', 'branches', 'functions', 'lines']
// A metric fails only when it falls under this line: at or above it on the
// merge base, under it on the PR. A package that is already under it never
// fails, and any other drop is reported but does not fail.
const THRESHOLD = 60
// Marks the PR comment so the workflow can find and update it.
const REPORT_MARKER = ''
function arg(name) {
const i = process.argv.indexOf(name)
return i === -1 ? undefined : process.argv[i + 1]
}
function totals(summary) {
// A package with no source loaded reports totals of 0/0 as 100%; that is not
// a number worth comparing.
if (summary.total.statements.total === 0) return undefined
return Object.fromEntries(
METRICS.map((metric) => [metric, summary.total[metric].pct]),
)
}
function packageDirs() {
const dirs = {}
for (const entry of readdirSync('packages', { withFileTypes: true })) {
if (!entry.isDirectory()) continue
dirs[entry.name] = entry.name
try {
const pkg = JSON.parse(
readFileSync(join('packages', entry.name, 'package.json'), 'utf8'),
)
if (pkg.name) dirs[pkg.name] = entry.name
} catch {
// Directory with no package.json is still collectable by folder name.
}
}
return dirs
}
function expectedDirs() {
const raw = arg('--expect')
if (raw === undefined) return undefined
const names = raw
.split(',')
.map((name) => name.trim())
.filter(Boolean)
const dirs = packageDirs()
const resolved = []
const unknown = []
for (const name of names) {
const dir = dirs[name]
if (!dir) unknown.push(name)
else resolved.push(dir)
}
if (unknown.length > 0) {
console.error(`Unknown package(s): ${unknown.join(', ')}`)
process.exit(1)
}
return resolved
}
function readSummary(dirName) {
try {
return JSON.parse(
readFileSync(
join('packages', dirName, 'coverage', 'coverage-summary.json'),
'utf8',
),
)
} catch (error) {
if (error && error.code === 'ENOENT') return null
throw error
}
}
function collect(dir) {
mkdirSync(dir, { recursive: true })
const expected = expectedDirs()
const names =
expected ??
readdirSync('packages', { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
const missing = []
let n = 0
for (const name of names) {
const summary = readSummary(name)
if (!summary) {
if (expected) missing.push(`${name} (no coverage-summary.json)`)
continue
}
const values = totals(summary)
if (!values) {
if (expected) missing.push(`${name} (0 statements)`)
continue
}
writeFileSync(join(dir, `${name}.json`), JSON.stringify(values))
n++
}
if (missing.length > 0) {
console.error(
`Coverage collect failed for ${missing.length} package(s):\n ${missing.join('\n ')}`,
)
process.exit(1)
}
console.log(`Collected ${n} coverage summary/summaries into ${dir}`)
}
function read(dir) {
let files
try {
files = readdirSync(dir)
} catch (error) {
if (error && error.code === 'ENOENT') return {}
throw error
}
return Object.fromEntries(
files
.filter((file) => file.endsWith('.json'))
.map((file) => [
file.replace(/\.json$/, ''),
JSON.parse(readFileSync(join(dir, file), 'utf8')),
]),
)
}
const collectDir = arg('--collect')
if (collectDir) {
collect(collectDir)
process.exit(0)
}
const baseDir = arg('--base')
const headDir = arg('--head')
if (!baseDir || !headDir) {
console.error(
'Usage: coverage-check.mjs --collect [--expect pkg,pkg] | --base --head [--report ]',
)
process.exit(2)
}
const base = read(baseDir)
const head = read(headDir)
const names = Object.keys(head).sort()
if (names.length === 0) {
console.error('No packages were measured — nothing to compare.')
process.exit(1)
}
const regressions = []
const additions = []
const rows = []
for (const name of names) {
const before = base[name]
const after = head[name]
if (!before) {
additions.push(name)
rows.push([
name,
...METRICS.map((metric) => `${after[metric].toFixed(2)}%`),
'new',
])
continue
}
const deltas = METRICS.map((metric) => after[metric] - before[metric])
const dropped = METRICS.filter(
(metric) => before[metric] >= THRESHOLD && after[metric] < THRESHOLD,
)
if (dropped.length > 0) regressions.push({ name, before, after, dropped })
rows.push([
name,
...METRICS.map((metric, i) => {
const sign = deltas[i] > 0 ? '+' : ''
return `${after[metric].toFixed(2)}% (${sign}${deltas[i].toFixed(2)})`
}),
dropped.length > 0 ? 'FAIL' : 'ok',
])
}
for (const name of Object.keys(base)
.filter((name) => !head[name])
.sort()) {
const before = base[name]
regressions.push({ name, before, after: null, dropped: ['missing'] })
rows.push([
name,
...METRICS.map((metric) => `missing (${before[metric].toFixed(2)}%)`),
'FAIL',
])
}
const compared = names.filter((name) => base[name]).length
const header = ['package', ...METRICS, '']
const widths = header.map((_, column) =>
Math.max(header[column].length, ...rows.map((row) => row[column].length)),
)
const line = (cells) =>
cells
.map((cell, i) => cell.padEnd(widths[i]))
.join(' ')
.trimEnd()
console.log(line(header))
console.log(widths.map((width) => '-'.repeat(width)).join(' '))
for (const row of rows) console.log(line(row))
if (additions.length > 0) {
console.log(
`\n${additions.length} package(s) had no coverage on the base commit` +
` (new, or unmeasurable there): ${additions.join(', ')}`,
)
}
const headline =
regressions.length > 0
? `❌ Coverage fell under ${THRESHOLD}% in ${regressions.length} package(s).`
: compared > 0
? `✅ Coverage held across ${compared} compared package(s).`
: `${additions.length} package(s) are new; nothing to compare against.`
const md = [
`## Coverage`,
``,
headline,
``,
`Each package is measured twice in this job — on this PR and on its merge-base` +
` with \`main\` — and compared. A metric fails only when it is at or above` +
` ${THRESHOLD}% on the merge-base and under ${THRESHOLD}% on this PR. Other` +
` drops are listed but do not fail. Packages your PR doesn't affect are not` +
` measured and not listed.`,
``,
`| package | ${METRICS.join(' | ')} | |`,
`| --- | ${METRICS.map(() => '---:').join(' | ')} | --- |`,
...rows.map((row) => `| ${row.join(' | ')} |`),
]
if (additions.length > 0) {
md.push(
``,
`> No coverage on the base commit, so nothing to compare against:` +
` ${additions.join(', ')}.`,
)
}
const markdown = `${md.join('\n')}\n`
if (process.env.GITHUB_STEP_SUMMARY) {
appendFileSync(process.env.GITHUB_STEP_SUMMARY, markdown)
}
const reportFile = arg('--report')
if (reportFile) writeFileSync(reportFile, `${REPORT_MARKER}\n${markdown}`)
if (regressions.length > 0) {
console.error(
`\nCoverage fell under ${THRESHOLD}% in ${regressions.length} package(s):`,
)
for (const { name, before, after, dropped } of regressions) {
if (!after) {
console.error(` ${name}: measured on base, missing on this PR`)
continue
}
for (const metric of dropped) {
console.error(
` ${name} ${metric}: ${before[metric].toFixed(2)}% -> ${after[metric].toFixed(2)}%`,
)
}
}
console.error(
`\nAdd tests covering the code this PR changed, so each metric is back at ${THRESHOLD}% or more.`,
)
process.exit(1)
}
if (compared === 0) {
console.log(
`\n${additions.length} package(s) are new; nothing to compare against.`,
)
} else {
console.log(`\nCoverage held across ${compared} compared package(s).`)
}