const assert = require('node:assert'); const { execFileSync } = require('node:child_process'); const fs = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const { test } = require('node:test'); const SCRIPT = path.join(__dirname, 'render-benchmark-comment.mjs'); // The script is ESM; load it lazily from the CJS test file. const loadModule = () => import('./render-benchmark-comment.mjs'); function sampleResult(overrides = {}) { return { version: 1, methodologyVersion: 2, app: 'nextjs-turbopack', backend: 'vercel', generatedAt: '2026-07-08T12:00:00.000Z', commit: 'abcdef1234567890', config: { streamIterations: 30, sequentialIterations: 1, sequentialStepCount: 1020, warmupIterations: 2, }, scenarios: [ { name: 'stream', description: 'one streaming step in turbo mode' }, { name: '1020 steps', description: 'trivial sequential steps' }, ], metrics: [ { metric: 'ttfs', scenario: 'stream', unit: 'ms', best: 320, avg: 412.3, p75: 398, p90: 512, p99: 634, samples: 30, targets: { p75: 200, p90: 300, p99: 600 }, }, { metric: 'stso', scenario: '1020 steps (101-120)', unit: 'ms', best: 60, avg: 91, p75: 85, p90: 120, p99: 200, samples: 19, targets: { p75: 30, p90: 45, p99: 90 }, }, { metric: 'wo', scenario: 'stream', unit: 'ms', best: 900, avg: 1200, p75: 1100, p90: 1500, p99: 1900, samples: 30, }, ], ...overrides, }; } test('renders a completed run with a table and embedded history', async () => { const { renderComment, extractHistory } = await loadModule(); const body = renderComment({ status: 'completed', results: [sampleResult()], history: [], commit: 'abcdef1234567890', runUrl: 'https://github.com/vercel/workflow/actions/runs/1', }); assert.match(body, //); assert.match(body, /## 📊 Workflow Benchmarks/); assert.match(body, /\*\*TTFS\*\*/); assert.match(body, /\| stream \|/); assert.match(body, /1020 steps \(101-120\)/); // "ms" lives in the column headers, not in the cells; no Avg column assert.match( body, /\| Best \(ms\) \| P75 \(ms\) \| P90 \(ms\) \| P99 \(ms\) \|/ ); assert.doesNotMatch(body, /Avg \(ms\)/); assert.doesNotMatch(body, /\d ms \|/); // Best cell (fastest sample) renders before P75 (warm-start floor for TTFS) assert.match(body, /\| 320 \| 398 🔴 \|/); // Metric definitions live in the footer, not in the table rows assert.doesNotMatch(body, /\| \*\*TTFS\*\* /); // The smallprint footer is collapsed into a dropdown, like "Previous results" assert.match( body, /
\nâ„šī¸ Metric definitions & methodology<\/summary>/ ); assert.match(body, /Metrics — \*\*TTFS\*\*: time to first step body/); // Scenario legend from the runner-provided descriptions assert.match( body, /Scenarios — \*\*stream\*\*: one streaming step in turbo mode/ ); // Target marks: TTFS p75 398 > 200 → 🔴; WO has no targets. assert.match(body, /398 🔴/); assert.doesNotMatch(body, /đŸŸĸ/); assert.match(body, /\| 1100 \|/); // Targets legend derived from row targets assert.match(body, /Targets \(p75\/p90\/p99, ms\) — TTFS 200\/300\/600/); assert.match(body, /STSO \(101-120\) 30\/45\/90/); assert.match(body, /commit `abcdef1`/); // No previous results yet assert.doesNotMatch(body, /Previous results/); // History round-trips through the embedded data block const history = extractHistory(body); assert.strictEqual(history.length, 1); assert.strictEqual(history[0].commit, 'abcdef1234567890'); assert.strictEqual(history[0].results[0].metrics.length, 3); }); test('drops retired stream metrics from old artifacts and sticky history', async () => { const { renderComment, extractHistory, loadResults } = await loadModule(); const result = sampleResult(); const retired = ['sl', 'so', 'crtt', 'cdv', 'slip', 'stream']; for (const metric of retired) { const scenario = `retired-${metric}`; result.metrics.push({ ...result.metrics[0], metric, scenario }); result.scenarios.push({ name: scenario, description: 'retired workload' }); } const oldEntry = { commit: 'old1234', results: [result] }; // Encode the old schema directly, not via the new renderer. const oldBody = ``; const history = extractHistory(oldBody); assert.deepStrictEqual( history[0].results[0].metrics.map((row) => row.metric), ['ttfs', 'stso', 'wo'] ); const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-retired-')); try { fs.writeFileSync( path.join(dir, 'bench-results-old.json'), JSON.stringify(result) ); assert.deepStrictEqual( loadResults(dir)[0].metrics, history[0].results[0].metrics ); for (const status of ['running', 'completed', 'failed']) { const body = renderComment({ status, results: status === 'running' ? [] : [result], baseline: [result], history: [oldEntry], commit: 'new1234', }); assert.match(body, /\*\*TTFS\*\*/); assert.doesNotMatch(body, /retired-|\*\*Streams\*\*|CRTT|CDV/); assert.deepStrictEqual( extractHistory(body)[0].results[0].metrics.map((row) => row.metric), ['ttfs', 'stso', 'wo'] ); } } finally { fs.rmSync(dir, { recursive: true, force: true }); } }); test('renders the fan-out scenario as one Fan-out TTFS row and one Fan-out TTLS row', async () => { const { renderComment } = await loadModule(); const fanOutRow = (metric, overrides) => ({ metric, scenario: 'Promise.all(100 steps)', unit: 'ms', best: 300, avg: 420, p75: 450, p90: 520, p99: 700, samples: 10, ...overrides, }); const result = sampleResult({ scenarios: [ { name: 'Promise.all(100 steps)', description: '100 trivial no-op steps started together', }, ], metrics: [ // Deliberately out of display order: the table sorts by METRIC_ORDER. fanOutRow('fanout-ttls', { best: 900, p75: 1400, p90: 1800, p99: 2600 }), fanOutRow('fanout-ttfs'), sampleResult().metrics[0], ], }); const body = renderComment({ status: 'completed', results: [result], history: [], commit: 'abcdef1234567890', }); assert.match( body, /\| \*\*Fan-out TTFS\*\* \| Promise\.all\(100 steps\) \| 300 \|/ ); assert.match( body, /\| \*\*Fan-out TTLS\*\* \| Promise\.all\(100 steps\) \| 900 \|/ ); // Neither row falls back to the raw metric id. assert.doesNotMatch(body, /\| fanout-ttfs \|/); assert.doesNotMatch(body, /\| fanout-ttls \|/); // Display order: single-step TTFS, then the fan-out pair with first before last. const order = ['**TTFS**', '**Fan-out TTFS**', '**Fan-out TTLS**'].map( (name) => body.indexOf(name) ); assert.ok( order.every((index, i) => index > -1 && (i === 0 || index > order[i - 1])), `unexpected row order: ${JSON.stringify(order)}` ); // Both metrics are defined in the collapsed footer. assert.match(body, /\*\*Fan-out TTFS\*\*: fan-out time to first step/); assert.match(body, /\*\*Fan-out TTLS\*\*: fan-out time to last step/); // No targets on the fan-out rows, so they contribute nothing to the targets // legend and carry no 🔴 marks. assert.doesNotMatch(body, /Targets \(p75\/p90\/p99, ms\) —[^<]*Fan-out/); assert.doesNotMatch(body, /\| \*\*Fan-out TT[FL]S\*\* \|[^\n]*🔴/); }); test('diffs fan-out rows against a baseline keyed on their own metric ids', async () => { const { renderComment } = await loadModule(); const fanOutRow = (metric, best) => ({ metric, scenario: 'Promise.all(100 steps)', unit: 'ms', best, avg: best, p75: best, p90: best, p99: best, samples: 10, }); const metricsFor = (ttfsBest, ttlsBest) => [ fanOutRow('fanout-ttfs', ttfsBest), fanOutRow('fanout-ttls', ttlsBest), ]; const body = renderComment({ status: 'completed', // First branch lands as fast as on main; the tail is 50% slower. results: [sampleResult({ metrics: metricsFor(300, 1500) })], baseline: [sampleResult({ metrics: metricsFor(300, 1000) })], history: [], commit: 'abcdef1234567890', }); // TTFS unchanged, TTLS regressed — the two rows carry independent deltas, // which is the point of splitting them. assert.match(body, /\| \*\*Fan-out TTFS\*\* \|[^\n]*300 \(Âą0%\)/); assert.match(body, /\| \*\*Fan-out TTLS\*\* \|[^\n]*1500 \(\+50%\) đŸ”ģ/); }); test('renders best/p75/p90/p99 deltas with đŸ”ģ/💚 threshold marks and embeds them', async () => { const { renderComment, extractHistory } = await loadModule(); const baseline = sampleResult({ metrics: sampleResult() .metrics.filter((row) => row.metric !== 'wo') // no baseline for WO .map((row) => ({ ...row, // ttfs: best 320 vs 250 → +28% đŸ”ģ, p75 398 vs 500 → -20% 💚, // p90 512 vs 512 → Âą0%, p99 634 vs 600 → +5.7% (no mark). // sl/stso baselines equal the run → Âą0% everywhere. best: { ttfs: 250, sl: 30, stso: 60 }[row.metric], p75: { ttfs: 500, sl: 48, stso: 85 }[row.metric], p90: { ttfs: 512, sl: 55, stso: 120 }[row.metric], p99: { ttfs: 600, sl: 120, stso: 200 }[row.metric], })), }); const body = renderComment({ status: 'completed', results: [sampleResult()], baseline: [baseline], history: [], commit: 'abcdef1234567890', }); // Best regression past +15% → đŸ”ģ assert.match(body, /\| 320 \(\+28%\) đŸ”ģ \|/); // P75 improvement past -15% → 💚 (alongside the 🔴 target miss) assert.match(body, /398 🔴 \(-20%\) 💚/); // P90 now carries a delta (previously undecorated); Âą0%, no threshold mark assert.match(body, /512 🔴 \(Âą0%\) \|/); // P99 small delta, no threshold mark assert.match(body, /634 🔴 \(\+5\.7%\) \|/); // WO has no baseline row → no delta on its Best cell assert.match(body, /\| 900 \|/); assert.match( body, /Best\/P75\/P90\/P99 deltas compare against the most recent benchmark run on `main`/ ); assert.match(body, /💚 one better than/); // The annotations are embedded so history re-renders keep the deltas const history = extractHistory(body); assert.strictEqual(history[0].results[0].metrics[0].baselineBest, 250); assert.strictEqual(history[0].results[0].metrics[0].baselineP90, 512); const rerendered = renderComment({ status: 'running', results: [], history, commit: 'ffffff1234567890', }); assert.match(rerendered, /\| 320 \(\+28%\) đŸ”ģ \|/); }); test('suppresses deltas when the baseline methodology version differs', async () => { const { renderComment } = await loadModule(); // Old-methodology baseline (e.g. proxy-inclusive TTFS) must not be diffed // against a new-methodology run, even though backend/app/metric/scenario // match — the numbers are not comparable. const baseline = sampleResult({ methodologyVersion: 1, metrics: sampleResult().metrics.map((row) => ({ ...row, best: 200 })), }); const body = renderComment({ status: 'completed', results: [sampleResult()], // methodologyVersion: 2 baseline: [baseline], history: [], commit: 'abcdef1234567890', }); // No percentage deltas, and the "compare against main" note is absent. assert.doesNotMatch(body, /%\)/); assert.doesNotMatch(body, /deltas compare against/); }); test('renders no deltas without a baseline', async () => { const { renderComment } = await loadModule(); const body = renderComment({ status: 'completed', results: [sampleResult()], history: [], commit: 'abcdef1234567890', }); assert.doesNotMatch(body, /%\)/); assert.doesNotMatch(body, /deltas compare against/); }); test('collapses previous results on re-runs', async () => { const { renderComment, extractHistory } = await loadModule(); const first = renderComment({ status: 'completed', results: [sampleResult()], history: [], commit: '1111111aaaaaaa', runUrl: 'https://example.com/run/1', }); const second = renderComment({ status: 'completed', results: [sampleResult()], history: extractHistory(first), commit: '2222222bbbbbbb', runUrl: 'https://example.com/run/2', }); // Latest commit shown prominently, previous one collapsed assert.match(second, /commit `2222222`/); assert.match( second, /
\n📜 Previous results \(1\)<\/summary>/ ); assert.match(second, /#### 1111111/); const history = extractHistory(second); assert.strictEqual(history.length, 2); assert.strictEqual(history[0].commit, '2222222bbbbbbb'); }); test('running status preserves previous results and history', async () => { const { renderComment, extractHistory } = await loadModule(); const first = renderComment({ status: 'completed', results: [sampleResult()], history: [], commit: '1111111aaaaaaa', }); const running = renderComment({ status: 'running', results: [], history: extractHistory(first), commit: '2222222bbbbbbb', }); assert.match(running, /Benchmarks are running for `2222222`/); assert.match(running, /Results below are from a previous run/); // Previous results still rendered and history unchanged assert.match(running, /398 🔴/); const history = extractHistory(running); assert.strictEqual(history.length, 1); assert.strictEqual(history[0].commit, '1111111aaaaaaa'); }); test('failed status renders a failure banner', async () => { const { renderComment } = await loadModule(); const body = renderComment({ status: 'failed', results: [], history: [], commit: '3333333ccccccc', runUrl: 'https://example.com/run/3', }); assert.match(body, /❌ \*\*The benchmark run for `3333333` failed\.\*\*/); assert.match(body, /No benchmark results were produced/); }); test('caps history at 10 entries', async () => { const { renderComment, extractHistory } = await loadModule(); let history = []; for (let i = 0; i < 12; i++) { const body = renderComment({ status: 'completed', results: [sampleResult()], history, commit: `${i}`.repeat(10), }); history = extractHistory(body); } assert.strictEqual(history.length, 10); assert.strictEqual(history[0].commit, '11'.repeat(10)); }); test('ignores malformed data blocks', async () => { const { extractHistory } = await loadModule(); assert.deepStrictEqual(extractHistory(undefined), []); assert.deepStrictEqual(extractHistory('no marker here'), []); assert.deepStrictEqual( extractHistory(''), [] ); const garbage = Buffer.from('not json', 'utf8').toString('base64'); assert.deepStrictEqual( extractHistory(``), [] ); }); test('CLI renders results from a directory and previous body file', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-render-')); const resultsDir = path.join(dir, 'results'); fs.mkdirSync(resultsDir); fs.writeFileSync( path.join(resultsDir, 'bench-results-nextjs-turbopack-vercel.json'), JSON.stringify(sampleResult()) ); const firstOut = path.join(dir, 'comment1.md'); execFileSync(process.execPath, [ SCRIPT, '--status', 'completed', '--results-dir', resultsDir, '--commit', '1111111aaaaaaa', '--run-url', 'https://example.com/run/1', '--output', firstOut, ]); const first = fs.readFileSync(firstOut, 'utf8'); assert.match(first, /398 🔴/); // Baseline files arrive nested in per-artifact subdirectories (that's how // the download action extracts them); loadResults must find them anyway. const baselineDir = path.join(dir, 'baseline'); fs.mkdirSync( path.join(baselineDir, 'bench-results-nextjs-turbopack-vercel'), { recursive: true, } ); const baseline = sampleResult(); baseline.metrics = baseline.metrics.map((row) => ({ ...row, best: 300 })); fs.writeFileSync( path.join( baselineDir, 'bench-results-nextjs-turbopack-vercel', 'bench-results-nextjs-turbopack-vercel.json' ), JSON.stringify(baseline) ); const secondOut = path.join(dir, 'comment2.md'); execFileSync(process.execPath, [ SCRIPT, '--status', 'completed', '--results-dir', resultsDir, '--baseline-dir', baselineDir, '--previous-body', firstOut, '--commit', '2222222bbbbbbb', '--output', secondOut, ]); const second = fs.readFileSync(secondOut, 'utf8'); assert.match(second, /Previous results \(1\)/); assert.match(second, /#### 1111111/); // ttfs best 320 vs baseline 300 → +6.7% assert.match(second, /\| 320 \(\+6\.7%\) \|/); }); test('CLI fails when completed with no results', async () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-render-empty-')); assert.throws(() => execFileSync( process.execPath, [SCRIPT, '--status', 'completed', '--results-dir', dir], { stdio: 'pipe' } ) ); }); /** A sequential-steps result whose STSO rows carry raw samples, as the * benchmark runner now records them (every gap, not a sampled window). */ function sequentialResult({ inline, queueHop }) { const stsoRow = (scenario, raw) => ({ metric: 'stso', scenario, unit: 'ms', best: Math.min(...raw), avg: raw.reduce((a, b) => a + b, 0) / raw.length, p75: raw[Math.ceil(0.75 * raw.length) - 1], p90: raw[Math.ceil(0.9 * raw.length) - 1], p99: raw[Math.ceil(0.99 * raw.length) - 1], samples: raw.length, raw, }); return sampleResult({ scenarios: [ { name: '1020 steps', description: 'trivial sequential steps' }, ], metrics: [ stsoRow('1020 steps (inline)', inline), stsoRow('1020 steps (queue-hop)', queueHop), ], }); } test('renders inline and queue-hop STSO histogram diffs against main', async () => { const { renderComment } = await loadModule(); const body = renderComment({ status: 'completed', // Two inline steps move from the 350-400ms bucket down to 150-200ms. results: [ sequentialResult({ inline: [160, 190, 360, 360], queueHop: [2100, 2600], }), ], baseline: [ sequentialResult({ inline: [360, 360, 360, 360], queueHop: [2100, 2100], }), ], history: [], commit: 'abcdef1234567890', }); // Collapsed by default — a drill-down, not the headline. assert.match( body, /
\n📈 STSO distribution vs main \(inline \/ queue-hop histograms\)<\/summary>/ ); assert.match(body, /_1020 steps \(inline\)_/); assert.match(body, /_1020 steps \(queue-hop\)_/); // Cumulative time diff: inline 1440ms → 1070ms. assert.match( body, /Cumulative STSO time: main 1440ms → this run 1070ms \(Δ -370ms, -26%\)/ ); // Inline rows are bucketed at a hardcoded 50ms, so the two fast samples // land in 150-200 and the bucket the baseline occupied loses them. Counts // and the per-bucket delta ride on the bar line; there is no second table. assert.match(body, /^ *150-200 ms .*main 0 +this 2 +\+2$/m); assert.match(body, /^ *350-400 ms .*main 4 +this 2 +-2$/m); assert.doesNotMatch(body, /Bucket \(ms\)/); // Queue-hop rows keep the adaptive (much coarser) bin width. assert.doesNotMatch(body, /2100-2150 ms/); assert.match(body, /^ *2000-2500 ms .*main 2 +this 1 +-1$/m); assert.match(body, /^ *2500-3000 ms .*main 0 +this 1 +\+1$/m); // Overlay bars: solid run for main, notch for this run. assert.match(body, /█+┃/); assert.match( body, /The collapsed \*\*STSO distribution\*\* section above buckets every/ ); }); test('shows the STSO distribution alone when main has no raw samples', async () => { const { renderComment } = await loadModule(); const run = sequentialResult({ inline: [160, 360], queueHop: [2100] }); // A pre-raw-samples baseline: same rows, percentiles only. const baseline = sequentialResult({ inline: [160, 360], queueHop: [2100] }); for (const row of baseline.metrics) delete row.raw; const body = renderComment({ status: 'completed', results: [run], baseline: [baseline], history: [], commit: 'abcdef1234567890', }); // Summary drops the "vs main" qualifier when there is nothing to diff. assert.match( body, /📈 STSO distribution \(inline \/ queue-hop histograms\)<\/summary>/ ); assert.doesNotMatch(body, /STSO distribution vs main/); assert.match(body, /No `main` baseline with raw samples yet/); assert.match(body, /Cumulative STSO time: 520ms over 2 samples/); // Single-series rendering: one count per bucket, no main/this or delta. assert.match(body, /^ *150-200 ms .*steps 1$/m); assert.doesNotMatch(body, /this \d/); assert.doesNotMatch(body, /Bucket \(ms\)/); }); test('strips raw samples from the embedded history data block', async () => { const { renderComment, extractHistory } = await loadModule(); const inline = Array.from({ length: 1000 }, (_, i) => 200 + (i % 300)); const body = renderComment({ status: 'completed', results: [sequentialResult({ inline, queueHop: [2100, 2600] })], baseline: [sequentialResult({ inline, queueHop: [2100, 2600] })], history: [], commit: 'abcdef1234567890', }); // The histogram renders for the current run... assert.match(body, /📈 STSO distribution vs main/); // ...but the ~1000 samples behind it never reach the embedded data block, // which would otherwise blow past GitHub's comment size limit as history // accumulates. const history = extractHistory(body); const row = history[0].results[0].metrics[0]; assert.strictEqual(row.raw, undefined); assert.strictEqual(row.baselineRaw, undefined); assert.strictEqual(row.samples, 1000); assert.ok(body.length < 20_000, `comment is ${body.length} chars`); // Re-rendering from that history keeps the table but drops the histogram. const rerendered = renderComment({ status: 'running', results: [], history, commit: 'ffffff1234567890', }); assert.match(rerendered, /1020 steps \(inline\)/); assert.doesNotMatch(rerendered, /STSO distribution/); }); test('buckets negative STSO gaps separately from the slow tail', async () => { const { renderComment } = await loadModule(); // Consecutive step timestamps come from different step bodies, so a gap can // come out slightly negative under clock skew — it must not be counted with // the slowest samples. const body = renderComment({ status: 'completed', results: [sequentialResult({ inline: [-40, 160, 360], queueHop: [2100] })], baseline: [sequentialResult({ inline: [160, 360], queueHop: [2100] })], history: [], commit: 'abcdef1234567890', }); assert.match(body, /^ *<0 \(skew\) ms .*main 0 +this 1 +\+1$/m); // ...and the top bucket still reflects only genuinely slow samples. assert.match(body, /^ *350-400 ms .*main 1 +this 1 +\+0$/m); });