#!/usr/bin/env -S npx tsx /** * ilb-diff-publish — render the differential bench results into a publication * artifact (roadmap item 3.2). Reads the latest `benchmarks/results/ilb-diff/` * JSON and emits a Markdown report ready to drop into `RESULTS.md` or the * public docs site. * * The headline isn't "we win" — it's the **disagreement region**. Where * Interlace fires and CodeQL/Semgrep/Snyk don't is the most credible * narrative for the leadership claim. This script computes that region and * formats it for human + agent consumption. * * Usage: * tsx scripts/ilb-diff-publish.ts # latest result * tsx scripts/ilb-diff-publish.ts --result # specific JSON * tsx scripts/ilb-diff-publish.ts --out path/to/output.md */ import fs from 'node:fs'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; const HERE = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(HERE, '..'); const DIFF_DIR = path.join(REPO_ROOT, 'benchmarks', 'results', 'ilb-diff'); const DEFAULT_OUT = path.join(REPO_ROOT, 'benchmark-results', 'differential.md'); const ARG = (flag) => { const i = process.argv.indexOf(flag); return i >= 0 ? process.argv[i + 1] : null; }; function findLatest() { if (!fs.existsSync(DIFF_DIR)) return null; const files = fs.readdirSync(DIFF_DIR).filter((f) => f.endsWith('.json')).toSorted(); return files.length ? path.join(DIFF_DIR, files[files.length - 1]) : null; } function renderMd(diff) { const lines = []; lines.push('# Differential Benchmark — Interlace × CodeQL × Semgrep × Snyk Code'); lines.push(''); lines.push(`> Generated by \`npm run ilb:diff:publish\` from \`${path.relative(REPO_ROOT, diff._sourcePath)}\`.`); lines.push(`> Source bench: ${diff.bench} v${diff.benchVersion} · run at ${diff.timestamp} · corpus ${diff.corpus}`); lines.push(''); const installed = (diff.adapters ?? []).filter((a) => a.installed); const skipped = (diff.adapters ?? []).filter((a) => !a.installed); lines.push(`**Tools compared:** ${installed.map((a) => a.tool).join(' · ') || ''}`); if (skipped.length) { lines.push(''); lines.push(`Tools skipped (not installed locally): ${skipped.map((a) => `${a.tool} (${a.note || 'no install hint'})`).join(' · ')}`); } lines.push(''); lines.push('## Top-line counts'); lines.push(''); lines.push('| Tool | Findings |'); lines.push('|---|---:|'); for (const a of installed) lines.push(`| **${a.tool}** | ${a.findingCount ?? 0} |`); lines.push(''); if (diff.matrix?.byCombo) { lines.push('## Agreement matrix'); lines.push(''); lines.push('Each row: a (file, line, CWE) location plus the set of tools that flagged it. The rows where only one tool fired are the **interesting region** — they\'re where the tools genuinely disagree, and the natural starting point for "is this a real bug or a false positive?" triage.'); lines.push(''); lines.push('| Tool combination | Locations |'); lines.push('|---|---:|'); const combos = (Object.entries(diff.matrix.byCombo) as Array<[string, number]>).toSorted((a, b) => b[1] - a[1]); for (const [combo, count] of combos) lines.push(`| ${combo} | ${count} |`); lines.push(''); const totalLocations: number = combos.reduce((s: number, [, n]) => s + n, 0); const interlaceOnly = combos.find(([combo]) => combo === 'interlace')?.[1] ?? 0; const consensusCombo = combos.find(([combo]) => combo.split('+').length === installed.length); lines.push(`**${totalLocations} unique locations** flagged across all tools.`); if (interlaceOnly > 0) lines.push(`- **Interlace-only**: ${interlaceOnly} locations. These are findings the rest of the field misses (or that we over-fire on — see triage).`); if (consensusCombo) lines.push(`- **Consensus** (${consensusCombo[0]}): ${consensusCombo[1]} locations. High-confidence true positives — every tool agrees.`); lines.push(''); } lines.push('## Toolchain pin'); lines.push(''); const tc = diff.toolchain ?? {}; lines.push(`- node: \`${tc.node ?? '?'}\``); lines.push(`- eslint: \`${tc.eslint ?? '?'}\``); lines.push(`- typescript: \`${tc.typescript ?? '?'}\` (\`${tc.tsCompiler ?? '?'}\`)`); lines.push(`- platform: \`${tc.platform ?? '?'}\``); lines.push(`- methodology commit: \`${diff.methodologyCommit ?? '?'}\``); lines.push(''); lines.push('## Reproducing this'); lines.push(''); lines.push('```bash'); lines.push(`# Install peer tools (once)`); for (const a of skipped) if (a.note?.startsWith('install:')) lines.push(`# ${a.tool}: ${a.note}`); lines.push(''); lines.push('# Run the bench against the same corpus'); lines.push(`npm run ilb:diff -- --corpus ${diff.corpus} --tools ${(diff.adapters ?? []).map((a) => a.tool).join(',')}`); lines.push(''); lines.push('# Re-publish'); lines.push('npm run ilb:diff:publish'); lines.push('```'); return lines.join('\n'); } function main() { const sourcePath = ARG('--result') ? path.resolve(ARG('--result')) : findLatest(); if (!sourcePath || !fs.existsSync(sourcePath)) { console.error(`ilb-diff-publish: no diff result found. Run \`npm run ilb:diff\` first.`); process.exit(1); } const diff = JSON.parse(fs.readFileSync(sourcePath, 'utf8')); diff._sourcePath = sourcePath; const out = ARG('--out') ? path.resolve(ARG('--out')) : DEFAULT_OUT; fs.mkdirSync(path.dirname(out), { recursive: true }); fs.writeFileSync(out, renderMd(diff) + '\n', 'utf8'); console.log(`ilb-diff-publish: wrote ${path.relative(REPO_ROOT, out)} from ${path.relative(REPO_ROOT, sourcePath)}`); } main();