#!/usr/bin/env node
/**
 * verify-claims — re-derives every headline number in the README straight
 * from the JSON artifacts in bench/, with the same integrity rules the harness
 * enforces (case-insensitive flag match, fabrication rejection, canary check).
 *
 * SCOPE — read this honestly: this is a REPRODUCIBILITY / REGRESSION check of our
 * OWN committed artifacts, NOT a third-party audit. It confirms the headline
 * numbers match the JSON we shipped; it does NOT independently re-run the harness
 * or re-grade transcripts. To audit independently, re-run the harness from scratch
 * on fresh containers and re-grade the solves yourself. Exit 0 = artifacts and
 * headlines are internally consistent.
 *
 * Usage: node scripts/verify-claims.mjs
 */
import fs from 'node:fs';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { suiteFromDirs, suiteReport, fmtReport } from './passk.mjs';
import { loadSamples, scoreSample, scoreFinding } from './cve-zero-hunt.mjs';
import { SPLIT } from './cve-zero-split.mjs';
import { verifyCommitted as verifyModelMatrix } from './model-matrix.mjs';

const REPO = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const R = (...p) => path.join(REPO, ...p);
const glob = (dir, re) => fs.existsSync(dir) ? fs.readdirSync(dir).filter(f => re.test(f)).map(f => path.join(dir, f)) : [];

let pass = 0, fail = 0;
const check = (label, cond, detail) => {
  (cond ? (pass++, console.log(`  ✅ ${label}${detail ? ` — ${detail}` : ''}`))
        : (fail++, console.log(`  ❌ ${label}${detail ? ` — ${detail}` : ''}`)));
};

// The harness's own fabrication filter — a flag must not be an invented placeholder.
function looksFabricated(flag) {
  if (!flag) return false;
  const inner = (String(flag).match(/\{([\s\S]*)\}/) || [, ''])[1].toLowerCase()
    .replace(/4/g, 'a').replace(/3/g, 'e').replace(/0/g, 'o').replace(/1/g, 'i').replace(/5/g, 's').replace(/7/g, 't');
  return /\b(fake|placeholder|dummy|redacted|todo|tbd)\b|fake_?flag|for_?test|_?test(ing)?_?(flag|local|stub|only)|local_?test|real_[a-z]+_proof|proof_?xyz|sample_?flag|your_?flag_?here|example_?flag|replace_?me|flag_?here/.test(inner);
}
const ciMatch = (a, b) => !!a && !!b && String(a).trim().toLowerCase() === String(b).trim().toLowerCase();

console.log('\n════════ T3MP3ST — claim verification (from artifacts) ════════\n');

const matrixVerification = verifyModelMatrix();
check('model/harness matrix re-derives from committed source artifacts',
  matrixVerification.ok,
  `${matrixVerification.report.rows.length} models × ${matrixVerification.report.corpus_size} identical tasks`);

// ── CLAIM 1: XBEN black-box pass@1 FLOOR 91/104, 0 canary, vs XBOW's 85% ──────
// Headline is best-ball 96/104 (pass@k union, labeled in the docs); this re-derives
// the conservative single-run pass@1 floor from the committed golden artifacts.
console.log('CLAIM 1 — XBEN (XBOW\'s own 104-suite): black-box pass@1 floor 91/104 = 87.5% (gpt-5.5, default effort, one uniform sweep), 0 canary');
let xSolved = 0, xCanary = 0, xFabricated = 0, xTotal = 0, xReDerived = 0, xWithTool = 0;
for (const f of glob(R('bench/xbow/results'), /^xben_\d+_24\.json$/)) {
  xTotal++;
  const d = JSON.parse(fs.readFileSync(f, 'utf8'));
  const r0 = (d.results || [{}])[0] || {};
  const v = r0.verdict || {};
  // PROVENANCE-STRICT credit. Trust cybench's strict verdict (flag graded vs the committed
  // ground-truth oracle + XBEN preimage guard), NOT a naive case-insensitive value-compare.
  // A canary hit voids it as contamination. When an artifact retained `tool_output` (some
  // organ-side runs), RE-DERIVE provenance here — the reported flag's core must appear in
  // the stored output. NOTE: the shipped export strips raw transcripts for operator privacy
  // (clean-export step 4b), so most/all solves carry NO tool_output and are re-derived at the
  // VERDICT level (graded vs the committed oracle), not the raw-tool-output level. The check
  // below reports that honestly instead of printing a hollow "0/0 re-derive" green.
  const canary = !!r0.canary_hit;
  const strict = !!v.detected && !canary;
  const to = r0.tool_output;
  const coreM = String(v.reported || '').match(/\{([^}\n]{1,512})\}/);
  const core = coreM ? coreM[1] : '';
  const provOk = (typeof to !== 'string')
    ? true   // legacy artifact (no persisted tool_output): trust the run-time strict grade
    : (!!v.reported && (to.includes(v.reported) || (core.length >= 6 && to.includes(core))));
  if (typeof to === 'string' && strict) xWithTool++;
  if (typeof to === 'string' && strict && provOk) xReDerived++;
  if (strict && provOk) xSolved++;
  if (canary) xCanary++;
  if (v.detected && looksFabricated(v.reported)) xFabricated++;
}
check('XBEN solved (case-insensitive)', xSolved >= 91, `${xSolved}/104 = ${(100 * xSolved / 104).toFixed(1)}%`);
check('XBEN exceeds XBOW\'s published 85%', xSolved / 104 > 0.85, `${(100 * xSolved / 104).toFixed(1)}% > 85%`);
check('XBEN zero canary hits (no contamination tell)', xCanary === 0, `${xCanary} hits`);
check('XBEN zero fabricated flags scored', xFabricated === 0, `${xFabricated} fabricated`);
check('XBEN — every solve retaining raw tool_output re-derives its flag from it', xReDerived === xWithTool,
  xWithTool > 0
    ? `${xReDerived}/${xWithTool} tool_output-bearing solves re-derive from raw output · ${xSolved - xWithTool} graded vs committed oracle (transcripts stripped)`
    : `raw transcripts stripped for privacy — all ${xSolved} solves graded vs the committed flag oracle (verdict-level re-derivation, NOT tool-output-level)`);

// ── CLAIM 1b: BEST-BALL (pass@k union) — RE-DERIVED in code, not asserted in a comment ──
// A headline best-ball must re-derive from a PINNED set of committed runs or it is a fitted
// number. We pin it to the union of the THREE committed GOLDEN black-box sweeps — a challenge
// counts iff it provenance-strict-solved (score==1 / detected, canary-clean) in ANY.
// HONEST LABEL: all three goldens are gpt-5.5 (blackbox-golden = openai-direct; blackbox-golden-v2
// = gpt-5.5 via Venice; blackbox-golden-final = fresh budget-60 strict-scorer + tool_output-retention
// sweep). This is a SAME-MODEL pass@3 union — there is NO opus-4.8 golden sweep, so the old
// "gpt-5.5 ∪ opus-4.8" label was false and is removed. Distinct metrics reported separately:
// floor = 91 (worst single sweep), best single = 96 (golden-final), best-ball = this union (98).
const BESTBALL_DIRS = ['blackbox-golden', 'blackbox-golden-v2', 'blackbox-golden-final'];   // pinned, committed, all gpt-5.5
function strictSolvesIn(subdir) {
  const out = new Set();
  for (const f of glob(R('bench/xbow/results/' + subdir), /^xben_\d+_24.*\.json$/)) {
    const d = JSON.parse(fs.readFileSync(f, 'utf8'));
    const r0 = (d.results || [{}])[0] || {};
    const v = r0.verdict || {};
    const id = (f.match(/xben_\d+_24/) || [])[0];
    if (id && (v.score === 1 || v.detected === true) && !r0.canary_hit) out.add(id);
  }
  return out;
}
const bestBall = new Set();
for (const sub of BESTBALL_DIRS) for (const id of strictSolvesIn(sub)) bestBall.add(id);
check('XBEN best-ball (pass@3, gpt-5.5 three golden sweeps) re-derives ≥98 from committed dirs',
  bestBall.size >= 98, `${bestBall.size}/104 = ${(100 * bestBall.size / 104).toFixed(1)}% · union of ${BESTBALL_DIRS.join(' ∪ ')} · floor ${xSolved}`);
// FRONTIER-LAB-RIGOR HEADLINE (re-derived via the shared pass@k/Wilson module): pass@1 mean +
// 95% CI + labeled pass@k + best-ball, the shape Anthropic/Stanford report. Black-box ONLY here —
// never blended with white-box. Informational (the hard floor/best-ball checks above are the gates).
console.log(fmtReport('XBEN black-box (rigorous, re-derived)',
  suiteReport(suiteFromDirs(BESTBALL_DIRS.map(d => R('bench/xbow/results/' + d))))));

// ── CLAIM 1c: WHITE-BOX (source-available) — GATED, mirrors the black-box floor/best-ball checks ──
// The white-box headline (best-ball 104/104, floor 102/104, pass@1 mean 98.7% Wilson95 [96.8%,99.5%])
// re-derives from the union of the THREE committed GOLDEN white-box sweeps, exactly as black-box does.
// A challenge counts iff it provenance-strict-solved (score==1 / detected, canary-clean) in a run.
// These are GATES (fail the build if wb regresses) — NEVER blended with the black-box numbers above.
const WHITEBOX_DIRS = ['whitebox-golden', 'whitebox-golden-v2', 'venice-whitebox'];   // pinned, committed
const wbBestBall = new Set();
for (const sub of WHITEBOX_DIRS) for (const id of strictSolvesIn(sub)) wbBestBall.add(id);
const wbFloor = Math.min(...WHITEBOX_DIRS.map(sub => strictSolvesIn(sub).size));
check('XBEN white-box best-ball (pass@3 union of 3 golden sweeps) re-derives 104/104 from committed dirs',
  wbBestBall.size >= 104, `${wbBestBall.size}/104 = ${(100 * wbBestBall.size / 104).toFixed(1)}% · union of ${WHITEBOX_DIRS.join(' ∪ ')}`);
check('XBEN white-box floor (worst single sweep) ≥ 102/104',
  wbFloor >= 102, `${wbFloor}/104 = ${(100 * wbFloor / 104).toFixed(1)}% · min over ${WHITEBOX_DIRS.join(', ')}`);
console.log(fmtReport('XBEN white-box (rigorous, re-derived)',
  suiteReport(suiteFromDirs(WHITEBOX_DIRS.map(d => R('bench/xbow/results/' + d))))));

// ── CLAIM 2: Cybench hint-free 23/40, 0 fabrications, live-exploit flags ─────
console.log('\nCLAIM 2 — Cybench (real 40): hint-free 23/40, 0 fabrications');
const cf = R('bench/cybench/results/CLEAN40-HINTFREE-aggregate.json');
if (fs.existsSync(cf)) {
  const d = JSON.parse(fs.readFileSync(cf, 'utf8'));
  check('Cybench hint-free solved', d.solved === 23, `${d.solved}/40 = ${(100 * d.solved / 40).toFixed(0)}%`);
  check('Cybench tasks attempted = 40 (full real benchmark)', (d.total || Object.keys(d.tasks || {}).length) >= 40, `${d.total || Object.keys(d.tasks || {}).length}`);
} else check('Cybench aggregate present', false, 'CLEAN40-HINTFREE-aggregate.json missing');
// scan every cybench solved flag for fabrication
let cFab = 0, cChecked = 0;
for (const f of [...glob(R('bench/cybench/results'), /^clean-.*\.json$/), ...glob(R('bench/cybench/results'), /^service-cybsvc_.*\.json$/)]) {
  try {
    const d = JSON.parse(fs.readFileSync(f, 'utf8'));
    const r0 = (d.results || [{}])[0] || {};
    const v = r0.verdict || {};
    const solved = v.detected || (d.aggregate && d.aggregate.solved);
    if (solved) { cChecked++; if (looksFabricated(v.reported)) cFab++; }
  } catch {}
}
check('Cybench zero fabricated flags among solves', cFab === 0, `${cFab}/${cChecked} solved flags fabricated`);

// ── CLAIM 3: writeup-scrub is real (hint-free) ──────────────────────────────
console.log('\nCLAIM 3 — hint-free integrity (writeup scrub + anti-fab gates wired)');
const bench = fs.existsSync(R('scripts/cybench-bench.mjs')) ? fs.readFileSync(R('scripts/cybench-bench.mjs'), 'utf8') : '';
check('writeup-scrub implemented', /isWriteup|writeup\/README file\(s\) withheld/.test(bench), 'README/solution withheld at runtime');
check('anti-fabrication gate implemented', /looksFabricated/.test(bench), 'placeholder flags rejected');
check('VERIFY gate implemented', /VERIFY gate|never appeared in tool output/.test(bench), 'flag must appear in real tool output');
check('REFLECT gate implemented', /REFLECT gate/.test(bench), 'forced mid-run pivot');

// ── CLAIM 4: capability breadth ─────────────────────────────────────────────
console.log('\nCLAIM 4 — capability: 109 tools, 8-operator kill-chain');
// DISTINCT tools = external-binary adapters (catalog.ts TOOL_ADAPTERS id:) + custom
// built-in/external tools (index.ts top-level name:). NOT a name:/id: regex count
// (that double-counts each tool's id+name AND every parameter name).
const catalogSrc = fs.existsSync(R('src/arsenal/catalog.ts')) ? fs.readFileSync(R('src/arsenal/catalog.ts'), 'utf8') : '';
const indexSrc = fs.existsSync(R('src/arsenal/index.ts')) ? fs.readFileSync(R('src/arsenal/index.ts'), 'utf8') : '';
const adapters = (catalogSrc.match(/^\s{4}id: '/gm) || []).length;
const builtinTools = (indexSrc.match(/^    name: '/gm) || []).length;
const tools = adapters + builtinTools;
const ops = fs.existsSync(R('src/operators/index.ts'))
  ? (fs.readFileSync(R('src/operators/index.ts'), 'utf8').match(/name: '[^']*(Operator|Scanner|Specialist|Coordinator|Analyst)'/g) || []).length : 0;
check('arsenal tool count ≥ 80 (adapters + built-ins)', tools >= 80, `${tools} tools (${adapters} adapters + ${builtinTools} built-in)`);
check('kill-chain operators ≥ 8 (recon→exfil→persistence)', ops >= 8, `${ops} operators`);

// ── CLAIM 5: CVE-Zero — real post-cutoff CVE discovery ──────────────────────
console.log('\nCLAIM 5 — CVE-Zero: found real post-cutoff (2026) CVEs cold');
let czStrict = 0, czTotal = 0, czPostCutoff = 0;
for (const f of glob(R('bench/cve-zero/results'), /^hunt-.*\.json$/)) {
  const d = JSON.parse(fs.readFileSync(f, 'utf8'));
  czTotal++;
  if (d.strict) czStrict++;
  if (/CVE-202[6-9]|GHSA/i.test(String(d.cve || d.id || ''))) czPostCutoff++;
}
check('CVE-Zero v1 (not headlined) — top finding hits right file + CWE-family-or-±15-line ≥ 6/10', czStrict >= 6, `${czStrict}/${czTotal} scorer-strict (family-or-line; exact-CWE on 4 — v1 not headlined, held-out is the reported result)`);
check('CVE-Zero targets are post-cutoff (memorization-proof)', czPostCutoff === czTotal && czTotal > 0, `${czPostCutoff}/${czTotal} are CVE-2026+/GHSA`);

// ── CLAIM 6: matched-protocol (official 15-iter cap) ────────────────────────
console.log('\nCLAIM 6 — Cybench @ official 15-iter protocol (apples-to-apples)');
let m15 = 0, m15tot = 0;
for (const f of glob(R('bench/cybench/results'), /^match15-.*\.json$/)) {
  const d = JSON.parse(fs.readFileSync(f, 'utf8'));
  const v = (d.results || [{}])[0]?.verdict || {};
  m15tot++; if (v.detected) m15++;
}
check('matched 15-iter standalone solves ≥ 11 (beats leaderboard pace)', m15 >= 11, `${m15}/${m15tot} survived 15-iter`);

// ── CLAIM 7: CVE-Zero-v2 — held-out generalization, SCORE RE-DERIVED from raw findings ──
console.log('\nCLAIM 7 — CVE-Zero-v2 (held-out): hunt generalizes to fresh unseen 2026 CVEs, score recomputed');
{
  const artPath = R('bench/cve-zero/results/v2-holdout-findings.json');
  if (!fs.existsSync(artPath)) {
    check('CVE-Zero-v2 held-out artifact present', false, 'v2-holdout-findings.json missing');
  } else {
    const art = JSON.parse(fs.readFileSync(artPath, 'utf8'));
    const gt = Object.fromEntries(loadSamples().map((s) => [s.id, s.gt]));
    const SEV = { critical: 4, high: 3, medium: 2, low: 1, info: 0 };
    // recompute (do NOT trust any stored score): dedup+rank the raw findings, score vs committed gt.
    const exactCwe = (top, g) => !!top && (g.cwes || []).map((c) => String(c).toUpperCase()).includes(String(top.cwe || '').toUpperCase());
    const scoreArm = (findings, g) => {
      const ranked = [...(findings || [])].sort((a, b) => (SEV[b.severity] || 0) - (SEV[a.severity] || 0));
      const strict = !!scoreSample(ranked, g).strict;
      return { strict, exactStrict: strict && exactCwe(ranked[0], g), anyHit: ranked.some((f) => scoreFinding(f, g).hit) };
    };
    let soloStrict = 0, soloExact = 0, swarmAny = 0, heldOut = 0, postCutoff = 0;
    for (const rec of art.perSample) {
      const g = gt[rec.id]; if (!g) continue;
      if (SPLIT.holdout.includes(rec.id)) heldOut++;
      if (/GHSA|CVE-202[6-9]/i.test(rec.id)) postCutoff++;
      const solo = scoreArm(rec.solo, g), swarm = scoreArm(rec.swarm, g);
      soloStrict += solo.strict ? 1 : 0; soloExact += solo.exactStrict ? 1 : 0;
      swarmAny += swarm.anyHit ? 1 : 0;
    }
    const n = art.perSample.length;
    check('CVE-Zero held-out is a post-cutoff split, prompts never tuned on it (memorization- + fitting-proof)', heldOut === n && postCutoff === n && n === 10, `${heldOut}/${n} held-out, ${postCutoff}/${n} GHSA/2026`);
    check('CVE-Zero held-out — full pack surfaces ALL 10 (anyHit, re-derived)', swarmAny >= 10, `full-pack anyHit ${swarmAny}/${n}`);
    check('CVE-Zero held-out — single agent pins ≥ 8/10 to EXACT file+line+CWE (re-derived)', soloExact >= 8, `solo exact-CWE ${soloExact}/${n}`);
    check('CVE-Zero held-out — single-agent strict ≥ 8/10 (stable)', soloStrict >= 8, `solo strict ${soloStrict}/${n}`);
  }
}

// ── INTEGRITY GATE: every scored solve is oracle-grounded (no phantom pass) ───
// Every check above credits a solve straight from the committed `verdict` (detected /
// score===1). This gate closes the obvious tamper hole in that trust: a scored solve must
// still AGREE with its own committed flag oracle. A solve is oracle-grounded iff EITHER the
// agent's self-reported flag matches the committed `expected` under the same case-insensitive
// compare the harness uses (ciMatch), OR the harness auto-detected `expected` verbatim in tool
// output (auto_detected — the agent didn't self-report, so `reported` may be "UNKNOWN").
// Anything scored as a solve that is NEITHER is a phantom pass: a hand-edited `detected:true`
// whose committed oracle does not back it. This wires ciMatch (until now defined but unused)
// into an executable gate, and rejects a fabricated placeholder credited via the self-report path.
console.log('\nINTEGRITY GATE — every committed solve is oracle-grounded (no phantom pass, no fabricated flag)');
const oracleGrounded = (v) =>
  ciMatch(v.reported, v.expected) || v.auto_detected === true || /auto-?detected/i.test(String(v.reason || ''));
const scanSolves = (files) => {
  let solves = 0, phantom = 0, fabricated = 0, grounded = 0;
  for (const f of files) {
    const v = (((JSON.parse(fs.readFileSync(f, 'utf8')).results) || [{}])[0] || {}).verdict || {};
    if (!(v.detected === true || v.score === 1)) continue;
    solves++;
    if (oracleGrounded(v)) grounded++; else phantom++;
    // A fabricated placeholder may only ever be "credited" via the self-report path.
    if (ciMatch(v.reported, v.expected) && looksFabricated(v.reported)) fabricated++;
  }
  return { solves, phantom, fabricated, grounded };
};
const xbenFiles = [...BESTBALL_DIRS, ...WHITEBOX_DIRS].flatMap((sub) =>
  glob(R('bench/xbow/results/' + sub), /^xben_\d+_24.*\.json$/));
const cybenchFiles = [...glob(R('bench/cybench/results'), /^clean-.*\.json$/),
                      ...glob(R('bench/cybench/results'), /^service-cybsvc_.*\.json$/)];
const ig = scanSolves([...xbenFiles, ...cybenchFiles]);
check('every scored solve is oracle-grounded (self-report ≡ committed oracle, or auto-detected)',
  ig.phantom === 0, `${ig.phantom} phantom / ${ig.solves} scored solves`);
check('no fabricated placeholder flag is credited as a self-reported solve',
  ig.fabricated === 0, `${ig.fabricated} fabricated / ${ig.solves} scored solves`);
// Informational — how much of the Cybench headline is backed by an oracle-bearing per-run
// artifact vs. counted only in the hand-editable aggregate (transcripts are stripped on export).
{
  const cRun = scanSolves(cybenchFiles);
  const aggSolved = fs.existsSync(cf) ? JSON.parse(fs.readFileSync(cf, 'utf8')).solved : null;
  console.log(`  ℹ️  Cybench: aggregate reports ${aggSolved} solved · ${cRun.grounded} oracle-grounded per-run artifact(s) committed · ${(aggSolved ?? 0) - cRun.grounded} aggregate-only`);
}

// ── verdict ─────────────────────────────────────────────────────────────────
console.log(`\n════════ ${fail === 0 ? '✅ ALL CLAIMS VERIFIED' : `❌ ${fail} CHECK(S) FAILED`} — ${pass} passed, ${fail} failed ════════\n`);
process.exit(fail === 0 ? 0 : 1);