// Materiality judge exam (Stage 2 noise yardstick, step 1 — NOT a benchmark arm).
//
// Before the judge labels a single engine finding, it must label anchors whose
// answer is already known, and clear the pass bar fixed in
// src/compare/materiality.ts:
// - material: Martian goldens rated High/Critical, all five public corpora
// (benchmarks/martian/corpora/martian-*-goldens.json). Labelled by Martian.
// - immaterial: planted claims that are true but not worth posting
// (docs/benchmarks/materiality-planted-anchors.json).
// - stability: a fixed subset judged three times must keep its label.
// Martian Low style/speculative/doc_defect comments are reported as a
// secondary immaterial signal and never gate.
//
// WHAT A RESULT MEANS
// passed → the judge can label engine findings for the noise baseline.
// material ✗ → it waves real bugs away; any noise number it produced would
// flatter a prompt that finds less.
// immaterial ✗→ it calls nits material; it would hide exactly the noise
// Stage 2 is trying to remove.
// stability ✗ → its labels are coin flips; no single-run number means much.
//
// One isolated spawn per claim: tools [], empty MCP config, model opus (the
// hunters run sonnet, so the judge never grades its own model's output).
//
// LIVE (`run`): spends real money; the result lands in the lab ledger the same
// day (charter rule 6). `plan` and `score` are $0.
//
// bun run scripts/materiality-probe.ts plan
// bun run scripts/materiality-probe.ts run [--out
] [--concurrency ]
// bun run scripts/materiality-probe.ts score
//
// `run` checkpoints every judgment to runs.partial.json next to results.json;
// `score runs.partial.json` rescores an interrupted run.
import { writeFileSync } from "node:fs";
import { mkdir } from "node:fs/promises";
import { homedir } from "node:os";
import path from "node:path";
import type { MartianGoldenPr } from "#compare/martian-adapter";
import {
type Anchor,
type ExamScore,
goldenAnchors,
type JudgeRun,
judgeSystemSha256,
PASS_BAR,
type PlantedAnchor,
plantedAnchors,
scoreExam,
stabilitySubset,
} from "#compare/materiality";
import { JUDGE_MODEL, judgeClaim, judgeRouteFields } from "./materiality-spawn";
const ROOT = path.join(import.meta.dir, "..");
const BENCH = path.join(ROOT, "docs", "benchmarks");
const MARTIAN_CORPORA = path.join(ROOT, "benchmarks", "martian", "corpora");
const CORPORA = ["cal", "discourse", "grafana", "keycloak", "sentry"];
const STABILITY_PER_LABEL = 15;
const REPLICATES = 3;
interface Job {
anchorId: string;
replicate: number;
}
interface ResultsFile {
judge_model: string;
// The judge's route and the plan's fingerprint. Optional: files written
// before the route seam carry judge_model alone.
judge_route?: ReturnType["judge_route"];
judge_route_fingerprint?: string;
judge_system_sha256: string;
pass_bar: typeof PASS_BAR;
started_at: string;
finished_at: string;
cost_usd_est: number;
anchors: Anchor[];
runs: JudgeRun[];
score: ExamScore;
}
function fail(message: string): never {
console.error(`materiality-probe: ${message}`);
process.exit(1);
}
async function loadAnchors(): Promise {
const anchors: Anchor[] = [];
for (const corpus of CORPORA) {
const goldens = (await Bun.file(
path.join(MARTIAN_CORPORA, `martian-${corpus}-goldens.json`),
).json()) as MartianGoldenPr[];
anchors.push(...goldenAnchors(corpus, goldens));
}
const planted = (await Bun.file(
path.join(BENCH, "materiality-planted-anchors.json"),
).json()) as { anchors: PlantedAnchor[] };
anchors.push(...plantedAnchors(planted.anchors));
return anchors;
}
function buildJobs(anchors: Anchor[]): Job[] {
const jobs: Job[] = anchors.map((a) => ({ anchorId: a.id, replicate: 0 }));
for (const id of stabilitySubset(anchors, STABILITY_PER_LABEL)) {
for (let r = 1; r < REPLICATES; r += 1) {
jobs.push({ anchorId: id, replicate: r });
}
}
return jobs;
}
function describeAnchors(anchors: Anchor[], jobs: Job[]): string[] {
const count = (pick: (a: Anchor) => boolean) => anchors.filter(pick).length;
return [
`material (gating): ${count((a) => a.role === "gating" && a.expected === "material")}`,
`immaterial (gating): ${count((a) => a.role === "gating" && a.expected === "immaterial")}`,
`immaterial (secondary): ${count((a) => a.role === "secondary")}`,
`stability subset: ${STABILITY_PER_LABEL} + ${STABILITY_PER_LABEL} anchors x ${REPLICATES} runs`,
`judge spawns: ${jobs.length}`,
`judge model: ${JUDGE_MODEL} (tools [], empty MCP)`,
`judge system sha256: ${judgeSystemSha256()}`,
`pass bar: material >= ${PASS_BAR.material * 100}%, immaterial >= ${PASS_BAR.immaterial * 100}%, stability >= ${PASS_BAR.stability * 100}%`,
];
}
function pct(rate: number | null): string {
return rate === null ? "n/a" : `${(rate * 100).toFixed(1)}%`;
}
function renderScore(score: ExamScore): string[] {
const line = (name: string, r: ExamScore["material"]) =>
`${name.padEnd(22)} ${pct(r.rate).padStart(6)} (${r.hits}/${r.judged}, ${r.failed} failed)`;
return [
line("material", score.material),
line("immaterial", score.immaterial),
line("stability", score.stability),
line("secondary immaterial", score.secondaryImmaterial),
score.passed ? "PASSED" : `FAILED: ${score.reasons.join("; ")}`,
];
}
async function judgeOne(
anchor: Anchor,
replicate: number,
): Promise<{ run: JudgeRun; cost: number }> {
const out = await judgeClaim({
name: `materiality-${anchor.id}-${replicate}`,
prTitle: anchor.prTitle,
claim: anchor.claim,
});
const run: JudgeRun = { anchorId: anchor.id, replicate, label: out.label };
if (out.reason !== undefined) run.reason = out.reason;
if (out.error !== undefined) run.error = out.error;
return { run, cost: out.cost };
}
function flag(args: string[], name: string): string | undefined {
const i = args.indexOf(name);
return i >= 0 ? args[i + 1] : undefined;
}
async function run(args: string[]): Promise {
const anchors = await loadAnchors();
const jobs = buildJobs(anchors);
const concurrency = Number(flag(args, "--concurrency") ?? "4");
if (!Number.isInteger(concurrency) || concurrency < 1) {
fail("--concurrency must be a positive integer");
}
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
const outDir =
flag(args, "--out") ??
path.join(homedir(), "Desktop", "materiality-probe", stamp);
await mkdir(outDir, { recursive: true });
const outPath = path.join(outDir, "results.json");
const partialPath = path.join(outDir, "runs.partial.json");
for (const line of describeAnchors(anchors, jobs)) console.log(line);
console.log(`writing ${outPath}`);
const byId = new Map(anchors.map((a) => [a.id, a]));
const startedAt = new Date().toISOString();
const runs: JudgeRun[] = [];
let cost = 0;
let next = 0;
const worker = async () => {
while (next < jobs.length) {
const job = jobs[next];
next += 1;
if (job === undefined) break;
const anchor = byId.get(job.anchorId);
if (anchor === undefined) continue;
let out: { run: JudgeRun; cost: number };
try {
out = await judgeOne(anchor, job.replicate);
} catch (error) {
out = {
run: {
anchorId: anchor.id,
replicate: job.replicate,
label: null,
error: error instanceof Error ? error.message : String(error),
},
cost: 0,
};
}
runs.push(out.run);
cost += out.cost;
// Checkpoint after every judgment, so a killed run keeps what it paid
// for. Sync on purpose: concurrent workers never interleave a write.
writeFileSync(
partialPath,
`${JSON.stringify({ judge_system_sha256: judgeSystemSha256(), anchors, runs }, null, 2)}\n`,
);
const mark =
out.run.label === null
? "ERR"
: out.run.label === anchor.expected
? "ok "
: "MISS";
console.log(
`[${runs.length}/${jobs.length}] ${mark} ${anchor.id} r${job.replicate} → ${out.run.label ?? out.run.error}`,
);
}
};
await Promise.all(Array.from({ length: concurrency }, worker));
const score = scoreExam(anchors, runs);
const results: ResultsFile = {
judge_model: JUDGE_MODEL,
...judgeRouteFields(),
judge_system_sha256: judgeSystemSha256(),
pass_bar: PASS_BAR,
started_at: startedAt,
finished_at: new Date().toISOString(),
cost_usd_est: cost,
anchors,
runs,
score,
};
await Bun.write(outPath, `${JSON.stringify(results, null, 2)}\n`);
for (const line of renderScore(score)) console.log(line);
console.log(`cost (est): $${cost.toFixed(2)}`);
}
async function score(args: string[]): Promise {
const file = args[0];
if (file === undefined) fail("score needs a results.json path");
const results = (await Bun.file(file).json()) as ResultsFile;
if (results.judge_system_sha256 !== judgeSystemSha256()) {
console.log(
"note: the judge prompt changed since this run; rescoring the recorded labels only",
);
}
for (const line of renderScore(scoreExam(results.anchors, results.runs))) {
console.log(line);
}
}
const [command, ...rest] = process.argv.slice(2);
if (command === "plan") {
const anchors = await loadAnchors();
for (const line of describeAnchors(anchors, buildJobs(anchors))) {
console.log(line);
}
} else if (command === "run") {
await run(rest);
} else if (command === "score") {
await score(rest);
} else {
fail(
"usage: materiality-probe.ts plan | run [--out ] [--concurrency ] | score ",
);
}