import { test } from "node:test"; import assert from "node:assert/strict"; import { mkdtemp, mkdir, writeFile, readFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import YAML from "yaml"; import { REPORT_SECTIONS, validateReport } from "../src/report.js"; import { sanitizeReportPlan, sectionSchema, validateSection, assembleReport, resolveReportInputs, generateReport, MAX_CUSTOM_SECTIONS, } from "../src/ai/report.js"; async function fixtureDir(t) { const dir = await mkdtemp(path.join(tmpdir(), "forge-reptgen-")); t.after(() => import("node:fs/promises").then(({ rm }) => rm(dir, { recursive: true, force: true }))); return dir; } async function writeDeckFiles(dir, { withResearch = true } = {}) { await mkdir(dir, { recursive: true }); await writeFile(path.join(dir, "meta.yaml"), YAML.stringify({ slug: "fixture", brief: "Compare GPU architecture: NVIDIA vs AMD", status: "ready", })); await writeFile(path.join(dir, "plan.yaml"), YAML.stringify({ title: "GPU Architectures: NVIDIA vs AMD", sections: ["Introduction", "Architecture", "Application"], slides: [{ type: "title", purpose: "open", section: 0 }], })); if (withResearch) { await mkdir(path.join(dir, "research"), { recursive: true }); await writeFile( path.join(dir, "research", "notes.md"), "## NVIDIA Ada Lovelace\nNVIDIA RTX 4090 packs 128 SMs totalling 16,384 CUDA cores.\n\n## AMD RDNA 3\nAMD RX 7900 XTX packs 96 CUs totalling 6,144 stream processors.", ); } } /** A model stand-in: inspects the schema to decide what section it is writing * and returns content that obeys the grammar, recording the research text it * was handed. */ function fakeChat(log) { return async ({ schema, messages }) => { const user = messages.at(-1).content; if (schema.properties?.sections) { return { data: { title: "GPU Architectures: NVIDIA vs AMD — A Comparative Study", subtitle: "Two bets on parallelism", sections: [ { name: "Abstract", focus: "Summarise the comparison." }, { name: "Introduction", focus: "Motivate the study." }, { name: "Theoretical Background", focus: "Explain the two execution models." }, { name: "Application", focus: "Map the architectures to real use." }, { name: "Conclusion", focus: "Weigh the two bets." }, { name: "References", focus: "List the sources." }, ], } }; } log.push(user); if (schema.required?.includes("entries")) { return { data: { entries: ["1. NVIDIA. (2022). Ada Lovelace Architecture Whitepaper.", "2. AMD. (2022). RDNA 3 Architecture Whitepaper.", "3. Kirk & Hwu. (2016). Programming Massively Parallel Processors.", "4. Hennessy & Patterson. (2019). Computer Architecture."] } }; } if (schema.properties?.paragraphs?.items?.maxLength < 1000) { return { data: { paragraphs: ["A headline statement of the section.", "One supporting sentence.", "A second supporting sentence.", "A third supporting sentence."] } }; } return { data: { paragraphs: ["First substantial paragraph.", "Second substantial paragraph.", "Third substantial paragraph.", "Fourth substantial paragraph."], table: { caption: "Table 1: Architectural comparison.", header: ["Parameter", "NVIDIA", "AMD"], rows: [["Cores", "16,384", "6,144"], ["Model", "SIMT", "SIMD"]] }, } }; }; } /* ------------------------------------------------------------ plan coercion */ test("sanitizeReportPlan snaps structural names onto the graded order, dropping duplicates", () => { const out = sanitizeReportPlan({ title: "T", subtitle: "S", sections: [ { name: "Conclusion", focus: "c" }, { name: "abstract", focus: "lowercase ok" }, { name: "Conclusion", focus: "duplicate dropped" }, { name: "References", focus: "r" }, ], }); assert.deepEqual(out.sections.map((s) => s.name), ["Abstract", "Introduction", "Conclusion", "References"]); }); test("sanitizeReportPlan guarantees the graded core even from an empty plan", () => { const out = sanitizeReportPlan({}); assert.deepEqual(out.sections.map((s) => s.name), ["Abstract", "Introduction", "Conclusion", "References"]); }); test("sanitizeReportPlan sorts into the fixed graded order regardless of input order", () => { const out = sanitizeReportPlan({ sections: [ { name: "Future Scope", focus: "" }, { name: "Introduction", focus: "" }, { name: "Abstract", focus: "" }, ], }); assert.deepEqual(out.sections.map((s) => s.name), ["Abstract", "Introduction", "Future Scope", "Conclusion", "References"]); }); /* ------------------------------------------------- sections a topic earns */ test("a topic-specific section is kept, in the position the plan gave it", () => { const out = sanitizeReportPlan({ sections: [ { name: "Abstract", focus: "" }, { name: "Introduction", focus: "" }, { name: "Application", focus: "" }, { name: "Grid Integration Case Study", focus: "One depot, measured." }, { name: "Future Scope", focus: "" }, { name: "Conclusion", focus: "" }, { name: "References", focus: "" }, ], }); assert.deepEqual(out.sections.map((s) => s.name), [ "Abstract", "Introduction", "Application", "Grid Integration Case Study", "Future Scope", "Conclusion", "References", ]); assert.equal(out.sections.find((s) => s.name === "Grid Integration Case Study").focus, "One depot, measured."); }); test("a custom section is never allowed to displace the closing section", () => { // References last is the format. A plan that puts an appendix after them // must not be able to produce a report that fails on its own structure. const out = sanitizeReportPlan({ sections: [ { name: "Abstract", focus: "" }, { name: "Conclusion", focus: "" }, { name: "References", focus: "" }, { name: "Cost Model Appendix", focus: "" }, ], }); assert.equal(out.sections.at(-1).name, "References"); assert.deepEqual(out.sections.map((s) => s.name), [ "Abstract", "Introduction", "Conclusion", "Cost Model Appendix", "References", ]); }); test("custom sections are capped, and the ones over the cap are dropped", () => { const extra = ["One", "Two", "Three", "Four", "Five"].map((n) => ({ name: `Custom ${n}`, focus: "" })); const out = sanitizeReportPlan({ sections: [{ name: "Introduction", focus: "" }, ...extra], }); const custom = out.sections.filter((s) => s.name.startsWith("Custom ")); assert.equal(custom.length, MAX_CUSTOM_SECTIONS); // The cap keeps the FIRST ones asked for, not an arbitrary subset. assert.deepEqual(custom.map((s) => s.name), ["Custom One", "Custom Two", "Custom Three"]); }); test("a custom section cannot claim the renderer's own appendix name", () => { const out = sanitizeReportPlan({ sections: [{ name: "Introduction", focus: "" }, { name: "Image Credits", focus: "steal it" }], }); assert.ok(!out.sections.some((s) => s.name === "Image Credits")); }); test("sanitizeReportPlan follows a donor's structure, not the default eight", () => { const structure = ["Abstract", "Introduction", "Methodology", "Results", "Discussion", "References"]; const out = sanitizeReportPlan({ sections: [ { name: "results", focus: "lowercase snaps to the donor's spelling" }, { name: "Methodology", focus: "" }, { name: "Ablation Study", focus: "custom, after methodology" }, ], }, { structure }); assert.deepEqual(out.sections.map((s) => s.name), [ "Abstract", "Introduction", "Methodology", "Ablation Study", "Results", "References", ]); // "Theoretical Background" is not this donor's section and is not invented. assert.ok(!out.sections.some((s) => s.name === "Theoretical Background")); }); test("assembleReport records the section order, including sections with no index", () => { const plan = sanitizeReportPlan({ sections: [ { name: "Abstract", focus: "" }, { name: "Application", focus: "" }, { name: "Depot Case Study", focus: "" }, { name: "Conclusion", focus: "" }, { name: "References", focus: "" }, ], }); const data = Object.fromEntries(plan.sections.map((s) => [s.name, { paragraphs: [`${s.name} prose.`] }])); const report = assembleReport(plan, data); assert.deepEqual(report.order, plan.sections.map((s) => s.name)); assert.ok(report.order.includes("Depot Case Study")); }); /* --------------------------------------------------------- depth as grammar */ test("brief depth forbids tables and caps paragraphs short; full depth allows both", () => { const brief = sectionSchema("Theoretical Background", "brief"); const full = sectionSchema("Theoretical Background", "full"); assert.equal(brief.properties.table, undefined); assert.equal(brief.properties.paragraphs.maxItems, 4); assert.equal(brief.properties.paragraphs.items.maxLength, 450); assert.ok(full.properties.table); assert.equal(full.properties.paragraphs.maxItems, 6); assert.equal(full.properties.paragraphs.items.maxLength, 1500); }); test("validateSection: a table is valid at full depth and rejected at brief", () => { const withTable = { paragraphs: ["A.", "B.", "C."], table: { caption: "Table 1.", header: ["X", "Y"], rows: [["1", "2"], ["3", "4"]] }, }; assert.equal(validateSection("Theoretical Background", "full", withTable).ok, true); assert.equal(validateSection("Theoretical Background", "brief", withTable).ok, false); }); test("validateSection: brief paragraphs must be short", () => { const long = { paragraphs: ["fine", "fine", "x".repeat(500)] }; assert.equal(validateSection("Introduction", "brief", long).ok, false); assert.equal(validateSection("Introduction", "full", long).ok, true); }); test("References are entries at any depth; Acknowledgement is short", () => { assert.ok(sectionSchema("References", "full").required.includes("entries")); assert.equal(sectionSchema("References", "brief").properties.table, undefined); assert.ok(sectionSchema("Acknowledgement", "full").required.includes("paragraphs")); assert.equal(sectionSchema("Acknowledgement", "full").properties.table, undefined); }); test("no section schema carries a maxLength at or above 2000 — Ollama's grammar silently dies there", () => { for (const depth of ["full", "brief"]) { for (const name of REPORT_SECTIONS) { const big = []; JSON.stringify(sectionSchema(name, depth), (k, v) => { if (k === "maxLength" && v >= 2000) big.push(`${name}/${depth} maxLength ${v}`); return v; }); assert.deepEqual(big, []); } } }); /* ------------------------------------------------------------- assembly */ test("assembleReport keeps only sections that produced valid content", () => { const plan = sanitizeReportPlan({ sections: [{ name: "Abstract", focus: "" }, { name: "Conclusion", focus: "" }] }); const report = assembleReport(plan, { Abstract: { paragraphs: ["A."] } }); assert.deepEqual(Object.keys(report.content), ["Abstract"]); assert.equal(report.content.Abstract.paragraphs[0], "A."); }); test("an assembled report validates against report.schema.json", async () => { const plan = sanitizeReportPlan({ sections: [{ name: "Abstract", focus: "" }] }); const report = assembleReport(plan, { Abstract: { paragraphs: ["A."] } }); const { ok, errors } = await validateReport(report); assert.equal(ok, true, errors.join("; ")); }); /* ------------------------------------------------- input resolution */ test("resolveReportInputs reads the shared research/notes.md from the deck dir", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir); const { meta, plan, research } = await resolveReportInputs(dir); assert.equal(meta.brief, "Compare GPU architecture: NVIDIA vs AMD"); assert.equal(plan.title, "GPU Architectures: NVIDIA vs AMD"); assert.match(research, /16,384 CUDA cores/); }); test("resolveReportInputs errors clearly when research is missing", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir, { withResearch: false }); await assert.rejects(resolveReportInputs(dir), /research\/notes\.md.*--research/); }); test("resolveReportInputs falls back to deck.yaml slides when research is missing", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir, { withResearch: false }); await writeFile(path.join(dir, "deck.yaml"), YAML.stringify({ title: "GPU Architectures: NVIDIA vs AMD", slides: [ { type: "title", headline: "GPU Architectures" }, { type: "bullets", headline: "NVIDIA packs 16,384 CUDA cores", bullets: ["Point one here.", "Point two here.", "Point three here.", "Point four here."] }, ], }), "utf8"); const { research } = await resolveReportInputs(dir); assert.match(research, /16,384 CUDA cores/); }); test("resolveReportInputs errors clearly when the outline gate was skipped", async (t) => { const dir = await fixtureDir(t); await mkdir(dir, { recursive: true }); await writeFile(path.join(dir, "meta.yaml"), YAML.stringify({ brief: "x" })); await assert.rejects(resolveReportInputs(dir), /plan\.yaml.*outline gate/); }); test("resolveReportInputs with requirePlan:false needs no plan.yaml — the standalone report", async (t) => { const dir = await fixtureDir(t); await mkdir(dir, { recursive: true }); await writeFile(path.join(dir, "meta.yaml"), YAML.stringify({ brief: "Standalone topic" })); await mkdir(path.join(dir, "research"), { recursive: true }); await writeFile(path.join(dir, "research", "notes.md"), "## Source\nFacts for the report."); const { meta, plan, research } = await resolveReportInputs(dir, { requirePlan: false }); assert.equal(meta.brief, "Standalone topic"); assert.deepEqual(plan, { title: "", sections: [] }); assert.match(research, /Facts for the report/); }); /* --------------------------------------- report size vs the team that writes it */ /** A model stand-in that OBEYS the plan grammar it is handed. * * `fakeChat` above returns six sections whatever the schema says, and every * generateReport test used it — so no test could see a `maxItems` that the * real, grammar-constrained model would have been held to. A stub that * ignores the bound tests the stub. This one spends its budget the way the * prompt directs: the graded core first, then the body sections. */ function obedientChat(planSchemas = []) { const CORE = ["Abstract", "Introduction", "Conclusion", "References"]; const order = [...CORE, ...REPORT_SECTIONS.filter((s) => !CORE.includes(s))]; return async ({ schema }) => { if (schema.properties?.sections) { planSchemas.push(schema); const max = schema.properties.sections.maxItems ?? REPORT_SECTIONS.length; return { data: { title: "T", sections: order.slice(0, max).map((name) => ({ name, focus: "f" })) } }; } // Obedience has to extend to the section grammars too: Acknowledgement // allows at most two paragraphs, so a stub that always returns four has // its Acknowledgement dropped and the test reads that as a planning bug. const fill = (spec, text) => Array.from({ length: Math.max(spec.minItems ?? 1, Math.min(spec.maxItems ?? 4, 4)) }, (_, i) => `${text} ${i + 1}.`); if (schema.required?.includes("entries")) { return { data: { entries: fill(schema.properties.entries, "Source") } }; } return { data: { paragraphs: fill(schema.properties.paragraphs, "Paragraph") } }; }; } async function withTeam(dir, members) { await writeFile(path.join(dir, "meta.yaml"), YAML.stringify({ slug: "fixture", brief: "x", status: "ready", team: { label: "G", members: members.map((name) => ({ name, presenting: true })) }, })); } test("a solo author's report still carries the substance sections", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir); await withTeam(dir, []); const planSchemas = []; const r = await generateReport({ dir, depth: "brief", chat: obedientChat(planSchemas) }); // The budget was the TOTAL section count floored at four, while the prompt // also requires the four-section graded core — so one author could plan // nothing but the frame and the report had no body at all. assert.equal(planSchemas[0].properties.sections.maxItems, REPORT_SECTIONS.length + MAX_CUSTOM_SECTIONS); const body = ["Theoretical Background", "Application", "Future Scope"]; assert.ok( body.some((s) => r.sections.includes(s)), `a report with no body section: ${r.sections.join(", ")}`, ); }); test("the report's section budget does not shrink with the team", async (t) => { // A talk is divided between the people giving it; a report's structure is // set by the institution grading it. One author writing up a large project // owes the same sections as six. const sizes = [[], ["A"], ["A", "B"], ["A", "B", "C", "D"], Array.from({ length: 11 }, (_, i) => `M${i}`)]; for (const members of sizes) { const dir = await fixtureDir(t); await writeDeckFiles(dir); await withTeam(dir, members); const planSchemas = []; const r = await generateReport({ dir, depth: "brief", chat: obedientChat(planSchemas) }); assert.equal( planSchemas[0].properties.sections.maxItems, REPORT_SECTIONS.length + MAX_CUSTOM_SECTIONS, `team of ${members.length} was handed a tightened cap`, ); assert.deepEqual(r.sections, REPORT_SECTIONS, `team of ${members.length}`); } }); test("generateReport with requirePlan:false builds a standalone report with no deck", async (t) => { const dir = await fixtureDir(t); await mkdir(dir, { recursive: true }); await writeFile(path.join(dir, "meta.yaml"), YAML.stringify({ brief: "Standalone topic" })); await mkdir(path.join(dir, "research"), { recursive: true }); await writeFile(path.join(dir, "research", "notes.md"), "## Source\nFacts for the report."); const r = await generateReport({ dir, depth: "full", chat: fakeChat([]), requirePlan: false }); assert.ok(r.sections.length >= 3); const onDisk = YAML.parse(await readFile(r.reportFile, "utf8")); const { ok } = await validateReport(onDisk); assert.equal(ok, true); }); /* ------------------------------------------------------- orchestration */ test("generateReport: full depth writes four paragraphs and a table, from shared research", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir); const seen = []; const r = await generateReport({ dir, depth: "full", chat: fakeChat(seen) }); assert.equal(r.depth, "full"); assert.equal(r.skipped.length, 0); assert.deepEqual(r.sections, ["Abstract", "Introduction", "Theoretical Background", "Application", "Conclusion", "References"]); assert.equal(r.reportFile, path.join(dir, "report.yaml")); // The same research/ dir the deck used fed every section writer. assert.ok(seen.length >= 4); for (const user of seen) assert.match(user, /16,384 CUDA cores/); // Persisted and schema-valid. const onDisk = YAML.parse(await readFile(r.reportFile, "utf8")); const { ok, errors } = await validateReport(onDisk); assert.equal(ok, true, errors.join("; ")); assert.equal(onDisk.content["Theoretical Background"].paragraphs.length, 4); assert.ok(onDisk.content["Theoretical Background"].table); }); test("generateReport: brief depth is visibly shorter — no tables, sentence-level paragraphs", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir); const seen = []; const r = await generateReport({ dir, depth: "brief", chat: fakeChat(seen) }); const onDisk = YAML.parse(await readFile(r.reportFile, "utf8")); for (const name of r.sections) { const sec = onDisk.content[name]; if (name === "References") continue; assert.equal(sec.table, undefined, `${name} must have no table at brief depth`); for (const p of sec.paragraphs) assert.ok(p.length <= 450, `${name} paragraph must be short`); } // And it is visibly lighter than the full report of the same fixture — read // the brief file before the full run overwrites it. const briefBytes = (await readFile(r.reportFile)).length; const full = await generateReport({ dir, depth: "full", chat: fakeChat([]) }); const fullBytes = (await readFile(full.reportFile)).length; assert.ok(briefBytes < fullBytes); }); test("generateReport fails clearly when research was never gathered", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir, { withResearch: false }); await assert.rejects( generateReport({ dir, depth: "full", chat: fakeChat([]) }), /research\/notes\.md/, ); }); test("generateReport rejects an unknown depth before touching the model", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir); await assert.rejects( generateReport({ dir, depth: "verbose", chat: fakeChat([]) }), /depth must be one of full, brief/, ); }); test("generateReport drops a section whose prose violates its depth grammar", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir); let calls = 0; const chat = async ({ schema }) => { if (schema.properties?.sections) { return { data: { title: "T", sections: [{ name: "Abstract", focus: "" }, { name: "Introduction", focus: "" }] } }; } calls++; if (schema.required?.includes("entries")) { return { data: { entries: ["1. S.", "2. S.", "3. S.", "4. S."] } }; } // Fail the Introduction at full depth: a table with a wrong-shaped row. if (calls === 2) return { data: { paragraphs: ["A.", "B.", "C."], table: { header: ["X"], rows: [["bad"]] } } }; return { data: { paragraphs: ["A.", "B.", "C.", "D."], table: { caption: "c", header: ["X", "Y"], rows: [["1", "2"], ["3", "4"]] } } }; }; const r = await generateReport({ dir, depth: "full", chat }); assert.deepEqual(r.sections, ["Abstract", "Conclusion", "References"]); assert.equal(r.skipped.length, 1); assert.equal(r.skipped[0].section, "Introduction"); }); /* ------------------------------------------ what one section knows of another */ test("each section is shown the outline and the prose already written", async (t) => { // "Do not repeat wording already used in another section" was in the system // prompt while the model was shown no other section, so it could not be // followed and near-duplicate sections are what the reports came out with. const dir = await fixtureDir(t); await writeDeckFiles(dir); const seen = []; const chat = async ({ schema, messages }) => { if (schema.properties?.sections) { return { data: { title: "T", sections: [ { name: "Abstract", focus: "Summarise." }, { name: "Introduction", focus: "Motivate." }, { name: "Conclusion", focus: "Close." }, { name: "References", focus: "Cite." }, ] } }; } seen.push(messages.at(-1).content); if (schema.required?.includes("entries")) { return { data: { entries: ["1. S.", "2. S.", "3. S."] } }; } return { data: { paragraphs: [`Prose for a section.`, "B.", "C."] } }; }; await generateReport({ dir, depth: "brief", chat }); // The FIRST section has nothing written yet, and still learns what the // other sections are for — the half no history can supply. assert.match(seen[0], /THE OTHER SECTIONS AND WHAT THEY COVER/); assert.match(seen[0], /Introduction: Motivate\./); assert.ok(!seen[0].includes("ALREADY WRITTEN"), "nothing is written before the first section"); // A section never appears in its own outline. assert.ok(!/- Abstract:/.test(seen[0]), "the Abstract was listed as another section"); // A later section is shown what the earlier ones actually said. assert.match(seen.at(-1), /ALREADY WRITTEN — do not restate these/); assert.match(seen.at(-1), /- Abstract: Prose for a section\./); }); test("a section that failed its grammar is not offered as prose to avoid", async (t) => { const dir = await fixtureDir(t); await writeDeckFiles(dir); const seen = []; const chat = async ({ schema, messages }) => { if (schema.properties?.sections) { return { data: { title: "T", sections: [ { name: "Abstract", focus: "Summarise." }, { name: "Conclusion", focus: "Close." }, { name: "References", focus: "Cite." }, ] } }; } seen.push(messages.at(-1).content); if (schema.required?.includes("entries")) { return { data: { entries: ["1. S.", "2. S.", "3. S."] } }; } // The Abstract comes back over its brief-depth length cap and is dropped. if (seen.length === 1) return { data: { paragraphs: ["x".repeat(600), "B.", "C."] } }; return { data: { paragraphs: ["Real prose.", "B.", "C."] } }; }; const r = await generateReport({ dir, depth: "brief", chat }); assert.ok(!r.sections.includes("Abstract"), "the over-length Abstract should have been dropped"); // The Abstract legitimately appears in every other section's OUTLINE — it // was planned. What must not appear is its prose, under "already written". for (const msg of seen.slice(1)) { const block = msg.split("ALREADY WRITTEN — do not restate these")[1] ?? ""; assert.ok(!block.includes("- Abstract:"), "prose that failed its grammar was fed forward"); } // …and the block is genuinely being built, or the check above proves nothing. const last = seen.at(-1).split("ALREADY WRITTEN — do not restate these")[1] ?? ""; assert.match(last, /- Conclusion: Real prose\./); });