import { createServer, type IncomingMessage, type Server } from "node:http"; import type { AddressInfo } from "node:net"; import { readFileSync, readdirSync } from "node:fs"; import { mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { PlaywrightBrowserPort } from "@jevitate/playwright"; import { FakeGenerationGateway, type GenInput, type GenTaskKind, type GenerationResult } from "@jevitate/ai-core"; import { runExploration } from "./explore-goal.js"; import { ScriptedJudge, useSkippingTime } from "../../explore/src/testkit.js"; // #304: Node and page time skip idle waits; assertions unchanged. // #547: screenshots need the browser's own requestAnimationFrame (the pixel mask), so the page keeps its clock. useSkippingTime({ per: "all", pageClock: false }); /** * #829: a goal-stated credential is in NO written artifact. A change-password page through * `runExploration`, NO `--secret`: the goal states the new password and the run types it. Every * file the run writes (result, Recording, transcript, evidence, logs) and every generator payload is * grepped for the MARKER. Scripted judge + fake generator: no model call. */ const MARKER = "MARKER-pw-829y"; const EMAIL = "morgan@pinecrest.example"; const FORM = `

Change your password

`; const DONE = `

Password changed

`; class CapturingGateway extends FakeGenerationGateway { readonly payloads: string[] = []; override async generate(kind: K, input: GenInput): Promise> { this.payloads.push(JSON.stringify(input)); return super.generate(kind, input); } } let server: Server; let origin: string; const changes: Array = []; function formBody(req: IncomingMessage): Promise { return new Promise((resolve) => { let body = ""; req.on("data", (c: Buffer) => (body += c.toString())); req.on("end", () => resolve(new URLSearchParams(body))); }); } beforeAll(async () => { server = createServer((req, res) => { const html = (status: number, page: string): void => void res.writeHead(status, { "content-type": "text/html; charset=utf-8" }).end(page); const path = (req.url ?? "/").split("?")[0]; if (req.method === "POST" && path === "/change") { void formBody(req).then((f) => { changes.push(f.get("pw")); res.writeHead(303, { location: "/done" }).end(); }); return; } if (path === "/done") return html(200, DONE); html(200, FORM); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); origin = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; }); afterAll(async () => { await new Promise((resolve) => server.close(() => resolve())); }); describe("#829 — a goal-stated password is in no written artifact", () => { it("result, Recording, transcript and every other file the run writes hold the MARKER 0 times; the email still records", async () => { const out = await mkdtemp(join(tmpdir(), "jev-goal-cred-")); try { // Form: [0] Email, [1] New password, [2] Save password. const judge = new ScriptedJudge([{ op: "type", target: "0" }, { op: "type", target: "1" }, { op: "click", target: "2" }, { op: "done" }]); const gen = new CapturingGateway(); const result = await runExploration({ url: `${origin}/`, allowlist: [origin], goal: `Change the password for email ${EMAIL} to new password ${MARKER}.`, successAssertion: { kind: "visible", target: { testId: "changed" } }, judge, gen, outDir: out, bounds: { maxDecisions: 8 }, screenshots: { mode: "steps" }, browserPortFactory: () => new PlaywrightBrowserPort(), }); const r = result as unknown as { goalOutcome: string; resultPath: string }; expect(r.goalOutcome).toBe("succeeded"); // The real value reached the server. expect(changes).toEqual([MARKER]); const files: string[] = []; const walk = (d: string): void => { for (const e of readdirSync(d, { withFileTypes: true })) { const f = join(d, e.name); if (e.isDirectory()) walk(f); else files.push(f); } }; walk(out); const text = files.filter((f) => /\.(json|md|txt|jsonl|log)$/.test(f)); expect(text).toContain(r.resultPath); expect(text.length).toBeGreaterThanOrEqual(3); let total = 0; for (const f of text) total += readFileSync(f, "utf8").split(MARKER).length - 1; expect(total).toBe(0); expect(JSON.stringify(result).split(MARKER).length - 1).toBe(0); // No over-redaction: the email field's value still records, in the result's Recording. expect(readFileSync(r.resultPath, "utf8")).toContain(`"value": "${EMAIL}"`); // The decision states after the password was typed (from the 3rd on) and every goal check. expect(JSON.stringify(judge.calls.slice(2)).split(MARKER).length - 1).toBe(0); expect(JSON.stringify(judge.goalCalls).split(MARKER).length - 1).toBe(0); expect(gen.payloads.filter((p) => p.includes(MARKER))).toEqual([]); } finally { await rm(out, { recursive: true, force: true }); } }, 180_000); });