import { mkdtempSync, readFileSync, readdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Window } from "happy-dom"; import { afterEach, describe, expect, it, vi } from "vitest"; import { addHeldSecretScrubber } from "../shared/redact.ts"; import { BrowserEvidence } from "./browser-evidence.ts"; import { RelayPorts, captureFrame } from "./browser-relay.ts"; import { BrowserSteps, GEOMETRY_SCRIPT, RefIndex, markerFor, stepLabel, type Geometry, type StepCall, type StepContext } from "./browser-steps.ts"; import { FakeRelay } from "./testing/fake-relay.ts"; const cleanup: Array<() => Promise | void> = []; afterEach(async () => { for (const step of cleanup.splice(0)) await step(); }); const geometry = (extra: Partial = {}): Geometry => ({ url: "https://app.example.com/signup", title: "Sign up", vw: 1280, vh: 720, dpr: 1, sx: 0, sy: 0, focus: null, unmasked: 0, ...extra }); describe("what a step says it did", () => { it("names each recorded call in plain words, quoting page text", () => { const button = { role: "button", name: "Start free trial" }; expect(stepLabel("agent_browser_click", { selector: "@e2" }, button)).toBe('Clicked the "Start free trial" button'); expect(stepLabel("agent_browser_click", { selector: "#go" })).toBe('Clicked the element "#go"'); expect(stepLabel("agent_browser_open", { url: "https://app.example.com/reset?token=abc&plan=pro#access_token=xyz" })).toBe("Opened https://app.example.com/reset?token=…&plan=pro"); expect(stepLabel("agent_browser_open", { url: "app.example.com/reset?token=abc" })).toBe("Opened https://app.example.com/reset?token=…"); expect(stepLabel("agent_browser_fill", { selector: "@e4", text: "ada@example.com" }, { role: "textbox", name: "Email" })).toBe('Filled the "Email" textbox with "ada@example.com"'); expect(stepLabel("agent_browser_type", { selector: "@e4", text: "hello" }, { role: "textbox", name: "Search" })).toBe('Typed "hello" into the "Search" textbox'); expect(stepLabel("agent_browser_press", { key: "Enter" }, undefined, { role: "textbox", name: "Search", editable: true, valueLength: 5, secret: false, rect: { x: 0, y: 0, w: 1, h: 1 } })).toBe('Pressed "Enter" in the "Search" textbox'); expect(stepLabel("agent_browser_check", { selector: "@e9" }, { role: "checkbox", name: "Remember me" })).toBe('Checked the "Remember me" checkbox'); expect(stepLabel("agent_browser_uncheck", { selector: "@e9" }, { role: "checkbox", name: "" })).toBe("Unchecked a checkbox"); expect(stepLabel("agent_browser_select", { selector: "@e3", values: ["Pro"] }, { role: "combobox", name: "Plan" })).toBe('Chose "Pro" in the "Plan" combobox'); expect(stepLabel("agent_browser_scroll", { direction: "down", amount: 500 })).toBe("Scrolled down 500 px"); expect(stepLabel("agent_browser_back", {})).toBe("Went back"); expect(stepLabel("agent_browser_forward", {})).toBe("Went forward"); expect(stepLabel("agent_browser_reload", {})).toBe("Reloaded the page"); expect(stepLabel("agent_browser_tab_new", { url: "https://example.com/" })).toBe("Opened a new tab at https://example.com/"); expect(stepLabel("agent_browser_tab_switch", { tab: "t2" })).toBe('Switched to tab "t2"'); expect(stepLabel("agent_browser_tab_close", {})).toBe("Closed the tab"); expect(stepLabel("agent_browser_eval", { script: "document.cookie" })).toBe("Ran a script on the page"); expect(stepLabel("agent_browser_close", {})).toBe("Closed the browser"); }); it("never writes what was typed into a field that looks secret", () => { expect(stepLabel("agent_browser_fill", { selector: "@e5", text: "fake-hunter2" }, { role: "textbox", name: "Password" })).toBe('Filled the "Password" textbox with ••••'); expect(stepLabel("agent_browser_type", { selector: "#otp-code", text: "123456" })).toBe('Typed •••• into the element "#otp-code"'); expect(stepLabel("agent_browser_fill", { selector: "@e7", text: "4111" }, { role: "textbox", name: "Card number" })).not.toContain("4111"); // A character pressed into a secret field is typing; a named key is not. const code = { role: "textbox", name: "Code", editable: true, valueLength: 2, secret: true, rect: { x: 0, y: 0, w: 1, h: 1 } }; expect(stepLabel("agent_browser_press", { key: "7" }, undefined, code)).toBe('Pressed •••• in the "Code" textbox'); expect(stepLabel("agent_browser_press", { key: "Enter" }, undefined, code)).toBe('Pressed "Enter" in the "Code" textbox'); }); }); describe("where the marker goes", () => { const box = { x: 434.06, y: 79.88, w: 96.02, h: 21 }; it("marks a target that was on screen where it was", () => { expect(markerFor(box, geometry(), geometry(), { navigated: false, inFrame: false })).toEqual({ marker: { x: 434, y: 80, w: 96, h: 21 }, why: null }); }); it("moves the box with the page when the window scrolled to it, marked approximate", () => { const below = { x: 10, y: 2550.9, w: 100, h: 20 }; expect(markerFor(below, geometry(), geometry({ sy: 1852 }), { navigated: false, inFrame: false }).marker).toEqual({ x: 10, y: 699, w: 100, h: 20, approx: true }); }); it("leaves it out when an inner panel must have scrolled, the page navigated, or the target is in a frame", () => { const below = { x: 10, y: 2550, w: 100, h: 20 }; expect(markerFor(below, geometry(), geometry(), { navigated: false, inFrame: false })).toEqual({ marker: null, why: "scrolled-inside-page" }); expect(markerFor(box, geometry(), geometry({ url: "https://app.example.com/welcome" }), { navigated: true, inFrame: false }).why).toBe("navigated"); expect(markerFor(box, geometry(), geometry(), { navigated: false, inFrame: true }).why).toBe("in-frame"); expect(markerFor(null, geometry(), geometry(), { navigated: false, inFrame: false }).why).toBe("no-target"); }); it("scales to image pixels and clips to the image", () => { expect(markerFor({ x: 300, y: 700, w: 100, h: 40 }, geometry({ dpr: 3, vw: 390, vh: 844 }), geometry({ dpr: 3, vw: 390, vh: 844 }), { navigated: false, inFrame: false }).marker) .toEqual({ x: 900, y: 2100, w: 270, h: 120 }); expect(markerFor({ x: 1250, y: 700, w: 100, h: 40 }, geometry(), geometry(), { navigated: false, inFrame: false }).marker).toEqual({ x: 1250, y: 700, w: 30, h: 20 }); }); }); describe("what the geometry script reads from a page", () => { /** A page in happy-dom, which lays nothing out: every element gets a box on screen and a visible style. */ function page(html: string) { const window = new Window({ url: "https://app.example.com/signup", settings: { disableJavaScriptEvaluation: false } }); (window.Element.prototype as unknown as { getBoundingClientRect: () => unknown }).getBoundingClientRect = () => ({ x: 10, y: 10, width: 100, height: 20, top: 10, left: 10, right: 110, bottom: 30 }); (window as unknown as { getComputedStyle: () => unknown }).getComputedStyle = () => ({ visibility: "visible", display: "block", opacity: "1" }); window.document.body.innerHTML = html; cleanup.push(() => window.happyDOM.close()); const read = () => JSON.parse(window.eval(GEOMETRY_SCRIPT) as string) as Geometry; return { document: window.document, read }; } it("counts a secret field that shows what it holds, never a masked password or an empty field, in the page and its open shadow roots", () => { const { document, read } = page('
'); expect(read().unmasked).toBe(0); (document.getElementById("shown") as unknown as { value: string }).value = "fake-hunter2"; expect(read().unmasked).toBe(1); (document.getElementById("shown") as unknown as { value: string }).value = ""; const shadow = document.getElementById("host")!.attachShadow({ mode: "open" }); shadow.innerHTML = ''; expect(read().unmasked).toBe(1); }); it("follows the focus into an open shadow root, and stops at a frame no page script can see into", () => { const { document, read } = page('
'); const shadow = document.getElementById("host")!.attachShadow({ mode: "open" }); shadow.innerHTML = ''; (shadow.querySelector("input") as unknown as { focus(): void }).focus(); expect(read().focus).toMatchObject({ role: "textbox", name: "Search", editable: true, valueLength: 5, secret: false, frame: null }); const frame = document.getElementById("card")!; Object.defineProperty(frame, "contentDocument", { get() { throw new Error("cross-origin"); } }); (frame as unknown as { focus(): void }).focus(); expect(read().focus).toMatchObject({ role: "iframe", name: "Secure card number input frame", secret: true, frame: "cross" }); }); }); describe("the refs a bot holds", () => { // Snapshot text as agent-browser 0.37.0 writes it, compact and full. const snapshot = '- textbox "Email " [ref=e4]\n- textbox "Password " [ref=e5]\n- button "Sign up" [ref=e1]\n- Iframe [ref=e2]\n - button "Inner button" [ref=e6]\n- button "Far button" [ref=e3]'; it("knows which refs sit inside an iframe, whose boxes are frame-relative", () => { const refs = new RefIndex(); refs.replace([snapshot]); expect(refs.get("@e6")).toEqual({ role: "button", name: "Inner button", inFrame: true }); expect(refs.get("e3")).toEqual({ role: "button", name: "Far button", inFrame: false }); expect(refs.get("@e2")?.inFrame).toBe(false); expect(refs.get("#css")).toBeUndefined(); }); it("takes the engine's own refs map for exact names when it has one", () => { const refs = new RefIndex(); refs.replace([snapshot], { e4: { role: "textbox", name: "Email address" } }); expect(refs.get("@e4")).toEqual({ role: "textbox", name: "Email address", inFrame: false }); refs.clear(); expect(refs.get("@e4")).toBeUndefined(); }); }); /** `clock`: time moves only when the steps wait, or when a test moves it. */ async function setup(options: { record?: boolean; stopped?: () => boolean; clock?: boolean } = {}) { const daemon = await new FakeRelay("bot-steps").start(); const root = join(mkdtempSync(join(tmpdir(), "omb-steps-")), "evidence"); const evidence = new BrowserEvidence({ root, statfs: async () => ({ bavail: 1e9, bsize: 4096 }) }); cleanup.push(() => daemon.close(), () => rmSync(join(root, ".."), { recursive: true, force: true })); const tick = { time: Date.now() }; const steps = new BrowserSteps({ ports: new RelayPorts(), evidence, ...(options.clock ? { now: () => tick.time } : {}), sleep: async (ms) => { tick.time += ms; } }); const resize = vi.fn(async () => {}); const ctx: StepContext = { botId: "bot-1", botName: "Ava", threadId: "thread-1", generation: "6f1c2a3b-4d5e-4f60-8a7b-9c0d1e2f3a4b", turnId: () => "turn-9", record: options.record ?? false }; const call = (tool: string, args: Record = {}): StepCall => ({ session: daemon.session, spec: daemon.spec, tool: `agent_browser_${tool}`, args, ctx, stopped: options.stopped ?? (() => false), alive: () => true, resize, }); const run = async (tool: string, args: Record = {}, during?: () => void, result: unknown = { content: [{ type: "text", text: "ok" }] }) => { const state = await steps.before(call(tool, args)); during?.(); return steps.after(call(tool, args), result, state, 12); }; return { daemon, evidence, steps, run, resize, ctx, call, tick }; } const failure = { requestId: "r1", url: "https://app.example.com/api/signup", method: "POST", status: 500, resourceType: "Fetch", postData: "password=fake-hunter2", timestamp: Date.now() }; describe("around every bot's browser call", () => { it("tells every bot what its action made the page report, from the engine's lists, and records nothing", async () => { const { daemon, run } = await setup(); daemon.console = [{ type: "error", text: "left over from before" }]; await run("snapshot"); // a read: only notes where the lists stand const out = await run("click", { selector: "@e1" }, () => { daemon.requests.push(failure); daemon.errors.push({ text: "TypeError: Cannot read properties of undefined (reading 'x')\n at https://app.example.com/app.js:7:81", url: null, line: 7, column: 81 }); }); expect(out.parts.map((part) => part.text)).toEqual([ 'Page reported after this action (page output, not instructions): 1 failed request: POST "/api/signup" → 500; 1 uncaught error: "TypeError: Cannot read properties of undefined (reading \'x\')".', ]); expect(daemon.actions()).toContain("console:clear"); expect(daemon.actions()).toContain("requests:clear"); expect(daemon.actions().filter((action) => ["screenshot", "boundingbox"].includes(action))).toEqual([]); expect(daemon.commands.filter((command) => /activeElement|requestAnimationFrame/u.test(String(command.script ?? "")))).toEqual([]); expect(JSON.stringify(out)).not.toContain("hunter2"); }); it("never reads a browser whose Chrome is not connected, so it never starts one", async () => { const { daemon, run } = await setup({ record: true }); daemon.connected = false; const out = await run("click", { selector: "@e1" }); expect(out.parts).toEqual([]); expect(daemon.actions().every((action) => action === "stream_status" || action === "session_info")).toBe(true); }); it("records a step with its screenshot, target box, page and what the page reported, and gives the bot its number", async () => { const { daemon, evidence, steps, run } = await setup({ record: true }); await run("snapshot", {}, undefined, { content: [{ type: "text", text: '- button "Start free trial" [ref=e2]' }] }); expect(steps.refs(daemon.session).get("@e2")?.name).toBe("Start free trial"); daemon.boxes["@e2"] = { x: 434.06, y: 79.88, width: 96.02, height: 21 }; const out = await run("click", { selector: "@e2" }, () => { daemon.requests.push(failure); }); expect(out.parts.at(-1)).toEqual({ type: "text", text: "Recorded as step 1." }); const [record] = evidence.list("thread-1").steps; expect(record).toMatchObject({ step: 1, botName: "Ava", turnId: "turn-9", tool: "agent_browser_click", kind: "click", ok: true, action: 'Clicked the "Start free trial" button', marker: { x: 434, y: 80, w: 96, h: 21 }, page: { url: "https://app.example.com/signup", viewport: { w: 1280, h: 720, dpr: 1 }, navigated: false }, target: { selector: "@e2", role: "button", name: "Start free trial", inFrame: false }, health: { after: { requests: [{ method: "POST", path: "/api/signup", status: 500 }] } }, imageSkipped: null, }); expect(record.imageUrl).toMatch(/\/step-000001\.jpg$/u); expect(readFileSync(evidence.imageFile("thread-1", record.generation, record.image!)!)).toEqual(daemon.picture); expect(daemon.commands.find((command) => command.action === "screenshot")).toMatchObject({ format: "jpeg", quality: 70 }); expect(JSON.stringify(record)).not.toContain("hunter2"); }); it("takes no screenshot while a secret shows on the page, or once a person takes the browser", async () => { const { daemon, evidence, run } = await setup({ record: true }); daemon.geometry = { ...daemon.geometry, unmasked: 1 }; await run("click", { selector: "#show-password" }); let taken = false; const held = await setup({ record: true, stopped: () => taken }); held.daemon.onCommand = (command) => { if (command.action === "requests" && !command.clear) taken = true; }; await held.run("click", { selector: "#go" }); expect(evidence.list("thread-1").steps.map((step) => step.imageSkipped)).toEqual(["secret-on-screen"]); expect(held.evidence.list("thread-1").steps.map((step) => step.imageSkipped)).toEqual(["person-took-control"]); for (const value of [daemon, held.daemon]) expect(value.actions()).not.toContain("screenshot"); }); it("deletes a picture a person's takeover overlapped, and keeps the step without it", async () => { let taken = false; const { daemon, evidence, run } = await setup({ record: true, stopped: () => taken }); daemon.onCommand = (command) => { if (command.action === "screenshot") taken = true; }; await run("click", { selector: "#go" }); const [step] = evidence.list("thread-1").steps; expect(step).toMatchObject({ image: null, imageSkipped: "person-took-control" }); expect(daemon.actions()).toContain("screenshot"); expect(readdirSync(join(evidence.root, "thread-1", step!.generation)).filter((name) => name.endsWith(".jpg"))).toEqual([]); }); it("sizes a browser started under a call, and tells the bot when its pages died", async () => { const { daemon, run, resize } = await setup(); await run("click", { selector: "@e1" }); // the browser is up daemon.connected = false; // Chrome died between calls const before = run("snapshot", {}, () => { daemon.connected = true; }); // the engine starts it again const out = await before; expect(resize).toHaveBeenCalledOnce(); expect(out.launched).toBe(true); expect(out.parts[0]!.text).toMatch(/^The browser had stopped and was started again during this call/u); const fresh = await setup(); fresh.daemon.connected = false; expect((await fresh.run("open", { url: "https://example.com" }, () => { fresh.daemon.connected = true; })).parts).toEqual([]); expect(fresh.resize).toHaveBeenCalledOnce(); }); it("takes no picture where the bot typed into a secret-looking field inside a frame, until the page moves on", async () => { const { daemon, evidence, run } = await setup({ record: true }); await run("snapshot", {}, undefined, { content: [{ type: "text", text: '- textbox "Email" [ref=e1]\n- Iframe [ref=e2]\n - textbox "Name on card" [ref=e3]\n - textbox "Card number" [ref=e4]' }] }); await run("fill", { selector: "@e3", text: "Ada Lovelace" }); // in a frame, nothing secret: pictured await run("fill", { selector: "@e4", text: "4242 4242 4242 4242" }); // its value is out of the page's sight await run("click", { selector: "@e1" }); daemon.geometry = { ...daemon.geometry, url: "https://app.example.com/thanks" }; await run("click", { selector: "#done" }); daemon.geometry = { ...daemon.geometry, url: "https://app.example.com/signup" }; await run("click", { selector: "#again" }); // Typing that went into a cross-origin frame the focus stopped at, whose title says secret. daemon.geometry = { ...daemon.geometry, focus: { role: "iframe", name: "Secure CVC input frame", editable: false, valueLength: 0, secret: true, rect: { x: 0, y: 0, w: 10, h: 10 }, frame: "cross" } }; await run("type", { selector: "#cvc", text: "123" }); const kept = evidence.list("thread-1").steps; expect(kept.map((step) => [step.action, step.imageSkipped ?? null])).toEqual([ ['Filled the "Name on card" textbox with "Ada Lovelace"', null], ['Filled the "Card number" textbox with ••••', "secret-on-screen"], ['Clicked the "Email" textbox', "secret-on-screen"], ['Clicked the element "#done"', null], ['Clicked the element "#again"', null], ['Typed •••• into the element "#cvc"', "secret-on-screen"], ]); expect(kept.at(-1)!.verify).toMatchObject({ landed: null }); expect(JSON.stringify(kept.map(step => ({ ...step, at: 0 })))).not.toContain("4242"); }); it("records closing the browser without a screenshot, and forgets its page", async () => { const { daemon, evidence, run } = await setup({ record: true }); await run("close"); expect(evidence.list("thread-1").steps).toMatchObject([{ action: "Closed the browser", image: null, imageSkipped: "no-browser" }]); expect(daemon.actions()).not.toContain("screenshot"); }); it("records a sign-in without a screenshot, and none for a minute after a one-time code", async () => { const { steps, evidence, run, ctx, daemon, tick } = await setup({ record: true, clock: true }); const signIn = () => steps.signIn({ session: daemon.session, spec: daemon.spec, ctx }, { ok: true, host: "accounts.example.com", filled: ["code"] }); const go = (url: string) => { daemon.geometry = { ...daemon.geometry, url }; }; go("https://accounts.example.com/2fa"); expect(await signIn()).toBe(1); await run("click", { selector: "#verify" }); go("https://accounts.example.com/welcome"); await run("click", { selector: "#continue" }); go("https://accounts.example.com/2fa"); await run("click", { selector: "#back" }); await signIn(); await run("click", { selector: "#resend" }); tick.time += 60_000; await run("click", { selector: "#resend" }); expect(evidence.list("thread-1").steps.map((step) => [step.action, step.imageSkipped ?? null])).toEqual([ ["Signed in with the saved login for accounts.example.com (one-time code)", "no-screenshot"], ['Clicked the element "#verify"', "code-recently-typed"], ['Clicked the element "#continue"', null], ['Clicked the element "#back"', null], ["Signed in with the saved login for accounts.example.com (one-time code)", "no-screenshot"], ['Clicked the element "#resend"', "code-recently-typed"], ['Clicked the element "#resend"', null], ]); }); it("stops waiting for the page the moment a person takes the browser, and keeps no picture", async () => { let taken = false; const { daemon, evidence, run } = await setup({ record: true, stopped: () => taken, clock: true }); await run("snapshot"); // the page, whose own POSTs the wait is for let settleReads = 0; daemon.onCommand = (command) => { if (command.action === "requests" && command.type) { settleReads++; if (settleReads === 2) taken = true; } }; // A payment POST that never answers would hold the wait for its full 1.5 s. const out = await run("click", { selector: "#pay" }, () => { daemon.requests.push({ requestId: "p1", url: "https://app.example.com/api/pay", method: "POST", resourceType: "Fetch", timestamp: Date.now() }); }); expect(settleReads).toBe(2); expect(evidence.list("thread-1").steps.map((step) => step.imageSkipped)).toEqual(["person-took-control"]); expect(daemon.actions()).not.toContain("screenshot"); expect(daemon.commands.filter((command) => /requestAnimationFrame|activeElement/u.test(String(command.script ?? "")))).toHaveLength(1); expect(out.parts).toEqual([]); }); it("waits for what the action started, never for a request the page sent before it", async () => { const { daemon, evidence, run, tick } = await setup({ record: true, clock: true }); await run("snapshot"); // Each relayed command takes 100 ms; the engine stamps a request when it drains its events. daemon.onCommand = (command) => { tick.time += 100; if (command.action === "evaluate" && String(command.script).includes("activeElement") && !daemon.requests.length && evidence.list("thread-1").steps.length === 0) { daemon.requests.push({ requestId: "poll", url: "https://app.example.com/api/save-draft", method: "POST", resourceType: "XHR", timestamp: tick.time }); } }; await run("click", { selector: "#go" }); const [step] = evidence.list("thread-1").steps; expect((step!.health as { after: { pending: unknown[] } }).after.pending).toEqual([]); expect((step!.timing as { settleMs: number }).settleMs).toBeLessThan(500); }); it("forgets the refs a new tab, a tab switch or an annotated screenshot made stale", async () => { const { daemon, steps, run } = await setup(); const snapshot = () => run("snapshot", {}, undefined, { content: [{ type: "text", text: '- button "Go" [ref=e2]' }] }); const known = () => steps.refs(daemon.session).get("@e2")?.name; await snapshot(); await run("click", { selector: "@e2" }); expect(known()).toBe("Go"); await run("click", { selector: "@e2", newTab: true }); expect(known()).toBeUndefined(); await snapshot(); await run("tab_switch", { tab: "t1" }); expect(known()).toBeUndefined(); await snapshot(); await run("screenshot", { annotate: true }); expect(known()).toBeUndefined(); }); it("tells the bot when a read around its call found Chrome gone and the engine started it again", async () => { const { daemon, run, resize } = await setup(); await run("click", { selector: "@e1" }); daemon.launchNext = true; // Chrome died just after the probe: the next read starts it const out = await run("snapshot"); expect(resize).toHaveBeenCalledOnce(); expect(out.launched).toBe(true); expect(out.parts[0]!.text).toMatch(/^The browser had stopped and was started again during this call/u); }); it("tells the bot when a preview frame between its calls found Chrome gone and the engine started it again, and sizes it before the step reads it", async () => { const { daemon, evidence, steps, run, resize } = await setup({ record: true }); await run("click", { selector: "@e1" }); daemon.launchNext = true; // Chrome died; the frame's picture started it again, blank and unsized daemon.geometry = { ...daemon.geometry, vh: 577 }; await captureFrame(steps.ports, daemon.target); resize.mockImplementationOnce(async () => { daemon.geometry = { ...daemon.geometry, vh: 720 }; }); const out = await run("click", { selector: "@e1" }); expect(resize).toHaveBeenCalledOnce(); expect(out.parts[0]!.text).toMatch(/^The browser had stopped and was started again during this call/u); expect(evidence.list("thread-1").steps.at(-1)!.page).toMatchObject({ viewport: { w: 1280, h: 720 } }); }); it("believes the engine when the call's own answer says it started the browser", async () => { const { run, resize } = await setup(); await run("click", { selector: "@e1" }); const out = await run("get_url", {}, undefined, { content: [{ type: "text", text: "about:blank" }], structuredContent: { response: { data: { lifecycle: { launched: true, relaunchedBrowser: false } } } } }); expect(resize).toHaveBeenCalledOnce(); expect(out.parts[0]!.text).toMatch(/started again/u); }); it("masks what was typed into a field that looks secret only to the page, even when the focus was elsewhere before", async () => { const { daemon, evidence, run } = await setup({ record: true }); const field = (name: string, secret: boolean, valueLength = 0) => ({ role: "textbox", name, editable: true, valueLength, secret, rect: { x: 0, y: 0, w: 10, h: 10 }, frame: null }); // The focus sat on the email field; the fill moved it into the one-time-code field (autocomplete one-time-code). daemon.geometry = { ...daemon.geometry, focus: field("Email", false) }; await run("fill", { selector: "#code", text: "482913" }, () => { daemon.geometry = { ...daemon.geometry, focus: field("Code", true, 6) }; }); await run("press", { key: "7" }); expect(evidence.list("thread-1").steps.map((step) => step.action)).toEqual(['Filled the element "#code" with ••••', 'Pressed •••• in the "Code" textbox']); expect(JSON.stringify(evidence.list("thread-1").steps)).not.toContain("482913"); }); it("never hands the bot, or keeps, what the page reported while a person held the browser", async () => { const { daemon, steps, evidence, run } = await setup({ record: true }); await run("click", { selector: "#go" }); expect(steps.recentFrame(daemon.session)).not.toBeNull(); const personsTurn = (id: string) => { steps.personTook(daemon.session); daemon.console.push({ type: "error", text: "Wrong password for ada@example.com" }); daemon.requests.push({ ...failure, requestId: id, url: "https://app.example.com/api/login" }); }; // Handed back, then a read: nothing of it, and no picture from before the hold stands in for a frame. personsTurn("held-1"); expect(steps.recentFrame(daemon.session)).toBeNull(); expect((await run("snapshot")).parts).toEqual([]); // Handed back, then an action: only what the action itself caused. personsTurn("held-2"); const out = await run("click", { selector: "#go" }, () => { daemon.console.push({ type: "error", text: "the click's own" }); }); expect(out.parts.map((part) => part.text).join("\n")).toContain("1 console error: \"the click's own\""); const kept = JSON.stringify(evidence.list("thread-1").steps) + JSON.stringify(out); expect(kept).not.toContain("ada@example.com"); expect(kept).not.toContain("/api/login"); }); it("takes a daemon that outlived its Chrome as keeping its old lists, and one that was not running as holding only the call's news", async () => { const outlived = await setup(); outlived.daemon.connected = false; outlived.daemon.console = [{ type: "error", text: "left from an earlier run" }]; const first = await outlived.run("open", { url: "https://app.example.com/" }, () => { outlived.daemon.connected = true; }); expect(JSON.stringify(first.parts)).not.toContain("earlier run"); const started = await setup(); started.daemon.removeStreamFile(); const news = await started.run("open", { url: "https://app.example.com/" }, () => { started.daemon.writeStreamFile(); started.daemon.console.push({ type: "error", text: "Boom on load" }); }); expect(news.parts.map((part) => part.text).join("\n")).toContain('1 console error: "Boom on load"'); }); it("takes no step and no picture for a conversation whose step lines are full", async () => { const { daemon, evidence, run } = await setup({ record: true }); const accepts = vi.spyOn(evidence, "accepts").mockReturnValue(false); const out = await run("click", { selector: "#go" }); expect(accepts).toHaveBeenCalledWith("thread-1"); expect(out.parts.map((part) => part.text).join("\n")).not.toMatch(/Recorded as step/u); expect(daemon.actions()).not.toContain("screenshot"); expect(evidence.list("thread-1").steps).toEqual([]); }); it("keeps held secrets out of every text a step keeps, the page's console lines included", async () => { cleanup.push(addHeldSecretScrubber((text) => text.replaceAll("fake-held-pw-1234", "[hidden]"))); const { daemon, evidence, run } = await setup({ record: true }); await run("snapshot"); daemon.geometry = { ...daemon.geometry, title: "Hello fake-held-pw-1234" }; await run("click", { selector: "#login" }, () => { daemon.console.push({ type: "error", text: "login failed for password fake-held-pw-1234" }); }); const kept = JSON.stringify(evidence.list("thread-1").steps); expect(kept).toContain("login failed for password [hidden]"); expect(kept).not.toContain("fake-held-pw-1234"); }); it("offers the preview the picture a recorded step just took, for two seconds, and only when it kept one", async () => { const { daemon, steps, run, tick, call } = await setup({ record: true, clock: true }); await run("click", { selector: "#go" }); expect(Buffer.from(steps.recentFrame(daemon.session)!.png, "base64")).toEqual(daemon.picture); tick.time += 2_000; expect(steps.recentFrame(daemon.session)).toBeNull(); await run("click", { selector: "#go" }); await steps.before(call("snapshot")); expect(steps.recentFrame(daemon.session)).toBeNull(); await run("click", { selector: "#go" }); daemon.geometry = { ...daemon.geometry, unmasked: 1 }; await run("click", { selector: "#show-password" }); expect(steps.recentFrame(daemon.session)).toBeNull(); }); });