/** * `agent.act()` coverage: the harness-owned step dispatch, * the raw StepExecutor socket driven by a hand-rolled executor with no AI SDK, * and the default ToolLoopAgent executor driven by a scripted tool-calling * model. Real Playwright observations and actions throughout. */ import { readFileSync } from 'node:fs'; import path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { startFixtureApp, type FixtureApp } from '../helpers/fixture-app.ts'; import { installFakeLoopModel, loopCalls, nodeIdFor } from '../helpers/fake-loop-model.ts'; import { resultByTitle, runProject, type FixtureProject } from '../helpers/run-project.ts'; import type { RunOutcome } from '../helpers/run-project.ts'; import type { StepExecutor, StepExecutorContext } from '../../src/agent/executor.ts'; const SUITE = `import { test, expect } from 'e2e'; test('scripted executor increments the counter', async ({ app, agent, screen }) => { await app.open(); await agent.act('increment the counter once and verify it shows 1'); await expect(screen.getByRole('status')).toHaveText('1'); }); `; const BLOCKED_SUITE = `import { test } from 'e2e'; test('executor reports a blocked step', async ({ app, agent }) => { await app.open(); await agent.act('log in with the staging account'); }); `; const BUDGET_SUITE = `import { test } from 'e2e'; test('executor overruns the action budget', async ({ app, agent }) => { await app.open(); await agent.act('keep clicking forever', { maxSteps: 2 }); }); `; const HANG_SUITE = `import { test } from 'e2e'; test('executor hangs past the step timeout', async ({ app, agent }) => { await app.open(); await agent.act('do something eventually', { timeout: 500 }); }); `; const HANGING_PAGE_SUITE = `import { test } from 'e2e'; test('a page that never settles costs one action timeout', async ({ app, agent }) => { await app.open(); await agent.act('open the hanging page', { timeout: 60_000 }); }); `; const OVERSPEND_SUITE = `import { test } from 'e2e'; test('executor overspends the model-call budget', async ({ app, agent }) => { await app.open(); await agent.act('think very hard', { maxModelCalls: 2 }); }); `; const INHERIT_SUITE = `import { test } from 'e2e'; test('failed verdict inherits the runtime code', async ({ app, agent }) => { await app.open(); await agent.act('tap twice', { maxSteps: 1 }); }); `; const LOOP_SUITE = `import { test, expect } from 'e2e'; test('default agent increments the counter', async ({ app, agent, screen }) => { await app.open(); await agent.act('increment the counter once and verify it shows 1'); await expect(screen.getByRole('status')).toHaveText('1'); }); `; /** Taps the Increment button, verifies the counter, and passes. No AI SDK. */ function scriptedExecutor(): StepExecutor { return { name: 'scripted-executor', version: 'test', async runStep(context: StepExecutorContext) { const turnStartedAt = new Date().toISOString(); let observation = await context.observe(); const id = nodeIdFor(observation.text, /button "Increment"/); await context.actions.tap({ id }); // Like a tool-using loop: the turn is reported after its tool ran. context.budgets.recordModelCall({ startedAt: turnStartedAt, durationMs: 5 }); observation = await context.observe(); if (!/status.*"1"|"Counter".*value="1"/.test(observation.text)) { return { status: 'failed' as const, summary: 'the counter did not show 1 after one tap' }; } return { status: 'passed' as const, summary: 'tapped Increment; the counter shows 1' }; }, }; } describe('agent.act with a hand-rolled step executor', () => { let app: FixtureApp; let outcome: RunOutcome; let project: FixtureProject; beforeAll(async () => { app = await startFixtureApp(); const result = await runProject( { 'tests/act.e2e.ts': SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor: scriptedExecutor() } }, }, }, ); outcome = result.outcome; project = result.project; }, 120_000); afterAll(async () => { project?.cleanup(); await app?.close(); }); it('passes the test through the socket without any AI SDK involvement', () => { expect(resultByTitle(outcome, 'scripted executor increments the counter').status).toBe( 'passed', ); expect(outcome.exitCode).toBe(0); }); it('records the act step with harness-owned accounting', () => { const attempt = resultByTitle( outcome, 'scripted executor increments the counter', ).attempts.at(-1)!; const step = attempt.steps.find((candidate) => candidate.api === 'agent.act'); expect(step).toBeDefined(); expect(step!.kind).toBe('agent'); expect(step!.status).toBe('passed'); expect(step!.metrics!.actionSteps).toBe(1); expect(step!.explanation).toContain('the counter shows 1'); const engineEvents = step!.events.filter((event) => event.kind === 'engine'); expect(engineEvents).toHaveLength(1); const modelEvents = step!.events.filter((event) => event.kind === 'model'); expect(modelEvents).toHaveLength(1); // Recorded after the tap, stamped with the executor's start: earlier in // time than the engine event, later in the list. expect(step!.events.indexOf(modelEvents[0]!)).toBeGreaterThan(step!.events.indexOf(engineEvents[0]!)); expect(Date.parse(modelEvents[0]!.startedAt)).toBeLessThanOrEqual(Date.parse(engineEvents[0]!.startedAt)); }); }); describe('agent.act verdict mapping', () => { let app: FixtureApp; beforeAll(async () => { app = await startFixtureApp(); }); afterAll(async () => { await app?.close(); }); it('maps a blocked verdict onto its error code and the configuration exit code', async () => { const executor: StepExecutor = { name: 'blocked-executor', async runStep() { return { status: 'blocked' as const, summary: 'the staging credential is not configured', errorCode: 'AUTH_CREDENTIAL_UNAVAILABLE' as const, }; }, }; const { outcome, project } = await runProject( { 'tests/blocked.e2e.ts': BLOCKED_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } }, ); try { const result = resultByTitle(outcome, 'executor reports a blocked step'); expect(result.status).toBe('failed'); const error = result.attempts.at(-1)!.error; expect(error?.code).toBe('AUTH_CREDENTIAL_UNAVAILABLE'); expect(error?.message).toContain('blocked'); // Blocked-for-configuration is distinguishable from a product failure at // the process boundary: exit 2, not the test-failure exit 1. expect(outcome.exitCode).toBe(2); // ...and blocked is first-class in the report: the step says blocked, // and a run whose every non-passing result is blockable says blocked. const step = result.attempts.at(-1)!.steps.find((s) => s.api === 'agent.act'); expect(step?.status).toBe('blocked'); expect(outcome.report.run.status).toBe('blocked'); } finally { project.cleanup(); } }, 120_000); it('fails closed when the executor claims success over an exhausted budget', async () => { const executor: StepExecutor = { name: 'over-budget-executor', async runStep(context: StepExecutorContext) { const observation = await context.observe(); const id = nodeIdFor(observation.text, /button "Increment"/); // Swallow every failure and claim success; the harness must not let // the verdict outrank its own budget accounting. for (let round = 0; round < 5; round += 1) { try { await context.actions.tap({ id }); } catch { break; } } return { status: 'passed' as const, summary: 'all good, nothing to see' }; }, }; const { outcome, project } = await runProject( { 'tests/budget.e2e.ts': BUDGET_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } }, ); try { const result = resultByTitle(outcome, 'executor overruns the action budget'); expect(result.status).toBe('failed'); expect(result.attempts.at(-1)!.error?.code).toBe('STEP_BUDGET_EXHAUSTED'); } finally { project.cleanup(); } }, 120_000); it('cuts a hanging executor at the step timeout', async () => { const executor: StepExecutor = { name: 'hanging-executor', runStep: () => new Promise(() => undefined), }; const { outcome, project } = await runProject( { 'tests/hang.e2e.ts': HANG_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } }, ); try { const result = resultByTitle(outcome, 'executor hangs past the step timeout'); expect(result.status).toBe('failed'); expect(result.attempts.at(-1)!.error?.code).toBe('STEP_TIMEOUT'); // The step settled at its own 500 ms deadline, not the test timeout. expect(result.attempts.at(-1)!.durationMs).toBeLessThan(30_000); } finally { project.cleanup(); } }, 120_000); it('hard-stops an executor that overspends the model-call budget', async () => { const executor: StepExecutor = { name: 'overspending-executor', async runStep(context: StepExecutorContext) { for (let call = 0; call < 10; call += 1) { context.budgets.recordModelCall({ inputTokens: 1, outputTokens: 1 }); } return { status: 'passed' as const, summary: 'thought about it a lot' }; }, }; const { outcome, project } = await runProject( { 'tests/overspend.e2e.ts': OVERSPEND_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } }, ); try { const result = resultByTitle(outcome, 'executor overspends the model-call budget'); expect(result.status).toBe('failed'); expect(result.attempts.at(-1)!.error?.code).toBe('STEP_BUDGET_EXHAUSTED'); } finally { project.cleanup(); } }, 120_000); it('stamps the runtime code onto a code-less failed verdict after a hard stop', async () => { const executor: StepExecutor = { name: 'code-less-executor', async runStep(context: StepExecutorContext) { const observation = await context.observe(); const id = nodeIdFor(observation.text, /button "Increment"/); try { await context.actions.tap({ id }); await context.actions.tap({ id }); } catch { // Swallow the budget error and report a bare product failure. } return { status: 'failed' as const, summary: 'the counter looked wrong' }; }, }; const { outcome, project } = await runProject( { 'tests/inherit.e2e.ts': INHERIT_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } }, ); try { const result = resultByTitle(outcome, 'failed verdict inherits the runtime code'); expect(result.status).toBe('failed'); const error = result.attempts.at(-1)!.error; expect(error?.code).toBe('STEP_BUDGET_EXHAUSTED'); expect(error?.message).toContain('the counter looked wrong'); } finally { project.cleanup(); } }, 120_000); it('fills a declared secret through typeSecret, never exposing the value', async () => { const executor: StepExecutor = { name: 'login-executor', async runStep(context: StepExecutorContext) { // The executor sees only the placeholder, never the plaintext. const params = JSON.stringify(context.step.params); if (params.includes('admin-pass')) { return { status: 'failed' as const, summary: 'plaintext leaked into params' }; } if (context.step.secrets[0]?.name !== 'admin.password') { return { status: 'failed' as const, summary: 'secret was not declared' }; } const observation = await context.observe(); const password = nodeIdFor(observation.text, /textbox "Password"/); await context.actions.typeSecret({ id: password }, 'admin.password'); // A configured secret the step never declared is refused before any policy check runs. try { await context.actions.typeSecret({ id: password }, 'stripe-key'); return { status: 'failed' as const, summary: 'undeclared secret was accepted' }; } catch (cause) { const { code, message } = cause as { code?: string; message?: string }; if (code !== 'POLICY_DENIED' || message?.includes('was not declared in this step') !== true) { return { status: 'failed' as const, summary: `unexpected refusal: ${String(code)} ${String(message)}` }; } return { status: 'passed' as const, summary: 'filled the declared secret only' }; } }, }; const { outcome, project } = await runProject( { 'tests/secret.e2e.ts': SECRET_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } }, credentials: { admin: { username: 'admin', password: 'admin-pass' } }, secrets: { 'stripe-key': 'sk_live_generic_4242' }, }, }, ); try { const result = resultByTitle(outcome, 'executor fills a declared secret'); expect(result.attempts.at(-1)!.error?.message ?? '').toBe(''); expect(result.status).toBe('passed'); } finally { project.cleanup(); } }, 120_000); it('fills a generic secret into a plain textbox through typeSecret and refuses a password there', async () => { const executor: StepExecutor = { name: 'api-key-executor', async runStep(context: StepExecutorContext) { if (JSON.stringify(context.step.params).includes('sk_live_generic')) { return { status: 'failed' as const, summary: 'plaintext leaked into params' }; } const declared = context.step.secrets.map((secret) => `${secret.name}:${secret.purpose}`).toSorted(); if (declared.join(',') !== 'admin.password:password,stripe-key:generic-secret') { return { status: 'failed' as const, summary: `unexpected secrets ${declared.join(',')}` }; } const observation = await context.observe(); const focusTarget = nodeIdFor(observation.text, /textbox "Focus target"/); // A password belongs in a password field only. try { await context.actions.typeSecret({ id: focusTarget }, 'admin.password'); return { status: 'failed' as const, summary: 'a password was accepted by a plain textbox' }; } catch (cause) { if (!(cause instanceof Error) || !cause.message.includes('incompatible')) { return { status: 'failed' as const, summary: `unexpected refusal: ${String(cause)}` }; } } // A generic secret goes wherever the test points it. await context.actions.typeSecret({ id: focusTarget }, 'stripe-key'); const after = await context.observe(); if (after.text.includes('sk_live_generic')) { return { status: 'failed' as const, summary: 'the generic secret value reached the observation' }; } if (!after.text.includes('')) { return { status: 'failed' as const, summary: 'the filled value was not redacted by name' }; } return { status: 'passed' as const, summary: 'filled the generic secret into a plain field' }; }, }; const { outcome, project } = await runProject( { 'tests/secret.e2e.ts': GENERIC_SECRET_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } }, credentials: { admin: { username: 'admin', password: 'admin-pass' } }, secrets: { 'stripe-key': 'sk_live_generic_4242' }, }, }, ); try { const result = resultByTitle(outcome, 'executor fills a generic secret'); expect(result.attempts.at(-1)!.error?.message ?? '').toBe(''); expect(result.status).toBe('passed'); expect(JSON.stringify(outcome.report)).not.toContain('sk_live_generic'); } finally { project.cleanup(); } }, 120_000); it('resolves a provider-backed secret at fill time and keeps the plaintext out of the report', async () => { const provider: { calls: number } = { calls: 0 }; // Read through a call so control-flow narrowing cannot pin the counter: // the provider mutates it from inside the runner, invisibly to tsc. const callsSoFar = () => provider.calls; const executor: StepExecutor = { name: 'provider-login-executor', async runStep(context: StepExecutorContext) { if (callsSoFar() !== 0) { return { status: 'failed' as const, summary: 'provider resolved before the fill' }; } const observation = await context.observe(); const password = nodeIdFor(observation.text, /textbox "Password"/); await context.actions.typeSecret({ id: password }, 'admin.password'); return callsSoFar() === 1 ? { status: 'passed' as const, summary: 'provider resolved exactly once, at fill time' } : { status: 'failed' as const, summary: `provider resolved ${callsSoFar()} times` }; }, }; const { outcome, project } = await runProject( { 'tests/secret.e2e.ts': SECRET_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } }, credentials: { admin: { username: 'admin', password: () => { provider.calls += 1; return Promise.resolve('provider-pass-1'); }, }, }, }, }, ); try { const result = resultByTitle(outcome, 'executor fills a declared secret'); expect(result.attempts.at(-1)!.error?.message ?? '').toBe(''); expect(result.status).toBe('passed'); expect(JSON.stringify(outcome.report)).not.toContain('provider-pass-1'); } finally { project.cleanup(); } }, 120_000); it('rejects a verdict outside the closed grammar', async () => { const executor: StepExecutor = { name: 'rogue-executor', async runStep() { return { status: 'passed' as const, summary: '' }; }, }; const { outcome, project } = await runProject( { 'tests/blocked.e2e.ts': BLOCKED_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } }, ); try { const result = resultByTitle(outcome, 'executor reports a blocked step'); expect(result.status).toBe('failed'); expect(result.attempts.at(-1)!.error?.code).toBe('MODEL_OUTPUT_INVALID'); } finally { project.cleanup(); } }, 120_000); }); describe('agent.act engine operations are bounded by actionTimeout', () => { it('a page that never settles costs one action timeout, not the step budget', async () => { const app = await startFixtureApp(); const executor: StepExecutor = { name: 'hang-navigator', version: 'test', async runStep(context: StepExecutorContext) { try { await context.actions.navigate('/hang'); } catch (cause) { return { status: 'failed' as const, summary: `navigation gave up: ${cause instanceof Error ? cause.message : String(cause)}`, }; } return { status: 'failed' as const, summary: 'the hanging page unexpectedly loaded' }; }, }; const { outcome, project } = await runProject( { 'tests/hang.e2e.ts': HANGING_PAGE_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } }, actionTimeout: 1_000, }, }, ); try { const result = resultByTitle(outcome, 'a page that never settles costs one action timeout'); expect(result.status).toBe('failed'); const attempt = result.attempts.at(-1)!; const step = attempt.steps.find((candidate) => candidate.api === 'agent.act')!; // The 60s act budget is untouched: the hung navigation fails within its // own operation bound and the executor concludes, well under the clock. expect(step.error?.code).not.toBe('STEP_TIMEOUT'); expect(step.durationMs).toBeLessThan(15_000); expect(step.explanation).toContain('navigation gave up'); } finally { project.cleanup(); await app.close(); } }, 120_000); }); const RESIZE_SUITE = `import { test } from 'e2e'; test('executor resizes the viewport', async ({ app, agent }) => { await app.open(); await agent.act('check the layout on a phone'); }); `; describe('agent.act resize', () => { it('resizes the page through the engine, refuses a size out of bounds, and records the action', async () => { const app = await startFixtureApp(); const executor: StepExecutor = { name: 'resizer', version: 'test', async runStep(context: StepExecutorContext) { if (!context.target.verbs.has('resize')) return { status: 'failed' as const, summary: 'the web target offers no resize verb' }; const refused = await context.actions.resize({ width: 50, height: 844 }).then( () => 'accepted', (cause: unknown) => (cause as { code?: string }).code, ); await context.actions.resize({ width: 390, height: 844 }); const { viewport } = await context.observe(); return { status: 'passed' as const, summary: `out of bounds: ${String(refused)}; viewport: ${String(viewport.width)}x${String(viewport.height)}`, }; }, }; const { outcome, project } = await runProject( { 'tests/resize.e2e.ts': RESIZE_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { executor } } } }, ); try { const result = resultByTitle(outcome, 'executor resizes the viewport'); expect(result.status).toBe('passed'); const step = result.attempts.at(-1)!.steps.find((candidate) => candidate.api === 'agent.act')!; expect(step.explanation).toContain('out of bounds: INVALID_ARGUMENT; viewport: 390x844'); expect(step.metrics!.actionSteps).toBe(1); expect(step.events.filter((event) => event.kind === 'engine').map((event) => event.name)).toEqual(['resize']); expect(step.viewport).toEqual({ width: 390, height: 844, scale: 1 }); } finally { project.cleanup(); await app.close(); } }, 120_000); }); const SECRET_SUITE = `import { test, credentials, expect } from 'e2e'; test('executor fills a declared secret', async ({ app, agent, screen }) => { await app.open(); await agent.act('sign in with the given credentials', { params: { username: 'admin', password: credentials.user('admin').password }, }); // Secure fields refuse value reads by design; visibility is the most a // deterministic assertion may observe. The executor verified the fill. await expect(screen.getByLabel('Password')).toBeVisible(); }); `; const GENERIC_SECRET_SUITE = `import { test, credentials, secrets, expect } from 'e2e'; test('executor fills a generic secret', async ({ app, agent, screen }) => { await app.open(); await agent.act('paste the API key into the focus target field', { params: { password: credentials.user('admin').password, apiKey: secrets.get('stripe-key') }, }); await expect(screen.getByLabel('Focus target')).toBeVisible(); }); `; const LOOP_GUARD_SUITE = `import { test } from 'e2e'; test('agent goes in circles', async ({ app, agent }) => { await app.open(); await agent.act('keep poking the same thing forever'); }); `; describe('loop guards and transcripts', () => { let app: FixtureApp; beforeAll(async () => { app = await startFixtureApp(); }); afterAll(async () => { await app?.close(); }); it('forces a verdict when the model repeats itself, and persists the transcript', async () => { const model = installFakeLoopModel((call) => { if (call.toolNames.length === 1 && call.toolNames[0] === 'complete_step') { return [ { toolName: 'complete_step', input: { status: 'failed', summary: 'stuck repeating the same tap' }, }, ]; } // Node ids are stable, so the same tap keeps succeeding. Results report // only what changed, and the opening screen is never elided while only // change updates follow it, so the id is read from the prompt, as a // model would. const id = nodeIdFor(call.prompt, /button "Increment"/); return [{ toolName: 'tap', input: { target: id } }]; }); const { outcome, project } = await runProject( { 'tests/loop-guard.e2e.ts': LOOP_GUARD_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } } }, runOptions: { debug: true }, }, ); try { const result = resultByTitle(outcome, 'agent goes in circles'); expect(result.status).toBe('failed'); const attempt = result.attempts.at(-1)!; expect(attempt.error?.message).toContain('stuck repeating the same tap'); const step = attempt.steps.find((candidate) => candidate.api === 'agent.act')!; // Five identical calls trip the stop; the forced turn concludes. Without // the guard this model would burn the whole 25-turn budget. expect(step.metrics!.modelCalls).toBeLessThanOrEqual(8); // --debug persists the executor transcript as a step-attributed log artifact. const log = attempt.artifacts.find((artifact) => artifact.kind === 'log'); expect(log).toBeDefined(); expect(log!.producer).toEqual({ kind: 'step', stepId: step.id }); expect(log!.path).toBeDefined(); const text = readFileSync( path.join(project.dir, '.e2e', 'results', ...log!.path!.split('/')), 'utf8', ); expect(text).toContain('tool call: tap'); expect(text).toContain('--- turn 1 ---'); } finally { project.cleanup(); } }, 120_000); }); describe('agent.act with the default ToolLoopAgent executor', () => { let app: FixtureApp; let outcome: RunOutcome; let project: FixtureProject; beforeAll(async () => { app = await startFixtureApp(); const model = installFakeLoopModel((call) => { if (call.lastToolResult === '') { // First turn: the prompt carries the instruction and initial screen. const id = nodeIdFor(call.prompt, /button "Increment"/); return [{ toolName: 'tap', input: { target: id } }]; } // The tap result reports the change; conclude once the counter reads 1. if (/text="1"/.test(call.lastToolResult)) { return [ { toolName: 'complete_step', input: { status: 'passed', summary: 'tapped Increment and the counter shows 1', }, }, ]; } throw new Error(`unexpected loop state: ${call.lastToolResult}`); }); const result = await runProject( { 'tests/loop.e2e.ts': LOOP_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model, context: 'This is the e2e fixture application.' } }, }, }, ); outcome = result.outcome; project = result.project; }, 120_000); afterAll(async () => { project?.cleanup(); await app?.close(); }); it('drives the step through the tool loop to a passed verdict', () => { expect(resultByTitle(outcome, 'default agent increments the counter').status).toBe('passed'); }); it('offers the default toolset and concludes through complete_step', () => { expect(loopCalls.length).toBeGreaterThanOrEqual(2); expect(loopCalls[0]!.toolNames).toEqual([ 'back', 'check', 'complete_step', 'double_tap', 'drag', 'drag_at', 'hover', 'hover_at', 'long_press', 'navigate', 'observe', 'press', 'press_at', 'resize_viewport', 'right_click', 'screenshot', 'scroll', 'scroll_to', 'select', 'select_at', 'tap', 'tap_at', 'type', 'type_at', 'upload', ]); expect(loopCalls[0]!.prompt).toContain('increment the counter once'); expect(loopCalls[0]!.prompt).toMatch(/Current screen \(revision b\d+, path \/, \d+ nodes\):/); // The result reports the change since the opening screen, not the tree again. expect(loopCalls[1]!.lastToolResult).toContain('Screen changes since revision'); expect(loopCalls[1]!.lastToolResult).toMatch(/changed #\S+ status "Counter" text="1" \(was: #\S+ status "Counter" text="0"\)/); expect(loopCalls[1]!.lastToolResult).not.toContain('button "Increment"'); }); it('accounts executor model calls in the step metrics and provenance', () => { const attempt = resultByTitle(outcome, 'default agent increments the counter').attempts.at( -1, )!; const step = attempt.steps.find((candidate) => candidate.api === 'agent.act'); expect(step!.metrics!.modelCalls).toBeGreaterThanOrEqual(2); expect(step!.metrics!.actionSteps).toBe(1); expect(step!.model).toMatchObject({ provider: 'fake-loop', model: 'scripted-loop', tokenAccounting: 'provider', calls: step!.metrics!.modelCalls, }); expect(step!.model!.inputTokens).toBeGreaterThan(0); expect(step!.events.filter((event) => event.kind === 'model')).toHaveLength( step!.metrics!.modelCalls, ); }); }); const DELAYED_SUITE = `import { test, expect } from 'e2e'; test('default agent reads the second view', async ({ app, agent, screen }) => { await app.open('/delayed'); await agent.act('continue to the second view'); await expect(screen.getByRole('heading', { name: 'Second view' })).toBeVisible(); }); `; const DEAD_END_SUITE = `import { test } from 'e2e'; test('default agent taps a control with no effect', async ({ app, agent }) => { await app.open('/delayed'); await agent.act('tap the dead end button once and report what happened'); }); `; const BATCH_SUITE = `import { test, expect } from 'e2e'; test('default agent fills two fields in one turn', async ({ app, agent, screen }) => { await app.open(); // Finish the fixture's unrelated timer before testing an unchanged screen. await expect(screen.getByRole('button', { name: 'Late arrival' })).toBeVisible(); await agent.act('fill the email and the focus target fields'); await expect(screen.getByLabel('Email')).toHaveValue('ada@example.test'); await expect(screen.getByLabel('Focus target')).toHaveValue('hello'); }); `; describe('the default agent reads action results after their effect', () => { let app: FixtureApp; beforeAll(async () => { app = await startFixtureApp(); }); afterAll(async () => { await app?.close(); }); it('waits for a change that lands after the tap and reports it as a diff', async () => { const model = installFakeLoopModel((call) => { if (call.lastToolResult === '') { return [{ toolName: 'tap', input: { target: nodeIdFor(call.prompt, /button "Continue"/) } }]; } return [ { toolName: 'complete_step', input: { status: 'passed', summary: 'the second view is showing' }, }, ]; }); const { outcome, project } = await runProject( { 'tests/delayed.e2e.ts': DELAYED_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } } } }, ); try { expect(resultByTitle(outcome, 'default agent reads the second view').status).toBe('passed'); // Two turns: the tap and the verdict. The tap's own result already shows // the view that arrived 700 ms after the click, as a change update. expect(loopCalls).toHaveLength(2); const result = loopCalls[1]!.lastToolResult; expect(result).toContain('Screen changes since revision'); expect(result).toMatch(/added #\S+ heading "Second view"/); expect(result).toMatch(/removed #\S+ button "Continue"/); expect(result).not.toContain('Current screen'); } finally { project.cleanup(); } }, 120_000); it('reports an action that changed nothing instead of a stale screen', async () => { const model = installFakeLoopModel((call) => { if (call.lastToolResult === '') { return [{ toolName: 'tap', input: { target: nodeIdFor(call.prompt, /button "Dead end"/) } }]; } return [ { toolName: 'complete_step', input: { status: 'passed', summary: 'tapped the dead end button; nothing changed' }, }, ]; }); const { outcome, project } = await runProject( { 'tests/dead-end.e2e.ts': DEAD_END_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } } } }, ); try { expect(resultByTitle(outcome, 'default agent taps a control with no effect').status).toBe('passed'); expect(loopCalls[1]!.lastToolResult).toContain('No listed node changed within the wait after this action'); // No tree and no change lines: the model has the screen already. expect(loopCalls[1]!.lastToolResult).not.toMatch(/^(added|changed|removed) #n\d+ /m); } finally { project.cleanup(); } }, 120_000); it('runs batched actions in order, each with its own changes, and answers a redundant observe cheaply', async () => { const model = installFakeLoopModel((call) => { if (call.turn === 1) { return [ { toolName: 'type', input: { target: nodeIdFor(call.prompt, /textbox "Email"/), value: 'ada@example.test' } }, { toolName: 'type', input: { target: nodeIdFor(call.prompt, /textbox "Focus target"/), value: 'hello' } }, ]; } if (call.turn === 2) return [{ toolName: 'observe', input: {} }]; return [ { toolName: 'complete_step', input: { status: 'passed', summary: 'both fields hold their values' }, }, ]; }); const { outcome, project } = await runProject( { 'tests/batch.e2e.ts': BATCH_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } } } }, ); try { const result = resultByTitle(outcome, 'default agent fills two fields in one turn'); expect(result.attempts.at(-1)!.error?.message ?? '').toBe(''); expect(result.status).toBe('passed'); const [first, second] = loopCalls[1]!.toolResults; // Each result reports the change of its own action and nothing else. expect(first).toMatch(/changed #\S+ textbox "Email"[^\n]*value="ada@example.test"/); expect(first).not.toContain('value="hello"'); expect(second).toMatch(/changed #\S+ textbox "Focus target"[^\n]*value="hello"/); expect(second).not.toContain('ada@example.test'); // Nothing moved between the second type and the observe. expect(loopCalls[2]!.lastToolResult).toContain('Screen unchanged since revision'); const step = result.attempts.at(-1)!.steps.find((candidate) => candidate.api === 'agent.act')!; expect(step.metrics!.actionSteps).toBe(2); expect(step.metrics!.modelCalls).toBe(3); } finally { project.cleanup(); } }, 120_000); }); const CHURN_SUITE = `import { test, expect } from 'e2e'; test('default agent taps a control that remounts under it', async ({ app, agent, screen }) => { await app.open('/churn'); await agent.act('tap the "Tap me" button three times'); await expect(screen.getByRole('status')).toHaveText('3 / 3'); }); `; describe('a targeted action re-finds a node that went stale', () => { it('relocates by descriptor and retries instead of failing the tap', async () => { const app = await startFixtureApp(); const model = installFakeLoopModel((call) => { if (call.turn <= 3) { // The id comes from the newest screen that lists the button, the way a // model reads its history: a result that reports only the progress // change leaves the earlier id valid. The page remounts the row when // the pointer enters it, so the handle is dead on arrival. const screens = call.toolResults.toReversed().concat(call.prompt); const screen = screens.find((text) => /button "Tap me"/.test(text)) ?? call.prompt; return [{ toolName: 'tap', input: { target: nodeIdFor(screen, /button "Tap me"/) } }]; } return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'tapped three times' } }]; }); const { outcome, project } = await runProject( { 'tests/churn.e2e.ts': CHURN_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } } } }, ); try { const result = resultByTitle(outcome, 'default agent taps a control that remounts under it'); expect(result.attempts.at(-1)!.error?.message ?? '').toBe(''); expect(result.status).toBe('passed'); const step = result.attempts.at(-1)!.steps.find((candidate) => candidate.api === 'agent.act')!; expect(step.metrics!.actionSteps).toBe(3); // A tap went stale and was re-found rather than failed: at least one // relocation capture, and no tap result reported a failure. const relocations = step.events.filter((event) => event.kind === 'observation' && event.name === 'relocate'); expect(relocations.length).toBeGreaterThanOrEqual(1); for (const call of loopCalls.slice(1, 4)) expect(call.lastToolResult).not.toContain('failed'); } finally { project.cleanup(); await app.close(); } }, 120_000); it('lets batched actions keep addressing the screen the turn saw after the rows remounted between them', async () => { const app = await startFixtureApp(); const model = installFakeLoopModel((call) => { if (call.turn === 1) { // Three taps in one turn, all by the opening screen's id. The first // tap remounts the rows, so by the second the id names an element // that no longer exists; the newest screen lists the button anew. const target = nodeIdFor(call.prompt, /button "Tap me"/); return [ { toolName: 'tap', input: { target } }, { toolName: 'tap', input: { target } }, { toolName: 'tap', input: { target } }, ]; } return [{ toolName: 'complete_step', input: { status: 'passed', summary: 'tapped three times' } }]; }); const { outcome, project } = await runProject( { 'tests/churn.e2e.ts': CHURN_SUITE }, { appUrl: app.url, config: { tests: 'tests/**/*.e2e.ts', agents: { default: { model } } } }, ); try { const result = resultByTitle(outcome, 'default agent taps a control that remounts under it'); expect(result.attempts.at(-1)!.error?.message ?? '').toBe(''); expect(result.status).toBe('passed'); const step = result.attempts.at(-1)!.steps.find((candidate) => candidate.api === 'agent.act')!; expect(step.metrics!.actionSteps).toBe(3); expect(step.metrics!.modelCalls).toBe(2); for (const text of loopCalls[1]!.toolResults) expect(text).not.toContain('failed'); } finally { project.cleanup(); await app.close(); } }, 120_000); });