/** * The assistant: one chat, one agent. A message starts a harness turn with the reply exit; the answer * (via reply() or plain prose) lands as the assistant's turn with its steps and cards; a message while * a turn runs is spoken into it; the history is digested into the next turn's orientation; a turn * resumed after a restart re-attaches to its chat. Scripted model, temp dir, no network. */ import { describe, it, expect } from 'vitest'; import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { OperatorRun } from '../src/operator/harness.js'; import { Registry } from '../src/operator/registry.js'; import { makeAssistant, REPLY_SPEC, labelOf } from '../src/operator/assistant.js'; import { assistantPrompt } from '../src/operator/assistantPrompt.js'; const tmp = () => fs.mkdtempSync(path.join(os.tmpdir(), 'as-')); const call = (name, args) => ({ content: '', toolCalls: [{ name, args }] }); const prose = (content) => ({ content, toolCalls: [] }); const tick = () => new Promise((r) => setTimeout(r, 5)); /** A harness turn factory over a scripted model; every turn gets the same script unless a queue is given. */ function factory(scripts, seen) { const jobDir = tmp(); return ({ goal, orientation, finishSpec, meta }) => { const turns = Array.isArray(scripts[0]) ? scripts.shift() : scripts; let i = 0; const chat = async ({ messages }) => { if (seen) seen.push(messages.map((m) => ({ ...m }))); const t = turns[Math.min(i, turns.length - 1)]; i++; return typeof t === 'function' ? t(messages) : t; }; const reg = new Registry(); reg.register('gb_watcher_feed', 'feed', { type: 'object', properties: { watcherId: { type: 'string' } } }, async ({ watcherId }) => ({ watcherId, items: [{ author: 'Ilya', state: 'waiting', draft: 'thanks Ilya' }] })); const run = new OperatorRun({ goal, chat, registry: reg, systemPrompt: 'sys', orientation, finishSpec, meta, persistDir: jobDir, startIterations: 20 }); run.done = run.run(); return run; }; } describe('assistant', () => { it('answers through reply() with cards and the steps it took, and titles the chat from the first message', async () => { const seen = []; const a = makeAssistant({ dir: tmp(), startTurn: factory([ call('gb_watcher_feed', { watcherId: 'facebook-post-watcher' }), call('reply', { text: '**Ilya** replied on your post — a draft is ready.', details: 'feed: 1 waiting', cards: [{ kind: 'results', watcherId: 'facebook-post-watcher', title: 'Open results' }] }), ], seen) }); const c = a.create(); expect(c.title).toBe('New chat'); const r = a.send(c.id, 'do I have replies to react to?'); expect(r.ok).toBe(true); expect(r.jobId).toMatch(/^op-/); expect(a.view(c.id).live).toBeTruthy(); while (a.live.has(c.id)) await tick(); const v = a.view(c.id); expect(v.title).toBe('do I have replies to react to?'); expect(v.live).toBeNull(); expect(v.turns.map((t) => t.role)).toEqual(['user', 'assistant']); const t = v.turns[1]; expect(t.text).toMatch(/Ilya/); expect(t.status).toBe('done'); expect(t.details).toBe('feed: 1 waiting'); expect(t.cards[0]).toMatchObject({ kind: 'results', watcherId: 'facebook-post-watcher' }); expect(t.steps.map((s) => s.name)).toEqual(['gb_watcher_feed']); expect(t.steps[0].label).toBe('Reading watcher results'); expect(t.steps[0].text).toBe('1 item · 1 waiting on you'); // a human line, never JSON // the model was told the goal and NOT asked for a task list first expect(seen[0][1].content).toMatch(/GOAL: do I have replies/); expect(seen[0][1].content).not.toMatch(/save_task_list first/); expect(a.list()[0]).toMatchObject({ id: c.id, turns: 2, running: false }); }); it('takes plain prose as the answer, digests the history into the next turn, and speaks into a running turn', async () => { const seen = []; let release; const gate = new Promise((r) => { release = r; }); const a = makeAssistant({ dir: tmp(), startTurn: factory([ [prose('Yes — two watchers run every 15 minutes.')], [async () => { await gate; return prose('Understood, keeping it short.'); }], ], seen) }); const c = a.create('Ops'); a.send(c.id, 'what runs right now?'); while (a.live.has(c.id)) await tick(); expect(a.view(c.id).turns[1].text).toBe('Yes — two watchers run every 15 minutes.'); expect(a.view(c.id).turns[1].steps).toEqual([]); // second turn: history in the orientation; a word while it runs is spoken into it a.send(c.id, 'shorter please'); const spoken = a.send(c.id, 'and no jargon'); expect(spoken.spoken).toBe(true); release(); while (a.live.has(c.id)) await tick(); const v = a.view(c.id); expect(v.turns.map((t) => t.role)).toEqual(['user', 'assistant', 'user', 'user', 'assistant']); expect(v.turns[3].spoken).toBe(true); const orient = seen[1][1].content; expect(orient).toMatch(/THE CONVERSATION SO FAR/); expect(orient).toMatch(/Owner: what runs right now\?/); expect(orient).toMatch(/You: Yes — two watchers/); expect(orient).toMatch(/Owner: shorter please/); }); it('re-attaches a resumed turn by its chat id so the answer still lands', async () => { const dir = tmp(); const a = makeAssistant({ dir, startTurn: factory([prose('never used')]) }); const c = a.create('Resume'); // a turn journal as a restart leaves it: the chat id in meta const reg = new Registry(); const run = new OperatorRun({ goal: 'x', chat: async () => call('reply', { text: 'back after the restart' }), registry: reg, systemPrompt: 'sys', finishSpec: REPLY_SPEC, meta: { chatId: c.id }, persistDir: tmp() }); run.done = run.run(); expect(a.attach(run)).toBe(true); while (a.live.has(c.id)) await tick(); expect(a.view(c.id).turns.at(-1)).toMatchObject({ role: 'assistant', text: 'back after the restart' }); expect(a.attach({ meta: { chatId: 'nope' } })).toBe(false); }); it('carries a picture a tool took onto its step and inlines it for the app', async () => { const shots = path.join(process.env.PROFILE_DIR || '/profiles', 'operator', 'shots'); let file = null; try { fs.mkdirSync(shots, { recursive: true }); file = path.join(shots, 'test-' + Date.now() + '.jpg'); fs.writeFileSync(file, Buffer.from([0xff, 0xd8, 0xff, 0xd9])); } catch { file = null; } const a = makeAssistant({ dir: tmp(), startTurn: ({ goal, orientation, finishSpec, meta }) => { const reg = new Registry(); // the real tools hand back JSON TEXT (clipped for the model), never an object — the picture must still be found reg.register('gb_look', 'look', { type: 'object', properties: {} }, async () => JSON.stringify({ url: 'https://x', title: 'Page', controls: [], text: '', screenshotUrl: file ? '/v1/operator/shots/' + path.basename(file) : null })); let i = 0; const chat = async () => (i++ === 0 ? call('gb_look', {}) : call('reply', { text: 'looked' })); const run = new OperatorRun({ goal, chat, registry: reg, systemPrompt: 'sys', orientation, finishSpec, meta, persistDir: tmp(), startIterations: 10 }); run.done = run.run(); return run; } }); const c = a.create('Look'); a.send(c.id, 'look at it'); while (a.live.has(c.id)) await tick(); const step = a.view(c.id).turns[1].steps[0]; expect(step.label).toBe('Looking at the page'); expect(step.text).toBe('Page'); if (file) { expect(step.image).toMatch(/\/v1\/operator\/shots\/test-/); expect(step.imageData).toMatch(/^data:image\/jpeg;base64,/); fs.unlinkSync(file); } else expect(step.image).toBeUndefined(); // no writable shots dir on this host: the step still lands }); it('records what a person asked, not the recipe it wrote', () => { /* * The training set takes its prompt from job.goal. The assistant writes a plan, and storing * that as the goal taught the model to expect an ordered list with the url already in it. * Measured on a live run: the owner wrote "on ns.nl look up a train from Amsterdam to Eindhoven * around nine" and the stored goal was a twelve-clause recipe starting from a url that 404'd. */ const p = assistantPrompt(); /* Plain string containment, not regex: the instruction contains backticks and an apostrophe, and both have to survive being written down in a test. */ expect(p.includes('ALWAYS PASS')).toBe(true); expect(p.includes('own words, verbatim')).toBe(true); }); it('never hands a walk a path it invented', () => { /* * The prompt used to say "say the start url", and the assistant duly wrote one. Measured on a * live run: ns.nl/en/travellers/journeyplanner looked entirely plausible, returned a 404, and the * walk that followed it burnt 120 steps, 23 looks, a click loop and a stall without reporting an * answer. A walk can find a page. It cannot know that the url it was handed was imagined. * * Asserted on the text because that is where the behaviour lives — there is no code that builds * the goal, only this instruction, so an instruction quietly dropped is the whole regression. */ const p = assistantPrompt(); expect(p).toMatch(/NEVER write a path you have not actually seen/); expect(p).toMatch(/Name the domain and let the walk find/); expect(p).not.toMatch(/say the start url/); }); it('labels steps for people and ships a prompt that carries the ladder and the operator method', () => { expect(labelOf('gb_watcher_run')).toBe('Running the watcher'); expect(labelOf('gb_zzz_thing')).toBe('zzz thing'); const p = assistantPrompt(); expect(p).toMatch(/IS THE ANSWER ALREADY GATHERED/); expect(p).toMatch(/gb_walk/); expect(p).toMatch(/CHANGE THE SMALLEST THING/); expect(p).not.toMatch(/You are THE GB OPERATOR/); expect(p).not.toMatch(/FINISH through the finish tool/); expect(REPLY_SPEC.map({ text: 'hi', cards: [{ kind: 'approvals' }] })).toMatchObject({ status: 'done', summary: 'hi', report: { answer: 'hi', cards: [{ kind: 'approvals' }] } }); }); });