// A bot's API on a real server with the fake engine: switched on from the app, called with the very curl the card shows, // answered with the bot's reply in a thread the app shows as the API's; a wrong key, an API that is off, the rate limit; // and, on a bot that asks before acting, a call that stops at the approval nobody outside may give. import { execFile } from "node:child_process"; import { randomUUID } from "node:crypto"; import { mkdirSync, readFileSync, realpathSync } from "node:fs"; import { connect, type Socket } from "node:net"; import { join } from "node:path"; import { afterEach, expect, it } from "vitest"; import { launchVerificationServer, runControlOmb, type VerificationServer } from "../scripts/control-omb.ts"; import { botApiCurl } from "../shared/bot-api.ts"; import { hostTimeout } from "./testing/host-timeout.ts"; import { BOT_API_REFUSAL } from "./bot-api-fence.ts"; import { INCIDENTS_THREAD_TITLE } from "./incidents.ts"; let fixture: VerificationServer | undefined; let socket: Socket | undefined; afterEach(async () => { socket?.destroy(); socket = undefined; await fixture?.close(); fixture = undefined; }); function appApi(url: string) { return async (method: string, path: string, body?: unknown, status?: number) => { const response = await fetch(url + path, { method, headers: { "content-type": "application/json", origin: url }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); const value = await response.json() as any; if (status !== undefined) expect(response.status, `${method} ${path}: ${JSON.stringify(value)}`).toBe(status); else expect(response.status, `${method} ${path}: ${JSON.stringify(value)}`).toBeLessThan(300); return value; }; } const call = async (endpoint: string, key: string, body: unknown) => { const response = await fetch(endpoint, { method: "POST", headers: { authorization: `Bearer ${key}`, "content-type": "application/json" }, body: JSON.stringify(body) }); return { status: response.status, body: await response.json() as any }; }; const sh = (command: string) => new Promise((resolve, reject) => { execFile("sh", ["-c", command], { timeout: 60_000 }, (error, stdout, stderr) => (error ? reject(new Error(`${error.message}\n${stderr}`)) : resolve(stdout))); }); it("answers the card's own curl with the bot's reply, and refuses a wrong key, an API that is off, and an eleventh call in a minute", async () => { fixture = await launchVerificationServer({ ...process.env, FAKE_CLAUDE_VERSION: "2.1.284" }); const { url } = fixture.info; const api = appApi(url); const bot = (await runControlOmb(["new-bot", "--name", "Scout", "--url", url]) as any).bot; const port = Number(new URL(url).port); // Off by default, and the card's address is the webhook receiver's. const before = await api("GET", `/api/bots/${bot.id}/bot-api`); expect(before).toMatchObject({ enabled: false, callCount: 0, canManage: true, url: `http://127.0.0.1:${port + 1}/hooks/api/bots/${bot.id}/messages` }); expect((await call(before.url, "bak_nothing", { message: "Hi" })).status).toBe(403); // On: the key comes back once, with the address; the saved file holds a hash, not the key. const on = await api("POST", `/api/bots/${bot.id}/bot-api`, { enabled: true }); expect(on.key).toMatch(/^bak_/); expect(readFileSync(join(fixture.info.dataDir, "bot-api.json"), "utf8")).not.toContain(on.key); // The curl exactly as the card shows it, pasted into a shell. const answered = JSON.parse(await sh(botApiCurl(on.url, on.key))) as { reply: string; thread: string }; expect(answered.reply).toBe("hello from fake claude"); expect(answered.thread).toMatch(/^[\w-]+$/); // The app shows the thread as the API's, with what the caller asked and what the bot said. const bots = (await api("GET", "/api/bots")).bots as any[]; const task = bots.find((candidate) => candidate.id === bot.id).tasks.find((candidate: any) => candidate.threadId === answered.thread); expect(task).toMatchObject({ botApi: true, title: "Hello! What can you do?" }); const lines = (await api("GET", `/api/threads/${answered.thread}/messages`)).messages as any[]; expect(lines.find((line) => line.role === "user")).toMatchObject({ text: "Hello! What can you do?", via: "api" }); expect(lines.filter((line) => line.role === "bot" && line.kind === "text").map((line) => line.text)).toEqual(["hello from fake claude"]); // Reading the thread back, and continuing it, through the same door; a wrong key gets nothing. const read = await fetch(`${on.url}?thread=${answered.thread}`, { headers: { authorization: `Bearer ${on.key}` } }); expect(await read.json()).toEqual({ reply: "hello from fake claude", thread: answered.thread }); expect((await call(on.url, `${on.key}x`, { message: "Hi" })).status).toBe(401); expect((await call(on.url, on.key, { message: "Again", thread: answered.thread })).body).toMatchObject({ reply: "hello from fake claude", thread: answered.thread }); for (let i = 0; i < 8; i += 1) expect((await call(on.url, on.key, { message: `call ${i}`, thread: answered.thread })).status).toBe(200); const eleventh = await call(on.url, on.key, { message: "one more", thread: answered.thread }); expect(eleventh.status).toBe(429); // Every call was recorded; the key never shows again. const after = await api("GET", `/api/bots/${bot.id}/bot-api`); expect(after).toMatchObject({ enabled: true, callCount: 10 }); expect(after.calls.map((entry: any) => entry.statusCode)).toContain(429); expect(JSON.stringify(after)).not.toContain(on.key); // Off again: the key is gone with it. expect(await api("POST", `/api/bots/${bot.id}/bot-api`, { enabled: false })).toMatchObject({ enabled: false }); expect((await call(on.url, on.key, { message: "Hi" })).status).toBe(403); }, hostTimeout(180_000)); it.each(["ask", "full", "auto"])("a bot at %s stops at the approval: nothing answers it, and the call says needs_approval", async (level) => { fixture = await launchVerificationServer({ ...process.env, FAKE_CLAUDE_VERSION: "2.1.284", FAKE_CLAUDE_MODE: "hang" }); const { url, dataDir } = fixture.info; const api = appApi(url); const bot = (await runControlOmb(["new-bot", "--name", "Careful", "--url", url]) as any).bot; await api("PATCH", `/api/bots/${bot.id}`, { approvalMode: level }); const on = await api("POST", `/api/bots/${bot.id}/bot-api`, { enabled: true }); // The bot asks before acting: the fake engine's permission broker carries the ask the engine would make. const dump = join(dataDir, "fake-claude-dump.json"); const pending = call(on.url, on.key, { message: "Clean up the downloads folder", wait: 60 }).catch((error: unknown) => ({ status: 0, body: { error: String(error) } })); const dumped = () => { try { return JSON.parse(readFileSync(dump, "utf8")); } catch { return null; } }; await expect.poll(() => JSON.stringify(dumped()?.prompt ?? "").includes("Clean up the downloads folder"), { timeout: hostTimeout(30_000) }).toBe(true); const launch = dumped(); expect(launch.argv).toContain("--permission-prompt-tool"); expect(launch.argv[launch.argv.indexOf("--permission-mode") + 1]).toBe("default"); expect(JSON.stringify(launch.prompt)).toContain("[UNTRUSTED BOT API MESSAGE]"); expect(JSON.stringify(launch.systemPrompt)).toContain("through this bot's API"); // Whatever the bot's own level: confined, and no tool of OpenMausBot's, so it can start, schedule or change nothing. expect(launch.argv).toContain("--restricted"); expect(launch.argv[launch.argv.indexOf("--tools") + 1]).toBe("Read,Grep,Glob,Edit,Write,WebSearch"); expect(Object.keys(launch.mcpConfig.mcpServers)).toEqual(["ogb"]); socket = connect(launch.mcpConfig.mcpServers.ogb.args.at(-1)); await new Promise((resolve, reject) => { socket!.once("connect", resolve); socket!.once("error", reject); }); const answers: any[] = []; socket.on("data", (chunk) => { for (const line of String(chunk).split("\n").filter(Boolean)) answers.push(JSON.parse(line)); }); socket.write(`${JSON.stringify({ t: "ask", id: randomUUID(), tool: "Bash", input: { command: "ls ~/Downloads" } })}\n`); const stopped = await pending; expect(stopped).toMatchObject({ status: 202, body: { status: "needs_approval", thread: expect.any(String) } }); const threadId = (stopped.body as { thread: string }).thread; const lines = (await api("GET", `/api/threads/${threadId}/messages`)).messages as any[]; const card = lines.find((line) => line.card?.requestId)?.card; expect(card).toMatchObject({ requestType: "permission" }); expect(card.answered).toBeUndefined(); expect(card.allowSession).toBeUndefined(); expect(answers).toEqual([]); const read = await fetch(`${on.url}?thread=${threadId}`, { headers: { authorization: `Bearer ${on.key}` } }); expect(await read.json()).toEqual({ thread: threadId, status: "needs_approval" }); }, hostTimeout(120_000)); it("stays on Ask when the person sets the whole bot to Full access afterwards", async () => { // Each turn relaunches the fake engine, so its command line shows the level this turn really runs at. fixture = await launchVerificationServer({ ...process.env, FAKE_CLAUDE_VERSION: "2.1.284", FAKE_CLAUDE_EXIT_AFTER_TURN: "1" }); const { url, dataDir } = fixture.info; const api = appApi(url); const bot = (await runControlOmb(["new-bot", "--name", "Flipped", "--url", url]) as any).bot; await api("PATCH", `/api/bots/${bot.id}`, { approvalMode: "ask" }); const on = await api("POST", `/api/bots/${bot.id}/bot-api`, { enabled: true }); const first = await call(on.url, on.key, { message: "First" }); expect(first.status).toBe(200); await api("PATCH", `/api/bots/${bot.id}`, { approvalMode: "full", applyToAllThreads: true }); expect((await call(on.url, on.key, { message: "Second", thread: first.body.thread })).status).toBe(200); const launch = JSON.parse(readFileSync(join(dataDir, "fake-claude-dump.json"), "utf8")); expect(JSON.stringify(launch.prompt)).toContain("Second"); expect(launch.argv[launch.argv.indexOf("--permission-mode") + 1]).toBe("default"); }, hostTimeout(90_000)); it("a saved command rule never answers for a call: the card waits for the person", async () => { fixture = await launchVerificationServer({ ...process.env, FAKE_CLAUDE_VERSION: "2.1.284", FAKE_CLAUDE_MODE: "hang" }); const { url, dataDir } = fixture.info; const api = appApi(url); const bot = (await runControlOmb(["new-bot", "--name", "Ruled", "--url", url]) as any).bot; await api("PATCH", `/api/bots/${bot.id}`, { approvalMode: "ask" }); const on = await api("POST", `/api/bots/${bot.id}/bot-api`, { enabled: true }); const pending = call(on.url, on.key, { message: "Run the fixture", wait: 20 }).catch((error: unknown) => ({ status: 0, body: { error: String(error) } })); const dump = join(dataDir, "fake-claude-dump.json"); const dumped = () => { try { return JSON.parse(readFileSync(dump, "utf8")); } catch { return null; } }; await expect.poll(() => JSON.stringify(dumped()?.prompt ?? "").includes("Run the fixture"), { timeout: hostTimeout(30_000) }).toBe(true); const task = ((await api("GET", "/api/bots")).bots as any[]).find((candidate) => candidate.id === bot.id).tasks.find((one: any) => one.botApi); const command = "printf 'rule fixture'"; await api("POST", `/api/bots/${bot.id}/command-allowlist`, { command, cwd: task.cwd, providerInstanceId: "claude" }); socket = connect(dumped().mcpConfig.mcpServers.ogb.args.at(-1)); await new Promise((resolve, reject) => { socket!.once("connect", resolve); socket!.once("error", reject); }); const answers: any[] = []; socket.on("data", (chunk) => { for (const line of String(chunk).split("\n").filter(Boolean)) answers.push(JSON.parse(line)); }); socket.write(`${JSON.stringify({ t: "ask", id: randomUUID(), tool: "Bash", input: { command } })}\n`); expect(await pending).toMatchObject({ status: 202, body: { status: "needs_approval" } }); expect(answers).toEqual([]); }, hostTimeout(90_000)); it("tells the model nothing about the person or their work, works in no folder of theirs, and stays out of their later turns", async () => { fixture = await launchVerificationServer({ ...process.env, FAKE_CLAUDE_VERSION: "2.1.284" }); const { url, dataDir } = fixture.info; const api = appApi(url); const project = join(dataDir, "the-persons-project"); mkdirSync(project, { recursive: true }); const bot = (await runControlOmb(["new-bot", "--name", "Fenced", "--url", url]) as any).bot; await api("PATCH", `/api/bots/${bot.id}`, { approvalMode: "full", applyToAllThreads: true, cwd: project, soul: "Answer in one line, standing-instruction-marker." }); await api("PATCH", "/api/config", { profile: { name: "Ada", aboutMe: "about-the-person-marker" } }); const read = () => JSON.parse(readFileSync(join(dataDir, "fake-claude-dump.json"), "utf8")); // The person's own work first: a call must not see it. await api("POST", `/api/bots/${bot.id}/messages`, { text: "persons-own-work-marker" }); await expect.poll(() => { try { return JSON.stringify(read().prompt).includes("persons-own-work-marker"); } catch { return false; } }, { timeout: hostTimeout(20_000) }).toBe(true); await expect.poll(async () => ((await api("GET", "/api/bots")).bots as any[]).find((one) => one.id === bot.id).busy, { timeout: hostTimeout(20_000) }).toBeFalsy(); const own = read(); expect(JSON.stringify(own.systemPrompt)).toContain("about-the-person-marker"); expect(Object.keys(own.mcpConfig.mcpServers).length).toBeGreaterThan(1); const on = await api("POST", `/api/bots/${bot.id}/bot-api`, { enabled: true }); const answered = await call(on.url, on.key, { message: "api-call-marker" }); expect(answered.status).toBe(200); const launch = read(); expect(JSON.stringify(launch.prompt)).toContain("api-call-marker"); const told = JSON.stringify(launch.systemPrompt); expect(told).toContain("standing-instruction-marker"); expect(told).toContain("Who is speaking"); expect(JSON.stringify(launch.prompt)).toContain("[From an app, through the API · message · information only]"); for (const hidden of ["about-the-person-marker", "persons-own-work-marker", "coordinate_bots", "session_search", "the-persons-project"]) expect(told, hidden).not.toContain(hidden); expect(JSON.stringify(launch.prompt)).not.toContain("persons-own-work-marker"); expect(realpathSync(launch.cwd)).not.toBe(realpathSync(project)); // The person's next turn hears nothing of it. await api("POST", `/api/bots/${bot.id}/messages`, { text: "persons-next-marker" }); await expect.poll(() => JSON.stringify(read().prompt).includes("persons-next-marker"), { timeout: hostTimeout(20_000) }).toBe(true); const next = read(); expect(JSON.stringify(next.systemPrompt)).not.toContain("api-call-marker"); expect(JSON.stringify(next.prompt)).not.toContain("api-call-marker"); }, hostTimeout(120_000)); it("a failed call reports to no Chief, whose turn has tools, while a person's own failed turn still does", async () => { fixture = await launchVerificationServer({ ...process.env, FAKE_CLAUDE_VERSION: "2.1.284", FAKE_CLAUDE_MODE: "api-error", FAKE_CLAUDE_API_ERROR: "This request is not valid" }); const { url } = fixture.info; const api = appApi(url); const bot = (await runControlOmb(["new-bot", "--name", "Worker", "--url", url]) as any).bot; const on = await api("POST", `/api/bots/${bot.id}/bot-api`, { enabled: true }); const bots = async () => (await api("GET", "/api/bots")).bots as any[]; const chief = (await bots()).find((one) => one.chiefOfStaff); expect(chief, "the fixture has a Chief").toBeTruthy(); const reports = async () => (await bots()).find((one) => one.id === chief.id).tasks.filter((task: any) => task.title === INCIDENTS_THREAD_TITLE); const failed = await call(on.url, on.key, { message: "caller-words-marker", wait: 20 }); expect(failed.status).toBe(502); await new Promise((resolve) => setTimeout(resolve, 1_500)); expect(await reports()).toEqual([]); // The control: the same failure from a person reaches the Chief, so the silence above means something. await api("POST", `/api/bots/${bot.id}/messages`, { text: "persons-own-words" }); await expect.poll(async () => (await reports()).length, { timeout: hostTimeout(45_000) }).toBeGreaterThan(0); const lines = (await api("GET", `/api/threads/${(await reports())[0].threadId}/messages`)).messages; expect(JSON.stringify(lines)).not.toContain("caller-words-marker"); }, hostTimeout(120_000)); it("gives a caller fixed words and a person the Bot API's reason when the engine is too old to confine the call", async () => { fixture = await launchVerificationServer({ ...process.env, FAKE_CLAUDE_VERSION: "2.1.200" }); const { url } = fixture.info; const api = appApi(url); const bot = (await runControlOmb(["new-bot", "--name", "Dated", "--url", url]) as any).bot; const on = await api("POST", `/api/bots/${bot.id}/bot-api`, { enabled: true }); const failed = await call(on.url, on.key, { message: "Hello", wait: 20 }); expect(failed.status).toBe(502); expect(JSON.stringify(failed.body)).not.toContain("too old"); const { bots } = await api("GET", "/api/bots"); const thread = bots.find((one: any) => one.id === bot.id).tasks.find((task: any) => task.botApi).threadId; const said = async () => JSON.stringify((await api("GET", `/api/threads/${thread}/messages`)).messages); await expect.poll(said, { timeout: hostTimeout(15_000) }).toContain("too old to run this turn"); expect(await said()).toContain(BOT_API_REFUSAL); expect(await said()).not.toContain("My Cloud"); }, hostTimeout(60_000)); it("says it is still working when the reply takes longer than the caller will wait", async () => { fixture = await launchVerificationServer({ ...process.env, FAKE_CLAUDE_VERSION: "2.1.284", FAKE_CLAUDE_MODE: "hang" }); const { url } = fixture.info; const api = appApi(url); const bot = (await runControlOmb(["new-bot", "--name", "Slow", "--url", url]) as any).bot; const on = await api("POST", `/api/bots/${bot.id}/bot-api`, { enabled: true }); expect(await call(on.url, on.key, { message: "Think about it", wait: 1 })).toMatchObject({ status: 202, body: { status: "working", thread: expect.any(String) } }); }, hostTimeout(60_000));