import { afterEach, describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { runDevinAcpProcess } from '../src/devin-acp.js'; // S4 coverage: the Devin ACP escape-hatch runner (src/devin-acp.js) had only // happy-path + thought-chunk coverage. These tests exercise the error and // boundary paths with a mocked stdio child (no real Devin CLI, no network). const ENV_KEYS = [ 'DEVIN_CLI_PATH', 'DEVIN_CLI_ACP_ARGS_JSON', 'DEVIN_TIMEOUT_MS', 'DEVIN_OUTPUT_LIMIT_BYTES', 'DEVIN_CLI_WORKDIR', ]; const originalEnv = Object.fromEntries(ENV_KEYS.map(k => [k, process.env[k]])); const tmpDirs = []; afterEach(() => { for (const k of ENV_KEYS) { if (originalEnv[k] === undefined) delete process.env[k]; else process.env[k] = originalEnv[k]; } while (tmpDirs.length) { try { rmSync(tmpDirs.pop(), { recursive: true, force: true }); } catch { /* ignore */ } } }); // Write a fake `devin acp` stdio server and point the runner at it via env. function installFakeAcp(source, { timeoutMs = 5000 } = {}) { const dir = mkdtempSync(join(tmpdir(), 'windsurfapi-acp-edge-')); tmpDirs.push(dir); const script = join(dir, 'fake-acp.mjs'); writeFileSync(script, source, 'utf8'); process.env.DEVIN_CLI_PATH = process.execPath; process.env.DEVIN_CLI_ACP_ARGS_JSON = JSON.stringify([script]); process.env.DEVIN_TIMEOUT_MS = String(timeoutMs); return script; } // Standard handshake preamble for fakes that only vary the prompt phase. const HANDSHAKE = ` import readline from 'node:readline'; const rl = readline.createInterface({ input: process.stdin }); function send(obj) { process.stdout.write(JSON.stringify(obj) + '\\n'); } function update(u) { send({ jsonrpc: '2.0', method: 'session/update', params: { sessionId: 'session-1', update: u } }); } `; describe('Devin ACP runner — config + auth boundaries', () => { it('rejects with 503 backend_unavailable when no apiKey is supplied (never spawns)', async () => { await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: '' }), (err) => { assert.equal(err.status, 503); assert.equal(err.type, 'backend_unavailable'); return true; }, ); }); it('rejects with 500 backend_misconfigured on malformed DEVIN_CLI_ACP_ARGS_JSON', async () => { process.env.DEVIN_CLI_PATH = process.execPath; process.env.DEVIN_CLI_ACP_ARGS_JSON = '{not-json'; await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }), (err) => { assert.equal(err.status, 500); assert.equal(err.type, 'backend_misconfigured'); return true; }, ); }); it('rejects DEVIN_CLI_ACP_ARGS_JSON that is not a string array', async () => { process.env.DEVIN_CLI_PATH = process.execPath; process.env.DEVIN_CLI_ACP_ARGS_JSON = JSON.stringify([1, 2, 3]); await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }), (err) => { assert.equal(err.status, 500); assert.equal(err.type, 'backend_misconfigured'); return true; }, ); }); it('rejects with 503 backend_unavailable when the CLI binary is missing (ENOENT)', async () => { process.env.DEVIN_CLI_PATH = join(tmpdir(), 'no-such-devin-binary-xyz-123'); process.env.DEVIN_CLI_ACP_ARGS_JSON = JSON.stringify(['acp']); await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }), (err) => { assert.equal(err.status, 503); assert.equal(err.type, 'backend_unavailable'); return true; }, ); }); it('surfaces an authenticate RPC error as 401 unauthorized (real dead-token shape)', async () => { installFakeAcp(`${HANDSHAKE} rl.on('line', line => { const msg = JSON.parse(line); if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; } if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, error: { code: -32603, message: 'invalid api key' } }); return; } }); `); // AC2 P0: "invalid api key" is a genuine auth failure → 401 unauthorized // (was a blanket 502 before transient-first classification landed). It still // routes to the pool's generic-error streak via reportRunFailure (401 is // neither 429/402 nor a 5xx internal-error), so a truly dead token is // disabled after the streak — but a transient blip in a 401 shell is caught // earlier by the CAPACITY / UPSTREAM_INTERNAL branches and never lands here. await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'bad-key' }), (err) => { assert.equal(err.status, 401); assert.equal(err.type, 'unauthorized'); assert.equal(err.code, 'UNAUTHORIZED'); return true; }, ); }); }); describe('Devin ACP runner — process + protocol failure paths', () => { it('maps a non-zero CLI exit to 502 backend_error', async () => { installFakeAcp(` process.exit(3); `); await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }), (err) => { assert.equal(err.status, 502); assert.equal(err.type, 'backend_error'); return true; }, ); }); it('rejects with 502 when session/new does not return a sessionId', async () => { installFakeAcp(`${HANDSHAKE} rl.on('line', line => { const msg = JSON.parse(line); if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; } if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; } if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; } }); `); await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }), (err) => { assert.equal(err.status, 502); assert.equal(err.type, 'backend_error'); assert.match(err.message, /sessionId/); return true; }, ); }); it('enforces the output byte cap with 502 backend_output_too_large', async () => { process.env.DEVIN_OUTPUT_LIMIT_BYTES = '1024'; installFakeAcp(` process.stdout.write('x'.repeat(20000) + '\\n'); setTimeout(() => {}, 2000); `); await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }), (err) => { assert.equal(err.status, 502); assert.equal(err.type, 'backend_output_too_large'); return true; }, ); }); it('times out a stalled prompt with 504 backend_timeout', async () => { // Handshake completes, but session/prompt never resolves. DEVIN_TIMEOUT_MS // drives the per-request timer (runTimeoutMs() has a 1000ms floor, so this // must stay >= 1000 or it silently falls back to the 10-minute default). installFakeAcp(`${HANDSHAKE} rl.on('line', line => { const msg = JSON.parse(line); if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; } if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; } if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; } // session/prompt: intentionally never answered. }); `, { timeoutMs: 1000 }); await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }), (err) => { assert.equal(err.status, 504); assert.equal(err.type, 'backend_timeout'); return true; }, ); }); it('aborts an in-flight prompt with 499 request_aborted when the signal fires', async () => { installFakeAcp(`${HANDSHAKE} rl.on('line', line => { const msg = JSON.parse(line); if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; } if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; } if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; } // session/prompt: never answered, wait for abort. }); `); const ac = new AbortController(); setTimeout(() => ac.abort(), 300); await assert.rejects( () => runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k', signal: ac.signal }), (err) => { assert.equal(err.status, 499); assert.equal(err.type, 'request_aborted'); return true; }, ); }); }); describe('Devin ACP runner — request_permission auto-decline + content shapes', () => { it('auto-declines session/request_permission so the agent never blocks on local tools', async () => { // The runner must reply {outcome:'cancelled'} to permission prompts (it // exposes no local fs/terminal). If it failed to, the fake would hang and // the test would time out instead of returning REPLY. installFakeAcp(`${HANDSHAKE} let asked = false; rl.on('line', line => { const msg = JSON.parse(line); if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; } if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; } if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; } if (msg.method === 'session/prompt') { // Ask for permission first; only continue once the client answers. send({ jsonrpc: '2.0', id: 9001, method: 'session/request_permission', params: { sessionId: 'session-1', toolCall: { name: 'write_file' } } }); return; } // The client's permission answer (id 9001) comes back as a result line. if (msg.id === 9001 && msg.result) { asked = true; if (msg.result.outcome !== 'cancelled') { send({ jsonrpc: '2.0', method: 'session/update', params: { sessionId: 'session-1', update: { sessionUpdate: 'agent_message_chunk', content: { text: 'WRONG_OUTCOME' } } } }); } else { update({ sessionUpdate: 'agent_message_chunk', content: { text: 'REPLY' } }); } // resolve the original prompt (id captured implicitly as 4 in the runner's sequence) send({ jsonrpc: '2.0', id: 4, result: { stopReason: 'end_turn' } }); return; } }); `); const result = await runDevinAcpProcess('do a thing', { modelKey: 'swe-1.6', apiKey: 'k' }); assert.equal(result.text, 'REPLY'); }); it('joins array-of-parts content blocks into the reply text', async () => { installFakeAcp(`${HANDSHAKE} rl.on('line', line => { const msg = JSON.parse(line); if (msg.method === 'initialize') { send({ jsonrpc: '2.0', id: msg.id, result: { protocolVersion: 1, authMethods: [{ id: 'windsurf-api-key' }] } }); return; } if (msg.method === 'authenticate') { send({ jsonrpc: '2.0', id: msg.id, result: {} }); return; } if (msg.method === 'session/new') { send({ jsonrpc: '2.0', id: msg.id, result: { sessionId: 'session-1' } }); return; } if (msg.method === 'session/prompt') { update({ sessionUpdate: 'agent_message_chunk', content: [{ type: 'text', text: 'AB' }, { type: 'text', text: 'CD' }] }); send({ jsonrpc: '2.0', id: msg.id, result: { stopReason: 'end_turn' } }); return; } }); `); const result = await runDevinAcpProcess('hi', { modelKey: 'swe-1.6', apiKey: 'k' }); assert.equal(result.text, 'ABCD'); assert.equal(result.reasoning, ''); }); });