import { afterEach, beforeEach, describe, it } from 'node:test';
import assert from 'node:assert/strict';
import {
toChatCompletion,
streamChatCompletion,
__setStreamChatForTest,
} from '../src/devin-connect-openai.js';
import { normalizeConnectUsage } from '../src/devin-connect.js';
afterEach(() => __setStreamChatForTest(null));
// Build a fake streamChat that yields a scripted event sequence.
function fakeStream(events) {
return async function* () {
for (const ev of events) yield ev;
};
}
const SAMPLE = [
{ type: 'reasoning', text: 'let me think ' },
{ type: 'reasoning', text: 'about it.' },
{ type: 'content', text: 'The answer ' },
{ type: 'content', text: 'is 42.' },
{ type: 'finish', reason: 'stop', usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 } },
];
describe('toChatCompletion (non-stream)', () => {
it('keeps both when reasoning and content differ', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'reasoning', text: 'the deliberation' },
{ type: 'content', text: 'the answer' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
const msg = body.choices[0].message;
assert.equal(msg.content, 'the answer');
assert.equal(msg.reasoning_content, 'the deliberation');
});
it('assembles a chat.completion with separated content and reasoning', async () => {
__setStreamChatForTest(fakeStream(SAMPLE));
const { status, body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(status, 200);
assert.equal(body.object, 'chat.completion');
assert.equal(body.model, 'swe-1-6-slow');
const msg = body.choices[0].message;
assert.equal(msg.role, 'assistant');
assert.equal(msg.content, 'The answer is 42.');
assert.equal(msg.reasoning_content, 'let me think about it.');
assert.equal(body.choices[0].finish_reason, 'stop');
assert.deepEqual(body.usage, { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 });
});
it('passes the normalized cache-write usage through untouched (O2)', async () => {
// omp reads exactly this field. The measured cache-write shape used to arrive
// as prompt_tokens=3 for an ~11.5k prompt; the adapter must not re-map it.
const usage = normalizeConnectUsage({ prompt: 3, completion: 170, cache_write_tokens: 11329 });
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'ok' },
{ type: 'finish', reason: 'stop', usage },
]));
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.deepEqual(body.usage, usage);
assert.equal(body.usage.prompt_tokens, 11332);
assert.deepEqual(body.usage.prompt_tokens_details, { cached_tokens: 0 });
});
it('echoes displayModel over the request model when given', async () => {
__setStreamChatForTest(fakeStream(SAMPLE));
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] }, { displayModel: 'claude-sonnet-4-6' });
assert.equal(body.model, 'claude-sonnet-4-6');
});
it('MOVES reasoning into content when the model returns no answer text', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'reasoning', text: 'hmm' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'm', messages: [] });
const message = body.choices[0].message;
assert.equal(message.content, 'hmm', 'the answer must be visible to the client');
// Moved, not copied. This assertion used to require BOTH fields to carry the
// same text, which made every client that renders reasoning show the answer
// twice — and on the Anthropic route produced a `thinking` block and a `text`
// block with byte-identical content. Nothing is lost by dropping the duplicate:
// the text is still delivered, once, in the field the client renders as the
// answer.
assert.equal(message.reasoning_content, undefined,
'reasoning_content must be dropped once its text has been promoted into content, '
+ 'or the same answer is rendered twice');
});
it('omits usage when the upstream gave none', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'hi' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'm', messages: [] });
assert.equal('usage' in body, false);
});
it('uses a stable id/created when supplied', async () => {
__setStreamChatForTest(fakeStream(SAMPLE));
const { body } = await toChatCompletion({ model: 'm', messages: [] }, { id: 'chatcmpl-fixed', created: 123 });
assert.equal(body.id, 'chatcmpl-fixed');
assert.equal(body.created, 123);
});
it('retries a transient failure then succeeds (no token duplication)', async () => {
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
if (calls === 1) {
// fail AFTER yielding a partial — the retry must discard it cleanly.
yield { type: 'content', text: 'PARTIAL' };
throw Object.assign(new Error('reset'), { code: 'ECONNRESET' });
}
yield { type: 'content', text: 'clean answer' };
yield { type: 'finish', reason: 'stop', usage: null };
});
const { body } = await toChatCompletion({ model: 'm', messages: [] }, { retryBaseMs: 1 });
assert.equal(calls, 2);
assert.equal(body.choices[0].message.content, 'clean answer'); // no leading PARTIAL
});
it('does not retry a terminal MODEL_BLOCKED error', async () => {
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
throw Object.assign(new Error('/upgrade required'), { code: 'MODEL_BLOCKED' });
});
await assert.rejects(
toChatCompletion({ model: 'm', messages: [] }, { retryBaseMs: 1 }),
/upgrade/,
);
assert.equal(calls, 1); // one attempt, no retry
});
it('gives up after maxRetries on a persistent transient error', async () => {
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
throw Object.assign(new Error('down'), { code: 'ETIMEDOUT' });
});
await assert.rejects(
toChatCompletion({ model: 'm', messages: [] }, { maxRetries: 2, retryBaseMs: 1 }),
/down/,
);
assert.equal(calls, 3); // initial + 2 retries
});
});
describe('streamChatCompletion (SSE)', () => {
function collectSend() {
const frames = [];
return { send: (d) => frames.push(d), frames };
}
it('emits role-prime, reasoning, content, finish, and usage chunks in order', async () => {
__setStreamChatForTest(fakeStream(SAMPLE));
const { send, frames } = collectSend();
// O1: the trailing usage frame is opt-in; this test asserts its shape.
const result = await streamChatCompletion({ model: 'swe-1-6-slow', messages: [] }, send, { id: 'x', created: 1, includeUsage: true });
// 1. role-prime
assert.deepEqual(frames[0].choices[0].delta, { role: 'assistant', content: '' });
// every chunk carries the chat.completion.chunk envelope
for (const f of frames) {
assert.equal(f.object, 'chat.completion.chunk');
assert.equal(f.id, 'x');
assert.equal(f.created, 1);
}
// reasoning chunks precede content chunks
const kinds = frames.map(f => {
const d = f.choices[0]?.delta;
if (d?.reasoning_content != null) return 'reasoning';
if (d?.content && f.choices[0].finish_reason == null && !('role' in d)) return 'content';
if (f.choices[0]?.finish_reason) return 'finish';
if (f.choices.length === 0) return 'usage';
return 'role';
});
assert.equal(kinds[0], 'role');
const firstReasoning = kinds.indexOf('reasoning');
const firstContent = kinds.indexOf('content');
assert.ok(firstReasoning > 0 && firstReasoning < firstContent, 'reasoning before content');
assert.equal(kinds.at(-2), 'finish');
assert.equal(kinds.at(-1), 'usage');
// finish chunk shape
const finish = frames.find(f => f.choices[0]?.finish_reason === 'stop');
assert.deepEqual(finish.choices[0].delta, {});
// usage chunk shape
const usageFrame = frames.find(f => f.choices.length === 0);
assert.deepEqual(usageFrame.usage, { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 });
// returns the assembled result for caching
assert.equal(result.content, 'The answer is 42.');
assert.equal(result.reasoning, 'let me think about it.');
assert.equal(result.finish_reason, 'stop');
});
it('does not emit a usage frame when upstream had no usage', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'hi' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { send, frames } = collectSend();
await streamChatCompletion({ model: 'm', messages: [] }, send);
assert.equal(frames.some(f => f.choices.length === 0), false);
});
it('streams each content delta as its own chunk (verbatim, not coalesced)', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'a' },
{ type: 'content', text: 'b' },
{ type: 'content', text: 'c' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { send, frames } = collectSend();
await streamChatCompletion({ model: 'm', messages: [] }, send);
const contentDeltas = frames
.map(f => f.choices[0]?.delta?.content)
.filter(c => c != null && c !== '');
assert.deepEqual(contentDeltas, ['a', 'b', 'c']);
});
});
// Tool emulation: the connect models have no native function-calling, so the
// adapter parses {...} markup out of the answer (the
// same machinery the Cascade path uses) and surfaces OpenAI tool_calls.
describe('toChatCompletion tool emulation', () => {
const TOOL_ANSWER = '{"name": "get_weather", "arguments": {"city": "Paris"}}';
it('extracts a tool_call and sets finish_reason=tool_calls, content=null', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: TOOL_ANSWER },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [], tools: [{ type: 'function', function: { name: 'get_weather' } }] }, { emulateTools: true });
const msg = body.choices[0].message;
assert.equal(body.choices[0].finish_reason, 'tool_calls');
assert.equal(msg.content, null);
assert.equal(msg.tool_calls.length, 1);
assert.equal(msg.tool_calls[0].type, 'function');
assert.equal(msg.tool_calls[0].function.name, 'get_weather');
assert.deepEqual(JSON.parse(msg.tool_calls[0].function.arguments), { city: 'Paris' });
assert.ok(msg.tool_calls[0].id, 'has an id');
});
it('leaves a plain answer untouched when emulateTools is on but no markup present', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'just a normal answer' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'm', messages: [], tools: [{ type: 'function', function: { name: 'search' } }] }, { emulateTools: true });
assert.equal(body.choices[0].finish_reason, 'stop');
assert.equal(body.choices[0].message.content, 'just a normal answer');
assert.equal('tool_calls' in body.choices[0].message, false);
});
it('does NOT parse tool markup when emulateTools is off (passes through as text)', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: TOOL_ANSWER },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'm', messages: [] }); // no emulateTools
assert.equal(body.choices[0].finish_reason, 'stop');
assert.equal(body.choices[0].message.content, TOOL_ANSWER);
});
});
describe('streamChatCompletion tool emulation', () => {
function collectSend() {
const frames = [];
return { send: (d) => frames.push(d), frames };
}
it('emits a tool_calls delta and finishes with finish_reason=tool_calls', async () => {
// Split the markup across deltas to exercise the streaming parser's
// cross-chunk buffering.
__setStreamChatForTest(fakeStream([
{ type: 'content', text: '{"name": "search", ' },
{ type: 'content', text: '"arguments": {"q": "cats"}}' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { send, frames } = collectSend();
const tools = [{ type: 'function', function: { name: 'search' } }];
const result = await streamChatCompletion({ model: 'swe-1-6-slow', messages: [], tools }, send, { emulateTools: true });
const toolFrame = frames.find(f => f.choices[0]?.delta?.tool_calls);
assert.ok(toolFrame, 'a tool_calls delta was emitted');
const tc = toolFrame.choices[0].delta.tool_calls[0];
assert.equal(tc.index, 0);
assert.equal(tc.type, 'function');
assert.equal(tc.function.name, 'search');
assert.deepEqual(JSON.parse(tc.function.arguments), { q: 'cats' });
const finish = frames.find(f => f.choices[0]?.finish_reason);
assert.equal(finish.choices[0].finish_reason, 'tool_calls');
assert.equal(result.finish_reason, 'tool_calls');
assert.equal(result.toolCalls.length, 1);
// The tool markup must NOT leak into content deltas.
const leaked = frames.map(f => f.choices[0]?.delta?.content || '').join('');
assert.equal(leaked.includes(''), false, 'markup leaked to content');
});
it('streams normal text untouched when emulateTools is on but no tool markup', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'hello ' },
{ type: 'content', text: 'world' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { send, frames } = collectSend();
const result = await streamChatCompletion({ model: 'm', messages: [] }, send, { emulateTools: true });
const text = frames.map(f => f.choices[0]?.delta?.content || '').join('');
assert.equal(text, 'hello world');
assert.equal(result.finish_reason, 'stop');
assert.equal(result.toolCalls.length, 0);
});
it('ToolGuard: drops tool_calls not in declared tools[] (P1 allowlist)', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: '{"name": "search", ' },
{ type: 'content', text: '"arguments": {"q": "cats"}}' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { send, frames } = collectSend();
// No tools[] declared → ToolGuard drops the parsed call entirely.
await streamChatCompletion({ model: 'swe-1-6-slow', messages: [] }, send, { emulateTools: true });
const toolFrame = frames.find(f => f.choices[0]?.delta?.tool_calls);
assert.ok(!toolFrame, 'undeclared tool_call must be filtered (ToolGuard parity)');
});
});
// Native tool calls: when DEVIN_CONNECT_TOOL_CALL_TAGS is calibrated, streamChat
// surfaces real ChatToolCall structs on the terminal finish event
// (devin-connect.js:927) as ev.toolCalls = [{ id, name, arguments }] where
// `arguments` is the raw JSON string. The adapter must translate these to
// OpenAI tool_calls WITHOUT also running text emulation (the two are mutually
// exclusive — calibrated native means no markup in the text).
describe('toChatCompletion native tool calls', () => {
const NATIVE = [
{ type: 'content', text: 'let me check that' },
{
type: 'finish', reason: 'stop', usage: null,
toolCalls: [{ id: 'call_abc', name: 'get_weather', arguments: '{"city":"Paris"}' }],
},
];
it('translates ev.toolCalls into OpenAI tool_calls and sets finish_reason', async () => {
__setStreamChatForTest(fakeStream(NATIVE));
const { body } = await toChatCompletion({ model: 'claude-sonnet-4-6', messages: [] });
const msg = body.choices[0].message;
assert.equal(body.choices[0].finish_reason, 'tool_calls');
assert.equal(msg.content, null);
assert.equal(msg.tool_calls.length, 1);
assert.equal(msg.tool_calls[0].id, 'call_abc');
assert.equal(msg.tool_calls[0].type, 'function');
assert.equal(msg.tool_calls[0].function.name, 'get_weather');
assert.deepEqual(JSON.parse(msg.tool_calls[0].function.arguments), { city: 'Paris' });
});
it('native wins over emulation — markup in text is NOT double-counted', async () => {
// Both a native tool call AND markup present: native takes the
// call, the text parser must not run (no second/duplicate tool_call).
__setStreamChatForTest(fakeStream([
{ type: 'content', text: '{"name":"shadow","arguments":{}}' },
{
type: 'finish', reason: 'stop', usage: null,
toolCalls: [{ id: 'call_real', name: 'real_tool', arguments: '{"a":1}' }],
},
]));
const { body } = await toChatCompletion({ model: 'm', messages: [], tools: [{ type: 'function', function: { name: 'search' } }] }, { emulateTools: true });
const msg = body.choices[0].message;
assert.equal(msg.tool_calls.length, 1);
assert.equal(msg.tool_calls[0].function.name, 'real_tool');
assert.equal(body.choices[0].finish_reason, 'tool_calls');
});
it('handles multiple (parallel) native tool calls', async () => {
__setStreamChatForTest(fakeStream([
{
type: 'finish', reason: 'stop', usage: null,
toolCalls: [
{ id: 'c1', name: 'a', arguments: '{"x":1}' },
{ id: 'c2', name: 'b', arguments: '{"y":2}' },
],
},
]));
const { body } = await toChatCompletion({ model: 'm', messages: [] });
const calls = body.choices[0].message.tool_calls;
assert.equal(calls.length, 2);
assert.equal(calls[0].function.name, 'a');
assert.equal(calls[1].function.name, 'b');
});
it('falls back to text emulation when finish carries no native tool calls', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: '{"name":"search","arguments":{"q":"x"}}' },
{ type: 'finish', reason: 'stop', usage: null }, // no toolCalls field
]));
const { body } = await toChatCompletion({ model: 'm', messages: [], tools: [{ type: 'function', function: { name: 'search' } }] }, { emulateTools: true });
const msg = body.choices[0].message;
assert.equal(msg.tool_calls.length, 1);
assert.equal(msg.tool_calls[0].function.name, 'search');
assert.equal(body.choices[0].finish_reason, 'tool_calls');
});
it('ignores an empty native toolCalls array (stays plain text)', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'plain answer' },
{ type: 'finish', reason: 'stop', usage: null, toolCalls: [] },
]));
const { body } = await toChatCompletion({ model: 'm', messages: [] });
assert.equal(body.choices[0].finish_reason, 'stop');
assert.equal(body.choices[0].message.content, 'plain answer');
assert.equal('tool_calls' in body.choices[0].message, false);
});
});
describe('streamChatCompletion native tool calls', () => {
function collectSend() {
const frames = [];
return { send: (d) => frames.push(d), frames };
}
it('emits a tool_calls delta from ev.toolCalls and finishes with tool_calls', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'checking…' },
{
type: 'finish', reason: 'stop', usage: null,
toolCalls: [{ id: 'call_xyz', name: 'lookup', arguments: '{"id":7}' }],
},
]));
const { send, frames } = collectSend();
const result = await streamChatCompletion({ model: 'm', messages: [] }, send);
const toolFrame = frames.find(f => f.choices[0]?.delta?.tool_calls);
assert.ok(toolFrame, 'a tool_calls delta was emitted');
const tc = toolFrame.choices[0].delta.tool_calls[0];
assert.equal(tc.index, 0);
assert.equal(tc.id, 'call_xyz');
assert.equal(tc.type, 'function');
assert.equal(tc.function.name, 'lookup');
assert.deepEqual(JSON.parse(tc.function.arguments), { id: 7 });
const finish = frames.find(f => f.choices[0]?.finish_reason);
assert.equal(finish.choices[0].finish_reason, 'tool_calls');
assert.equal(result.finish_reason, 'tool_calls');
assert.equal(result.toolCalls.length, 1);
// The plain content still streams through (native doesn't strip text).
const text = frames.map(f => f.choices[0]?.delta?.content || '').join('');
assert.equal(text, 'checking…');
});
it('native wins over emulation in the stream — no duplicate tool_calls delta', async () => {
// On the wire native and emulation never coexist (calibrated native means
// the model emits structured calls, not text markup). The
// de-dup guard makes that structural: if emulation already streamed a call
// inline, native is suppressed at finish so the call is never counted twice.
__setStreamChatForTest(fakeStream([
{ type: 'content', text: '{"name":"emulated","arguments":{}}' },
{
type: 'finish', reason: 'stop', usage: null,
toolCalls: [{ id: 'call_real', name: 'real_tool', arguments: '{"a":1}' }],
},
]));
const { send, frames } = collectSend();
const result = await streamChatCompletion({ model: 'm', messages: [] }, send, { emulateTools: true });
const toolFrames = frames.filter(f => f.choices[0]?.delta?.tool_calls);
assert.equal(toolFrames.length, 1, 'exactly one tool_calls delta (no double count)');
assert.equal(result.toolCalls.length, 1);
assert.equal(result.finish_reason, 'tool_calls');
});
it('native emits in the stream when emulation produced nothing', async () => {
// Realistic calibrated-native shape: structured call at finish, no markup.
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'checking…' },
{
type: 'finish', reason: 'stop', usage: null,
toolCalls: [{ id: 'call_real', name: 'real_tool', arguments: '{"a":1}' }],
},
]));
const { send, frames } = collectSend();
const result = await streamChatCompletion({ model: 'm', messages: [] }, send, { emulateTools: true });
const toolFrames = frames.filter(f => f.choices[0]?.delta?.tool_calls);
assert.equal(toolFrames.length, 1);
assert.equal(toolFrames[0].choices[0].delta.tool_calls[0].function.name, 'real_tool');
assert.equal(result.toolCalls.length, 1);
assert.equal(result.finish_reason, 'tool_calls');
});
it('emits multiple parallel native tool calls on distinct indices', async () => {
__setStreamChatForTest(fakeStream([
{
type: 'finish', reason: 'stop', usage: null,
toolCalls: [
{ id: 'c1', name: 'a', arguments: '{}' },
{ id: 'c2', name: 'b', arguments: '{}' },
],
},
]));
const { send, frames } = collectSend();
const result = await streamChatCompletion({ model: 'm', messages: [] }, send);
const toolFrames = frames.filter(f => f.choices[0]?.delta?.tool_calls);
const indices = toolFrames.map(f => f.choices[0].delta.tool_calls[0].index);
assert.deepEqual(indices, [0, 1]);
assert.equal(result.toolCalls.length, 2);
});
});
// retry-on-empty: NON-weak models occasionally return a COMPLETED turn
// (finish=stop) with zero content — probabilistic upstream capacity jitter. The
// adapter transparently re-issues the identical request a bounded number of
// times. It must (a) heal a subsequent real answer, (b) never trim tools, (c) not
// retry a genuine terminal state, (d) be a no-op on the hot path. NOTE: weak
// models (fable) are EXEMPT — their empties are deterministic (paid 27/27), so
// retry only amplifies rate limits; see the dedicated weak-model test below.
describe('retry-on-empty (fable capacity-jitter self-heal)', () => {
const RETRY_ENV = ['DEVIN_CONNECT_RETRY_ON_EMPTY', 'DEVIN_CONNECT_RETRY_ON_EMPTY_MAX', 'DEVIN_CONNECT_RETRY_ON_EMPTY_MS'];
afterEach(() => { for (const k of RETRY_ENV) delete process.env[k]; });
const EMPTY = [{ type: 'finish', reason: 'stop', usage: { prompt_tokens: 8, completion_tokens: 1, total_tokens: 9 } }];
const REAL = [
{ type: 'content', text: 'real answer' },
{ type: 'finish', reason: 'stop', usage: { prompt_tokens: 8, completion_tokens: 3, total_tokens: 11 } },
];
function collectSend() {
const frames = [];
return { send: (d) => frames.push(d), frames };
}
it('non-stream: retries an empty completion then returns the real answer', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
for (const ev of (calls === 1 ? EMPTY : REAL)) yield ev;
});
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(calls, 2, 'one empty + one heal');
assert.equal(body.choices[0].message.content, 'real answer');
});
// Weak-model exemption (paid E2E 2026-07-08): fable empties are DETERMINISTIC,
// retry never heals + triples upstream load → 3h rate limit. So a weak model
// must NOT retry on empty (single call, empty result returned as-is).
it('weak model (fable) does NOT retry on empty — single call, no amplification', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
for (const ev of EMPTY) yield ev;
});
const { body } = await toChatCompletion({ model: 'claude-5-fable-medium', messages: [] });
assert.equal(calls, 1, 'weak model fired exactly once (no retry)');
assert.equal(body.choices[0].message.content, '', 'empty returned as-is, not errored');
});
// REGRESSION (live paid probe 2026-07-08): genuine fable empties came back with
// completion_tokens of 3/5/8/9, NOT <=2. An earlier `ct <= 2` gate vetoed the
// retry on every one (15/15 empty, zero heals in production). The empty OUTPUT
// is the signature, not the token count.
it('retries an empty completion even when completion_tokens > 2 (no ct gate)', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
if (calls === 1) yield { type: 'finish', reason: 'stop', usage: { prompt_tokens: 8, completion_tokens: 9, total_tokens: 17 } };
else for (const ev of REAL) yield ev;
});
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(calls, 2, 'ct=9 empty is still healed');
assert.equal(body.choices[0].message.content, 'real answer');
});
it('stream: retries an empty completion without emitting a premature role/finish frame', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
for (const ev of (calls === 1 ? EMPTY : REAL)) yield ev;
});
const { send, frames } = collectSend();
const result = await streamChatCompletion({ model: 'swe-1-6-slow', messages: [] }, send);
assert.equal(calls, 2);
assert.equal(result.content, 'real answer');
// Exactly ONE role-prime frame (the empty attempt must not have primed/emitted).
const roleFrames = frames.filter(f => 'role' in (f.choices[0]?.delta || {}));
assert.equal(roleFrames.length, 1, 'no premature role frame from the discarded empty attempt');
// Exactly ONE terminal finish frame.
const finishFrames = frames.filter(f => f.choices[0]?.finish_reason);
assert.equal(finishFrames.length, 1);
const text = frames.map(f => f.choices[0]?.delta?.content || '').join('');
assert.equal(text, 'real answer');
});
it('gives up after RETRY_ON_EMPTY_MAX and returns the empty completion (never errors)', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MAX = '2';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
for (const ev of EMPTY) yield ev;
});
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(calls, 3, 'initial + 2 retries');
assert.equal(body.choices[0].message.content, ''); // degrades to empty, no throw
assert.equal(body.choices[0].finish_reason, 'stop');
});
it('is disabled by DEVIN_CONNECT_RETRY_ON_EMPTY=0 (single attempt)', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY = '0';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
for (const ev of EMPTY) yield ev;
});
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(calls, 1, 'no retry when disabled');
assert.equal(body.choices[0].message.content, '');
});
it('does NOT retry a real answer (completion_tokens>2 with content) — hot path is a no-op', async () => {
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
for (const ev of REAL) yield ev;
});
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(calls, 1);
assert.equal(body.choices[0].message.content, 'real answer');
});
it('does NOT retry a finish_reason=length truncation (real terminal state)', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
yield { type: 'finish', reason: 'length', usage: { prompt_tokens: 8, completion_tokens: 1, total_tokens: 9 } };
});
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(calls, 1, 'length is a genuine terminal state, not empty-jitter');
assert.equal(body.choices[0].finish_reason, 'length');
});
it('does NOT retry an empty-text turn that carries native tool calls', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
yield {
type: 'finish', reason: 'stop', usage: { prompt_tokens: 8, completion_tokens: 1, total_tokens: 9 },
toolCalls: [{ id: 'c1', name: 'do_thing', arguments: '{}' }],
};
});
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(calls, 1, 'a tool call is a real answer even with no visible text');
assert.equal(body.choices[0].finish_reason, 'tool_calls');
});
it('does NOT retry when reasoning-only content arrived (thinking counts as real)', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
yield { type: 'reasoning', text: 'thinking…' };
yield { type: 'finish', reason: 'stop', usage: { prompt_tokens: 8, completion_tokens: 1, total_tokens: 9 } };
});
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(calls, 1, 'a reasoning-only turn must not trigger the empty-completion retry');
// This test's subject is the retry decision above. It used to assert the text
// sits in `reasoning_content`, which was incidental — a reasoning-only turn is
// now PROMOTED (the text moves into content so the client is never handed an
// empty answer), so pin what actually matters here: the text is not lost.
const message = body.choices[0].message;
assert.equal(message.content, 'thinking…', 'the reasoning text must still reach the client');
});
it('treats a stop with no usage (free tier) + no content as empty and heals it', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
if (calls === 1) yield { type: 'finish', reason: 'stop', usage: null };
else for (const ev of REAL) yield ev;
});
const { body } = await toChatCompletion({ model: 'swe-1-6-slow', messages: [] });
assert.equal(calls, 2, 'no usage + no content still qualifies as empty');
assert.equal(body.choices[0].message.content, 'real answer');
});
});
describe('proto-openai-03: stop enforcement', () => {
function collectSend() {
const frames = [];
return { send: (d) => frames.push(d), frames };
}
const streamText = (frames) => frames
.filter(f => f.choices[0]?.delta?.content && f.choices[0].finish_reason == null && !('role' in f.choices[0].delta))
.map(f => f.choices[0].delta.content).join('');
it('non-stream: truncates content at the stop sequence and reports finish_reason stop', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'keep this' },
{ type: 'content', text: ' STOP drop this' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'm', messages: [] }, { stop: 'STOP' });
assert.equal(body.choices[0].message.content, 'keep this ');
assert.equal(body.choices[0].finish_reason, 'stop');
});
it('non-stream: no stop configured leaves content whole', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'a STOP b' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'm', messages: [] });
assert.equal(body.choices[0].message.content, 'a STOP b');
});
it('stream: emits only the prefix before the stop, even split across chunks', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'answer ST' },
{ type: 'content', text: 'OP leaked' },
{ type: 'content', text: ' more leaked' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { send, frames } = collectSend();
const result = await streamChatCompletion({ model: 'm', messages: [] }, send, { stop: 'STOP' });
assert.equal(streamText(frames), 'answer ');
assert.equal(result.finish_reason, 'stop');
// nothing after the stop leaked
assert.ok(!streamText(frames).includes('leaked'));
});
it('stream: without stop, all content flows (regression)', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: 'a STOP ' },
{ type: 'content', text: 'b' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { send, frames } = collectSend();
await streamChatCompletion({ model: 'm', messages: [] }, send, {});
assert.equal(streamText(frames), 'a STOP b');
});
});
describe('thinking-only rescue & promotion', () => {
const SAMPLE_TOOLS = [{ type: 'function', function: { name: 'read_file' } }];
it('rescues thinking-only response in streamChatCompletion when tools are present (includes digest in nudge)', async () => {
let callCount = 0;
let lastParams = null;
__setStreamChatForTest(async function* (params) {
callCount++;
lastParams = params;
if (callCount === 1) {
yield { type: 'reasoning', text: "I'll use the read_file tool" };
yield { type: 'finish', reason: 'stop', usage: null };
} else {
yield {
type: 'finish',
reason: 'stop',
usage: null,
toolCalls: [{ id: 't1', name: 'read_file', arguments: '{"path":"/tmp/x"}' }],
};
}
});
const initialMessages = [
{ role: 'user', content: 'hello' },
{ role: 'assistant', content: ' ' },
];
const frames = [];
const send = (frame) => frames.push(frame);
const result = await streamChatCompletion(
{ model: 'swe-1-7', messages: initialMessages, tools: SAMPLE_TOOLS },
send,
{ emulateTools: true },
);
assert.equal(callCount, 2);
const passedMsgs = lastParams.messages;
assert.equal(passedMsgs.length, 2);
assert.equal(passedMsgs[0].role, 'user');
assert.equal(passedMsgs[0].content, 'hello');
assert.equal(passedMsgs[1].role, 'user');
assert.equal(
passedMsgs[1].content,
'Your previous reasoning ended with: """I\'ll use the read_file tool"""\nStop reasoning. Emit the tool call markup now.',
);
assert.equal(result.finish_reason, 'tool_calls');
});
it('caps reasoning digest at DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS (tail kept)', async () => {
let callCount = 0;
let lastParams = null;
const longReasoning = 'A'.repeat(150) + 'B'.repeat(50);
__setStreamChatForTest(async function* (params) {
callCount++;
lastParams = params;
if (callCount === 1) {
yield { type: 'reasoning', text: longReasoning };
yield { type: 'finish', reason: 'stop', usage: null };
} else {
yield { type: 'content', text: '{"name":"read_file"}' };
yield { type: 'finish', reason: 'stop', usage: null };
}
});
const oldLimit = process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS;
process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS = '50';
try {
await streamChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'hi' }], tools: SAMPLE_TOOLS },
() => {},
{ emulateTools: true },
);
} finally {
if (oldLimit === undefined) delete process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS;
else process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS = oldLimit;
}
assert.equal(callCount, 2);
const nudge = lastParams.messages.at(-1).content;
// Buffer is a capped tail: no truncation marker, just the last 50 chars.
const expectedDigest = 'B'.repeat(50);
assert.equal(
nudge,
`Your previous reasoning ended with: """${expectedDigest}"""\nStop reasoning. Emit the tool call markup now.`,
);
});
it('does not accumulate nudges across consecutive rescues (one fresh nudge per rescue)', async () => {
// Review finding on this PR: rebuilding filteredMsgs from attemptParams kept every
// prior rescue's nudge (role:'user' survives the empty-assistant filter), so rescue
// call k carried k-1 nudges. Invisible at 46 chars/nudge pre-digest, ~2.1KB each
// with one. Rebuild from the untouched original instead — see devin-connect-openai.js.
const seen = [];
let callCount = 0;
__setStreamChatForTest(async function* (params) {
callCount++;
seen.push(params.messages.map((m) => ({ role: m.role, content: m.content })));
if (callCount <= 3) {
yield { type: 'reasoning', text: `reasoning tail ${callCount}` };
yield { type: 'finish', reason: 'stop', usage: null };
} else {
yield { type: 'content', text: '{"name":"read_file"}' };
yield { type: 'finish', reason: 'stop', usage: null };
}
});
const oldMax = process.env.DEVIN_CONNECT_RESCUE_MAX;
process.env.DEVIN_CONNECT_RESCUE_MAX = '5';
try {
await streamChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'hi' }], tools: SAMPLE_TOOLS },
() => {},
{ emulateTools: true },
);
} finally {
if (oldMax === undefined) delete process.env.DEVIN_CONNECT_RESCUE_MAX;
else process.env.DEVIN_CONNECT_RESCUE_MAX = oldMax;
}
assert.equal(callCount, 4, 'three thinking-only attempts, then the healed one');
for (let k = 2; k <= 4; k++) {
const msgs = seen[k - 1];
const nudges = msgs.filter(
(m) => m.role === 'user' && String(m.content).startsWith('Your previous reasoning ended with'),
);
assert.equal(nudges.length, 1, `rescue call ${k} carries ${nudges.length} nudge(s), expected exactly 1`);
assert.equal(msgs.length, 2, `rescue call ${k} must be original message + one nudge, got ${msgs.length} messages`);
assert.ok(
nudges[0].content.includes(`reasoning tail ${k - 1}`),
`rescue call ${k} must quote the immediately-preceding failed attempt`,
);
if (k >= 3) {
assert.ok(
!nudges[0].content.includes(`reasoning tail ${k - 2}`),
`rescue call ${k} still carries the stale digest from attempt ${k - 2}`,
);
}
}
});
it('disables reasoning digest when DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS=0', async () => {
let callCount = 0;
let lastParams = null;
__setStreamChatForTest(async function* (params) {
callCount++;
lastParams = params;
if (callCount === 1) {
yield { type: 'reasoning', text: 'some reasoning' };
yield { type: 'finish', reason: 'stop', usage: null };
} else {
yield { type: 'content', text: 'ok' };
yield { type: 'finish', reason: 'stop', usage: null };
}
});
const oldLimit = process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS;
process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS = '0';
try {
await streamChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'hi' }], tools: SAMPLE_TOOLS },
() => {},
{ emulateTools: true },
);
} finally {
if (oldLimit === undefined) delete process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS;
else process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS = oldLimit;
}
assert.equal(callCount, 2);
assert.equal(lastParams.messages.at(-1).content, 'Stop reasoning. Emit the tool call markup now.');
});
// Post-merge review of #241. `Number.isFinite` lets `1e9` through, exactly as it did
// for the RESCUE_MAX knob twelve lines above the digest cap in the source — that one
// already carries RESCUE_MAX_CEILING because an unbounded value hung the request for
// hours of upstream calls. The digest cap shipped without the equivalent clamp.
//
// These three pin the whole sanitisation surface as ONE unit, because the failing
// direction differs per input and asserting only the clamp would leave the other two
// free to drift:
// 1e9 → clamped to the ceiling (the defect this adds the clamp for)
// '' → 0, i.e. digest OFF (Number('') === 0; the file-wide convention)
// abc → NaN, i.e. the 2000 default (the only input the fallback actually serves)
describe('digest cap sanitisation (post-merge hardening)', () => {
const digestFor = async (envValue, reasoningLen) => {
let callCount = 0;
let lastParams = null;
__setStreamChatForTest(async function* (params) {
callCount++;
lastParams = params;
if (callCount === 1) {
yield { type: 'reasoning', text: 'R'.repeat(reasoningLen) };
yield { type: 'finish', reason: 'stop', usage: null };
} else {
yield { type: 'content', text: 'ok' };
yield { type: 'finish', reason: 'stop', usage: null };
}
});
const old = process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS;
process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS = envValue;
try {
await streamChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'hi' }], tools: SAMPLE_TOOLS },
() => {},
{ emulateTools: true },
);
} finally {
if (old === undefined) delete process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS;
else process.env.DEVIN_CONNECT_RESCUE_REASONING_MAX_CHARS = old;
}
// Precondition, not decoration — but NOT for the reason the first version of this
// comment claimed. It said the length assertions "would pass on a run where nothing
// happened"; they would not, they would THROW: with no rescue, .at(-1) is the original
// user turn, the `"""` regex returns null, and `m[1]` is a TypeError. What this buys is
// a NAMED failure ("the rescue must actually have fired") instead of a bare
// "Cannot read properties of null", which is what sent an earlier debugging round
// looking at the regex instead of at the fixture.
assert.equal(callCount, 2, 'precondition: the rescue must actually have fired');
const nudge = lastParams.messages.at(-1).content;
const m = nudge.match(/"""([\s\S]*)"""/);
return { nudge, quoted: m ? m[1] : null };
};
it('clamps an unbounded value (1e9) to the ceiling instead of quoting everything', async () => {
const { quoted } = await digestFor('1e9', 50000);
assert.equal(
quoted.length, 32000,
'Number.isFinite accepts 1e9; without the clamp the whole 50000-char reasoning ships in the nudge',
);
});
it('an empty value disables the digest — same convention as the sibling knobs', async () => {
const { nudge, quoted } = await digestFor('', 3000);
assert.equal(quoted, null, 'no quoted block at all');
assert.equal(
nudge, 'Stop reasoning. Emit the tool call markup now.',
"Number('') === 0, so an empty value reads as 0 (digest off), NOT as the 2000 default",
);
});
it('a genuinely non-numeric value falls back to the 2000 default', async () => {
const { quoted } = await digestFor('abc', 3000);
assert.equal(quoted.length, 2000, 'NaN is the only input the default fallback serves');
});
// Found by the post-release review pass, and it is the OTHER HALF of the 1e9 hole: the
// clamp alone does not cover it, because Math.min(0.5, 32000) is correctly 0.5 — the
// inversion happens one step later, in slice(). Measured before the fix: MAX_CHARS=0.5
// shipped all 50000 chars. Fractional values above 1 are the benign direction (they
// truncate downward), so both are pinned to keep the asymmetry visible.
it('a fractional cap below 1 does not invert into "no cap" (slice(-0.5) === slice(0))', async () => {
const { nudge, quoted } = await digestFor('0.5', 5000);
// Math.floor(0.5) === 0, and cap 0 takes the same branch as an explicit 0: bare nudge,
// no fence at all. The assertion that matters is the FIRST one — before the fix this
// shipped the entire 5000-char reasoning.
assert.equal(quoted, null, 'no fence: a sub-1 cap floors to 0, i.e. digest disabled');
assert.equal(nudge, 'Stop reasoning. Emit the tool call markup now.');
assert.ok(!nudge.includes('R'.repeat(50)), 'the reasoning body must not appear anywhere in the nudge');
});
it('a fractional cap above 1 truncates downward rather than throwing', async () => {
const { quoted } = await digestFor('100.9', 5000);
assert.equal(quoted.length, 100, 'Math.floor(100.9) === 100');
});
});
it('rescues in native tool mode when tools are present (emulateTools=false)', async () => {
let callCount = 0;
__setStreamChatForTest(async function* () {
callCount++;
if (callCount === 1) {
yield { type: 'reasoning', text: 'I should call a tool' };
yield { type: 'finish', reason: 'stop', usage: null };
} else {
yield {
type: 'finish',
reason: 'stop',
usage: null,
toolCalls: [{ id: 'call_1', name: 'read_file', arguments: '{}' }],
};
}
});
const result = await streamChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'hi' }], tools: SAMPLE_TOOLS },
() => {},
{ emulateTools: false },
);
assert.equal(callCount, 2, 'the rescue fires on the native tool path too, not only under emulation');
assert.equal(result.finish_reason, 'tool_calls');
});
it('skips the rescue when the request carries no tools', async () => {
let callCount = 0;
__setStreamChatForTest(async function* () {
callCount++;
yield { type: 'reasoning', text: 'just thinking out loud' };
yield { type: 'finish', reason: 'stop', usage: null };
});
await streamChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'hi' }] },
() => {},
{ emulateTools: true },
);
assert.equal(callCount, 1, 'no tools in the request — a reasoning-only finish is legitimate, not a trap');
});
it('respects rescue budget when DEVIN_CONNECT_RESCUE_MAX=0', async () => {
let callCount = 0;
__setStreamChatForTest(async function* () {
callCount++;
yield { type: 'reasoning', text: 'thinking only' };
yield { type: 'finish', reason: 'stop', usage: null };
});
const frames = [];
const send = (frame) => frames.push(frame);
const oldVal = process.env.DEVIN_CONNECT_RESCUE_MAX;
process.env.DEVIN_CONNECT_RESCUE_MAX = '0';
try {
await streamChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'hi' }], tools: SAMPLE_TOOLS },
send,
{ emulateTools: true },
);
} finally {
if (oldVal === undefined) delete process.env.DEVIN_CONNECT_RESCUE_MAX;
else process.env.DEVIN_CONNECT_RESCUE_MAX = oldVal;
}
assert.equal(callCount, 1);
});
it('promotes reasoning to content in toChatCompletion when content is empty and no tool calls', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'reasoning', text: 'standalone reasoning answer' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'swe-1-7', messages: [] }, { emulateTools: false });
assert.equal(body.choices[0].message.content, 'standalone reasoning answer');
});
it('promotes reasoning to content in streamChatCompletion when content is empty and no tool calls', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'reasoning', text: 'streamed reasoning answer' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const frames = [];
const send = (frame) => frames.push(frame);
const result = await streamChatCompletion({ model: 'swe-1-7', messages: [] }, send, { emulateTools: false });
assert.equal(result.content, 'streamed reasoning answer');
const contentDeltas = frames
.flatMap((f) => f.choices || [])
.map((c) => c.delta?.content)
.filter(Boolean);
assert.ok(contentDeltas.includes('streamed reasoning answer'));
});
// Promotion MOVES the text; it must not leave a copy behind.
//
// The review of #238 measured the duplicate on all three surfaces: non-stream
// (`content === reasoning_content`), stream (same after accumulating deltas), and
// the Anthropic route, where it became a `thinking` block and a `text` block with
// byte-identical content — i.e. the reporter's own client (kimi CLI) rendering the
// answer twice. The non-stream half is fixed here; the stream half cannot be
// (deltas already left) and is documented as an accepted cost at the call site.
it('does not leave a reasoning_content copy behind after promoting (non-stream)', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'reasoning', text: 'the whole answer lives in reasoning' },
{ type: 'finish', reason: 'stop', usage: { completion_tokens: 12 } },
]));
const { body } = await toChatCompletion({ model: 'swe-1-7', messages: [] });
const message = body.choices[0].message;
assert.equal(message.content, 'the whole answer lives in reasoning');
assert.notEqual(
message.content, message.reasoning_content,
'content and reasoning_content must not carry the same text — a client that renders '
+ 'reasoning would show the answer twice',
);
assert.equal(message.reasoning_content, undefined, 'the promoted copy must be dropped');
});
// A rescue attempt REPLACES the previous one; it must not be appended to it.
//
// The rescue loop lives inside streamChatWithEmptyRetry, while the accumulators
// live in its two consumers — which reset per THEIR OWN retry, so they never saw a
// rescue boundary. Measured before the fix: three thinking-only attempts produced
// `"PASS1. PASS2. PASS3. "`, i.e. the user was shown all three tries glued
// together as the answer. The generator now emits an `attempt_reset` sentinel.
for (const [label, run] of [
['non-stream', async (params, opts) => {
const { body } = await toChatCompletion(params, opts);
return body.choices[0].message.content;
}],
['stream', async (params, opts) => {
const result = await streamChatCompletion(params, () => {}, opts);
return result.content;
}],
]) {
it(`discards an abandoned rescue attempt instead of concatenating it (${label})`, async () => {
let attempts = 0;
__setStreamChatForTest(async function* () {
attempts++;
yield { type: 'reasoning', text: `PASS${attempts}. ` };
yield { type: 'finish', reason: 'stop', usage: { completion_tokens: 10 } };
});
const content = await run(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'x' }], tools: [{ type: 'function', function: { name: 'f', parameters: {} } }] },
{ emulateTools: true },
);
assert.ok(attempts > 1, `precondition: the rescue must have fired (attempts=${attempts})`);
assert.doesNotMatch(
String(content || ''), /PASS1/,
`the first attempt's output leaked into the final answer (${JSON.stringify(content)}) — each `
+ 'rescue replaces the previous attempt, it does not continue it',
);
assert.match(String(content || ''), new RegExp(`PASS${attempts}`),
'the LAST attempt is the one whose output the client should see');
});
}
// P-04: the emitted reasoning must come from the SAME attempt as the
// returned/cached/continuity value. The attempt-reset above only fixed the
// `content` accumulator; `reasoning_content` deltas had already gone out on the
// wire from the abandoned attempt, so the client saw ABANDONED + FINAL while
// result.reasoning (and therefore the cache and session continuity) held only
// FINAL. A stream cannot retract a delta, so the generator must not emit
// provisional reasoning until the attempt is committed.
it("emits only the committed attempt's reasoning — exact frame sequence", async () => {
let attempts = 0;
__setStreamChatForTest(async function* () {
attempts++;
if (attempts === 1) {
yield { type: 'reasoning', text: 'ABANDONED-1 ' };
yield { type: 'reasoning', text: 'ABANDONED-2 ' };
yield { type: 'finish', reason: 'stop', usage: { prompt_tokens: 4, completion_tokens: 5, total_tokens: 9 } };
return;
}
yield { type: 'reasoning', text: 'FINAL-1 ' };
yield { type: 'reasoning', text: 'FINAL-2' };
yield { type: 'content', text: 'the answer' };
yield { type: 'finish', reason: 'stop', usage: { prompt_tokens: 6, completion_tokens: 9, total_tokens: 15 } };
});
const frames = [];
const result = await streamChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'go' }], tools: SAMPLE_TOOLS },
(f) => frames.push(f),
{ emulateTools: true, includeUsage: true },
);
assert.equal(attempts, 2, 'precondition: the rescue must have fired');
assert.deepEqual(
frames.map((f) => {
const c = f.choices?.[0];
if (!c) return { usage: f.usage };
return { delta: c.delta, finish: c.finish_reason };
}),
[
{ delta: { role: 'assistant', content: '' }, finish: null },
{ delta: { reasoning_content: 'FINAL-1 ' }, finish: null },
{ delta: { reasoning_content: 'FINAL-2' }, finish: null },
{ delta: { content: 'the answer' }, finish: null },
{ delta: {}, finish: 'stop' },
{ usage: { prompt_tokens: 6, completion_tokens: 9, total_tokens: 15 } },
],
'the wire must carry exactly one attempt: the committed FINAL reasoning, its answer, '
+ 'and its usage — never the abandoned ABANDONED deltas',
);
assert.equal(result.reasoning, 'FINAL-1 FINAL-2',
'the returned value used by the cache and session continuity must match what the client saw');
assert.equal(result.content, 'the answer');
});
// Promotion must run the promoted text through the SAME tool-call extraction the
// content path uses.
//
// The rescue nudge literally says "Emit the tool call markup now." A model that
// complies but keeps writing into the REASONING channel produces markup that the
// promotion previously delivered unparsed: raw `` XML as the visible
// answer with finish_reason='stop'. The agent loop does not advance and the user
// sees markup — worse than the empty turn the promotion exists to prevent, and on
// the exact model this whole change set targets.
for (const [label, run] of [
['non-stream', async (params, opts) => {
const { body } = await toChatCompletion(params, opts);
return {
finish: body.choices[0].finish_reason,
toolCalls: (body.choices[0].message.tool_calls || []).length,
content: String(body.choices[0].message.content || ''),
};
}],
['stream', async (params, opts) => {
const frames = [];
const result = await streamChatCompletion(params, (f) => frames.push(f), opts);
const deltas = frames.flatMap((f) => f.choices || []);
return {
finish: deltas.map((c) => c.finish_reason).filter(Boolean).join(','),
toolCalls: deltas.reduce((n, c) => n + (c.delta?.tool_calls?.length || 0), 0),
content: deltas.map((c) => c.delta?.content).filter(Boolean).join(''),
};
}],
]) {
it(`extracts a tool call that arrived in the reasoning channel (${label})`, async () => {
__setStreamChatForTest(fakeStream([
{ type: 'reasoning', text: '\n{"name":"read_file","arguments":{"path":"a.js"}}\n' },
{ type: 'finish', reason: 'stop', usage: { completion_tokens: 20 } },
]));
const out = await run(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'read a.js' }], tools: [{ type: 'function', function: { name: 'read_file', parameters: {} } }] },
{ emulateTools: true },
);
assert.doesNotMatch(
out.content, //,
'raw tool markup reached the client as the visible answer — the promoted text must go '
+ 'through the same extraction the content path uses',
);
assert.equal(out.toolCalls, 1, 'the tool call must be delivered as a tool call');
assert.match(String(out.finish), /tool_calls/, 'and the finish reason must say so');
});
}
it('a successful rescue delivers the tool call with no stale reasoning attached', async () => {
let attempts = 0;
__setStreamChatForTest(async function* () {
attempts++;
if (attempts === 1) {
yield { type: 'reasoning', text: 'I should use read_file.' };
yield { type: 'finish', reason: 'stop', usage: { completion_tokens: 10 } };
return;
}
yield { type: 'content', text: '\n{"name":"read_file","arguments":{"path":"a.js"}}\n' };
yield { type: 'finish', reason: 'stop', usage: { completion_tokens: 20 } };
});
const { body } = await toChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'read a.js' }], tools: [{ type: 'function', function: { name: 'read_file', parameters: {} } }] },
{ emulateTools: true },
);
const message = body.choices[0].message;
assert.equal(attempts, 2, 'the rescue should heal on the first retry');
assert.equal(body.choices[0].finish_reason, 'tool_calls');
assert.equal((message.tool_calls || []).length, 1);
assert.doesNotMatch(
String(message.content || ''), /I should use read_file/,
"the abandoned attempt's reasoning must not ride along with the healed turn",
);
});
// The rescue budget must be bounded and must honour the existing off switches.
//
// Measured before this was hardened: DEVIN_CONNECT_RESCUE_MAX=Infinity and =1e9
// both defeated the `rescueAttempt < rescueMax` bound (the 1e9 probe had to be
// killed), a non-numeric value produced NaN and silently turned the whole feature
// OFF with no log line, the documented empty-retry master off switch did not cover
// the rescue at all, and the weak-model veto — written against a paid E2E that
// proved retrying the fable family only burns the account into a 3h rate limit —
// was bypassed.
for (const [label, env, expected] of [
['default', {}, 3],
['RESCUE_MAX=Infinity', { DEVIN_CONNECT_RESCUE_MAX: 'Infinity' }, 3],
['RESCUE_MAX=1e9 (hard ceiling)', { DEVIN_CONNECT_RESCUE_MAX: '1e9' }, 6],
['RESCUE_MAX=abc (must stay ON at the default)', { DEVIN_CONNECT_RESCUE_MAX: 'abc' }, 3],
['RESCUE_MAX=0 (explicit off)', { DEVIN_CONNECT_RESCUE_MAX: '0' }, 1],
['RETRY_ON_EMPTY=0 (master off switch)', { DEVIN_CONNECT_RETRY_ON_EMPTY: '0' }, 1],
]) {
it(`bounds upstream calls: ${label}`, async () => {
const saved = {};
for (const k of Object.keys(env)) { saved[k] = process.env[k]; process.env[k] = env[k]; }
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
try {
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
yield { type: 'reasoning', text: 'thinking. ' };
yield { type: 'finish', reason: 'stop', usage: { completion_tokens: 5 } };
});
await toChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'x' }], tools: [{ type: 'function', function: { name: 'f', parameters: {} } }] },
{ emulateTools: true },
);
assert.equal(calls, expected,
`${label}: expected ${expected} upstream call(s), got ${calls} — an unbounded rescue turns `
+ 'one client request into a loop that burns the account into a rate limit');
} finally {
for (const k of Object.keys(saved)) {
if (saved[k] === undefined) delete process.env[k]; else process.env[k] = saved[k];
}
delete process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS;
}
});
}
it('does not rescue a weak model, honouring the paid-probe account-burn veto', async () => {
process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS = '0';
try {
let calls = 0;
__setStreamChatForTest(async function* () {
calls++;
yield { type: 'reasoning', text: 'thinking. ' };
yield { type: 'finish', reason: 'stop', usage: { completion_tokens: 5 } };
});
await toChatCompletion(
{ model: 'claude-5-fable-medium', messages: [{ role: 'user', content: 'x' }], tools: [{ type: 'function', function: { name: 'f', parameters: {} } }] },
{ emulateTools: true },
);
assert.equal(calls, 1,
'the fable family must not be retried — a paid E2E (27/27) proved retries never heal it and '
+ 'only triple upstream load into a 3h rate limit');
} finally {
delete process.env.DEVIN_CONNECT_RETRY_ON_EMPTY_MS;
}
});
it('keeps reasoning_content when there is a real answer to distinguish it from', async () => {
// The promotion must not fire when the model DID answer: reasoning and content
// are genuinely different text and both belong in the response.
__setStreamChatForTest(fakeStream([
{ type: 'reasoning', text: 'let me think' },
{ type: 'content', text: 'the answer is 255' },
{ type: 'finish', reason: 'stop', usage: { completion_tokens: 20 } },
]));
const { body } = await toChatCompletion({ model: 'swe-1-7', messages: [] });
const message = body.choices[0].message;
assert.equal(message.content, 'the answer is 255');
assert.equal(message.reasoning_content, 'let me think',
'a genuine reasoning trace must survive alongside a real answer');
});
});
// Pin tests — GOAL-2026-08-06-PR-REWORK item 3. The leading think-tag reroute
// moved from the messages.js egress translators into the connect layer, at the
// stream-event level (streamChatWithEmptyRetry), BEFORE the #238 rescue
// decision. These pin the combined behavior: reclassification + rescue + the
// egress invariant "text block always present".
describe('think-text reroute (connect layer, DEVIN_CONNECT_THINKTEXT_REROUTE)', () => {
const OPEN = '<' + 'think' + '>';
const CLOSE = '<' + '/' + 'think' + '>';
const SAMPLE_TOOLS = [{ type: 'function', function: { name: 'read_file' } }];
let prev;
beforeEach(() => { prev = process.env.DEVIN_CONNECT_THINKTEXT_REROUTE; process.env.DEVIN_CONNECT_THINKTEXT_REROUTE = '1'; });
afterEach(() => { if (prev === undefined) delete process.env.DEVIN_CONNECT_THINKTEXT_REROUTE; else process.env.DEVIN_CONNECT_THINKTEXT_REROUTE = prev; });
it('(a) whole turn = think block: the #238 rescue fires and the client gets an answer', async () => {
// Attempt 1 is an ENTIRE turn on the content channel, think-tagged. With the
// classifier at the event level this decodes to reasoning-only, so sawText
// stays false and the rescue fires naturally — it never did before the move:
// the egress reroute ran AFTER the rescue had already given up, leaving a
// reasoning-only finish (#238 form, APIEmptyResponseError on strict clients).
let callCount = 0;
let lastParams = null;
__setStreamChatForTest(async function* (params) {
callCount++;
lastParams = params;
if (callCount === 1) {
yield { type: 'content', text: OPEN + 'Let me think through the whole request. ' + CLOSE };
yield { type: 'finish', reason: 'stop', usage: null };
} else {
yield { type: 'content', text: 'Here is the answer.' };
yield { type: 'finish', reason: 'stop', usage: null };
}
});
const frames = [];
const result = await streamChatCompletion(
{ model: 'swe-1-7', messages: [{ role: 'user', content: 'hi' }], tools: SAMPLE_TOOLS },
(f) => frames.push(f),
{ emulateTools: true },
);
assert.equal(callCount, 2, 'the #238 rescue must fire for a whole-think turn');
assert.ok(
lastParams.messages.at(-1).content.includes('Stop reasoning. Emit the tool call markup now.'),
'the rescue nudge reaches the second attempt',
);
// The rescued attempt's answer is what reaches the client — the abandoned
// think-only attempt is not concatenated onto it.
assert.equal(result.content, 'Here is the answer.');
});
it('(a) whole-think turn without tools: promotion keeps a visible answer (no empty message)', async () => {
// No tools -> no rescue (plain chat legitimately ends in reasoning/text). The
// think-only content must still reach the client as the visible answer via
// the existing reasoning->content promotion, so the anthropic egress below
// (openAIToAnthropic) always has a text block to push.
__setStreamChatForTest(fakeStream([
{ type: 'content', text: OPEN + 'reasoning only, no tools. ' + CLOSE },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'swe-1-7', messages: [] });
const msg = body.choices[0].message;
assert.equal(msg.content, 'reasoning only, no tools. ', 'promoted into the visible content');
assert.equal(msg.reasoning_content, undefined, 'promotion moves, not copies');
});
it('(b) think + answer still splits into thinking and text', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: OPEN + 'inner reasoning. ' + CLOSE },
{ type: 'content', text: 'The answer.' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'swe-1-7', messages: [] });
const msg = body.choices[0].message;
assert.equal(msg.reasoning_content, 'inner reasoning. ', 'think span lands on the reasoning channel');
assert.equal(msg.content, 'The answer.', 'the answer stays on the content channel');
});
it('(c) stream variant: think + answer renders as reasoning_content deltas then content deltas', async () => {
__setStreamChatForTest(fakeStream([
{ type: 'content', text: OPEN + 'inner reasoning. ' + CLOSE },
{ type: 'content', text: 'The answer.' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const frames = [];
const result = await streamChatCompletion(
{ model: 'swe-1-7', messages: [] },
(f) => frames.push(f),
{},
);
assert.equal(result.reasoning, 'inner reasoning. ');
assert.equal(result.content, 'The answer.');
const reasoningDeltas = frames.filter((f) => f.choices?.[0]?.delta?.reasoning_content).map((f) => f.choices[0].delta.reasoning_content).join('');
assert.equal(reasoningDeltas, 'inner reasoning. ');
const contentDeltas = frames.filter((f) => f.choices?.[0]?.delta?.content).map((f) => f.choices[0].delta.content).join('');
assert.equal(contentDeltas, 'The answer.');
});
it('respects the gate: with DEVIN_CONNECT_THINKTEXT_REROUTE off, think-tagged content stays text', async () => {
process.env.DEVIN_CONNECT_THINKTEXT_REROUTE = '0';
__setStreamChatForTest(fakeStream([
{ type: 'content', text: OPEN + 'inner reasoning. ' + CLOSE + 'The answer.' },
{ type: 'finish', reason: 'stop', usage: null },
]));
const { body } = await toChatCompletion({ model: 'swe-1-7', messages: [] });
const msg = body.choices[0].message;
assert.equal(msg.content, OPEN + 'inner reasoning. ' + CLOSE + 'The answer.');
assert.equal(msg.reasoning_content, undefined);
});
it('history isolation: a leading think block inside an INBOUND history message is forwarded upstream untouched, even with the gate on', async () => {
// The classifier runs ONLY on live upstream output events. Caller-pasted
// content (a quoted transcript, a literal "what does the tag mean") rides
// the inbound message list and must never be reclassified — that would
// lose attribution, the mirror image of the leak this feature fixes.
process.env.DEVIN_CONNECT_THINKTEXT_REROUTE = '1';
let captured = null;
__setStreamChatForTest(async function* (params) {
captured = params;
yield { type: 'content', text: 'ok' };
yield { type: 'finish', reason: 'stop', usage: null };
});
const history = [
{ role: 'user', content: 'what does ' + OPEN + CLOSE + ' mean?' },
{ role: 'assistant', content: OPEN + 'quoted from a log' + CLOSE + ' and here is my real answer' },
{ role: 'user', content: 'go on' },
];
const { status, body } = await toChatCompletion({ model: 'swe-1-7', messages: history });
assert.equal(status, 200);
assert.deepEqual(captured.messages, history);
assert.equal(body.choices[0].message.content, 'ok');
});
});