// WINDSURFAPI_STRICT_MODEL=0 degrades an unmapped model to the free selector. The // degrade is a deliberate operator opt-out. Reporting the PAID name back as if it ran // is not — and that is what the response used to do. // // Measured before this fix: `claude-opus-4.9` returned 200 with // `model: "claude-opus-4.9"` on the non-stream body AND on every stream chunk, plus a // system_fingerprint derived from that name, while `swe-1-6-slow` ran upstream. Wrong // output, wrong billing attribution, and a client that trusts the echoed name is // silently misled. #234's acceptance criteria call this out explicitly. // // Nothing in the suite asserted the echoed name on this path — verified by instrumenting // the `mapped === false` branch and running all test files: exactly one test reaches it // (devin-connect-strict-model.test.js:62) and it only asserts "not a 400". So this file // is the first coverage of the contract. // // The load-bearing subtlety is WHICH name to report. See the router-hop test below. import { afterEach, beforeEach, describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { handleChatCompletions, __setConnectDeps } from '../src/handlers/chat.js'; import { addAccountByKey, removeAccount, getAccountInternal } from '../src/auth.js'; // The fingerprint case below runs the REAL adapter (no toChatCompletion stub) so it // asserts the production derivation, not a fixture's copy of it: the adapter builds // system_fingerprint as systemFingerprint(displayModel || params.model) // (src/devin-connect-openai.js:565), and __setStreamChatForTest is that module's // documented offline seam (same usage as test/devin-connect-openai.test.js). import { toChatCompletion as realToChatCompletion, __setStreamChatForTest } from '../src/devin-connect-openai.js'; import { systemFingerprint } from '../src/system-fingerprint.js'; const FREE_SELECTOR = 'swe-1-6-slow'; const UNMAPPED_PAID = 'claude-opus-4.9'; const MAPPED_ALIAS = 'claude-sonnet-4.6'; const SAVED = {}; const ENV_KEYS = ['DEVIN_CONNECT', 'WINDSURFAPI_STRICT_MODEL', 'DEVIN_CONNECT_TOKEN', 'WINDSURF_API_KEY', 'DEVIN_CONNECT_ASSIGN_MODEL']; for (const k of ENV_KEYS) SAVED[k] = process.env[k]; /** Records what the adapter was handed, so we can compare wire vs echoed name. */ let seen = null; const seeded = []; /** * Put one usable account in the pool. * * Required for the AssignModel router-hop tests: connectParams.token is only set from an * ACQUIRED account (chat.js, `if (ccAcct) connectParams.token = ccAcct.apiKey`), and the * hop is gated on that token being present. With an env token alone the hop is skipped * and the router assertions fail on their own precondition rather than on the contract — * which is indistinguishable from a passing mutation if you only read pass/fail counts. */ function seedAccount() { const a = addAccountByKey('sk-honesty-' + Math.random().toString(36).slice(2, 12), 'honesty'); const acct = getAccountInternal(a.id); acct.status = 'active'; acct.tier = 'pro'; seeded.push(a.id); return acct; } function stubAdapter() { seen = null; __setConnectDeps({ toChatCompletion: async (params, meta) => { seen = { wire: params.model, echoed: meta.displayModel }; return { status: 200, body: { id: 'chatcmpl-test', object: 'chat.completion', created: 0, // Mirror the real adapter's precedence: displayModel wins when provided. model: meta.displayModel || params.model, choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, }, }; }, }); } async function chat(model) { const res = await handleChatCompletions( { model, messages: [{ role: 'user', content: 'hi' }], stream: false }, { reqId: 'honesty', callerKey: 'honesty-caller', specialAgent: {} }, ); return { status: res?.status ?? null, bodyModel: res?.body?.model ?? null, fingerprint: res?.body?.system_fingerprint ?? null, errCode: res?.body?.error?.code ?? null, wire: seen?.wire ?? null, echoed: seen?.echoed ?? null, }; } beforeEach(() => { process.env.DEVIN_CONNECT = '1'; process.env.DEVIN_CONNECT_TOKEN = 'test-token-shape-only'; delete process.env.WINDSURF_API_KEY; stubAdapter(); }); afterEach(() => { while (seeded.length) removeAccount(seeded.pop()); __setConnectDeps(null); __setStreamChatForTest(null); seen = null; for (const k of ENV_KEYS) { if (SAVED[k] === undefined) delete process.env[k]; else process.env[k] = SAVED[k]; } }); describe('degrade honesty: the echoed model must be what ran (#234)', () => { it('reports the free selector, not the requested paid name, when degrading', async () => { process.env.WINDSURFAPI_STRICT_MODEL = '0'; const r = await chat(UNMAPPED_PAID); assert.equal(r.status, 200, 'the opt-out still serves the request'); assert.equal(r.wire, FREE_SELECTOR, 'precondition: the free selector is what actually ran'); assert.equal(r.bodyModel, FREE_SELECTOR, `the response must not claim to have run ${UNMAPPED_PAID} when ${FREE_SELECTOR} ran`); assert.notEqual(r.bodyModel, UNMAPPED_PAID); }); it('still echoes the REQUESTED name for a mapped alias', async () => { // Alias resolution is not a degrade: claude-sonnet-4.6 → claude-sonnet-4-6-thinking // is the same model under its canonical selector. OpenAI echoes the requested alias // here, and so must we — a fix that reported the resolved selector for mapped names // would be a behaviour change with no defect behind it. process.env.WINDSURFAPI_STRICT_MODEL = '0'; const r = await chat(MAPPED_ALIAS); assert.equal(r.status, 200); assert.notEqual(r.wire, MAPPED_ALIAS, 'precondition: the wire name differs from the alias'); assert.equal(r.bodyModel, MAPPED_ALIAS, 'a mapped alias must keep echoing the requested name'); }); it('is consistent when the requested name IS the selector', async () => { process.env.WINDSURFAPI_STRICT_MODEL = '0'; const r = await chat(FREE_SELECTOR); assert.equal(r.wire, FREE_SELECTOR); assert.equal(r.bodyModel, FREE_SELECTOR); }); it('leaves strict mode rejecting rather than degrading', async () => { // With the guard ON (the default) an unmapped name never reaches the degrade at all, // so honesty is moot — it 400s. Pinned so the honesty fix can't be mistaken for a // reason to relax the guard. process.env.WINDSURFAPI_STRICT_MODEL = '1'; const r = await chat(UNMAPPED_PAID); assert.equal(r.status, 400); assert.equal(r.errCode, 'model_not_found'); assert.equal(r.wire, null, 'nothing reached the adapter'); }); it('reports the ROUTER-ASSIGNED model, not the free selector, after an AssignModel hop', async () => { // The trap. A router name like `adaptive` also resolves mapped:false, so it takes the // same branch — but the AssignModel hop reassigns connectParams.model to a concrete // uid AFTER `selector` was captured. Reporting `selector` here (which is what // HANDOFF-2026-08-03-B.md §3.6 literally prescribed) swaps one lie for another: a // request that really ran claude-opus-4-8-medium gets reported as the free // swe-1-6-slow. Reproduced both ways before choosing connectParams.model. // // Without this test the distinction is invisible: a mutation replacing // connectParams.model with selector passes every other assertion in this file. process.env.WINDSURFAPI_STRICT_MODEL = '0'; process.env.DEVIN_CONNECT_ASSIGN_MODEL = '1'; seedAccount(); const ASSIGNED = 'claude-opus-4-8-medium'; __setConnectDeps({ assignModel: async () => ({ model_uid: ASSIGNED }), }); const r = await chat('adaptive'); assert.equal(r.wire, ASSIGNED, 'precondition: the router hop put the assigned uid on the wire'); assert.equal(r.bodyModel, ASSIGNED, 'the echoed name must be the model the router actually assigned — reporting the ' + 'pre-hop selector would report a free model for a paid run'); assert.notEqual(r.bodyModel, FREE_SELECTOR); assert.notEqual(r.bodyModel, 'adaptive'); }); it('falls back to the selector when the AssignModel hop fails', async () => { // A failed resolve degrades to the original selector, so THAT is what ran and what // must be reported. Pins the other side of the branch. process.env.WINDSURFAPI_STRICT_MODEL = '0'; process.env.DEVIN_CONNECT_ASSIGN_MODEL = '1'; seedAccount(); __setConnectDeps({ assignModel: async () => { throw Object.assign(new Error('nope'), { code: 'ASSIGN_FAILED' }); }, }); const r = await chat('adaptive'); assert.equal(r.wire, FREE_SELECTOR, 'precondition: it degraded to the selector'); assert.equal(r.bodyModel, FREE_SELECTOR, 'and that is what must be reported'); }); it('derives system_fingerprint from the echoed name, not the requested one', async () => { // The fingerprint is keyed to the model. If it stayed keyed to the requested paid // name it would contradict the model field we now report honestly. // // Runs the REAL adapter (the toChatCompletion stub is replaced for this case), so // the fingerprint comes from production code — devin-connect-openai.js:565 keys it // to `displayModel || params.model` — rather than from a fixture that could only // ever agree with itself. The upstream is faked through that module's documented // seam; the fake yields content so the retry-on-empty wrapper does not re-issue. // // The guard this replaced (`if (typeof systemFingerprint !== 'function') return;`) // was never false: chat.js imports systemFingerprint but does not re-export it, so // the case returned early and asserted nothing. Mutating chat.js's connectDisplayModel // back to reqModelName turns it red (measured). process.env.WINDSURFAPI_STRICT_MODEL = '0'; __setStreamChatForTest(async function* () { yield { type: 'content', text: 'ok' }; yield { type: 'finish', reason: 'stop', usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }; }); __setConnectDeps({ toChatCompletion: realToChatCompletion }); const r = await chat(UNMAPPED_PAID); // Fingerprint first: on the degrade path the honest model name and the honest // fingerprint source are the same string, so a displayModel mutation breaks both — // asserting the fingerprint before the model field is what makes a failure report // the fingerprint contract rather than the sibling one. assert.equal(r.fingerprint, systemFingerprint(FREE_SELECTOR), 'the fingerprint must be keyed to the name we report, not to the requested one'); assert.notEqual(r.fingerprint, systemFingerprint(UNMAPPED_PAID)); assert.equal(r.bodyModel, FREE_SELECTOR); // Precondition for the check above: the two names must hash differently, or a // swapped fingerprint would be invisible to it. assert.notEqual(systemFingerprint(FREE_SELECTOR), systemFingerprint(UNMAPPED_PAID), 'precondition: the two names produce different fingerprints'); }); }); describe('degrade honesty reaches the Anthropic and Gemini routes too (#234)', () => { // The partial-path check. chat.js alone fixes /v1/chat/completions and leaves the other // two protocol routes reporting the requested name — measured before this: a // /v1/messages request for claude-opus-4.9 ran swe-1-6-slow and reported // claude-opus-4.9. Both files had ZERO references to displayModel. it('/v1/messages reports what ran, not what was asked for', async () => { process.env.WINDSURFAPI_STRICT_MODEL = '0'; const { handleMessages } = await import('../src/handlers/messages.js'); const res = await handleMessages( { model: UNMAPPED_PAID, max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, { callerKey: 'honesty-caller' }, ); assert.equal(seen?.wire, FREE_SELECTOR, 'precondition: the free selector ran'); assert.equal(res?.body?.model, FREE_SELECTOR, 'the Anthropic route must not report the paid name either'); }); it('/v1/messages still echoes the requested name for a mapped alias', async () => { process.env.WINDSURFAPI_STRICT_MODEL = '0'; const { handleMessages } = await import('../src/handlers/messages.js'); const res = await handleMessages( { model: MAPPED_ALIAS, max_tokens: 16, messages: [{ role: 'user', content: 'hi' }] }, { callerKey: 'honesty-caller' }, ); assert.equal(res?.body?.model, MAPPED_ALIAS, 'alias resolution is not a degrade — the requested name still stands'); }); });