/** * Local response cache for chat completions. * * Cascade/Windsurf upstream does not expose Anthropic-style prompt caching, * so we add an in-memory, exact-match cache keyed on the normalized request * body. This only helps with duplicate requests (Claude Code retries, parallel * identical calls), not prefix-caching. */ import { createHash } from 'crypto'; import { log } from './config.js'; const TTL_MS = 5 * 60 * 1000; const MAX_ENTRIES = 500; const DEFAULT_MAX_BYTES = 16 * 1024 * 1024; export function isCacheEnabled() { const raw = String(process.env.RESPONSE_CACHE_ENABLED ?? process.env.WINDSURFAPI_RESPONSE_CACHE ?? '1') .trim() .toLowerCase(); return !['0', 'false', 'off', 'no'].includes(raw); } // Map preserves insertion order → we evict the oldest when over capacity. const _store = new Map(); const _stats = { hits: 0, misses: 0, stores: 0, evictions: 0, skips: 0 }; let _bytes = 0; function bytesEnv(names, fallback) { for (const name of names) { const raw = String(process.env[name] || '').trim(); if (!raw) continue; const m = raw.match(/^(\d+(?:\.\d+)?)\s*(b|kb|kib|k|mb|mib|m|gb|gib|g)?$/i); if (!m) continue; const n = Number(m[1]); if (!Number.isFinite(n) || n <= 0) continue; const unit = (m[2] || 'b').toLowerCase(); const mul = unit === 'gb' || unit === 'gib' || unit === 'g' ? 1024 ** 3 : unit === 'mb' || unit === 'mib' || unit === 'm' ? 1024 ** 2 : unit === 'kb' || unit === 'kib' || unit === 'k' ? 1024 : 1; return Math.floor(n * mul); } return fallback; } function maxBytes() { return bytesEnv( ['RESPONSE_CACHE_MAX_BYTES', 'WINDSURFAPI_RESPONSE_CACHE_MAX_BYTES'], DEFAULT_MAX_BYTES ); } function valueBytes(value) { try { return Buffer.byteLength(JSON.stringify(value), 'utf8'); } catch { return Number.POSITIVE_INFINITY; } } function deleteEntry(key) { const entry = _store.get(key); if (!entry) return false; _store.delete(key); _bytes = Math.max(0, _bytes - (Number(entry.bytes) || 0)); return true; } function digestBase64Data(data = '', mime = '') { const compact = String(data).replace(/\s/g, ''); const bytes = Math.floor(compact.length * 3 / 4) - (compact.endsWith('==') ? 2 : compact.endsWith('=') ? 1 : 0); const hash = createHash('sha256').update(compact).digest('hex').slice(0, 32); return `[base64:${String(mime || 'application/octet-stream').toLowerCase()}:sha256=${hash}:bytes=${Math.max(0, bytes)}]`; } function normalizeDataUrl(url) { const clean = String(url || '').replace(/\s/g, ''); const m = clean.match(/^data:([^;,]+)(?:;[^,]*)?;base64,(.*)$/i); if (!m) return url; return `data:${m[1].toLowerCase()};base64,${digestBase64Data(m[2], m[1])}`; } // Cache-key normalisation must be TOTAL: it runs on the RAW request body // before any validation (the wrapper computes the original key first, by // design — chat.js:2840), so every JSON value a client can actually send must // produce a key rather than a throw. Non-object entries — a null part // (`content:[null]`), a null message, a bare string/number — pass through // unchanged; the shared tail guard rejects or the wire builder coerces them // later. Before this guard, `content:[null]` from any caller with a // trustworthy per-user scope threw // `TypeError: Cannot read properties of null (reading 'type')` here and the // route answered 500 before the guard could return its 400. function normalizeBinary(messages) { if (!Array.isArray(messages)) return messages; return messages.map(m => { if (!m || typeof m !== 'object') return m; if (!Array.isArray(m.content)) return m; return { ...m, content: m.content.map(p => { if (!p || typeof p !== 'object') return p; if (p.type === 'image_url' && typeof p.image_url?.url === 'string' && p.image_url.url.startsWith('data:')) return { ...p, image_url: { ...p.image_url, url: normalizeDataUrl(p.image_url.url) } }; if (p.type === 'image' && p.source?.type === 'base64') return { ...p, source: { ...p.source, data: digestBase64Data(p.source.data, p.source.media_type) } }; if ((p.type === 'file' || p.type === 'input_file') && typeof p.file?.file_data === 'string' && p.file.file_data.startsWith('data:')) return { ...p, file: { ...p.file, file_data: normalizeDataUrl(p.file.file_data) } }; return p; })}; }); } // Deep, key-sorted clone so structurally-equal objects serialize to the same // string regardless of key insertion order. JSON.stringify preserves insertion // order, so `{"50256":-100,"100":1}` and `{"100":1,"50256":-100}` — the SAME // logit_bias map — hashed to different cache keys and split the slot, silently // halving the hit rate (never wrong data, just a miss). Arrays keep their order // (order is semantic there); only object keys are sorted. (audit #10) // JSON.parse accepts deeper nesting than an unguarded recursive clone can // survive: a body nested ~10k deep parsed fine, then the clone overflowed the // stack into a per-request RangeError. Same depth idiom as retainedBytes / // SUB_DUMP_MAX_DEPTH (S4-D14). Literal stays inline: test harnesses evaluate // function text without module scope. function stableClone(value, depth = 0) { if (depth > 64) throw new Error('stableClone: value exceeds maximum nesting depth (64)'); if (Array.isArray(value)) return value.map((v) => stableClone(v, depth + 1)); if (value && typeof value === 'object') { const out = {}; for (const k of Object.keys(value).sort()) out[k] = stableClone(value[k], depth + 1); return out; } return value; } function normalize(body) { return { model: body.model || '', messages: normalizeBinary(body.messages || []), // tools / tool_choice / response_format / thinking / stream_options are // nested objects whose inner JSON-Schema key order is not canonical across // clients. Sort keys (same fix as logit_bias below, audit #10) so two // semantically-identical requests differing only in key order share a cache // slot instead of splitting it (a miss, never wrong data). Arrays keep order. tools: body.tools ? stableClone(body.tools) : null, tool_choice: body.tool_choice ? stableClone(body.tool_choice) : null, response_format: body.response_format ? stableClone(body.response_format) : null, // cache-01: resolve the reasoning effort the SAME way the router does // (chat.js mergeReasoningEffortIntoModel: reasoning_effort || reasoning.effort). // The nested `reasoning:{effort}` form (codex CLI, OpenAI Responses) merges // into the model id before routing but never mutates body.model, so keying // only the flat field collapsed two different reasoning tiers into one slot → // the second request got the first's wrong-tier answer. reasoning_effort: (body.reasoning_effort ?? body.reasoning?.effort) ?? null, thinking: body.thinking ? stableClone(body.thinking) : null, stream_options: body.stream_options ? stableClone(body.stream_options) : null, temperature: body.temperature ?? null, top_p: body.top_p ?? null, // top_k is a live sampling knob on the DEVIN_CONNECT completion config // (chat.js reads body.top_k → completionOverrides.topK; messages.js threads // the Anthropic-side top_k through). Omitting it from the key meant two // requests differing ONLY in top_k shared a cache slot → the second got a // reply sampled under the first's top_k. (audit #9) top_k: body.top_k ?? null, // O3: resolve max_completion_tokens (modern OpenAI spelling) with the same // precedence handleChatCompletions uses, so the two field names collapse to // one cache dimension — a request sending max_completion_tokens:N and one // sending max_tokens:N are the same generation and share a cache slot, while // differing caps still key apart. max_tokens: (Number.isFinite(body.max_completion_tokens) ? body.max_completion_tokens : body.max_tokens) ?? null, // Output-affecting params — omitting these served a response generated // under a different stop/seed/penalty config for an otherwise-identical // body. `stop` is set from Anthropic stop_sequences in messages.js. stop: body.stop ?? null, seed: body.seed ?? null, frequency_penalty: body.frequency_penalty ?? null, presence_penalty: body.presence_penalty ?? null, // logit_bias is a token-id→bias map with no canonical key order; sort keys // so two key-order-permuted-but-equal maps land in the same cache slot (#10). logit_bias: body.logit_bias ? stableClone(body.logit_bias) : null, n: body.n ?? null, }; } /** * Build a cache key for a chat request. * * `callerKey` is required to scope the cache to the specific upstream * tenant — earlier versions hashed only the request body, which let one * caller's "hi" return another caller's cached response from the same * model. Pass an empty string only for tests; production callers must * thread the request's authenticated callerKey through. * * Implementation note: prefix the JSON with the caller scope and a * separator so two distinct callers can't collide by crafting bodies * that serialize to identical strings. */ export function cacheKey(body, callerKey = '') { const scope = String(callerKey || ''); const json = JSON.stringify(normalize(body)); return createHash('sha256').update(scope).update('\0').update(json).digest('hex'); } export function cacheGet(key) { if (!isCacheEnabled()) return null; const entry = _store.get(key); if (!entry) { _stats.misses++; return null; } if (entry.expiresAt < Date.now()) { deleteEntry(key); _stats.misses++; return null; } // Refresh LRU position _store.delete(key); _store.set(key, entry); _stats.hits++; return entry.value; } export function cacheSet(key, value) { if (!isCacheEnabled()) return; // Don't cache empty or partial results if (!value || (!value.text && !(value.chunks && value.chunks.length))) return; const bytes = valueBytes(value); const limit = maxBytes(); deleteEntry(key); if (!Number.isFinite(bytes) || bytes > limit) { _stats.skips++; return; } _store.set(key, { value, expiresAt: Date.now() + TTL_MS, bytes }); _bytes += bytes; _stats.stores++; while (_store.size > MAX_ENTRIES || _bytes > limit) { const oldest = _store.keys().next().value; if (oldest === undefined) break; deleteEntry(oldest); _stats.evictions++; } } export function cacheStats() { const total = _stats.hits + _stats.misses; return { enabled: isCacheEnabled(), size: _store.size, maxSize: MAX_ENTRIES, bytes: _bytes, maxBytes: maxBytes(), ttlMs: TTL_MS, hits: _stats.hits, misses: _stats.misses, stores: _stats.stores, evictions: _stats.evictions, skips: _stats.skips, hitRate: total > 0 ? ((_stats.hits / total) * 100).toFixed(1) : '0.0', }; } export function cacheClear() { _store.clear(); _bytes = 0; _stats.hits = 0; _stats.misses = 0; _stats.stores = 0; _stats.evictions = 0; _stats.skips = 0; log.info('Response cache cleared'); }