import { clientFacingModelText, type ClientFacingModelLabel } from "../../lib/error-destination"; /** * Pre-dispatch input admission (#1412). * * Refuses a turn whose estimated input cannot plausibly fit the model context window, * BEFORE auth resolution, circuit admission, or any upstream I/O. #1412 reported ~127k of * real context compounding to 1.3M-1.6M tokens and crashing the proxy; the provider would * reject such a turn anyway, so paying for the round trip buys nothing. * * Deliberately narrow. This is not a context manager and not a compaction trigger: it * catches the pathological case and stays out of the way otherwise. Every uncertainty * resolves toward admitting. */ import { collapseDevinModelUid, DEVIN_MODEL_CONTEXT_WINDOWS } from "../../adapters/devin/live-models"; import { nativeOpenAiContextWindow, nativeOpenAiMaxInputTokens, nativeOpenAiMaxOutputTokens, type NativeContextLimitsInput, } from "../../codex/catalog/metadata"; import { getModelMetadata } from "../../generated/model-metadata"; import { estimateTokens } from "../../lib/token-estimate"; import { isCanonicalOpenAiForwardProvider, OPENAI_CODEX_PROVIDER_ID } from "../../providers/openai-tiers"; import { modelRecordValue } from "../../reasoning-effort"; import { modelInList, type OcxContentPart, type OcxParsedRequest, type OcxProviderConfig } from "../../types"; /** * Multiplier applied to the ceiling before refusing. * * 2.5, not something tighter, because the estimate is a heuristic and the cost of being wrong * is asymmetric: a false refusal fails a turn the provider would have answered, while an * over-admission merely pays for one round trip the provider then rejects itself. * * The original margin was sized against a sampling artifact — `cjkRatio` read every stride-th * character, so a payload of fixed-width records could sample as 100% CJK while being 1.6% CJK * and inflate the estimate by 4.0/2.5 = 1.6x. That sampler is gone: CJK characters are now * counted exactly, so that particular 1.6x divergence cannot occur and the headroom it bought is * no longer spent on it. * * The margin is still 2.5 because the estimator it guards got LARGER, not smaller. Counting the * two scripts separately raises a pure-Latin estimate by 1.25x and a Korean one by up to 1.67x * against the previous model, which consumes real headroom: measured against the old estimator's * scale, 2.5 now behaves like roughly 2.0x for Latin and 1.5x for Korean-dominant input. That is * the intended direction — the estimates are closer to what providers actually charge, so the * same multiplier is a tighter and more honest bound — and it still refuses the #1412 shape * (10x compounding) several times over. */ export const ADMISSION_TOLERANCE = 2.5; /** * Token cost charged for a remote image URL. The bytes are not in this request — the * provider fetches them — so the URL own length is not the cost. A small flat charge * acknowledges the tiles the image will occupy without pretending to know its dimensions. */ const REMOTE_IMAGE_TOKENS = 850; /** Decoded image bytes per token. Coarse tile-count proxy, not a provider formula. */ const IMAGE_BYTES_PER_TOKEN = 750; /** Compressed-audio bytes per token. More conservative than the image proxy. */ const AUDIO_BYTES_PER_TOKEN = 256; export interface InputAdmissionResult { admitted: boolean; estimatedTokens: number; /** Resolved ceiling, or null when nothing could be resolved (=> always admitted). */ ceiling: number | null; /** Actual shared context window when an input-only ceiling differs. */ contextWindow?: number | null; /** Output space reserved by the combo preflight; absent on the loose direct gate. */ requiredOutputHeadroom?: number; } function positive(value: unknown): number | null { return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : null; } /** * Charge a `data:` URL by its DECODED size rather than its character length: base64 inflates * by 4/3, so charging the string would overcount by a third. A remote URL is charged flat. */ function imageTokens(imageUrl: string): number { if (!imageUrl.startsWith("data:")) return REMOTE_IMAGE_TOKENS; const comma = imageUrl.indexOf(","); if (comma < 0) return REMOTE_IMAGE_TOKENS; const payload = imageUrl.length - comma - 1; if (payload <= 0) return 0; const decoded = Math.floor((payload * 3) / 4); return Math.max(1, Math.ceil(decoded / IMAGE_BYTES_PER_TOKEN)); } function contentPartTokens(part: OcxContentPart, modelId: string): number { if (part.type === "image") return imageTokens(part.imageUrl); if (part.type === "video") return imageTokens(part.videoUrl); if (part.type === "audio") return base64PayloadTokens(part.data, AUDIO_BYTES_PER_TOKEN); // An inline document is a base64 payload, not a sentence: estimating it from its marker would // admit a request whose real input is orders of magnitude larger. Counted arithmetically — // rebuilding the data URL here would materialize a second request-sized string just to measure it. if (part.type === "document") return base64PayloadTokens(part.data); return estimateTokens(part.text, modelId); } function base64PayloadTokens(base64: string, bytesPerToken = IMAGE_BYTES_PER_TOKEN): number { if (base64.length === 0) return 0; const decoded = Math.floor((base64.length * 3) / 4); return Math.max(1, Math.ceil(decoded / bytesPerToken)); } function contentTokens(content: string | readonly OcxContentPart[], modelId: string): number { if (typeof content === "string") return estimateTokens(content, modelId); let total = 0; for (const part of content) total += contentPartTokens(part, modelId); return total; } /** * Estimate the input tokens of a parsed request. * * Walks the whole `OcxMessage` union rather than user text alone. Assistant turns carry * their content as `OcxAssistantContentPart[]` — text, thinking blocks, and tool calls whose * JSON arguments are frequently the largest single item in an agent conversation. A walk * that counted only `{type:"text"}` would undercount exactly the turns that trigger this * gate. The routed provider determines whether replayed thinking reaches the wire. */ export function estimateInputTokens(parsed: OcxParsedRequest, modelId: string, provider?: OcxProviderConfig): number { const { context } = parsed; const countThinking = provider?.adapter !== "openai-chat" || modelInList(provider.preserveReasoningContentModels, parsed.modelId); let total = 0; for (const prompt of context.systemPrompt ?? []) total += estimateTokens(prompt, modelId); for (const message of context.messages) { if (message.role === "assistant") { for (const part of message.content) { if (part.type === "text") total += estimateTokens(part.text, modelId); else if (part.type === "thinking") { if (countThinking) total += estimateTokens(part.thinking, modelId); } else total += estimateTokens(part.name, modelId) + estimateTokens(JSON.stringify(part.arguments), modelId); } // Opaque provider blob replayed verbatim upstream, so it costs real input tokens. if (message.kiroRedactedReasoning) total += estimateTokens(message.kiroRedactedReasoning, modelId); continue; } total += contentTokens(message.content, modelId); } // Tool schemas ride every turn: name, description, and the JSON parameter schema all // reach the upstream, and a large MCP catalog can dominate a short conversation. for (const tool of context.tools ?? []) { total += estimateTokens(tool.name, modelId) + estimateTokens(tool.description, modelId) + estimateTokens(JSON.stringify(tool.parameters), modelId); } return total; } /** * Resolve the admission ceiling. Pure: no filesystem, no catalog, no registry scan. * * `provider` must be the ROUTED config (`route.provider`), which `routedProviderConfig` * has already transport-guarded and merged. Re-deriving from `config.providers[name]` would * reject a user-defined provider that merely shares a built-in name using limits that * belong to a different service. */ interface ResolvedContextLimits { /** The target's total context window: input and output share it. */ window: number | null; /** Largest admissible input, which input-only caps may tighten below the window. */ ceiling: number | null; } /** * Adapter-owned static window, consulted only when the routed provider config declares none. * Devin is the case that matters: its live catalog windows reach the Codex-facing model list * but never `provider.modelContextWindows`, so without this fallback every devin target * resolved `window: null` and the strict combo gate stayed fail-open on exactly the turns it * exists to skip. Explicit provider config still wins; the effort-suffix collapse covers * catalog UIDs like `swe-2-flash-medium`. */ function adapterContextWindow(provider: OcxProviderConfig, modelId: string): number | null { if (provider.adapter !== "devin") return null; // Devin spells the same model dotted in catalog families (`swe-1.7`) and hyphenated // in uids (`swe-1-7`); normalize before either lookup so both spellings resolve. const normalized = modelId.replace(/\./g, "-"); const collapsed = collapseDevinModelUid(normalized); // A configured window for the collapsed base (operator override or registry seed) still // wins over the static table for effort-suffixed ids like swe-2-flash-medium. return positive(modelRecordValue(provider.modelContextWindows, collapsed)) ?? positive(modelRecordValue(DEVIN_MODEL_CONTEXT_WINDOWS, normalized)) ?? positive(modelRecordValue(DEVIN_MODEL_CONTEXT_WINDOWS, collapsed)); } function resolveContextLimits( provider: OcxProviderConfig, providerName: string, modelId: string, nativeContextCap?: NativeContextLimitsInput, ): ResolvedContextLimits { // `modelRecordValue`, not a bare lookup: the catalog resolves these same two maps that // way, so a `gpt-oss` entry covers `gpt-oss:120b`. Reading raw here made the gate fall // back to the provider-wide window and refuse turns the model can plainly hold. const configured = positive(modelRecordValue(provider.modelContextWindows, modelId)) ?? positive(provider.contextWindow); // The canonical `openai` registry entry declares no context fields, so without this the // gate would be inert on the default Codex route. All three clauses are load-bearing: a // transport-mismatched custom provider named "openai" is preserved verbatim by routing // and must not inherit built-in native limits, and a routed `provider/model` id is not a // native slug. Static maps only — no catalog read. const canonicalNativeBare = providerName === OPENAI_CODEX_PROVIDER_ID && isCanonicalOpenAiForwardProvider(provider) && !modelId.includes("/"); const nativeLimits = canonicalNativeBare && configured !== null ? { ...(typeof nativeContextCap === "number" ? { cap: nativeContextCap } : (nativeContextCap ?? {})), modelWindows: { [modelId]: configured }, } : nativeContextCap; const native = canonicalNativeBare ? positive(nativeOpenAiContextWindow(modelId, nativeLimits)) : null; const nativeMaxInput = canonicalNativeBare ? positive(nativeOpenAiMaxInputTokens(modelId, nativeLimits)) : null; const window = canonicalNativeBare ? (native ?? generatedNativeWindow(modelId, configured, nativeContextCap)) : (configured ?? adapterContextWindow(provider, modelId)); // modelMaxInputTokens is an input-only cap, so it can only tighten the window. const configuredMaxInput = positive(modelRecordValue(provider.modelMaxInputTokens, modelId)); const limits = [window, configuredMaxInput, nativeMaxInput].filter((v): v is number => v !== null); return { window, ceiling: limits.length === 0 ? null : Math.min(...limits) }; } /** * Generated-catalog keys, not routing provider names. `OPENAI_CODEX_PROVIDER_ID` is the string * `"openai"` -- the canonical Codex forward route -- so using it to index the generated bundle * would silently skip the native Codex rows and read the public API rows instead. */ const NATIVE_METADATA_CATALOGS = ["openai-codex", "openai"] as const; /** * Static in-tree metadata for a canonical native slug the narrower override and pinned-native * tables do not carry. Falling through to null made input admission completely blind for * exactly those models, which is how a 128k target accepted a turn it could not finish. * * This deliberately covers slugs that are no longer offered in the picker: a retired slug is * still dispatchable when an operator names it explicitly in a combo target, and that is the * configuration where the gate was inert. This is a generated bundle compiled into the binary, * not a live catalog read, so it adds no I/O. Explicit provider and operator caps may only * narrow the result, never widen it. */ function generatedNativeWindow( modelId: string, configured: number | null, nativeContextCap: NativeContextLimitsInput | undefined, ): number | null { let generated: number | null = null; for (const catalog of NATIVE_METADATA_CATALOGS) { generated = positive(getModelMetadata(catalog, modelId)?.contextWindow); if (generated !== null) break; } if (generated === null) return null; const cap = typeof nativeContextCap === "number" ? positive(nativeContextCap) : positive(nativeContextCap?.cap); return Math.min(generated, configured ?? generated, cap ?? generated); } export function resolveInputCeiling( provider: OcxProviderConfig, providerName: string, modelId: string, // Operator cap for the canonical native provider. Passed in rather than read from a // config here so this stays pure: no filesystem, no catalog, no registry scan. nativeContextCap?: NativeContextLimitsInput, ): number | null { return resolveContextLimits(provider, providerName, modelId, nativeContextCap).ceiling; } /** * Largest output the concrete target can emit. Used only to avoid reserving MORE than the * target could ever produce when a client asks for a bigger allowance than the model has. * Unknown stays unknown rather than inventing a capability. */ export function resolveOutputCeiling( provider: OcxProviderConfig, providerName: string, modelId: string, ): number | null { const configured = positive(modelRecordValue(provider.modelMaxOutputTokens, modelId)) ?? positive(provider.defaultMaxOutputTokens); const canonicalNativeBare = providerName === OPENAI_CODEX_PROVIDER_ID && isCanonicalOpenAiForwardProvider(provider) && !modelId.includes("/"); const native = canonicalNativeBare ? positive(nativeOpenAiMaxOutputTokens(modelId)) : null; const limits = [configured, native].filter((v): v is number => v !== null); return limits.length === 0 ? null : Math.min(...limits); } /** * Combo-only admission. A fallback must be able to satisfy the caller's declared output * allowance inside its OWN context window. Otherwise it returns 200, emits a few hundred * tokens, and terminates on `finish_reason: length` — which the Anthropic surface renders as * "response exceeded the output token maximum" even though the real cause was the total * window. By then the next target cannot be tried, because output has already committed. * * Two budgets are checked separately so the reserve is counted exactly once. `ceiling` is an * input-only budget once `modelMaxInputTokens` tightens it below the window, so the output * reserve belongs against `window`, not against `ceiling`. * * Direct and single-target requests keep the deliberately loose 2.5x pathological-input gate. * This stricter rule applies only to synthetic combo children, where skipping one known-small * target is safe and the ladder continues before any upstream bytes are sent. Unknown context * stays fail-open, and a caller that declared no output allowance is unaffected. */ export function checkComboTargetInputAdmission( parsed: OcxParsedRequest, provider: OcxProviderConfig, providerName: string, modelId: string, nativeContextCap?: NativeContextLimitsInput, ): InputAdmissionResult { const { window, ceiling } = resolveContextLimits(provider, providerName, modelId, nativeContextCap); const requestedOutput = positive(parsed.options.maxOutputTokens); if (window === null || ceiling === null || requestedOutput === null) { return checkInputAdmission(parsed, provider, providerName, modelId, nativeContextCap); } const targetOutput = resolveOutputCeiling(provider, providerName, modelId); const requiredOutputHeadroom = targetOutput === null ? requestedOutput : Math.min(requestedOutput, targetOutput); const estimatedTokens = estimateInputTokens(parsed, modelId, provider); return { admitted: estimatedTokens <= ceiling && estimatedTokens + requiredOutputHeadroom <= window, estimatedTokens, ceiling, requiredOutputHeadroom, ...(window !== ceiling ? { contextWindow: window } : {}), }; } /** * Fail-open when no ceiling is known; refuse only past `ceiling * ADMISSION_TOLERANCE`. * * The caller is responsible for skipping compaction turns — see the call site in core.ts. */ export function checkInputAdmission( parsed: OcxParsedRequest, provider: OcxProviderConfig, providerName: string, modelId: string, nativeContextCap?: NativeContextLimitsInput, ): InputAdmissionResult { const { window, ceiling } = resolveContextLimits(provider, providerName, modelId, nativeContextCap); if (ceiling === null) return { admitted: true, estimatedTokens: 0, ceiling: null }; const estimatedTokens = estimateInputTokens(parsed, modelId, provider); return { admitted: estimatedTokens <= ceiling * ADMISSION_TOLERANCE, estimatedTokens, ceiling, ...(window !== ceiling ? { contextWindow: window } : {}) }; } export function inputAdmissionRefusalMessage(input: InputAdmissionResult, model: ClientFacingModelLabel): string { const count = (value: number | null) => value !== null && Number.isFinite(value) && value >= 0 ? Math.trunc(value) : 0; const label = clientFacingModelText(model); const estimate = count(input.estimatedTokens); const ceiling = count(input.ceiling); if (input.contextWindow !== undefined && input.estimatedTokens > ceiling) { const window = input.contextWindow === null ? "" : `; context window: ${count(input.contextWindow)} tokens`; return `Estimated input (~${estimate} tokens) exceeds the input limit of ${label} (${ceiling} tokens${window}). ` + "Start a new session or choose a model with a larger input limit."; } const window = count(input.contextWindow ?? input.ceiling); const inputLimit = input.contextWindow !== undefined ? `; input limit: ${ceiling} tokens` : ""; return input.requiredOutputHeadroom !== undefined ? `Estimated input (~${estimate} tokens) plus ${count(input.requiredOutputHeadroom)} ` + `tokens of requested output headroom cannot fit the context window of ${label} ` + `(${window} tokens${inputLimit}).` : `Estimated input (~${estimate} tokens) is far past the context window ` + `of ${label} (${window} tokens${inputLimit}). Start a new session or choose a ` + `model with a larger context window.`; }