/** * Retry policy for local wrapper event-processing failures (#782). * * The kernel deliberately re-delivers an event that the wrapper does not * acknowledge. That preserves at-least-once handling, but a flat poll loop * turns a model-provider outage into repeated subprocess launches. Keep the * retry policy here so every adapter gets the same bounded behavior. */ import { PUBLIC_MCP_NOT_READY_CODE, PUBLIC_TOOL_FAILED_CODE } from './public-tool-error.js'; import { PUBLIC_CODEX_VERSION_CODE } from './adapters/tested-version.js'; import { modelUnavailableFailure } from './model-unavailable.js'; export const SPAWN_FAILURE_CLASS = Object.freeze({ // The provider refused the request's CONTENT. Re-sending the same content // can only be refused again, so this class is never retried: the run loop // acks the event as refused instead (lib/content-refusal.js). CONTENT_REFUSED: 'content_refused', QUOTA: 'quota', RATE_LIMIT: 'rate_limit', CONFIGURATION: 'configuration', RUNTIME: 'runtime', // The spawn never started: no transcript row, stdout or stderr inside the // first-output window (lib/first-output-watchdog.js). A host condition such as // a locked console, so the circuit opens at once and the probe waits. STALLED: 'stalled', // F293: the child closed on a signal from outside the wrapper (a person, a // supervisor, a walk). The provider never refused anything, so the retry // comes at the normal poll interval and the circuit stays shut. SIGNALED: 'signaled', }); // How many outside signals in a row go uncounted before they count again. A // walk that cuts three turns short is a walk; a child killed on every spawn is // a fault, and after this many the usual schedule backs the seat off. export const EXTERNAL_SIGNAL_GRACE = 3; // Set by the adapter (`markExternalSignal`), only on a failure with no cause of // its own: a stream's error or the adapter's own kill never carries it. export const isExternalSignal = (error) => error?.externalSignal === true; // A stalled probe costs one first-output window, so the wait starts short and // doubles to the usual ceiling: 2, 4, 8, then 15 minutes. export const SPAWN_STALL_RETRY_BASE_MS = 2 * 60 * 1000; export const SPAWN_CIRCUIT_THRESHOLD = 3; // Fallback ceiling when an error has no usable reset clock. export const SPAWN_RETRY_MAX_MS = 15 * 60 * 1000; export const SPAWN_RETRY_JITTER_MAX_RATIO = 0.2; // This list is a per-provider allowlist, and it only ever grows after an // outage has already been misclassified. Twice now: // 2026-08-03 codex "Your workspace is out of credits." → `out of credits` // 2026-08-18 claude "You've hit your session limit" → `session limit` // 2026-08-25 claude "You've hit your weekly limit" → the phrase below // The second one is the instructive failure: `usage limit` was already here — // it is Claude's OTHER exhaustion wording — so the fleet stalled for an hour on // a string one word away from a pattern we had. Both times the miss meant // RUNTIME, the weakest class with the shortest backoff, against a provider that // was not going to answer for hours. // // What a miss costs, measured against these constants (intervalMs 5000, // jitter 0; `*` = circuit open): // // quota n=1,2,3 -> 900s* 900s* 900s* // configuration n=1,2,3 -> 900s* 900s* 900s* // rate_limit n=1,2,3 -> 60s* 120s* 240s* // runtime n=1,2,3 -> 5s 10s 60s* // // So an unmatched wording is not one class off — it is 180x faster on the // first retry than the class it belonged in, and the only class that does not // open the circuit at n=1. That is the price of a missing string, and it is // why the entry below is a list of exact wordings rather than a loose pattern. // // Note RUNTIME is also `classifySpawnFailure`'s fallthrough, so "unrecognised" // and "transient local fault" resolve to the same, most aggressive schedule. // That is a structural issue rather than a vocabulary one — see #996. // // Deliberately NOT loosened to a bare `limit`: QUOTA is tested before // RATE_LIMIT, so that would swallow every "rate limit" error into the 15-minute // cooldown. Add exact wordings, not looser ones. // Measured 2026-10-16 from a codex seat (gpt-6-astra over the ChatGPT login), // on a security-gate request carrying SSRF/exfil/ReDoS test cases: "codex turn // failed: This content was flagged for possible cybersecurity risk. If this // seems wrong, try rephrasing your request. If you're doing authorized security // work … apply for Daybreak access …". Classed as `runtime`, it was re-sent on // backoff 5 times in 52 minutes, every one refused, and the seat handled // nothing else until the kernel retired the event. Only the refusal wordings // themselves are matched, so an ordinary failure that mentions a flag is not. const CONTENT_REFUSED_RE = /(?:flagged for possible cybersecurity risk|this content was flagged|content[_ ]policy[_ ]violation|cyber[_ ]policy)/i; // pi writes its own line for a provider's content stop (pi-ai 0.84.1: // openai-completions.js, anthropic-messages.js) as the whole errorMessage, so // only the pi failure that IS that line matches. Anywhere else the wording is // text someone wrote, such as a seat quoting this file, and a match is terminal. const PI_CONTENT_STOP_RE = /^pi exited with code \d+(?: \(upstream \d+\))?: Provider (?:finish_reason: content_filter|stopped with: sensitive)$/; // The third miss is why there is now a PHRASE as well as a word list, and it is // a different failure from the first two. Measured across the fleet's logs on // 2026-08-25: 283 failures reading "You've hit your weekly limit", 55 reading // "You've hit your session limit" (matched), and 6 reading "You've reached your // Fable 5 limit" (not matched). Three wordings, one sentence shape, and the list // had caught exactly one of them. Enumerating per-wording keeps failing because // the variable part is a BILLING PERIOD or a MODEL NAME, both of which the // providers keep adding, so a list is structurally one release behind. // // `QUOTA_POSSESSIVE_RE` matches that shape and nothing looser: the subject must // be the caller's own allowance ("you've hit/reached YOUR ... limit"), which is // what distinguishes exhaustion from a server-side throttle. The negative // lookahead keeps "you've hit your rate limit" out — QUOTA is tested first, and // without it that string would take the 15-minute cooldown instead of the // 60-second rate-limit backoff. The length bound stops it spanning sentences. // // The character class admits a dot ONLY when a digit follows it, and that // carve-out is load-bearing rather than cosmetic (sprint-review, 2026-08-25): // a plain `[^.\n]` reads as "stay inside one sentence" and silently also means // "no model names". Every model we run is dotted, so "you've reached your Haiku // 4.5 limit" fell through to RUNTIME, the shortest backoff — the class excluded // the exact variable part this paragraph says keeps changing. // // Known limit of `\.(?=\d)`, kept rather than papered over: a period with a // digit immediately after it is admitted even at a sentence boundary, so // "You've reached your goal.5 things have no limit" classes as QUOTA. The space // in ordinary prose ("goal. 5 things") is what terminates the match, and // adjacent `.`+digit is rare enough in provider copy to accept. The test below // pins what the pattern ACTUALLY has — a space after the period ends the match — // rather than the stronger property it does not have. // The gateway's two budget refusals. A key over its budget answers 422 with this // exact text (walked on a real key, 2026-11-06); a model group over its monthly // pool answers 429 "crossed budget" (our gateway config). Case-sensitive, so a // "thinking budget has been exceeded" stays out. Unmatched, the 422 is RUNTIME // and the seat respawns every few seconds against a key that stays refused. export const GATEWAY_BUDGET_RE = /Budget has been exceeded!|crossed budget/; // `credit_balance_exhausted` is OpenAI's code for a prepaid balance at zero, sent // as a 429. `credit balance` has a space, so it needs its own entry. OpenAI: // "Retrying billing, spend, or quota errors won't restore API access." // This reaches the code only while it's in the text, as a bare code or a body // kept whole. A body the keep-list reduces keeps its message and moves the code // to `error.providerCode`, which isn't read here because it can come from a // tool's JSON. That refusal stays RATE_LIMIT and probes in 60 seconds. const QUOTA_RE = /(?:quota|usage limit|session limit|weekly limit|credit balance|credit_balance_exhausted|out of credits|billing|insufficient[_ -]?quota|resource exhausted|spending limit)/i; const QUOTA_POSSESSIVE_RE = /you'?ve (?:hit|reached) your (?!rate[ -]?limit)(?:[^.\n]|\.(?=\d)){0,40}?limit/i; // The status code is not a throttle wording: "429: You've reached your Fable 5 // limit" is an exhausted allowance that a provider reports as HTTP 429. const THROTTLE_WORDING_RE = /(?:rate[ -]?limit|too many requests|overloaded|capacity)/i; const RATE_LIMIT_RE = /(?:rate[ -]?limit|too many requests|\b429\b|overloaded|capacity)/i; const CONFIGURATION_RE = /(?:ENOENT|command not found|not on PATH|login required|not logged in|invalid api key|authentication failed|unauthori[sz]ed|forbidden|\b40[13]\b)/i; // The zone is an IANA name: Area/Location (America/Port-au-Prince, Etc/GMT+5) // or a bare UTC/GMT, as a CI box or container seat reports. An unknown name // fails Intl below and keeps the fallback. const QUOTA_RESET_RE = /\bresets?\s+(\d{1,2})(?::(\d{2}))?\s*(am|pm)\s*\((UTC|GMT|[A-Za-z_]+(?:\/[A-Za-z0-9_+-]+)+)\)/i; const QUOTA_RESET_MARGIN_MS = 2 * 60 * 1000; const CONNECTION_FAULT_RE = /(?:ECONNRESET|ECONNREFUSED|ETIMEDOUT|EPIPE|socket hang up|fetch failed)/; const errorText = (error) => [ error?.message, error?.stderr, error?.body?.error, error?.body?.message, ] .filter(Boolean) .map(String) .join('\n'); export const classifySpawnFailure = (error) => { const text = errorText(error); // A public seat's withheld turn (TASK-256) quotes the tool's own error, which // is vendor text: the code decides, so the event stays unacked for retry and // is never acked as refused or parked behind a quota or configuration // cooldown by that text. A throttle keeps its backoff, which also retries. if (error?.code === PUBLIC_TOOL_FAILED_CODE || error?.code === PUBLIC_MCP_NOT_READY_CODE) { return error?.status === 429 || RATE_LIMIT_RE.test(text) ? SPAWN_FAILURE_CLASS.RATE_LIMIT : SPAWN_FAILURE_CLASS.RUNTIME; } // Only whoever runs the seat can change this, so it waits as a missing CLI does. if (error?.code === PUBLIC_CODEX_VERSION_CODE) return SPAWN_FAILURE_CLASS.CONFIGURATION; // First: the refusal text carries none of the words below today, but a // refusal must never be read as a transient condition and retried. if (CONTENT_REFUSED_RE.test(text) || PI_CONTENT_STOP_RE.test(String(error?.message || '').trim())) { return SPAWN_FAILURE_CLASS.CONTENT_REFUSED; } if (error?.code === 'spawn_stalled') return SPAWN_FAILURE_CLASS.STALLED; // Provider APIs commonly report an exhausted account quota as HTTP 429. // Prefer the more specific body/message over the generic status code so a // hard quota failure gets the long cooldown rather than a one-minute probe. if (QUOTA_RE.test(text) || GATEWAY_BUDGET_RE.test(text)) return SPAWN_FAILURE_CLASS.QUOTA; // The possessive shape is deliberately loose about what sits between "your" // and "limit", and its `(?!rate[ -]?limit)` lookahead only guards the word // IMMEDIATELY after "your" — so "you've hit your API rate limit" walked past // it into the 15-minute circuit, where main correctly gives the 60-second // rate-limit probe (gate on #2491; it was a regression, not a gap). // A throttle wording anywhere in the text therefore wins over the shape. // QUOTA_RE is still tested first, so an explicit quota word keeps QUOTA even // when the same message also mentions rate limits. if (QUOTA_POSSESSIVE_RE.test(text) && !THROTTLE_WORDING_RE.test(text)) { return SPAWN_FAILURE_CLASS.QUOTA; } if (error?.status === 429 || RATE_LIMIT_RE.test(text)) { return SPAWN_FAILURE_CLASS.RATE_LIMIT; } // A model the provider won't run fails the same way until someone changes it. // A 403 for that reads as configuration below; a 404 has no status of its own // there, so without this it retried in seconds against the same answer. A // turn's text can hold a tool's 404 from before a dropped connection killed // the turn, so a connection fault keeps the quick retry. if ( error?.code === 'ENOENT' || error?.status === 401 || error?.status === 403 || CONFIGURATION_RE.test(text) || (modelUnavailableFailure(error) && !CONNECTION_FAULT_RE.test(text)) ) { return SPAWN_FAILURE_CLASS.CONFIGURATION; } return SPAWN_FAILURE_CLASS.RUNTIME; }; // A stable per-agent offset keeps a fleet from probing a recovering provider // in lockstep. Stability matters: random jitter makes operator logs and tests // harder to reason about, while distinct agent names already provide entropy. export const spawnRetryJitter = (agentName) => { let hash = 2166136261; for (const char of String(agentName || 'agent')) { hash ^= char.charCodeAt(0); hash = Math.imul(hash, 16777619); } return ((hash >>> 0) % 2001) / 10000; }; const applyJitter = (delayMs, jitterRatio) => { const safeJitter = Number.isFinite(jitterRatio) ? Math.min(SPAWN_RETRY_JITTER_MAX_RATIO, Math.max(0, jitterRatio)) : 0; return Math.round(delayMs * (1 + safeJitter)); }; // Claude names the local reset clock in its quota error. Waiting for that // clock avoids repeatedly launching a seat that cannot answer before then. // If the hint is absent or malformed, keep the ordinary bounded retry path. const quotaResetDelay = (error, now = Date.now()) => { const match = QUOTA_RESET_RE.exec(errorText(error)); if (!match) return null; const hour = Number(match[1]); const minute = Number(match[2] || 0); if (hour < 1 || hour > 12 || minute > 59) return null; const target = ((hour % 12) + (match[3].toLowerCase() === 'pm' ? 12 : 0)) * 60 + minute; let formatter; try { formatter = new Intl.DateTimeFormat('en-US', { timeZone: match[4], hour: '2-digit', minute: '2-digit', hourCycle: 'h23', }); } catch { return null; } const localMinute = (instant) => { const parts = Object.fromEntries(formatter.formatToParts(instant).map(({ type, value }) => [type, value])); return Number(parts.hour) * 60 + Number(parts.minute); }; const nowMinute = localMinute(now); const wallMinutes = (target - nowMinute + 1440) % 1440; // At or just after the named minute, the message may be a stale failure // from the old quota window. Probe on the ordinary cooldown instead of // assuming the next reset is nearly 24 hours away. if (wallMinutes === 0 || wallMinutes > 1430) return null; let resetAt = now - (now % 60000) + wallMinutes * 60 * 1000; // Crossing a daylight-saving boundary can shift the initial wall-clock // estimate by an hour. Reconcile against the target in the named zone. for (let attempt = 0; attempt < 2; attempt += 1) { const difference = (target - localMinute(resetAt) + 1440) % 1440; if (difference === 0) break; resetAt += (difference > 720 ? difference - 1440 : difference) * 60 * 1000; } if (resetAt <= now || localMinute(resetAt) !== target) return null; return resetAt - now + QUOTA_RESET_MARGIN_MS; }; /** * Return the next probe delay and whether the circuit is open. * * Known non-transient failures open immediately. Unknown runtime failures get * two quick retries, then open the circuit on the third consecutive failure. * Later probes back off exponentially to the same 15-minute base ceiling. */ export const spawnRetryPolicy = ({ error, consecutiveFailures, intervalMs, jitterRatio = 0, // The run loop's call: an outside signal still inside its grace (F293). externalSignal = false, }) => { const safeIntervalMs = Number.isFinite(intervalMs) && intervalMs > 0 ? intervalMs : 5000; if (externalSignal) { return { failureClass: SPAWN_FAILURE_CLASS.SIGNALED, circuitOpen: false, delayMs: applyJitter(safeIntervalMs, jitterRatio), }; } const failureClass = classifySpawnFailure(error); const failureCount = Number.isInteger(consecutiveFailures) && consecutiveFailures > 0 ? consecutiveFailures : 1; if ( failureClass === SPAWN_FAILURE_CLASS.QUOTA || failureClass === SPAWN_FAILURE_CLASS.CONFIGURATION ) { const resetDelay = failureClass === SPAWN_FAILURE_CLASS.QUOTA ? quotaResetDelay(error) : null; return { failureClass, circuitOpen: true, // Spread the reset probe by at most 24 seconds, not a percentage of // the entire wait: a six-hour hint must not add an hour of lag. delayMs: resetDelay === null ? applyJitter(SPAWN_RETRY_MAX_MS, jitterRatio) : resetDelay + applyJitter(QUOTA_RESET_MARGIN_MS, jitterRatio) - QUOTA_RESET_MARGIN_MS, }; } if (failureClass === SPAWN_FAILURE_CLASS.STALLED) { return { failureClass, circuitOpen: true, delayMs: applyJitter( Math.min(SPAWN_RETRY_MAX_MS, SPAWN_STALL_RETRY_BASE_MS * (2 ** (failureCount - 1))), jitterRatio, ), }; } if (failureClass === SPAWN_FAILURE_CLASS.RATE_LIMIT) { return { failureClass, circuitOpen: true, delayMs: applyJitter( Math.min( SPAWN_RETRY_MAX_MS, 60 * 1000 * (2 ** (failureCount - 1)), ), jitterRatio, ), }; } if (failureCount < SPAWN_CIRCUIT_THRESHOLD) { return { failureClass, circuitOpen: false, delayMs: applyJitter( Math.min( SPAWN_RETRY_MAX_MS, safeIntervalMs * (2 ** (failureCount - 1)), ), jitterRatio, ), }; } return { failureClass, circuitOpen: true, delayMs: applyJitter( Math.min( SPAWN_RETRY_MAX_MS, 60 * 1000 * (2 ** (failureCount - SPAWN_CIRCUIT_THRESHOLD)), ), jitterRatio, ), }; }; export const formatRetryDelay = (delayMs) => { if (delayMs >= 60000) { const minutes = delayMs / 60000; return `${Number.isInteger(minutes) ? minutes : minutes.toFixed(1)}m`; } if (delayMs >= 1000) { const seconds = delayMs / 1000; return `${Number.isInteger(seconds) ? seconds : seconds.toFixed(1)}s`; } return `${delayMs}ms`; };