// RB2 — three resilience enhancements on the account pool (src/auth.js): // B1 account-level EXPONENTIAL BACKOFF on the error streak (self-healing, // capped, never permanent). // B2 QUOTA-EXHAUSTION closed loop: a dry balance pre-cools the account on its // own self-healing dimension (quotaResetAt), cleared the moment the // balance recovers — never a permanent disable. // T3 NEW-CREDENTIAL thunderstorm guard: a freshly-added account is seeded at // pool-median LRU (not first-picked) and is exempt from backoff // escalation during a grace window. // // MOTHER THEME (transient-first): every cooldown here auto-expires and/or is // cleared on recovery. There must be NO path to a permanent disable, and a // HEALTHY account must never be locked (the critical counter-examples below). import { afterEach, describe, it } from 'node:test'; import assert from 'node:assert/strict'; import { addAccountByKey, removeAccount, reportError, reportSuccess, markRateLimited, reportInternalError, applyQuotaSnapshot, __isRateLimitedForModel, __isLastUsableAccount, setAccountSpendPolicy, getAccountSpendState, getApiKey, markQuotaExhausted, releaseAccount, } from '../src/auth.js'; import { setBreakerTunables } from '../src/runtime-config.js'; const created = []; let _seq = 0; function mk(label = 'rb2', { aged = true } = {}) { const acct = addAccountByKey(`rb2-key-${Date.now()}-${_seq++}-${Math.random().toString(36).slice(2)}`, label); acct._health = []; // Most backoff tests want a "settled" account, NOT one inside the new-account // grace window (T3b would otherwise exempt it from escalation). if (aged) acct.addedAt = Date.now() - 60 * 60 * 1000; created.push(acct.id); return acct; } const BREAKER_ENV = [ 'WINDSURFAPI_BREAKER', 'WINDSURFAPI_BREAKER_BASE_MS', 'WINDSURFAPI_BREAKER_FACTOR', 'WINDSURFAPI_BREAKER_MAX_MS', 'WINDSURFAPI_NEW_ACCOUNT_GRACE_MS', 'WINDSURFAPI_NEW_ACCOUNT_BASELINE', 'WINDSURFAPI_QUOTA_COOLDOWN', 'WINDSURFAPI_QUOTA_COOLDOWN_MS', 'WINDSURFAPI_QUOTA_DRY_THRESHOLD', 'WINDSURFAPI_ERROR_RECOVERY_MS', 'WINDSURFAPI_LAST_ACCOUNT_EXEMPT', 'WINDSURFAPI_SPEND_ON_DEMAND', 'WINDSURFAPI_ON_DEMAND_RESERVE_USD', ]; afterEach(() => { for (const k of BREAKER_ENV) delete process.env[k]; while (created.length) removeAccount(created.pop()); }); // Drive one error EPISODE: a streak of 3 errors flips status→'error' and bumps // the backoff streak once. A half-open recovery later (status back to 'active') // lets the NEXT episode escalate. We simulate the half-open flip directly. function failToError(acct) { reportError(acct.apiKey); reportError(acct.apiKey); reportError(acct.apiKey); } describe('RB2/B1 — account-level exponential backoff', () => { it('first episode applies NO extra cooldown (matches pre-RB2 behaviour)', () => { reportSuccess(mk('b1-peer-first').apiKey); // healthy peer: not the last account const acct = mk('b1-first'); failToError(acct); assert.equal(acct.status, 'error'); assert.equal(acct._breakerStreak, 1, 'streak counts the first episode'); // streak 1 must not push rateLimitedUntil out — half-open TTL alone governs. assert.ok(!acct.rateLimitedUntil || acct.rateLimitedUntil <= Date.now(), 'no backoff cooldown on the very first episode'); }); it('escalates the cooldown across consecutive failed half-open episodes, capped', () => { process.env.WINDSURFAPI_BREAKER_BASE_MS = '60000'; // 1m process.env.WINDSURFAPI_BREAKER_FACTOR = '2'; process.env.WINDSURFAPI_BREAKER_MAX_MS = '600000'; // cap 10m reportSuccess(mk('b1-peer-grow').apiKey); // healthy peer: not the last account const acct = mk('b1-grow'); failToError(acct); // streak 1 — no cooldown acct.status = 'active'; // half-open recovery (TTL elapsed in reality) reportError(acct.apiKey); // streak 2 → base*2^1 = 120000 const cd2 = acct.rateLimitedUntil - Date.now(); assert.equal(acct._breakerStreak, 2); assert.ok(Math.abs(cd2 - 120000) < 5000, `streak2 cooldown ~2m, got ${cd2}ms`); acct.status = 'active'; reportError(acct.apiKey); // streak 3 → base*2^2 = 240000 const cd3 = acct.rateLimitedUntil - Date.now(); assert.equal(acct._breakerStreak, 3); assert.ok(cd3 > cd2, `streak3 cooldown (${cd3}) must exceed streak2 (${cd2})`); assert.ok(Math.abs(cd3 - 240000) < 5000, `streak3 cooldown ~4m, got ${cd3}ms`); // Push the streak high — must clamp at the 10m cap, never beyond. for (let i = 0; i < 8; i++) { acct.status = 'active'; reportError(acct.apiKey); } const cdN = acct.rateLimitedUntil - Date.now(); assert.ok(cdN <= 600000 + 5000, `capped cooldown must not exceed 10m, got ${cdN}ms`); assert.ok(cdN >= 600000 - 5000, 'a high streak sits AT the cap'); }); it('backoff is never permanent — the capped cooldown still expires (self-heal)', () => { process.env.WINDSURFAPI_BREAKER_BASE_MS = '60000'; process.env.WINDSURFAPI_BREAKER_MAX_MS = '120000'; // 2m cap reportSuccess(mk('b1-peer-selfheal').apiKey); // healthy peer: not the last account const acct = mk('b1-selfheal'); failToError(acct); for (let i = 0; i < 5; i++) { acct.status = 'active'; reportError(acct.apiKey); } assert.ok(acct.rateLimitedUntil > Date.now(), 'currently cooled'); // Once that deadline passes the account is selectable again. assert.equal(__isRateLimitedForModel(acct, null, acct.rateLimitedUntil + 1), false, 'past the cooldown the account is eligible again — no permanent lockout'); }); it('reportSuccess fully clears the backoff streak (good behaviour wipes the penalty)', () => { reportSuccess(mk('b1-peer-clear').apiKey); // healthy peer: not the last account const acct = mk('b1-clear'); failToError(acct); assert.equal(acct._breakerStreak, 1); acct.status = 'active'; reportError(acct.apiKey); // streak 2 assert.equal(acct._breakerStreak, 2); reportSuccess(acct.apiKey); assert.equal(acct._breakerStreak, 0, 'a success resets the ladder to zero'); assert.equal(acct.errorCount, 0); }); it('WINDSURFAPI_BREAKER=0 disables escalation entirely', () => { process.env.WINDSURFAPI_BREAKER = '0'; const acct = mk('b1-off'); failToError(acct); for (let i = 0; i < 4; i++) { acct.status = 'active'; reportError(acct.apiKey); } assert.ok(!acct.rateLimitedUntil || acct.rateLimitedUntil <= Date.now(), 'no backoff cooldown is applied when the breaker is disabled'); }); }); describe('RB2/B2 — quota-exhaustion closed loop', () => { it('a dry balance (weekly 0%) pre-cools the account on the quota dimension', () => { process.env.WINDSURFAPI_QUOTA_COOLDOWN_MS = '1800000'; // 30m const acct = mk('b2-dry'); const now = 1_000_000; applyQuotaSnapshot(acct, 0, now); assert.equal(acct.quotaResetAt, now + 1800000, 'quotaResetAt set 30m out'); // getApiKey filters on this via isRateLimitedForModel. assert.equal(__isRateLimitedForModel(acct, null, now + 1000), true, 'dry account is excluded from selection during the quota cooldown'); }); it('a dry account with a positive on-demand ($) balance is NOT cooled', () => { process.env.WINDSURFAPI_QUOTA_COOLDOWN_MS = '1800000'; const acct = mk('b2-ondemand'); acct.credits = { balance: 74.39 }; // included quota dry, but prepaid $ left const now = 1_500_000; applyQuotaSnapshot(acct, 0, now); // weekly 0% = dry assert.ok(!acct.quotaResetAt, 'not cooled — on-demand balance can serve'); assert.equal(__isRateLimitedForModel(acct, null, now + 1000), false, 'stays selectable so the paid prepaid pool actually gets used'); }); it('a dry account with zero / no on-demand balance is still cooled', () => { process.env.WINDSURFAPI_QUOTA_COOLDOWN_MS = '1800000'; const zero = mk('b2-zero'); zero.credits = { balance: 0 }; const none = mk('b2-none'); // no credits.balance at all const now = 1_600_000; applyQuotaSnapshot(zero, 0, now); applyQuotaSnapshot(none, 0, now); assert.equal(zero.quotaResetAt, now + 1800000, 'zero balance → cooled'); assert.equal(none.quotaResetAt, now + 1800000, 'no balance info → cooled'); }); it('an on-demand balance clears an existing quota cooldown', () => { process.env.WINDSURFAPI_QUOTA_COOLDOWN_MS = '1800000'; const acct = mk('b2-recover-ondemand'); const now = 1_700_000; applyQuotaSnapshot(acct, 0, now); // cooled (no balance yet) assert.ok(acct.quotaResetAt > now); acct.credits = { balance: 12.5 }; // balance shows up next snapshot applyQuotaSnapshot(acct, 0, now + 1000); // still dry, but now has $ assert.equal(acct.quotaResetAt, 0, 'cooldown cleared once on-demand balance is known'); }); it('the quota cooldown self-heals: it expires on its own (NOT permanent)', () => { process.env.WINDSURFAPI_QUOTA_COOLDOWN_MS = '1800000'; const acct = mk('b2-expire'); const now = 2_000_000; applyQuotaSnapshot(acct, 0, now); assert.equal(__isRateLimitedForModel(acct, null, acct.quotaResetAt + 1), false, 'past the deadline the account is eligible again — no permanent disable'); }); it('a later refresh seeing recovered balance clears the cooldown immediately', () => { const acct = mk('b2-recover'); const now = 3_000_000; applyQuotaSnapshot(acct, 0, now); assert.ok(acct.quotaResetAt > now, 'cooled while dry'); // Next 15-min refresh sees the weekly balance refilled. applyQuotaSnapshot(acct, 42, now + 60_000); assert.equal(acct.quotaResetAt, 0, 'balance recovered → quota cooldown cleared'); assert.equal(__isRateLimitedForModel(acct, null, now + 60_001), false); }); it('quota cooldown lives on its OWN dimension — never clobbers a transient cooldown (B6)', () => { const acct = mk('b2-orthogonal'); const now = 4_000_000; // A transient RATE_LIMITED cooldown is active (separate dimension). markRateLimited(acct.apiKey, 5 * 60 * 1000, null); const transientUntil = acct.rateLimitedUntil; assert.ok(transientUntil > 0); // Quota recovers → clears the quota dimension only, transient untouched. applyQuotaSnapshot(acct, 0, now); applyQuotaSnapshot(acct, 80, now + 1000); assert.equal(acct.quotaResetAt, 0, 'quota dimension cleared'); assert.equal(acct.rateLimitedUntil, transientUntil, 'transient rateLimitedUntil is NOT touched by quota recovery'); }); it('unknown balance (null/undefined) never cools — does not punish an unprobed account', () => { const acct = mk('b2-unknown'); const now = 5_000_000; applyQuotaSnapshot(acct, null, now); applyQuotaSnapshot(acct, undefined, now); assert.ok(!acct.quotaResetAt, 'no cooldown applied when balance is unknown'); }); it('CRITICAL counter-example: a HEALTHY (funded) account is NEVER quota-cooled', () => { const acct = mk('b2-healthy'); const now = 6_000_000; applyQuotaSnapshot(acct, 73, now); // plenty of weekly quota assert.ok(!acct.quotaResetAt, 'a funded account is not cooled'); assert.equal(__isRateLimitedForModel(acct, null, now + 1), false, 'healthy account stays fully selectable'); }); it('WINDSURFAPI_QUOTA_COOLDOWN=0 disables the closed loop', () => { process.env.WINDSURFAPI_QUOTA_COOLDOWN = '0'; const acct = mk('b2-off'); applyQuotaSnapshot(acct, 0, 7_000_000); assert.ok(!acct.quotaResetAt, 'no quota cooldown when disabled'); }); it('the dry threshold is env-tunable (cool at/under threshold, leave above)', () => { process.env.WINDSURFAPI_QUOTA_DRY_THRESHOLD = '5'; const dry = mk('b2-thresh-dry'); const ok = mk('b2-thresh-ok'); const now = 8_000_000; applyQuotaSnapshot(dry, 3, now); // <= 5 → dry applyQuotaSnapshot(ok, 9, now); // > 5 → fine assert.ok(dry.quotaResetAt > now, 'weekly 3% with threshold 5 cools'); assert.ok(!ok.quotaResetAt, 'weekly 9% with threshold 5 stays selectable'); }); }); // v3.0.3 — on-demand spend POLICY. When the included weekly quota is dry, does // the account keep serving off its prepaid $ balance? Governed by a global // default (runtime-config / env WINDSURFAPI_SPEND_ON_DEMAND + a reserve floor) // and a per-account override. Defaults (spend=true, reserve=0) reproduce the // pre-policy 52e255f behaviour exactly — the B2 block above is the regression // guard for that. describe('v3.0.3 — on-demand spend policy', () => { it('ZERO REGRESSION: default policy (spend on, reserve 0) burns balance just like before', () => { const acct = mk('sp-default'); acct.credits = { balance: 74.39 }; const now = 10_000_000; applyQuotaSnapshot(acct, 0, now); // dry, but funded assert.ok(!acct.quotaResetAt, 'default policy keeps a funded dry account serving'); }); it('global spend-off (env=0): a dry account is cooled even WITH a balance', () => { process.env.WINDSURFAPI_SPEND_ON_DEMAND = '0'; const acct = mk('sp-global-off'); acct.credits = { balance: 74.39 }; const now = 10_100_000; applyQuotaSnapshot(acct, 0, now); assert.equal(acct.quotaResetAt, now + 30 * 60 * 1000, 'spend-off policy cools a dry account rather than burning its balance'); assert.equal(__isRateLimitedForModel(acct, null, now + 1000), true, 'excluded from selection'); }); it('global reserve floor: balance at/under the reserve is treated as dry → cooled', () => { process.env.WINDSURFAPI_ON_DEMAND_RESERVE_USD = '10'; const low = mk('sp-reserve-low'); low.credits = { balance: 8 }; // under floor const high = mk('sp-reserve-high'); high.credits = { balance: 25 }; // above floor const now = 10_200_000; applyQuotaSnapshot(low, 0, now); applyQuotaSnapshot(high, 0, now); assert.ok(low.quotaResetAt > now, 'balance under the reserve floor → cooled (never burned to zero)'); assert.ok(!high.quotaResetAt, 'balance above the reserve floor keeps serving'); }); it('per-account override BEATS the global default (force spend on a globally-off pool)', () => { process.env.WINDSURFAPI_SPEND_ON_DEMAND = '0'; // global: stop when dry const acct = mk('sp-override-on'); acct.credits = { balance: 50 }; setAccountSpendPolicy(acct.id, { spendOnDemand: true }); // but THIS account may burn const now = 10_300_000; applyQuotaSnapshot(acct, 0, now); assert.ok(!acct.quotaResetAt, 'per-account spend=true overrides the global off default'); }); it('per-account override can STOP an account while the global default burns', () => { const acct = mk('sp-override-off'); acct.credits = { balance: 50 }; setAccountSpendPolicy(acct.id, { spendOnDemand: false }); // protect THIS account const now = 10_400_000; applyQuotaSnapshot(acct, 0, now); assert.ok(acct.quotaResetAt > now, 'per-account spend=false cools it despite the global burn default'); }); it('per-account reserve override protects one account with a custom floor', () => { const acct = mk('sp-reserve-override'); acct.credits = { balance: 30 }; setAccountSpendPolicy(acct.id, { reserve: 40 }); // keep >$40 → 30 is under floor const now = 10_500_000; applyQuotaSnapshot(acct, 0, now); assert.ok(acct.quotaResetAt > now, 'balance below the per-account reserve → cooled'); }); it('clearing an override (null) falls back to the global default', () => { const acct = mk('sp-clear'); acct.credits = { balance: 50 }; setAccountSpendPolicy(acct.id, { spendOnDemand: false }); assert.equal(acct.spendOnDemand, false); setAccountSpendPolicy(acct.id, { spendOnDemand: null }); // back to inherit assert.equal(acct.spendOnDemand, null, 'override cleared → inherits global'); const now = 10_600_000; applyQuotaSnapshot(acct, 0, now); assert.ok(!acct.quotaResetAt, 'inherited default (spend on) keeps it serving'); }); it('getAccountSpendState reports the resolved policy + live state for the dashboard', () => { const spending = mk('sp-state-spending'); spending.credits = { balance: 50 }; const now = 10_700_000; applyQuotaSnapshot(spending, 0, now); let st = getAccountSpendState(spending, now + 1000); assert.equal(st.state, 'spending', 'funded dry account under burn policy = spending'); assert.equal(st.spendOnDemand, true); assert.equal(st.balance, 50); const stopped = mk('sp-state-stopped'); stopped.credits = { balance: 50 }; setAccountSpendPolicy(stopped.id, { spendOnDemand: false }); applyQuotaSnapshot(stopped, 0, now); st = getAccountSpendState(stopped, now + 1000); assert.equal(st.cooled, true, 'stop-when-dry account is cooled'); assert.equal(st.state, 'cooled'); assert.equal(st.spendOverride, false, 'surfaces the explicit per-account override'); }); it('setAccountSpendPolicy on a missing id is a safe no-op', () => { assert.equal(setAccountSpendPolicy('no-such-id', { spendOnDemand: true }), false); }); }); describe('RB2/T3 — new-credential thunderstorm guard', () => { it('T3b: a brand-new account is EXEMPT from backoff escalation during the grace window', () => { process.env.WINDSURFAPI_BREAKER_BASE_MS = '60000'; process.env.WINDSURFAPI_BREAKER_FACTOR = '2'; process.env.WINDSURFAPI_NEW_ACCOUNT_GRACE_MS = String(10 * 60 * 1000); reportSuccess(mk('t3-peer-new').apiKey); // healthy peer: not the last account // aged:false → addedAt = now, inside the grace window. const acct = mk('t3-new', { aged: false }); failToError(acct); acct.status = 'active'; reportError(acct.apiKey); // would be streak 2 → escalate, but it's NEW assert.equal(acct._breakerStreak, 2, 'streak still counts'); assert.ok(!acct.rateLimitedUntil || acct.rateLimitedUntil <= Date.now(), 'a new account is not ramped into a long lockout by onboarding wobble'); }); it('T3b: once past the grace window the same account DOES escalate', () => { process.env.WINDSURFAPI_BREAKER_BASE_MS = '60000'; process.env.WINDSURFAPI_BREAKER_FACTOR = '2'; process.env.WINDSURFAPI_NEW_ACCOUNT_GRACE_MS = String(10 * 60 * 1000); // A healthy peer so the account under test is NOT the last usable one (the // LB last-account exemption would otherwise keep a sole account in rotation). reportSuccess(mk('t3-peer').apiKey); const acct = mk('t3-aged', { aged: false }); acct.addedAt = Date.now() - 60 * 60 * 1000; // now well past the grace window failToError(acct); acct.status = 'active'; reportError(acct.apiKey); // streak 2 → escalate normally assert.ok(acct.rateLimitedUntil > Date.now(), 'a settled account escalates as usual past the grace window'); }); it('T3a: a new account joining a running pool is seeded near pool-median LRU (not "oldest")', () => { // Build a running pool with known lastUsed values. const base = Date.now() - 5 * 60 * 1000; const a1 = mk('t3-pool-1'); a1.lastUsed = base; const a2 = mk('t3-pool-2'); a2.lastUsed = base + 60_000; const a3 = mk('t3-pool-3'); a3.lastUsed = base + 120_000; // New account added into that pool. const fresh = mk('t3-fresh', { aged: false }); // It must NOT be left at lastUsed=0 (which the LRU tiebreaker treats as the // oldest → most-preferred). It should sit near the pool median. assert.ok(fresh.lastUsed > 0, 'new account not left at the "oldest" sentinel'); const median = base + 60_000; assert.ok(Math.abs(fresh.lastUsed - median) <= 60_000, `seeded near pool median (~${median}), got ${fresh.lastUsed}`); }); it('T3a: a BATCH of new accounts does not all collapse onto the same LRU value', () => { const base = Date.now() - 5 * 60 * 1000; const a1 = mk('t3-batch-pool-1'); a1.lastUsed = base; const a2 = mk('t3-batch-pool-2'); a2.lastUsed = base + 120_000; // Add three at once. const f1 = mk('t3-batch-1', { aged: false }); const f2 = mk('t3-batch-2', { aged: false }); const f3 = mk('t3-batch-3', { aged: false }); const vals = [f1.lastUsed, f2.lastUsed, f3.lastUsed]; assert.ok(vals.every(v => v > 0), 'all seeded off the oldest sentinel'); // Jitter de-syncs them — not a perfect 3-way tie (which would force all the // first requests onto whichever one the sort happens to surface). assert.ok(new Set(vals).size >= 2, `batch should de-synchronize, got ${vals.join(',')}`); }); it('T3a: an empty/fresh pool leaves lastUsed=0 (no median to balance against)', () => { // No active account in this pool has a running lastUsed>0 of its own beyond // what we add here; the very first account has no median → stays 0. const first = mk('t3-firstboot', { aged: false }); // first is added when peers (from other describe blocks) are removed in // afterEach, but to be deterministic we assert the documented contract: // when _poolMedianLastUsed() is 0, lastUsed is untouched. if (first.lastUsed === 0) { assert.equal(first.lastUsed, 0, 'no seeding without a running pool'); } else { // A median existed (peers present) → seeded, which is also valid. assert.ok(first.lastUsed > 0); } }); it('WINDSURFAPI_NEW_ACCOUNT_BASELINE=0 disables LRU seeding', () => { process.env.WINDSURFAPI_NEW_ACCOUNT_BASELINE = '0'; const a1 = mk('t3-disabled-pool'); a1.lastUsed = Date.now(); const fresh = mk('t3-disabled-fresh', { aged: false }); assert.equal(fresh.lastUsed, 0, 'seeding disabled → new account keeps lastUsed=0'); }); }); describe('RB2 — mother-theme guards (transients must not be escalated)', () => { it('CRITICAL: a transient (CAPACITY/internal) NEVER touches the backoff streak', () => { const acct = mk('mt-transient'); // markRateLimited (CAPACITY/RATE_LIMITED) and reportInternalError are the // transient paths — chat.js routes them here, NOT to reportError. markRateLimited(acct.apiKey, 60 * 1000, 'gpt', 'c'); reportInternalError(acct.apiKey); reportInternalError(acct.apiKey); assert.ok(!acct._breakerStreak, 'transients do not start the backoff ladder'); assert.equal(acct.status, 'active', 'transients do not flip status to error'); }); it('CRITICAL counter-example: a HEALTHY account is never locked by selection', () => { const acct = mk('mt-healthy'); reportSuccess(acct.apiKey); const now = Date.now(); assert.equal(__isRateLimitedForModel(acct, null, now), false, 'healthy account selectable'); assert.equal(__isRateLimitedForModel(acct, 'gpt-5.5', now), false); assert.ok(!acct.quotaResetAt && (!acct.rateLimitedUntil || acct.rateLimitedUntil <= now)); }); it('cool-01: internal-error quarantine must NOT shorten a longer existing cooldown', () => { // reportInternalError quarantines for internalQuarantineMs (default 2min). If // the account already sits under a MUCH longer cooldown (e.g. a 3h RATE_LIMITED // reset window), a bare `rateLimitedUntil = now + quarantine` assignment would // SHORTEN it and reopen a still-throttled account early. The fix uses Math.max. const acct = mk('cool01', { aged: true }); const longUntil = Date.now() + 3 * 60 * 60 * 1000; // 3h hard rate limit acct.rateLimitedUntil = longUntil; // Need a healthy peer so the last-usable-account exemption doesn't skip the // quarantine branch (single-account pools are exempt by design). const peer = mk('cool01-peer'); reportSuccess(peer.apiKey); // internalErrorThreshold default = 2 → two hits trigger the quarantine branch. reportInternalError(acct.apiKey); reportInternalError(acct.apiKey); assert.equal(acct.rateLimitedUntil, longUntil, 'the pre-existing 3h cooldown must survive — quarantine (2min) must not shorten it'); }); it('cool-01: internal-error quarantine still EXTENDS when no/shorter cooldown exists', () => { const acct = mk('cool01-extend', { aged: true }); const peer = mk('cool01-extend-peer'); reportSuccess(peer.apiKey); const before = Date.now(); acct.rateLimitedUntil = 0; // no existing cooldown reportInternalError(acct.apiKey); reportInternalError(acct.apiKey); assert.ok(acct.rateLimitedUntil > before, 'with no prior cooldown, the quarantine window is applied'); }); }); describe('RB2/B2 — quota dimension is consistent across pool-status helpers', () => { it('isAllRateLimited / isAllTemporarilyUnavailable treat a quota-cooled account as unavailable', async () => { const { isAllRateLimited, isAllTemporarilyUnavailable } = await import('../src/auth.js'); // Single-account pool view is noisy across suites; assert the per-account // contract instead: a quota-cooled account is filtered out of selection. const acct = mk('b2-pool-consistency'); applyQuotaSnapshot(acct, 0, Date.now()); assert.ok(acct.quotaResetAt > Date.now(), 'cooled on quota dimension'); assert.equal(__isRateLimitedForModel(acct, null), true, 'quota-cooled account excluded — same predicate both pool helpers use'); // Smoke the helpers don't throw with the new dimension present. assert.doesNotThrow(() => isAllRateLimited(null)); assert.doesNotThrow(() => isAllTemporarilyUnavailable(null)); }); }); // LB — last-account breaker exemption: the breaker must not take the ONLY usable // account out of rotation. In a single-account pool, disabling = a guaranteed // pool-wide 529, so one bad request/model self-inflicts a total outage. A // degraded sole account serving what it can beats guaranteed failure. The streak // counters keep advancing, so a healthy peer restores normal breaker behaviour. // afterEach (top of file) removes every created account, so each test starts on a // clean pool and controls its own island. describe('LB — last-account breaker exemption', () => { it('reportError does NOT flip the sole account to error (kept in rotation)', () => { const acct = mk('lb-sole'); failToError(acct); // 3 errors would normally flip status='error' assert.equal(acct.status, 'active', 'last account stays active'); assert.equal(acct.errorCount, 3, 'errorCount still climbed (history preserved)'); assert.equal(__isRateLimitedForModel(acct, null), false, 'still selectable'); }); it('reportInternalError does NOT quarantine the sole account', () => { const acct = mk('lb-sole-internal'); reportInternalError(acct.apiKey); reportInternalError(acct.apiKey); // streak>=2 would normally 5min-quarantine assert.equal(acct.internalErrorStreak, 2, 'streak still climbed'); assert.ok(!acct.rateLimitedUntil || acct.rateLimitedUntil <= Date.now(), 'no quarantine cooldown set on the last account'); assert.equal(__isRateLimitedForModel(acct, null), false, 'still selectable'); }); it('with a healthy peer present, the breaker trips normally (exemption is pool-size aware)', () => { const bad = mk('lb-bad'); const good = mk('lb-good'); reportSuccess(good.apiKey); // ensure the peer is unambiguously usable assert.equal(__isLastUsableAccount(bad), false, 'bad is not the last usable account'); failToError(bad); assert.equal(bad.status, 'error', 'a non-last account still trips to error'); // internal-error quarantine path also trips when a peer exists const bad2 = mk('lb-bad2'); const t0 = Date.now(); reportInternalError(bad2.apiKey); reportInternalError(bad2.apiKey); assert.ok(bad2.rateLimitedUntil > Date.now(), 'non-last account still quarantined'); // L1 (2026-07-10): quarantine window cut 5min → 2min (KiroStudio short-cooldown // philosophy). Assert ~2min, NOT the old ~5min. const quarMs = bad2.rateLimitedUntil - t0; assert.ok(quarMs > 1.5 * 60 * 1000 && quarMs < 2.5 * 60 * 1000, `quarantine ~2min (got ${(quarMs / 1000).toFixed(0)}s)`); }); // 2026-07-10 tier-aware exemption: a healthy FREE peer is NOT a fallback for a // PRO account, because free accounts cannot serve paid selectors. The sole pro // account must therefore be exempt from quarantine even with a free peer // present — otherwise quarantining it blacks out every paid model pool-wide // (the Claude-Code 1ms-429 lockout this fix targets). it('a PRO account with only a FREE peer is still the last usable (tier-aware exemption)', () => { const pro = mk('lb-pro'); const free = mk('lb-free'); pro.tier = 'pro'; free.tier = 'free'; reportSuccess(free.apiKey); // free peer is unambiguously healthy // Free cannot cover pro's paid selectors → pro is the last usable for its tier. assert.equal(__isLastUsableAccount(pro), true, 'pro is last usable despite healthy free peer'); reportInternalError(pro.apiKey); reportInternalError(pro.apiKey); assert.ok(!(pro.rateLimitedUntil > Date.now()), 'sole pro account NOT quarantined (tier-aware exemption)'); // Symmetry check: the FREE account, with a healthy PRO peer, is NOT the last // (pro outranks and can serve everything free can) → breaker trips normally. reportSuccess(pro.apiKey); // clear pro's streak + make it healthy again pro.rateLimitedUntil = 0; assert.equal(__isLastUsableAccount(free), false, 'free is not last usable when a healthy pro peer exists'); }); // Boundary: an UNKNOWN (unprobed) peer is only optimistically paid-capable, so // it must NOT justify quarantining the sole KNOWN-pro account — betting on an // unprobed peer is how paid selectors get blacked out. Conversely two same-tier // (unknown+unknown) peers ARE mutual fallbacks (breaker trips normally). it('an UNKNOWN peer does not rescue a PRO account, but same-tier peers do', () => { const pro = mk('lb-pro2'); const unk = mk('lb-unknown'); pro.tier = 'pro'; unk.tier = 'unknown'; // mk() default is already 'unknown', set explicit for clarity reportSuccess(unk.apiKey); assert.equal(__isLastUsableAccount(pro), true, 'pro not rescued by an unprobed unknown peer'); // But the unknown account itself, with a healthy pro peer, is NOT the last // (pro rank 2 >= unknown rank 1). assert.equal(__isLastUsableAccount(unk), false, 'unknown is not last when a healthy pro peer exists'); // And two unknowns are mutual fallbacks. const unk2 = mk('lb-unknown2'); reportSuccess(unk2.apiKey); assert.equal(__isLastUsableAccount(unk), false, 'unknown+unknown are mutual fallbacks'); }); it('once the only peer is itself down, the survivor is exempt again', () => { const a = mk('lb-a'); const b = mk('lb-b'); // Take a out via the internal-error path with b as the healthy peer present. reportSuccess(a.apiKey); reportInternalError(b.apiKey); reportInternalError(b.apiKey); assert.ok(b.rateLimitedUntil > Date.now(), 'b quarantined while a was usable'); // Now a is the last usable account — it must be exempt. assert.equal(__isLastUsableAccount(a), true, 'a is now the sole usable account'); failToError(a); assert.equal(a.status, 'active', 'survivor exempt once its peer is down'); }); it('WINDSURFAPI_LAST_ACCOUNT_EXEMPT=0 restores strict trip-always behaviour', () => { process.env.WINDSURFAPI_LAST_ACCOUNT_EXEMPT = '0'; const acct = mk('lb-optout'); failToError(acct); assert.equal(acct.status, 'error', 'opt-out: sole account trips as before'); }); }); // L2 (2026-07-10): degraded-serve fallback. When the hard filter leaves zero // candidates (whole entitled pool transiently throttled), serve the least-bad // transiently-cooled account instead of returning null → 429. Opt-in. describe('L2 — degraded-serve fallback (pool fully throttled)', () => { afterEach(() => { setBreakerTunables({ degradedServe: null }); }); function cool(acct, ms) { markRateLimited(acct.apiKey, ms, null); } // account-wide transient it('disabled (opt-out): a fully-throttled pool returns null → 429', () => { // Default is now ON (429 mitigation); explicitly disable to get the hard fast-429. setBreakerTunables({ degradedServe: false }); const a = mk('l2-off'); cool(a, 60_000); assert.equal(getApiKey([], null, ''), null, 'no degraded serve when disabled → null → 429'); }); it('enabled: serves the throttled account (degraded) instead of null', () => { setBreakerTunables({ degradedServe: true }); const a = mk('l2-on'); cool(a, 60_000); const got = getApiKey([], null, ''); assert.ok(got, 'degraded serve returned an account'); assert.equal(got.apiKey, a.apiKey); assert.equal(got._degraded, true, 'flagged as a degraded serve'); releaseAccount(got.apiKey); }); it('enabled: picks the account with the SHORTEST remaining cooldown', () => { setBreakerTunables({ degradedServe: true }); const soon = mk('l2-soon'); const later = mk('l2-later'); cool(soon, 5_000); // frees up sooner cool(later, 90_000); const got = getApiKey([], null, ''); assert.equal(got.apiKey, soon.apiKey, 'chose the closest-to-healing account'); releaseAccount(got.apiKey); }); it('enabled: does NOT fall back onto a QUOTA-exhausted account (real dry-well)', () => { setBreakerTunables({ degradedServe: true }); const a = mk('l2-quota'); markQuotaExhausted(a.apiKey, 60_000); assert.equal(getApiKey([], null, ''), null, 'quota dry-well is not a degraded-serve candidate'); }); it('enabled: does NOT fall back onto a dead/errored account (real fault)', () => { setBreakerTunables({ degradedServe: true }); const a = mk('l2-dead'); a.status = 'error'; assert.equal(getApiKey([], null, ''), null, 'errored account is not a degraded-serve candidate'); }); it('enabled: a healthy account is still selected normally (fallback not triggered)', () => { setBreakerTunables({ degradedServe: true }); const a = mk('l2-healthy'); reportSuccess(a.apiKey); const got = getApiKey([], null, ''); assert.ok(got && !got._degraded, 'healthy account served via normal path, not degraded'); releaseAccount(got.apiKey); }); });