import { CommandId, MessageId, type OrchestrationV2Command } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Stream from "effect/Stream"; import type { OrchestrationV2DomainEvent } from "@t3tools/contracts"; import * as DueHint from "../scheduling/DueHint.ts"; import * as Scheduler from "../scheduling/Scheduler.ts"; import { forkParked } from "../serverActivation.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ThreadManagement from "./ThreadManagementService.ts"; const MAX_OVERDUE_ARM_MS = 24 * 60 * 60 * 1000; export function limitRecoveryCommand( thread: ProjectionStore.ProjectionLimitRecoveryCandidate, autoResume: boolean, nowMs: number, snooze = false, ): OrchestrationV2Command | null { if ( thread.status !== "failed" || thread.lastErrorClass !== "usage_limit" || !thread.latestRunId || !thread.usageLimitResetAt || thread.archivedAt !== null || thread.settledOverride === "settled" || thread.pendingRuntimeRequest !== null ) return null; const resetMs = Date.parse(thread.usageLimitResetAt); // An already-expired window reported with a fresh failure cannot start a retry loop. if ( !Number.isFinite(resetMs) || resetMs <= DateTime.toEpochMillis(thread.latestRunCompletedAt ?? thread.updatedAt) ) return null; const identity = `${thread.id}:${thread.latestRunId}:${resetMs}`; const recovery = thread.limitRecovery; if (recovery?.runId !== thread.latestRunId || recovery.resetAt !== thread.usageLimitResetAt) { if (nowMs - resetMs > MAX_OVERDUE_ARM_MS) return null; if (!autoResume && (!snooze || resetMs <= nowMs)) return null; return { type: "thread.metadata.update", commandId: CommandId.make(`limit-arm:${identity}`), threadId: thread.id, limitRecovery: { runId: thread.latestRunId, resetAt: thread.usageLimitResetAt, autoResume, snooze: snooze && resetMs > nowMs, }, }; } if ( !recovery.autoResume || resetMs > nowMs || (thread.snoozedUntil != null && DateTime.toEpochMillis(thread.snoozedUntil) > nowMs) ) return null; const deliveryIdentity = `${identity}:${recovery.requestId ?? "legacy"}`; return { type: "message.dispatch", commandId: CommandId.make(`limit-resume:${deliveryIdentity}`), messageId: MessageId.make(`limit-resume:${deliveryIdentity}`), threadId: thread.id, usageLimitContinuationOfRunId: thread.latestRunId, ...(recovery.requestId === undefined ? {} : { usageLimitRecoveryRequestId: recovery.requestId }), text: "Continue where you left off.", attachments: [], dispatchMode: { type: "start_immediately" }, createdBy: "user", creationSource: "server", }; } // Events that can make a thread a candidate or change its command; streaming items cannot. const affectsLimitRecovery = (event: OrchestrationV2DomainEvent) => event.type.startsWith("thread.") || event.type === "run.created" || event.type === "run.updated" || event.type === "runtime-request.updated" || event.type.startsWith("provider-session.") || (event.type === "turn-item.updated" && event.payload.type === "error"); const makeSweep = Effect.gen(function* () { const projections = yield* ProjectionStore.ProjectionStoreV2; const threads = yield* ThreadManagement.ThreadManagementService; const orchestrator = yield* Orchestrator.OrchestratorV2; const settings = yield* ServerSettings.ServerSettingsService; const due = DueHint.make(); yield* forkParked( Stream.runForEach(orchestrator.streamDomainEvents, (event) => affectsLimitRecovery(event) ? due.invalidate : Effect.void, ), ); yield* forkParked(Stream.runForEach(settings.streamChanges, () => due.invalidate)); return Effect.fn("UsageLimitRecoveryWorker.sweep")(function* () { const now = yield* DateTime.now; const nowMs = DateTime.toEpochMillis(now); if (yield* due.isIdle(nowMs)) return; const readVersion = yield* due.version; const preferences = yield* settings.getSettings; const candidates = yield* projections.getLimitRecoveryCandidates({ now, autoResume: preferences.autoResumeLimitedThreads, snooze: preferences.snoozeLimitedThreads, }); let acted = false; for (const thread of candidates) { const command = limitRecoveryCommand( thread, preferences.autoResumeLimitedThreads, nowMs, preferences.snoozeLimitedThreads, ); if (command === null) continue; acted = true; yield* Effect.gen(function* () { if (command.type === "thread.metadata.update" && command.limitRecovery != null) { yield* orchestrator.autoArmLimitRecovery({ commandId: command.commandId, threadId: command.threadId, limitRecovery: command.limitRecovery, }); } else { yield* threads.dispatch(command); } }).pipe( Effect.catchCause((cause) => Effect.logWarning("orchestration-v2.limit-recovery.dispatch-failed", { threadId: thread.id, cause, }), ), ); } // A sweep that acted, or failed to, re-checks on the next tick. if (acted) return; const wakeAt = yield* projections.getLimitRecoveryWakeAt(now); yield* due.record(readVersion, wakeAt ?? Infinity, nowMs); }); }); export const layer = Layer.effectDiscard( Effect.gen(function* () { const sweep = yield* makeSweep; const scheduler = yield* Scheduler.Scheduler; yield* scheduler.register("usage-limit-recovery", sweep()); }), );