import { expect, it } from "@effect/vitest"; import { CommandId, EnvironmentId, MessageId, type OrchestrationV2Command, type OrchestrationV2Run, type OrchestrationV2ProviderThread, ProviderThreadId, ProviderDriverKind, ProviderSessionId, type OrchestrationV2StoredEvent, type OrchestrationV2ThreadProjection, ProjectId, ProviderInstanceId, RunId, ThreadId, } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Option from "effect/Option"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as ServerSettings from "../serverSettings.ts"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import * as ThreadManagementService from "./ThreadManagementService.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; it("stamps authoritative provenance on commands that create threads or messages", () => { const command: OrchestrationV2Command = { type: "thread.create", createdBy: "agent", creationSource: "mcp", commandId: CommandId.make("command:thread-management:create"), threadId: ThreadId.make("thread:thread-management:create"), projectId: ProjectId.make("project:thread-management"), title: "Thread management", modelSelection: { instanceId: ProviderInstanceId.make("codex"), model: "gpt-5-codex", }, runtimeMode: "full-access", interactionMode: "default", branch: null, worktreePath: null, // A client cannot claim a linked environment's origin, or a parent there. linkOrigin: { sessionId: "forged", label: "Somewhere else" }, delegatedFrom: { environmentId: EnvironmentId.make("environment:elsewhere"), threadId: ThreadId.make("thread:elsewhere"), title: "Somewhere else", }, }; const stamped = ThreadManagementService.withCreationProvenance(command, { createdBy: "user", creationSource: "web", }); expect(stamped).toMatchObject({ createdBy: "user", creationSource: "web", }); expect(stamped).not.toHaveProperty("linkOrigin"); expect(stamped).not.toHaveProperty("delegatedFrom"); }); const layerReceiptsTest = CommandReceiptStore.layer.pipe( Layer.provide(SqlitePersistence.layerMemory), ); const layerDispatchChokepointTest = (settingsLayer: ReturnType) => (dispatched: Array) => ThreadManagementService.layer.pipe( Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), Layer.provide( Layer.mock(Orchestrator.OrchestratorV2)({ getThreadRecords: () => Effect.succeed({ thread: { id: ThreadId.make("thread:thread-management:chokepoint"), projectId: ProjectId.make("project:thread-management:chokepoint"), deletedAt: null, modelSelection: { instanceId: ProviderInstanceId.make("codex_work"), model: "test", }, }, } as unknown as OrchestrationV2ThreadProjection), dispatch: (command) => Effect.sync(() => { dispatched.push(command.type); return { sequence: 1, storedEvents: [] } as never; }), }), ), Layer.provide(settingsLayer), Layer.provideMerge(layerReceiptsTest), ); it.effect( "the dispatch chokepoint refuses a continuation message.dispatch on a project-disabled instance before reaching the orchestrator", () => { const projectId = ProjectId.make("project:thread-management:chokepoint"); const threadId = ThreadId.make("thread:thread-management:chokepoint"); const instanceId = ProviderInstanceId.make("codex_work"); const commandId = CommandId.make("command:thread-management:chokepoint-disabled"); const dispatched: Array = []; const layerTest = layerDispatchChokepointTest( ServerSettings.layerTest({ providerInstances: { [instanceId]: { driver: "codex", enabled: true } }, projectSettingsOverrides: { [projectId]: { providerInstances: { [instanceId]: { enabled: false } } }, }, }), )(dispatched); return Effect.gen(function* () { const threads = yield* ThreadManagementService.ThreadManagementService; const error = yield* Effect.flip( threads.dispatch({ type: "message.dispatch", commandId, threadId, messageId: MessageId.make("message:thread-management:chokepoint-disabled"), text: "Background task completed.", attachments: [], dispatchMode: { type: "queue_after_active" }, createdBy: "agent", creationSource: "server", }), ); expect(error).toBeInstanceOf( ThreadManagementService.ThreadManagementProviderInstanceDisabledError, ); expect(error).toMatchObject({ projectId, instanceId }); expect(dispatched).toEqual([]); }).pipe(Effect.provide(layerTest)); }, ); it.effect( "a replay of an already-accepted command id dispatches despite the instance now being disabled", () => { const projectId = ProjectId.make("project:thread-management:chokepoint"); const threadId = ThreadId.make("thread:thread-management:chokepoint"); const instanceId = ProviderInstanceId.make("codex_work"); const commandId = CommandId.make("command:thread-management:chokepoint-replay"); const dispatched: Array = []; const layerTest = layerDispatchChokepointTest( ServerSettings.layerTest({ providerInstances: { [instanceId]: { driver: "codex", enabled: true } }, projectSettingsOverrides: { [projectId]: { providerInstances: { [instanceId]: { enabled: false } } }, }, }), )(dispatched); return Effect.gen(function* () { const receipts = yield* CommandReceiptStore.CommandReceiptStoreV2; yield* receipts.insertIfAbsent({ commandId, threadId, commandType: "message.dispatch", acceptedAt: yield* DateTime.now, resultSequence: 1, status: "accepted", error: null, }); const threads = yield* ThreadManagementService.ThreadManagementService; yield* threads.dispatch({ type: "message.dispatch", commandId, threadId, messageId: MessageId.make("message:thread-management:chokepoint-replay"), text: "Background task completed.", attachments: [], dispatchMode: { type: "queue_after_active" }, createdBy: "agent", creationSource: "server", }); expect(dispatched).toEqual(["message.dispatch"]); }).pipe(Effect.provide(layerTest)); }, ); it.effect("waitForThread follows a restart continuation instead of the cut run", () => Effect.gen(function* () { const projectId = ProjectId.make("project:thread-management:wait-restart"); const threadId = ThreadId.make("thread:thread-management:wait-restart"); const runId = RunId.make("run:thread-management:wait-restart"); const continuationId = RunId.make("run:thread-management:wait-restart:2"); const subscribed = yield* Deferred.make(); const events = yield* Queue.unbounded(); let runs: Array<{ id: RunId; status: OrchestrationV2Run["status"]; restartContinuationOfRunId?: RunId; }> = [{ id: runId, status: "cancelled" }]; let pending = true; const stored = (sequence: number, event: object) => ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent; const layerTest = ThreadManagementService.layer.pipe( Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), Layer.provide( Layer.mock(Orchestrator.OrchestratorV2)({ getThreadEventSequence: () => Effect.succeed(0), restartContinuationPending: (run) => Effect.succeed(pending && run.id === runId), awaitRestartContinuation: () => Effect.never, streamStoredEventsFrom: (input) => input?.eventType === "run.updated" ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( Stream.drain, Stream.concat(Stream.fromQueue(events)), ) : Stream.never, getThreadRecords: () => Effect.sync( () => ({ thread: { id: threadId, projectId, deletedAt: null }, runs, }) as unknown as OrchestrationV2ThreadProjection, ), }), ), Layer.provide(ServerSettings.layerTest()), Layer.provide(layerReceiptsTest), ); const service = yield* ThreadManagementService.ThreadManagementService.pipe( Effect.provide(layerTest), ); const fiber = yield* service .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }) .pipe(Effect.forkChild); yield* Deferred.await(subscribed); const continuation = { id: continuationId, restartContinuationOfRunId: runId }; runs = [...runs, { ...continuation, status: "running" }]; pending = false; yield* Queue.offer( events, stored(1, { type: "run.updated", payload: { ...continuation, status: "running" } }), ); runs = [runs[0]!, { ...continuation, status: "completed" }]; yield* Queue.offer( events, stored(2, { type: "run.updated", payload: { ...continuation, status: "completed" } }), ); const result = yield* Fiber.join(fiber); expect(result).toMatchObject({ timedOut: false, run: { id: continuationId, status: "completed" }, }); }), ); it.effect("waitForThread can wait past an ended run for the thread's delegated work", () => Effect.gen(function* () { const projectId = ProjectId.make("project:thread-management:wait-delegated"); const threadId = ThreadId.make("thread:thread-management:wait-delegated"); const runId = RunId.make("run:thread-management:wait-delegated"); const wakeRunId = RunId.make("run:thread-management:wait-delegated:wake"); const subscribed = yield* Deferred.make(); const events = yield* Queue.unbounded(); const at = (seconds: number) => DateTime.makeUnsafe(seconds * 1_000); const run = (id: RunId, ordinal: number, status: OrchestrationV2Run["status"], end: number) => ({ id, ordinal, status, startedAt: at(end - 1), completedAt: status === "running" ? null : at(end), }) as unknown as OrchestrationV2Run; // The run ended, but a task it delegated still runs and will wake the thread. let runs = [run(runId, 1, "completed", 1)]; let subagents = [{ status: "running" }]; const stored = (sequence: number, event: object) => ({ sequence, event: { threadId, ...event } }) as unknown as OrchestrationV2StoredEvent; const layerTest = ThreadManagementService.layer.pipe( Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), Layer.provide( Layer.mock(Orchestrator.OrchestratorV2)({ getThreadEventSequence: () => Effect.succeed(0), restartContinuationPending: () => Effect.succeed(false), delegatedTaskResultPending: () => Effect.succeed(false), streamStoredEventsFrom: (input) => input?.eventType === "subagent.updated" ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( Stream.drain, Stream.concat(Stream.fromQueue(events)), ) : Stream.never, getThreadRecords: () => Effect.sync( () => ({ thread: { id: threadId, projectId, deletedAt: null }, runs, messages: [], subagents, providerThreads: [], }) as unknown as OrchestrationV2ThreadProjection, ), }), ), Layer.provide(ServerSettings.layerTest()), Layer.provide(layerReceiptsTest), ); const service = yield* ThreadManagementService.ThreadManagementService.pipe( Effect.provide(layerTest), ); // Without the option, the ended run is the answer, as before. expect( yield* service.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }), ).toMatchObject({ timedOut: false, run: { id: runId } }); expect( yield* service.waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }), ).not.toHaveProperty("resultRun"); const fiber = yield* service .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000, untilDelegatedWorkSettled: true, }) .pipe(Effect.forkChild); // A wait that returns early fails below instead of hanging here. yield* Effect.raceFirst(Deferred.await(subscribed), Fiber.await(fiber)); // The task finishes; its delivery wakes the thread, whose next run answers. subagents = [{ status: "completed" }]; runs = [...runs, run(wakeRunId, 2, "completed", 3)]; yield* Queue.offer(events, stored(1, { type: "subagent.updated", payload: {} })); const result = yield* Fiber.join(fiber); expect(result).toMatchObject({ timedOut: false, run: { id: runId, status: "completed" }, resultRun: { id: wakeRunId, status: "completed" }, }); }), ); it.effect("waitForThread settles delegated work on a thread rolled back past its only run", () => Effect.gen(function* () { const projectId = ProjectId.make("project:thread-management:wait-rolled-back"); const threadId = ThreadId.make("thread:thread-management:wait-rolled-back"); const runId = RunId.make("run:thread-management:wait-rolled-back"); const rolledBack = { id: runId, ordinal: 1, status: "rolled_back", startedAt: DateTime.makeUnsafe(0), completedAt: DateTime.makeUnsafe(1_000), } as unknown as OrchestrationV2Run; const layerTest = ThreadManagementService.layer.pipe( Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), Layer.provide( Layer.mock(Orchestrator.OrchestratorV2)({ getThreadEventSequence: () => Effect.succeed(0), restartContinuationPending: () => Effect.succeed(false), delegatedTaskResultPending: () => Effect.succeed(false), streamStoredEventsFrom: () => Stream.never, getThreadRecords: () => Effect.succeed({ thread: { id: threadId, projectId, deletedAt: null }, runs: [rolledBack], messages: [], subagents: [], providerThreads: [], } as unknown as OrchestrationV2ThreadProjection), }), ), Layer.provide(ServerSettings.layerTest()), Layer.provide(layerReceiptsTest), ); const service = yield* ThreadManagementService.ThreadManagementService.pipe( Effect.provide(layerTest), ); const fiber = yield* service .waitForThread({ projectId, threadId, runId, timeoutMs: 60_000, untilDelegatedWorkSettled: true, }) .pipe(Effect.forkChild); // Nothing will ever run there again, so a wait still open runs out here. yield* TestClock.adjust(Duration.millis(60_000)); const result = yield* Fiber.join(fiber); expect(result).toMatchObject({ timedOut: false, run: { id: runId, status: "rolled_back" }, resultRun: { id: runId, status: "rolled_back" }, }); }), ); it.effect("waitForThread returns the cut run once its continuation is declined", () => Effect.gen(function* () { const projectId = ProjectId.make("project:thread-management:wait-declined"); const threadId = ThreadId.make("thread:thread-management:wait-declined"); const runId = RunId.make("run:thread-management:wait-declined"); const subscribed = yield* Deferred.make(); const declined = yield* Deferred.make(); let pending = true; const layerTest = ThreadManagementService.layer.pipe( Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), Layer.provide( Layer.mock(Orchestrator.OrchestratorV2)({ getThreadEventSequence: () => Effect.succeed(0), restartContinuationPending: () => Effect.succeed(pending), awaitRestartContinuation: () => Deferred.await(declined), streamStoredEventsFrom: (input) => input?.eventType === "run.updated" ? Stream.fromEffect(Deferred.succeed(subscribed, undefined)).pipe( Stream.drain, Stream.concat(Stream.never), ) : Stream.never, getThreadRecords: () => Effect.succeed({ thread: { id: threadId, projectId, deletedAt: null }, runs: [{ id: runId, status: "cancelled" }], } as unknown as OrchestrationV2ThreadProjection), }), ), Layer.provide(ServerSettings.layerTest()), Layer.provide(layerReceiptsTest), ); const service = yield* ThreadManagementService.ThreadManagementService.pipe( Effect.provide(layerTest), ); const fiber = yield* service .waitForThread({ projectId, threadId, runId, timeoutMs: 60 * 60 * 1_000 }) .pipe(Effect.forkChild); yield* Deferred.await(subscribed); pending = false; yield* Deferred.succeed(declined, undefined); const result = yield* Fiber.join(fiber); expect(result).toMatchObject({ timedOut: false, run: { id: runId, status: "cancelled" } }); }), ); it.effect.each(["live", "no-session", "unsupported-runtime"] as const)( "marks background work awaited on the thread's active provider thread with state %s", (state) => Effect.gen(function* () { const threadId = ThreadId.make("thread:background-wait"); const providerThreadId = ProviderThreadId.make("provider-thread:background-wait"); const providerSessionId = ProviderSessionId.make("provider-session:background-wait"); const driver = ProviderDriverKind.make("claudeAgent"); const now = yield* DateTime.now; const providerThread: OrchestrationV2ProviderThread = { id: providerThreadId, driver, providerInstanceId: ProviderInstanceId.make("claudeAgent"), providerSessionId, appThreadId: threadId, ownerNodeId: null, nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" }, nativeConversationHeadRef: null, status: "idle", firstRunOrdinal: 1, lastRunOrdinal: 1, handoffIds: [], forkedFrom: null, pendingBackgroundTasks: [{ taskId: "tests", kind: "command" }], contextUsage: null, nativeMetadata: null, createdAt: now, updatedAt: now, }; const calls: Array<{ providerThreadId: ProviderThreadId; awaited: boolean }> = []; const runtime = { driver, providerSessionId, ...(state === "unsupported-runtime" ? {} : { awaitBackgroundTasks: (input: { readonly providerThread: OrchestrationV2ProviderThread; readonly taskIds: ReadonlyArray; readonly awaited: boolean; }) => Effect.sync(() => { calls.push({ providerThreadId: input.providerThread.id, awaited: input.awaited, }); return { matched: input.taskIds.filter((taskId) => taskId === "tests") }; }), }), } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime; const layerTest = ThreadManagementService.layer.pipe( Layer.provide( Layer.mergeAll( Layer.mock(Orchestrator.OrchestratorV2)({ getThreadRecords: () => Effect.succeed({ thread: { id: threadId, deletedAt: null, activeProviderThreadId: providerThreadId, }, providerThreads: [providerThread], } as unknown as OrchestrationV2ThreadProjection), }), Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({ get: () => Effect.succeed(state === "no-session" ? Option.none() : Option.some(runtime)), }), ), ), Layer.provide(ServerSettings.layerTest()), Layer.provide(layerReceiptsTest), ); const result = yield* Effect.gen(function* () { const service = yield* ThreadManagementService.ThreadManagementService; return yield* service .awaitBackgroundTasks({ threadId, taskIds: ["tests", "finished"], awaited: true }) .pipe(Effect.result); }).pipe(Effect.provide(Layer.merge(layerTest, ThreadCommandExecutor.layer))); if (state === "live") { expect(result).toMatchObject({ success: { matched: ["tests"], unmatched: ["finished"] }, }); expect(calls).toEqual([{ providerThreadId, awaited: true }]); } else { expect(result).toMatchObject({ failure: { _tag: "ThreadManagementBackgroundWaitError", reason: state === "no-session" ? "unavailable" : "unsupported", }, }); expect(calls).toEqual([]); } }), ); it.effect.each([ { status: "completed" as const, settles: true }, { status: "failed" as const, settles: false }, { status: "interrupted" as const, settles: false }, ])("settleAfterRun settles only when the run $status", ({ status, settles }) => Effect.gen(function* () { const projectId = ProjectId.make("project:thread-management:settle-after-run"); const threadId = ThreadId.make("thread:thread-management:settle-after-run"); const runId = RunId.make("run:thread-management:settle-after-run"); const dispatched: Array = []; const layerTest = ThreadManagementService.layer.pipe( Layer.provide(Layer.mock(ProviderSessionManager.ProviderSessionManagerV2)({})), Layer.provide( Layer.mock(Orchestrator.OrchestratorV2)({ getThreadEventSequence: () => Effect.succeed(0), restartContinuationPending: () => Effect.succeed(false), getThreadRecords: () => Effect.succeed({ thread: { id: threadId, projectId, deletedAt: null }, runs: [{ id: runId, status }], } as unknown as OrchestrationV2ThreadProjection), dispatch: (command) => Effect.sync(() => { dispatched.push(command.type); return { sequence: 1, storedEvents: [] } as never; }), }), ), Layer.provide(ServerSettings.layerTest()), Layer.provide(layerReceiptsTest), ); const service = yield* ThreadManagementService.ThreadManagementService.pipe( Effect.provide(layerTest), ); yield* service.settleAfterRun({ projectId, threadId, runId }); expect(dispatched).toEqual(settles ? ["thread.settle"] : []); }), );