import { CommandId, type EnvironmentId, NodeId, OrchestratorMcpFailure, type OrchestratorMcpDelegateTaskInput, type OrchestratorMcpTaskCancelResult, type OrchestratorMcpThreadTimelineItem, type OrchestrationV2RemoteTaskChild, type OrchestrationV2Subagent, type ProjectId, type ProviderInstanceId, ProviderDriverKind, type RunId, type ThreadId, } from "@t3tools/contracts"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import type * as Tool from "effect/ai/Tool"; import type * as McpInvocationContext from "../mcp/McpInvocationContext.ts"; import { ProjectToolkit } from "../mcp/toolkits/project/tools.ts"; import { ThreadToolkit } from "../mcp/toolkits/thread/tools.ts"; import { resolveInteractionMode, resolveRuntimeMode, taskPrompt, } from "../mcp/delegatedTaskModes.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as ProjectService from "../project/ProjectService.ts"; import * as RepositoryIdentityResolver from "../project/RepositoryIdentityResolver.ts"; import { forkParked } from "../serverActivation.ts"; import * as PeerForwarding from "./PeerForwarding.ts"; import * as PeerLinks from "./PeerLinks.ts"; /** How long one follow of a remote task waits on its thread there before checking again. */ const FOLLOW_WAIT_MS = 10 * 60 * 1_000; /** A linked environment that stops answering is retried, backing off to this. */ const FOLLOW_MAX_BACKOFF = Duration.minutes(5); /** How long settling a task there keeps retrying, say while that machine sleeps. */ const SETTLE_RETRY_WINDOW = Duration.days(1); /** How a follower retries; tests step it by hand. */ export const FollowRetrySchedule = Context.Reference>( "t3/peer/RemoteDelegation/FollowRetrySchedule", { defaultValue: () => Schedule.exponential(Duration.seconds(2)).pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, FOLLOW_MAX_BACKOFF)), ), ), }, ); type ThreadScope = McpInvocationContext.McpThreadInvocationScope; /** * `delegate_task` to a linked environment. The task's child is an ordinary * thread there, launched through the link; the parent here records the task * like a local one, without a child thread, and a follower completes it when * the thread there ends, so the parent wakes as it would for a local child. */ export class RemoteDelegation extends Context.Service< RemoteDelegation, { readonly delegate: ( scope: ThreadScope, input: OrchestratorMcpDelegateTaskInput & { readonly target: NonNullable & { readonly environmentId: EnvironmentId; }; }, ) => Effect.Effect<{ readonly taskId: NodeId }, OrchestratorMcpFailure>; /** * Stops the task's thread there, then completes it here as cancelled. A * task that already has its result keeps it; its thread there is still * stopped, as cancelling a local task stops follow-up work on its child. */ readonly cancel: ( scope: ThreadScope, task: OrchestrationV2Subagent, input: { readonly reason?: string | undefined; readonly clientRequestId?: string | undefined; }, ) => Effect.Effect; /** * Follows every open remote task until it ends, including after a * restart, and a finished one again when a Stop here owes it a stop there. */ readonly start: () => Effect.Effect; } >()("t3/peer/RemoteDelegation") {} const isMcpFailure = Schema.is(OrchestratorMcpFailure); /** The other side refused because the thread there runs above this caller's modes. */ const isModeRefusal = (error: OrchestratorMcpFailure) => error.code === "runtime_mode_escalation_denied" || error.code === "interaction_mode_escalation_denied"; /** * Refusals that following again cannot change: the link was revoked, expired * or forgotten, or the thread or run there is gone. Anything else (the other * machine asleep, a dropped route) is worth retrying. */ const isFinal = (error: OrchestratorMcpFailure) => error.code === "capability_denied" || error.code === "thread_not_found" || error.code === "run_not_found"; const make = Effect.gen(function* () { const followRetry = yield* FollowRetrySchedule; const forwarding = yield* PeerForwarding.PeerForwarding; const links = yield* PeerLinks.PeerLinks; const threads = yield* ThreadManagement.ThreadManagementService; const projections = yield* ProjectionStore.ProjectionStoreV2; const projects = yield* ProjectService.ProjectService; const repositoryIdentities = yield* RepositoryIdentityResolver.RepositoryIdentityResolver; const crypto = yield* Crypto.Crypto; const hereId = yield* ServerEnvironment.ServerEnvironment.pipe( Effect.flatMap((environment) => environment.getEnvironmentId), ); // Followers live as long as this service, so shutting down stops them. const followers = yield* Scope.Scope; // One follower runs per open task and holds what it read across a wait of // up to FOLLOW_WAIT_MS, so it reads its own task, never the parent's whole // task list: a parent with thousands of tasks made every restart decode // that list once per open task at the same time and ran out of heap. const readTask = (parentThreadId: ThreadId, taskId: NodeId) => threads .getThreadRecords(parentThreadId, ["subagents"], { subagentIds: [taskId] }) .pipe(Effect.map(({ thread, subagents }) => ({ thread, task: subagents[0] }))); /** The peer's project with this thread's repository, or the one the caller named. */ const resolveRemoteProject = Effect.fn("RemoteDelegation.resolveRemoteProject")(function* ( scope: ThreadScope, environmentId: EnvironmentId, parentProjectId: ProjectId, requested: ProjectId | undefined, ) { if (requested !== undefined) return requested; const here = yield* projects.getById(parentProjectId).pipe( Effect.map(Option.getOrUndefined), Effect.orElseSucceed(() => undefined), ); // The project's own identity is a one-minute cache that reads blank when // cold, so resolve it here, as the other side's t3_project_list does. const key = here === undefined ? undefined : (yield* repositoryIdentities.resolve(here.workspaceRoot))?.canonicalKey; if (key === undefined) { return yield* new OrchestratorMcpFailure({ code: "target_required", message: "Pass target.projectId: this thread's project has no repository to match there.", }); } const matches: Array<{ readonly id: ProjectId; readonly title: string }> = []; let cursor: number | null = 0; while (cursor !== null) { const listed: Tool.Success = yield* forwarding.call(scope, ProjectToolkit.tools.t3_project_list, environmentId, { cursor, limit: 100, }); matches.push( ...listed.projects.filter((project) => project.repositoryIdentity?.canonicalKey === key), ); cursor = listed.nextCursor; } if (matches.length === 1) return matches[0]!.id; return yield* new OrchestratorMcpFailure({ code: "invalid_request", message: matches.length === 0 ? `No project there has this repository (${key}). Pass target.projectId; t3_project_list with this environmentId lists them. To bring the repository there, use t3_project_clone and then t3_project_create with this environmentId.` : `Several projects there have this repository: ${matches.map((project) => `${project.title} (${project.id})`).join(", ")}. Pass target.projectId.`, }); }); const delegate: RemoteDelegation["Service"]["delegate"] = Effect.fn("RemoteDelegation.delegate")( function* (scope, input) { const parent = yield* threads.getThreadRecords(scope.thread.threadId, ["runs"]).pipe( Effect.mapError( () => new OrchestratorMcpFailure({ code: "thread_not_found", message: "The calling thread was not found.", }), ), ); const parentRun = parent.runs .filter(ThreadManagement.isActiveRun) .toSorted((left, right) => right.ordinal - left.ordinal)[0]; if ( parentRun === undefined || parentRun.rootNodeId === null || parentRun.providerInstanceId !== scope.thread.providerInstanceId ) { return yield* new OrchestratorMcpFailure({ code: "parent_not_active", message: "Delegated tasks require an active run owned by this MCP provider session.", }); } const parentNodeId = parentRun.rootNodeId; const { target } = input; // A worktree there would branch from a commit this checkout has and the // other may not, so a remote task runs in that project's checkout. if (input.isolation === "worktree") { return yield* new OrchestratorMcpFailure({ code: "invalid_request", message: 'A task in a linked environment runs in that project\'s checkout; isolation "worktree" is not supported there. Omit isolation, or use t3_thread_launch with environmentId and a worktree workspaceStrategy.', }); } // The provider and model inherit as they do for a task here. Another // provider there has no model to inherit, so the caller names one. const inherited = parent.thread.modelSelection; const instanceId = target.providerInstanceId ?? inherited.instanceId; const model = target.model ?? (instanceId === inherited.instanceId ? inherited.model : undefined); if (model === undefined) { return yield* new OrchestratorMcpFailure({ code: "target_required", message: "Pass target.model from orchestrator_capabilities with this environmentId.", }); } const options = target.options ?? (instanceId === inherited.instanceId && model === inherited.model ? inherited.options : undefined); // Modes asked for are checked against this thread. Omitted ones are left // to the other side, which inherits this thread's modes capped by the // link's access, so a narrower link still takes the task. const runtimeMode = input.runtimeMode === undefined || input.runtimeMode === "inherit" ? undefined : yield* resolveRuntimeMode(parent.thread.runtimeMode, input.runtimeMode); const interactionMode = input.interactionMode === undefined || input.interactionMode === "inherit" ? undefined : yield* resolveInteractionMode(parent.thread.interactionMode, input.interactionMode); const projectId = yield* resolveRemoteProject( scope, target.environmentId, parent.thread.projectId, target.projectId, ); // One key for the launch there and the record here, so a retry finds both. // Both are scoped to the provider session, as every key a caller passes is: // the forwarder scopes the launch's, and the record's names it. const key = input.clientRequestId ?? (yield* crypto.randomUUIDv4().pipe(Effect.orDie)); const task = taskPrompt(input); const launched = yield* forwarding.call( scope, ProjectToolkit.tools.t3_thread_launch, target.environmentId, { projectId, title: input.title ?? parent.thread.title, modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) }, ...(runtimeMode === undefined ? {} : { runtimeMode }), ...(interactionMode === undefined ? {} : { interactionMode }), message: task, clientRequestId: `delegate:${scope.thread.threadId}:${key}`, // The thread there shows it is this thread's subagent, with a way back. delegatedFrom: { environmentId: hereId, threadId: parent.thread.id, title: parent.thread.title, }, }, ); const link = yield* links.get(target.environmentId).pipe( Effect.map(Option.getOrUndefined), Effect.orElseSucceed(() => undefined), ); const commandId = CommandId.make( `command:mcp:${encodeURIComponent(scope.requestNamespace)}:remote-delegate:${encodeURIComponent(key)}`, ); const remoteChild: OrchestrationV2RemoteTaskChild = { environmentId: target.environmentId, threadId: launched.threadId, label: link?.label ?? target.environmentId, ...(launched.runId === null ? {} : { runId: launched.runId }), }; const recorded = yield* threads .dispatch({ type: "delegated_task.remote.request", commandId, parentThreadId: scope.thread.threadId, parentRunId: parentRun.id, parentNodeId, task, ...(input.title === undefined ? {} : { title: input.title }), // The driver runs there; this side only labels the task with it. driver: ProviderDriverKind.make("remote"), modelSelection: launched.modelSelection, remoteChild, completionWake: input.mode === "wait" ? "settled_only" : "always", }) .pipe( // The cause goes to the log; the caller learns only that recording failed. Effect.tapError((cause) => Effect.logWarning("remote-delegation.record-failed", { parentThreadId: scope.thread.threadId, cause, }), ), Effect.mapError( () => new OrchestratorMcpFailure({ code: "orchestration_error", message: "Unable to record the delegated task.", }), ), // Nothing here follows a thread the task was never recorded for, so stop it there. // A crash between the launch and this record still leaves it running there // until a retry with the same clientRequestId records it. Effect.tapError(() => stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop").pipe( Effect.ignoreCause({ log: true }), ), ), ); const taskEvent = recorded.storedEvents.find( (stored) => stored.event.type === "subagent.updated", ); const taskId = taskEvent?.event.type === "subagent.updated" ? taskEvent.event.payload.id : // A replayed request recorded the task the first time. yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( Effect.map( ({ subagents }) => subagents.find((task) => task.remoteChild?.threadId === launched.threadId)?.id, ), Effect.orElseSucceed(() => undefined), ); if (taskId === undefined) { return yield* new OrchestratorMcpFailure({ code: "orchestration_error", message: "The delegated task was not recorded.", }); } yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers)); return { taskId }; }, ); /** * Completes `task` here with its thread there's final state. Each attempt is its own * command, so a failed one never blocks the next; a report refused because the task already * has its result (a cancel, a Stop or an earlier report) is done. `stoppedThere` records that * nothing is left to stop there, even on a task that already has its result. */ const complete = Effect.fn("RemoteDelegation.complete")(function* ( parentThreadId: ThreadId, taskId: NodeId, status: "completed" | "failed" | "cancelled" | "interrupted", result: string, stoppedThere: "ran-out" | "stopped" | undefined, stopCount?: number, ) { const attempt = yield* crypto.randomUUIDv4().pipe(Effect.orDie); yield* threads .dispatch({ type: "delegated_task.remote.complete", commandId: CommandId.make(`command:remote-task-complete:${taskId}:${attempt}`), parentThreadId, taskId, status, result, ...(stoppedThere === undefined ? {} : { stoppedThere }), ...(stopCount === undefined ? {} : { stopCount }), }) .pipe( Effect.catch((error) => readTask(parentThreadId, taskId).pipe( Effect.flatMap(({ task }) => (task?.result != null ? Effect.void : Effect.fail(error))), ), ), ); }); /** * How this environment reaches a task's thread there: as the parent * thread, under one namespace per parent, so a stop sent again after a * restart or a dropped route is recorded there once. */ const followScope = ( parent: { readonly threadId: ThreadId; readonly providerInstanceId: ProviderInstanceId }, remote: OrchestrationV2RemoteTaskChild, ): ThreadScope => ({ environmentId: hereId, requestNamespace: `remote-task:${parent.threadId}`, thread: { threadId: parent.threadId, providerSessionId: `remote-task:${remote.threadId}`, providerInstanceId: parent.providerInstanceId, }, client: undefined, capabilities: new Set(["orchestration"]), issuedAt: 0, }); /** * Stops the task's own thread there as its Stop button does: its running turn, queued * messages, pull request watches and delegated tasks. `key` names the stop (the follower * sends one, each cancel its own), and each is recorded there once however often it is sent. */ const stopThere = ( scope: ThreadScope, remote: OrchestrationV2RemoteTaskChild, key: string, reason?: string, ) => forwarding .interruptThread(scope, remote.environmentId, { threadId: remote.threadId, stop: true, ...(reason === undefined ? {} : { reason }), clientRequestId: `${key}:${remote.threadId}`, }) .pipe(Effect.asVoid); /** * The last reply `runId` gave on the task's thread there, which is its * result, read whole however long it is. A failed run prefers its error * from the activity view over an earlier reply, without taking another run's. */ const lastReply = Effect.fn("RemoteDelegation.lastReply")(function* ( scope: ThreadScope, remote: OrchestrationV2RemoteTaskChild, runId: RunId | null, failed = false, ) { let reply: OrchestratorMcpThreadTimelineItem | undefined; let error: OrchestratorMcpThreadTimelineItem | undefined; let afterPosition: number | undefined; while (true) { const read = yield* forwarding.readThread(scope, remote.environmentId, { threadId: remote.threadId, view: failed ? "activity" : "messages", limit: 100, ...(afterPosition === undefined ? {} : { afterPosition }), }); reply = read.items.findLast( (item) => item.type === "assistant_message" && (runId === null || item.runId === runId), ) ?? reply; if (failed) { error = read.items.findLast( (item) => item.type === "error" && (runId === null || item.runId === runId), ) ?? error; } if (!read.hasMore || read.nextPosition === null) break; afterPosition = read.nextPosition; } reply = error ?? reply; if (reply === undefined) return undefined; let text = reply.text ?? ""; let textOffset = reply.textTruncated ? (reply.nextTextOffset ?? null) : null; while (textOffset !== null) { const rest = yield* forwarding.readThread(scope, remote.environmentId, { threadId: remote.threadId, itemId: reply.itemId, textOffset, maxCharsPerItem: 50_000, }); const part = rest.items[0]; if (part === undefined) break; text += part.text ?? ""; textOffset = part.textTruncated ? (part.nextTextOffset ?? null) : null; } return text; }); /** * Why following a task can never succeed, if that is what `error` means. * The far side's message already says what was refused and what to do; a * link forgotten here only reads as a bad request, so it is named. */ const finalReason = Effect.fn("RemoteDelegation.finalReason")( function* (parentThreadId: ThreadId, taskId: NodeId, error: OrchestratorMcpFailure) { if (isFinal(error)) return error.message; if (error.code !== "invalid_request") return undefined; const { task } = yield* readTask(parentThreadId, taskId); const environmentId = task?.remoteChild?.environmentId; if (environmentId === undefined) return error.message; const link = yield* links.get(environmentId); return Option.isSome(link) ? undefined : error.message; }, Effect.orElseSucceed(() => undefined), ); /** A task's thread there that ended without a reply. */ const noReply = "(The thread in the linked environment ended without a reply.)"; /** * Waits for the task's run there to end, completes the task here, and stops it there if it * ended here first. A revoked, expired or forgotten link, or a thread or run gone there, * fails the task with nothing left to stop; anything else (the other machine asleep, a * dropped route) is retried with back-off, and so is completing. */ const follow = (parentThreadId: ThreadId, taskId: NodeId): Effect.Effect => followOnce(parentThreadId, taskId).pipe( // `false` means it is still running there: follow again straight away. Effect.repeat({ while: (done) => !done }), Effect.catchTags({ OrchestratorMcpFailure: Effect.fn("RemoteDelegation.followFailed")(function* (error) { const reason = yield* finalReason(parentThreadId, taskId, error); if (reason === undefined) return yield* error; // Gone there, or no link to reach it by: nothing left to stop there. const { task } = yield* readTask(parentThreadId, taskId); const stopCount = task?.remoteChild?.stopCount; yield* complete(parentThreadId, taskId, "failed", reason, "stopped", stopCount); }), }), Effect.retry(followRetry), Effect.ignoreCause({ log: true }), ); /** * One wait on the task's thread there; true once the task here is done. * A task completed here first (cancelled, or its parent stopped) is * stopped there, and done once that stop is recorded here. */ const followOnce = Effect.fn("RemoteDelegation.followOnce")(function* ( parentThreadId: ThreadId, taskId: NodeId, ) { const sequence = yield* threads.getThreadEventSequence(parentThreadId); const { thread, task } = yield* readTask(parentThreadId, taskId); if (task?.remoteChild === undefined) return true; const remote = task.remoteChild; const scope = followScope( { threadId: parentThreadId, providerInstanceId: thread.providerInstanceId }, remote, ); // Ended here, maybe while this follower backed off from an unreachable // machine or before a restart: it still has to be stopped there. So // does one its parent here was stopped after, for what followed there. if (task.result !== null) { if ( remote.stoppedThere === false || ((task.status === "cancelled" || task.status === "interrupted") && remote.stoppedThere !== true) ) { const key = remote.stopCount === undefined ? "stop" : `stop-${remote.stopCount}`; yield* stopThere(scope, remote, key).pipe( // Its thread there now runs above what this side may touch, say // because the parent here was lowered since; no retry changes that. Effect.catchIf(isModeRefusal, (error) => Effect.logWarning("A linked environment refused to stop a delegated task", { taskId, reason: error.message, }), ), ); yield* complete( parentThreadId, taskId, terminalStatus(task.status), task.result, "stopped", remote.stopCount, ); return yield* settled(parentThreadId, taskId); } return true; } const endedHere = threads .streamStoredEventsFrom({ threadId: parentThreadId, afterSequence: sequence, eventType: "subagent.updated", }) .pipe( Stream.filter( (stored) => stored.event.type === "subagent.updated" && stored.event.payload.id === taskId && stored.event.payload.result !== null, ), Stream.runHead, Effect.flatMap((ended) => Option.isSome(ended) ? Effect.succeed({ endedHere: true } as const) : Effect.never, ), ); const waited = yield* forwarding .waitForThread(scope, remote.environmentId, { threadId: remote.threadId, ...(remote.runId === undefined ? {} : { runId: remote.runId }), timeoutMs: FOLLOW_WAIT_MS, // A thread there whose run ended may still be waiting on its own // tasks to wake it for the answer, which a task here waits for too. // An older peer ignores this and reports the run alone. untilDelegatedWorkSettled: true, }) .pipe(Effect.raceFirst(endedHere)); // The next look reads the result and stops it there. if ("endedHere" in waited) return false; if (waited.timedOut) return false; // A task recorded without its run waits on the thread's latest; a // thread there with none has nothing left to wait for. if (waited.runId === null) { const reply = yield* lastReply(scope, remote, null); yield* complete( parentThreadId, taskId, reply === undefined ? "failed" : "completed", reply ?? "The thread in the linked environment has no run to follow.", "ran-out", ); return yield* settled(parentThreadId, taskId); } if (!isTerminal(waited.status)) return false; const resultRun = waited.resultRun ?? { runId: remote.runId ?? waited.runId, status: waited.status, }; const result = yield* lastReply(scope, remote, resultRun.runId, resultRun.status === "failed"); yield* complete( parentThreadId, taskId, terminalStatus(resultRun.status), result ?? noReply, "ran-out", ); return yield* settled(parentThreadId, taskId); }); /** * Whether the task is done here after the follower reported its run ran * out there. A cancel or Stop that landed while the reply was read keeps * its result, and still owes a stop there for what may follow the run; * the next look sends it. */ const settled = (parentThreadId: ThreadId, taskId: NodeId) => readTask(parentThreadId, taskId).pipe( Effect.map(({ task }) => task?.remoteChild?.stoppedThere === true), ); const following = new Map(); /** * `follow`, at most once per task. Asked again while it runs, say by a * Stop here on a task that already has its result, it looks once more * before it ends, so that stop is not missed. */ const followOne = (parentThreadId: ThreadId, taskId: NodeId) => Effect.suspend(() => { const running = following.get(taskId); if (running !== undefined) { running.again = true; return Effect.void; } const state = { again: false }; following.set(taskId, state); return follow(parentThreadId, taskId).pipe( // Checked and released in one step, so a request between them is never lost. Effect.repeat({ while: () => { if (state.again) { state.again = false; return true; } following.delete(taskId); return false; }, }), Effect.ensuring( Effect.sync(() => { if (following.get(taskId) === state) following.delete(taskId); }), ), ); }); const cancel: RemoteDelegation["Service"]["cancel"] = Effect.fn("RemoteDelegation.cancel")( function* (scope, task, input) { const remote = task.remoteChild; if (remote === undefined) { return yield* new OrchestratorMcpFailure({ code: "task_not_found", message: `Delegated task ${task.id} is not remote.`, }); } const { reason } = input; const there = followScope(scope.thread, remote); // A later cancel stops what started there since; a retry of this one, nothing new. const key = `cancel:${encodeURIComponent(scope.requestNamespace)}:${ input.clientRequestId ?? (yield* crypto.randomUUIDv4().pipe(Effect.orDie)) }`; if (task.result !== null) { // Best effort: the result stands whether or not the stop gets there. yield* stopThere(there, remote, key, reason).pipe( Effect.andThen( complete( task.threadId, task.id, terminalStatus(task.status), task.result, "stopped", remote.stopCount, ), ), Effect.ignoreCause({ log: true }), ); return { taskId: task.id, status: task.status === "completed" || task.status === "failed" || task.status === "interrupted" ? task.status : "cancelled", } satisfies OrchestratorMcpTaskCancelResult; } // A link that is gone or a machine that is down still cancels the task // here; its follower keeps trying to stop it there, across restarts. const unstopped = yield* stopThere(there, remote, key, reason).pipe( Effect.as(undefined), Effect.catch((error) => Effect.succeed(error.message)), ); const cancelled = reason === undefined ? "Cancelled." : `Cancelled: ${reason}`; yield* complete( task.threadId, task.id, "cancelled", unstopped === undefined ? cancelled : `${cancelled} It could not be stopped in ${remote.label} and may still run there: ${unstopped}`, unstopped === undefined ? "stopped" : undefined, remote.stopCount, ).pipe( Effect.tapError((cause) => Effect.logWarning("remote-delegation.cancel-record-failed", { taskId: task.id, cause }), ), Effect.mapError( () => new OrchestratorMcpFailure({ code: "orchestration_error", message: "Unable to record the cancellation.", }), ), ); return { taskId: task.id, status: "cancel_requested", } satisfies OrchestratorMcpTaskCancelResult; }, ); /** * Settles the task's thread there while its parent here stays settled. Clients list that thread * under its parent only while they hold the parent, and a settled parent drops out of their * snapshots once newer threads settle, so an unsettled task would resurface as active there. */ const settleThere = Effect.fn("RemoteDelegation.settleThere")( function* (parentThreadId: ThreadId, taskId: NodeId) { const { thread, task } = yield* readTask(parentThreadId, taskId); const remote = task?.remoteChild; if (thread.settledOverride !== "settled" || remote === undefined) return; const scope = followScope( { threadId: parentThreadId, providerInstanceId: thread.providerInstanceId }, remote, ); yield* forwarding.call(scope, ThreadToolkit.tools.t3_thread_organize, remote.environmentId, { threadId: remote.threadId, action: "settle", }); }, // A task still running there refuses until it ends; a machine asleep, until it wakes. Effect.retry({ schedule: followRetry.pipe(Schedule.upTo({ duration: SETTLE_RETRY_WINDOW })), while: (error) => isMcpFailure(error) && error.code !== "invalid_request" && !(isFinal(error) || isModeRefusal(error)), }), Effect.ignoreCause({ log: true }), ); // Each retrying settle re-reads its parent, so a repeated settle here needs no second one. const settling = new Set(); const settleAllThere = Effect.fn("RemoteDelegation.settleAllThere")( function* (parentThreadId: ThreadId) { const { subagents } = yield* threads.getThreadRecords(parentThreadId, ["subagents"]); for (const task of subagents) { if (task.remoteChild === undefined || settling.has(task.id)) continue; settling.add(task.id); yield* settleThere(parentThreadId, task.id).pipe( Effect.ensuring(Effect.sync(() => settling.delete(task.id))), Effect.forkIn(followers), ); } }, Effect.ignoreCause({ log: true }), ); const start: RemoteDelegation["Service"]["start"] = () => forkParked( Effect.gen(function* () { // A finished task's follower has ended, so a Stop here that owes its // thread there a stop starts it again. Started before the scan below; // a Stop that slips between the two stays owed for the next start. yield* Stream.runForEach(threads.streamDomainEvents, (event) => event.type === "subagent.updated" && event.payload.remoteChild?.stoppedThere === false ? followOne(event.threadId, event.payload.id).pipe(Effect.forkIn(followers)) : event.type === "thread.settled" ? settleAllThere(event.threadId) : Effect.void, ).pipe( Effect.ignoreCause({ log: true }), Effect.forkIn(followers, { startImmediately: true }), ); const open = yield* projections.getOpenRemoteDelegatedTasks; yield* Effect.forEach( open, ({ parentThreadId, taskId }) => followOne(parentThreadId, taskId), { concurrency: "unbounded", discard: true, }, ); }).pipe(Effect.ignoreCause({ log: true })), ); return RemoteDelegation.of({ delegate, cancel, start }); }); const isTerminal = (status: string) => status === "completed" || status === "failed" || status === "cancelled" || status === "interrupted" || status === "rolled_back"; const terminalStatus = (status: string): "completed" | "failed" | "cancelled" | "interrupted" => status === "completed" || status === "failed" || status === "interrupted" ? status : "cancelled"; export const layer = Layer.effect(RemoteDelegation, make);