import type { AgentAwarenessThread } from "../orchestration-v2/ProjectionStore.ts"; import { isListedThreadLineage, type EnvironmentId, type OrchestrationV2DomainEvent, type OrchestrationV2TurnItem, type Project, type RuntimeRequestId, type ThreadId, } from "@t3tools/contracts"; import { RelayApi, type RelayAgentActivityPublishProofPayload, type RelayAgentActivityState, } from "@t3tools/contracts/relay"; import { type SubagentRequestAwareness, projectThreadAwarenessV2, subagentAwaitsApproval, } from "@t3tools/shared/agentAwareness"; import { turnItemUpdateCanEndBackgroundWork } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; import { withRelayClientTracing } from "@t3tools/shared/relayTracing"; import { normalizeRelayIssuer, RELAY_ACTIVITY_PUBLISH_TYP } from "@t3tools/shared/relayJwt"; import * as Cause from "effect/Cause"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import type * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; import * as FetchHttpClient from "effect/http/FetchHttpClient"; import * as HttpClient from "effect/http/HttpClient"; import * as HttpClientRequest from "effect/http/HttpClientRequest"; import * as HttpApiClient from "effect/http-api/HttpApiClient"; import * as ServerSecretStore from "../auth/ServerSecretStore.ts"; import { isAgentActivityPublishingEnabledValue, PUBLISH_AGENT_ACTIVITY_SECRET, RELAY_ENVIRONMENT_CREDENTIAL_SECRET, RELAY_ISSUER_SECRET, RELAY_URL_SECRET, } from "../cloud/config.ts"; import { EnvironmentSigningKey } from "../cloud/environmentKeys.ts"; import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as ProjectService from "../project/ProjectService.ts"; import { forkParked } from "../serverActivation.ts"; export class AgentAwarenessRelay extends Context.Service< AgentAwarenessRelay, { readonly publishThread: (threadId: ThreadId) => Effect.Effect; readonly drain: Effect.Effect; readonly requestCatchUp: () => Effect.Effect; readonly start: () => Effect.Effect; } >()("t3/relay/AgentAwarenessRelay") {} function eventThreadId(event: OrchestrationV2DomainEvent): ThreadId { return event.threadId; } function isTurnItemPayload( payload: unknown, ): payload is Pick { return ( typeof payload === "object" && payload !== null && "type" in payload && "status" in payload ); } function shouldPublishAgentAwarenessEvent( event: Pick & { readonly payload?: unknown }, ): boolean { if ( event.type === "thread.created" && typeof event.payload === "object" && event.payload !== null && "historyOrigin" in event.payload && event.payload.historyOrigin === "v1_import" ) { return false; } switch (event.type) { case "thread.created": case "thread.archived": case "thread.unarchived": case "thread.deleted": case "thread.metadata-updated": case "thread.pull-request-link-updated": case "thread.pull-request-synced": case "thread.settled": case "thread.model-selection-updated": case "thread.provider-switched": case "run.created": case "run.updated": case "runtime-request.updated": case "subagent.updated": case "provider-thread.updated": return true; // Unsettling must not re-alert a dismissed result. case "thread.unsettled": case "thread.guest-share-updated": case "thread.snoozed": case "thread.unsnoozed": case "thread.auto-settle-set": case "thread.pinned": case "thread.unpinned": case "thread.promoted": case "thread.demoted": case "thread.pin-reordered": case "thread.active-reordered": case "thread.tags-updated": case "thread.visited": case "thread.marked-unread": case "thread.runtime-mode-updated": case "thread.interaction-mode-updated": case "run.background-work-cancelled": case "run-attempt.created": case "run-attempt.updated": case "node.updated": case "provider-session.attached": case "provider-session.updated": case "provider-session.detached": case "provider-turn.updated": case "turn-item.updated": return isTurnItemPayload(event.payload) && turnItemUpdateCanEndBackgroundWork(event.payload); case "message.updated": case "plan.updated": case "checkpoint-scope.created": case "checkpoint.captured": case "checkpoint.rollback-requested": case "context-handoff.updated": case "context-transfer.created": case "context-transfer.updated": return false; } } export const makeAgentAwarenessPublishWorker = Effect.fnUntraced(function* ( publish: (threadId: ThreadId) => Effect.Effect, ) { const queued = new Set(); const worker = yield* makeDrainableWorker((threadId: ThreadId) => Effect.sync(() => queued.delete(threadId)).pipe(Effect.andThen(publish(threadId))), ); const enqueue = (threadId: ThreadId) => Effect.suspend(() => { if (queued.has(threadId)) return Effect.void; // Remove queued IDs when processing starts so publication can queue one later update. queued.add(threadId); return worker.enqueue(threadId); }).pipe(Effect.uninterruptible); return { enqueue, drain: worker.drain }; }); function agentAwarenessPublishIdentity( state: RelayAgentActivityState | null, subagentRequest: SubagentRequestAwareness | null = null, ): string { if (state === null) { return "null"; } const { updatedAt: _updatedAt, ...meaningfulState } = state; // A child's next request reads the same as its last one, but it is a new // wait and must reach the phone. return JSON.stringify( subagentRequest === null ? meaningfulState : [meaningfulState, subagentRequest.requestId], ); } const CLEARED_PUBLISH_IDENTITY = agentAwarenessPublishIdentity(null); // A published tombstone stays recorded so repeats are skipped, but it is not live state. function hasLivePublishedState( publishedStateByThread: ReadonlyMap, threadId: ThreadId, ): boolean { const identity = publishedStateByThread.get(threadId); return identity !== undefined && identity !== CLEARED_PUBLISH_IDENTITY; } function resolveAgentActivityPublishingStartupState(input: { readonly relayConfigured: boolean; readonly publishEnabled: boolean; }): "waiting-for-link" | "disabled" | "enabled" { if (!input.relayConfigured) { return "waiting-for-link"; } return input.publishEnabled ? "enabled" : "disabled"; } const RELAY_AGENT_ACTIVITY_DETAIL_MAX_LENGTH = 160; const REDACTED_RELAY_AGENT_FAILURE_DETAIL = "The agent run failed."; const RELAY_AGENT_ACTIVITY_RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000] as const; function sanitizeRelayAgentActivityState( state: RelayAgentActivityState | null, ): RelayAgentActivityState | null { if (state === null) { return null; } const { detail: _detail, ...rest } = state; const detail = (state.phase === "failed" ? REDACTED_RELAY_AGENT_FAILURE_DETAIL : state.detail) ?.trim() .slice(0, RELAY_AGENT_ACTIVITY_DETAIL_MAX_LENGTH) .trim(); return detail ? { ...rest, detail } : rest; } function relayEnvironmentClient(token: string) { return HttpClient.mapRequest(HttpClientRequest.setHeader("authorization", `Bearer ${token}`)); } function deliveryStats( deliveries: ReadonlyArray<{ readonly ok: boolean; readonly queued?: boolean | undefined; readonly kind: string; readonly apnsStatus?: number | null; readonly apnsReason?: string | null; }>, ) { let queued = 0; let successful = 0; let failed = 0; const failedReasons: string[] = []; const kinds = new Set(); for (const delivery of deliveries) { kinds.add(delivery.kind); if (delivery.queued) { queued += 1; continue; } if (delivery.ok) { successful += 1; continue; } failed += 1; failedReasons.push(`${delivery.apnsStatus ?? "transport"}:${delivery.apnsReason ?? "unknown"}`); } return { total: deliveries.length, queued, successful, failed, kinds: [...kinds], failedReasons, }; } const makePublishProof = Effect.fn("makePublishProof")(function* (input: { readonly relayIssuer: string; readonly environmentId: string; readonly threadId: ThreadId; readonly state: RelayAgentActivityState | null; readonly jti: string; }) { const now = yield* DateTime.now; const expiresAt = DateTime.add(now, { minutes: 5 }); const payload = { iss: `t3-env:${input.environmentId}`, aud: normalizeRelayIssuer(input.relayIssuer), sub: input.environmentId, jti: input.jti, iat: Math.floor(now.epochMilliseconds / 1_000), exp: Math.floor(expiresAt.epochMilliseconds / 1_000), environmentId: input.environmentId as RelayAgentActivityPublishProofPayload["environmentId"], threadId: input.threadId, state: input.state, } satisfies RelayAgentActivityPublishProofPayload; const signingKey = yield* EnvironmentSigningKey; return yield* signingKey.sign({ typ: RELAY_ACTIVITY_PUBLISH_TYP, payload }); }); function describeThreadShellForAwareness( thread: Option.Option, ): Record { if (Option.isNone(thread)) { return { found: false }; } const shell = thread.value; return { found: true, status: shell.status, activityRunStatus: shell.activityRunStatus ?? null, activeRunId: shell.activeRunId ?? null, latestRunId: shell.latestRunId ?? null, pendingRuntimeRequestKind: shell.pendingRuntimeRequest?.kind ?? null, hasActionableProposedPlan: shell.hasActionableProposedPlan, }; } interface SubagentRequest { readonly topLevelThreadId: ThreadId; readonly requestId: RuntimeRequestId; } interface SubagentAlert { readonly requestId: RuntimeRequestId; readonly updatedAt: string; } const waitingApprovalId = (thread: AgentAwarenessThread | null) => thread !== null && thread.archivedAt === null && subagentAwaitsApproval(thread) ? thread.pendingRuntimeRequest?.id : undefined; function resolveAgentAwarenessRelayPublishSnapshot(input: { readonly environmentId: EnvironmentId; readonly thread: Option.Option; readonly project: Option.Option; readonly pendingSubagentRequest: SubagentRequestAwareness | null; }): { readonly projectId: string | null; readonly state: RelayAgentActivityState | null; /** The child request behind the state's wait, when the parent isn't waiting itself. */ readonly subagentWait: SubagentRequestAwareness | null; readonly reason: "snapshot" | "thread-not-found" | "project-not-found"; } { if (Option.isNone(input.thread)) { return { projectId: null, state: null, subagentWait: null, reason: "thread-not-found", }; } if (Option.isNone(input.project)) { return { projectId: input.thread.value.projectId, state: null, subagentWait: null, reason: "project-not-found", }; } const awarenessInput = { environmentId: input.environmentId, project: input.project.value, thread: input.thread.value, }; const state = projectThreadAwarenessV2({ ...awarenessInput, pendingSubagentRequest: input.pendingSubagentRequest, }); const childDrivesWait = input.pendingSubagentRequest !== null && state?.phase !== projectThreadAwarenessV2(awarenessInput)?.phase; return { projectId: input.thread.value.projectId, state: sanitizeRelayAgentActivityState(state), subagentWait: childDrivesWait ? input.pendingSubagentRequest : null, reason: "snapshot", }; } function terminalWorkSinceStart(thread: AgentAwarenessThread, startedAt: number): boolean { return ( thread.latestRunCompletedAt != null && DateTime.toEpochMillis(thread.latestRunCompletedAt) > startedAt ); } export function resolveAgentAwarenessRelayActiveThreadIds(input: { readonly environmentId: EnvironmentId; readonly startedAt: number; readonly projects: ReadonlyArray>; readonly threads: ReadonlyArray; }): ReadonlyArray { const projectById = new Map(input.projects.map((project) => [project.id, project])); return input.threads .filter((thread) => { // Publishing a waiting subagent republishes its top-level thread. if (subagentAwaitsApproval(thread)) return true; const project = projectById.get(thread.projectId); if (!project) { return false; } const state = projectThreadAwarenessV2({ environmentId: input.environmentId, project, thread, }); return ( state !== null && (state.phase !== "completed" && state.phase !== "failed" ? true : terminalWorkSinceStart(thread, input.startedAt)) ); }) .map((thread) => thread.id); } /** @public */ export const make = Effect.gen(function* () { const secrets = yield* ServerSecretStore.ServerSecretStore; const serverEnvironment = yield* ServerEnvironment.ServerEnvironment; const threads = yield* ThreadManagement.ThreadManagementService; const projects = yield* ProjectService.ProjectService; const random = yield* SecureRandom.SecureRandom; const signingKey = yield* EnvironmentSigningKey; const scope = yield* Effect.scope; const startedAt = (yield* DateTime.now).epochMilliseconds; const catchUpRequests = yield* Queue.dropping(1); const publishedStateByThreadRef = yield* Ref.make(new Map()); // Android keys alerts by phase and time, so a child's unchanged request keeps // its first alert time through parent renames, relinks and the parent's own requests. const childAlertByThread = new Map(); const subagentRequests = new Map(); const readSecretString = (name: string) => secrets .get(name) .pipe( Effect.map((bytes) => Option.isSome(bytes) ? new TextDecoder().decode(bytes.value) : null, ), ); const readRelayConfig = Effect.gen(function* () { const [url, issuer, environmentCredential] = yield* Effect.all([ readSecretString(RELAY_URL_SECRET), readSecretString(RELAY_ISSUER_SECRET), readSecretString(RELAY_ENVIRONMENT_CREDENTIAL_SECRET), ]); return url && environmentCredential ? { url, issuer: issuer ?? url, environmentCredential } : null; }); const readPublishAgentActivityEnabled = readSecretString(PUBLISH_AGENT_ACTIVITY_SECRET).pipe( Effect.map(isAgentActivityPublishingEnabledValue), ); const makeRelayClient = (relayConfig: { readonly url: string; readonly environmentCredential: string; }) => HttpApiClient.make(RelayApi, { baseUrl: relayConfig.url, transformClient: relayEnvironmentClient(relayConfig.environmentCredential), }).pipe(Effect.provide(FetchHttpClient.layer)); let cachedRelayClient: Effect.Success> | undefined; let publishedRelayConfig: NonNullable> | undefined; // Confirm through the same worker so tombstones cannot race live updates; recovery clears the deadline. const publishConfirmDeadlines = new Map(); // Threads whose confirmation holds a completion, so a thread that settles during the hold still alerts. const heldCompletions = new Set(); let schedulePublishConfirm: (threadId: ThreadId) => Effect.Effect = () => Effect.void; const publishRetries = new Map< ThreadId, { readonly attempts: number; timer: Fiber.Fiber | undefined } >(); const cancelPublishRetry = Effect.fnUntraced(function* (threadId: ThreadId) { const retry = publishRetries.get(threadId); publishRetries.delete(threadId); if (retry?.timer !== undefined) yield* Fiber.interrupt(retry.timer); }); const cancelPublishRetries = Effect.suspend(() => Effect.forEach([...publishRetries.keys()], cancelPublishRetry, { discard: true }), ); let schedulePublishRetry: (threadId: ThreadId) => Effect.Effect = () => Effect.void; let enqueueTopLevelPublish: (threadId: ThreadId) => Effect.Effect = () => Effect.void; const resetPublishedConnection = Effect.gen(function* () { publishedRelayConfig = undefined; cachedRelayClient = undefined; publishConfirmDeadlines.clear(); heldCompletions.clear(); yield* Ref.set(publishedStateByThreadRef, new Map()); }); const publishThreadUnsafe = Effect.fn("publishThreadUnsafe")(function* (threadId: ThreadId) { const publishAgentActivity = yield* readPublishAgentActivityEnabled; if (!publishAgentActivity) { yield* cancelPublishRetries; yield* resetPublishedConnection; yield* Effect.logDebug("agent activity publish skipped; publication disabled", { threadId, }); return; } const relayConfig = yield* readRelayConfig; if (!relayConfig) { yield* cancelPublishRetries; yield* resetPublishedConnection; yield* Effect.logDebug("agent activity publish skipped; relay link credentials unavailable", { threadId, }); return; } if ( publishedRelayConfig?.url !== relayConfig.url || publishedRelayConfig.issuer !== relayConfig.issuer || publishedRelayConfig.environmentCredential !== relayConfig.environmentCredential ) { yield* resetPublishedConnection; publishedRelayConfig = relayConfig; } const environmentId = yield* serverEnvironment.getEnvironmentId; const publishState = Effect.fn("relay.agent_activity.publish")(function* (input: { readonly projectId: string | null; readonly state: RelayAgentActivityState | null; readonly reason: string; }) { if (cachedRelayClient === undefined) { cachedRelayClient = yield* makeRelayClient(relayConfig); } const relayClient = cachedRelayClient; const proof = yield* makePublishProof({ relayIssuer: relayConfig.issuer, environmentId, threadId, state: input.state, jti: yield* random.uuidV4(), }).pipe(Effect.provideService(EnvironmentSigningKey, signingKey)); yield* Effect.logInfo("publishing agent activity for thread", { environmentId, threadId, projectId: input.projectId, statePhase: input.state?.phase ?? null, hasState: input.state !== null, reason: input.reason, }); const response = yield* relayClient.server .publishAgentActivity({ params: { environmentId, threadId, }, payload: { state: input.state, proof, }, }) .pipe(Effect.timeout("10 seconds")); yield* Effect.logInfo("agent activity publish completed", { environmentId, threadId, ok: response.ok, deliveries: deliveryStats(response.deliveries), }); }); // Read only this thread; full shell reads would make each activity event scale with all threads. const threadShell = yield* readAwarenessThread(threadId); yield* trackSubagentRequest(threadId, threadShell); if ( threadShell !== null && !isListedThreadLineage(threadShell.lineage) && !hasLivePublishedState(yield* Ref.get(publishedStateByThreadRef), threadId) ) { // Subagents and side chats have no relay rows to clear; per-event tombstones would needlessly redeliver user aggregates. return; } const thread = threadShell === null || threadShell.archivedAt !== null ? Option.none() : Option.some(threadShell); const project = Option.isSome(thread) ? yield* projects.getById(thread.value.projectId) : Option.none(); const snapshot = resolveAgentAwarenessRelayPublishSnapshot({ environmentId, thread, project, pendingSubagentRequest: yield* currentSubagentRequestFor(threadId), }); const publishIdentity = agentAwarenessPublishIdentity(snapshot.state, snapshot.subagentWait); const publishedStateByThread = yield* Ref.get(publishedStateByThreadRef); if ( (snapshot.state?.phase === "completed" || snapshot.state?.phase === "failed") && !hasLivePublishedState(publishedStateByThread, threadId) ) { if (Option.isNone(thread) || !terminalWorkSinceStart(thread.value, startedAt)) return; } if ( publishedStateByThread.get(threadId) === publishIdentity && // A held completion settled after an earlier clear still has to alert. !(snapshot.state === null && heldCompletions.has(threadId)) ) { // Clear moot deadlines so later transient nulls cannot skip their deferral window. publishConfirmDeadlines.delete(threadId); heldCompletions.delete(threadId); yield* Effect.logDebug("agent activity publish skipped; projected state unchanged", { environmentId, threadId, reason: snapshot.reason, }); return; } // Confirm tombstones over live state to avoid removing active cards. // Confirm completion: a run can end moments before its continuation starts, and the relay // alerts on every move into completed. const requiresConfirmation = snapshot.state === null || snapshot.state?.phase === "completed"; if (requiresConfirmation) { if (snapshot.state?.phase === "completed") heldCompletions.add(threadId); const nowMs = (yield* DateTime.now).epochMilliseconds; const deadline = publishConfirmDeadlines.get(threadId); if (deadline === undefined) { publishConfirmDeadlines.set(threadId, nowMs + 5_000); yield* Effect.logInfo("agent activity publish deferred pending confirmation", { environmentId, threadId, reason: snapshot.reason, statePhase: snapshot.state?.phase ?? null, shell: describeThreadShellForAwareness(thread), }); yield* schedulePublishConfirm(threadId); return; } if (nowMs < deadline) { return; } yield* Effect.logInfo("agent activity deferred publish confirmed", { environmentId, threadId, reason: snapshot.reason, statePhase: snapshot.state?.phase ?? null, shell: describeThreadShellForAwareness(thread), }); // Settled during the hold, not resumed: alert the current completion, then clear the card // on a new window. Only settlement may hide it; queued or silent runs also project to null. const unsettled = snapshot.state === null && heldCompletions.has(threadId) && Option.isSome(thread) && thread.value.settledOverride === "settled" ? resolveAgentAwarenessRelayPublishSnapshot({ environmentId, thread: Option.some({ ...thread.value, settledOverride: null }), project, pendingSubagentRequest: null, }) : null; if (unsettled?.state?.phase === "completed") { const completion = unsettled.state; yield* publishState({ projectId: unsettled.projectId, state: completion, reason: unsettled.reason, }).pipe(withRelayClientTracing); heldCompletions.delete(threadId); publishConfirmDeadlines.set(threadId, nowMs + 5_000); yield* Ref.update(publishedStateByThreadRef, (publishedStates) => new Map(publishedStates).set(threadId, agentAwarenessPublishIdentity(completion)), ); yield* schedulePublishConfirm(threadId); return; } } else { publishConfirmDeadlines.delete(threadId); heldCompletions.delete(threadId); } if (snapshot.reason === "thread-not-found") { yield* Effect.logDebug("publishing agent activity tombstone; thread not found", { environmentId, threadId, }); } else if (snapshot.reason === "project-not-found") { yield* Effect.logDebug("publishing agent activity tombstone; project not found", { environmentId, threadId, projectId: snapshot.projectId, }); } const childRequestId = snapshot.subagentWait?.requestId; const childAlert = childAlertByThread.get(threadId); const state = snapshot.state !== null && childRequestId !== undefined && childAlert?.requestId === childRequestId ? { ...snapshot.state, updatedAt: childAlert.updatedAt } : snapshot.state; yield* publishState({ projectId: snapshot.projectId, state, reason: snapshot.reason, }).pipe(withRelayClientTracing); if (state !== null && childRequestId !== undefined) { childAlertByThread.set(threadId, { requestId: childRequestId, updatedAt: state.updatedAt }); } publishConfirmDeadlines.delete(threadId); heldCompletions.delete(threadId); yield* Ref.update(publishedStateByThreadRef, (publishedStates) => new Map(publishedStates).set(threadId, publishIdentity), ); }); const readAwarenessThread = (threadId: ThreadId) => threads .getAgentAwarenessSnapshot(threadId) .pipe(Effect.map((snapshot) => snapshot.threads[0] ?? null)); // The subagent helpers below are untraced: they run per event inside // publishThreadUnsafe, whose span already covers them. const findTopLevelThreadId = Effect.fnUntraced(function* (child: AgentAwarenessThread) { const seen = new Set([child.id]); let current: AgentAwarenessThread | null = child; while ( current?.lineage.relationshipToParent === "subagent" || current?.lineage.sideChat === true ) { const parentId: ThreadId | null = current.lineage.parentThreadId; if (parentId === null || seen.has(parentId)) return null; seen.add(parentId); current = yield* readAwarenessThread(parentId); } return current?.id ?? null; }); const trackSubagentRequest = Effect.fnUntraced(function* ( threadId: ThreadId, threadShell: AgentAwarenessThread | null, ) { const requestId = waitingApprovalId(threadShell); const previous = subagentRequests.get(threadId); // An unchanged wait still republishes its parent: a relink catch-up reaches it only here. if (threadShell === null || requestId === undefined) { if (previous === undefined) return; subagentRequests.delete(threadId); yield* enqueueTopLevelPublish(previous.topLevelThreadId); return; } const topLevelThreadId = previous?.topLevelThreadId ?? (yield* findTopLevelThreadId(threadShell)); if (topLevelThreadId === null) return; subagentRequests.set(threadId, { topLevelThreadId, requestId }); yield* enqueueTopLevelPublish(topLevelThreadId); }); // The map can miss a child's update while publishing is off or the relay is // unlinked, so each waiting child is re-read before its parent is published. const currentSubagentRequestFor = Effect.fnUntraced(function* (topLevelThreadId: ThreadId) { let pending: SubagentRequestAwareness | null = null; for (const [childId, entry] of subagentRequests) { if (entry.topLevelThreadId !== topLevelThreadId) continue; const child = yield* readAwarenessThread(childId); const requestId = waitingApprovalId(child); if (child === null || requestId === undefined) { subagentRequests.delete(childId); continue; } if (requestId !== entry.requestId) subagentRequests.set(childId, { ...entry, requestId }); // The newest wait is the one to announce. if (pending === null || DateTime.isGreaterThan(child.updatedAt, pending.updatedAt)) { pending = { requestId, updatedAt: child.updatedAt }; } } return pending; }); const processThreadPublish = Effect.fnUntraced(function* (threadId: ThreadId) { const retry = publishRetries.get(threadId); if (retry?.timer !== undefined) { const timer = retry.timer; retry.timer = undefined; yield* Fiber.interrupt(timer); } yield* publishThreadUnsafe(threadId).pipe( Effect.tap(() => cancelPublishRetry(threadId)), Effect.catchCause((cause) => { if (Cause.hasInterruptsOnly(cause)) return Effect.void; return Effect.logWarning("agent activity publish failed", { threadId, cause: Cause.pretty(cause), }).pipe(Effect.andThen(schedulePublishRetry(threadId))); }), Effect.withSpan("AgentAwarenessRelay.publishThread"), ); }); const worker = yield* makeAgentAwarenessPublishWorker(processThreadPublish); const enqueueThreadPublish = (threadId: ThreadId) => cancelPublishRetry(threadId).pipe(Effect.andThen(worker.enqueue(threadId))); enqueueTopLevelPublish = enqueueThreadPublish; const publishThread: AgentAwarenessRelay["Service"]["publishThread"] = (threadId) => enqueueThreadPublish(threadId).pipe(Effect.andThen(worker.drain)); schedulePublishRetry = Effect.fnUntraced(function* (threadId: ThreadId) { const attempts = publishRetries.get(threadId)?.attempts ?? 0; const delayMs = RELAY_AGENT_ACTIVITY_RETRY_DELAYS_MS[attempts]; if (delayMs === undefined) { // Retain exhausted budgets until activity or success resets them; old timers must not restart retries. yield* Effect.logWarning("agent activity publish retry budget exhausted", { threadId }); return; } const retry = { attempts: attempts + 1, timer: undefined as Fiber.Fiber | undefined, }; publishRetries.set(threadId, retry); const timer = yield* Effect.sleep(delayMs).pipe( Effect.andThen( Effect.suspend(() => { if (publishRetries.get(threadId) !== retry) return Effect.void; retry.timer = undefined; return worker.enqueue(threadId); }), ), Effect.forkIn(scope), ); if (publishRetries.get(threadId) !== retry) { yield* Fiber.interrupt(timer); } else { retry.timer = timer; } }); const publishActiveThreadsUnsafe = Effect.gen(function* () { const relayUrl = yield* readSecretString(RELAY_URL_SECRET).pipe( Effect.orElseSucceed(() => null), ); if (!relayUrl) { yield* Effect.logDebug("agent activity snapshot skipped; relay link credentials unavailable"); return "unlinked" as const; } const publishAgentActivity = yield* readPublishAgentActivityEnabled.pipe( Effect.orElseSucceed(() => false), ); if (!publishAgentActivity) { yield* Effect.logDebug("agent activity snapshot skipped; publication disabled"); return "disabled" as const; } const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null)); if (!relayConfig) { yield* Effect.logDebug("agent activity snapshot skipped; relay link credentials unavailable"); return "unlinked" as const; } const environmentId = yield* serverEnvironment.getEnvironmentId; const projectSnapshot = yield* projects.snapshot; const activitySnapshot = yield* threads.getAgentAwarenessSnapshot(); // Live children aren't listed, but a waiting one makes its top-level thread wait. const liveChildren = yield* Effect.forEach( (yield* threads.listActiveShellIndex()).filter( (thread) => thread.lineage.relationshipToParent === "subagent" && thread.mayBeLive, ), (thread) => readAwarenessThread(thread.id), ); const activeThreadIds = resolveAgentAwarenessRelayActiveThreadIds({ environmentId, startedAt, projects: projectSnapshot.projects, threads: [...activitySnapshot.threads, ...liveChildren.filter((thread) => thread !== null)], }); if (activeThreadIds.length === 0) { yield* Effect.logDebug("agent activity snapshot has no publishable threads"); return "published" as const; } yield* Effect.logInfo("publishing active agent activity snapshot", { count: activeThreadIds.length, }); yield* Effect.forEach(activeThreadIds, enqueueThreadPublish, { discard: true }); yield* worker.drain; return "published" as const; }); // Many environments never link, so while unlinked the retry backs off from 5 // s to 60 s. A linked environment keeps the 5 s retry, because `t3 connect // publish` can turn publishing on from another process. Keep this untraced: // an unlinked environment can retry for the lifetime of the process. const publishActiveThreadsWhenConfigured = Effect.fnUntraced(function* ( logEnabledWhenReady: boolean, ) { let unlinkedRetryDelayMs = 5_000; while (true) { const result = yield* publishActiveThreadsUnsafe.pipe( Effect.orElseSucceed(() => "failed" as const), ); if (result === "published") { if (logEnabledWhenReady) { const relayConfig = yield* readRelayConfig.pipe(Effect.orElseSucceed(() => null)); yield* Effect.logInfo("agent activity publishing enabled after link reconciliation", { relayUrl: relayConfig?.url, }); } logEnabledWhenReady = false; unlinkedRetryDelayMs = 5_000; yield* Queue.take(catchUpRequests); continue; } const retryDelayMs = result === "unlinked" ? unlinkedRetryDelayMs : 5_000; yield* Effect.race(Effect.sleep(retryDelayMs), Queue.take(catchUpRequests)); if (result === "unlinked") { unlinkedRetryDelayMs = Math.min(unlinkedRetryDelayMs * 2, 60_000); } } }); schedulePublishConfirm = (threadId) => Effect.forkIn( Effect.sleep("5 seconds").pipe( Effect.andThen(worker.enqueue(threadId)), Effect.catchCause((cause) => Effect.logWarning("deferred agent activity confirmation failed", { threadId, cause: Cause.pretty(cause), }), ), ), scope, ).pipe(Effect.asVoid); const start: AgentAwarenessRelay["Service"]["start"] = Effect.fn("AgentAwarenessRelay.start")( function* () { const [relayConfig, publishEnabled] = yield* Effect.all([ readRelayConfig.pipe(Effect.orElseSucceed(() => null)), readPublishAgentActivityEnabled.pipe(Effect.orElseSucceed(() => false)), ]); const startupState = resolveAgentActivityPublishingStartupState({ relayConfigured: relayConfig !== null, publishEnabled, }); switch (startupState) { case "waiting-for-link": yield* Effect.logInfo( "agent activity publishing standby; waiting for T3 Connect link reconciliation", ); break; case "disabled": yield* Effect.logInfo("agent activity publishing disabled by T3 Connect configuration"); break; case "enabled": yield* Effect.logInfo("agent activity publishing enabled", { relayUrl: relayConfig?.url, }); break; } yield* forkParked( Effect.sleep("1 second").pipe( Effect.andThen(publishActiveThreadsWhenConfigured(startupState !== "enabled")), ), ); yield* forkParked( Stream.runForEach(threads.streamDomainEvents, (event) => { const threadId = eventThreadId(event); if (!shouldPublishAgentAwarenessEvent(event)) { return Effect.void; } return Effect.logDebug("agent activity publishing queued thread publish", { eventType: event.type, threadId, }).pipe(Effect.andThen(enqueueThreadPublish(threadId))); }), ); }, ); return AgentAwarenessRelay.of({ publishThread, drain: worker.drain, requestCatchUp: Effect.fn("AgentAwarenessRelay.requestCatchUp")(function () { return Queue.offer(catchUpRequests, undefined).pipe(Effect.asVoid); }), start, }); }); export const layer = Layer.effect(AgentAwarenessRelay, make);