import { CommandId, MessageId, type OrchestrationV2Notification, type PullRequestActivity, type PullRequestComment, type PullRequestRef, type PullRequestThreadCommentsResult, type ThreadPullRequestLink, type ThreadPullRequestWatch, } from "@t3tools/contracts"; import { normalizeThreadPullRequestKey, threadPullRequestKeyOf, visibleThreadPullRequests, } from "../sourceControl/threadPullRequests.ts"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import * as Context from "effect/Context"; import * as SecureRandom from "effect/crypto/SecureRandom"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import type * as Scope from "effect/Scope"; import { type ProviderChangeRequestWatchFingerprint, PullRequestProviderError, } from "@t3tools/source-control-core/server/PullRequestProvider"; import * as PullRequestService from "../pullRequest/PullRequestService.ts"; import { forkParked } from "../serverActivation.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import { evaluatePullRequestWatch, PULL_REQUEST_WATCH_NO_CHECKS_MS, pullRequestWatchMessage, } from "./pullRequestWatch.ts"; /** Checks take minutes; faster sweeps waste the account's shared host quota. */ const SWEEP_MINUTES = 2; const READ_FAILURE_LIMIT = 8; /** Group reads in flight at once, inside the host's own request limit. */ const GROUP_READS = 4; /** Without fingerprints, reread quiet PRs on snapshot changes or this interval to catch edits. */ const QUIET_REREAD_MS = 10 * 60_000; /** Reread despite unchanged fingerprints; they miss review-thread comment edits. */ const FINGERPRINT_REREAD_MS = 30 * 60_000; const isProviderError = Schema.is(PullRequestProviderError); const providerFailureReason = (cause: Cause.Cause) => Option.match(Cause.findErrorOption(cause), { onNone: () => null, onSome: (error) => error._tag === "PullRequestOperationError" && isProviderError(error.cause) ? error.cause.reason : null, }); const logFailure = (message: string, fields: Record) => (cause: Cause.Cause): Effect.Effect => Cause.hasInterruptsOnly(cause) ? Effect.interrupt : Effect.logWarning(message, { ...fields, cause }); interface WatchTarget { readonly thread: ProjectionStore.ProjectionThreadPullRequests; readonly link: ThreadPullRequestLink; readonly watch: ThreadPullRequestWatch; } interface WatchGroup { readonly key: string; readonly targets: ReadonlyArray; readonly fingerprint: string; } interface LastRead { readonly at: number; readonly fingerprint: string; readonly inFlight: boolean; readonly remarksComplete: boolean; readonly watches: ReadonlySet; readonly status: string | null; readonly remarks: string | null; } interface ReadPlan { readonly detail: boolean; readonly activity: boolean; } const watchKey = ({ thread, watch }: WatchTarget) => `${thread.id} ${watch.startedAt}`; type WatchEndReason = | "merged" | "closed" | "unreadable" | "comment-limit" | "settled" | "subagent" | "stopped"; interface WatchLife { readonly threadId: string; readonly pullRequest: string; readonly startedAt: number; readonly headSha: string | null; readonly pushedAt: number; readonly longestQuietMs: number; readonly wakes: number; readonly reads: number; } const lifeKey = ({ thread, link, watch }: WatchTarget) => `${thread.id} ${threadPullRequestKeyOf(link)} ${watch.startedAt}`; const minutes = (ms: number) => Math.max(0, Math.round(ms / 60_000)); const snapshotFingerprint = ({ link }: WatchTarget) => { const snapshot = link.snapshot; return snapshot === null ? "" : [ snapshot.state, snapshot.updatedAt, snapshot.checksState, snapshot.mergeability, snapshot.reviewDecision, snapshot.isDraft, ].join(" "); }; function planRead( group: WatchGroup, last: LastRead | undefined, fingerprint: ProviderChangeRequestWatchFingerprint | null, now: number, ): ReadPlan { if (last === undefined || group.targets.some((target) => !last.watches.has(watchKey(target)))) { return { detail: true, activity: true }; } // Nothing on the host changes when the no-checks deadline passes, so it alone prompts a read. const checksDue = group.targets.some( ({ watch }) => watch.checksAwaitedSince != null && now - Date.parse(watch.checksAwaitedSince) >= PULL_REQUEST_WATCH_NO_CHECKS_MS, ); if (fingerprint === null) { const read = last.inFlight || !last.remarksComplete || last.fingerprint !== group.fingerprint || now - last.at >= QUIET_REREAD_MS; return { detail: read || checksDue, activity: read }; } const activity = last.remarks !== fingerprint.remarks || !last.remarksComplete || now - last.at >= FINGERPRINT_REREAD_MS; return { detail: activity || checksDue || last.status !== fingerprint.status || last.inFlight, activity, }; } function watchesEqual(left: ThreadPullRequestWatch, right: ThreadPullRequestWatch): boolean { return ( left.startedAt === right.startedAt && left.headSha === right.headSha && left.failedChecks.join("\n") === right.failedChecks.join("\n") && left.passed === right.passed && left.passedChecks.join("\n") === right.passedChecks.join("\n") && (left.owedChecks ?? []).join("\n") === (right.owedChecks ?? []).join("\n") && left.remarksThrough === right.remarksThrough && left.remarkIds.join("\n") === right.remarkIds.join("\n") && left.conflicting === right.conflicting && left.wakes === right.wakes && left.checksAwaitedSince === right.checksAwaitedSince ); } export class PullRequestWatchReactor extends Context.Service< PullRequestWatchReactor, { readonly start: () => Effect.Effect; readonly sweep: Effect.Effect; } >()("t3/orchestration-v2/PullRequestWatchReactor") {} /** @public */ export const make = Effect.gen(function* () { const engine = yield* Orchestrator.OrchestratorV2; const projections = yield* ProjectionStore.ProjectionStoreV2; const pullRequests = yield* PullRequestService.PullRequestService; const random = yield* SecureRandom.SecureRandom; const bootedAt = yield* Clock.currentTimeMillis; const lives = new Map(); const lifeOf = (target: WatchTarget): WatchLife => { const existing = lives.get(lifeKey(target)); if (existing !== undefined) return existing; const startedAt = Date.parse(target.watch.startedAt); const life: WatchLife = { threadId: target.thread.id, pullRequest: threadPullRequestKeyOf(target.link), startedAt, headSha: target.watch.headSha, pushedAt: startedAt, longestQuietMs: 0, wakes: 0, reads: 0, }; lives.set(lifeKey(target), life); return life; }; const woke = (target: WatchTarget) => Effect.sync(() => { const life = lifeOf(target); lives.set(lifeKey(target), { ...life, wakes: life.wakes + 1 }); }); const reportEnd = Effect.fn("PullRequestWatchReactor.reportEnd")(function* ( key: string, life: WatchLife, reason: WatchEndReason, ) { lives.delete(key); const now = yield* Clock.currentTimeMillis; const quietMs = now - life.pushedAt; yield* Effect.logInfo("pull request watch ended", { threadId: life.threadId, pullRequest: life.pullRequest, reason, minutes: minutes(now - life.startedAt), quietMinutes: minutes(quietMs), longestQuietMinutes: minutes(Math.max(life.longestQuietMs, quietMs)), wakes: life.wakes, reads: life.reads, partial: life.startedAt < bootedAt, }); }); const ended = (target: WatchTarget, reason: WatchEndReason) => Effect.suspend(() => reportEnd(lifeKey(target), lifeOf(target), reason)); const readFailures = new Map(); const lastReads = new Map(); const threadTails = new Map< string, { readonly fetchedAt: number; readonly visibleThrough: number; readonly tails: ReadonlyMap< string, { readonly count: number; readonly comments: ReadonlyArray } >; } >(); const identityOf = (link: ThreadPullRequestLink) => ({ host: normalizeThreadPullRequestKey(link).host, repository: link.repository, number: link.number, }); /** * The orchestrator applies this only while the same watch is on, so a stop or restart * that lands during the host read wins. */ const record = Effect.fn("PullRequestWatchReactor.record")(function* ( target: WatchTarget, next: ThreadPullRequestWatch | null, wake?: { readonly text: string; readonly notification: OrchestrationV2Notification }, endedWatch?: ThreadPullRequestWatch, ) { const uuid = yield* random.uuidV4(); yield* engine.dispatch({ type: "thread.pull-request-watch.sync", commandId: CommandId.make(`server:pr-watch:${target.thread.id}:${uuid}`), threadId: target.thread.id, ...identityOf(target.link), startedAt: target.watch.startedAt, watch: next, ...(endedWatch === undefined ? {} : { endedWatch }), ...(wake === undefined ? {} : { wake: { ...wake, messageId: MessageId.make(`message:pr-watch:${uuid}`) } }), }); }); const giveUp = (target: WatchTarget) => record(target, null, { text: `T3 Code stopped watching pull request #${target.link.number} (${target.link.url}) because it failed to read it from the host ${READ_FAILURE_LIMIT} times in a row. Check it yourself, and call watch_pull_request to watch it again.`, notification: { source: { kind: "monitor" }, outcome: "failed", summary: `#${target.link.number}: stopped watching, could not read it`, }, }).pipe( Effect.tap(() => woke(target)), Effect.catch(() => record(target, null)), Effect.tap(() => ended(target, "unreadable")), ); const readRemarks = Effect.fn("PullRequestWatchReactor.readRemarks")( function* ( key: string, reference: PullRequestRef, activity: PullRequestActivity, now: number, rereadMs: number, ) { // Comment cursors cannot account for missing threads. Only finish a truncated read // when the host confirms that every thread was listed. if (activity.commentsTruncated && activity.reviewThreadsTruncated !== false) return null; const visibleThrough = activity.comments.reduce( (latest, comment) => Math.max(latest, Date.parse(comment.editedAt ?? comment.createdAt)), 0, ); const paginated = activity.reviewThreads.filter( (thread) => thread.nextCommentsCursor != null, ); const cached = threadTails.get(key); // All tails share one watermark, so newer visible remarks or a changed thread must // refresh the entire group before any remarks can advance it. const reuse = cached !== undefined && now - cached.fetchedAt < rereadMs && cached.visibleThrough === visibleThrough && cached.tails.size === paginated.length && paginated.every( (thread) => cached.tails.get(thread.id)?.count === (thread.commentCount ?? 0), ); const tails = new Map< string, { count: number; comments: ReadonlyArray } >(); const remarks = [...activity.comments]; for (const thread of paginated) { let cursor = thread.nextCommentsCursor ?? null; const count = thread.commentCount ?? 0; let tail = reuse ? cached.tails.get(thread.id) : undefined; if (tail === undefined) { const comments = new Map(); const cursors = new Set(); while (cursor !== null) { if (cursors.has(cursor)) return null; cursors.add(cursor); const page: PullRequestThreadCommentsResult = yield* pullRequests.threadComments({ ...reference, threadId: thread.id, cursor, }); for (const comment of page.comments) { comments.set(comment.id, { ...comment, kind: "review-comment", path: thread.path, reviewState: null, }); } cursor = page.nextCursor; } if (thread.comments.length + comments.size < count) return null; tail = { count, comments: [...comments.values()] }; } tails.set(thread.id, tail); remarks.push(...tail.comments); } threadTails.set(key, { fetchedAt: reuse ? cached.fetchedAt : now, visibleThrough, tails, }); return remarks.sort((left, right) => left.createdAt.localeCompare(right.createdAt)); }, Effect.catch((error) => Effect.logWarning("pull request watch comment pagination failed", { error }).pipe( Effect.as(null), ), ), (effect, key) => effect.pipe( Effect.onExit((exit) => Effect.sync(() => { // A retry must refresh every tail before advancing the shared remark watermark. if (Exit.isFailure(exit) || exit.value === null) threadTails.delete(key); }), ), ), ); const closed = (target: WatchTarget) => record(target, null, { text: `Pull request #${target.link.number} (${target.link.url}) was closed, so T3 Code stopped watching it. Call watch_pull_request if it reopens.`, notification: { source: { kind: "monitor" }, outcome: "updated", summary: `#${target.link.number}: closed, stopped watching`, }, }).pipe( Effect.tap(() => woke(target)), Effect.tap(() => ended(target, "closed")), ); const merged = (target: WatchTarget) => record(target, null, { text: `Pull request #${target.link.number} (${target.link.url}) was merged, so T3 Code stopped watching it. Finish any cleanup for it now.`, notification: { source: { kind: "monitor" }, outcome: "completed", summary: `#${target.link.number}: merged, stopped watching`, }, }).pipe( Effect.tap(() => woke(target)), Effect.tap(() => ended(target, "merged")), ); // The orchestrator refuses to wake a settled thread, so only an unsettled one is told. const canHearMerge = ({ thread }: WatchTarget) => thread.settledOverride !== "settled" && thread.settledAt === null; const endsWithoutRead = ({ thread, link }: WatchTarget): WatchEndReason | undefined => link.snapshot?.state === "merged" ? "merged" : thread.settledOverride === "settled" || thread.settledAt !== null ? "settled" : thread.lineage.relationshipToParent === "subagent" ? "subagent" : undefined; const eachTarget = ( group: WatchGroup, step: (target: WatchTarget) => Effect.Effect, ) => Effect.forEach(group.targets, (target) => step(target).pipe( Effect.as(true), Effect.catchCause((cause) => { lastReads.delete(group.key); return logFailure("pull request watch update failed", { threadId: target.thread.id, pullRequest: group.key, })(cause).pipe(Effect.as(false)); }), ), ).pipe(Effect.map((landed) => landed.every(Boolean))); const referenceOf = (group: WatchGroup) => { const first = group.targets[0]!; return { projectId: first.thread.projectId, ...identityOf(first.link) }; }; const fingerprintOf = (group: WatchGroup) => pullRequests.watchFingerprint(referenceOf(group)).pipe( Effect.catchCause( (cause): Effect.Effect => Cause.hasInterruptsOnly(cause) ? Effect.interrupt : providerFailureReason(cause) === "rate-limited" ? Effect.succeed("paused") : Effect.logDebug("pull request watch fingerprint failed", { pullRequest: group.key, cause, }).pipe(Effect.as(null)), ), ); const readGroup = Effect.fn("PullRequestWatchReactor.readGroup")(function* ( group: WatchGroup, fingerprint: ProviderChangeRequestWatchFingerprint | null, ) { const now = yield* Clock.currentTimeMillis; const last = lastReads.get(group.key); const plan = planRead(group, last, fingerprint, now); if (!plan.detail) return; if ( last === undefined || group.targets.some((target) => !last.watches.has(watchKey(target))) || (fingerprint !== null && last.remarks !== fingerprint.remarks) ) { // A new watch or changed host remarks needs a fresh view of every tail. threadTails.delete(group.key); } const reference = referenceOf(group); // Invalidate cached reads before using the newer fingerprint; clients receive no notification. if (fingerprint !== null) yield* pullRequests.invalidate({ reference }); const [detailRead, activityRead] = yield* Effect.all( [ Effect.exit(pullRequests.detail({ ...reference, allowStale: false })), Effect.exit( plan.activity ? pullRequests.activity(reference).pipe(Effect.asSome) : Effect.succeed(Option.none()), ), ], { concurrency: 2 }, ); // Activity failures must not cancel or hide a permanent detail failure. const read = Exit.isFailure(detailRead) ? Exit.failCause(detailRead.cause) : Exit.isFailure(activityRead) ? Exit.failCause(activityRead.cause) : Exit.succeed([detailRead.value, activityRead.value] as const); if (Exit.isFailure(read)) { if (Cause.hasInterruptsOnly(read.cause)) return yield* Effect.failCause(read.cause); lastReads.delete(group.key); const reason = providerFailureReason(read.cause); if (reason === "rate-limited") return; // Transport and host failures can outlast the failure limit; keep the watch for recovery. if (reason === "failed") { readFailures.delete(group.key); return yield* Effect.failCause(read.cause); } const failures = (readFailures.get(group.key) ?? 0) + 1; readFailures.set(group.key, failures); if (failures >= READ_FAILURE_LIMIT && (yield* eachTarget(group, giveUp))) { readFailures.delete(group.key); } return yield* Effect.failCause(read.cause); } readFailures.delete(group.key); const [detail, activity] = read.value; for (const target of group.targets) { const life = lifeOf(target); const headSha = detail.headSha ?? life.headSha; const pushed = life.headSha !== null && headSha !== life.headSha; lives.set(lifeKey(target), { ...life, headSha, reads: life.reads + 1, ...(pushed ? { pushedAt: now, longestQuietMs: Math.max(life.longestQuietMs, now - life.pushedAt) } : {}), }); } if (detail.state !== "open") { lastReads.delete(group.key); return yield* eachTarget(group, (target) => detail.state === "closed" ? closed(target) : canHearMerge(target) ? merged(target) : record(target, null).pipe(Effect.tap(() => ended(target, "merged"))), ); } // Never advance the remark watermark past comments an incomplete read could have missed. const remarks = Option.isSome(activity) ? yield* readRemarks( group.key, reference, activity.value, now, fingerprint === null ? QUIET_REREAD_MS : FINGERPRINT_REREAD_MS, ) : null; lastReads.set(group.key, { at: plan.activity || last === undefined ? (threadTails.get(group.key)?.fetchedAt ?? now) : last.at, fingerprint: group.fingerprint, inFlight: detail.mergeability === "unknown" || detail.checks.some((check) => check.status === "pending"), remarksComplete: plan.activity ? remarks !== null : (last?.remarksComplete ?? false), watches: new Set(group.targets.map(watchKey)), status: fingerprint?.status ?? null, remarks: fingerprint?.remarks ?? null, }); yield* eachTarget(group, (target) => { const report = evaluatePullRequestWatch(target.watch, detail, remarks, now); if (report.changes.length > 0) { const wake = pullRequestWatchMessage({ number: target.link.number, url: target.link.url, baseBranch: detail.baseBranch, headSha: report.next.headSha, report, }); // An ending watch keeps what this pass saw, so watching again starts from it. return ( report.exhausted ? record(target, null, wake, report.next) : record(target, report.next, wake) ).pipe( Effect.tap(() => woke(target)), Effect.tap(() => (report.exhausted ? ended(target, "comment-limit") : Effect.void)), ); } return watchesEqual(report.next, target.watch) ? Effect.void : record(target, report.next); }); }); const sweep = Effect.gen(function* () { const threads = yield* projections.getThreadsWithPullRequests(); const targets = threads.flatMap((thread) => visibleThreadPullRequests(thread.pullRequests ?? []).flatMap((link) => link.watch === undefined ? [] : [{ thread, link, watch: link.watch }], ), ); const present = new Set(targets.map(lifeKey)); yield* Effect.forEach( [...lives].filter(([key]) => !present.has(key)), ([key, life]) => reportEnd(key, life, "stopped"), { discard: true }, ); for (const target of targets) lifeOf(target); const byPullRequest = new Map>(); const ending: Array = []; for (const target of targets) { const reason = endsWithoutRead(target); if (reason !== undefined) { ending.push([target, reason]); continue; } // Group by project because checkout access differs; one failure must not end another project's watches. const key = `${target.thread.projectId} ${threadPullRequestKeyOf(target.link)}`; byPullRequest.set(key, [...(byPullRequest.get(key) ?? []), target]); } const groups = [...byPullRequest].map(([key, members]): WatchGroup => ({ key, targets: members, fingerprint: [...new Set(members.map(snapshotFingerprint))].toSorted().join("\n"), })); for (const cache of [readFailures, lastReads, threadTails]) { for (const key of cache.keys()) if (!byPullRequest.has(key)) cache.delete(key); } yield* Effect.forEach( ending, ([target, reason]) => (reason === "merged" && canHearMerge(target) ? merged(target) : record(target, null).pipe(Effect.tap(() => ended(target, reason))) ).pipe( Effect.catchCause( logFailure("pull request watch stop failed", { threadId: target.thread.id, pullRequest: threadPullRequestKeyOf(target.link), }), ), ), { discard: true }, ); // Every fingerprint is asked for at once so the host can batch them, and each group reads // as soon as its own answer lands rather than waiting on the slowest host. const reads = yield* Semaphore.make(GROUP_READS); yield* Effect.forEach( groups, (group) => fingerprintOf(group).pipe( Effect.flatMap((fingerprint) => fingerprint === "paused" ? Effect.void : reads.withPermit( readGroup(group, fingerprint).pipe( Effect.catchCause( logFailure("pull request watch check failed", { pullRequest: group.key }), ), ), ), ), ), { concurrency: "unbounded", discard: true }, ); }).pipe( Effect.catchCause(logFailure("pull request watch sweep failed", {})), Effect.withSpan("PullRequestWatchReactor.sweep"), ); const start: PullRequestWatchReactor["Service"]["start"] = () => forkParked( sweep.pipe(Effect.repeat(Schedule.spaced(`${SWEEP_MINUTES} minutes`)), Effect.asVoid), ); return { start, sweep } satisfies PullRequestWatchReactor["Service"]; }); export const layer = Layer.effect(PullRequestWatchReactor, make);