@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 996 lines 37.4 KB
Raw
1 import {
2 CommandId,
3 isOrchestrationV2WorkActive,
4 MessageId,
5 type OrchestrationV2Notification,
6 type PullRequestActivity,
7 type PullRequestComment,
8 type PullRequestDetail,
9 type PullRequestRef,
10 type PullRequestThreadCommentsResult,
11 type ThreadPullRequestLink,
12 type ThreadId,
13 type ThreadPullRequestWatch,
14 } from "@t3tools/contracts";
15 import { commandIndicatesFailure } from "@t3tools/shared/toolOutput";
16 import {
17 changeRequestsClosedBy,
18 isChangeRequestCloseCommand,
19 } from "../sourceControl/sourceControlDefinitions.ts";
20 import {
21 normalizeThreadPullRequestKey,
22 threadPullRequestKeyOf,
23 visibleThreadPullRequests,
24 } from "../sourceControl/threadPullRequests.ts";
25 import * as Cause from "effect/Cause";
26 import * as Clock from "effect/Clock";
27 import * as Context from "effect/Context";
28 import * as SecureRandom from "effect/crypto/SecureRandom";
29 import * as Effect from "effect/Effect";
30 import * as Exit from "effect/Exit";
31 import * as FiberMap from "effect/FiberMap";
32 import * as Layer from "effect/Layer";
33 import * as Option from "effect/Option";
34 import * as Schedule from "effect/Schedule";
35 import * as Schema from "effect/Schema";
36 import * as Semaphore from "effect/Semaphore";
37 import type * as Scope from "effect/Scope";
38 import * as Stream from "effect/Stream";
39
40 import {
41 type ProviderChangeRequestWatchFingerprint,
42 PullRequestProviderError,
43 } from "@t3tools/source-control-core/server/PullRequestProvider";
44 import * as PullRequestService from "../pullRequest/PullRequestService.ts";
45 import { forkParked } from "../serverActivation.ts";
46 import * as Orchestrator from "./Orchestrator.ts";
47 import * as ProjectionStore from "./ProjectionStore.ts";
48 import { isTerminalRunStatus } from "./ThreadManagementService.ts";
49 import {
50 combinePullRequestWatchWakes,
51 evaluatePullRequestWatch,
52 PULL_REQUEST_WATCH_NO_CHECKS_MS,
53 pullRequestWatchEndMessage,
54 pullRequestWatchMessage,
55 } from "./pullRequestWatch.ts";
56
57 /** Checks take minutes; faster sweeps waste the account's shared host quota. */
58 const SWEEP_MINUTES = 2;
59 const READ_FAILURE_LIMIT = 8;
60 /** Group reads in flight at once, inside the host's own request limit. */
61 const GROUP_READS = 4;
62 /** Without fingerprints, reread quiet PRs on snapshot changes or this interval to catch edits. */
63 const QUIET_REREAD_MS = 10 * 60_000;
64 /** Reread despite unchanged fingerprints; they miss review-thread comment edits. */
65 const FINGERPRINT_REREAD_MS = 30 * 60_000;
66
67 const isProviderError = Schema.is(PullRequestProviderError);
68
69 const providerFailureReason = (cause: Cause.Cause<PullRequestService.PullRequestError>) =>
70 Option.match(Cause.findErrorOption(cause), {
71 onNone: () => null,
72 onSome: (error) =>
73 error._tag === "PullRequestOperationError" && isProviderError(error.cause)
74 ? error.cause.reason
75 : null,
76 });
77
78 const logFailure =
79 (message: string, fields: Record<string, unknown>) =>
80 <E>(cause: Cause.Cause<E>): Effect.Effect<void> =>
81 Cause.hasInterruptsOnly(cause)
82 ? Effect.interrupt
83 : Effect.logWarning(message, { ...fields, cause });
84
85 interface WatchTarget {
86 readonly thread: ProjectionStore.ProjectionThreadPullRequests;
87 readonly link: ThreadPullRequestLink;
88 readonly watch: ThreadPullRequestWatch;
89 }
90
91 interface WatchGroup {
92 readonly key: string;
93 readonly targets: ReadonlyArray<WatchTarget>;
94 readonly fingerprint: string;
95 }
96
97 interface LastRead {
98 readonly at: number;
99 readonly fingerprint: string;
100 readonly inFlight: boolean;
101 readonly remarksComplete: boolean;
102 readonly watches: ReadonlySet<string>;
103 readonly status: string | null;
104 readonly remarks: string | null;
105 }
106
107 interface ReadPlan {
108 readonly detail: boolean;
109 readonly activity: boolean;
110 }
111
112 interface Wake {
113 readonly text: string;
114 readonly notification: OrchestrationV2Notification;
115 readonly includesRemarks?: boolean;
116 }
117
118 /** A wake a pass holds back so each thread gets one for everything the pass found. */
119 interface HeldWake {
120 readonly groupKey: string;
121 readonly target: WatchTarget;
122 readonly next: ThreadPullRequestWatch | null;
123 readonly endedWatch?: ThreadPullRequestWatch;
124 readonly wake: Wake;
125 readonly reason?: WatchEndReason;
126 }
127
128 const watchKey = ({ thread, watch }: WatchTarget) => `${thread.id} ${watch.startedAt}`;
129
130 type WatchEndReason =
131 | "merged"
132 | "closed"
133 | "unreadable"
134 | "comment-limit"
135 | "settled"
136 | "subagent"
137 | "stopped";
138
139 interface WatchLife {
140 readonly threadId: string;
141 readonly pullRequest: string;
142 readonly startedAt: number;
143 readonly headSha: string | null;
144 readonly pushedAt: number;
145 readonly longestQuietMs: number;
146 readonly wakes: number;
147 readonly reads: number;
148 }
149
150 const lifeKeyOf = (threadId: string, link: ThreadPullRequestLink, watch: ThreadPullRequestWatch) =>
151 `${threadId} ${threadPullRequestKeyOf(link)} ${watch.startedAt}`;
152 const lifeKey = ({ thread, link, watch }: WatchTarget) => lifeKeyOf(thread.id, link, watch);
153
154 const minutes = (ms: number) => Math.max(0, Math.round(ms / 60_000));
155
156 const snapshotFingerprint = ({ link }: WatchTarget) => {
157 const snapshot = link.snapshot;
158 return snapshot === null
159 ? ""
160 : [
161 snapshot.state,
162 snapshot.updatedAt,
163 snapshot.checksState,
164 snapshot.mergeability,
165 snapshot.rebaseRequired,
166 snapshot.reviewDecision,
167 snapshot.isDraft,
168 ].join(" ");
169 };
170
171 function planRead(
172 group: WatchGroup,
173 last: LastRead | undefined,
174 fingerprint: ProviderChangeRequestWatchFingerprint | null,
175 now: number,
176 ): ReadPlan {
177 if (last === undefined || group.targets.some((target) => !last.watches.has(watchKey(target)))) {
178 return { detail: true, activity: true };
179 }
180 // Nothing on the host changes when the no-checks deadline passes, or when a push to another
181 // branch clears a watch's answer, so either alone prompts a read.
182 const checksDue = group.targets.some(
183 ({ watch }) =>
184 watch.checksPending === undefined ||
185 (watch.checksAwaitedSince != null &&
186 now - Date.parse(watch.checksAwaitedSince) >= PULL_REQUEST_WATCH_NO_CHECKS_MS),
187 );
188 if (fingerprint === null) {
189 const read =
190 last.inFlight ||
191 !last.remarksComplete ||
192 last.fingerprint !== group.fingerprint ||
193 now - last.at >= QUIET_REREAD_MS;
194 return { detail: read || checksDue, activity: read };
195 }
196 const activity =
197 last.remarks !== fingerprint.remarks ||
198 !last.remarksComplete ||
199 now - last.at >= FINGERPRINT_REREAD_MS;
200 return {
201 detail:
202 activity ||
203 checksDue ||
204 last.status !== fingerprint.status ||
205 (last.inFlight && fingerprint.coversDetail !== true),
206 activity,
207 };
208 }
209
210 // Lets the orchestrator tell that a push cleared the answer while this read was in flight.
211 const checksPendingBeforeOf = ({ watch }: WatchTarget) =>
212 watch.checksPending === undefined ? {} : { checksPendingBefore: watch.checksPending };
213
214 function watchesEqual(left: ThreadPullRequestWatch, right: ThreadPullRequestWatch): boolean {
215 return (
216 left.startedAt === right.startedAt &&
217 left.headSha === right.headSha &&
218 left.failedChecks.join("\n") === right.failedChecks.join("\n") &&
219 left.passed === right.passed &&
220 left.passedChecks.join("\n") === right.passedChecks.join("\n") &&
221 (left.owedChecks ?? []).join("\n") === (right.owedChecks ?? []).join("\n") &&
222 (left.awaitingApproval ?? []).join("\n") === (right.awaitingApproval ?? []).join("\n") &&
223 (left.approvalAnnounced ?? false) === (right.approvalAnnounced ?? false) &&
224 left.remarksThrough === right.remarksThrough &&
225 left.remarkIds.join("\n") === right.remarkIds.join("\n") &&
226 left.conflicting === right.conflicting &&
227 left.rebaseRequired === right.rebaseRequired &&
228 left.mergeQueueRemovalId === right.mergeQueueRemovalId &&
229 left.wakes === right.wakes &&
230 left.awaitingBaseline === right.awaitingBaseline &&
231 left.checksAwaitedSince === right.checksAwaitedSince &&
232 left.checksPending === right.checksPending &&
233 left.until === right.until
234 );
235 }
236
237 export class PullRequestWatchReactor extends Context.Service<
238 PullRequestWatchReactor,
239 {
240 readonly start: () => Effect.Effect<void, never, Scope.Scope>;
241 readonly sweep: Effect.Effect<void>;
242 }
243 >()("t3/orchestration-v2/PullRequestWatchReactor") {}
244
245 /** @public */
246 export const make = Effect.gen(function* () {
247 const engine = yield* Orchestrator.OrchestratorV2;
248 const projections = yield* ProjectionStore.ProjectionStoreV2;
249 const pullRequests = yield* PullRequestService.PullRequestService;
250 const random = yield* SecureRandom.SecureRandom;
251 const bootedAt = yield* Clock.currentTimeMillis;
252 const lives = new Map<string, WatchLife>();
253
254 const lifeOf = (target: WatchTarget): WatchLife => {
255 const existing = lives.get(lifeKey(target));
256 if (existing !== undefined) return existing;
257 const startedAt = Date.parse(target.watch.startedAt);
258 const life: WatchLife = {
259 threadId: target.thread.id,
260 pullRequest: threadPullRequestKeyOf(target.link),
261 startedAt,
262 headSha: target.watch.headSha,
263 pushedAt: startedAt,
264 longestQuietMs: 0,
265 wakes: 0,
266 reads: 0,
267 };
268 lives.set(lifeKey(target), life);
269 return life;
270 };
271 const woke = (target: WatchTarget) =>
272 Effect.sync(() => {
273 const life = lifeOf(target);
274 lives.set(lifeKey(target), { ...life, wakes: life.wakes + 1 });
275 });
276
277 const reportEnd = Effect.fn("PullRequestWatchReactor.reportEnd")(function* (
278 key: string,
279 life: WatchLife,
280 reason: WatchEndReason,
281 ) {
282 lives.delete(key);
283 const now = yield* Clock.currentTimeMillis;
284 const quietMs = now - life.pushedAt;
285 yield* Effect.logInfo("pull request watch ended", {
286 threadId: life.threadId,
287 pullRequest: life.pullRequest,
288 reason,
289 minutes: minutes(now - life.startedAt),
290 quietMinutes: minutes(quietMs),
291 longestQuietMinutes: minutes(Math.max(life.longestQuietMs, quietMs)),
292 wakes: life.wakes,
293 reads: life.reads,
294 partial: life.startedAt < bootedAt,
295 });
296 });
297 const ended = (target: WatchTarget, reason: WatchEndReason) =>
298 Effect.suspend(() => reportEnd(lifeKey(target), lifeOf(target), reason));
299
300 const readFailures = new Map<string, number>();
301 /** The last sweep's groups, whose pull requests get host change feeds. */
302 let currentGroups: ReadonlyArray<WatchGroup> = [];
303 // A pass reads watches and records new ones, so passes must not overlap.
304 const passes = Semaphore.makeUnsafe(1);
305 const lastReads = new Map<string, LastRead>();
306 /**
307 * Each thread's merge and close commands, by item, with the pull requests each names. One counts
308 * while it runs and, if it succeeded, until the read at its end is done.
309 */
310 const agentEnds = new Map<ThreadId, Map<string, ReadonlySet<number> | "any">>();
311 // The agent needs no wake for a merge or close it made itself. Anything ending later, as a
312 // `--auto` merge does, is news.
313 const endedByAgent = (target: WatchTarget) =>
314 [...(agentEnds.get(target.thread.id)?.values() ?? [])].some(
315 (named) => named === "any" || named.has(target.link.number),
316 );
317 const threadTails = new Map<
318 string,
319 {
320 readonly fetchedAt: number;
321 readonly visibleThrough: number;
322 readonly tails: ReadonlyMap<
323 string,
324 { readonly count: number; readonly comments: ReadonlyArray<PullRequestComment> }
325 >;
326 }
327 >();
328
329 const identityOf = (link: ThreadPullRequestLink) => ({
330 host: normalizeThreadPullRequestKey(link).host,
331 repository: link.repository,
332 number: link.number,
333 });
334
335 /**
336 * The orchestrator applies this only while the same watch is on, so a stop or restart
337 * that lands during the host read wins.
338 */
339 const record = Effect.fn("PullRequestWatchReactor.record")(function* (
340 target: WatchTarget,
341 next: ThreadPullRequestWatch | null,
342 wake?: Wake,
343 endedWatch?: ThreadPullRequestWatch,
344 ) {
345 const uuid = yield* random.uuidV4();
346 yield* engine.dispatch({
347 type: "thread.pull-request-watch.sync",
348 commandId: CommandId.make(`server:pr-watch:${target.thread.id}:${uuid}`),
349 threadId: target.thread.id,
350 ...identityOf(target.link),
351 startedAt: target.watch.startedAt,
352 watch: next,
353 ...checksPendingBeforeOf(target),
354 ...(endedWatch === undefined ? {} : { endedWatch }),
355 ...(wake === undefined
356 ? {}
357 : { wake: { ...wake, messageId: MessageId.make(`message:pr-watch:${uuid}`) } }),
358 });
359 });
360
361 const giveUp = (target: WatchTarget) =>
362 record(target, null, {
363 text: `T3 Code stopped watching pull request #${target.link.number} (${target.link.url}) because it failed to read it from the host ${READ_FAILURE_LIMIT} times in a row. Check it yourself, and call watch_pull_request only if it is still open.`,
364 notification: {
365 source: { kind: "monitor" },
366 outcome: "failed",
367 summary: `#${target.link.number}: stopped watching, could not read it`,
368 },
369 }).pipe(
370 Effect.tap(() => woke(target)),
371 Effect.catch(() => record(target, null)),
372 Effect.tap(() => ended(target, "unreadable")),
373 );
374
375 const readRemarks = Effect.fn("PullRequestWatchReactor.readRemarks")(
376 function* (
377 key: string,
378 reference: PullRequestRef,
379 activity: PullRequestActivity,
380 now: number,
381 rereadMs: number,
382 ) {
383 // Comment cursors cannot account for missing threads. Only finish a truncated read
384 // when the host confirms that every thread was listed.
385 if (activity.commentsTruncated && activity.reviewThreadsTruncated !== false) return null;
386
387 const visibleThrough = activity.comments.reduce(
388 (latest, comment) => Math.max(latest, Date.parse(comment.editedAt ?? comment.createdAt)),
389 0,
390 );
391 const paginated = activity.reviewThreads.filter(
392 (thread) => thread.nextCommentsCursor != null,
393 );
394 const cached = threadTails.get(key);
395 // All tails share one watermark, so newer visible remarks or a changed thread must
396 // refresh the entire group before any remarks can advance it.
397 const reuse =
398 cached !== undefined &&
399 now - cached.fetchedAt < rereadMs &&
400 cached.visibleThrough === visibleThrough &&
401 cached.tails.size === paginated.length &&
402 paginated.every(
403 (thread) => cached.tails.get(thread.id)?.count === (thread.commentCount ?? 0),
404 );
405 const tails = new Map<
406 string,
407 { count: number; comments: ReadonlyArray<PullRequestComment> }
408 >();
409 const remarks = [...activity.comments];
410 for (const thread of paginated) {
411 let cursor = thread.nextCommentsCursor ?? null;
412 const count = thread.commentCount ?? 0;
413 let tail = reuse ? cached.tails.get(thread.id) : undefined;
414 if (tail === undefined) {
415 const comments = new Map<string, PullRequestComment>();
416 const cursors = new Set<string>();
417 let hidden = 0;
418 while (cursor !== null) {
419 if (cursors.has(cursor)) return null;
420 cursors.add(cursor);
421 const page: PullRequestThreadCommentsResult = yield* pullRequests.threadComments({
422 ...reference,
423 threadId: thread.id,
424 cursor,
425 });
426 for (const comment of page.comments) {
427 comments.set(comment.id, {
428 ...comment,
429 kind: "review-comment",
430 path: thread.path,
431 reviewState: null,
432 });
433 }
434 hidden += page.hiddenCount ?? 0;
435 cursor = page.nextCursor;
436 }
437 if (thread.comments.length + comments.size + hidden < count) return null;
438 tail = { count, comments: [...comments.values()] };
439 }
440 tails.set(thread.id, tail);
441 remarks.push(...tail.comments);
442 }
443 threadTails.set(key, {
444 fetchedAt: reuse ? cached.fetchedAt : now,
445 visibleThrough,
446 tails,
447 });
448 return remarks.sort((left, right) => left.createdAt.localeCompare(right.createdAt));
449 },
450 Effect.catch((error) =>
451 Effect.logWarning("pull request watch comment pagination failed", { error }).pipe(
452 Effect.as(null),
453 ),
454 ),
455 (effect, key) =>
456 effect.pipe(
457 Effect.onExit((exit) =>
458 Effect.sync(() => {
459 // A retry must refresh every tail before advancing the shared remark watermark.
460 if (Exit.isFailure(exit) || exit.value === null) threadTails.delete(key);
461 }),
462 ),
463 ),
464 );
465
466 /** Wakes the current pass found, sent by `sendHeld` once every read of the pass is done. */
467 let held: Array<HeldWake> = [];
468 const hold = (wake: HeldWake) => Effect.sync(() => void held.push(wake));
469
470 const sendOne = (item: HeldWake) =>
471 record(item.target, item.next, item.wake, item.endedWatch).pipe(
472 Effect.tap(() => woke(item.target)),
473 Effect.tap(() => (item.reason === undefined ? Effect.void : ended(item.target, item.reason))),
474 Effect.catchCause((cause) => {
475 lastReads.delete(item.groupKey);
476 return logFailure("pull request watch update failed", {
477 threadId: item.target.thread.id,
478 pullRequest: item.groupKey,
479 })(cause);
480 }),
481 );
482
483 const sendTogether = Effect.fn("PullRequestWatchReactor.sendTogether")(function* (
484 items: readonly [HeldWake, ...ReadonlyArray<HeldWake>],
485 ) {
486 const [first, ...rest] = items;
487 const uuid = yield* random.uuidV4();
488 yield* engine.dispatch({
489 type: "thread.pull-request-watch.sync",
490 commandId: CommandId.make(`server:pr-watch:${first.target.thread.id}:${uuid}`),
491 threadId: first.target.thread.id,
492 ...identityOf(first.target.link),
493 startedAt: first.target.watch.startedAt,
494 watch: first.next,
495 ...checksPendingBeforeOf(first.target),
496 ...(first.endedWatch === undefined ? {} : { endedWatch: first.endedWatch }),
497 others: rest.map((item) => ({
498 ...identityOf(item.target.link),
499 startedAt: item.target.watch.startedAt,
500 watch: item.next,
501 ...checksPendingBeforeOf(item.target),
502 ...(item.endedWatch === undefined ? {} : { endedWatch: item.endedWatch }),
503 ...(item.wake.includesRemarks === true ? { includesRemarks: true } : {}),
504 })),
505 wake: {
506 ...combinePullRequestWatchWakes([first.wake, ...rest.map((item) => item.wake)]),
507 ...(first.wake.includesRemarks === true ? { includesRemarks: true } : {}),
508 sections: items.map(({ wake }) => ({
509 text: wake.text,
510 outcome: wake.notification.outcome,
511 summary: wake.notification.summary,
512 })),
513 messageId: MessageId.make(`message:pr-watch:${uuid}`),
514 },
515 });
516 yield* Effect.forEach(
517 items,
518 (item) =>
519 woke(item.target).pipe(
520 Effect.andThen(item.reason === undefined ? Effect.void : ended(item.target, item.reason)),
521 ),
522 { discard: true },
523 );
524 });
525
526 const sendHeld = Effect.suspend(() => {
527 const byThread = new Map<string, Array<HeldWake>>();
528 for (const item of held) {
529 byThread.set(item.target.thread.id, [...(byThread.get(item.target.thread.id) ?? []), item]);
530 }
531 held = [];
532 return Effect.forEach(
533 byThread.values(),
534 ([first, ...rest]) =>
535 rest.length === 0
536 ? sendOne(first!)
537 : // A watch that ended or moved during the pass fails the whole command; each then
538 // goes on its own, as the orchestrator decides for it alone.
539 sendTogether([first!, ...rest]).pipe(
540 Effect.catchCauseIf(
541 (cause) => !Cause.hasInterruptsOnly(cause),
542 () => Effect.forEach([first!, ...rest], sendOne, { discard: true }),
543 ),
544 ),
545 { discard: true },
546 );
547 });
548
549 const finished = (
550 groupKey: string,
551 target: WatchTarget,
552 detail: Pick<PullRequestDetail, "state" | "baseBranch" | "mergeCommitSha" | "mergedBy">,
553 ) => {
554 const reason = detail.state === "merged" ? "merged" : "closed";
555 return endedByAgent(target)
556 ? record(target, null).pipe(Effect.tap(() => ended(target, reason)))
557 : hold({
558 groupKey,
559 target,
560 next: null,
561 wake: pullRequestWatchEndMessage({
562 number: target.link.number,
563 url: target.link.url,
564 detail,
565 }),
566 reason,
567 });
568 };
569
570 // A merged snapshot still gets one read, so the wake can name the merge commit.
571 const endsWithoutRead = ({ thread }: WatchTarget): WatchEndReason | undefined =>
572 thread.settledOverride === "settled" || thread.settledAt !== null
573 ? "settled"
574 : thread.lineage.relationshipToParent === "subagent"
575 ? "subagent"
576 : undefined;
577
578 const eachTarget = <E>(
579 group: WatchGroup,
580 step: (target: WatchTarget) => Effect.Effect<void, E>,
581 ) =>
582 Effect.forEach(group.targets, (target) =>
583 step(target).pipe(
584 Effect.as(true),
585 Effect.catchCause((cause) => {
586 lastReads.delete(group.key);
587 return logFailure("pull request watch update failed", {
588 threadId: target.thread.id,
589 pullRequest: group.key,
590 })(cause).pipe(Effect.as(false));
591 }),
592 ),
593 ).pipe(Effect.map((landed) => landed.every(Boolean)));
594
595 // A merged PR's watch holds its thread from settling, so it must not wait out the host.
596 const endMergedSnapshots = (group: WatchGroup) => {
597 const merged = group.targets.filter((target) => target.link.snapshot?.state === "merged");
598 return eachTarget({ ...group, targets: merged }, (target) =>
599 target.link.snapshot ? finished(group.key, target, target.link.snapshot) : Effect.void,
600 ).pipe(
601 Effect.as({ ...group, targets: group.targets.filter((target) => !merged.includes(target)) }),
602 );
603 };
604
605 const referenceOf = (group: WatchGroup) => {
606 const first = group.targets[0]!;
607 return { projectId: first.thread.projectId, ...identityOf(first.link) };
608 };
609
610 const fingerprintOf = (group: WatchGroup) =>
611 pullRequests.watchFingerprint(referenceOf(group)).pipe(
612 Effect.catchCause(
613 (cause): Effect.Effect<ProviderChangeRequestWatchFingerprint | null | "paused"> =>
614 Cause.hasInterruptsOnly(cause)
615 ? Effect.interrupt
616 : providerFailureReason(cause) === "rate-limited"
617 ? Effect.succeed("paused")
618 : Effect.logDebug("pull request watch fingerprint failed", {
619 pullRequest: group.key,
620 cause,
621 }).pipe(Effect.as(null)),
622 ),
623 );
624
625 const readGroup = Effect.fn("PullRequestWatchReactor.readGroup")(function* (
626 group: WatchGroup,
627 fingerprint: ProviderChangeRequestWatchFingerprint | null,
628 ) {
629 const now = yield* Clock.currentTimeMillis;
630 const last = lastReads.get(group.key);
631 const plan = planRead(group, last, fingerprint, now);
632 if (!plan.detail) return;
633 if (
634 last === undefined ||
635 group.targets.some((target) => !last.watches.has(watchKey(target))) ||
636 (fingerprint !== null && last.remarks !== fingerprint.remarks)
637 ) {
638 // A new watch or changed host remarks needs a fresh view of every tail.
639 threadTails.delete(group.key);
640 }
641 const reference = referenceOf(group);
642 // Invalidate cached reads before using the newer fingerprint; clients receive no notification.
643 if (fingerprint !== null) yield* pullRequests.invalidate({ reference });
644 const [detailRead, activityRead] = yield* Effect.all(
645 [
646 Effect.exit(pullRequests.detail({ ...reference, allowStale: false })),
647 Effect.exit(
648 plan.activity
649 ? pullRequests.activity(reference).pipe(Effect.asSome)
650 : Effect.succeed(Option.none<PullRequestActivity>()),
651 ),
652 ],
653 { concurrency: 2 },
654 );
655 if (Exit.isFailure(detailRead)) {
656 if (Cause.hasInterruptsOnly(detailRead.cause))
657 return yield* Effect.failCause(detailRead.cause);
658 lastReads.delete(group.key);
659 const unended = yield* endMergedSnapshots(group);
660 if (unended.targets.length === 0) {
661 readFailures.delete(group.key);
662 return;
663 }
664 const reason = providerFailureReason(detailRead.cause);
665 if (reason === "rate-limited") return;
666 // Transport and host failures can outlast the failure limit; keep the watch for recovery.
667 if (reason === "failed") {
668 readFailures.delete(group.key);
669 return yield* Effect.failCause(detailRead.cause);
670 }
671 const failures = (readFailures.get(group.key) ?? 0) + 1;
672 readFailures.set(group.key, failures);
673 if (failures >= READ_FAILURE_LIMIT && (yield* eachTarget(unended, giveUp))) {
674 readFailures.delete(group.key);
675 }
676 return yield* Effect.failCause(detailRead.cause);
677 }
678 readFailures.delete(group.key);
679 const detail = detailRead.value;
680 for (const target of group.targets) {
681 const life = lifeOf(target);
682 const headSha = detail.headSha ?? life.headSha;
683 const pushed = life.headSha !== null && headSha !== life.headSha;
684 lives.set(lifeKey(target), {
685 ...life,
686 headSha,
687 reads: life.reads + 1,
688 ...(pushed
689 ? { pushedAt: now, longestQuietMs: Math.max(life.longestQuietMs, now - life.pushedAt) }
690 : {}),
691 });
692 }
693 if (detail.state !== "open") {
694 lastReads.delete(group.key);
695 return yield* eachTarget(group, (target) => finished(group.key, target, detail));
696 }
697
698 // A failed comments read leaves remarks incomplete, so the next pass reads them again;
699 // checks, head and conflicts from the detail still reach the evaluator.
700 if (Exit.isFailure(activityRead)) {
701 threadTails.delete(group.key);
702 if (providerFailureReason(activityRead.cause) !== "rate-limited") {
703 yield* logFailure("pull request watch activity read failed", { pullRequest: group.key })(
704 activityRead.cause,
705 );
706 }
707 }
708 const activity = Exit.isSuccess(activityRead)
709 ? activityRead.value
710 : Option.none<PullRequestActivity>();
711
712 // Never advance the remark watermark past comments an incomplete read could have missed.
713 const remarks = Option.isSome(activity)
714 ? yield* readRemarks(
715 group.key,
716 reference,
717 activity.value,
718 now,
719 fingerprint === null ? QUIET_REREAD_MS : FINGERPRINT_REREAD_MS,
720 )
721 : null;
722 lastReads.set(group.key, {
723 at:
724 plan.activity || last === undefined
725 ? (threadTails.get(group.key)?.fetchedAt ?? now)
726 : last.at,
727 fingerprint: group.fingerprint,
728 inFlight:
729 detail.mergeability === "unknown" ||
730 detail.checks.some((check) => check.status === "pending"),
731 remarksComplete: plan.activity ? remarks !== null : (last?.remarksComplete ?? false),
732 watches: new Set(group.targets.map(watchKey)),
733 status: fingerprint?.status ?? null,
734 remarks: fingerprint?.remarks ?? null,
735 });
736 yield* eachTarget(group, (target) => {
737 const report = evaluatePullRequestWatch(target.watch, detail, remarks, now);
738 if (report.changes.length > 0) {
739 const wake = {
740 ...pullRequestWatchMessage({
741 number: target.link.number,
742 url: target.link.url,
743 baseBranch: detail.baseBranch,
744 headSha: report.next.headSha,
745 report,
746 }),
747 includesRemarks: report.changes.some((change) => change.kind === "remarks"),
748 };
749 // An ending watch keeps what this pass saw, so watching again starts from it.
750 return hold(
751 report.exhausted
752 ? {
753 groupKey: group.key,
754 target,
755 next: null,
756 endedWatch: report.next,
757 wake,
758 reason: "comment-limit",
759 }
760 : { groupKey: group.key, target, next: report.next, wake },
761 );
762 }
763 return watchesEqual(report.next, target.watch) ? Effect.void : record(target, report.next);
764 });
765 });
766
767 /** Every watch, or with `changed`, only that pull request's after its host reported a change. */
768 // The sweep and change reads each name their own span.
769 const pass = Effect.fnUntraced(function* (changed: string | undefined) {
770 held = [];
771 const threads = yield* projections.getThreadsWithPullRequests();
772 const targets = threads.flatMap((thread) =>
773 visibleThreadPullRequests(thread.pullRequests ?? []).flatMap((link) =>
774 link.watch === undefined ||
775 (changed !== undefined && threadPullRequestKeyOf(link) !== changed)
776 ? []
777 : [{ thread, link, watch: link.watch }],
778 ),
779 );
780 if (changed === undefined) {
781 const present = new Set(targets.map(lifeKey));
782 yield* Effect.forEach(
783 [...lives].filter(([key]) => !present.has(key)),
784 ([key, life]) => reportEnd(key, life, "stopped"),
785 { discard: true },
786 );
787 }
788 for (const target of targets) lifeOf(target);
789 const byPullRequest = new Map<string, Array<WatchTarget>>();
790 const ending: Array<readonly [WatchTarget, WatchEndReason]> = [];
791 for (const target of targets) {
792 const reason = endsWithoutRead(target);
793 if (reason !== undefined) {
794 ending.push([target, reason]);
795 continue;
796 }
797 // Group by project because checkout access differs; one failure must not end another project's watches.
798 const key = `${target.thread.projectId} ${threadPullRequestKeyOf(target.link)}`;
799 byPullRequest.set(key, [...(byPullRequest.get(key) ?? []), target]);
800 }
801 const groups = [...byPullRequest].map(([key, members]): WatchGroup => ({
802 key,
803 targets: members,
804 fingerprint: [...new Set(members.map(snapshotFingerprint))].toSorted().join("\n"),
805 }));
806 if (changed === undefined) {
807 currentGroups = groups;
808 for (const cache of [readFailures, lastReads, threadTails]) {
809 for (const key of cache.keys()) if (!byPullRequest.has(key)) cache.delete(key);
810 }
811 }
812 yield* Effect.forEach(
813 ending,
814 ([target, reason]) =>
815 record(target, null).pipe(
816 Effect.tap(() => ended(target, reason)),
817 Effect.catchCause(
818 logFailure("pull request watch stop failed", {
819 threadId: target.thread.id,
820 pullRequest: threadPullRequestKeyOf(target.link),
821 }),
822 ),
823 ),
824 { discard: true },
825 );
826 // Every fingerprint is asked for at once so the host can batch them, and each group reads
827 // as soon as its own answer lands rather than waiting on the slowest host.
828 const reads = yield* Semaphore.make(GROUP_READS);
829 yield* Effect.forEach(
830 groups,
831 (group) =>
832 // The host just reported a change, so a cached fingerprint may predate it.
833 (changed === undefined
834 ? Effect.void
835 : pullRequests.invalidate({ reference: referenceOf(group) })
836 ).pipe(
837 Effect.andThen(fingerprintOf(group)),
838 Effect.tap((fingerprint) => {
839 const last = lastReads.get(group.key);
840 // Open pull request views refresh only when the host state moved.
841 return changed !== undefined &&
842 fingerprint !== null &&
843 fingerprint !== "paused" &&
844 (last === undefined ||
845 last.status !== fingerprint.status ||
846 last.remarks !== fingerprint.remarks)
847 ? pullRequests.invalidate({ reference: referenceOf(group) }, { notifyReaders: true })
848 : Effect.void;
849 }),
850 Effect.flatMap((fingerprint) =>
851 fingerprint === "paused"
852 ? endMergedSnapshots(group).pipe(Effect.asVoid)
853 : reads.withPermit(
854 readGroup(group, fingerprint).pipe(
855 Effect.catchCause(
856 logFailure("pull request watch check failed", { pullRequest: group.key }),
857 ),
858 ),
859 ),
860 ),
861 ),
862 { concurrency: "unbounded", discard: true },
863 );
864 yield* sendHeld;
865 }, passes.withPermit);
866
867 const sweep = pass(undefined).pipe(
868 Effect.catchCause(logFailure("pull request watch sweep failed", {})),
869 Effect.withSpan("PullRequestWatchReactor.sweep"),
870 );
871
872 const pullRequestOf = (group: WatchGroup) => threadPullRequestKeyOf(group.targets[0]!.link);
873
874 const readOnChange = (pullRequest: string) =>
875 pass(pullRequest).pipe(
876 Effect.catchCause(logFailure("pull request watch change read failed", { pullRequest })),
877 Effect.withSpan("PullRequestWatchReactor.readOnChange"),
878 );
879
880 // A full read, since an unchanged fingerprint or snapshot would otherwise skip the detail.
881 const readThreadWatches = (threadId: ThreadId) =>
882 projections.getThreadsWithPullRequests(threadId).pipe(
883 Effect.flatMap((threads) =>
884 Effect.forEach(
885 threads.flatMap((thread) =>
886 visibleThreadPullRequests(thread.pullRequests ?? [])
887 .filter((link) => link.watch !== undefined)
888 .map((link) => [thread.projectId, threadPullRequestKeyOf(link)] as const),
889 ),
890 ([projectId, pullRequest]) =>
891 // After any pass in flight, which could record a read from before the merge.
892 passes
893 .withPermit(Effect.sync(() => lastReads.delete(`${projectId} ${pullRequest}`)))
894 .pipe(Effect.andThen(readOnChange(pullRequest))),
895 { discard: true },
896 ),
897 ),
898 Effect.catchCause(logFailure("pull request watch read after a merge failed", { threadId })),
899 );
900
901 const start: PullRequestWatchReactor["Service"]["start"] = Effect.fn(
902 "PullRequestWatchReactor.start",
903 )(function* () {
904 // Reading a thread's watches as its agent's merge or close ends lets them end at once.
905 const agentReads = yield* FiberMap.make<string>();
906 const finished = new Set<string>();
907 yield* forkParked(
908 Stream.runForEach(engine.streamDomainEvents, (event) => {
909 if (event.type === "run.updated") {
910 // A run can stop before its command reports that it ended.
911 const commands = agentEnds.get(event.threadId);
912 if (commands === undefined || !isTerminalRunStatus(event.payload.status)) {
913 return Effect.void;
914 }
915 for (const id of commands.keys()) if (!finished.has(id)) commands.delete(id);
916 if (commands.size === 0) agentEnds.delete(event.threadId);
917 return Effect.void;
918 }
919 if (event.type !== "turn-item.updated") return Effect.void;
920 const item = event.payload;
921 if (
922 item.type !== "command_execution" ||
923 finished.has(item.id) ||
924 !isChangeRequestCloseCommand(item.input)
925 ) {
926 return Effect.void;
927 }
928 const commands = agentEnds.get(event.threadId) ?? new Map();
929 agentEnds.set(event.threadId, commands);
930 commands.set(item.id, changeRequestsClosedBy(item.input));
931 if (isOrchestrationV2WorkActive(item.status)) return Effect.void;
932 finished.add(item.id);
933 const done = Effect.sync(() => {
934 commands.delete(item.id);
935 if (commands.size === 0 && agentEnds.get(event.threadId) === commands) {
936 agentEnds.delete(event.threadId);
937 }
938 });
939 // Some providers report a command that failed as completed.
940 return Effect.andThen(
941 item.status === "completed" && !commandIndicatesFailure(item) ? Effect.void : done,
942 FiberMap.run(
943 agentReads,
944 item.id,
945 )(readThreadWatches(event.threadId).pipe(Effect.ensuring(done))),
946 );
947 }).pipe(Effect.catchCause(logFailure("pull request watch command stream failed", {}))),
948 );
949 // One feed per pull request: the host keeps one subscription per pull request and account.
950 const feeds = yield* FiberMap.make<string>();
951 const follow = Effect.gen(function* () {
952 const wanted = new Map(currentGroups.map((group) => [pullRequestOf(group), group]));
953 const stale = Array.from(feeds, ([pullRequest]) => pullRequest).filter(
954 (pullRequest) => !wanted.has(pullRequest),
955 );
956 yield* Effect.forEach(stale, (pullRequest) => FiberMap.remove(feeds, pullRequest), {
957 discard: true,
958 });
959 for (const [pullRequest, group] of wanted) {
960 // An ended feed restarts here, so a host that refused it is asked again each sweep.
961 yield* FiberMap.run(feeds, pullRequest, { onlyIfMissing: true })(
962 pullRequests.changes(referenceOf(group)).pipe(
963 // A review sends an event per comment; those landing during a read make one more.
964 Stream.buffer({ capacity: 1, strategy: "sliding" }),
965 Stream.runForEach(() => readOnChange(pullRequest)),
966 ),
967 );
968 }
969 });
970 // A watch's first read is its baseline, so it reads now: CI that finishes before the next
971 // sweep would otherwise be taken as old news and never wake the thread.
972 yield* forkParked(
973 Stream.runForEach(engine.streamDomainEvents, (event) => {
974 if (event.type !== "thread.pull-request-synced") return Effect.void;
975 const unread = visibleThreadPullRequests(event.payload.pullRequests ?? []).flatMap(
976 (link) =>
977 link.watch === undefined || lives.has(lifeKeyOf(event.threadId, link, link.watch))
978 ? []
979 : [threadPullRequestKeyOf(link)],
980 );
981 return Effect.forEach(new Set(unread), readOnChange, { discard: true });
982 }).pipe(Effect.catchCause(logFailure("pull request watch event stream failed", {}))),
983 );
984 yield* forkParked(
985 sweep.pipe(
986 Effect.andThen(follow),
987 Effect.repeat(Schedule.spaced(`${SWEEP_MINUTES} minutes`)),
988 Effect.asVoid,
989 ),
990 );
991 });
992
993 return { start, sweep } satisfies PullRequestWatchReactor["Service"];
994 });
995
996 export const layer = Layer.effect(PullRequestWatchReactor, make);