The best way to control Claude Code, Codex and more from any computer or phone

main
ts 907 lines 39.1 KB
Raw
1 import {
2 CommandId,
3 isOrchestrationV2WorkActive,
4 type OrchestrationV2TurnItem,
5 type PullRequestSummary,
6 type ThreadId,
7 type ThreadPullRequestKey,
8 type ThreadPullRequestLink,
9 type ThreadPullRequestSnapshot,
10 type ThreadPullRequestStack,
11 } from "@t3tools/contracts";
12 import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
13 import * as KeyedLock from "@t3tools/shared/KeyedLock";
14 import {
15 threadPullRequestKeyOf,
16 normalizeThreadPullRequestKey,
17 threadPullRequestKeysEqual,
18 visibleThreadPullRequests,
19 } from "../sourceControl/threadPullRequests.ts";
20 import {
21 isChangeRequestCloseCommand,
22 sourceControlDefinitions,
23 } from "../sourceControl/sourceControlDefinitions.ts";
24 import * as Cause from "effect/Cause";
25 import * as Clock from "effect/Clock";
26 import * as Context from "effect/Context";
27 import * as SecureRandom from "effect/crypto/SecureRandom";
28 import * as DateTime from "effect/DateTime";
29 import * as Effect from "effect/Effect";
30 import * as Layer from "effect/Layer";
31 import * as Schema from "effect/Schema";
32 import * as Schedule from "effect/Schedule";
33 import type * as Scope from "effect/Scope";
34 import * as Semaphore from "effect/Semaphore";
35 import * as Stream from "effect/Stream";
36
37 import * as GitManager from "../git/GitManager.ts";
38 import { PullRequestProviderError } from "@t3tools/source-control-core/server/PullRequestProvider";
39 import * as PullRequestService from "../pullRequest/PullRequestService.ts";
40 import { forkParked } from "../serverActivation.ts";
41 import * as VcsProcess from "../vcs/VcsProcess.ts";
42 import * as Orchestrator from "./Orchestrator.ts";
43 import * as ProjectionStore from "./ProjectionStore.ts";
44 import { isTerminalRunStatus } from "./ThreadManagementService.ts";
45
46 const SLOW_SYNC_INTERVAL_MS = 15 * 60 * 1_000;
47 // Every server polls every open linked pull request, and most of them sit untouched for days.
48 // One nobody has updated for QUIET_AFTER_MS is read on the quiet cadence instead of each sweep;
49 // requested refreshes, watches and a host CLI's merge command still read it at once.
50 export const QUIET_AFTER_MS = 30 * 60 * 1_000;
51 export const QUIET_SYNC_INTERVAL_MS = 5 * 60 * 1_000;
52 // A pull request closed this long ago is rarely reopened, so the sweep stops reading it, as it
53 // does merged ones. A reader that sees it open, or a requested refresh, still reads it at once.
54 const CLOSED_FINAL_AFTER_MS = 24 * 60 * 60 * 1_000;
55 // A link whose host read failed waits a sweep, then twice as long after each further failure.
56 const FAILED_READ_BACKOFF_MS = 60 * 1_000;
57
58 // Where a shell starts a command, past `NAME=value` assignments and wrappers like `timeout 60`.
59 // A create command anywhere else, as in `echo "gh pr create"`, is text, not a command that runs.
60 const COMMAND_START =
61 "(?:^|[;&|({`\\n]|\\b(?:do|then|else)\\s)\\s*(?:(?:\\w+=\\S*|env|time|nohup|timeout\\s+\\S+)\\s+)*";
62 const CREATE_COMMANDS = sourceControlDefinitions.definitions.flatMap(
63 ({ kind, changeRequestCreateCommand: command }) =>
64 command === undefined
65 ? []
66 : // The subcommand must end its shell word: `gh pr create#x` runs another one.
67 [
68 {
69 kind,
70 command,
71 run: new RegExp(`${COMMAND_START}${command.source}(?=$|[\\s;&|)])`, "gu"),
72 },
73 ],
74 );
75 const createCommandCount = (code: string) =>
76 CREATE_COMMANDS.reduce((count, { run }) => count + [...code.matchAll(run)].length, 0);
77 // A push, directly or through a stack tool, that can start checks on a linked pull request.
78 const PUSH_COMMAND = new RegExp(
79 `${COMMAND_START}(?:git(?:\\s+-[cC]\\s+\\S+)*\\s+push|gh\\s+stack\\s+(?:push|submit)|cafe\\s+stack\\s+submit|jj\\s+git\\s+push)(?=$|[\\s;&|)])`,
80 "u",
81 );
82
83 // Codex runs each command as `/bin/zsh -lc <script>`, quoting the script in pieces such as
84 // `"a "'`b`'" c"`.
85 const SHELL_WRAPPER = /^\s*(?:\S*\/)?(?:ba|z|da|k)?sh\s+-l?c\s+(?=["'])([\s\S]*)$/u;
86 const SHELL_WORD_PIECE = /"((?:\\[\s\S]|[^"\\])*)"|'([^']*)'|\\([\s\S])|(\S)/gu;
87 // Heredoc bodies, quoted strings and comments are text a command carries, not commands it runs.
88 const HEREDOC_BODY =
89 /<<-?[ \t]*(['"]?)(\w+)\1[^\n]*\n(?:(?:[\s\S]*?\n)?[ \t]*\2[ \t]*(?=\n|$)|[\s\S]*$)/gu;
90 const QUOTED_OR_COMMENT = /'[^']*'|"(?:\\[\s\S]|[^"\\])*"|(?<=^|[\s;&|(])#[^\n]*/gmu;
91 const COMMAND_SEPARATOR = /&&|\|\||[;|\n]/u;
92 // Commands that print other change requests' URLs, so output beside them proves nothing.
93 const LISTS_CHANGE_REQUESTS =
94 /\b(?:gh\s+(?:pr\s+(?:list|status)|search\s+prs|api)|glab\s+(?:mr\s+list|api))\b/u;
95 const URL_LINE = /^[ \t]*(https?:\/\/\S+)[ \t]*$/gmu;
96
97 /** The script a shell input runs, unwrapped from Codex's `sh -lc`. */
98 function shellScript(input: string): string {
99 const [, wrapped] = SHELL_WRAPPER.exec(input) ?? [];
100 return wrapped === undefined
101 ? input
102 : Array.from(
103 wrapped.matchAll(SHELL_WORD_PIECE),
104 ([, double, single, escaped, bare]) =>
105 double?.replace(/\\(["\\$`])/gu, "$1") ?? single ?? escaped ?? bare,
106 ).join("");
107 }
108
109 /** A script split at command separators that sit outside quotes. */
110 function shellSegments(script: string): ReadonlyArray<string> {
111 const segments: Array<string> = [];
112 let current = "";
113 let quote: string | null = null;
114 for (let index = 0; index < script.length; index++) {
115 const char = script[index] ?? "";
116 if (quote !== null) {
117 current += char;
118 if (char === "\\" && quote === '"') current += script[++index] ?? "";
119 else if (char === quote) quote = null;
120 continue;
121 }
122 if (char === "'" || char === '"') quote = char;
123 if (char === "\\") {
124 current += char + (script[++index] ?? "");
125 continue;
126 }
127 const pair = script.slice(index, index + 2);
128 if (pair === "&&" || pair === "||" || char === ";" || char === "|" || char === "\n") {
129 segments.push(current);
130 current = "";
131 if (pair === "&&" || pair === "||") index++;
132 continue;
133 }
134 current += char;
135 }
136 segments.push(current);
137 return segments;
138 }
139
140 /** The commands a script runs, without the text it carries. */
141 const shellCode = (script: string) =>
142 script.replace(HEREDOC_BODY, "<<").replace(QUOTED_OR_COMMENT, "''");
143
144 const isPushCommand = (input: string) => PUSH_COMMAND.test(shellCode(shellScript(input)));
145
146 type CommandExecution = Extract<OrchestrationV2TurnItem, { readonly type: "command_execution" }>;
147
148 /**
149 * The change requests a shell command opened: URLs on lines of their own in the output of a
150 * command that runs a host's create command. One the command also names, as `gh pr view 12`
151 * does, existed before it ran. More URLs than create commands means something else printed
152 * some, such as `gh pr list`, so none are taken.
153 */
154 export function changeRequestsCreatedBy(input: string, output: string) {
155 if (!CREATE_COMMANDS.some(({ command }) => command.test(input))) return [];
156 const script = shellScript(input);
157 const code = shellCode(script);
158 const creates = createCommandCount(code);
159 if (creates === 0) return [];
160 const otherCommands = code
161 .split(COMMAND_SEPARATOR)
162 .filter((part) => createCommandCount(part) === 0)
163 .join("\n");
164 if (LISTS_CHANGE_REQUESTS.test(otherCommands)) return [];
165 const otherScript = shellSegments(script)
166 .filter((segment) => createCommandCount(shellCode(segment)) === 0)
167 .join("\n");
168 const creators = new Set(
169 CREATE_COMMANDS.filter(({ run }) => code.match(run) !== null).map(({ kind }) => kind),
170 );
171 const created = new Map<string, ThreadPullRequestKey & { readonly url: string }>();
172 for (const [, url = ""] of output.matchAll(URL_LINE)) {
173 const parsed = sourceControlDefinitions.parseChangeRequestUrl(url);
174 // Only the host whose create command ran can have printed a new change request.
175 if (parsed === null || !creators.has(parsed.definition.kind)) continue;
176 const { link } = parsed;
177 const named = new RegExp(`(?:^|(?<=[\\s#/=("']))${link.number}(?=$|[\\s);&|/"'])`, "u");
178 // Quoted text in the other commands counts, as in `printf '.../pull/12'`.
179 if (named.test(otherScript)) continue;
180 const key = normalizeThreadPullRequestKey(link);
181 created.set(threadPullRequestKeyOf(key), { ...key, url });
182 }
183 return created.size > creates ? [] : [...created.values()];
184 }
185
186 const isPullRequestProviderError = Schema.is(PullRequestProviderError);
187
188 type SnapshotFields = Omit<ThreadPullRequestSnapshot, "syncedAt">;
189
190 interface LinkEntry {
191 readonly thread: ProjectionStore.ProjectionThreadPullRequests;
192 readonly link: ThreadPullRequestLink;
193 }
194
195 const referenceOf = ({ thread, link }: LinkEntry) => ({
196 projectId: thread.projectId,
197 host: normalizeThreadPullRequestKey(link).host,
198 repository: link.repository,
199 number: link.number,
200 });
201
202 /** Pauses and batching both belong to a project's host. */
203 const pauseKeyOf = (entry: LinkEntry) => {
204 const { projectId, host } = referenceOf(entry);
205 return `${projectId}\0${host}`;
206 };
207
208 /** Another layer of the stack `url` belongs to, in its host's canonical shape. */
209 function siblingChangeRequestUrl(url: string, number: number): string | null {
210 if (!Number.isSafeInteger(number) || number < 1) return null;
211 const parsed = sourceControlDefinitions.parseChangeRequestUrl(url);
212 if (parsed === null) return null;
213 const { definition, link } = parsed;
214 return definition.changeRequestUrl({
215 host: link.host,
216 repository: link.repository,
217 number,
218 webUrl: link.repositoryUrl,
219 });
220 }
221
222 function snapshotFieldsOf(summary: PullRequestSummary): SnapshotFields {
223 return {
224 state: summary.state,
225 title: summary.title,
226 headBranch: summary.headBranch,
227 baseBranch: summary.baseBranch,
228 isDraft: summary.isDraft ?? false,
229 updatedAt: summary.updatedAt,
230 closedAt: summary.closedAt ?? null,
231 mergedAt: summary.mergedAt ?? null,
232 ...(summary.author === undefined ? {} : { author: summary.author }),
233 ...(summary.additions === undefined ? {} : { additions: summary.additions }),
234 ...(summary.deletions === undefined ? {} : { deletions: summary.deletions }),
235 ...(summary.changedFiles === undefined ? {} : { changedFiles: summary.changedFiles }),
236 ...(summary.reviewDecision === undefined ? {} : { reviewDecision: summary.reviewDecision }),
237 ...(summary.checksState === undefined ? {} : { checksState: summary.checksState }),
238 ...(summary.mergeability === undefined ? {} : { mergeability: summary.mergeability }),
239 ...(summary.rebaseRequired === undefined ? {} : { rebaseRequired: summary.rebaseRequired }),
240 };
241 }
242
243 // GitHub briefly returns unknown after base pushes; other hosts may mean a lasting condition.
244 function withKnownMergeability(
245 previous: ThreadPullRequestSnapshot | null,
246 fields: SnapshotFields,
247 provider: PullRequestSummary["provider"],
248 ): SnapshotFields {
249 if (
250 provider !== "github" ||
251 fields.state !== "open" ||
252 fields.mergeability !== "unknown" ||
253 previous?.updatedAt !== fields.updatedAt
254 )
255 return fields;
256 const known = previous?.state === "open" ? previous.mergeability : undefined;
257 return known === undefined || known === "unknown" ? fields : { ...fields, mergeability: known };
258 }
259
260 // Rotating ghe.com avatar tokens should not make otherwise identical snapshots differ.
261 function avatarIdentity(avatarUrl: string | null | undefined): string | null {
262 if (avatarUrl === null || avatarUrl === undefined) return null;
263 const url = URL.parse(avatarUrl);
264 if (url === null || (url.hostname !== "ghe.com" && !url.hostname.endsWith(".ghe.com"))) {
265 return avatarUrl;
266 }
267 url.searchParams.delete("token");
268 return url.href;
269 }
270
271 function snapshotFieldsEqual(left: SnapshotFields, right: SnapshotFields): boolean {
272 return (
273 left.state === right.state &&
274 left.title === right.title &&
275 left.headBranch === right.headBranch &&
276 left.baseBranch === right.baseBranch &&
277 left.isDraft === right.isDraft &&
278 left.updatedAt === right.updatedAt &&
279 (left.closedAt ?? null) === (right.closedAt ?? null) &&
280 (left.mergedAt ?? null) === (right.mergedAt ?? null) &&
281 (left.author?.login ?? null) === (right.author?.login ?? null) &&
282 avatarIdentity(left.author?.avatarUrl) === avatarIdentity(right.author?.avatarUrl) &&
283 left.additions === right.additions &&
284 left.deletions === right.deletions &&
285 left.changedFiles === right.changedFiles &&
286 (left.reviewDecision ?? null) === (right.reviewDecision ?? null) &&
287 (left.checksState ?? null) === (right.checksState ?? null) &&
288 left.mergeability === right.mergeability &&
289 left.rebaseRequired === right.rebaseRequired
290 );
291 }
292
293 function stacksEqual(
294 left: ThreadPullRequestStack | null,
295 right: ThreadPullRequestStack | null,
296 ): boolean {
297 if (left === null || right === null) return left === right;
298 return (
299 left.kind === right.kind &&
300 left.id === right.id &&
301 left.number === right.number &&
302 left.url === right.url &&
303 left.base === right.base &&
304 left.layers.length === right.layers.length &&
305 left.layers.every((layer, index) => {
306 const other = right.layers[index]!;
307 return (
308 layer.number === other.number &&
309 layer.headBranch === other.headBranch &&
310 layer.state === other.state
311 );
312 })
313 );
314 }
315
316 /** Stack ids are unique only within a repository, e.g. GitHub's fallback to the stack number. */
317 function nativeStackKeyOf(link: ThreadPullRequestLink, stack: ThreadPullRequestStack): string {
318 const key = normalizeThreadPullRequestKey(link);
319 return `${key.host}/${key.repository}#stack:${stack.id}`;
320 }
321
322 function sameLayerNumbers(left: ThreadPullRequestStack, right: ThreadPullRequestStack): boolean {
323 return (
324 left.layers.length === right.layers.length &&
325 left.layers.every((layer, index) => layer.number === right.layers[index]!.number)
326 );
327 }
328
329 function skipReason(cause: Cause.Cause<unknown>): string {
330 const error = Cause.squash(cause);
331 return error instanceof Error ? error.message : String(error);
332 }
333
334 function rateLimitRetryAt(cause: Cause.Cause<unknown>): number | undefined {
335 let error: unknown = Cause.squash(cause);
336 while (error instanceof Error) {
337 if (isPullRequestProviderError(error) && error.reason === "rate-limited") return error.retryAt;
338 error = error.cause;
339 }
340 return undefined;
341 }
342
343 function isUnsettled(thread: ProjectionStore.ProjectionThreadPullRequests): boolean {
344 return thread.settledOverride !== "settled" && thread.settledAt === null;
345 }
346
347 export class PullRequestSyncReactor extends Context.Service<
348 PullRequestSyncReactor,
349 {
350 readonly start: () => Effect.Effect<void, never, Scope.Scope>;
351 readonly drain: Effect.Effect<void>;
352 /** Forces reread on the next sweep, including terminal snapshots; rate-limited hosts wait until the pause ends. */
353 readonly requestSync: (key: ThreadPullRequestKey) => Effect.Effect<void>;
354 }
355 >()("t3/orchestration-v2/PullRequestSyncReactor") {}
356
357 /** @public */
358 export const make = Effect.gen(function* () {
359 const engine = yield* Orchestrator.OrchestratorV2;
360 const projections = yield* ProjectionStore.ProjectionStoreV2;
361 const pullRequests = yield* PullRequestService.PullRequestService;
362 const git = yield* GitManager.GitManager;
363 const random = yield* SecureRandom.SecureRandom;
364
365 const lastSyncedAt = new Map<string, number>();
366 const requested = new Set<string>();
367 // Requests arriving during a host read wait for the next sweep.
368 const pendingRequests = new Set<string>();
369 // PR keys wait in `pendingRequests` for one queued sweep, so a burst of links (an agent
370 // linking dozens of pull requests) is read together and shares the summary batches.
371 let requestedSweepQueued = false;
372 const retryStacks = new Set<string>();
373 const failedReads = new Map<string, { readonly retryAtMs: number; readonly backoffMs: number }>();
374 // Group keys the host answered not-found for, or this server cannot read (no CLI, signed out,
375 // unsupported host); they are read on the slow cadence until a read succeeds.
376 const missing = new Set<string>();
377 // Pause per project and host because credentials are project-specific; leave paused PRs due until reset.
378 const pausedUntil = new Map<string, number>();
379
380 const isDue = (key: string, entries: ReadonlyArray<LinkEntry>, nowMs: number): boolean => {
381 if ((failedReads.get(key)?.retryAtMs ?? 0) > nowMs) return false;
382 if (requested.has(key)) return true;
383 const active = entries.filter((entry) => isUnsettled(entry.thread));
384 if (active.length === 0) return false;
385 const last = lastSyncedAt.get(key);
386 const sinceLastMs = last === undefined ? Number.POSITIVE_INFINITY : nowMs - last;
387 if (missing.has(key)) return sinceLastMs >= SLOW_SYNC_INTERVAL_MS;
388 if (retryStacks.has(key) || active.some((entry) => entry.link.snapshot === null)) return true;
389 // Merged snapshots without mergedAt cannot settle threads; reread slowly until supplied.
390 const isFinal = ({ link: { snapshot } }: LinkEntry) =>
391 snapshot?.state === "merged"
392 ? snapshot.mergedAt != null
393 : snapshot?.state === "closed" &&
394 nowMs - Date.parse(snapshot.closedAt ?? "") >= CLOSED_FINAL_AFTER_MS;
395 if (active.every(isFinal)) return false;
396 const open = active.filter((entry) => entry.link.snapshot?.state === "open");
397 if (open.length > 0) {
398 const quiet = open.every((entry) => {
399 const updatedAt = entry.link.snapshot?.updatedAt;
400 return updatedAt != null && nowMs - Date.parse(updatedAt) >= QUIET_AFTER_MS;
401 });
402 return !quiet || sinceLastMs >= QUIET_SYNC_INTERVAL_MS;
403 }
404 return sinceLastMs >= SLOW_SYNC_INTERVAL_MS;
405 };
406
407 const logSkipped =
408 (message: string, fields: Record<string, unknown>) =>
409 <E>(cause: Cause.Cause<E>): Effect.Effect<void, E> =>
410 Cause.hasInterruptsOnly(cause) ? Effect.failCause(cause) : Effect.logWarning(message, fields);
411
412 const sweep = Effect.fn("PullRequestSyncReactor.sweep")(function* (scope: "all" | "requested") {
413 const threads = yield* projections.getThreadsWithPullRequests();
414 const now = yield* DateTime.now;
415 const nowMs = DateTime.toEpochMillis(now);
416 const nowIso = DateTime.formatIso(now);
417
418 const groups = new Map<string, Array<LinkEntry>>();
419 for (const thread of threads) {
420 for (const link of visibleThreadPullRequests(thread.pullRequests ?? [])) {
421 // Reads depend on the project's checkout and credentials, including their failures.
422 const key = `${thread.projectId}\0${threadPullRequestKeyOf(link)}`;
423 const entries = groups.get(key) ?? [];
424 entries.push({ thread, link });
425 groups.set(key, entries);
426 }
427 }
428
429 // A host-level refresh reaches every project, each retaining its own pending retry.
430 for (const [key, entries] of groups) {
431 if (!pendingRequests.has(threadPullRequestKeyOf(entries[0]!.link))) continue;
432 requested.add(key);
433 // A new request reads at once; a failure after it backs off further, not from the start.
434 const failed = failedReads.get(key);
435 if (failed !== undefined) failedReads.set(key, { ...failed, retryAtMs: nowMs });
436 }
437 pendingRequests.clear();
438
439 for (const key of lastSyncedAt.keys()) if (!groups.has(key)) lastSyncedAt.delete(key);
440 for (const key of retryStacks) if (!groups.has(key)) retryStacks.delete(key);
441 for (const key of failedReads.keys()) if (!groups.has(key)) failedReads.delete(key);
442 for (const key of missing) if (!groups.has(key)) missing.delete(key);
443 for (const key of requested.keys()) if (!groups.has(key)) requested.delete(key);
444
445 const linkedThisSweep = new Set<string>();
446 // Fresh stack reads by group, so a member flagged by a sibling is compared against what the
447 // host answered this sweep, not the saved layers that sibling is about to replace.
448 const stacksReadThisSweep = new Map<string, ThreadPullRequestStack | null>();
449 const persistence = yield* Semaphore.make(1);
450
451 // Effect.fn's stack frames close over these sweep-scoped bodies and their arguments, and a
452 // cache lookup forked inside keeps those frames, so a plain span leaves the sweep collectable.
453 const syncEntry = Effect.fnUntraced(function* (
454 entry: LinkEntry,
455 hostFields: SnapshotFields,
456 fetchedStack: { readonly stack: ThreadPullRequestStack | null } | null,
457 provider: PullRequestSummary["provider"],
458 ) {
459 const { thread, link } = entry;
460 const fields = withKnownMergeability(link.snapshot, hostFields, provider);
461 const nextStack = fetchedStack === null ? link.stack : fetchedStack.stack;
462 const changed =
463 link.snapshot === null ||
464 !snapshotFieldsEqual(link.snapshot, fields) ||
465 !stacksEqual(link.stack, nextStack);
466 // Persist siblings before terminal snapshots trigger settlement; settled threads get snapshots without new links.
467 for (const layer of isUnsettled(thread) ? (fetchedStack?.stack?.layers ?? []) : []) {
468 const layerKey = {
469 host: normalizeThreadPullRequestKey(link).host,
470 repository: link.repository,
471 number: layer.number,
472 };
473 const dedupeKey = `${thread.id}:${threadPullRequestKeyOf(layerKey)}`;
474 if (linkedThisSweep.has(dedupeKey)) continue;
475 // Tombstones count as present: a dismissed layer is never re-added.
476 if (
477 (thread.pullRequests ?? []).some((existing) =>
478 threadPullRequestKeysEqual(existing, layerKey),
479 )
480 ) {
481 continue;
482 }
483 const url = siblingChangeRequestUrl(link.url, layer.number);
484 if (url === null) continue;
485 const uuid = yield* random.uuidV4();
486 yield* engine.dispatch({
487 type: "thread.pull-request.link",
488 commandId: CommandId.make(`server:pr-stack-link:${thread.id}:${uuid}`),
489 threadId: thread.id,
490 ...layerKey,
491 url,
492 source: "stack",
493 });
494 linkedThisSweep.add(dedupeKey);
495 }
496 if (changed) {
497 const uuid = yield* random.uuidV4();
498 yield* engine.dispatch({
499 type: "thread.pull-request-link.sync",
500 commandId: CommandId.make(`server:pr-sync:${thread.id}:${uuid}`),
501 threadId: thread.id,
502 host: normalizeThreadPullRequestKey(link).host,
503 repository: link.repository,
504 number: link.number,
505 snapshot: { ...fields, syncedAt: nowIso },
506 stack: nextStack,
507 });
508 }
509 }, Effect.withSpan("PullRequestSyncReactor.syncEntry"));
510
511 const syncGroup = Effect.fnUntraced(function* (key: string, entries: ReadonlyArray<LinkEntry>) {
512 const first = entries[0]!;
513 const ref = referenceOf(first);
514 const refreshRequested = requested.has(key);
515 // A held merged summary is served without asking the host, so a snapshot still missing its
516 // merge time has to drop it or the slow-cadence re-read would hand back the same gap.
517 const missingMergedAt = entries.some(
518 (entry) => entry.link.snapshot?.state === "merged" && entry.link.snapshot.mergedAt == null,
519 );
520 // A stack retry reads past the cache, or it could get back the layer list it is retrying.
521 const invalidated = refreshRequested || missingMergedAt || retryStacks.has(key);
522 if (invalidated) yield* pullRequests.invalidate({ reference: ref });
523 const summary = yield* pullRequests.summary(ref, { recoverTransientFailure: false }).pipe(
524 Effect.tapError((error) =>
525 error.reason === "not-found" || error._tag === "PullRequestUnavailableError"
526 ? Effect.sync(() => {
527 // Not-found, or a host this server cannot read, won't change by asking again
528 // next sweep: it consumes the refresh and starts the backoff.
529 missing.add(key);
530 lastSyncedAt.set(key, nowMs);
531 requested.delete(key);
532 })
533 : Effect.void,
534 ),
535 );
536 missing.delete(key);
537 const fields = snapshotFieldsOf(summary);
538 const needsStack =
539 refreshRequested ||
540 retryStacks.has(key) ||
541 entries.some(
542 (entry) =>
543 entry.link.snapshot === null ||
544 !snapshotFieldsEqual(
545 entry.link.snapshot,
546 withKnownMergeability(entry.link.snapshot, fields, summary.provider),
547 ) ||
548 (summary.stack !== undefined &&
549 (entry.link.stack?.number ?? null) !== (summary.stack?.number ?? null)),
550 );
551 const knownUnstacked =
552 summary.stack === null && entries.every((entry) => entry.link.stack === null);
553 // A sibling can flag this group while its summary read is in flight.
554 if (!invalidated && retryStacks.has(key) && !knownUnstacked) {
555 yield* pullRequests.invalidate({ reference: ref });
556 }
557 // The retry this read answers is consumed before the read, not after it: a sibling may
558 // flag this group again while the read is in flight, and that flag must survive to the
559 // next sweep because the answer below may predate the sibling's layers.
560 if (needsStack) retryStacks.delete(key);
561 const fetchedStack = !needsStack
562 ? null
563 : knownUnstacked
564 ? { stack: null }
565 : yield* pullRequests.stack(ref, { includeDetails: false }).pipe(
566 Effect.map((stack) => ({
567 stack: stack === null ? null : ({ kind: "native", ...stack } as const),
568 })),
569 Effect.catchCauseIf(
570 (cause) => !Cause.hasInterruptsOnly(cause),
571 (cause) =>
572 rateLimitRetryAt(cause) !== undefined
573 ? Effect.sync(() => retryStacks.add(key)).pipe(
574 Effect.andThen(Effect.failCause(cause)),
575 )
576 : Effect.logWarning("pull request stack lookup failed", {
577 key,
578 }).pipe(Effect.as(null)),
579 ),
580 );
581 if (needsStack) {
582 if (fetchedStack === null) {
583 retryStacks.add(key);
584 return;
585 }
586 const fetched = fetchedStack.stack;
587 stacksReadThisSweep.set(key, fetched);
588 // New upper layers leave older members' fields unchanged; differing layer lists need a stack read.
589 // Only flag siblings in the same project: different credentials can yield lists that never agree.
590 if (fetched !== null) {
591 const fetchedKey = nativeStackKeyOf(first.link, fetched);
592 for (const [siblingKey, siblingEntries] of groups) {
593 if (siblingKey === key) continue;
594 if (siblingEntries[0]!.thread.projectId !== first.thread.projectId) continue;
595 const readThisSweep = stacksReadThisSweep.get(siblingKey);
596 const stale = siblingEntries.some(({ link }) => {
597 const stack = readThisSweep === undefined ? link.stack : readThisSweep;
598 return (
599 stack !== null &&
600 nativeStackKeyOf(link, stack) === fetchedKey &&
601 !sameLayerNumbers(stack, fetched)
602 );
603 });
604 if (stale) retryStacks.add(siblingKey);
605 }
606 }
607 }
608 // The host answered, so the cadence clock ticks even if a dispatch below is rejected.
609 lastSyncedAt.set(key, nowMs);
610 failedReads.delete(key);
611 requested.delete(key);
612 yield* Effect.forEach(
613 entries,
614 (entry) =>
615 syncEntry(entry, fields, fetchedStack, summary.provider).pipe(
616 persistence.withPermits(1),
617 Effect.catchCause((cause) => {
618 if (!Cause.hasInterruptsOnly(cause)) retryStacks.add(key);
619 return logSkipped("pull request sync skipped", { threadId: entry.thread.id, key })(
620 cause,
621 );
622 }),
623 ),
624 { discard: true },
625 );
626 }, Effect.withSpan("PullRequestSyncReactor.syncGroup"));
627
628 const skips = new Map<string, { count: number; readonly key: string }>();
629 const readGroup = (key: string, entries: ReadonlyArray<LinkEntry>, pauseKey: string) =>
630 syncGroup(key, entries).pipe(
631 Effect.catchCause((cause) => {
632 if (Cause.hasInterruptsOnly(cause)) return Effect.failCause(cause);
633 const retryAt = rateLimitRetryAt(cause);
634 if (retryAt !== undefined) {
635 pausedUntil.set(pauseKey, Math.max(retryAt, pausedUntil.get(pauseKey) ?? 0));
636 } else {
637 const previous = failedReads.get(key);
638 const backoffMs =
639 previous === undefined
640 ? FAILED_READ_BACKOFF_MS
641 : Math.min(previous.backoffMs * 2, SLOW_SYNC_INTERVAL_MS);
642 failedReads.set(key, { retryAtMs: nowMs + backoffMs, backoffMs });
643 }
644 const reason = skipReason(cause);
645 const skip = skips.get(reason);
646 if (skip === undefined) skips.set(reason, { count: 1, key });
647 else skip.count += 1;
648 return Effect.void;
649 }),
650 );
651 const pools = {
652 // Wider than one summary batch, so reads that finish their cache checks at different
653 // times still share batches.
654 batched: { width: 100, reads: yield* Semaphore.make(100) },
655 // One read per pull request, often a CLI process, so these keep the narrower width.
656 unbatched: { width: 25, reads: yield* Semaphore.make(25) },
657 };
658 const isPaused = (pauseKey: string, atMs: number) => (pausedUntil.get(pauseKey) ?? 0) > atMs;
659 // Judged once a slot is free, so a stack retry a sibling flags meanwhile still counts, and
660 // pauses set during the sweep apply at once.
661 const readIfDue = (
662 key: string,
663 entries: ReadonlyArray<LinkEntry>,
664 pauseKey: string,
665 reads: Semaphore.Semaphore,
666 ) =>
667 reads.withPermits(1)(
668 Effect.suspend(() =>
669 isDue(key, entries, nowMs)
670 ? Clock.currentTimeMillis.pipe(
671 Effect.flatMap((startedAtMs) =>
672 isPaused(pauseKey, startedAtMs) ? Effect.void : readGroup(key, entries, pauseKey),
673 ),
674 )
675 : Effect.void,
676 ),
677 );
678 const groupsByHost = Map.groupBy(
679 Array.from(groups).filter(([key]) => scope === "all" || requested.has(key)),
680 ([, entries]) => pauseKeyOf(entries[0]!),
681 );
682 yield* Effect.forEach(
683 groupsByHost,
684 ([pauseKey, hostGroups]) => {
685 // A host paused or with nothing due skips the lookup. Stack retries only flag groups on
686 // the flagging group's own host, so nothing on it becomes due this sweep.
687 if (
688 isPaused(pauseKey, nowMs) ||
689 !hostGroups.some(([key, entries]) => isDue(key, entries, nowMs))
690 ) {
691 return Effect.void;
692 }
693 return pullRequests.batchesReads(referenceOf(hostGroups[0]![1][0]!)).pipe(
694 Effect.flatMap((batches) => {
695 const { width, reads } = batches ? pools.batched : pools.unbatched;
696 return Effect.forEach(
697 hostGroups,
698 ([key, entries]) => readIfDue(key, entries, pauseKey, reads),
699 {
700 concurrency: width,
701 discard: true,
702 },
703 );
704 }),
705 );
706 },
707 { concurrency: "unbounded", discard: true },
708 );
709 for (const [reason, { count, key }] of skips) {
710 yield* Effect.logWarning("pull request sync skipped", { count, key, reason });
711 }
712 });
713
714 const worker = yield* makeDrainableWorker((scope: "all" | "requested") =>
715 Effect.suspend(() => {
716 if (scope === "requested") requestedSweepQueued = false;
717 return pullRequests.withSharedProjectReads(sweep(scope));
718 }).pipe(
719 Effect.provideService(VcsProcess.Background, true),
720 Effect.catchCause(logSkipped("pull request sync sweep failed", {})),
721 ),
722 );
723
724 // Threads that ran a close command this run, with the commands already read for.
725 const closeCommands = new Map<ThreadId, Set<string>>();
726 const refreshUnmergedLinks = (threadId: ThreadId) =>
727 projections.getThreadsWithPullRequests(threadId).pipe(
728 Effect.flatMap((threads) =>
729 Effect.forEach(
730 threads.flatMap((thread) =>
731 visibleThreadPullRequests(thread.pullRequests ?? []).filter(
732 (link) => link.snapshot?.state === "open" || link.snapshot?.state === "closed",
733 ),
734 ),
735 requestSync,
736 { discard: true },
737 ),
738 ),
739 Effect.catchCause(logSkipped("pull request refresh after run skipped", { threadId })),
740 );
741
742 // A thread's sibling syncs and relinks must not re-read a pull request its project's host
743 // could not find; the slow cadence and explicit refreshes still do.
744 const requestUnsyncedLinks = Effect.fnUntraced(
745 function* (threadId: ThreadId, links: ReadonlyArray<ThreadPullRequestLink>) {
746 const unsynced = links.filter((link) => link.snapshot === null);
747 if (unsynced.length === 0) return;
748 const [thread] =
749 missing.size === 0 ? [] : yield* projections.getThreadsWithPullRequests(threadId);
750 for (const link of unsynced) {
751 if (
752 thread !== undefined &&
753 missing.has(`${thread.projectId}\0${threadPullRequestKeyOf(link)}`)
754 ) {
755 continue;
756 }
757 yield* requestSync(link);
758 }
759 },
760 Effect.catchCause(logSkipped("pull request refresh after link update skipped", {})),
761 );
762
763 // An agent that opens a pull request from a shell gets it linked as link_pull_request would.
764 const linkCreatedChangeRequests = Effect.fn("PullRequestSyncReactor.linkCreatedChangeRequests")(
765 function* (threadId: ThreadId, item: CommandExecution) {
766 if (isOrchestrationV2WorkActive(item.status)) return;
767 yield* Effect.forEach(
768 changeRequestsCreatedBy(item.input, item.output ?? ""),
769 (created) =>
770 engine
771 .dispatch({
772 type: "thread.pull-request.link",
773 // Stable per command, so a repeated settlement cannot relink one the user unlinked.
774 commandId: CommandId.make(
775 `server:pr-created-link:${threadId}:${item.id}:${threadPullRequestKeyOf(created)}`,
776 ),
777 threadId,
778 ...created,
779 source: "created",
780 })
781 .pipe(
782 Effect.catchCause(
783 logSkipped("linking a pull request a command created failed", { threadId }),
784 ),
785 ),
786 { discard: true },
787 );
788 },
789 );
790
791 // A push can start new checks, so the thread's watches forget whether checks were running and
792 // hold the thread until they read the pull request again. The run is still going, so the
793 // thread never shows as done in between.
794 const reopenWatchedChecks = Effect.fn("PullRequestSyncReactor.reopenWatchedChecks")(
795 function* (threadId: ThreadId, item: CommandExecution) {
796 if (isOrchestrationV2WorkActive(item.status) || !isPushCommand(item.input)) return;
797 const threads = yield* projections.getThreadsWithPullRequests(threadId);
798 const answered = threads.some((thread) =>
799 (thread.pullRequests ?? []).some((link) => link.watch?.checksPending !== undefined),
800 );
801 if (!answered) return;
802 yield* engine.dispatch({
803 type: "thread.pull-request-watch.checks-reopen",
804 // Stable per command, so a repeated settlement clears an answer read since only once.
805 commandId: CommandId.make(`server:pr-push-checks:${threadId}:${item.id}`),
806 threadId,
807 });
808 },
809 (effect, threadId) =>
810 effect.pipe(
811 Effect.catchCause(
812 logSkipped("reopening a watch's checks after a push failed", { threadId }),
813 ),
814 ),
815 );
816
817 const start: PullRequestSyncReactor["Service"]["start"] = Effect.fn(
818 "PullRequestSyncReactor.start",
819 )(function* () {
820 const events = engine.streamDomainEvents;
821 // Linking and reopening dispatch under the thread's lock, so a busy thread delays only its
822 // own reactions. Refreshes queue with them, so each thread's reactions keep event order.
823 const threadReactions = yield* KeyedLock.makeRunner<ThreadId>();
824 const inThreadOrder = <E>(threadId: ThreadId, reaction: Effect.Effect<void, E>) =>
825 threadReactions.run(
826 threadId,
827 // The reactions log their own failures, so this sees only defects and shutdown.
828 Effect.catchCause(reaction, (cause) =>
829 Cause.hasInterruptsOnly(cause)
830 ? Effect.interrupt
831 : Effect.logWarning("pull request sync reaction failed", { threadId, cause }),
832 ),
833 );
834 const stateChanges = Stream.merge(
835 yield* pullRequests.subscribeStateChanges,
836 yield* git.subscribePullRequestStateChanges,
837 );
838 yield* forkParked(
839 Stream.runForEach(stateChanges, requestSync).pipe(
840 Effect.catchCause(logSkipped("pull request state change stream failed", {})),
841 ),
842 );
843 yield* forkParked(
844 Stream.runForEach(events, (event) => {
845 switch (event.type) {
846 case "thread.pull-request-link-updated":
847 return requestUnsyncedLinks(event.threadId, [event.payload]);
848 case "thread.pull-request-synced":
849 return requestUnsyncedLinks(
850 event.threadId,
851 visibleThreadPullRequests(event.payload.pullRequests ?? []),
852 );
853 // An agent can merge or close its pull request from a shell with its host's CLI, which
854 // sends no merge notification. Read the thread's unmerged links fresh once that command
855 // ends, while the agent usually still replies, so settlement can land before the run
856 // ends. Read again when the run ends, since `merge --auto` returns before it merges.
857 case "turn-item.updated": {
858 const item = event.payload;
859 if (item.type !== "command_execution") return Effect.void;
860 let reaction = Effect.andThen(
861 linkCreatedChangeRequests(event.threadId, item),
862 reopenWatchedChecks(event.threadId, item),
863 );
864 const active = isOrchestrationV2WorkActive(item.status);
865 if (isChangeRequestCloseCommand(item.input)) {
866 const ended = closeCommands.get(event.threadId) ?? new Set<string>();
867 closeCommands.set(event.threadId, ended);
868 if (!active && !ended.has(item.id)) {
869 ended.add(item.id);
870 reaction = Effect.andThen(reaction, refreshUnmergedLinks(event.threadId));
871 }
872 }
873 // Every reaction skips a running command, so only an ended one forks.
874 return active ? Effect.void : inThreadOrder(event.threadId, reaction);
875 }
876 // Settled threads are not swept, so read their links once to settle with fresh status.
877 case "thread.settled":
878 return inThreadOrder(event.threadId, refreshUnmergedLinks(event.threadId));
879 case "run.updated":
880 return isTerminalRunStatus(event.payload.status) && closeCommands.delete(event.threadId)
881 ? inThreadOrder(event.threadId, refreshUnmergedLinks(event.threadId))
882 : Effect.void;
883 default:
884 return Effect.void;
885 }
886 }).pipe(Effect.catchCause(logSkipped("pull request sync event stream failed", {}))),
887 );
888 yield* forkParked(
889 Effect.gen(function* () {
890 yield* worker.enqueue("all");
891 yield* worker.drain;
892 }).pipe(Effect.repeat(Schedule.spaced("1 minute")), Effect.asVoid),
893 );
894 });
895
896 const requestSync: PullRequestSyncReactor["Service"]["requestSync"] = (key) =>
897 Effect.suspend(() => {
898 pendingRequests.add(threadPullRequestKeyOf(key));
899 if (requestedSweepQueued) return Effect.void;
900 requestedSweepQueued = true;
901 return worker.enqueue("requested");
902 });
903
904 return { start, drain: worker.drain, requestSync } satisfies PullRequestSyncReactor["Service"];
905 });
906
907 export const layer = Layer.effect(PullRequestSyncReactor, make);