@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 844 lines 34.8 KB
Raw
1 import {
2 backgroundWorkHoldsCompletion,
3 isPullRequestWatchTask,
4 turnItemUpdateCanEndBackgroundWork,
5 } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
6 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
7 import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests";
8 import {
9 CommandId,
10 isOrchestrationV2WorkActive,
11 type ThreadId,
12 type OrchestrationV2DomainEvent,
13 type OrchestrationV2ThreadShell,
14 } from "@t3tools/contracts";
15 import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
16 import * as Cause from "effect/Cause";
17 import * as Context from "effect/Context";
18 import * as SecureRandom from "effect/crypto/SecureRandom";
19 import * as DateTime from "effect/DateTime";
20 import * as Effect from "effect/Effect";
21 import * as FiberSet from "effect/FiberSet";
22 import * as FileSystem from "effect/FileSystem";
23 import * as Path from "effect/Path";
24 import * as Schedule from "effect/Schedule";
25 import * as Semaphore from "effect/Semaphore";
26 import type * as Scope from "effect/Scope";
27 import * as Stream from "effect/Stream";
28
29 import * as GitManager from "../git/GitManager.ts";
30 import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts";
31 import * as PullRequestService from "../pullRequest/PullRequestService.ts";
32 import * as ServerSettings from "../serverSettings.ts";
33 import { forkParked } from "../serverActivation.ts";
34 import * as TerminalManager from "../terminal/Manager.ts";
35 import * as GitVcsDriver from "../vcs/GitVcsDriver.ts";
36 import * as ProjectStore from "./ProjectStore.ts";
37 import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts";
38 import { makeNoPullRequestRechecks } from "./noPullRequestRechecks.ts";
39 import * as Orchestrator from "./Orchestrator.ts";
40 import * as ProjectionStore from "./ProjectionStore.ts";
41 import { QUIET_AFTER_MS, QUIET_SYNC_INTERVAL_MS } from "./PullRequestSyncReactor.ts";
42
43 export interface SettlementPullRequest {
44 readonly state: "open" | "closed" | "merged";
45 readonly closedAt?: string | null;
46 readonly mergedAt?: string | null;
47 }
48
49 const DAY_MS = 24 * 60 * 60 * 1_000;
50 const QUEUED_TURN_START_GRACE_MS = 2 * 60 * 1_000;
51
52 function toMillis(value: DateTime.Utc | null | undefined): number | null {
53 return value == null ? null : DateTime.toEpochMillis(value);
54 }
55
56 function latestMillis(values: ReadonlyArray<number | null>): number | null {
57 let latest: number | null = null;
58 for (const value of values) {
59 if (value === null) continue;
60 if (latest === null || value > latest) latest = value;
61 }
62 return latest;
63 }
64
65 function threadActivityAtMs(thread: ProjectionStore.ProjectionSettlementCandidate): number {
66 return (
67 latestMillis([
68 toMillis(thread.latestUserMessageAt),
69 toMillis(thread.latestRunRequestedAt),
70 toMillis(thread.latestRunStartedAt),
71 toMillis(thread.latestRunCompletedAt),
72 ]) ?? DateTime.toEpochMillis(thread.createdAt)
73 );
74 }
75
76 function pullRequestMatchesProject(
77 pullRequest: GitManager.GitBranchPullRequest,
78 project: {
79 readonly repositoryIdentity?: { readonly canonicalKey: string } | null | undefined;
80 },
81 ): boolean {
82 return (
83 pullRequest.repositoryKey !== null &&
84 project.repositoryIdentity != null &&
85 sourceControlDefinitions.canonicalRepositoryKey(pullRequest.repositoryKey) ===
86 sourceControlDefinitions.canonicalRepositoryKey(project.repositoryIdentity.canonicalKey)
87 );
88 }
89
90 type SettlementPullRequestLink = NonNullable<
91 ProjectionStore.ProjectionSettlementCandidate["pullRequests"]
92 >[number];
93
94 /** When the linked pull request merged or closed, or -Infinity if it has not. */
95 function linkTerminalAtMs(link: SettlementPullRequestLink): number {
96 const snapshot = link.snapshot;
97 const value = snapshot?.state === "merged" ? snapshot.mergedAt : snapshot?.closedAt;
98 const timestamp = Date.parse(value ?? "");
99 return Number.isNaN(timestamp) ? Number.NEGATIVE_INFINITY : timestamp;
100 }
101
102 /** Exclude links added after merge or close; they record completed work rather than outcomes this thread awaits. */
103 function settlementPullRequestLinks(
104 links: ReadonlyArray<SettlementPullRequestLink>,
105 ): ReadonlyArray<SettlementPullRequestLink> {
106 return visibleThreadPullRequests(links).filter((link) => {
107 if (link.snapshot === null || link.snapshot.state === "open") return true;
108 const linkedAtMs = Date.parse(link.linkedAt);
109 return Number.isNaN(linkedAtMs) || linkTerminalAtMs(link) >= linkedAtMs;
110 });
111 }
112
113 /** Absolute age bounds clock skew in both directions and expires stale pre-adoption data. */
114 export function threadHasQueuedTurnStart(
115 thread: Pick<
116 OrchestrationV2ThreadShell,
117 | "latestUserMessageAt"
118 | "latestRunRequestedAt"
119 | "latestRunStartedAt"
120 | "latestRunCompletedAt"
121 | "latestRunId"
122 | "status"
123 >,
124 nowMs: number,
125 ): boolean {
126 const messageAtMs = toMillis(thread.latestUserMessageAt);
127 if (messageAtMs === null || thread.status === "failed") return false;
128 const age = nowMs - messageAtMs;
129 if (Number.isNaN(age) || Math.abs(age) > QUEUED_TURN_START_GRACE_MS) return false;
130 if (thread.latestRunId === null) return true;
131 return [
132 toMillis(thread.latestRunRequestedAt),
133 toMillis(thread.latestRunStartedAt),
134 toMillis(thread.latestRunCompletedAt),
135 ].every((value) => value === null || value < messageAtMs);
136 }
137
138 function pullRequestSettles(
139 thread: Pick<
140 ProjectionStore.ProjectionSettlementCandidate,
141 "createdAt" | "latestUserAuthoredMessageAt"
142 >,
143 pullRequest: SettlementPullRequest,
144 autoSettleOnMerge: boolean,
145 ): boolean {
146 if (pullRequest.state !== "closed" && (pullRequest.state !== "merged" || !autoSettleOnMerge)) {
147 return false;
148 }
149 const terminalAt = pullRequest.state === "merged" ? pullRequest.mergedAt : pullRequest.closedAt;
150 if (terminalAt == null) return false;
151 const userAnchorMs = latestMillis([
152 toMillis(thread.createdAt),
153 toMillis(thread.latestUserAuthoredMessageAt),
154 ]);
155 if (userAnchorMs === null) return false;
156 const pullRequestAtMs = Date.parse(terminalAt);
157 if (Number.isNaN(pullRequestAtMs)) return false;
158 return pullRequestAtMs >= userAnchorMs;
159 }
160
161 export function isAutoSettlementCandidate(
162 thread: Omit<ProjectionStore.ProjectionSettlementCandidate, "latestUserAuthoredMessageAt">,
163 nowMs: number,
164 ): boolean {
165 if (thread.archivedAt !== null || thread.settledOverride !== null) return false;
166 if (thread.pinnedAt != null || thread.autoSettleDisabledAt != null) return false;
167 if (thread.pendingRuntimeRequest !== null) return false;
168 // A guest is waiting on the owner; settling would hide the Request.
169 if ((thread.pendingGuestMessageCount ?? 0) > 0) return false;
170 if (thread.activityRunStatus != null) return false;
171 const pendingTasks = thread.pendingBackgroundTasks ?? [];
172 if (backgroundWorkHoldsCompletion(pendingTasks)) return false;
173 // Settling ends a watch, which would silently drop a reviewer's later comments.
174 if (pendingTasks.some(isPullRequestWatchTask)) return false;
175 if (threadHasQueuedTurnStart(thread, nowMs)) return false;
176 // Interrupted runs must keep the same snooze protection the sidebar shows.
177 return !isSnoozed(thread, nowMs);
178 }
179
180 /** Match client effectiveSnoozed so agents and sidebar agree. */
181 export function isSnoozed(
182 thread: Pick<
183 ProjectionStore.ProjectionSettlementCandidate,
184 | "snoozedUntil"
185 | "snoozedAt"
186 | "latestRunCompletedAt"
187 | "status"
188 | "pendingRuntimeRequest"
189 | "pendingGuestMessageCount"
190 >,
191 nowMs: number,
192 ): boolean {
193 const snoozedUntilMs = toMillis(thread.snoozedUntil);
194 if (snoozedUntilMs === null || snoozedUntilMs <= nowMs) return false;
195 // Same as the client: a waiting guest message wakes the thread, and the snooze stays stored.
196 if ((thread.pendingGuestMessageCount ?? 0) > 0) return false;
197 const snoozedAtMs = toMillis(thread.snoozedAt);
198 if (
199 thread.pendingRuntimeRequest !== null &&
200 (thread.pendingRuntimeRequest.kind !== "user_input" ||
201 snoozedAtMs === null ||
202 DateTime.toEpochMillis(thread.pendingRuntimeRequest.createdAt) > snoozedAtMs)
203 ) {
204 return false;
205 }
206 const completedAtMs = toMillis(thread.latestRunCompletedAt);
207 const wokeOnError =
208 thread.status === "failed" &&
209 (snoozedAtMs === null || (completedAtMs !== null && completedAtMs > snoozedAtMs));
210 return !wokeOnError;
211 }
212
213 export function resolveAutoSettlementAt(input: {
214 readonly thread: ProjectionStore.ProjectionSettlementCandidate;
215 readonly pullRequest: SettlementPullRequest | null;
216 readonly nowMs: number;
217 readonly autoSettleAfterDays: number | null;
218 readonly autoSettleOnMerge: boolean;
219 readonly autoSettleKeepPullRequests?: boolean;
220 }): DateTime.Utc | null {
221 const { thread } = input;
222 let pullRequest = input.pullRequest;
223 const links = settlementPullRequestLinks(thread.pullRequests ?? []);
224 if (links.some((link) => link.snapshot === null)) return null;
225 // An open link means the pull request has not finished, so only inactivity
226 // can settle the thread, never a merged or closed lookup.
227 const hasOpenLink = links.some((link) => link.snapshot?.state === "open");
228 if (hasOpenLink) {
229 pullRequest = null;
230 } else if (links.length > 0) {
231 const latest = links.reduce((current, candidate) =>
232 linkTerminalAtMs(candidate) > linkTerminalAtMs(current) ? candidate : current,
233 );
234 pullRequest =
235 latest.snapshot === null
236 ? null
237 : {
238 state: latest.snapshot.state,
239 mergedAt: latest.snapshot.mergedAt ?? null,
240 closedAt: latest.snapshot.closedAt ?? null,
241 };
242 }
243 if (!isAutoSettlementCandidate(thread, input.nowMs)) return null;
244 const activityAtMs = threadActivityAtMs(thread);
245 if (pullRequest !== null && pullRequestSettles(thread, pullRequest, input.autoSettleOnMerge)) {
246 return DateTime.makeUnsafe(activityAtMs);
247 }
248 const hasPullRequest =
249 hasOpenLink ||
250 pullRequest !== null ||
251 thread.linkedPullRequest != null ||
252 thread.branchPullRequest != null;
253 if (input.autoSettleKeepPullRequests === true && hasPullRequest) return null;
254 if (input.autoSettleAfterDays === null) return null;
255 const snoozedUntilMs = toMillis(thread.snoozedUntil);
256 const timerWakeAtMs =
257 snoozedUntilMs !== null && snoozedUntilMs <= input.nowMs ? snoozedUntilMs : null;
258 const inactivityAtMs =
259 latestMillis([activityAtMs, toMillis(thread.lastSnoozeWakeAt), timerWakeAtMs]) ?? activityAtMs;
260 return inactivityAtMs < input.nowMs - input.autoSettleAfterDays * DAY_MS
261 ? DateTime.makeUnsafe(inactivityAtMs)
262 : null;
263 }
264
265 export class ThreadSettlementServiceV2 extends Context.Service<
266 ThreadSettlementServiceV2,
267 {
268 readonly start: () => Effect.Effect<void, never, Scope.Scope>;
269 readonly drain: Effect.Effect<void>;
270 }
271 >()("t3/orchestration-v2/ThreadSettlementService/ThreadSettlementServiceV2") {}
272
273 function autoSettlementConfigured(settings: import("@t3tools/contracts").ServerSettings): boolean {
274 if (settings.sidebarAutoSettleOnMerge || settings.sidebarAutoSettleAfterDays !== null) {
275 return true;
276 }
277 return Object.values(settings.projectSettingsOverrides).some(
278 (entry) =>
279 entry.sidebarAutoSettleOnMerge === true ||
280 (entry.sidebarAutoSettleAfterDays !== undefined && entry.sidebarAutoSettleAfterDays !== null),
281 );
282 }
283
284 /** @internal Exported for tests. */
285 export function autoSettlementSettingsKey(
286 settings: import("@t3tools/contracts").ServerSettings,
287 ): string {
288 return JSON.stringify([
289 settings.sidebarAutoSettleOnMerge,
290 settings.sidebarAutoSettleAfterDays,
291 settings.sidebarAutoSettleKeepPullRequests,
292 Object.entries(settings.projectSettingsOverrides)
293 .filter(
294 ([, entry]) =>
295 entry.sidebarAutoSettleOnMerge !== undefined ||
296 entry.sidebarAutoSettleAfterDays !== undefined ||
297 entry.sidebarAutoSettleKeepPullRequests !== undefined,
298 )
299 .sort(([left], [right]) => left.localeCompare(right))
300 .map(([projectId, entry]) => [
301 projectId,
302 entry.sidebarAutoSettleOnMerge ?? "inherit",
303 entry.sidebarAutoSettleAfterDays === undefined
304 ? "inherit"
305 : entry.sidebarAutoSettleAfterDays,
306 entry.sidebarAutoSettleKeepPullRequests ?? "inherit",
307 ]),
308 ]);
309 }
310
311 export const make = Effect.gen(function* () {
312 const orchestrator = yield* Orchestrator.OrchestratorV2;
313 const projections = yield* ProjectionStore.ProjectionStoreV2;
314 const projectStore = yield* ProjectStore.ProjectStoreV2;
315 const settingsService = yield* ServerSettings.ServerSettingsService;
316 const git = yield* GitManager.GitManager;
317 const pullRequests = yield* PullRequestService.PullRequestService;
318 const random = yield* SecureRandom.SecureRandom;
319 const fileSystem = yield* FileSystem.FileSystem;
320 const path = yield* Path.Path;
321 // A shell merge or close sends no event, and the branch lookup caches an open pull request for
322 // a minute, so these threads' next lookup reads the host fresh.
323 const shellClosedThreads = new Set<ThreadId>();
324 // Each summary read can be a host CLI process (one `az` per read on Azure DevOps), so a quiet
325 // open pull request is read on the sync reactor's quiet cadence rather than every sweep.
326 const quietOpenReadAt = new Map<string, number>();
327 const quietReadKey = (reference: {
328 readonly projectId: string;
329 readonly repository: string;
330 readonly number: number;
331 }) => `${reference.projectId}\0${reference.repository.toLowerCase()}\0${reference.number}`;
332 // A host can finish a merge minutes after the app's merge action and its refresh, and Azure's
333 // updatedAt never moves while open, so pull requests stay on the fast cadence for a while after.
334 // Sweeps that started before the pause fail the same check, so their reads record nothing.
335 let quietPausedAtMs = Number.NEGATIVE_INFINITY;
336 const pauseQuietReads = Effect.fn("ThreadSettlementServiceV2.pauseQuietReads")(function* () {
337 quietPausedAtMs = DateTime.toEpochMillis(yield* DateTime.now);
338 quietOpenReadAt.clear();
339 });
340 const terminals = yield* TerminalManager.TerminalManager;
341 const projectScripts = yield* ProjectSetupScriptRunner.ProjectSetupScriptRunner;
342 // Repeated thread.settled retains settledAt; run the action only once per settlement.
343 const settleActionRunAt = new Map<ThreadId, number>();
344 const noPullRequestRechecks = makeNoPullRequestRechecks();
345
346 const sweep = Effect.fn("ThreadSettlementServiceV2.sweep")(function* (
347 mergedPullRequest: PullRequestService.PullRequestMergeEvent | null,
348 threadId?: ThreadId,
349 ) {
350 // A thread that is not a candidate now (still running) must not keep a merged PR's quiet entry.
351 if (mergedPullRequest !== null) yield* pauseQuietReads();
352 const settings = yield* settingsService.getSettings;
353 const automaticSettlementEnabled = autoSettlementConfigured(settings);
354 const threads = yield* projections.getSettlementCandidates(threadId);
355 if (threads.length === 0) return;
356 const projectShells = yield* projectStore.listShells();
357 const nowMs = DateTime.toEpochMillis(yield* DateTime.now);
358 const projects = new Map(projectShells.map((project) => [project.id, project]));
359 for (const [key, readAtMs] of quietOpenReadAt) {
360 if (nowMs - readAtMs >= QUIET_SYNC_INTERVAL_MS) quietOpenReadAt.delete(key);
361 }
362 // Merge events resweep all candidates because unlinked threads settle through branch lookup.
363 const candidates = threads.filter(
364 (thread) =>
365 (automaticSettlementEnabled || thread.lineage.relationshipToParent === "subagent") &&
366 isAutoSettlementCandidate(thread, nowMs),
367 );
368
369 // Runs per candidate thread, thousands per sweep on busy servers; the sweep owns the span.
370 const settleThread = Effect.fnUntraced(
371 function* (thread: (typeof candidates)[number], pullRequest: SettlementPullRequest | null) {
372 if (thread.lineage.relationshipToParent === "subagent") {
373 if (yield* terminals.hasActiveSubprocess({ threadId: thread.id })) return null;
374 // The store returns a subagent only once its parent is done, so it
375 // settles with the parent and never runs its own lookup.
376 const uuid = yield* random.uuidV4();
377 yield* orchestrator.dispatch({
378 type: "thread.auto-settle",
379 commandId: CommandId.make(`server:auto-settle:${thread.id}:${uuid}`),
380 threadId: thread.id,
381 snapshotAt: thread.updatedAt,
382 });
383 return null;
384 }
385 // thread.auto-settle does not recheck settings, so read them per decision; reads are cached.
386 const currentSettings = resolveProjectSettings(
387 yield* settingsService.getSettings,
388 thread.projectId,
389 ).settings;
390 const decisionNow = yield* DateTime.now;
391 const settledAt = resolveAutoSettlementAt({
392 thread,
393 pullRequest,
394 nowMs: DateTime.toEpochMillis(decisionNow),
395 autoSettleAfterDays: currentSettings.sidebarAutoSettleAfterDays,
396 autoSettleOnMerge: currentSettings.sidebarAutoSettleOnMerge,
397 autoSettleKeepPullRequests: currentSettings.sidebarAutoSettleKeepPullRequests,
398 });
399 if (settledAt === null) return thread;
400 // A terminal still running a program keeps its thread open; a later sweep settles it.
401 if (yield* terminals.hasActiveSubprocess({ threadId: thread.id })) return null;
402 const uuid = yield* random.uuidV4();
403 yield* orchestrator.dispatch({
404 type: "thread.auto-settle",
405 commandId: CommandId.make(`server:auto-settle:${thread.id}:${uuid}`),
406 threadId: thread.id,
407 snapshotAt: thread.updatedAt,
408 settledAt,
409 });
410 return null;
411 },
412 (effect, thread) =>
413 effect.pipe(
414 Effect.catchCauseIf(
415 (cause) => !Cause.hasInterruptsOnly(cause),
416 (cause) =>
417 Effect.logWarning("automatic thread settlement skipped", {
418 threadId: thread.id,
419 cause: Cause.pretty(cause),
420 }).pipe(Effect.as(null)),
421 ),
422 ),
423 );
424
425 const lookupCandidates = (yield* Effect.forEach(
426 candidates,
427 (thread) => {
428 if (thread.lineage.relationshipToParent === "subagent") return settleThread(thread, null);
429 const projectSettings = resolveProjectSettings(settings, thread.projectId).settings;
430 // Skip no-op decisions; actual settlement still reads fresh settings before dispatch.
431 const settledAt = resolveAutoSettlementAt({
432 thread,
433 pullRequest: null,
434 nowMs,
435 autoSettleAfterDays: projectSettings.sidebarAutoSettleAfterDays,
436 autoSettleOnMerge: projectSettings.sidebarAutoSettleOnMerge,
437 autoSettleKeepPullRequests: projectSettings.sidebarAutoSettleKeepPullRequests,
438 });
439 return settledAt === null ? Effect.succeed(thread) : settleThread(thread, null);
440 },
441 { concurrency: 8 },
442 ))
443 .filter((thread) => thread !== null)
444 .filter((thread) => visibleThreadPullRequests(thread.pullRequests ?? []).length === 0)
445 .filter(
446 (thread) =>
447 thread.branch !== null ||
448 thread.linkedPullRequest != null ||
449 thread.branchPullRequest != null,
450 );
451 const lookupCwdByThreadId = new Map<string, string>();
452 yield* Effect.forEach(
453 lookupCandidates,
454 (thread) =>
455 Effect.gen(function* () {
456 const project = projects.get(thread.projectId);
457 if (project === undefined || thread.branch === null) return;
458 // A worktree folder can outlive its .git, and git there fails with not_a_repository.
459 // An unreadable .git keeps the worktree, so git fails there and the thread is skipped.
460 lookupCwdByThreadId.set(
461 thread.id,
462 thread.worktreePath !== null &&
463 (yield* fileSystem
464 .exists(path.join(thread.worktreePath, ".git"))
465 .pipe(Effect.orElseSucceed(() => true)))
466 ? thread.worktreePath
467 : project.workspaceRoot,
468 );
469 }),
470 { concurrency: 8, discard: true },
471 );
472 if (mergedPullRequest !== null) {
473 // Invalidate checkout caches on merge; cached open branches would delay settlement beyond this sweep.
474 const cwds = [...new Set(lookupCwdByThreadId.values())];
475 yield* Effect.forEach(cwds, (cwd) => git.invalidateStatus(cwd), {
476 concurrency: 8,
477 discard: true,
478 });
479 }
480 const lookupKey = (thread: (typeof lookupCandidates)[number]) => {
481 const reference = thread.linkedPullRequest ?? thread.branchPullRequest;
482 if (reference != null) {
483 return JSON.stringify([
484 "linked",
485 reference.projectId,
486 reference.repository,
487 reference.number,
488 lookupCwdByThreadId.get(thread.id),
489 thread.branch,
490 ]);
491 }
492 if (thread.branch === null) return JSON.stringify(["none", thread.id]);
493 const cwd = lookupCwdByThreadId.get(thread.id);
494 return JSON.stringify(
495 cwd === undefined ? ["missing-project", thread.id] : ["branch", cwd, thread.branch],
496 );
497 };
498 const groups = Map.groupBy(lookupCandidates, lookupKey);
499
500 // wouldSettle and pullRequestFor run per lookup group; the git and pull request reads they make carry the spans.
501 const wouldSettle = Effect.fnUntraced(function* (
502 group: ReadonlyArray<(typeof lookupCandidates)[number]>,
503 pullRequest: SettlementPullRequest,
504 ) {
505 const currentSettings = yield* settingsService.getSettings;
506 const decisionNow = yield* DateTime.now;
507 return group.some((thread) => {
508 const { settings } = resolveProjectSettings(currentSettings, thread.projectId);
509 return (
510 resolveAutoSettlementAt({
511 thread,
512 pullRequest,
513 nowMs: DateTime.toEpochMillis(decisionNow),
514 autoSettleAfterDays: settings.sidebarAutoSettleAfterDays,
515 autoSettleOnMerge: settings.sidebarAutoSettleOnMerge,
516 autoSettleKeepPullRequests: settings.sidebarAutoSettleKeepPullRequests,
517 }) !== null
518 );
519 });
520 });
521
522 const pullRequestFor = Effect.fnUntraced(function* (
523 group: ReadonlyArray<(typeof lookupCandidates)[number]>,
524 ) {
525 const thread = group[0]!;
526 const reference = thread.linkedPullRequest ?? thread.branchPullRequest;
527 if (reference != null) {
528 // Only matching PR links settle from this merge event; unrelated links require fresh lookup.
529 const matchesMerge =
530 mergedPullRequest !== null &&
531 reference.projectId === mergedPullRequest.projectId &&
532 reference.repository.toLowerCase() === mergedPullRequest.repository.toLowerCase() &&
533 reference.number === mergedPullRequest.number;
534 if (!matchesMerge && !projects.has(reference.projectId)) {
535 return yield* Effect.die(new Error("linked pull request project not found"));
536 }
537 const readKey = quietReadKey(reference);
538 const shellClosed =
539 group.filter((member) => shellClosedThreads.delete(member.id)).length > 0;
540 if (shellClosed) quietOpenReadAt.delete(readKey);
541 else if (!matchesMerge && quietOpenReadAt.has(readKey)) {
542 return { state: "open", closedAt: null, mergedAt: null } satisfies SettlementPullRequest;
543 }
544 const summary = matchesMerge
545 ? ({
546 state: "merged",
547 closedAt: null,
548 mergedAt: mergedPullRequest.mergedAt,
549 } satisfies SettlementPullRequest)
550 : yield* pullRequests.summary(
551 {
552 projectId: reference.projectId,
553 repository: reference.repository,
554 number: reference.number,
555 },
556 { recoverTransientFailure: false, bypassCache: shellClosed },
557 );
558 if (
559 "updatedAt" in summary &&
560 summary.state === "open" &&
561 nowMs - quietPausedAtMs >= QUIET_SYNC_INTERVAL_MS &&
562 nowMs - Date.parse(summary.updatedAt) >= QUIET_AFTER_MS
563 ) {
564 quietOpenReadAt.set(readKey, nowMs);
565 }
566 const terminal = {
567 state: summary.state,
568 closedAt: summary.closedAt ?? null,
569 mergedAt: summary.mergedAt ?? null,
570 } satisfies SettlementPullRequest;
571 const cwd = lookupCwdByThreadId.get(thread.id);
572 if (summary.state !== "open" && thread.branch !== null && cwd !== undefined) {
573 if (!(yield* wouldSettle(group, terminal))) return undefined;
574 const current = yield* git.branchPullRequest(
575 { cwd, branch: thread.branch },
576 { refresh: true },
577 );
578 const project = projects.get(thread.projectId);
579 if (
580 current?.state === "open" &&
581 project !== undefined &&
582 pullRequestMatchesProject(current, project)
583 ) {
584 return current;
585 }
586 }
587 return terminal;
588 }
589 if (thread.branch === null) return null;
590 const cwd = lookupCwdByThreadId.get(thread.id);
591 if (cwd === undefined) {
592 return yield* Effect.die(new Error("thread project not found"));
593 }
594 const refresh = group.filter((member) => shellClosedThreads.delete(member.id)).length > 0;
595 const key = lookupKey(thread);
596 if (scheduledLookups !== null && !refresh && !scheduledLookups.has(key)) return undefined;
597 // Remembered git state is enough to rule a thread out; only an answer
598 // that would settle it is confirmed against fresh git state.
599 const remembered = yield* git.branchPullRequest(
600 { cwd, branch: thread.branch },
601 { refresh, confirmTerminal: false },
602 );
603 noPullRequestRechecks.record(key, nowMs, remembered !== null);
604 if (refresh || remembered === null || remembered.state === "open") return remembered;
605 if (!(yield* wouldSettle(group, remembered))) return undefined;
606 return yield* git.branchPullRequest({ cwd, branch: thread.branch }, { freshGitState: true });
607 });
608
609 // A merge, a shell close and a thread's own events read at once; only scheduled sweeps wait.
610 const scheduledLookups =
611 mergedPullRequest !== null || threadId !== undefined
612 ? null
613 : noPullRequestRechecks.select(
614 nowMs,
615 [...groups]
616 .filter(([key]) => key.startsWith('["branch",'))
617 .map(([key, group]) => ({
618 key,
619 activityAtMs: Math.max(...group.map(threadActivityAtMs)),
620 })),
621 );
622
623 yield* Effect.forEach(
624 groups.values(),
625 (group) =>
626 Effect.gen(function* () {
627 const pullRequest = yield* pullRequestFor(group);
628 if (pullRequest === undefined) return;
629 yield* Effect.forEach(group, (thread) => settleThread(thread, pullRequest), {
630 discard: true,
631 });
632 }).pipe(
633 Effect.catchCauseIf(
634 (cause) => !Cause.hasInterruptsOnly(cause),
635 (cause) =>
636 Effect.logWarning("automatic thread settlement skipped", {
637 threadIds: group.map((thread) => thread.id),
638 cause: Cause.pretty(cause),
639 }),
640 ),
641 ),
642 { concurrency: 8, discard: true },
643 );
644 if (threadId === undefined) noPullRequestRechecks.retain(groups);
645 });
646
647 const runSweep = (
648 mergedPullRequest: PullRequestService.PullRequestMergeEvent | null,
649 threadId?: ThreadId,
650 ) =>
651 sweep(mergedPullRequest, threadId).pipe(
652 Effect.provideService(GitVcsDriver.BackgroundGitWork, true),
653 Effect.catchCauseIf(
654 (cause) => !Cause.hasInterruptsOnly(cause),
655 (cause) =>
656 Effect.logWarning("automatic thread settlement sweep failed", {
657 cause: Cause.pretty(cause),
658 }),
659 ),
660 );
661 let fullSweepPending = false;
662 // A thread that just stopped must not wait behind a full sweep or another thread's host
663 // reads: clients hold it in Working only briefly, so a late settle makes it appear and then vanish.
664 const fullSweeps = yield* makeDrainableWorker(() => {
665 // Clear at the start so events during this sweep can request one follow-up.
666 fullSweepPending = false;
667 return runSweep(null);
668 });
669 const enqueueFullSweep = () =>
670 Effect.suspend(() => {
671 if (fullSweepPending) return Effect.void;
672 fullSweepPending = true;
673 return fullSweeps.enqueue(undefined);
674 });
675 const threadSweeps = yield* FiberSet.make<void>();
676 const threadSweepSlots = yield* Semaphore.make(8);
677 // One sweep per thread at a time; events during a running sweep coalesce into one more pass.
678 const threadSweepStates = new Map<ThreadId, "queued" | "running" | "rerun">();
679 const sweepThread = (
680 threadId: ThreadId,
681 ): Effect.Effect<void, Effect.Error<ReturnType<typeof runSweep>>> =>
682 Effect.sync(() => threadSweepStates.set(threadId, "running")).pipe(
683 Effect.andThen(runSweep(null, threadId)),
684 threadSweepSlots.withPermits(1),
685 Effect.andThen(
686 Effect.suspend(() => {
687 if (threadSweepStates.get(threadId) === "rerun") return sweepThread(threadId);
688 threadSweepStates.delete(threadId);
689 return Effect.void;
690 }),
691 ),
692 );
693 const enqueueThread = (threadId: ThreadId) =>
694 Effect.suspend(() => {
695 const state = threadSweepStates.get(threadId);
696 if (state === "running") threadSweepStates.set(threadId, "rerun");
697 if (state !== undefined) return Effect.void;
698 threadSweepStates.set(threadId, "queued");
699 return sweepThread(threadId).pipe(FiberSet.run(threadSweeps), Effect.asVoid);
700 });
701
702 // Skip settle scripts in shared checkouts where other threads may still work.
703 const cleanUpSettledThread = Effect.fn("ThreadSettlementServiceV2.cleanUpSettledThread")(
704 function* (threadId: ThreadId) {
705 const settled = yield* projections.getThread(threadId);
706 if (settled.deletedAt !== null || settled.settledOverride !== "settled") return;
707 yield* terminals.closeOnSettle({ threadId });
708 const worktreePath = settled.worktreePath;
709 if (worktreePath === null || !(yield* fileSystem.exists(worktreePath))) return;
710 // Recheck after closing and worktree I/O; only live, settled threads may run scripts.
711 const thread = yield* projections.getThread(threadId);
712 if (thread.deletedAt !== null || thread.settledOverride !== "settled") return;
713 const settledAtMs = toMillis(thread.settledAt);
714 if (settledAtMs === null || settleActionRunAt.get(threadId) === settledAtMs) return;
715 // The parent's worktree belongs to the parent; shared-worktree children
716 // skip duplicate cleanup there.
717 if (
718 thread.lineage.relationshipToParent === "subagent" &&
719 thread.lineage.parentThreadId !== null
720 ) {
721 const parent = yield* projections.getThread(thread.lineage.parentThreadId).pipe(
722 Effect.catchTags({
723 ProjectionStoreThreadNotFoundError: () => Effect.succeed(null),
724 }),
725 );
726 if (parent?.worktreePath === worktreePath) return;
727 }
728 const run = yield* projectScripts.runForThread({
729 threadId,
730 projectId: thread.projectId,
731 worktreePath,
732 trigger: "settle",
733 observeCompletion: {},
734 });
735 settleActionRunAt.set(threadId, settledAtMs);
736 if (run.status === "started" && run.completion) {
737 yield* run.completion.pipe(Effect.forkDetach);
738 }
739 },
740 (effect, threadId) =>
741 effect.pipe(
742 Effect.catchCauseIf(
743 (cause) => !Cause.hasInterruptsOnly(cause),
744 (cause) =>
745 Effect.logWarning("cleaning up a settled thread failed", {
746 threadId,
747 cause: Cause.pretty(cause),
748 }),
749 ),
750 ),
751 );
752
753 const processEvent = (event: OrchestrationV2DomainEvent) => {
754 switch (event.type) {
755 case "thread.settled":
756 return cleanUpSettledThread(event.threadId).pipe(Effect.andThen(enqueueFullSweep()));
757 case "thread.archived":
758 case "thread.deleted":
759 return enqueueFullSweep();
760 case "thread.pull-request-link-updated":
761 case "thread.pull-request-synced":
762 case "thread.issue-synced":
763 case "provider-session.detached":
764 case "run.background-work-cancelled":
765 return enqueueThread(event.threadId);
766 // Sweep only when the work can have ended: progress updates cannot settle anything.
767 case "subagent.updated":
768 return isOrchestrationV2WorkActive(event.payload.status)
769 ? Effect.void
770 : enqueueThread(event.threadId);
771 case "provider-thread.updated":
772 return backgroundWorkHoldsCompletion(event.payload.pendingBackgroundTasks ?? [])
773 ? Effect.void
774 : enqueueThread(event.threadId);
775 case "turn-item.updated": {
776 const item = event.payload;
777 if (
778 item.type === "command_execution" &&
779 !isOrchestrationV2WorkActive(item.status) &&
780 sourceControlDefinitions.definitions.some(
781 (definition) => definition.changeRequestCloseCommand?.test(item.input) === true,
782 )
783 ) {
784 shellClosedThreads.add(event.threadId);
785 }
786 return turnItemUpdateCanEndBackgroundWork(item)
787 ? enqueueThread(event.threadId)
788 : Effect.void;
789 }
790 case "provider-session.updated":
791 return event.payload.status !== "starting" && event.payload.status !== "running"
792 ? enqueueThread(event.threadId)
793 : Effect.void;
794 case "run.updated":
795 return ["completed", "interrupted", "failed", "cancelled", "rolled_back"].includes(
796 event.payload.status,
797 )
798 ? enqueueThread(event.threadId)
799 : Effect.void;
800 default:
801 return Effect.void;
802 }
803 };
804
805 const start: ThreadSettlementServiceV2["Service"]["start"] = Effect.fn(
806 "ThreadSettlementServiceV2.start",
807 )(function* () {
808 const settingsChanges = yield* settingsService.subscribeChanges;
809 const mergedPullRequests = yield* pullRequests.subscribeMerges;
810 const events = orchestrator.streamDomainEvents;
811 const initialSettings = yield* settingsService.getSettings.pipe(Effect.orDie);
812 let lastSettlementSettings = autoSettlementSettingsKey(initialSettings);
813 yield* forkParked(
814 Effect.gen(function* () {
815 yield* enqueueFullSweep();
816 yield* fullSweeps.drain;
817 }).pipe(Effect.repeat(Schedule.spaced("1 minute")), Effect.asVoid),
818 );
819 yield* forkParked(
820 Stream.runForEach(settingsChanges, (settings) => {
821 const key = autoSettlementSettingsKey(settings);
822 if (key === lastSettlementSettings) {
823 return Effect.void;
824 }
825 lastSettlementSettings = key;
826 return enqueueFullSweep();
827 }),
828 );
829 yield* forkParked(Stream.runForEach(mergedPullRequests, (event) => runSweep(event)));
830 yield* forkParked(Stream.runForEach(pullRequests.subscribeRefreshes, pauseQuietReads));
831 yield* forkParked(
832 Stream.runForEach(events, processEvent).pipe(
833 Effect.catchCause((cause) =>
834 Effect.logWarning("Thread settlement event stream failed", { cause }),
835 ),
836 ),
837 );
838 });
839
840 return {
841 start,
842 drain: Effect.andThen(fullSweeps.drain, FiberSet.awaitEmpty(threadSweeps)),
843 } satisfies ThreadSettlementServiceV2["Service"];
844 });