The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | import { |
| 2 | backgroundWorkHoldsCompletion, |
| 3 | isPullRequestWatchTask, |
| 4 | turnItemUpdateCanEndBackgroundWork, |
| 5 | } from "@t3tools/shared/orchestrationV2PendingBackgroundWork"; |
| 6 | import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; |
| 7 | import { visibleThreadPullRequests } from "@t3tools/shared/threadPullRequests"; |
| 8 | import { |
| 9 | CommandId, |
| 10 | isOrchestrationV2WorkActive, |
| 11 | type ThreadId, |
| 12 | type OrchestrationV2DomainEvent, |
| 13 | type OrchestrationV2ThreadShell, |
| 14 | } from "@t3tools/contracts"; |
| 15 | import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker"; |
| 16 | import * as Cause from "effect/Cause"; |
| 17 | import * as Context from "effect/Context"; |
| 18 | import * as SecureRandom from "effect/crypto/SecureRandom"; |
| 19 | import * as DateTime from "effect/DateTime"; |
| 20 | import * as Effect from "effect/Effect"; |
| 21 | import * as FiberSet from "effect/FiberSet"; |
| 22 | import * as FileSystem from "effect/FileSystem"; |
| 23 | import * as Path from "effect/Path"; |
| 24 | import * as Schedule from "effect/Schedule"; |
| 25 | import * as Semaphore from "effect/Semaphore"; |
| 26 | import type * as Scope from "effect/Scope"; |
| 27 | import * as Stream from "effect/Stream"; |
| 28 | |
| 29 | import * as GitManager from "../git/GitManager.ts"; |
| 30 | import * as ProjectSetupScriptRunner from "../project/ProjectSetupScriptRunner.ts"; |
| 31 | import * as PullRequestService from "../pullRequest/PullRequestService.ts"; |
| 32 | import * as ServerSettings from "../serverSettings.ts"; |
| 33 | import { forkParked } from "../serverActivation.ts"; |
| 34 | import * as TerminalManager from "../terminal/Manager.ts"; |
| 35 | import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; |
| 36 | import * as ProjectStore from "./ProjectStore.ts"; |
| 37 | import { sourceControlDefinitions } from "../sourceControl/sourceControlDefinitions.ts"; |
| 38 | import { makeNoPullRequestRechecks } from "./noPullRequestRechecks.ts"; |
| 39 | import * as Orchestrator from "./Orchestrator.ts"; |
| 40 | import * as ProjectionStore from "./ProjectionStore.ts"; |
| 41 | import { QUIET_AFTER_MS, QUIET_SYNC_INTERVAL_MS } from "./PullRequestSyncReactor.ts"; |
| 42 | |
| 43 | export interface SettlementPullRequest { |
| 44 | readonly state: "open" | "closed" | "merged"; |
| 45 | readonly closedAt?: string | null; |
| 46 | readonly mergedAt?: string | null; |
| 47 | } |
| 48 | |
| 49 | const DAY_MS = 24 * 60 * 60 * 1_000; |
| 50 | const QUEUED_TURN_START_GRACE_MS = 2 * 60 * 1_000; |
| 51 | |
| 52 | function toMillis(value: DateTime.Utc | null | undefined): number | null { |
| 53 | return value == null ? null : DateTime.toEpochMillis(value); |
| 54 | } |
| 55 | |
| 56 | function latestMillis(values: ReadonlyArray<number | null>): number | null { |
| 57 | let latest: number | null = null; |
| 58 | for (const value of values) { |
| 59 | if (value === null) continue; |
| 60 | if (latest === null || value > latest) latest = value; |
| 61 | } |
| 62 | return latest; |
| 63 | } |
| 64 | |
| 65 | function threadActivityAtMs(thread: ProjectionStore.ProjectionSettlementCandidate): number { |
| 66 | return ( |
| 67 | latestMillis([ |
| 68 | toMillis(thread.latestUserMessageAt), |
| 69 | toMillis(thread.latestRunRequestedAt), |
| 70 | toMillis(thread.latestRunStartedAt), |
| 71 | toMillis(thread.latestRunCompletedAt), |
| 72 | ]) ?? DateTime.toEpochMillis(thread.createdAt) |
| 73 | ); |
| 74 | } |
| 75 | |
| 76 | function pullRequestMatchesProject( |
| 77 | pullRequest: GitManager.GitBranchPullRequest, |
| 78 | project: { |
| 79 | readonly repositoryIdentity?: { readonly canonicalKey: string } | null | undefined; |
| 80 | }, |
| 81 | ): boolean { |
| 82 | return ( |
| 83 | pullRequest.repositoryKey !== null && |
| 84 | project.repositoryIdentity != null && |
| 85 | sourceControlDefinitions.canonicalRepositoryKey(pullRequest.repositoryKey) === |
| 86 | sourceControlDefinitions.canonicalRepositoryKey(project.repositoryIdentity.canonicalKey) |
| 87 | ); |
| 88 | } |
| 89 | |
| 90 | type SettlementPullRequestLink = NonNullable< |
| 91 | ProjectionStore.ProjectionSettlementCandidate["pullRequests"] |
| 92 | >[number]; |
| 93 | |
| 94 | /** When the linked pull request merged or closed, or -Infinity if it has not. */ |
| 95 | function linkTerminalAtMs(link: SettlementPullRequestLink): number { |
| 96 | const snapshot = link.snapshot; |
| 97 | const value = snapshot?.state === "merged" ? snapshot.mergedAt : snapshot?.closedAt; |
| 98 | const timestamp = Date.parse(value ?? ""); |
| 99 | return Number.isNaN(timestamp) ? Number.NEGATIVE_INFINITY : timestamp; |
| 100 | } |
| 101 | |
| 102 | /** Exclude links added after merge or close; they record completed work rather than outcomes this thread awaits. */ |
| 103 | function settlementPullRequestLinks( |
| 104 | links: ReadonlyArray<SettlementPullRequestLink>, |
| 105 | ): ReadonlyArray<SettlementPullRequestLink> { |
| 106 | return visibleThreadPullRequests(links).filter((link) => { |
| 107 | if (link.snapshot === null || link.snapshot.state === "open") return true; |
| 108 | const linkedAtMs = Date.parse(link.linkedAt); |
| 109 | return Number.isNaN(linkedAtMs) || linkTerminalAtMs(link) >= linkedAtMs; |
| 110 | }); |
| 111 | } |
| 112 | |
| 113 | /** Absolute age bounds clock skew in both directions and expires stale pre-adoption data. */ |
| 114 | export function threadHasQueuedTurnStart( |
| 115 | thread: Pick< |
| 116 | OrchestrationV2ThreadShell, |
| 117 | | "latestUserMessageAt" |
| 118 | | "latestRunRequestedAt" |
| 119 | | "latestRunStartedAt" |
| 120 | | "latestRunCompletedAt" |
| 121 | | "latestRunId" |
| 122 | | "status" |
| 123 | >, |
| 124 | nowMs: number, |
| 125 | ): boolean { |
| 126 | const messageAtMs = toMillis(thread.latestUserMessageAt); |
| 127 | if (messageAtMs === null || thread.status === "failed") return false; |
| 128 | const age = nowMs - messageAtMs; |
| 129 | if (Number.isNaN(age) || Math.abs(age) > QUEUED_TURN_START_GRACE_MS) return false; |
| 130 | if (thread.latestRunId === null) return true; |
| 131 | return [ |
| 132 | toMillis(thread.latestRunRequestedAt), |
| 133 | toMillis(thread.latestRunStartedAt), |
| 134 | toMillis(thread.latestRunCompletedAt), |
| 135 | ].every((value) => value === null || value < messageAtMs); |
| 136 | } |
| 137 | |
| 138 | function pullRequestSettles( |
| 139 | thread: Pick< |
| 140 | ProjectionStore.ProjectionSettlementCandidate, |
| 141 | "createdAt" | "latestUserAuthoredMessageAt" |
| 142 | >, |
| 143 | pullRequest: SettlementPullRequest, |
| 144 | autoSettleOnMerge: boolean, |
| 145 | ): boolean { |
| 146 | if (pullRequest.state !== "closed" && (pullRequest.state !== "merged" || !autoSettleOnMerge)) { |
| 147 | return false; |
| 148 | } |
| 149 | const terminalAt = pullRequest.state === "merged" ? pullRequest.mergedAt : pullRequest.closedAt; |
| 150 | if (terminalAt == null) return false; |
| 151 | const userAnchorMs = latestMillis([ |
| 152 | toMillis(thread.createdAt), |
| 153 | toMillis(thread.latestUserAuthoredMessageAt), |
| 154 | ]); |
| 155 | if (userAnchorMs === null) return false; |
| 156 | const pullRequestAtMs = Date.parse(terminalAt); |
| 157 | if (Number.isNaN(pullRequestAtMs)) return false; |
| 158 | return pullRequestAtMs >= userAnchorMs; |
| 159 | } |
| 160 | |
| 161 | export function isAutoSettlementCandidate( |
| 162 | thread: Omit<ProjectionStore.ProjectionSettlementCandidate, "latestUserAuthoredMessageAt">, |
| 163 | nowMs: number, |
| 164 | ): boolean { |
| 165 | if (thread.archivedAt !== null || thread.settledOverride !== null) return false; |
| 166 | if (thread.pinnedAt != null || thread.autoSettleDisabledAt != null) return false; |
| 167 | if (thread.pendingRuntimeRequest !== null) return false; |
| 168 | // A guest is waiting on the owner; settling would hide the Request. |
| 169 | if ((thread.pendingGuestMessageCount ?? 0) > 0) return false; |
| 170 | if (thread.activityRunStatus != null) return false; |
| 171 | const pendingTasks = thread.pendingBackgroundTasks ?? []; |
| 172 | if (backgroundWorkHoldsCompletion(pendingTasks)) return false; |
| 173 | // Settling ends a watch, which would silently drop a reviewer's later comments. |
| 174 | if (pendingTasks.some(isPullRequestWatchTask)) return false; |
| 175 | if (threadHasQueuedTurnStart(thread, nowMs)) return false; |
| 176 | // Interrupted runs must keep the same snooze protection the sidebar shows. |
| 177 | return !isSnoozed(thread, nowMs); |
| 178 | } |
| 179 | |
| 180 | /** Match client effectiveSnoozed so agents and sidebar agree. */ |
| 181 | export function isSnoozed( |
| 182 | thread: Pick< |
| 183 | ProjectionStore.ProjectionSettlementCandidate, |
| 184 | | "snoozedUntil" |
| 185 | | "snoozedAt" |
| 186 | | "latestRunCompletedAt" |
| 187 | | "status" |
| 188 | | "pendingRuntimeRequest" |
| 189 | | "pendingGuestMessageCount" |
| 190 | >, |
| 191 | nowMs: number, |
| 192 | ): boolean { |
| 193 | const snoozedUntilMs = toMillis(thread.snoozedUntil); |
| 194 | if (snoozedUntilMs === null || snoozedUntilMs <= nowMs) return false; |
| 195 | // Same as the client: a waiting guest message wakes the thread, and the snooze stays stored. |
| 196 | if ((thread.pendingGuestMessageCount ?? 0) > 0) return false; |
| 197 | const snoozedAtMs = toMillis(thread.snoozedAt); |
| 198 | if ( |
| 199 | thread.pendingRuntimeRequest !== null && |
| 200 | (thread.pendingRuntimeRequest.kind !== "user_input" || |
| 201 | snoozedAtMs === null || |
| 202 | DateTime.toEpochMillis(thread.pendingRuntimeRequest.createdAt) > snoozedAtMs) |
| 203 | ) { |
| 204 | return false; |
| 205 | } |
| 206 | const completedAtMs = toMillis(thread.latestRunCompletedAt); |
| 207 | const wokeOnError = |
| 208 | thread.status === "failed" && |
| 209 | (snoozedAtMs === null || (completedAtMs !== null && completedAtMs > snoozedAtMs)); |
| 210 | return !wokeOnError; |
| 211 | } |
| 212 | |
| 213 | export function resolveAutoSettlementAt(input: { |
| 214 | readonly thread: ProjectionStore.ProjectionSettlementCandidate; |
| 215 | readonly pullRequest: SettlementPullRequest | null; |
| 216 | readonly nowMs: number; |
| 217 | readonly autoSettleAfterDays: number | null; |
| 218 | readonly autoSettleOnMerge: boolean; |
| 219 | readonly autoSettleKeepPullRequests?: boolean; |
| 220 | }): DateTime.Utc | null { |
| 221 | const { thread } = input; |
| 222 | let pullRequest = input.pullRequest; |
| 223 | const links = settlementPullRequestLinks(thread.pullRequests ?? []); |
| 224 | if (links.some((link) => link.snapshot === null)) return null; |
| 225 | // An open link means the pull request has not finished, so only inactivity |
| 226 | // can settle the thread, never a merged or closed lookup. |
| 227 | const hasOpenLink = links.some((link) => link.snapshot?.state === "open"); |
| 228 | if (hasOpenLink) { |
| 229 | pullRequest = null; |
| 230 | } else if (links.length > 0) { |
| 231 | const latest = links.reduce((current, candidate) => |
| 232 | linkTerminalAtMs(candidate) > linkTerminalAtMs(current) ? candidate : current, |
| 233 | ); |
| 234 | pullRequest = |
| 235 | latest.snapshot === null |
| 236 | ? null |
| 237 | : { |
| 238 | state: latest.snapshot.state, |
| 239 | mergedAt: latest.snapshot.mergedAt ?? null, |
| 240 | closedAt: latest.snapshot.closedAt ?? null, |
| 241 | }; |
| 242 | } |
| 243 | if (!isAutoSettlementCandidate(thread, input.nowMs)) return null; |
| 244 | const activityAtMs = threadActivityAtMs(thread); |
| 245 | if (pullRequest !== null && pullRequestSettles(thread, pullRequest, input.autoSettleOnMerge)) { |
| 246 | return DateTime.makeUnsafe(activityAtMs); |
| 247 | } |
| 248 | const hasPullRequest = |
| 249 | hasOpenLink || |
| 250 | pullRequest !== null || |
| 251 | thread.linkedPullRequest != null || |
| 252 | thread.branchPullRequest != null; |
| 253 | if (input.autoSettleKeepPullRequests === true && hasPullRequest) return null; |
| 254 | if (input.autoSettleAfterDays === null) return null; |
| 255 | const snoozedUntilMs = toMillis(thread.snoozedUntil); |
| 256 | const timerWakeAtMs = |
| 257 | snoozedUntilMs !== null && snoozedUntilMs <= input.nowMs ? snoozedUntilMs : null; |
| 258 | const inactivityAtMs = |
| 259 | latestMillis([activityAtMs, toMillis(thread.lastSnoozeWakeAt), timerWakeAtMs]) ?? activityAtMs; |
| 260 | return inactivityAtMs < input.nowMs - input.autoSettleAfterDays * DAY_MS |
| 261 | ? DateTime.makeUnsafe(inactivityAtMs) |
| 262 | : null; |
| 263 | } |
| 264 | |
| 265 | export class ThreadSettlementServiceV2 extends Context.Service< |
| 266 | ThreadSettlementServiceV2, |
| 267 | { |
| 268 | readonly start: () => Effect.Effect<void, never, Scope.Scope>; |
| 269 | readonly drain: Effect.Effect<void>; |
| 270 | } |
| 271 | >()("t3/orchestration-v2/ThreadSettlementService/ThreadSettlementServiceV2") {} |
| 272 | |
| 273 | function autoSettlementConfigured(settings: import("@t3tools/contracts").ServerSettings): boolean { |
| 274 | if (settings.sidebarAutoSettleOnMerge || settings.sidebarAutoSettleAfterDays !== null) { |
| 275 | return true; |
| 276 | } |
| 277 | return Object.values(settings.projectSettingsOverrides).some( |
| 278 | (entry) => |
| 279 | entry.sidebarAutoSettleOnMerge === true || |
| 280 | (entry.sidebarAutoSettleAfterDays !== undefined && entry.sidebarAutoSettleAfterDays !== null), |
| 281 | ); |
| 282 | } |
| 283 | |
| 284 | /** @internal Exported for tests. */ |
| 285 | export function autoSettlementSettingsKey( |
| 286 | settings: import("@t3tools/contracts").ServerSettings, |
| 287 | ): string { |
| 288 | return JSON.stringify([ |
| 289 | settings.sidebarAutoSettleOnMerge, |
| 290 | settings.sidebarAutoSettleAfterDays, |
| 291 | settings.sidebarAutoSettleKeepPullRequests, |
| 292 | Object.entries(settings.projectSettingsOverrides) |
| 293 | .filter( |
| 294 | ([, entry]) => |
| 295 | entry.sidebarAutoSettleOnMerge !== undefined || |
| 296 | entry.sidebarAutoSettleAfterDays !== undefined || |
| 297 | entry.sidebarAutoSettleKeepPullRequests !== undefined, |
| 298 | ) |
| 299 | .sort(([left], [right]) => left.localeCompare(right)) |
| 300 | .map(([projectId, entry]) => [ |
| 301 | projectId, |
| 302 | entry.sidebarAutoSettleOnMerge ?? "inherit", |
| 303 | entry.sidebarAutoSettleAfterDays === undefined |
| 304 | ? "inherit" |
| 305 | : entry.sidebarAutoSettleAfterDays, |
| 306 | entry.sidebarAutoSettleKeepPullRequests ?? "inherit", |
| 307 | ]), |
| 308 | ]); |
| 309 | } |
| 310 | |
| 311 | export const make = Effect.gen(function* () { |
| 312 | const orchestrator = yield* Orchestrator.OrchestratorV2; |
| 313 | const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 314 | const projectStore = yield* ProjectStore.ProjectStoreV2; |
| 315 | const settingsService = yield* ServerSettings.ServerSettingsService; |
| 316 | const git = yield* GitManager.GitManager; |
| 317 | const pullRequests = yield* PullRequestService.PullRequestService; |
| 318 | const random = yield* SecureRandom.SecureRandom; |
| 319 | const fileSystem = yield* FileSystem.FileSystem; |
| 320 | const path = yield* Path.Path; |
| 321 | // A shell merge or close sends no event, and the branch lookup caches an open pull request for |
| 322 | // a minute, so these threads' next lookup reads the host fresh. |
| 323 | const shellClosedThreads = new Set<ThreadId>(); |
| 324 | // Each summary read can be a host CLI process (one `az` per read on Azure DevOps), so a quiet |
| 325 | // open pull request is read on the sync reactor's quiet cadence rather than every sweep. |
| 326 | const quietOpenReadAt = new Map<string, number>(); |
| 327 | const quietReadKey = (reference: { |
| 328 | readonly projectId: string; |
| 329 | readonly repository: string; |
| 330 | readonly number: number; |
| 331 | }) => `${reference.projectId}\0${reference.repository.toLowerCase()}\0${reference.number}`; |
| 332 | // A host can finish a merge minutes after the app's merge action and its refresh, and Azure's |
| 333 | // updatedAt never moves while open, so pull requests stay on the fast cadence for a while after. |
| 334 | // Sweeps that started before the pause fail the same check, so their reads record nothing. |
| 335 | let quietPausedAtMs = Number.NEGATIVE_INFINITY; |
| 336 | const pauseQuietReads = Effect.fn("ThreadSettlementServiceV2.pauseQuietReads")(function* () { |
| 337 | quietPausedAtMs = DateTime.toEpochMillis(yield* DateTime.now); |
| 338 | quietOpenReadAt.clear(); |
| 339 | }); |
| 340 | const terminals = yield* TerminalManager.TerminalManager; |
| 341 | const projectScripts = yield* ProjectSetupScriptRunner.ProjectSetupScriptRunner; |
| 342 | // Repeated thread.settled retains settledAt; run the action only once per settlement. |
| 343 | const settleActionRunAt = new Map<ThreadId, number>(); |
| 344 | const noPullRequestRechecks = makeNoPullRequestRechecks(); |
| 345 | |
| 346 | const sweep = Effect.fn("ThreadSettlementServiceV2.sweep")(function* ( |
| 347 | mergedPullRequest: PullRequestService.PullRequestMergeEvent | null, |
| 348 | threadId?: ThreadId, |
| 349 | ) { |
| 350 | // A thread that is not a candidate now (still running) must not keep a merged PR's quiet entry. |
| 351 | if (mergedPullRequest !== null) yield* pauseQuietReads(); |
| 352 | const settings = yield* settingsService.getSettings; |
| 353 | const automaticSettlementEnabled = autoSettlementConfigured(settings); |
| 354 | const threads = yield* projections.getSettlementCandidates(threadId); |
| 355 | if (threads.length === 0) return; |
| 356 | const projectShells = yield* projectStore.listShells(); |
| 357 | const nowMs = DateTime.toEpochMillis(yield* DateTime.now); |
| 358 | const projects = new Map(projectShells.map((project) => [project.id, project])); |
| 359 | for (const [key, readAtMs] of quietOpenReadAt) { |
| 360 | if (nowMs - readAtMs >= QUIET_SYNC_INTERVAL_MS) quietOpenReadAt.delete(key); |
| 361 | } |
| 362 | // Merge events resweep all candidates because unlinked threads settle through branch lookup. |
| 363 | const candidates = threads.filter( |
| 364 | (thread) => |
| 365 | (automaticSettlementEnabled || thread.lineage.relationshipToParent === "subagent") && |
| 366 | isAutoSettlementCandidate(thread, nowMs), |
| 367 | ); |
| 368 | |
| 369 | // Runs per candidate thread, thousands per sweep on busy servers; the sweep owns the span. |
| 370 | const settleThread = Effect.fnUntraced( |
| 371 | function* (thread: (typeof candidates)[number], pullRequest: SettlementPullRequest | null) { |
| 372 | if (thread.lineage.relationshipToParent === "subagent") { |
| 373 | if (yield* terminals.hasActiveSubprocess({ threadId: thread.id })) return null; |
| 374 | // The store returns a subagent only once its parent is done, so it |
| 375 | // settles with the parent and never runs its own lookup. |
| 376 | const uuid = yield* random.uuidV4(); |
| 377 | yield* orchestrator.dispatch({ |
| 378 | type: "thread.auto-settle", |
| 379 | commandId: CommandId.make(`server:auto-settle:${thread.id}:${uuid}`), |
| 380 | threadId: thread.id, |
| 381 | snapshotAt: thread.updatedAt, |
| 382 | }); |
| 383 | return null; |
| 384 | } |
| 385 | // thread.auto-settle does not recheck settings, so read them per decision; reads are cached. |
| 386 | const currentSettings = resolveProjectSettings( |
| 387 | yield* settingsService.getSettings, |
| 388 | thread.projectId, |
| 389 | ).settings; |
| 390 | const decisionNow = yield* DateTime.now; |
| 391 | const settledAt = resolveAutoSettlementAt({ |
| 392 | thread, |
| 393 | pullRequest, |
| 394 | nowMs: DateTime.toEpochMillis(decisionNow), |
| 395 | autoSettleAfterDays: currentSettings.sidebarAutoSettleAfterDays, |
| 396 | autoSettleOnMerge: currentSettings.sidebarAutoSettleOnMerge, |
| 397 | autoSettleKeepPullRequests: currentSettings.sidebarAutoSettleKeepPullRequests, |
| 398 | }); |
| 399 | if (settledAt === null) return thread; |
| 400 | // A terminal still running a program keeps its thread open; a later sweep settles it. |
| 401 | if (yield* terminals.hasActiveSubprocess({ threadId: thread.id })) return null; |
| 402 | const uuid = yield* random.uuidV4(); |
| 403 | yield* orchestrator.dispatch({ |
| 404 | type: "thread.auto-settle", |
| 405 | commandId: CommandId.make(`server:auto-settle:${thread.id}:${uuid}`), |
| 406 | threadId: thread.id, |
| 407 | snapshotAt: thread.updatedAt, |
| 408 | settledAt, |
| 409 | }); |
| 410 | return null; |
| 411 | }, |
| 412 | (effect, thread) => |
| 413 | effect.pipe( |
| 414 | Effect.catchCauseIf( |
| 415 | (cause) => !Cause.hasInterruptsOnly(cause), |
| 416 | (cause) => |
| 417 | Effect.logWarning("automatic thread settlement skipped", { |
| 418 | threadId: thread.id, |
| 419 | cause: Cause.pretty(cause), |
| 420 | }).pipe(Effect.as(null)), |
| 421 | ), |
| 422 | ), |
| 423 | ); |
| 424 | |
| 425 | const lookupCandidates = (yield* Effect.forEach( |
| 426 | candidates, |
| 427 | (thread) => { |
| 428 | if (thread.lineage.relationshipToParent === "subagent") return settleThread(thread, null); |
| 429 | const projectSettings = resolveProjectSettings(settings, thread.projectId).settings; |
| 430 | // Skip no-op decisions; actual settlement still reads fresh settings before dispatch. |
| 431 | const settledAt = resolveAutoSettlementAt({ |
| 432 | thread, |
| 433 | pullRequest: null, |
| 434 | nowMs, |
| 435 | autoSettleAfterDays: projectSettings.sidebarAutoSettleAfterDays, |
| 436 | autoSettleOnMerge: projectSettings.sidebarAutoSettleOnMerge, |
| 437 | autoSettleKeepPullRequests: projectSettings.sidebarAutoSettleKeepPullRequests, |
| 438 | }); |
| 439 | return settledAt === null ? Effect.succeed(thread) : settleThread(thread, null); |
| 440 | }, |
| 441 | { concurrency: 8 }, |
| 442 | )) |
| 443 | .filter((thread) => thread !== null) |
| 444 | .filter((thread) => visibleThreadPullRequests(thread.pullRequests ?? []).length === 0) |
| 445 | .filter( |
| 446 | (thread) => |
| 447 | thread.branch !== null || |
| 448 | thread.linkedPullRequest != null || |
| 449 | thread.branchPullRequest != null, |
| 450 | ); |
| 451 | const lookupCwdByThreadId = new Map<string, string>(); |
| 452 | yield* Effect.forEach( |
| 453 | lookupCandidates, |
| 454 | (thread) => |
| 455 | Effect.gen(function* () { |
| 456 | const project = projects.get(thread.projectId); |
| 457 | if (project === undefined || thread.branch === null) return; |
| 458 | // A worktree folder can outlive its .git, and git there fails with not_a_repository. |
| 459 | // An unreadable .git keeps the worktree, so git fails there and the thread is skipped. |
| 460 | lookupCwdByThreadId.set( |
| 461 | thread.id, |
| 462 | thread.worktreePath !== null && |
| 463 | (yield* fileSystem |
| 464 | .exists(path.join(thread.worktreePath, ".git")) |
| 465 | .pipe(Effect.orElseSucceed(() => true))) |
| 466 | ? thread.worktreePath |
| 467 | : project.workspaceRoot, |
| 468 | ); |
| 469 | }), |
| 470 | { concurrency: 8, discard: true }, |
| 471 | ); |
| 472 | if (mergedPullRequest !== null) { |
| 473 | // Invalidate checkout caches on merge; cached open branches would delay settlement beyond this sweep. |
| 474 | const cwds = [...new Set(lookupCwdByThreadId.values())]; |
| 475 | yield* Effect.forEach(cwds, (cwd) => git.invalidateStatus(cwd), { |
| 476 | concurrency: 8, |
| 477 | discard: true, |
| 478 | }); |
| 479 | } |
| 480 | const lookupKey = (thread: (typeof lookupCandidates)[number]) => { |
| 481 | const reference = thread.linkedPullRequest ?? thread.branchPullRequest; |
| 482 | if (reference != null) { |
| 483 | return JSON.stringify([ |
| 484 | "linked", |
| 485 | reference.projectId, |
| 486 | reference.repository, |
| 487 | reference.number, |
| 488 | lookupCwdByThreadId.get(thread.id), |
| 489 | thread.branch, |
| 490 | ]); |
| 491 | } |
| 492 | if (thread.branch === null) return JSON.stringify(["none", thread.id]); |
| 493 | const cwd = lookupCwdByThreadId.get(thread.id); |
| 494 | return JSON.stringify( |
| 495 | cwd === undefined ? ["missing-project", thread.id] : ["branch", cwd, thread.branch], |
| 496 | ); |
| 497 | }; |
| 498 | const groups = Map.groupBy(lookupCandidates, lookupKey); |
| 499 | |
| 500 | // wouldSettle and pullRequestFor run per lookup group; the git and pull request reads they make carry the spans. |
| 501 | const wouldSettle = Effect.fnUntraced(function* ( |
| 502 | group: ReadonlyArray<(typeof lookupCandidates)[number]>, |
| 503 | pullRequest: SettlementPullRequest, |
| 504 | ) { |
| 505 | const currentSettings = yield* settingsService.getSettings; |
| 506 | const decisionNow = yield* DateTime.now; |
| 507 | return group.some((thread) => { |
| 508 | const { settings } = resolveProjectSettings(currentSettings, thread.projectId); |
| 509 | return ( |
| 510 | resolveAutoSettlementAt({ |
| 511 | thread, |
| 512 | pullRequest, |
| 513 | nowMs: DateTime.toEpochMillis(decisionNow), |
| 514 | autoSettleAfterDays: settings.sidebarAutoSettleAfterDays, |
| 515 | autoSettleOnMerge: settings.sidebarAutoSettleOnMerge, |
| 516 | autoSettleKeepPullRequests: settings.sidebarAutoSettleKeepPullRequests, |
| 517 | }) !== null |
| 518 | ); |
| 519 | }); |
| 520 | }); |
| 521 | |
| 522 | const pullRequestFor = Effect.fnUntraced(function* ( |
| 523 | group: ReadonlyArray<(typeof lookupCandidates)[number]>, |
| 524 | ) { |
| 525 | const thread = group[0]!; |
| 526 | const reference = thread.linkedPullRequest ?? thread.branchPullRequest; |
| 527 | if (reference != null) { |
| 528 | // Only matching PR links settle from this merge event; unrelated links require fresh lookup. |
| 529 | const matchesMerge = |
| 530 | mergedPullRequest !== null && |
| 531 | reference.projectId === mergedPullRequest.projectId && |
| 532 | reference.repository.toLowerCase() === mergedPullRequest.repository.toLowerCase() && |
| 533 | reference.number === mergedPullRequest.number; |
| 534 | if (!matchesMerge && !projects.has(reference.projectId)) { |
| 535 | return yield* Effect.die(new Error("linked pull request project not found")); |
| 536 | } |
| 537 | const readKey = quietReadKey(reference); |
| 538 | const shellClosed = |
| 539 | group.filter((member) => shellClosedThreads.delete(member.id)).length > 0; |
| 540 | if (shellClosed) quietOpenReadAt.delete(readKey); |
| 541 | else if (!matchesMerge && quietOpenReadAt.has(readKey)) { |
| 542 | return { state: "open", closedAt: null, mergedAt: null } satisfies SettlementPullRequest; |
| 543 | } |
| 544 | const summary = matchesMerge |
| 545 | ? ({ |
| 546 | state: "merged", |
| 547 | closedAt: null, |
| 548 | mergedAt: mergedPullRequest.mergedAt, |
| 549 | } satisfies SettlementPullRequest) |
| 550 | : yield* pullRequests.summary( |
| 551 | { |
| 552 | projectId: reference.projectId, |
| 553 | repository: reference.repository, |
| 554 | number: reference.number, |
| 555 | }, |
| 556 | { recoverTransientFailure: false, bypassCache: shellClosed }, |
| 557 | ); |
| 558 | if ( |
| 559 | "updatedAt" in summary && |
| 560 | summary.state === "open" && |
| 561 | nowMs - quietPausedAtMs >= QUIET_SYNC_INTERVAL_MS && |
| 562 | nowMs - Date.parse(summary.updatedAt) >= QUIET_AFTER_MS |
| 563 | ) { |
| 564 | quietOpenReadAt.set(readKey, nowMs); |
| 565 | } |
| 566 | const terminal = { |
| 567 | state: summary.state, |
| 568 | closedAt: summary.closedAt ?? null, |
| 569 | mergedAt: summary.mergedAt ?? null, |
| 570 | } satisfies SettlementPullRequest; |
| 571 | const cwd = lookupCwdByThreadId.get(thread.id); |
| 572 | if (summary.state !== "open" && thread.branch !== null && cwd !== undefined) { |
| 573 | if (!(yield* wouldSettle(group, terminal))) return undefined; |
| 574 | const current = yield* git.branchPullRequest( |
| 575 | { cwd, branch: thread.branch }, |
| 576 | { refresh: true }, |
| 577 | ); |
| 578 | const project = projects.get(thread.projectId); |
| 579 | if ( |
| 580 | current?.state === "open" && |
| 581 | project !== undefined && |
| 582 | pullRequestMatchesProject(current, project) |
| 583 | ) { |
| 584 | return current; |
| 585 | } |
| 586 | } |
| 587 | return terminal; |
| 588 | } |
| 589 | if (thread.branch === null) return null; |
| 590 | const cwd = lookupCwdByThreadId.get(thread.id); |
| 591 | if (cwd === undefined) { |
| 592 | return yield* Effect.die(new Error("thread project not found")); |
| 593 | } |
| 594 | const refresh = group.filter((member) => shellClosedThreads.delete(member.id)).length > 0; |
| 595 | const key = lookupKey(thread); |
| 596 | if (scheduledLookups !== null && !refresh && !scheduledLookups.has(key)) return undefined; |
| 597 | // Remembered git state is enough to rule a thread out; only an answer |
| 598 | // that would settle it is confirmed against fresh git state. |
| 599 | const remembered = yield* git.branchPullRequest( |
| 600 | { cwd, branch: thread.branch }, |
| 601 | { refresh, confirmTerminal: false }, |
| 602 | ); |
| 603 | noPullRequestRechecks.record(key, nowMs, remembered !== null); |
| 604 | if (refresh || remembered === null || remembered.state === "open") return remembered; |
| 605 | if (!(yield* wouldSettle(group, remembered))) return undefined; |
| 606 | return yield* git.branchPullRequest({ cwd, branch: thread.branch }, { freshGitState: true }); |
| 607 | }); |
| 608 | |
| 609 | // A merge, a shell close and a thread's own events read at once; only scheduled sweeps wait. |
| 610 | const scheduledLookups = |
| 611 | mergedPullRequest !== null || threadId !== undefined |
| 612 | ? null |
| 613 | : noPullRequestRechecks.select( |
| 614 | nowMs, |
| 615 | [...groups] |
| 616 | .filter(([key]) => key.startsWith('["branch",')) |
| 617 | .map(([key, group]) => ({ |
| 618 | key, |
| 619 | activityAtMs: Math.max(...group.map(threadActivityAtMs)), |
| 620 | })), |
| 621 | ); |
| 622 | |
| 623 | yield* Effect.forEach( |
| 624 | groups.values(), |
| 625 | (group) => |
| 626 | Effect.gen(function* () { |
| 627 | const pullRequest = yield* pullRequestFor(group); |
| 628 | if (pullRequest === undefined) return; |
| 629 | yield* Effect.forEach(group, (thread) => settleThread(thread, pullRequest), { |
| 630 | discard: true, |
| 631 | }); |
| 632 | }).pipe( |
| 633 | Effect.catchCauseIf( |
| 634 | (cause) => !Cause.hasInterruptsOnly(cause), |
| 635 | (cause) => |
| 636 | Effect.logWarning("automatic thread settlement skipped", { |
| 637 | threadIds: group.map((thread) => thread.id), |
| 638 | cause: Cause.pretty(cause), |
| 639 | }), |
| 640 | ), |
| 641 | ), |
| 642 | { concurrency: 8, discard: true }, |
| 643 | ); |
| 644 | if (threadId === undefined) noPullRequestRechecks.retain(groups); |
| 645 | }); |
| 646 | |
| 647 | const runSweep = ( |
| 648 | mergedPullRequest: PullRequestService.PullRequestMergeEvent | null, |
| 649 | threadId?: ThreadId, |
| 650 | ) => |
| 651 | sweep(mergedPullRequest, threadId).pipe( |
| 652 | Effect.provideService(GitVcsDriver.BackgroundGitWork, true), |
| 653 | Effect.catchCauseIf( |
| 654 | (cause) => !Cause.hasInterruptsOnly(cause), |
| 655 | (cause) => |
| 656 | Effect.logWarning("automatic thread settlement sweep failed", { |
| 657 | cause: Cause.pretty(cause), |
| 658 | }), |
| 659 | ), |
| 660 | ); |
| 661 | let fullSweepPending = false; |
| 662 | // A thread that just stopped must not wait behind a full sweep or another thread's host |
| 663 | // reads: clients hold it in Working only briefly, so a late settle makes it appear and then vanish. |
| 664 | const fullSweeps = yield* makeDrainableWorker(() => { |
| 665 | // Clear at the start so events during this sweep can request one follow-up. |
| 666 | fullSweepPending = false; |
| 667 | return runSweep(null); |
| 668 | }); |
| 669 | const enqueueFullSweep = () => |
| 670 | Effect.suspend(() => { |
| 671 | if (fullSweepPending) return Effect.void; |
| 672 | fullSweepPending = true; |
| 673 | return fullSweeps.enqueue(undefined); |
| 674 | }); |
| 675 | const threadSweeps = yield* FiberSet.make<void>(); |
| 676 | const threadSweepSlots = yield* Semaphore.make(8); |
| 677 | // One sweep per thread at a time; events during a running sweep coalesce into one more pass. |
| 678 | const threadSweepStates = new Map<ThreadId, "queued" | "running" | "rerun">(); |
| 679 | const sweepThread = ( |
| 680 | threadId: ThreadId, |
| 681 | ): Effect.Effect<void, Effect.Error<ReturnType<typeof runSweep>>> => |
| 682 | Effect.sync(() => threadSweepStates.set(threadId, "running")).pipe( |
| 683 | Effect.andThen(runSweep(null, threadId)), |
| 684 | threadSweepSlots.withPermits(1), |
| 685 | Effect.andThen( |
| 686 | Effect.suspend(() => { |
| 687 | if (threadSweepStates.get(threadId) === "rerun") return sweepThread(threadId); |
| 688 | threadSweepStates.delete(threadId); |
| 689 | return Effect.void; |
| 690 | }), |
| 691 | ), |
| 692 | ); |
| 693 | const enqueueThread = (threadId: ThreadId) => |
| 694 | Effect.suspend(() => { |
| 695 | const state = threadSweepStates.get(threadId); |
| 696 | if (state === "running") threadSweepStates.set(threadId, "rerun"); |
| 697 | if (state !== undefined) return Effect.void; |
| 698 | threadSweepStates.set(threadId, "queued"); |
| 699 | return sweepThread(threadId).pipe(FiberSet.run(threadSweeps), Effect.asVoid); |
| 700 | }); |
| 701 | |
| 702 | // Skip settle scripts in shared checkouts where other threads may still work. |
| 703 | const cleanUpSettledThread = Effect.fn("ThreadSettlementServiceV2.cleanUpSettledThread")( |
| 704 | function* (threadId: ThreadId) { |
| 705 | const settled = yield* projections.getThread(threadId); |
| 706 | if (settled.deletedAt !== null || settled.settledOverride !== "settled") return; |
| 707 | yield* terminals.closeOnSettle({ threadId }); |
| 708 | const worktreePath = settled.worktreePath; |
| 709 | if (worktreePath === null || !(yield* fileSystem.exists(worktreePath))) return; |
| 710 | // Recheck after closing and worktree I/O; only live, settled threads may run scripts. |
| 711 | const thread = yield* projections.getThread(threadId); |
| 712 | if (thread.deletedAt !== null || thread.settledOverride !== "settled") return; |
| 713 | const settledAtMs = toMillis(thread.settledAt); |
| 714 | if (settledAtMs === null || settleActionRunAt.get(threadId) === settledAtMs) return; |
| 715 | // The parent's worktree belongs to the parent; shared-worktree children |
| 716 | // skip duplicate cleanup there. |
| 717 | if ( |
| 718 | thread.lineage.relationshipToParent === "subagent" && |
| 719 | thread.lineage.parentThreadId !== null |
| 720 | ) { |
| 721 | const parent = yield* projections.getThread(thread.lineage.parentThreadId).pipe( |
| 722 | Effect.catchTags({ |
| 723 | ProjectionStoreThreadNotFoundError: () => Effect.succeed(null), |
| 724 | }), |
| 725 | ); |
| 726 | if (parent?.worktreePath === worktreePath) return; |
| 727 | } |
| 728 | const run = yield* projectScripts.runForThread({ |
| 729 | threadId, |
| 730 | projectId: thread.projectId, |
| 731 | worktreePath, |
| 732 | trigger: "settle", |
| 733 | observeCompletion: {}, |
| 734 | }); |
| 735 | settleActionRunAt.set(threadId, settledAtMs); |
| 736 | if (run.status === "started" && run.completion) { |
| 737 | yield* run.completion.pipe(Effect.forkDetach); |
| 738 | } |
| 739 | }, |
| 740 | (effect, threadId) => |
| 741 | effect.pipe( |
| 742 | Effect.catchCauseIf( |
| 743 | (cause) => !Cause.hasInterruptsOnly(cause), |
| 744 | (cause) => |
| 745 | Effect.logWarning("cleaning up a settled thread failed", { |
| 746 | threadId, |
| 747 | cause: Cause.pretty(cause), |
| 748 | }), |
| 749 | ), |
| 750 | ), |
| 751 | ); |
| 752 | |
| 753 | const processEvent = (event: OrchestrationV2DomainEvent) => { |
| 754 | switch (event.type) { |
| 755 | case "thread.settled": |
| 756 | return cleanUpSettledThread(event.threadId).pipe(Effect.andThen(enqueueFullSweep())); |
| 757 | case "thread.archived": |
| 758 | case "thread.deleted": |
| 759 | return enqueueFullSweep(); |
| 760 | case "thread.pull-request-link-updated": |
| 761 | case "thread.pull-request-synced": |
| 762 | case "thread.issue-synced": |
| 763 | case "provider-session.detached": |
| 764 | case "run.background-work-cancelled": |
| 765 | return enqueueThread(event.threadId); |
| 766 | // Sweep only when the work can have ended: progress updates cannot settle anything. |
| 767 | case "subagent.updated": |
| 768 | return isOrchestrationV2WorkActive(event.payload.status) |
| 769 | ? Effect.void |
| 770 | : enqueueThread(event.threadId); |
| 771 | case "provider-thread.updated": |
| 772 | return backgroundWorkHoldsCompletion(event.payload.pendingBackgroundTasks ?? []) |
| 773 | ? Effect.void |
| 774 | : enqueueThread(event.threadId); |
| 775 | case "turn-item.updated": { |
| 776 | const item = event.payload; |
| 777 | if ( |
| 778 | item.type === "command_execution" && |
| 779 | !isOrchestrationV2WorkActive(item.status) && |
| 780 | sourceControlDefinitions.definitions.some( |
| 781 | (definition) => definition.changeRequestCloseCommand?.test(item.input) === true, |
| 782 | ) |
| 783 | ) { |
| 784 | shellClosedThreads.add(event.threadId); |
| 785 | } |
| 786 | return turnItemUpdateCanEndBackgroundWork(item) |
| 787 | ? enqueueThread(event.threadId) |
| 788 | : Effect.void; |
| 789 | } |
| 790 | case "provider-session.updated": |
| 791 | return event.payload.status !== "starting" && event.payload.status !== "running" |
| 792 | ? enqueueThread(event.threadId) |
| 793 | : Effect.void; |
| 794 | case "run.updated": |
| 795 | return ["completed", "interrupted", "failed", "cancelled", "rolled_back"].includes( |
| 796 | event.payload.status, |
| 797 | ) |
| 798 | ? enqueueThread(event.threadId) |
| 799 | : Effect.void; |
| 800 | default: |
| 801 | return Effect.void; |
| 802 | } |
| 803 | }; |
| 804 | |
| 805 | const start: ThreadSettlementServiceV2["Service"]["start"] = Effect.fn( |
| 806 | "ThreadSettlementServiceV2.start", |
| 807 | )(function* () { |
| 808 | const settingsChanges = yield* settingsService.subscribeChanges; |
| 809 | const mergedPullRequests = yield* pullRequests.subscribeMerges; |
| 810 | const events = orchestrator.streamDomainEvents; |
| 811 | const initialSettings = yield* settingsService.getSettings.pipe(Effect.orDie); |
| 812 | let lastSettlementSettings = autoSettlementSettingsKey(initialSettings); |
| 813 | yield* forkParked( |
| 814 | Effect.gen(function* () { |
| 815 | yield* enqueueFullSweep(); |
| 816 | yield* fullSweeps.drain; |
| 817 | }).pipe(Effect.repeat(Schedule.spaced("1 minute")), Effect.asVoid), |
| 818 | ); |
| 819 | yield* forkParked( |
| 820 | Stream.runForEach(settingsChanges, (settings) => { |
| 821 | const key = autoSettlementSettingsKey(settings); |
| 822 | if (key === lastSettlementSettings) { |
| 823 | return Effect.void; |
| 824 | } |
| 825 | lastSettlementSettings = key; |
| 826 | return enqueueFullSweep(); |
| 827 | }), |
| 828 | ); |
| 829 | yield* forkParked(Stream.runForEach(mergedPullRequests, (event) => runSweep(event))); |
| 830 | yield* forkParked(Stream.runForEach(pullRequests.subscribeRefreshes, pauseQuietReads)); |
| 831 | yield* forkParked( |
| 832 | Stream.runForEach(events, processEvent).pipe( |
| 833 | Effect.catchCause((cause) => |
| 834 | Effect.logWarning("Thread settlement event stream failed", { cause }), |
| 835 | ), |
| 836 | ), |
| 837 | ); |
| 838 | }); |
| 839 | |
| 840 | return { |
| 841 | start, |
| 842 | drain: Effect.andThen(fullSweeps.drain, FiberSet.awaitEmpty(threadSweeps)), |
| 843 | } satisfies ThreadSettlementServiceV2["Service"]; |
| 844 | }); |