The best way to control Claude Code, Codex and more from any computer or phone

main
ts 17,137 lines 646 KB
Raw
Large file — syntax highlighting disabled.
1 import type {
2 OrchestrationV2ThreadLinkMetadata,
3 GuestShareId,
4 OrchestrationV2SearchThreadInput,
5 OrchestrationV2SearchThreadResult,
6 OrchestrationV2ThreadHistoryPage,
7 ProjectId,
8 } from "@t3tools/contracts";
9 import {
10 latestExecutedRun,
11 latestRootProviderFailure,
12 orchestrationV2WorktreePending,
13 runRanAfter,
14 usageLimitBlockedRun,
15 } from "@t3tools/shared/orchestrationV2ThreadError";
16 import { threadIssueKeysEqual } from "@t3tools/contracts";
17 import { canonicalizeGitCafePullRequestUrl } from "@t3tools/source-control-definitions/registry";
18 import { threadPullRequestsOf } from "../sourceControl/threadPullRequests.ts";
19 import {
20 normalizeThreadPullRequestKey,
21 visibleThreadPullRequests,
22 threadPullRequestKeysEqual,
23 legacyThreadPullRequestKey,
24 } from "../sourceControl/threadPullRequests.ts";
25 import {
26 ORCHESTRATION_V2_WORKSPACE_PREPARATION_FAILURE_CODE,
27 type ChatAttachment,
28 CommandId,
29 type GuestMessageId,
30 isCompactionCommandMessage,
31 isOrchestrationV2WorkActive,
32 isProviderNativeSubagentThread,
33 MessageId,
34 type NodeId,
35 type ModelSelection,
36 OrchestrationV2Command,
37 type OrchestrationV2InternalCommand,
38 type OrchestrationV2LimitRecoveryUpdate,
39 type OrchestrationV2Notification,
40 type OrchestrationV2ServerCommand,
41 type ThreadPullRequestLink,
42 type ThreadPullRequestWatch,
43 type PullRequestWatchUntil,
44 type OrchestrationV2AppThread,
45 type OrchestrationV2ContextHandoff,
46 type OrchestrationV2ContextSourcePoint,
47 type OrchestrationV2ContextTransfer,
48 type OrchestrationV2ContextTransferResolution,
49 type OrchestrationV2ConversationMessage,
50 type OrchestrationV2DelegatedCompletionCohort,
51 type OrchestrationV2DelegatedCompletionDelivery,
52 type OrchestrationV2DomainEvent,
53 type OrchestrationV2ExecutionNode,
54 type OrchestrationV2ProviderSession,
55 type OrchestrationV2ProviderThread,
56 type OrchestrationV2ProviderTurn,
57 type OrchestrationV2RemoteTaskChild,
58 type OrchestrationV2ModelSwitch,
59 type OrchestrationV2Run,
60 type OrchestrationV2RunAttempt,
61 type OrchestrationV2ThreadShell,
62 type OrchestrationV2ThreadShellSnapshot,
63 type OrchestrationV2StoredEvent,
64 type OrchestrationV2Subagent,
65 type OrchestrationV2ThreadProjection,
66 type OrchestrationV2TurnItem,
67 latestProviderTurnForAttempt,
68 parseGoalCommand,
69 orchestrationV2RunWorkStartedAt,
70 ProviderInstanceId,
71 PROVIDER_SEND_TURN_MAX_INPUT_CHARS,
72 ProviderInteractionMode,
73 isRuntimeModeAbove,
74 resolveProviderInstanceEnabled,
75 resolveProjectProviderInstanceEnabled,
76 type ProviderSessionId,
77 RunId,
78 RuntimeMode,
79 ThreadLinkedPullRequest,
80 ThreadId,
81 TurnItemId,
82 TrimmedNonEmptyString,
83 managedRuntimeModeCeilingMessage,
84 } from "@t3tools/contracts";
85 import * as KeyedLock from "@t3tools/shared/KeyedLock";
86 import { modelSelectionsEqual } from "@t3tools/shared/model";
87 import { expandAssistantCitationsForProvider } from "@t3tools/shared/assistantCitations";
88 import { modelSwitchSince } from "./modelSwitch.ts";
89 import { contextBoundaryOrdinal, isScheduledFreshRun } from "./contextBoundary.ts";
90 import { isPlaceholderThreadTitle } from "./placeholderThreadTitle.ts";
91 import { combinePullRequestWatchWakes, type PullRequestWatchWake } from "./pullRequestWatch.ts";
92 import { outputRequestProblem } from "./runOutput.ts";
93 import { recordThreadOpen } from "@t3tools/shared/paletteSearch";
94 import {
95 backgroundWorkHoldsCompletion,
96 derivePendingBackgroundWork,
97 hasPendingBackgroundSubagent,
98 pendingBackgroundTurnItems,
99 } from "@t3tools/shared/orchestrationV2PendingBackgroundWork";
100 import * as Context from "effect/Context";
101 import * as DateTime from "effect/DateTime";
102 import * as Effect from "effect/Effect";
103 import * as Exit from "effect/Exit";
104 import * as FileSystem from "effect/FileSystem";
105 import * as Path from "effect/Path";
106 import * as Layer from "effect/Layer";
107 import * as Option from "effect/Option";
108 import * as Ref from "effect/Ref";
109 import * as Result from "effect/Result";
110 import * as Schema from "effect/Schema";
111 import * as Stream from "effect/Stream";
112
113 import * as ProjectStore from "./ProjectStore.ts";
114 import {
115 isCheckpointRestoreIsolated,
116 SHARED_WORKSPACE_RESTORE_MESSAGE,
117 } from "./CheckpointRestoreSafety.ts";
118 import { CheckpointServiceV2 } from "./CheckpointService.ts";
119 import {
120 CommandPolicyV2,
121 followUpOwnerRun,
122 goalCommandStopsActiveRun,
123 isActiveRun,
124 isNativeMaintenanceCommand,
125 providerSwitchWaitsForTurn,
126 providerTurnCapabilities,
127 resolveMessageDispatchIntent,
128 } from "./CommandPolicy.ts";
129 import { CommandReceiptStoreV2, type CommandReceiptV2 } from "./CommandReceiptStore.ts";
130 import {
131 ContextHandoffServiceV2,
132 readForkHistoryItems,
133 readInheritedHandoffItems,
134 visibleConversationItems,
135 visibleDeltaRunOrdinals,
136 } from "./ContextHandoffService.ts";
137 import { notificationTurnItem } from "@t3tools/provider-core/server/notification";
138 import {
139 cancelledBeforeStart,
140 continuationFailedBeforePrompt,
141 continuationStartRetryDelayMs,
142 delegatedTaskAwaitsWork,
143 delegatedTaskResultRun,
144 delegatedTaskRuns,
145 isRestartContinuationSource,
146 runOutcome,
147 } from "./RestartBackgroundNote.ts";
148 import {
149 isBufferedProviderNotification,
150 isUndeliveredMailboxSteer,
151 } from "./NotificationMailbox.ts";
152 import { EventSinkV2 } from "./EventSink.ts";
153 import {
154 decideGuestMessageSend,
155 guestShareAfterCancelledSend,
156 releasedGuestAttachmentIds,
157 withoutGuestAttribution,
158 decideGuestShareCommand,
159 type GuestShareCommand,
160 } from "./GuestShareDecider.ts";
161 import * as EffectOutbox from "./EffectOutbox.ts";
162 import { withCommandMetrics } from "../observability/Metrics.ts";
163 import type { OrchestrationEffectRequestV2, PendingOrchestrationEffectV2 } from "./EffectOutbox.ts";
164 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
165 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
166 import {
167 DispatchModeLimit,
168 exceededDispatchModeLimit,
169 narrowToCeiling,
170 type RuntimeModeCeiling,
171 runtimeModeCeilingRefusal,
172 threadsStartingWork,
173 } from "./DispatchModeLimit.ts";
174 import {
175 applyToProjection,
176 emptyProjection,
177 threadShellFromProjection,
178 ProjectionStoreV2,
179 type ProjectionRecordField,
180 type ProjectionTimelinePage,
181 type ProjectionTimelinePageOptions,
182 type ProjectionRecordFilter,
183 type ProjectionRecords,
184 type ProjectionCheckpointContext,
185 type OlderSettledThreadsListing,
186 type ActiveShellIndexEntry,
187 type ActiveThreadListEntry,
188 type OlderSettledThreadsQuery,
189 type ShellSnapshotOptions,
190 } from "./ProjectionStore.ts";
191 import { ProviderAdapterRegistryV2 } from "./ProviderAdapterRegistry.ts";
192 import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
193 import { makeProviderFailure } from "@t3tools/provider-core/server/failure";
194 import * as RunExecutionService from "./RunExecutionService.ts";
195 import { ProviderSessionManagerV2 } from "./ProviderSessionManager.ts";
196 import { ProviderSwitchServiceV2 } from "./ProviderSwitchService.ts";
197 import { isAutomaticCompletionRun, queuedRunsInDeliveryOrder } from "./QueuedRunOrder.ts";
198 import { isRuntimePolicyAboveCeilingError, RuntimePolicyV2 } from "./RuntimePolicy.ts";
199 import {
200 endOrphanedNativeSubagent,
201 endRunlessRootTurns,
202 makeSubagentChildThread,
203 subagentResultForRun,
204 delegatedTaskProgress,
205 delegatedTaskReportRuns,
206 monitorUpdateRunIds,
207 subagentThreadTitle,
208 } from "@t3tools/provider-core/server/subagentProjection";
209 import {
210 forkableSourceRunStatusError,
211 isForkableSourceRunStatus,
212 ThreadForkServiceV2,
213 } from "./ThreadForkService.ts";
214 import { planThreadDeletion } from "./ThreadDeletion.ts";
215 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
216 import * as ServerSettings from "../serverSettings.ts";
217
218 const isMessageQuestionAnswer = Schema.is(
219 Schema.Union([TrimmedNonEmptyString, Schema.NonEmptyArray(TrimmedNonEmptyString)]),
220 );
221
222 export class OrchestratorDispatchError extends Schema.TaggedError<OrchestratorDispatchError>()(
223 "OrchestratorDispatchError",
224 {
225 commandId: CommandId,
226 commandType: Schema.String,
227 cause: Schema.optional(Schema.Defect()),
228 },
229 ) {
230 override get message(): string {
231 return `Failed to dispatch orchestration command ${this.commandType} (${this.commandId}).`;
232 }
233 }
234
235 export class OrchestratorThreadAlreadyExistsError extends Schema.TaggedError<OrchestratorThreadAlreadyExistsError>()(
236 "OrchestratorThreadAlreadyExistsError",
237 { commandId: CommandId, threadId: ThreadId },
238 ) {
239 override get message(): string {
240 return `Thread ${this.threadId} already exists. Use a new thread id.`;
241 }
242 }
243
244 export class OrchestratorCommandRejectedError extends Schema.TaggedError<OrchestratorCommandRejectedError>()(
245 "OrchestratorCommandRejectedError",
246 { commandId: CommandId, commandType: Schema.String, cause: Schema.optional(Schema.Defect()) },
247 ) {
248 override get message(): string {
249 return `Orchestration command ${this.commandType} (${this.commandId}) was rejected before commit.`;
250 }
251 }
252
253 export class OrchestratorProjectionError extends Schema.TaggedError<OrchestratorProjectionError>()(
254 "OrchestratorProjectionError",
255 {
256 threadId: ThreadId,
257 cause: Schema.optional(Schema.Defect()),
258 },
259 ) {
260 override get message(): string {
261 return `Failed to load orchestration projection for thread ${this.threadId}.`;
262 }
263 }
264
265 const isOrchestratorProjectionError = Schema.is(OrchestratorProjectionError);
266
267 export class OrchestratorDomainEventStreamError extends Schema.TaggedError<OrchestratorDomainEventStreamError>()(
268 "OrchestratorDomainEventStreamError",
269 {
270 cause: Schema.optional(Schema.Defect()),
271 },
272 ) {
273 override get message(): string {
274 return "Failed while streaming orchestration domain events.";
275 }
276 }
277
278 export class OrchestratorProviderAdapterError extends Schema.TaggedError<OrchestratorProviderAdapterError>()(
279 "OrchestratorProviderAdapterError",
280 {
281 commandId: CommandId,
282 providerInstanceId: ProviderInstanceId,
283 cause: Schema.optional(Schema.Defect()),
284 },
285 ) {
286 override get message(): string {
287 return `Provider adapter failed while dispatching orchestration command ${this.commandId}.`;
288 }
289 }
290
291 export class OrchestratorSubagentThreadReadOnlyError extends Schema.TaggedError<OrchestratorSubagentThreadReadOnlyError>()(
292 "OrchestratorSubagentThreadReadOnlyError",
293 { commandId: CommandId, threadId: ThreadId },
294 ) {
295 override get message(): string {
296 return "This subagent is run by its provider and cannot take messages. Message the parent thread instead.";
297 }
298 }
299
300 export class OrchestratorThreadAboveModeLimitError extends Schema.TaggedError<OrchestratorThreadAboveModeLimitError>()(
301 "OrchestratorThreadAboveModeLimitError",
302 {
303 commandId: CommandId,
304 threadId: ThreadId,
305 mode: Schema.Literals(["runtime", "interaction"]),
306 runtimeMode: RuntimeMode,
307 interactionMode: ProviderInteractionMode,
308 },
309 ) {
310 override get message(): string {
311 return `Thread ${this.threadId} now runs in ${this.runtimeMode}/${this.interactionMode} mode, above what this caller may change.`;
312 }
313 }
314
315 export class OrchestratorProviderSelectionBusyError extends Schema.TaggedError<OrchestratorProviderSelectionBusyError>()(
316 "OrchestratorProviderSelectionBusyError",
317 { commandId: CommandId, threadId: ThreadId },
318 ) {
319 override get message(): string {
320 return `Thread ${this.threadId} has an active turn. Wait for it to finish, then retry the provider selection change.`;
321 }
322 }
323
324 const isOrchestratorDispatchError = Schema.is(OrchestratorDispatchError);
325
326 /** The ceiling refusal under however many dispatch errors wrapped it on the way out. */
327 function runtimePolicyCeilingCause(error: unknown) {
328 let current = error;
329 while (isOrchestratorDispatchError(current)) current = current.cause;
330 return isRuntimePolicyAboveCeilingError(current) ? current : undefined;
331 }
332
333 export class OrchestratorRuntimeModeAboveCeilingError extends Schema.TaggedError<OrchestratorRuntimeModeAboveCeilingError>()(
334 "OrchestratorRuntimeModeAboveCeilingError",
335 {
336 commandId: CommandId,
337 threadId: ThreadId,
338 refused: Schema.Literals(["mode", "turn"]),
339 runtimeMode: RuntimeMode,
340 maxRuntimeMode: RuntimeMode,
341 organization: Schema.String,
342 },
343 ) {
344 override get message(): string {
345 const limit = managedRuntimeModeCeilingMessage(this.organization, this.maxRuntimeMode);
346 return this.refused === "turn" ? `${limit} Switch this thread's mode to continue.` : limit;
347 }
348 }
349
350 export class OrchestratorCommandPreviouslyRejectedError extends Schema.TaggedError<OrchestratorCommandPreviouslyRejectedError>()(
351 "OrchestratorCommandPreviouslyRejectedError",
352 {
353 commandId: CommandId,
354 commandType: Schema.String,
355 detail: Schema.String,
356 },
357 ) {
358 override get message(): string {
359 return `Command ${this.commandId} was previously rejected: ${this.detail}`;
360 }
361 }
362
363 export class OrchestratorCommandIdConflictError extends Schema.TaggedError<OrchestratorCommandIdConflictError>()(
364 "OrchestratorCommandIdConflictError",
365 {
366 commandId: CommandId,
367 commandType: Schema.String,
368 receiptThreadId: ThreadId,
369 commandThreadId: ThreadId,
370 },
371 ) {
372 override get message(): string {
373 return `Command ${this.commandId} was already handled for thread ${this.receiptThreadId} and cannot be replayed for ${this.commandThreadId}.`;
374 }
375 }
376
377 export const OrchestratorV2Error = Schema.Union([
378 OrchestratorDispatchError,
379 OrchestratorThreadAlreadyExistsError,
380 OrchestratorCommandRejectedError,
381 OrchestratorProjectionError,
382 OrchestratorDomainEventStreamError,
383 OrchestratorProviderAdapterError,
384 OrchestratorCommandPreviouslyRejectedError,
385 OrchestratorCommandIdConflictError,
386 OrchestratorSubagentThreadReadOnlyError,
387 OrchestratorThreadAboveModeLimitError,
388 OrchestratorProviderSelectionBusyError,
389 OrchestratorRuntimeModeAboveCeilingError,
390 ]);
391 export type OrchestratorV2Error = typeof OrchestratorV2Error.Type;
392
393 export interface OrchestratorV2DispatchResult {
394 readonly sequence: number;
395 readonly storedEvents: ReadonlyArray<OrchestrationV2StoredEvent>;
396 /** The command id was already accepted; analytics skip it. */
397 readonly replayed?: true;
398 }
399
400 export interface OrchestratorV2Shape {
401 readonly resumeQueuedRuns: Effect.Effect<number, OrchestratorV2Error>;
402 readonly recoverDelegatedTasks: Effect.Effect<void>;
403 readonly recoverDelegatedTask: (
404 threadId: ThreadId,
405 options?: { readonly sourceRunId?: RunId; readonly completedRunId?: RunId },
406 ) => Effect.Effect<void, OrchestratorProjectionError>;
407 /** Commits cancellation with the child's Stop after descendants accept stopping. */
408 readonly cancelDelegatedTask: <E, R>(input: {
409 readonly parentThreadId: ThreadId;
410 readonly taskId: NodeId;
411 readonly command: Extract<OrchestrationV2ServerCommand, { readonly type: "thread.stop" }>;
412 readonly stopDescendants: (
413 dispatch: OrchestratorV2Shape["dispatch"],
414 ) => Effect.Effect<void, E, R>;
415 }) => Effect.Effect<void, E | OrchestratorV2Error, R>;
416 readonly delegatedTaskResultPending: (
417 childThreadId: ThreadId,
418 taskId?: NodeId,
419 ) => Effect.Effect<boolean, OrchestratorProjectionError>;
420 readonly restartContinuationPending: (run: OrchestrationV2Run) => Effect.Effect<boolean>;
421 /** Declined continuations emit no thread event; this still wakes waiters. */
422 readonly awaitRestartContinuation: (run: OrchestrationV2Run) => Effect.Effect<void>;
423 readonly dispatch: (
424 command: OrchestrationV2ServerCommand,
425 ) => Effect.Effect<OrchestratorV2DispatchResult, OrchestratorV2Error>;
426 readonly autoArmLimitRecovery: (input: {
427 readonly commandId: CommandId;
428 readonly threadId: ThreadId;
429 readonly limitRecovery: OrchestrationV2LimitRecoveryUpdate;
430 }) => Effect.Effect<void, OrchestratorV2Error>;
431 readonly searchThreadStream: (
432 input: OrchestrationV2SearchThreadInput,
433 ) => Stream.Stream<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
434 readonly searchThread: (
435 input: OrchestrationV2SearchThreadInput,
436 ) => Effect.Effect<OrchestrationV2SearchThreadResult, OrchestratorV2Error>;
437 readonly getThreadHistoryPage: (
438 threadId: ThreadId,
439 cursor: string,
440 throughEntryId?: string,
441 conversationOnly?: boolean,
442 workMode?: "deferred" | "work",
443 ) => Effect.Effect<OrchestrationV2ThreadHistoryPage, OrchestratorV2Error>;
444 readonly getTimelinePage: (
445 threadId: ThreadId,
446 options: ProjectionTimelinePageOptions,
447 ) => Effect.Effect<ProjectionTimelinePage, OrchestratorProjectionError>;
448 readonly getMessageCount: (threadId: ThreadId) => Effect.Effect<number, OrchestratorV2Error>;
449 readonly getRunCount: (threadId: ThreadId) => Effect.Effect<number, OrchestratorV2Error>;
450 readonly getMessagesById: (
451 threadId: ThreadId,
452 messageIds: ReadonlyArray<MessageId>,
453 ) => Effect.Effect<
454 ReadonlyMap<MessageId, OrchestrationV2ConversationMessage>,
455 OrchestratorV2Error
456 >;
457 readonly getTranscriptByteFloor: (
458 threadId: ThreadId,
459 limit: number,
460 ) => Effect.Effect<number, OrchestratorV2Error>;
461 readonly getTurnItem: (input: {
462 readonly threadId: ThreadId;
463 readonly itemId: TurnItemId;
464 }) => Effect.Effect<OrchestrationV2TurnItem | null, OrchestratorV2Error>;
465 readonly getThreadRecords: <K extends ProjectionRecordField>(
466 threadId: ThreadId,
467 fields: ReadonlyArray<K>,
468 filter?: ProjectionRecordFilter,
469 ) => Effect.Effect<ProjectionRecords<K>, OrchestratorV2Error>;
470 readonly listAppOwnedChildThreadIds: (
471 threadId: ThreadId,
472 ) => Effect.Effect<ReadonlyArray<ThreadId>, OrchestratorV2Error>;
473 readonly getThreadProjection: (
474 threadId: ThreadId,
475 ) => Effect.Effect<OrchestrationV2ThreadProjection, OrchestratorV2Error>;
476 readonly getCheckpointContext: (
477 threadId: ThreadId,
478 ) => Effect.Effect<ProjectionCheckpointContext, OrchestratorV2Error>;
479 readonly getThreadSnapshot: (threadId: ThreadId) => Effect.Effect<
480 {
481 readonly schemaVersion: number;
482 readonly snapshotSequence: number;
483 readonly projection: OrchestrationV2ThreadProjection;
484 },
485 OrchestratorV2Error
486 >;
487 readonly getThreadSnapshotWindow: (
488 threadId: ThreadId,
489 options: Parameters<ProjectionStoreV2["Service"]["getThreadSnapshotWindow"]>[1],
490 ) => Effect.Effect<
491 {
492 readonly schemaVersion: number;
493 readonly snapshotSequence: number;
494 readonly projection: OrchestrationV2ThreadProjection;
495 },
496 OrchestratorV2Error
497 >;
498 readonly getPullRequestSnapshot: (
499 options: Parameters<ProjectionStoreV2["Service"]["getPullRequestSnapshot"]>[0],
500 ) => Effect.Effect<
501 Effect.Success<ReturnType<ProjectionStoreV2["Service"]["getPullRequestSnapshot"]>>,
502 OrchestratorV2Error
503 >;
504 readonly getAgentAwarenessSnapshot: (
505 threadId?: ThreadId,
506 ) => Effect.Effect<
507 Effect.Success<ReturnType<ProjectionStoreV2["Service"]["getAgentAwarenessSnapshot"]>>,
508 OrchestratorV2Error
509 >;
510 readonly getShellSnapshot: (
511 options?: ShellSnapshotOptions,
512 ) => Effect.Effect<OrchestrationV2ThreadShellSnapshot, OrchestratorV2Error>;
513 readonly listActiveSubagentThreadIds: () => Effect.Effect<
514 ReadonlyArray<ThreadId>,
515 OrchestratorV2Error
516 >;
517 readonly countThreads: () => Effect.Effect<number, OrchestratorV2Error>;
518 readonly listActiveShellIndex: () => Effect.Effect<
519 ReadonlyArray<ActiveShellIndexEntry>,
520 OrchestratorV2Error
521 >;
522 readonly listActiveThreads: (
523 projectId?: ProjectId,
524 ) => Effect.Effect<ReadonlyArray<ActiveThreadListEntry>, OrchestratorV2Error>;
525 readonly listOlderSettledThreads: (
526 query: OlderSettledThreadsQuery,
527 ) => Effect.Effect<OlderSettledThreadsListing, OrchestratorV2Error>;
528 readonly readShellSnapshot: (
529 options?: ShellSnapshotOptions,
530 ) => Effect.Effect<
531 Effect.Effect<OrchestrationV2ThreadShellSnapshot, OrchestratorV2Error>,
532 OrchestratorV2Error
533 >;
534 readonly getThreadShell: (
535 threadId: ThreadId,
536 ) => Effect.Effect<OrchestrationV2ThreadShell | null, OrchestratorV2Error>;
537 readonly findGuestShareThreadId: (
538 shareId: GuestShareId,
539 ) => Effect.Effect<ThreadId | null, OrchestratorV2Error>;
540 /** Titles of the given threads that exist, archived ones included. */
541 readonly getThreadLinkMetadata: (
542 threadIds: ReadonlyArray<ThreadId>,
543 ) => Effect.Effect<Record<ThreadId, OrchestrationV2ThreadLinkMetadata>, OrchestratorV2Error>;
544 readonly getThreadTitles: (
545 threadIds: ReadonlyArray<ThreadId>,
546 ) => Effect.Effect<Record<ThreadId, string>, OrchestratorV2Error>;
547 readonly getThreadEventSequence: (
548 threadId: ThreadId,
549 ) => Effect.Effect<number, OrchestratorV2Error>;
550 readonly streamStoredEvents: Stream.Stream<OrchestrationV2StoredEvent, OrchestratorV2Error>;
551 /** Events arrive projected for the wire and shared by every subscriber; never mutate them. */
552 readonly streamStoredEventsFrom: (input?: {
553 readonly threadId?: ThreadId;
554 readonly afterSequence?: number;
555 readonly eventType?: OrchestrationV2DomainEvent["type"];
556 }) => Stream.Stream<OrchestrationV2StoredEvent, OrchestratorV2Error>;
557 readonly streamDomainEvents: Stream.Stream<OrchestrationV2DomainEvent, OrchestratorV2Error>;
558 }
559
560 export class OrchestratorV2 extends Context.Service<OrchestratorV2, OrchestratorV2Shape>()(
561 "t3/orchestration-v2/Orchestrator/OrchestratorV2",
562 ) {}
563
564 /** Run ordinals can skip values: a queued run moves past newer runs when it starts. */
565 function nextRunOrdinal(projection: Pick<OrchestrationV2ThreadProjection, "runs">): number {
566 return projection.runs.reduce((max, run) => Math.max(max, run.ordinal), 0) + 1;
567 }
568
569 /**
570 * Stamp at wake start; queued prompts lack start times and delegated results jump the queue.
571 * Auto-drained agent messages continue active work; explicit queue starts reset it.
572 */
573 function wakeWorkStartedAt(
574 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "subagents">,
575 trigger: Pick<
576 OrchestrationV2ConversationMessage,
577 "createdBy" | "creationSource" | "senderThreadId"
578 > & {
579 readonly notification?: unknown;
580 readonly delegatedCompletion?: unknown;
581 readonly restartContinuationOfRunId?: RunId | undefined;
582 },
583 delivery: "immediate" | "automatic-queue" | "explicit-queue" = "immediate",
584 ): Pick<OrchestrationV2Run, "workStartedAt"> {
585 if (
586 trigger.notification === undefined &&
587 trigger.delegatedCompletion === undefined &&
588 trigger.restartContinuationOfRunId === undefined &&
589 !(
590 trigger.createdBy === "agent" &&
591 trigger.creationSource === "mcp" &&
592 trigger.senderThreadId !== undefined &&
593 (delivery === "automatic-queue" ||
594 (delivery === "immediate" &&
595 projection.subagents.some(
596 (subagent) =>
597 subagent.origin === "app_owned" && subagent.childThreadId === trigger.senderThreadId,
598 )))
599 )
600 ) {
601 return {};
602 }
603 const previous = projection.runs
604 .flatMap((run) => (run.startedAt === null ? [] : [{ run, startedAt: run.startedAt }]))
605 .toSorted(
606 (left, right) =>
607 DateTime.Order(right.startedAt, left.startedAt) || right.run.ordinal - left.run.ordinal,
608 )[0]?.run;
609 return previous === undefined ? {} : { workStartedAt: orchestrationV2RunWorkStartedAt(previous) };
610 }
611
612 const threadPullRequestLinksEqual = Schema.toEquivalence(Schema.NullOr(ThreadLinkedPullRequest));
613
614 /** Unfinished runs (including queued) or open requests block moving a thread to another project. */
615 export function threadHasUnfinishedWork(records: {
616 readonly runs: ReadonlyArray<{ readonly status: string }>;
617 readonly runtimeRequests: ReadonlyArray<{ readonly status: string }>;
618 }): boolean {
619 return (
620 records.runs.some(
621 (run) =>
622 run.status !== "completed" &&
623 run.status !== "failed" &&
624 run.status !== "cancelled" &&
625 run.status !== "interrupted" &&
626 run.status !== "rolled_back",
627 ) || records.runtimeRequests.some((request) => request.status === "pending")
628 );
629 }
630
631 function commandThreadId(command: OrchestrationV2ServerCommand): ThreadId {
632 switch (command.type) {
633 case "thread.create":
634 case "thread.archive":
635 case "thread.unarchive":
636 case "thread.delete":
637 case "thread.guest-share.open":
638 case "thread.guest-share.redeem":
639 case "thread.guest-share.close":
640 case "thread.guest-message.submit":
641 case "thread.guest-message.dismiss":
642 case "thread.settle":
643 case "thread.auto-settle":
644 case "thread.unsettle":
645 case "thread.snooze":
646 case "thread.unsnooze":
647 case "thread.auto-settle.set":
648 case "thread.pin":
649 case "thread.unpin":
650 case "thread.pin.reorder":
651 case "thread.active.reorder":
652 case "thread.tag.set":
653 case "thread.visit":
654 case "thread.mark-unread":
655 case "thread.metadata.update":
656 case "thread.pull-request.link":
657 case "thread.pull-request.unlink":
658 case "thread.pull-request-link.sync":
659 case "thread.issue.link":
660 case "thread.issue.unlink":
661 case "thread.pull-request.watch":
662 case "thread.pull-request-watch.sync":
663 case "thread.pull-request-watch.checks-reopen":
664 case "thread.pull-request.sync":
665 case "thread.title.regeneration.complete":
666 case "thread.project.adopt-folder":
667 case "thread.runtime-mode.set":
668 case "thread.interaction-mode.set":
669 case "thread.model-selection.set":
670 case "provider-session.detach":
671 case "message.dispatch":
672 case "notification.delivery.accept":
673 case "prepared-run.release":
674 case "prepared-run.progress":
675 case "prepared-run.fail":
676 case "prepared-run.retry":
677 case "subagent.stop":
678 case "run.interrupt":
679 case "run.output.submit":
680 case "queued-message.promote-to-steer":
681 case "queued-run.start":
682 case "queue.resume":
683 case "queued-run.reorder":
684 case "queued-run.cancel":
685 case "queued-run.edit":
686 case "runtime-request.respond":
687 case "runtime-request.delivery.fail":
688 case "thread.user-input.dismiss":
689 case "checkpoint.rollback":
690 case "checkpoint.rollback.fail":
691 case "thread.background-work.settle":
692 case "thread.interrupt.unresponsive":
693 case "thread.stop":
694 case "provider.switch":
695 return command.threadId;
696 case "delegated_task.request":
697 case "delegated_task.remote.request":
698 case "delegated_task.remote.complete":
699 case "delegated_task.remote.stop":
700 case "delegated_task.wake-policy":
701 case "delegated_task.completion-delivery.acknowledge":
702 case "delegated_task.completion-delivery.dispose":
703 case "thread.created.record":
704 return command.parentThreadId;
705 case "secret_request.record":
706 case "link_request.record":
707 return command.threadId;
708 case "thread.fork":
709 case "thread.merge_back":
710 return command.targetThreadId;
711 }
712 }
713
714 // Validate before resolving the request: adapters may otherwise coerce unusable answers.
715 const decodeUserInputAnswers = Schema.decodeUnknownEffect(
716 Schema.Record(
717 Schema.String,
718 Schema.Union([
719 Schema.String,
720 Schema.Array(Schema.String),
721 Schema.Struct({ answers: Schema.Array(Schema.String) }),
722 ]),
723 ),
724 );
725
726 function pendingThreadTitleGenerationEffect(
727 commandId: CommandId,
728 threadId: ThreadId,
729 kind:
730 | { readonly type: "initial"; readonly messageId: MessageId }
731 | { readonly type: "regenerate" },
732 ): PendingOrchestrationEffectV2 {
733 return {
734 id: `effect:${commandId}:thread-title.generate`,
735 commandId,
736 threadId,
737 request: { type: "thread-title.generate", kind },
738 };
739 }
740
741 // Clients seed a new thread with its first message, and empty threads start as "New thread" or
742 // an agent's placeholder. Any other title on a thread without messages was chosen on purpose.
743 // A fork or side chat starts with a title its first message replaces, so only a rename keeps it.
744 function hasTitleSetBeforeFirstMessage(
745 thread: OrchestrationV2AppThread,
746 titleSeed: string | undefined,
747 ): boolean {
748 if (thread.titleRenamed === true) return true;
749 const title = thread.title.trim();
750 return (
751 thread.forkedFrom === null &&
752 thread.lineage.sideChat !== true &&
753 title.length > 0 &&
754 title !== "New thread" &&
755 !isPlaceholderThreadTitle(title) &&
756 title !== titleSeed?.trim()
757 );
758 }
759
760 function delegatedTasksStopEffect(command: {
761 readonly commandId: CommandId;
762 readonly threadId: ThreadId;
763 readonly reason?: string | undefined;
764 }): PendingOrchestrationEffectV2 {
765 return {
766 id: `effect:${command.commandId}:delegated-tasks.stop`,
767 commandId: command.commandId,
768 threadId: command.threadId,
769 request: {
770 type: "delegated-tasks.stop",
771 ...(command.reason === undefined ? {} : { reason: command.reason }),
772 },
773 };
774 }
775
776 const WORKSPACE_PREPARATION_INPUT = "Preparing workspace";
777 const RUNTIME_REQUEST_DELIVERY_FAILED_MESSAGE =
778 "T3 Code could not confirm that your answer reached the agent. If the run stays stuck, stop it and try again.";
779
780 function withoutPreparationResult(
781 item: Extract<OrchestrationV2TurnItem, { readonly type: "command_execution" }>,
782 ) {
783 const { output: _output, exitCode: _exitCode, outputIndicatesFailure: _failure, ...rest } = item;
784 return rest;
785 }
786
787 /** Stop ended this task, so its result is not delivered to the parent agent. */
788 function cancelledAppOwnedTask(
789 task: OrchestrationV2Subagent,
790 now: DateTime.Utc,
791 ): OrchestrationV2Subagent {
792 return {
793 ...task,
794 status: "cancelled",
795 completedAt: now,
796 updatedAt: now,
797 completionDelivery: { state: "disposed", observedByRunId: null },
798 };
799 }
800
801 function askedForLinkedMachine(
802 card: Extract<OrchestrationV2TurnItem, { readonly type: "link_request" }>,
803 linked: Extract<OrchestrationV2InternalCommand, { readonly type: "link_request.record" }>,
804 ): boolean {
805 if (card.requestedAccess !== linked.requestedAccess) return false;
806 if (card.environmentId !== undefined) return card.environmentId === linked.linkedEnvironmentId;
807 if (card.url !== undefined) return linked.url !== undefined && card.url === linked.url;
808 const normalize = (text: string | undefined) => text?.trim().toLowerCase() ?? "";
809 const hint = normalize(card.hint);
810 return hint !== "" && hint === normalize(linked.hint) && hint === normalize(linked.linkedLabel);
811 }
812
813 // A workspace change detaches the session, which can settle the calling run at once. Queuing its
814 // follow-up in a later command would let a delegated task report its result before that exists.
815 function metadataUpdateContinuation(
816 command: Extract<OrchestrationV2Command, { readonly type: "thread.metadata.update" }>,
817 ): Extract<OrchestrationV2Command, { readonly type: "message.dispatch" }> | undefined {
818 return command.continuation === undefined
819 ? undefined
820 : {
821 type: "message.dispatch",
822 commandId: command.commandId,
823 threadId: command.threadId,
824 messageId: command.continuation.messageId,
825 text: command.continuation.text,
826 attachments: [],
827 dispatchMode: { type: "queue_after_active" },
828 createdBy: "agent",
829 creationSource: "mcp",
830 };
831 }
832
833 // A later command for the first message would leave the new thread committed with no run,
834 // and every client would show it idle until that command landed.
835 function threadCreateInitialMessage(
836 command: Extract<OrchestrationV2Command, { readonly type: "thread.create" }>,
837 ): Extract<OrchestrationV2Command, { readonly type: "message.dispatch" }> | undefined {
838 if (command.initialMessage === undefined) return undefined;
839 const { workspaceStrategy, ...message } = command.initialMessage;
840 return {
841 ...message,
842 type: "message.dispatch",
843 commandId: command.commandId,
844 threadId: command.threadId,
845 modelSelection: command.modelSelection,
846 dispatchMode:
847 workspaceStrategy === undefined
848 ? { type: "start_immediately" }
849 : { type: "defer_start", workspaceStrategy },
850 createdBy: command.createdBy,
851 creationSource: command.creationSource,
852 };
853 }
854
855 // Provider wakes can start follow-up work; monitor updates only report progress.
856 // Dispatch takes the parent's lock for follow-ups because intake writes the parent.
857 function mayStartSubagentFollowup(
858 command: Extract<OrchestrationV2Command, { readonly type: "message.dispatch" }>,
859 ): boolean {
860 if (command.scheduledTaskId !== undefined || command.restartContinuationOfRunId !== undefined)
861 return false;
862 if (isCompactionCommandMessage({ role: "user", ...command })) return false;
863 if (command.notification !== undefined)
864 return (
865 command.notification.source.kind !== "monitor" || command.notification.outcome !== "updated"
866 );
867 return (
868 command.createdBy === "user" ||
869 command.creationSource === "mcp" ||
870 command.usageLimitContinuationOfRunId !== undefined
871 );
872 }
873
874 /** Waiting runs only drain post-terminal output; their agent turn is over and still needs a wake. */
875 function isLiveRun(run: OrchestrationV2Run): boolean {
876 return run.status === "preparing" || run.status === "starting" || run.status === "running";
877 }
878
879 function hasLiveRun(projection: Pick<OrchestrationV2ThreadProjection, "runs">): boolean {
880 return projection.runs.some(isLiveRun);
881 }
882
883 function withPullRequestWatch(
884 link: ThreadPullRequestLink,
885 watch: ThreadPullRequestWatch | undefined,
886 ): ThreadPullRequestLink {
887 const { watch: previous, endedWatch, ...rest } = link;
888 if (watch !== undefined) return { ...rest, watch };
889 const ended = previous ?? endedWatch;
890 return ended === undefined ? rest : { ...rest, endedWatch: ended };
891 }
892
893 function syncedPullRequestWatch(
894 existing: ThreadPullRequestWatch | undefined,
895 command: {
896 readonly watch: ThreadPullRequestWatch | null;
897 readonly checksPendingBefore?: boolean | undefined;
898 },
899 ): ThreadPullRequestWatch | undefined {
900 if (command.watch === null) return undefined;
901 // Only the agent sets `until`, so a read in flight keeps what the watch has now.
902 const watch = withUntil(command.watch, existing?.until ?? "checks");
903 // A push cleared the answer while this read was in flight, so its answer is the old head's.
904 if (command.checksPendingBefore === undefined || existing?.checksPending !== undefined) {
905 return watch;
906 }
907 return withoutChecksAnswer(watch);
908 }
909
910 function withUntil(
911 watch: ThreadPullRequestWatch,
912 until: PullRequestWatchUntil | undefined,
913 ): ThreadPullRequestWatch {
914 if (until === undefined || until === (watch.until ?? "checks")) return watch;
915 if (until === "merged") return { ...watch, until };
916 const { until: _checks, ...rest } = watch;
917 return rest;
918 }
919
920 const withoutChecksAnswer = ({
921 checksPending: _answer,
922 ...watch
923 }: ThreadPullRequestWatch): ThreadPullRequestWatch => watch;
924
925 function legacyPullRequestLink(
926 thread: OrchestrationV2AppThread,
927 linked: ThreadLinkedPullRequest,
928 now: DateTime.Utc,
929 ): ThreadPullRequestLink {
930 const key = legacyThreadPullRequestKey(linked);
931 const existing = threadPullRequestsOf(thread).find((link) =>
932 threadPullRequestKeysEqual(link, key),
933 );
934 return withPullRequestWatch(
935 {
936 ...key,
937 url: canonicalizeGitCafePullRequestUrl(linked.url),
938 source: "manual",
939 linkedAt: DateTime.formatIso(now),
940 snapshot: null,
941 stack: null,
942 ...(existing?.repositoryWebUrl !== undefined && {
943 repositoryWebUrl: existing.repositoryWebUrl,
944 }),
945 ...(existing?.endedWatch && { endedWatch: existing.endedWatch }),
946 },
947 existing?.watch,
948 );
949 }
950
951 const DELEGATED_RESULT_INLINE_BYTES = 16_000;
952
953 function delegatedCompletionWakeDetail(
954 taskIds: ReadonlyArray<string>,
955 tasks: ReadonlyArray<OrchestrationV2Subagent>,
956 ): string {
957 let remainingBytes = DELEGATED_RESULT_INLINE_BYTES;
958 return taskIds
959 .map((taskId) => {
960 const task = tasks.find((candidate) => candidate.id === taskId);
961 // A follow-up gets a fresh task id; the child thread links it to the task delegate_task returned.
962 const thread = task?.childThreadId == null ? "" : ` in child thread ${task.childThreadId}`;
963 const label = `Delegated task ${taskId}${task?.title == null ? "" : ` (${task.title})`}${thread}${task === undefined ? " reached a terminal state" : ` [${task.status}]`}`;
964 if (task?.result != null) {
965 const detail = `${label}:\n${task.result}`;
966 const bytes = Buffer.byteLength(detail);
967 if (bytes <= remainingBytes) {
968 remainingBytes -= bytes;
969 return detail;
970 }
971 }
972 return `${label}. Use task_status with taskId ${taskId} to read the result.`;
973 })
974 .join("\n\n");
975 }
976
977 function isTerminalDelegatedTaskStatus(status: OrchestrationV2Subagent["status"]): boolean {
978 return (
979 status === "completed" ||
980 status === "failed" ||
981 status === "cancelled" ||
982 status === "interrupted"
983 );
984 }
985
986 function delegatedTaskTerminalStatus(
987 status: OrchestrationV2Run["status"],
988 ): OrchestrationV2Subagent["status"] | null {
989 switch (status) {
990 case "completed":
991 case "failed":
992 case "cancelled":
993 case "interrupted":
994 return status;
995 case "rolled_back":
996 return "cancelled";
997 case "preparing":
998 case "queued":
999 case "starting":
1000 case "running":
1001 case "waiting":
1002 return null;
1003 }
1004 }
1005
1006 /** Failed worktree setup keeps the queue held so queued messages cannot start in the project checkout. */
1007 function worktreeSetupPending(
1008 projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
1009 ): boolean {
1010 return (
1011 projection.thread.worktreePath === null &&
1012 projection.runs.some(
1013 (run) =>
1014 (run.status === "failed" || run.status === "interrupted") &&
1015 run.workspacePreparation?.type === "worktree",
1016 )
1017 );
1018 }
1019
1020 function nextQueuedRun(
1021 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "messages">,
1022 ): OrchestrationV2Run | undefined {
1023 return queuedRunsInDeliveryOrder(projection).find((run) => run.queueHeld !== true);
1024 }
1025
1026 // Delegated completions, monitor notifications and unpinned scheduled deliveries never choose a
1027 // model, so a queued one runs on the thread's current pick rather than the one frozen at queue time.
1028 function followsCurrentConversation(
1029 message:
1030 | Pick<
1031 OrchestrationV2ConversationMessage,
1032 | "createdBy"
1033 | "creationSource"
1034 | "notification"
1035 | "delegatedCompletion"
1036 | "followsThreadSelection"
1037 >
1038 | undefined,
1039 ): boolean {
1040 return (
1041 message?.delegatedCompletion !== undefined ||
1042 message?.followsThreadSelection === true ||
1043 (message?.notification?.source.kind === "monitor" && !isBufferedProviderNotification(message))
1044 );
1045 }
1046
1047 function queuedRunModelSelection(
1048 projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
1049 run: OrchestrationV2Run,
1050 ): ModelSelection {
1051 const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
1052 return followsCurrentConversation(message)
1053 ? projection.thread.modelSelection
1054 : run.modelSelection;
1055 }
1056
1057 /** The selection the thread will be on once every queued run has started. */
1058 function queueTailModelSelection(
1059 projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs" | "messages">,
1060 ): ModelSelection {
1061 return (
1062 queuedRunsInDeliveryOrder(projection).findLast(
1063 (run) =>
1064 !followsCurrentConversation(
1065 projection.messages.find((message) => message.id === run.userMessageId),
1066 ),
1067 )?.modelSelection ?? projection.thread.modelSelection
1068 );
1069 }
1070
1071 /**
1072 * The provider's latest run stopped on its usage limit and nothing ran after it.
1073 * With `now` (automatic starts), a run on another provider since does not lift
1074 * a reported reset still in the future; an unknown reset is lifted, as transient 429s are.
1075 */
1076 function usageLimitHoldsQueue(
1077 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems" | "providerSessions">,
1078 providerInstanceId: ModelSelection["instanceId"],
1079 now?: DateTime.Utc,
1080 ): boolean {
1081 let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
1082 for (const session of projection.providerSessions) {
1083 if (
1084 session.providerInstanceId === providerInstanceId &&
1085 (latestSession === undefined ||
1086 DateTime.isGreaterThan(session.updatedAt, latestSession.updatedAt))
1087 ) {
1088 latestSession = session;
1089 }
1090 }
1091 const limited = usageLimitBlockedRun(
1092 projection.runs.filter((run) => run.providerInstanceId === providerInstanceId),
1093 projection.turnItems,
1094 latestSession?.lastError ?? null,
1095 );
1096 if (limited === null) return false;
1097 if (latestExecutedRun(projection.runs)?.id === limited.id) return true;
1098 if (now === undefined) return false;
1099 const resetAt = latestRootProviderFailure(limited, projection.turnItems)?.resetAt;
1100 return resetAt != null && Date.parse(resetAt) > DateTime.toEpochMillis(now);
1101 }
1102
1103 function pullRequestSnapshotEndsWatch(
1104 snapshot: ThreadPullRequestLink["snapshot"],
1105 watchStartedAt: ThreadPullRequestWatch["startedAt"],
1106 ): boolean {
1107 return (
1108 snapshot?.state === "merged" ||
1109 // A restarted watch can see a closed snapshot from before the PR reopened.
1110 (snapshot?.state === "closed" &&
1111 DateTime.isGreaterThanOrEqualTo(
1112 DateTime.makeUnsafe(snapshot.syncedAt),
1113 DateTime.makeUnsafe(watchStartedAt),
1114 ))
1115 );
1116 }
1117
1118 type MonitorPullRequestStamp = NonNullable<
1119 Extract<OrchestrationV2Notification["source"], { readonly kind: "monitor" }>["pullRequest"]
1120 >;
1121
1122 function monitorPullRequestStamps(
1123 source: OrchestrationV2Notification["source"] | undefined,
1124 ): ReadonlyArray<MonitorPullRequestStamp> {
1125 if (source?.kind !== "monitor" || source.pullRequest === undefined) return [];
1126 return [source.pullRequest, ...(source.otherPullRequests ?? [])];
1127 }
1128
1129 function isStalePullRequestWatchStamp(
1130 projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
1131 stamp: MonitorPullRequestStamp,
1132 ): boolean {
1133 const link = threadPullRequestsOf(projection.thread).find(
1134 (candidate) =>
1135 candidate.source !== "stack-dismissed" && threadPullRequestKeysEqual(candidate, stamp),
1136 );
1137 if (link === undefined) return true;
1138 if (stamp.watchEnded === true) {
1139 // The final notice outlives its watch, but a replacement watch supersedes it.
1140 const latestWatch = link.watch ?? link.endedWatch;
1141 return latestWatch?.startedAt !== stamp.watchStartedAt;
1142 }
1143 if (link.watch === undefined) {
1144 // A watch the server ended with a final notice still owes the wakes it queued before it.
1145 const endedWithNotice = projection.messages.some((message) =>
1146 monitorPullRequestStamps(message.notification?.source).some(
1147 (ending) =>
1148 ending.watchEnded === true &&
1149 ending.watchStartedAt === stamp.watchStartedAt &&
1150 threadPullRequestKeysEqual(ending, stamp),
1151 ),
1152 );
1153 return (
1154 !endedWithNotice ||
1155 link.endedWatch?.startedAt !== stamp.watchStartedAt ||
1156 pullRequestSnapshotEndsWatch(link.snapshot, stamp.watchStartedAt)
1157 );
1158 }
1159 return (
1160 link.watch.startedAt !== stamp.watchStartedAt ||
1161 (stamp.headSha !== null && link.watch.headSha !== stamp.headSha) ||
1162 pullRequestSnapshotEndsWatch(link.snapshot, link.watch.startedAt)
1163 );
1164 }
1165
1166 function isStalePullRequestWatchWake(
1167 projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
1168 run: OrchestrationV2Run,
1169 ): boolean {
1170 const stamps = monitorPullRequestStamps(
1171 projection.messages.find((message) => message.id === run.userMessageId)?.notification?.source,
1172 );
1173 return (
1174 stamps.length > 0 && stamps.every((stamp) => isStalePullRequestWatchStamp(projection, stamp))
1175 );
1176 }
1177
1178 /**
1179 * A queued wake covering several pull requests, rebuilt from the ones still current, or
1180 * undefined when it needs no change.
1181 */
1182 function currentPullRequestWatchWake(
1183 projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
1184 run: OrchestrationV2Run,
1185 ): OrchestrationV2ConversationMessage | undefined {
1186 const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
1187 const stamps = monitorPullRequestStamps(message?.notification?.source);
1188 if (message === undefined || stamps.length < 2) return undefined;
1189 const current = stamps.filter((stamp) => !isStalePullRequestWatchStamp(projection, stamp));
1190 const [first, ...rest] = current;
1191 if (first === undefined || rest.length === stamps.length - 1) return undefined;
1192 const sections = current.flatMap(({ section }) => (section === undefined ? [] : [section]));
1193 if (sections.length !== current.length) return undefined;
1194 const wake = combinePullRequestWatchWakes(
1195 sections.map(({ text, outcome, summary }) => ({
1196 text,
1197 notification: { source: { kind: "monitor" }, outcome, summary },
1198 })) as [PullRequestWatchWake, ...Array<PullRequestWatchWake>],
1199 );
1200 return {
1201 ...message,
1202 text: wake.text,
1203 notification: {
1204 ...wake.notification,
1205 source: {
1206 kind: "monitor",
1207 pullRequest: rest.length === 0 ? withoutSection(first) : first,
1208 ...(rest.length === 0 ? {} : { otherPullRequests: rest }),
1209 },
1210 },
1211 };
1212 }
1213
1214 function withoutSection({ section: _, ...stamp }: MonitorPullRequestStamp) {
1215 return stamp;
1216 }
1217
1218 function queuedRunCancellationEvents(
1219 projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "attempts">,
1220 run: OrchestrationV2Run,
1221 now: DateTime.Utc,
1222 ): ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> {
1223 const rootNode = projection.nodes.find((node) => node.id === run.rootNodeId);
1224 const attempt = projection.attempts.find((candidate) => candidate.id === run.activeAttemptId);
1225 const events: Array<Omit<OrchestrationV2DomainEvent, "id">> = [
1226 {
1227 type: "run.updated",
1228 threadId: run.threadId,
1229 runId: run.id,
1230 ...(run.rootNodeId === null ? {} : { nodeId: run.rootNodeId }),
1231 providerInstanceId: run.providerInstanceId,
1232 occurredAt: now,
1233 payload: { ...run, status: "cancelled", queuePosition: null, completedAt: now },
1234 },
1235 ];
1236 if (attempt !== undefined && rootNode !== undefined) {
1237 events.push({
1238 type: "run-attempt.updated",
1239 threadId: run.threadId,
1240 runId: run.id,
1241 nodeId: rootNode.id,
1242 providerInstanceId: run.providerInstanceId,
1243 occurredAt: now,
1244 payload: { ...attempt, status: "cancelled", completedAt: now },
1245 });
1246 }
1247 if (rootNode !== undefined) {
1248 events.push({
1249 type: "node.updated",
1250 threadId: run.threadId,
1251 runId: run.id,
1252 nodeId: rootNode.id,
1253 providerInstanceId: run.providerInstanceId,
1254 occurredAt: now,
1255 payload: { ...rootNode, status: "cancelled", completedAt: now },
1256 });
1257 }
1258 return events;
1259 }
1260
1261 function latestStableRun(
1262 projection: Pick<OrchestrationV2ThreadProjection, "runs">,
1263 ): OrchestrationV2Run | null {
1264 return (
1265 projection.runs
1266 .filter((run) => run.status === "completed" && run.checkpointId !== null)
1267 .toSorted((left, right) => right.ordinal - left.ordinal)[0] ?? null
1268 );
1269 }
1270
1271 function runForSourcePoint(
1272 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "checkpoints">,
1273 sourcePoint: Extract<
1274 OrchestrationV2Command,
1275 { readonly type: "thread.fork" | "thread.merge_back" }
1276 >["sourcePoint"],
1277 ): OrchestrationV2Run | null {
1278 switch (sourcePoint.type) {
1279 case "latest_stable":
1280 return latestStableRun(projection);
1281 case "run":
1282 return projection.runs.find((run) => run.id === sourcePoint.runId) ?? null;
1283 case "checkpoint": {
1284 const checkpoint = projection.checkpoints.find(
1285 (candidate) => candidate.id === sourcePoint.checkpointId,
1286 );
1287 return checkpoint?.runId === null || checkpoint === undefined
1288 ? null
1289 : (projection.runs.find((run) => run.id === checkpoint.runId) ?? null);
1290 }
1291 }
1292 }
1293
1294 const DELEGATED_CHILD_RUN_WINDOW = 20;
1295
1296 type DelegatedChildControls = ProjectionRecords<
1297 "runs" | "messages" | "subagents" | "providerThreads"
1298 > & {
1299 /** Set when `runs` holds only what ran since this run, plus older unfinished and task runs. */
1300 readonly runsFloor?: OrchestrationV2Run | undefined;
1301 };
1302
1303 function ranSince(run: OrchestrationV2Run, floor: OrchestrationV2Run): boolean {
1304 return run.id === floor.id || runRanAfter(run, floor);
1305 }
1306
1307 /** The walk back to a report is exact when it stops at a run that ran since the floor. */
1308 function reportRunsRead(
1309 controls: DelegatedChildControls,
1310 run: OrchestrationV2Run,
1311 reportRuns: ReadonlyArray<OrchestrationV2Run>,
1312 ): boolean {
1313 const floor = controls.runsFloor;
1314 if (floor === undefined || run.status !== "completed") return true;
1315 const oldest = reportRuns.at(-1) ?? run;
1316 if (!ranSince(oldest, floor)) return false;
1317 const monitorRuns = monitorUpdateRunIds(controls.messages);
1318 return controls.runs.some(
1319 (candidate) =>
1320 candidate.id !== run.id &&
1321 candidate.startedAt !== null &&
1322 candidate.status !== "rolled_back" &&
1323 !monitorRuns.has(candidate.id) &&
1324 runRanAfter(oldest, candidate) &&
1325 ranSince(candidate, floor),
1326 );
1327 }
1328
1329 function providerThreadForRun(
1330 projection: Pick<OrchestrationV2ThreadProjection, "providerThreads">,
1331 run: OrchestrationV2Run,
1332 ): OrchestrationV2ProviderThread | undefined {
1333 return run.providerThreadId === null
1334 ? undefined
1335 : projection.providerThreads.find((candidate) => candidate.id === run.providerThreadId);
1336 }
1337
1338 function providerTurnForRun(
1339 projection: Pick<OrchestrationV2ThreadProjection, "providerTurns" | "attempts">,
1340 run: OrchestrationV2Run,
1341 ): OrchestrationV2ProviderTurn | undefined {
1342 if (run.activeAttemptId === null) {
1343 return undefined;
1344 }
1345
1346 return (
1347 latestProviderTurnForAttempt(projection.providerTurns, run.activeAttemptId) ??
1348 projection.providerTurns.find((turn) => {
1349 const attempt = projection.attempts.find((candidate) => candidate.id === run.activeAttemptId);
1350 return attempt?.providerTurnId === turn.id;
1351 })
1352 );
1353 }
1354
1355 function contextSourcePointForRun(
1356 projection: Pick<
1357 OrchestrationV2ThreadProjection,
1358 "thread" | "providerThreads" | "providerTurns" | "attempts"
1359 >,
1360 run: OrchestrationV2Run,
1361 ): OrchestrationV2ContextSourcePoint {
1362 const providerThread = providerThreadForRun(projection, run);
1363 const providerTurn = providerTurnForRun(projection, run);
1364 return {
1365 threadId: projection.thread.id,
1366 runId: run.id,
1367 ...(run.checkpointId === null ? {} : { checkpointId: run.checkpointId }),
1368 ...(providerThread?.nativeThreadRef === null || providerThread?.nativeThreadRef === undefined
1369 ? {}
1370 : { providerThreadRef: providerThread.nativeThreadRef }),
1371 ...(providerTurn?.nativeTurnRef === null || providerTurn?.nativeTurnRef === undefined
1372 ? {}
1373 : { providerTurnRef: providerTurn.nativeTurnRef }),
1374 };
1375 }
1376
1377 function pendingForkTransferForThread(
1378 projection: Pick<OrchestrationV2ThreadProjection, "contextTransfers" | "thread">,
1379 ): OrchestrationV2ContextTransfer | undefined {
1380 return projection.contextTransfers.find(
1381 (transfer) =>
1382 transfer.type === "fork" &&
1383 transfer.targetThreadId === projection.thread.id &&
1384 transfer.status === "pending",
1385 );
1386 }
1387
1388 /** A fork transfer still pending when a fresh run starts would bring the parent's context back. */
1389 function forkTransferSupersededEvent(
1390 transfer: OrchestrationV2ContextTransfer,
1391 runId: RunId,
1392 providerInstanceId: ProviderInstanceId,
1393 now: DateTime.Utc,
1394 ): Omit<OrchestrationV2DomainEvent, "id"> {
1395 return {
1396 type: "context-transfer.updated",
1397 threadId: transfer.targetThreadId,
1398 runId,
1399 providerInstanceId,
1400 occurredAt: now,
1401 payload: {
1402 ...transfer,
1403 status: "superseded",
1404 error: "Superseded by a fresh session.",
1405 updatedAt: now,
1406 },
1407 };
1408 }
1409
1410 function pendingMergeBackTransfersForThread(
1411 projection: Pick<OrchestrationV2ThreadProjection, "contextTransfers" | "thread">,
1412 ): ReadonlyArray<OrchestrationV2ContextTransfer> {
1413 return projection.contextTransfers.filter(
1414 (transfer) =>
1415 transfer.type === "merge_back" &&
1416 transfer.targetThreadId === projection.thread.id &&
1417 transfer.status === "pending",
1418 );
1419 }
1420
1421 function latestContextTransfer(
1422 transfers: ReadonlyArray<OrchestrationV2ContextTransfer>,
1423 ): OrchestrationV2ContextTransfer | undefined {
1424 return transfers.reduce<OrchestrationV2ContextTransfer | undefined>((latest, transfer) => {
1425 if (latest === undefined) {
1426 return transfer;
1427 }
1428 return DateTime.toEpochMillis(transfer.updatedAt) >= DateTime.toEpochMillis(latest.updatedAt)
1429 ? transfer
1430 : latest;
1431 }, undefined);
1432 }
1433
1434 export function shouldPrepareLegacyImportHandoff(input: {
1435 readonly hasNativeHistory?: boolean;
1436 readonly hasCompletedRun: boolean;
1437 readonly historyOrigin: OrchestrationV2AppThread["historyOrigin"];
1438 readonly legacyImportItemCount: number;
1439 }): boolean {
1440 return (
1441 input.historyOrigin === "v1_import" &&
1442 !input.hasNativeHistory &&
1443 !input.hasCompletedRun &&
1444 input.legacyImportItemCount > 0
1445 );
1446 }
1447
1448 function appendContextHandoffId(
1449 handoffIds: OrchestrationV2ProviderThread["handoffIds"],
1450 handoffId: OrchestrationV2ContextHandoff["id"] | null,
1451 ): OrchestrationV2ProviderThread["handoffIds"] {
1452 return handoffId === null ? handoffIds : Array.from(new Set([...handoffIds, handoffId]));
1453 }
1454
1455 /** The provider thread a message continues: the active one, unless a fresh-session schedule run started it or it predates the latest fresh start. */
1456 function conversationProviderThread(
1457 projection: Pick<
1458 OrchestrationV2ThreadProjection,
1459 "thread" | "providerThreads" | "runs" | "turnItems"
1460 >,
1461 ): OrchestrationV2ProviderThread | undefined {
1462 const startedByScheduledFreshRun = (providerThread: OrchestrationV2ProviderThread) =>
1463 projection.runs.some(
1464 (run) =>
1465 run.ordinal === providerThread.firstRunOrdinal &&
1466 isScheduledFreshRun(run, projection.turnItems),
1467 );
1468 const active = projection.providerThreads.find(
1469 (candidate) => candidate.id === projection.thread.activeProviderThreadId,
1470 );
1471 if (active === undefined) return active;
1472 const boundary = contextBoundaryOrdinal(projection);
1473 // A pre-clear session that is still alive (background work) can become active again on its updates.
1474 if (!startedByScheduledFreshRun(active) && !providerThreadPredatesBoundary(active, boundary)) {
1475 return active;
1476 }
1477 return (
1478 projection.providerThreads
1479 .filter(
1480 (candidate) =>
1481 candidate.appThreadId === projection.thread.id &&
1482 candidate.ownerNodeId === null &&
1483 candidate.lastRunOrdinal !== null &&
1484 !providerThreadPredatesBoundary(candidate, boundary) &&
1485 !startedByScheduledFreshRun(candidate),
1486 )
1487 .toSorted((left, right) => (right.lastRunOrdinal ?? 0) - (left.lastRunOrdinal ?? 0))[0] ??
1488 active
1489 );
1490 }
1491
1492 /** Wakes and fresh starts act on the active session; a message continues the conversation. */
1493 function dispatchProviderThread(
1494 projection: Pick<
1495 OrchestrationV2ThreadProjection,
1496 "thread" | "providerThreads" | "runs" | "turnItems"
1497 >,
1498 message: Pick<OrchestrationV2ConversationMessage, "notification" | "delegatedCompletion">,
1499 freshContext: boolean,
1500 ): OrchestrationV2ProviderThread | undefined {
1501 return freshContext ||
1502 message.notification !== undefined ||
1503 message.delegatedCompletion !== undefined
1504 ? projection.providerThreads.find(
1505 (candidate) => candidate.id === projection.thread.activeProviderThreadId,
1506 )
1507 : conversationProviderThread(projection);
1508 }
1509
1510 /** Threads with no started run (imported, or reserved by another queued run) never cross it. */
1511 function providerThreadPredatesBoundary(
1512 providerThread: Pick<OrchestrationV2ProviderThread, "firstRunOrdinal">,
1513 boundary: number,
1514 ): boolean {
1515 return (
1516 boundary > 0 &&
1517 (providerThread.firstRunOrdinal === null || providerThread.firstRunOrdinal < boundary)
1518 );
1519 }
1520
1521 /** A provider that hosts one native thread per session keeps the old conversation alive otherwise. */
1522 function sessionsReleasedByFreshContext(
1523 projection: Pick<OrchestrationV2ThreadProjection, "providerSessions">,
1524 activeProviderThread: OrchestrationV2ProviderThread | undefined,
1525 ): ReadonlyArray<OrchestrationV2ProviderSession> {
1526 const sessionId = activeProviderThread?.providerSessionId ?? null;
1527 return projection.providerSessions.filter(
1528 (session) =>
1529 session.id === sessionId &&
1530 session.status !== "stopped" &&
1531 session.status !== "error" &&
1532 !session.capabilities.sessions.supportsMultipleProviderThreadsPerSession,
1533 );
1534 }
1535
1536 const FRESH_CONTEXT_DETACH_REASON = "Started a fresh session.";
1537
1538 function uniqueSessions(
1539 sessions: ReadonlyArray<OrchestrationV2ProviderSession>,
1540 ): ReadonlyArray<OrchestrationV2ProviderSession> {
1541 return Array.from(new Map(sessions.map((session) => [session.id, session])).values());
1542 }
1543
1544 function rootProviderThreadsForProvider(
1545 projection: Pick<
1546 OrchestrationV2ThreadProjection,
1547 "providerThreads" | "thread" | "runs" | "turnItems"
1548 >,
1549 providerInstanceId: ModelSelection["instanceId"],
1550 ): ReadonlyArray<OrchestrationV2ProviderThread> {
1551 const boundary = contextBoundaryOrdinal(projection);
1552 return projection.providerThreads
1553 .filter(
1554 (providerThread) =>
1555 providerThread.providerInstanceId === providerInstanceId &&
1556 providerThread.appThreadId === projection.thread.id &&
1557 providerThread.ownerNodeId === null &&
1558 !providerThreadPredatesBoundary(providerThread, boundary),
1559 )
1560 .toSorted(
1561 (left, right) =>
1562 (right.lastRunOrdinal ?? 0) - (left.lastRunOrdinal ?? 0) ||
1563 DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
1564 );
1565 }
1566
1567 function isHandoffSourceRun(run: OrchestrationV2Run): boolean {
1568 return (
1569 run.status === "completed" ||
1570 run.status === "failed" ||
1571 run.status === "interrupted" ||
1572 // A queued request cancelled before it started never reached any provider.
1573 (run.status === "cancelled" && run.startedAt !== null)
1574 );
1575 }
1576
1577 function lastDeliveredRunForProviderThread(
1578 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "providerTurns">,
1579 providerThreadId: OrchestrationV2ProviderThread["id"],
1580 ): OrchestrationV2Run | undefined {
1581 return projection.runs.findLast(
1582 (run) =>
1583 isHandoffSourceRun(run) &&
1584 run.providerThreadId === providerThreadId &&
1585 (run.status === "completed" ||
1586 projection.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId)),
1587 );
1588 }
1589
1590 type DyingNativeSubagent = {
1591 readonly projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "subagents" | "turnItems">;
1592 readonly subagent: Pick<
1593 Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>,
1594 "subagentId" | "origin" | "childThreadId" | "providerInstanceId"
1595 >;
1596 };
1597
1598 const nativeSubagentRef = (subagent: OrchestrationV2Subagent): DyingNativeSubagent["subagent"] => ({
1599 subagentId: subagent.id,
1600 origin: subagent.origin,
1601 childThreadId: subagent.childThreadId,
1602 providerInstanceId: subagent.providerInstanceId,
1603 });
1604
1605 const hasRunningDelegatedTasks = (subagents: ReadonlyArray<OrchestrationV2Subagent>) =>
1606 subagents.some(
1607 (task) =>
1608 task.origin === "app_owned" &&
1609 task.childThreadId !== null &&
1610 isOrchestrationV2WorkActive(task.status),
1611 );
1612
1613 const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(function* () {
1614 const checkpointService = yield* CheckpointServiceV2;
1615 const commandPolicy = yield* CommandPolicyV2;
1616 const contextHandoffService = yield* ContextHandoffServiceV2;
1617 const eventSink = yield* EventSinkV2;
1618 const commandReceipts = yield* CommandReceiptStoreV2;
1619 const idAllocator = yield* IdAllocator.IdAllocatorV2;
1620 const projects = yield* ProjectStore.ProjectStoreV2;
1621 const projectionStore = yield* ProjectionStoreV2;
1622 const effectOutbox = yield* EffectOutbox.EffectOutboxV2;
1623 const nextTurnItemOrdinal = (
1624 projection: Pick<OrchestrationV2ThreadProjection, "thread"> &
1625 Partial<Pick<OrchestrationV2ThreadProjection, "turnItems">>,
1626 ) =>
1627 projectionStore.getNextTurnItemOrdinal(projection.thread.id).pipe(
1628 Effect.mapError(
1629 (cause) => new OrchestratorProjectionError({ threadId: projection.thread.id, cause }),
1630 ),
1631 Effect.map((next) =>
1632 (projection.turnItems ?? []).reduce(
1633 (ordinal, item) => Math.max(ordinal, item.ordinal + 1),
1634 next,
1635 ),
1636 ),
1637 );
1638
1639 const fileSystem = yield* FileSystem.FileSystem;
1640 const path = yield* Path.Path;
1641 const providerAdapters = yield* ProviderAdapterRegistryV2;
1642 const continuationRequests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
1643 const providerSessions = yield* ProviderSessionManagerV2;
1644 const providerSwitchService = yield* ProviderSwitchServiceV2;
1645 const serverSettings = yield* ServerSettings.ServerSettingsService;
1646 const runtimePolicy = yield* RuntimePolicyV2;
1647 const threadForkService = yield* ThreadForkServiceV2;
1648 const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor;
1649 const messageIntake = yield* KeyedLock.make<ThreadId>();
1650 // Read once: the policy is fixed for the server's life, and dispatch takes it as an input.
1651 const { managedPolicy } = serverSettings;
1652 const runtimeModeCeiling: RuntimeModeCeiling | undefined =
1653 managedPolicy?.limits?.maxRuntimeMode === undefined
1654 ? undefined
1655 : {
1656 organization: managedPolicy.organization,
1657 maxRuntimeMode: managedPolicy.limits.maxRuntimeMode,
1658 };
1659
1660 /** The driver when this provider runs `/goal` natively, else undefined. */
1661 const goalDriverFor = Effect.fn("Orchestrator.goalDriverFor")(function* (
1662 command: Pick<OrchestrationV2ServerCommand, "commandId" | "type">,
1663 providerInstanceId: ProviderInstanceId,
1664 ) {
1665 const adapter = yield* providerAdapters.get(providerInstanceId).pipe(mapDispatchError(command));
1666 const { driver } = adapter;
1667 if (driver === "opencode" || driver === "antigravity") {
1668 const capabilities = yield* adapter.getCapabilities().pipe(mapDispatchError(command));
1669 return capabilities.turns.supportsNativeGoals === true ? driver : undefined;
1670 }
1671 return driver === "codex" || driver === "claudeAgent" ? driver : undefined;
1672 });
1673
1674 /** Unsupported goal controls can become new objectives in native providers. */
1675 const ensureGoalCommandSupported = Effect.fn("Orchestrator.ensureGoalCommandSupported")(
1676 function* (
1677 command: Pick<OrchestrationV2ServerCommand, "commandId" | "type">,
1678 message: { readonly text: string; readonly attachments: ReadonlyArray<ChatAttachment> },
1679 providerInstanceId: ProviderInstanceId,
1680 ) {
1681 const goalCommand = parseGoalCommand(message);
1682 if (goalCommand === null || goalCommand.type === "set") return;
1683 const driver = yield* goalDriverFor(command, providerInstanceId);
1684 if (driver === "antigravity") {
1685 return yield* new OrchestratorDispatchError({
1686 commandId: command.commandId,
1687 commandType: command.type,
1688 cause:
1689 "Antigravity accepts /goal followed by an objective. Interrupt the running turn, then ask Antigravity to stop the current goal.",
1690 });
1691 }
1692 if (goalCommand.type !== "pause" && goalCommand.type !== "resume") return;
1693 if (driver !== "claudeAgent") return;
1694 return yield* new OrchestratorDispatchError({
1695 commandId: command.commandId,
1696 commandType: command.type,
1697 cause: "Claude goals cannot be paused or resumed. Send /goal clear to remove the goal.",
1698 });
1699 },
1700 );
1701
1702 const mapDispatchError =
1703 (command: Pick<OrchestrationV2ServerCommand, "commandId" | "type">) =>
1704 <A, E, R>(effect: Effect.Effect<A, E, R>): Effect.Effect<A, OrchestratorDispatchError, R> =>
1705 effect.pipe(
1706 Effect.mapError(
1707 (cause) =>
1708 new OrchestratorDispatchError({
1709 commandId: command.commandId,
1710 commandType: command.type,
1711 cause,
1712 }),
1713 ),
1714 );
1715
1716 const mapDelegatedCompletionError = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
1717 effect.pipe(
1718 Effect.mapError(
1719 (cause) =>
1720 new OrchestratorDispatchError({
1721 commandId: CommandId.make("command:system:delegated-completion-delivery"),
1722 commandType: "delegated_task.completion-delivery",
1723 cause,
1724 }),
1725 ),
1726 );
1727
1728 const providerSessionIdFor = (input: {
1729 readonly adapter: ProviderAdapter.ProviderAdapterV2["Service"];
1730 readonly providerInstanceId: ProviderInstanceId;
1731 readonly threadId: ThreadId;
1732 }) =>
1733 input.adapter.getCapabilities().pipe(
1734 Effect.flatMap((capabilities) =>
1735 capabilities.sessions.supportsMultipleProviderThreadsPerSession
1736 ? Effect.succeed(
1737 idAllocator.derive.providerSession({
1738 providerInstanceId: input.providerInstanceId,
1739 }),
1740 )
1741 : idAllocator.allocate.providerSession({
1742 providerInstanceId: input.providerInstanceId,
1743 threadId: input.threadId,
1744 }),
1745 ),
1746 );
1747
1748 const makeEvent =
1749 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
1750 Effect.fnUntraced(function* <Event extends OrchestrationV2DomainEvent>(
1751 command: OrchestrationV2ServerCommand,
1752 event: Omit<Event, "id">,
1753 ) {
1754 const eventId = yield* mapDispatchError(command)(
1755 idAllocator.allocate.event({
1756 threadId: event.threadId,
1757 commandId: command.commandId,
1758 }),
1759 );
1760 return {
1761 ...event,
1762 id: eventId,
1763 } as Event;
1764 });
1765
1766 const emit =
1767 (events: Ref.Ref<Array<OrchestrationV2DomainEvent>>, command: OrchestrationV2ServerCommand) =>
1768 <Event extends OrchestrationV2DomainEvent>(event: Omit<Event, "id">) =>
1769 Effect.gen(function* () {
1770 const withId = yield* makeEvent(command, event);
1771 yield* Ref.update(events, (existing) => [...existing, withId]);
1772 return withId;
1773 });
1774
1775 // Detaching kills background work running inside the provider process.
1776 // Probe even with an empty roster: Claude's native subagents never reach
1777 // pendingBackgroundTasks. Sessions shared across threads probe per thread.
1778 const sessionHasBackgroundWork = Effect.fn("Orchestrator.sessionHasBackgroundWork")(function* (
1779 detaching: OrchestrationV2ThreadProjection["providerSessions"][number],
1780 providerThreads: OrchestrationV2ThreadProjection["providerThreads"],
1781 ) {
1782 const session = yield* providerSessions
1783 .get(detaching.id)
1784 .pipe(Effect.orElseSucceed(() => Option.none()));
1785 if (Option.isNone(session)) return false;
1786 const runtime = session.value;
1787 const probes =
1788 !detaching.capabilities.sessions.supportsMultipleProviderThreadsPerSession &&
1789 runtime.hasPendingBackgroundWork !== undefined
1790 ? [runtime.hasPendingBackgroundWork]
1791 : providerThreads.flatMap((providerThread) =>
1792 providerThread.providerSessionId === detaching.id &&
1793 runtime.hasPendingBackgroundWorkForThread !== undefined
1794 ? [runtime.hasPendingBackgroundWorkForThread(providerThread)]
1795 : [],
1796 );
1797 for (const probe of probes) {
1798 // A failed probe cannot confirm that the work ended.
1799 if (yield* probe.pipe(Effect.catchCause(() => Effect.succeed(true)))) return true;
1800 }
1801 return false;
1802 });
1803
1804 // Runs in every command transaction; avoid reading thousands of finished leaf nodes.
1805 const readCommandProjection = (
1806 threadId: ThreadId,
1807 recent?: ProjectionRecordFilter["sinceLastCompaction"],
1808 ) =>
1809 projectionStore
1810 .getThreadRecords(
1811 threadId,
1812 [
1813 "runs",
1814 "attempts",
1815 "nodes",
1816 "subagents",
1817 "providerSessions",
1818 "providerThreads",
1819 "providerTurns",
1820 "runtimeRequests",
1821 "messages",
1822 "turnItems",
1823 "checkpointScopes",
1824 "contextTransfers",
1825 ],
1826 {
1827 turnItemTypes: ["user_message", "error"],
1828 messageRoles: ["user"],
1829 omitSettledLeafNodes: true,
1830 // Commands here read only undelivered or active tasks; a dispatcher thread keeps
1831 // thousands of delivered ones, each carrying its full prompt.
1832 omitSettledSubagents: true,
1833 // No command reads a settled task's node or its child's provider thread.
1834 omitSettledSubagentRecords: true,
1835 // Commands act on pending forks and merge-backs only; consumed ones stay unread.
1836 contextTransferStatuses: ["pending"],
1837 ...(recent === undefined ? {} : { sinceLastCompaction: recent }),
1838 },
1839 )
1840 .pipe(
1841 Effect.map((records): OrchestrationV2ThreadProjection => ({
1842 ...records,
1843 checkpoints: [],
1844 plans: [],
1845 contextHandoffs: [],
1846 visibleTurnItems: [],
1847 updatedAt: records.thread.updatedAt,
1848 })),
1849 Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
1850 );
1851
1852 // Command reads omit settled tasks, but a completion notice counts its whole cohort.
1853 const notificationTasks = (
1854 threadId: ThreadId,
1855 message: Pick<OrchestrationV2ConversationMessage, "delegatedCompletion">,
1856 loaded: ReadonlyArray<OrchestrationV2Subagent>,
1857 ) =>
1858 message.delegatedCompletion === undefined
1859 ? Effect.succeed(loaded)
1860 : projectionStore
1861 .getThreadRecords(threadId, ["subagents"], {
1862 subagentRunIds: [message.delegatedCompletion.parentRunId],
1863 })
1864 .pipe(
1865 Effect.map((records) => records.subagents),
1866 Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
1867 );
1868
1869 const readHandoffItems = (threadId: ThreadId, runIds?: ReadonlyArray<RunId | null>) =>
1870 projectionStore
1871 .getThreadRecords(
1872 threadId,
1873 ["turnItems"],
1874 runIds === undefined ? undefined : { turnItemRunIds: runIds },
1875 )
1876 .pipe(
1877 Effect.map((records) => records.turnItems),
1878 Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
1879 );
1880
1881 const readInheritedItems = (thread: OrchestrationV2AppThread) =>
1882 readInheritedHandoffItems(projectionStore, thread).pipe(
1883 Effect.mapError((cause) => new OrchestratorProjectionError({ threadId: thread.id, cause })),
1884 );
1885
1886 /** Includes tombstones, which still reserve their id. */
1887 const findThread = (threadId: ThreadId) =>
1888 projectionStore
1889 .findThread(threadId)
1890 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })));
1891
1892 /** Side chats live inside their parent, so deleting or archiving the parent takes them too. */
1893 const liveSideChatIds = Effect.fn("Orchestrator.liveSideChatIds")(
1894 function* (threadId: ThreadId) {
1895 const { contextTransfers } = yield* projectionStore
1896 .getThreadRecords(threadId, ["contextTransfers"])
1897 .pipe(
1898 Effect.catchTags({
1899 ProjectionStoreThreadNotFoundError: () => Effect.succeed({ contextTransfers: [] }),
1900 }),
1901 );
1902 const sideChatIds: Array<ThreadId> = [];
1903 for (const transfer of contextTransfers) {
1904 if (transfer.type !== "fork" || transfer.sourceThreadId !== threadId) continue;
1905 const child = yield* projectionStore.findThread(transfer.targetThreadId);
1906 if (child?.lineage.sideChat === true && child.deletedAt === null) {
1907 sideChatIds.push(child.id);
1908 }
1909 }
1910 return sideChatIds;
1911 },
1912 (effect, threadId) =>
1913 effect.pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))),
1914 );
1915
1916 const getProjectionWithPendingEvents =
1917 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
1918 Effect.fnUntraced(function* (
1919 threadId: ThreadId,
1920 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
1921 recent?: ProjectionRecordFilter["sinceLastCompaction"],
1922 ) {
1923 const pending = (yield* Ref.get(events)).filter((event) => event.threadId === threadId);
1924 const created = pending.find(
1925 (
1926 event,
1927 ): event is Extract<OrchestrationV2DomainEvent, { readonly type: "thread.created" }> =>
1928 event.type === "thread.created",
1929 );
1930 // A thread created here has no stored row: create and fork refuse used ids, delegation derives a fresh one.
1931 let projection: OrchestrationV2ThreadProjection;
1932 if (created === undefined) {
1933 projection = yield* readCommandProjection(threadId, recent);
1934 } else {
1935 projection = emptyProjection(created);
1936 }
1937
1938 for (const event of pending) {
1939 if (event.type === "thread.created" && projection.thread.id === event.payload.id) {
1940 projection = { ...projection, thread: event.payload, updatedAt: event.occurredAt };
1941 continue;
1942 }
1943 projection = applyToProjection(projection, event);
1944 }
1945 return projection;
1946 });
1947
1948 const makeSystemEvent =
1949 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
1950 Effect.fnUntraced(function* <Event extends OrchestrationV2DomainEvent>(
1951 event: Omit<Event, "id">,
1952 ) {
1953 const eventId = yield* idAllocator.allocate.event({
1954 threadId: event.threadId,
1955 });
1956 return {
1957 ...event,
1958 id: eventId,
1959 } as Event;
1960 });
1961
1962 const writeSystemEvents =
1963 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
1964 Effect.fnUntraced(function* (
1965 events: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">>,
1966 effects: ReadonlyArray<PendingOrchestrationEffectV2> = [],
1967 options: { readonly updateDelegatedCompletionOnly?: boolean } = {},
1968 ) {
1969 const withIds = yield* Effect.forEach(events, (event) => makeSystemEvent(event));
1970 yield* eventSink.writeWithEffects({ events: withIds, effects, ...options });
1971 });
1972
1973 const completionDeliveryRun = (
1974 projection: Pick<OrchestrationV2ThreadProjection, "messages" | "runs">,
1975 delivery: OrchestrationV2DelegatedCompletionDelivery | null | undefined,
1976 ) => {
1977 if (delivery == null) return undefined;
1978 const message = projection.messages.find((candidate) => candidate.id === delivery.messageId);
1979 return projection.runs.find((candidate) => candidate.id === message?.runId);
1980 };
1981
1982 const completionDeliveryMessage = (
1983 projection: Pick<OrchestrationV2ThreadProjection, "messages">,
1984 delivery: OrchestrationV2DelegatedCompletionDelivery | null | undefined,
1985 ) =>
1986 delivery === null || delivery === undefined
1987 ? undefined
1988 : projection.messages.find((candidate) => candidate.id === delivery.messageId);
1989
1990 const isUnobservedAcknowledgement = (
1991 command: OrchestrationV2ServerCommand,
1992 ): command is Extract<
1993 OrchestrationV2ServerCommand,
1994 { readonly type: "delegated_task.completion-delivery.acknowledge" }
1995 > & { readonly observedByRunId: null } =>
1996 command.type === "delegated_task.completion-delivery.acknowledge" &&
1997 command.observedByRunId === null;
1998
1999 /** Parent run and wake message events that commit a planDelegatedCompletionDelivery result. */
2000 const delegatedCompletionPlanEvents = (
2001 threadId: ThreadId,
2002 plan: {
2003 readonly parentRun: OrchestrationV2Run | undefined;
2004 readonly message: OrchestrationV2ThreadProjection["messages"][number] | undefined;
2005 },
2006 threadProviderInstanceId: ProviderInstanceId,
2007 occurredAt: DateTime.Utc,
2008 ): ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> => [
2009 ...(plan.parentRun === undefined
2010 ? []
2011 : [
2012 {
2013 type: "run.updated" as const,
2014 threadId,
2015 runId: plan.parentRun.id,
2016 ...(plan.parentRun.rootNodeId === null ? {} : { nodeId: plan.parentRun.rootNodeId }),
2017 providerInstanceId: plan.parentRun.providerInstanceId,
2018 occurredAt,
2019 payload: plan.parentRun,
2020 },
2021 ]),
2022 ...(plan.message === undefined
2023 ? []
2024 : [
2025 {
2026 type: "message.updated" as const,
2027 threadId,
2028 ...(plan.message.runId === null ? {} : { runId: plan.message.runId }),
2029 ...(plan.message.nodeId === null ? {} : { nodeId: plan.message.nodeId }),
2030 providerInstanceId: plan.parentRun?.providerInstanceId ?? threadProviderInstanceId,
2031 occurredAt,
2032 payload: plan.message,
2033 },
2034 ]),
2035 ];
2036
2037 /** The parent records `planDelegatedCompletionDelivery` reads for one task: its run, and the queued wake's message, run and tasks. */
2038 const readDelegatedCompletionPlanRecords =
2039 /* The delegated task operation that calls this owns the span. */
2040 Effect.fnUntraced(function* (parentThreadId: ThreadId, task: OrchestrationV2Subagent) {
2041 const {
2042 thread,
2043 runs: [parentRun],
2044 } = yield* projectionStore.getThreadRecords(parentThreadId, ["runs"], {
2045 runIds: task.runId === null ? [] : [task.runId],
2046 });
2047 const delivery = parentRun?.delegatedCompletion?.delivery;
2048 if (delivery == null) return { thread, parentRun, runs: [], messages: [], subagents: [] };
2049 const { messages } = yield* projectionStore.getThreadRecords(parentThreadId, ["messages"], {
2050 messageIds: [delivery.messageId],
2051 messageRoles: ["user"],
2052 });
2053 const deliveryRunId = messages[0]?.runId;
2054 const { runs, subagents } = yield* projectionStore.getThreadRecords(
2055 parentThreadId,
2056 ["runs", "subagents"],
2057 {
2058 runIds: deliveryRunId == null ? [] : [deliveryRunId],
2059 subagentIds: [...delivery.taskIds, task.id],
2060 },
2061 );
2062 return { thread, parentRun, runs, messages, subagents };
2063 });
2064
2065 const offerDelegatedCompletionDelivery =
2066 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
2067 Effect.fnUntraced(function* (threadId: ThreadId, parentRunId: RunId) {
2068 const controls = yield* projectionStore.getThreadRecords(threadId, ["runs"], {
2069 runIds: [parentRunId],
2070 });
2071 const parentRun = controls.runs[0];
2072 const cohort = parentRun?.delegatedCompletion;
2073 const delivery = cohort?.delivery;
2074 if (
2075 parentRun === undefined ||
2076 cohort?.disposition !== "open" ||
2077 delivery == null ||
2078 delivery.taskIds.length === 0 ||
2079 controls.thread.archivedAt !== null ||
2080 controls.thread.deletedAt !== null
2081 )
2082 return;
2083 const messages = yield* projectionStore.getThreadRecords(threadId, ["messages"], {
2084 messageIds: [delivery.messageId],
2085 });
2086 const message = messages.messages[0];
2087 if (message !== undefined) {
2088 if (message.delegatedCompletion === undefined || message.runId === null) return;
2089 const runs = yield* projectionStore.getThreadRecords(threadId, ["runs"], {
2090 runIds: [message.runId],
2091 });
2092 const run = runs.runs[0];
2093 const turns = yield* projectionStore.getThreadRecords(threadId, ["providerTurns"], {
2094 providerTurnAttemptIds: run?.activeAttemptId == null ? [] : [run.activeAttemptId],
2095 });
2096 if (!isUndeliveredMailboxSteer({ ...messages, ...runs, ...turns }, delivery.messageId))
2097 return;
2098 }
2099 const projection = yield* projectionStore.getThreadRecords(threadId, ["providerThreads"], {
2100 providerThreadIds: parentRun.providerThreadId === null ? [] : [parentRun.providerThreadId],
2101 });
2102 const providerThread =
2103 parentRun.providerThreadId === null
2104 ? undefined
2105 : projection.providerThreads.find(
2106 (candidate) => candidate.id === parentRun.providerThreadId,
2107 );
2108 if (providerThread === undefined) {
2109 return;
2110 }
2111 yield* continuationRequests.offer({
2112 threadId,
2113 providerThreadId: providerThread.id,
2114 driver: providerThread.driver,
2115 detail: null,
2116 delivery: "message_text",
2117 delegatedCompletion: {
2118 parentRunId,
2119 generation: delivery.generation,
2120 messageId: delivery.messageId,
2121 },
2122 });
2123 });
2124
2125 const offerDelegatedCompletionDeliveries =
2126 /* This helper runs for individual orchestration events; the enclosing operation owns the span. */
2127 Effect.fnUntraced(function* (threadId: ThreadId) {
2128 const projection = yield* projectionStore.getThreadRecords(threadId, ["runs"]);
2129 for (const run of projection.runs) {
2130 if (
2131 run.delegatedCompletion?.delivery !== undefined &&
2132 run.delegatedCompletion.delivery !== null
2133 ) {
2134 yield* offerDelegatedCompletionDelivery(threadId, run.id);
2135 }
2136 }
2137 });
2138
2139 /** Marks a wake the provider received delivered and reserves the next one for pending siblings. */
2140 const acceptDelegatedCompletionDelivery = Effect.fn(
2141 "Orchestrator.acceptDelegatedCompletionDelivery",
2142 )(function* (input: {
2143 readonly command: OrchestrationV2ServerCommand;
2144 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
2145 readonly projection: Pick<
2146 OrchestrationV2ThreadProjection,
2147 "thread" | "runs" | "messages" | "subagents"
2148 >;
2149 readonly messageId: OrchestrationV2ConversationMessage["id"];
2150 readonly now: DateTime.Utc;
2151 }) {
2152 const { projection, now } = input;
2153 const message = projection.messages.find((row) => row.id === input.messageId);
2154 const ownership = message?.delegatedCompletion;
2155 const parentRun = projection.runs.find((run) => run.id === ownership?.parentRunId);
2156 const cohort = parentRun?.delegatedCompletion;
2157 const delivery = cohort?.delivery;
2158 if (
2159 parentRun === undefined ||
2160 cohort === undefined ||
2161 cohort.disposition !== "open" ||
2162 delivery == null ||
2163 delivery.messageId !== input.messageId ||
2164 delivery.generation !== ownership?.generation
2165 ) {
2166 return undefined;
2167 }
2168 const threadId = projection.thread.id;
2169 const parentIsLive = hasLiveRun({ runs: [parentRun] });
2170 const pendingTaskIds =
2171 projection.thread.archivedAt === null && projection.thread.deletedAt === null
2172 ? projection.subagents
2173 .filter(
2174 (task) =>
2175 task.origin === "app_owned" &&
2176 task.runId === parentRun.id &&
2177 task.completionDelivery?.state === "pending" &&
2178 isTerminalDelegatedTaskStatus(task.status) &&
2179 (!parentIsLive || task.completionWake === "always"),
2180 )
2181 .map((task) => task.id)
2182 : [];
2183 const nextDelivery =
2184 pendingTaskIds.length === 0
2185 ? null
2186 : {
2187 generation: cohort.nextGeneration,
2188 messageId: yield* mapDelegatedCompletionError(
2189 idAllocator.allocate.message({
2190 threadId,
2191 ordinal:
2192 (yield* mapDelegatedCompletionError(projectionStore.getMessageCount(threadId))) +
2193 1,
2194 }),
2195 ),
2196 taskIds: pendingTaskIds,
2197 };
2198 const emitEvent = emit(input.events, input.command);
2199 const acceptedIds = new Set(delivery.taskIds);
2200 const pendingIds = new Set(pendingTaskIds);
2201 for (const task of projection.subagents) {
2202 const state = pendingIds.has(task.id)
2203 ? "claimed"
2204 : acceptedIds.has(task.id) && task.completionDelivery?.state === "claimed"
2205 ? "delivered"
2206 : undefined;
2207 if (state === undefined) continue;
2208 yield* emitEvent({
2209 type: "subagent.updated",
2210 threadId,
2211 runId: parentRun.id,
2212 nodeId: task.id,
2213 driver: task.driver,
2214 providerInstanceId: task.providerInstanceId,
2215 occurredAt: now,
2216 payload: { ...task, completionDelivery: { state, observedByRunId: null }, updatedAt: now },
2217 });
2218 }
2219 yield* emitEvent({
2220 type: "run.updated",
2221 threadId,
2222 runId: parentRun.id,
2223 providerInstanceId: parentRun.providerInstanceId,
2224 occurredAt: now,
2225 payload: {
2226 ...parentRun,
2227 delegatedCompletion: {
2228 ...cohort,
2229 nextGeneration: cohort.nextGeneration + (nextDelivery === null ? 0 : 1),
2230 delivery: nextDelivery,
2231 },
2232 },
2233 });
2234 return parentRun.id;
2235 });
2236
2237 const emitQueuedRunCancellation = Effect.fn("Orchestrator.emitQueuedRunCancellation")(
2238 function* (input: {
2239 readonly command: OrchestrationV2ServerCommand;
2240 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
2241 readonly projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "attempts">;
2242 readonly run: OrchestrationV2Run;
2243 readonly now: DateTime.Utc;
2244 }) {
2245 const emitEvent = emit(input.events, input.command);
2246 for (const event of queuedRunCancellationEvents(input.projection, input.run, input.now)) {
2247 yield* emitEvent(event);
2248 }
2249 },
2250 );
2251
2252 const failQueuedRunStart = Effect.fn("Orchestrator.failQueuedRunStart")(function* (
2253 threadId: ThreadId,
2254 cause: unknown,
2255 queuedRunId?: RunId,
2256 ) {
2257 const projection = yield* projectionStore.getThreadRecords(
2258 threadId,
2259 ["runs", "nodes", "attempts", "providerThreads", "turnItems", "messages"],
2260 { turnItemTypes: ["user_message"], messageRoles: ["user"] },
2261 );
2262 const queuedRun =
2263 queuedRunId === undefined
2264 ? nextQueuedRun(projection)
2265 : projection.runs.find((run) => run.id === queuedRunId && run.status === "queued");
2266 if (queuedRun === undefined) return;
2267 // A steer that fell back to the queue keeps its user item, hidden while the
2268 // run is queued. Republish it before the failure so clients whose bounded
2269 // snapshot omitted it can show the prompt beside the error.
2270 const retainedUserItems = projection.turnItems.filter(
2271 (item) => item.type === "user_message" && item.runId === queuedRun.id,
2272 );
2273 const now = yield* DateTime.now;
2274 const rootNode = projection.nodes.find((node) => node.id === queuedRun.rootNodeId);
2275 const attempt = projection.attempts.find((entry) => entry.id === queuedRun.activeAttemptId);
2276 const providerThread = projection.providerThreads.find(
2277 (entry) => entry.id === queuedRun.providerThreadId,
2278 );
2279 const handoffUnsupported =
2280 typeof cause === "object" &&
2281 cause !== null &&
2282 "_tag" in cause &&
2283 cause._tag === "CommandPolicyCapabilityUnsupportedError";
2284 const failureCause =
2285 typeof cause === "object" &&
2286 cause !== null &&
2287 "_tag" in cause &&
2288 cause._tag === "OrchestratorDispatchError" &&
2289 "cause" in cause
2290 ? cause.cause
2291 : cause;
2292 const failure = makeProviderFailure({
2293 cause: failureCause,
2294 code: handoffUnsupported ? "context_handoff_unsupported" : "queued_start_failed",
2295 class: handoffUnsupported ? "validation_error" : "unknown",
2296 });
2297 yield* writeSystemEvents([
2298 ...retainedUserItems.map((item) => ({
2299 type: "turn-item.updated" as const,
2300 threadId,
2301 runId: queuedRun.id,
2302 ...(rootNode === undefined ? {} : { nodeId: rootNode.id }),
2303 providerInstanceId: queuedRun.providerInstanceId,
2304 occurredAt: now,
2305 payload: { ...item, status: "failed" as const, completedAt: now, updatedAt: now },
2306 })),
2307 ...(attempt !== undefined && rootNode !== undefined
2308 ? [
2309 {
2310 type: "run-attempt.updated" as const,
2311 threadId,
2312 runId: queuedRun.id,
2313 nodeId: rootNode.id,
2314 providerInstanceId: queuedRun.providerInstanceId,
2315 occurredAt: now,
2316 payload: { ...attempt, status: "failed" as const, completedAt: now },
2317 },
2318 ]
2319 : []),
2320 ...(rootNode === undefined
2321 ? []
2322 : [
2323 {
2324 type: "node.updated" as const,
2325 threadId,
2326 runId: queuedRun.id,
2327 nodeId: rootNode.id,
2328 providerInstanceId: queuedRun.providerInstanceId,
2329 occurredAt: now,
2330 payload: { ...rootNode, status: "failed" as const, completedAt: now },
2331 },
2332 ]),
2333 ...(rootNode === undefined || providerThread === undefined
2334 ? []
2335 : [
2336 {
2337 type: "turn-item.updated" as const,
2338 threadId,
2339 runId: queuedRun.id,
2340 nodeId: rootNode.id,
2341 providerInstanceId: queuedRun.providerInstanceId,
2342 occurredAt: now,
2343 payload: {
2344 id: idAllocator.derive.turnItemFromProviderItem({
2345 driver: providerThread.driver,
2346 nativeItemId: `queued-start-failure:${queuedRun.id}`,
2347 }),
2348 threadId,
2349 runId: queuedRun.id,
2350 nodeId: rootNode.id,
2351 providerThreadId: providerThread.id,
2352 providerTurnId: null,
2353 nativeItemRef: null,
2354 parentItemId: null,
2355 ordinal: yield* nextTurnItemOrdinal(projection),
2356 status: "failed" as const,
2357 title: "Queued provider could not start",
2358 startedAt: now,
2359 completedAt: now,
2360 updatedAt: now,
2361 type: "error" as const,
2362 failure,
2363 },
2364 },
2365 ]),
2366 {
2367 type: "run.updated",
2368 threadId,
2369 runId: queuedRun.id,
2370 ...(rootNode === undefined ? {} : { nodeId: rootNode.id }),
2371 providerInstanceId: queuedRun.providerInstanceId,
2372 occurredAt: now,
2373 payload: { ...queuedRun, status: "failed", queuePosition: null, completedAt: now },
2374 },
2375 ]);
2376 });
2377
2378 const isStaleQuestionWake = Effect.fn("Orchestrator.isStaleQuestionWake")(function* (
2379 projection: Pick<OrchestrationV2ThreadProjection, "thread" | "subagents">,
2380 messageId: MessageId,
2381 notification: OrchestrationV2ThreadProjection["messages"][number]["notification"],
2382 ) {
2383 if (notification?.source.kind !== "delegated_task" || notification.outcome !== "updated")
2384 return false;
2385 const source = notification.source;
2386 const threadId = projection.thread.id;
2387 const task =
2388 projection.subagents.find((candidate) => source.taskIds.includes(candidate.id)) ??
2389 // Command projections omit settled tasks whose result was already handed over.
2390 (yield* projectionStore
2391 .getThreadRecords(threadId, ["subagents"], { subagentIds: source.taskIds })
2392 .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))))
2393 .subagents[0];
2394 // An unknown task gives no evidence the question is obsolete.
2395 if (task === undefined) return false;
2396 const childThreadId = task.childThreadId;
2397 if (
2398 childThreadId === null ||
2399 ["completed", "failed", "cancelled", "interrupted"].includes(task.status)
2400 )
2401 return true;
2402 const child = yield* projectionStore
2403 .getThreadRecords(childThreadId, ["runtimeRequests"])
2404 .pipe(
2405 Effect.mapError(
2406 (cause) => new OrchestratorProjectionError({ threadId: childThreadId, cause }),
2407 ),
2408 );
2409 return !child.runtimeRequests.some(
2410 (request) =>
2411 request.kind === "user_input" &&
2412 request.status === "pending" &&
2413 messageId === `message:delegated-task-question:${threadId}:${request.id}`,
2414 );
2415 });
2416
2417 const cancelStaleQuestionWakes = Effect.fn("Orchestrator.cancelStaleQuestionWakes")(function* (
2418 projection: OrchestrationV2ThreadProjection,
2419 ) {
2420 const events = yield* Ref.make<Array<OrchestrationV2DomainEvent>>([]);
2421 for (const run of projection.runs) {
2422 if (run.status !== "queued") continue;
2423 const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
2424 if (
2425 message === undefined ||
2426 !(yield* isStaleQuestionWake(projection, message.id, message.notification).pipe(
2427 // A child thread read must not fail the parent's unrelated next queued run.
2428 Effect.catchTags({
2429 OrchestratorProjectionError: (cause) =>
2430 Effect.logWarning("Failed to check a queued task question wake", {
2431 runId: run.id,
2432 cause,
2433 }).pipe(Effect.as(false)),
2434 }),
2435 ))
2436 )
2437 continue;
2438 yield* emitQueuedRunCancellation({
2439 command: {
2440 type: "queued-run.cancel",
2441 commandId: CommandId.make(`command:system:stale-question:${run.id}`),
2442 threadId: projection.thread.id,
2443 runId: run.id,
2444 },
2445 events,
2446 projection,
2447 run,
2448 now: yield* DateTime.now,
2449 });
2450 }
2451 const cancellations = yield* Ref.get(events);
2452 if (cancellations.length === 0) return false;
2453 yield* writeSystemEvents(cancellations);
2454 return true;
2455 });
2456
2457 const startNextQueuedRunOnce = Effect.fn("Orchestrator.startNextQueuedRun")(
2458 function* (
2459 threadId: ThreadId,
2460 options?: {
2461 readonly failedRunId?: RunId;
2462 readonly endedRunId?: RunId;
2463 readonly queuedRunId?: RunId;
2464 /** Treat the latest executed run as the one that just ended, as its run-end handler would. */
2465 readonly latestRunEnded?: boolean;
2466 readonly userStarted?: boolean;
2467 },
2468 ) {
2469 if (!(yield* projectionStore.canStartQueuedRun(threadId))) return;
2470 let projection = yield* readCommandProjection(threadId);
2471 if (
2472 projection.thread.archivedAt !== null ||
2473 projection.thread.deletedAt !== null ||
2474 projection.runs.some(isActiveRun)
2475 ) {
2476 return;
2477 }
2478
2479 if (yield* cancelStaleQuestionWakes(projection)) {
2480 projection = yield* readCommandProjection(threadId);
2481 }
2482 let storedQueuedRun =
2483 options?.queuedRunId === undefined
2484 ? nextQueuedRun(projection)
2485 : projection.runs.find(
2486 (run) => run.id === options.queuedRunId && run.status === "queued",
2487 );
2488 while (
2489 storedQueuedRun !== undefined &&
2490 isStalePullRequestWatchWake(projection, storedQueuedRun)
2491 ) {
2492 yield* writeSystemEvents(
2493 queuedRunCancellationEvents(projection, storedQueuedRun, yield* DateTime.now),
2494 );
2495 projection = yield* readCommandProjection(threadId);
2496 storedQueuedRun = nextQueuedRun(projection);
2497 }
2498 // A wake that outlived some of its pull requests tells only the current ones.
2499 const currentWake =
2500 storedQueuedRun === undefined
2501 ? undefined
2502 : currentPullRequestWatchWake(projection, storedQueuedRun);
2503 if (storedQueuedRun !== undefined && currentWake !== undefined) {
2504 const now = yield* DateTime.now;
2505 yield* writeSystemEvents([
2506 {
2507 type: "message.updated",
2508 threadId,
2509 runId: storedQueuedRun.id,
2510 ...(storedQueuedRun.rootNodeId === null ? {} : { nodeId: storedQueuedRun.rootNodeId }),
2511 providerInstanceId: storedQueuedRun.providerInstanceId,
2512 occurredAt: now,
2513 payload: { ...currentWake, updatedAt: now },
2514 },
2515 ]);
2516 const runId = storedQueuedRun.id;
2517 projection = yield* readCommandProjection(threadId);
2518 storedQueuedRun = projection.runs.find((run) => run.id === runId);
2519 }
2520 if (storedQueuedRun === undefined) {
2521 return;
2522 }
2523 const queuedMessage = projection.messages.find(
2524 (candidate) => candidate.id === storedQueuedRun.userMessageId,
2525 );
2526 const modelSelection = queuedRunModelSelection(projection, storedQueuedRun);
2527 const queuedRun = {
2528 ...storedQueuedRun,
2529 modelSelection,
2530 providerInstanceId: modelSelection.instanceId,
2531 };
2532 // A limited provider would fail and consume every waiting message.
2533 if (
2534 usageLimitHoldsQueue(
2535 projection,
2536 queuedRun.providerInstanceId,
2537 options?.userStarted === true ? undefined : yield* DateTime.now,
2538 )
2539 ) {
2540 return;
2541 }
2542 // Waits without a hold: lowering the thread's mode starts the queue again.
2543 if (isRuntimeModeAbove(projection.thread.runtimeMode, runtimeModeCeiling?.maxRuntimeMode)) {
2544 return;
2545 }
2546 // Failed worktree creation would run queued work in the project checkout; wait for the user.
2547 const failedRun = latestExecutedRun(projection.runs);
2548 const endedRunId = options?.latestRunEnded === true ? failedRun?.id : options?.endedRunId;
2549 const failedRunId =
2550 options?.latestRunEnded === true && failedRun?.status === "failed"
2551 ? failedRun.id
2552 : options?.failedRunId;
2553 const failureClass =
2554 failedRun && failedRunId !== undefined && failedRun.id === failedRunId
2555 ? latestRootProviderFailure(failedRun, projection.turnItems)?.class
2556 : undefined;
2557 const worktreeMissing =
2558 endedRunId !== undefined &&
2559 failedRun?.id === endedRunId &&
2560 failedRun?.workspacePreparation?.type === "worktree" &&
2561 projection.thread.worktreePath === null;
2562 const settings = yield* serverSettings.getSettings.pipe(Effect.orDie);
2563 const projectOverride =
2564 settings.projectSettingsOverrides[projection.thread.projectId]?.providerInstances?.[
2565 queuedRun.providerInstanceId
2566 ]?.enabled;
2567 const machineInstance = Object.hasOwn(
2568 settings.providerInstances,
2569 queuedRun.providerInstanceId,
2570 )
2571 ? settings.providerInstances[queuedRun.providerInstanceId]
2572 : undefined;
2573 const instanceDisabledForProject =
2574 projectOverride !== undefined
2575 ? !projectOverride
2576 : machineInstance !== undefined && !resolveProviderInstanceEnabled(machineInstance);
2577 const providerFailed =
2578 failureClass !== undefined &&
2579 failureClass !== "validation_error" &&
2580 failedRun?.providerInstanceId === queuedRun.providerInstanceId;
2581 if (worktreeMissing || instanceDisabledForProject || providerFailed) {
2582 // A buffered wake prompts nothing: it replays output the CLI already produced.
2583 const wakes =
2584 worktreeMissing || instanceDisabledForProject
2585 ? []
2586 : queuedRunsInDeliveryOrder(projection).filter((run) => {
2587 const message = projection.messages.find(
2588 (candidate) => candidate.id === run.userMessageId,
2589 );
2590 return (
2591 run.queueHeld !== true &&
2592 message !== undefined &&
2593 isBufferedProviderNotification(message)
2594 );
2595 });
2596 const now = yield* DateTime.now;
2597 yield* writeSystemEvents(
2598 projection.runs
2599 .filter((run) => run.status === "queued" && !wakes.some((wake) => wake.id === run.id))
2600 .map((run) => ({
2601 type: "run.updated" as const,
2602 threadId,
2603 runId: run.id,
2604 providerInstanceId: run.providerInstanceId,
2605 occurredAt: now,
2606 payload: { ...run, queueHeld: true },
2607 })),
2608 );
2609 return wakes.length > 0;
2610 }
2611 // Move queued ordinals after runs that started while they waited to preserve transcript and checkpoint order.
2612 // Started runs have prompts; cancelled queued messages do not.
2613 const newerRunIds = projection.runs
2614 .filter((run) => run.status !== "queued" && run.ordinal > queuedRun.ordinal)
2615 .map((run) => run.id);
2616 const startedAhead =
2617 newerRunIds.length > 0 &&
2618 (yield* projectionStore.getThreadRecords(threadId, ["turnItems"], {
2619 turnItemTypes: ["user_message", "notification"],
2620 turnItemRunIds: newerRunIds,
2621 })).turnItems.length > 0;
2622 const ordinal = startedAhead ? nextRunOrdinal(projection) : queuedRun.ordinal;
2623 const commandId = CommandId.make(`command:system:start-queued:${queuedRun.id}`);
2624 const rootNodeId = queuedRun.rootNodeId;
2625 const attemptId = queuedRun.activeAttemptId;
2626 const providerThreadId = queuedRun.providerThreadId;
2627 if (rootNodeId === null || attemptId === null || providerThreadId === null) {
2628 return yield* new OrchestratorDispatchError({
2629 commandId,
2630 commandType: "message.dispatch",
2631 cause: `Queued run ${queuedRun.id} is missing execution identity.`,
2632 });
2633 }
2634
2635 const rootNode = projection.nodes.find((candidate) => candidate.id === rootNodeId);
2636 const attempt = projection.attempts.find((candidate) => candidate.id === attemptId);
2637 const legacyQueuedTurnItem = projection.turnItems.find(
2638 (
2639 candidate,
2640 ): candidate is Extract<OrchestrationV2TurnItem, { readonly type: "user_message" }> =>
2641 candidate.type === "user_message" &&
2642 candidate.runId === queuedRun.id &&
2643 candidate.messageId === queuedRun.userMessageId,
2644 );
2645 const storedProviderThread = projection.providerThreads.find(
2646 (candidate) => candidate.id === providerThreadId,
2647 );
2648 const storedCheckpointScope = projection.checkpointScopes.find(
2649 (scope) => scope.id === rootNode?.checkpointScopeId,
2650 );
2651 if (
2652 rootNode === undefined ||
2653 attempt === undefined ||
2654 queuedMessage === undefined ||
2655 storedProviderThread === undefined ||
2656 (rootNode.checkpointScopeId !== null && storedCheckpointScope === undefined)
2657 ) {
2658 return yield* new OrchestratorDispatchError({
2659 commandId,
2660 commandType: "message.dispatch",
2661 cause: `Queued run ${queuedRun.id} is missing projection state.`,
2662 });
2663 }
2664 // A scheduled delivery starts on the thread's current pick, so /goal must suit that provider.
2665 yield* ensureGoalCommandSupported(
2666 { commandId, type: "message.dispatch" },
2667 queuedMessage,
2668 queuedRun.providerInstanceId,
2669 );
2670
2671 const now = yield* DateTime.now;
2672 const targetAdapter = yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
2673 Effect.mapError(
2674 (cause) =>
2675 new OrchestratorDispatchError({
2676 commandId,
2677 commandType: "message.dispatch",
2678 cause,
2679 }),
2680 ),
2681 );
2682 const followsCurrent = followsCurrentConversation(queuedMessage);
2683 const freshContext = queuedRun.freshContext === true;
2684 // A scheduled fresh run is a side conversation, so it must not drop the fork's context.
2685 const supersededForkTransfer =
2686 freshContext && queuedMessage.scheduledTaskId === undefined
2687 ? pendingForkTransferForThread(projection)
2688 : undefined;
2689 const activeProviderThread = dispatchProviderThread(projection, queuedMessage, freshContext);
2690 const contextBoundary = freshContext ? ordinal : contextBoundaryOrdinal(projection);
2691 // A run queued before a fresh run started is bound to the conversation that run left behind.
2692 const followsLatestContext =
2693 !freshContext &&
2694 (followsCurrent || providerThreadPredatesBoundary(storedProviderThread, contextBoundary));
2695 const targetProviderThread = followsLatestContext
2696 ? activeProviderThread?.providerInstanceId === queuedRun.providerInstanceId &&
2697 !providerThreadPredatesBoundary(activeProviderThread, contextBoundary)
2698 ? activeProviderThread
2699 : rootProviderThreadsForProvider(projection, queuedRun.providerInstanceId)[0]
2700 : freshContext &&
2701 // The reserved row belongs to the instance the run queued on, which may have changed.
2702 (storedProviderThread.providerInstanceId !== queuedRun.providerInstanceId ||
2703 storedProviderThread.firstRunOrdinal !== null ||
2704 storedProviderThread.nativeThreadRef !== null)
2705 ? undefined
2706 : storedProviderThread;
2707 const queuedProviderThread: OrchestrationV2ProviderThread = targetProviderThread ?? {
2708 id: idAllocator.derive.providerThread({
2709 driver: targetAdapter.driver,
2710 // The queue reserved `pending:<run>` for this run; another run may have taken it.
2711 nativeThreadId: freshContext
2712 ? `pending:${queuedRun.id}:fresh`
2713 : `pending:${queuedRun.id}`,
2714 }),
2715 driver: targetAdapter.driver,
2716 providerInstanceId: queuedRun.providerInstanceId,
2717 providerSessionId: null,
2718 appThreadId: threadId,
2719 ownerNodeId: null,
2720 nativeThreadRef: null,
2721 nativeConversationHeadRef: null,
2722 status: "not_loaded",
2723 firstRunOrdinal: null,
2724 lastRunOrdinal: null,
2725 handoffIds: [],
2726 forkedFrom: null,
2727 createdAt: now,
2728 updatedAt: now,
2729 };
2730 const selectionChanged = !modelSelectionsEqual(
2731 projection.thread.modelSelection,
2732 queuedRun.modelSelection,
2733 );
2734 // The picker may already have moved ahead of the native conversation.
2735 const switchFromActiveProvider =
2736 followsCurrent &&
2737 activeProviderThread !== undefined &&
2738 activeProviderThread.providerInstanceId !== queuedRun.providerInstanceId;
2739 const switchPlan =
2740 selectionChanged || switchFromActiveProvider
2741 ? yield* providerSwitchService
2742 .plan({
2743 projection: switchFromActiveProvider
2744 ? {
2745 ...projection,
2746 thread: {
2747 ...projection.thread,
2748 modelSelection: {
2749 ...projection.thread.modelSelection,
2750 instanceId: activeProviderThread.providerInstanceId,
2751 },
2752 },
2753 }
2754 : projection,
2755 targetModelSelection: queuedRun.modelSelection,
2756 })
2757 .pipe(
2758 Effect.mapError(
2759 (cause) =>
2760 new OrchestratorDispatchError({
2761 commandId,
2762 commandType: "message.dispatch",
2763 cause,
2764 }),
2765 ),
2766 )
2767 : null;
2768 const releasableSessions = uniqueSessions([
2769 ...projection.providerSessions.filter(
2770 (session) =>
2771 switchPlan?.releaseProviderSessionIds.includes(session.id) &&
2772 session.status !== "stopped" &&
2773 session.status !== "error",
2774 ),
2775 ...(freshContext ? sessionsReleasedByFreshContext(projection, activeProviderThread) : []),
2776 ]);
2777 const busySessionIds = new Set<ProviderSessionId>();
2778 for (const session of releasableSessions) {
2779 if (yield* sessionHasBackgroundWork(session, projection.providerThreads)) {
2780 busySessionIds.add(session.id);
2781 }
2782 }
2783 // Nobody waits on a queued start to refuse it. A handoff runs in its own session, so a
2784 // busy one stays up; a restart resumes the busy session's native thread, so wait instead.
2785 // A fresh run resumes nothing, so it never waits.
2786 if (
2787 busySessionIds.size > 0 &&
2788 !freshContext &&
2789 switchPlan?.transition.type === "restart_and_resume"
2790 ) {
2791 yield* Effect.logInfo(
2792 "Holding the queue: the replaced provider session still runs background work",
2793 { threadId, runId: queuedRun.id, providerSessionIds: Array.from(busySessionIds) },
2794 );
2795 yield* writeSystemEvents(
2796 projection.runs
2797 .filter((run) => run.status === "queued")
2798 .map((run) => ({
2799 type: "run.updated" as const,
2800 threadId,
2801 runId: run.id,
2802 providerInstanceId: run.providerInstanceId,
2803 occurredAt: now,
2804 payload: { ...run, queueHeld: true },
2805 })),
2806 );
2807 return;
2808 }
2809 const sessionsToDetach = releasableSessions.filter(
2810 (session) => !busySessionIds.has(session.id),
2811 );
2812 const canResumeAcrossInstances =
2813 !freshContext &&
2814 switchPlan?.instanceChanged === true &&
2815 switchPlan.transition.type === "restart_and_resume" &&
2816 activeProviderThread !== undefined &&
2817 activeProviderThread.nativeThreadRef !== null;
2818 const deliveryProviderThread = canResumeAcrossInstances
2819 ? {
2820 ...queuedProviderThread,
2821 status: activeProviderThread.status,
2822 nativeThreadRef: activeProviderThread.nativeThreadRef,
2823 nativeConversationHeadRef: activeProviderThread.nativeConversationHeadRef,
2824 nativeMetadata: activeProviderThread.nativeMetadata,
2825 }
2826 : queuedProviderThread;
2827 const targetCapabilities = yield* targetAdapter.getCapabilities().pipe(
2828 Effect.mapError(
2829 (cause) =>
2830 new OrchestratorDispatchError({
2831 commandId,
2832 commandType: "message.dispatch",
2833 cause,
2834 }),
2835 ),
2836 );
2837 const latestCompletedRun = projection.runs.findLast((run) => run.status === "completed");
2838 const latestHandoffRun = projection.runs.findLast(isHandoffSourceRun);
2839 const targetLastCompletedRun = lastDeliveredRunForProviderThread(
2840 projection,
2841 queuedProviderThread.id,
2842 );
2843 const coveredRuns =
2844 freshContext ||
2845 canResumeAcrossInstances ||
2846 latestHandoffRun === undefined ||
2847 latestHandoffRun.providerInstanceId === queuedRun.providerInstanceId
2848 ? []
2849 : projection.runs.filter(
2850 (run) =>
2851 isHandoffSourceRun(run) &&
2852 run.ordinal >= contextBoundary &&
2853 run.ordinal > (targetLastCompletedRun?.ordinal ?? 0) &&
2854 run.ordinal <= latestHandoffRun.ordinal,
2855 );
2856 const needsFullContext = deliveryProviderThread.nativeThreadRef === null;
2857 const legacyImportItems =
2858 projection.thread.historyOrigin === "v1_import" && contextBoundary === 0
2859 ? yield* readHandoffItems(threadId, [null])
2860 : [];
2861 const handoffStrategy = needsFullContext
2862 ? ("full_thread_summary" as const)
2863 : ("delta_since_target_last_seen" as const);
2864 const transferId =
2865 coveredRuns.length === 0
2866 ? null
2867 : yield* idAllocator.allocate
2868 .contextTransfer({
2869 sourceThreadId: threadId,
2870 targetThreadId: threadId,
2871 type: "provider_handoff",
2872 })
2873 .pipe(
2874 Effect.mapError(
2875 (cause) =>
2876 new OrchestratorDispatchError({
2877 commandId,
2878 commandType: "message.dispatch",
2879 cause,
2880 }),
2881 ),
2882 );
2883 if (transferId !== null) {
2884 yield* commandPolicy.ensureContextHandoff({
2885 commandId,
2886 threadId,
2887 providerInstanceId: queuedRun.providerInstanceId,
2888 capabilities: targetCapabilities,
2889 strategy: needsFullContext ? "full_thread_summary" : "delta_context",
2890 });
2891 }
2892 const handoff =
2893 transferId === null || latestHandoffRun === undefined
2894 ? null
2895 : yield* contextHandoffService
2896 .prepareProviderHandoff({
2897 threadId,
2898 targetRunId: queuedRun.id,
2899 transferId,
2900 fromProviderThreadIds: Array.from(
2901 new Set(
2902 coveredRuns.flatMap((run) =>
2903 run.providerThreadId === null ? [] : [run.providerThreadId],
2904 ),
2905 ),
2906 ),
2907 toProviderThreadId: queuedProviderThread.id,
2908 fromProviderInstanceId: latestHandoffRun.providerInstanceId,
2909 toProviderInstanceId: queuedRun.providerInstanceId,
2910 coveredRunOrdinals: {
2911 from: coveredRuns[0]!.ordinal,
2912 to: coveredRuns.at(-1)!.ordinal,
2913 },
2914 runs: projection.runs,
2915 strategy: handoffStrategy,
2916 items: [
2917 ...(needsFullContext && contextBoundary === 0
2918 ? yield* readInheritedItems(projection.thread)
2919 : []),
2920 ...(needsFullContext && latestCompletedRun !== undefined
2921 ? legacyImportItems
2922 : []),
2923 ...(yield* readHandoffItems(
2924 threadId,
2925 coveredRuns.map((run) => run.id),
2926 )),
2927 ],
2928 createdAt: now,
2929 })
2930 .pipe(
2931 Effect.mapError(
2932 (cause) =>
2933 new OrchestratorDispatchError({
2934 commandId,
2935 commandType: "message.dispatch",
2936 cause,
2937 }),
2938 ),
2939 );
2940 const legacyImportRecoveryHandoff =
2941 latestCompletedRun === undefined && needsFullContext && legacyImportItems.length > 0
2942 ? yield* contextHandoffService
2943 .prepareLegacyImport({
2944 threadId,
2945 targetRunId: queuedRun.id,
2946 toProviderThreadId: queuedProviderThread.id,
2947 toProviderInstanceId: queuedRun.providerInstanceId,
2948 items: legacyImportItems,
2949 createdAt: now,
2950 })
2951 .pipe(
2952 Effect.mapError(
2953 (cause) =>
2954 new OrchestratorDispatchError({
2955 commandId,
2956 commandType: "message.dispatch",
2957 cause,
2958 }),
2959 ),
2960 )
2961 : null;
2962 const activeHandoff = handoff ?? legacyImportRecoveryHandoff;
2963 const checkpointScope =
2964 storedCheckpointScope ??
2965 (yield* runtimePolicy
2966 .resolve({ thread: projection.thread, modelSelection: queuedRun.modelSelection })
2967 .pipe(
2968 Effect.flatMap((resolvedRuntimePolicy) =>
2969 checkpointService.prepareRootRunScope({
2970 threadId,
2971 runId: queuedRun.id,
2972 rootNodeId: rootNode.id,
2973 providerThreadId: queuedProviderThread.id,
2974 cwd: resolvedRuntimePolicy.cwd ?? projection.thread.worktreePath ?? process.cwd(),
2975 createdAt: now,
2976 }),
2977 ),
2978 Effect.mapError(
2979 (cause) =>
2980 new OrchestratorDispatchError({
2981 commandId,
2982 commandType: "message.dispatch",
2983 cause,
2984 }),
2985 ),
2986 ));
2987 const providerSessionId =
2988 (!canResumeAcrossInstances &&
2989 queuedProviderThread.providerSessionId !== null &&
2990 !switchPlan?.releaseProviderSessionIds.includes(queuedProviderThread.providerSessionId)
2991 ? queuedProviderThread.providerSessionId
2992 : null) ??
2993 (yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
2994 Effect.flatMap((adapter) =>
2995 providerSessionIdFor({
2996 adapter,
2997 providerInstanceId: queuedRun.providerInstanceId,
2998 threadId,
2999 }),
3000 ),
3001 Effect.mapError(
3002 (cause) =>
3003 new OrchestratorDispatchError({
3004 commandId,
3005 commandType: "message.dispatch",
3006 cause,
3007 }),
3008 ),
3009 ));
3010 const providerThread: OrchestrationV2ProviderThread = {
3011 ...deliveryProviderThread,
3012 providerSessionId,
3013 status: deliveryProviderThread.status === "error" ? "error" : "not_loaded",
3014 firstRunOrdinal: queuedProviderThread.firstRunOrdinal ?? ordinal,
3015 lastRunOrdinal: ordinal,
3016 handoffIds: appendContextHandoffId(
3017 appendContextHandoffId(queuedProviderThread.handoffIds, handoff?.id ?? null),
3018 legacyImportRecoveryHandoff?.id ?? null,
3019 ),
3020 updatedAt: now,
3021 };
3022 // Resume can clear the hold before the stopped run ends; its Stop marker still makes the drain explicit.
3023 const queueDelivery =
3024 endedRunId === undefined ||
3025 queuedRun.queueHeld === true ||
3026 (yield* projectionStore
3027 .getThreadRecords(threadId, ["turnItems"], {
3028 turnItemTypes: ["run_interrupt_request"],
3029 turnItemRunIds: [endedRunId],
3030 })
3031 .pipe(
3032 Effect.map(({ turnItems }) => turnItems.length > 0),
3033 Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
3034 ))
3035 ? "explicit-queue"
3036 : "automatic-queue";
3037 const startingRun: OrchestrationV2Run = {
3038 ...queuedRun,
3039 providerThreadId: queuedProviderThread.id,
3040 ordinal,
3041 status: "starting",
3042 queuePosition: null,
3043 startedAt: null,
3044 contextHandoffId: activeHandoff?.id ?? null,
3045 ...wakeWorkStartedAt(
3046 projection,
3047 {
3048 createdBy: queuedMessage.createdBy,
3049 creationSource: queuedMessage.creationSource,
3050 senderThreadId: queuedMessage.senderThreadId,
3051 notification: queuedMessage.notification,
3052 delegatedCompletion: queuedMessage.delegatedCompletion,
3053 restartContinuationOfRunId: queuedRun.restartContinuationOfRunId,
3054 },
3055 queueDelivery,
3056 ),
3057 };
3058 const userTurnItem: OrchestrationV2TurnItem = {
3059 ...(legacyQueuedTurnItem ?? {
3060 id: idAllocator.derive.userTurnItem({ messageId: queuedMessage.id }),
3061 threadId,
3062 runId: queuedRun.id,
3063 nodeId: rootNodeId,
3064 providerThreadId: queuedProviderThread.id,
3065 providerTurnId: null,
3066 nativeItemRef: null,
3067 parentItemId: null,
3068 ordinal: ordinal * 100,
3069 status: "completed",
3070 title: null,
3071 type: "user_message",
3072 messageId: queuedMessage.id,
3073 text: queuedMessage.text,
3074 attachments: queuedMessage.attachments,
3075 ...(queuedMessage.context ? { context: queuedMessage.context } : {}),
3076 createdBy: queuedMessage.createdBy,
3077 creationSource: queuedMessage.creationSource,
3078 ...(queuedMessage.scheduledTaskId === undefined
3079 ? {}
3080 : { scheduledTaskId: queuedMessage.scheduledTaskId }),
3081 ...(queuedMessage.senderThreadId === undefined
3082 ? {}
3083 : { senderThreadId: queuedMessage.senderThreadId }),
3084 ...(queuedMessage.linkedSender === undefined
3085 ? {}
3086 : { linkedSender: queuedMessage.linkedSender }),
3087 ...(queuedMessage.idleCompaction === undefined
3088 ? {}
3089 : { idleCompaction: queuedMessage.idleCompaction }),
3090 }),
3091 ...(freshContext ? { freshContext: true } : {}),
3092 providerThreadId: queuedProviderThread.id,
3093 inputIntent: "queued_turn",
3094 status: "completed",
3095 startedAt: now,
3096 completedAt: now,
3097 updatedAt: now,
3098 ...(activeHandoff === null ? modelSwitchSince(projection, queuedRun.modelSelection) : {}),
3099 };
3100 const handoffTurnItem: OrchestrationV2TurnItem | null =
3101 activeHandoff === null
3102 ? null
3103 : {
3104 id: idAllocator.derive.runSignalTurnItem({
3105 runId: queuedRun.id,
3106 signal: `context-handoff:${activeHandoff.id}`,
3107 }),
3108 threadId,
3109 runId: queuedRun.id,
3110 nodeId: rootNodeId,
3111 providerThreadId: queuedProviderThread.id,
3112 providerTurnId: null,
3113 nativeItemRef: null,
3114 parentItemId: null,
3115 ordinal: ordinal * 100 - 1,
3116 status: "completed",
3117 title: handoff === null ? "Imported context" : "Provider handoff",
3118 startedAt: now,
3119 completedAt: now,
3120 updatedAt: now,
3121 type: "handoff",
3122 contextHandoffId: activeHandoff.id,
3123 fromProviderThreadIds: activeHandoff.fromProviderThreadIds,
3124 toProviderThreadId: activeHandoff.toProviderThreadId,
3125 fromProviderInstanceIds: Array.from(
3126 new Set(coveredRuns.map((run) => run.providerInstanceId)),
3127 ),
3128 toProviderInstanceId: queuedRun.providerInstanceId,
3129 fromModelSelections: Array.from(
3130 new Map(
3131 coveredRuns.map((run) => [
3132 `${run.modelSelection.instanceId}\0${run.modelSelection.model}`,
3133 run.modelSelection,
3134 ]),
3135 ).values(),
3136 ),
3137 toModel: queuedRun.modelSelection.model,
3138 strategy: activeHandoff.strategy,
3139 summary: activeHandoff.summaryText,
3140 };
3141 const executionIdentityEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> =
3142 storedCheckpointScope === undefined ||
3143 storedProviderThread.id !== queuedProviderThread.id ||
3144 attempt.providerInstanceId !== queuedRun.providerInstanceId
3145 ? [
3146 {
3147 type: "checkpoint-scope.created",
3148 threadId,
3149 runId: queuedRun.id,
3150 nodeId: rootNodeId,
3151 providerInstanceId: queuedRun.providerInstanceId,
3152 occurredAt: now,
3153 payload: { ...checkpointScope, providerThreadId: queuedProviderThread.id },
3154 },
3155 {
3156 type: "node.updated",
3157 threadId,
3158 runId: queuedRun.id,
3159 nodeId: rootNodeId,
3160 providerInstanceId: queuedRun.providerInstanceId,
3161 occurredAt: now,
3162 payload: {
3163 ...rootNode,
3164 providerThreadId: queuedProviderThread.id,
3165 checkpointScopeId: checkpointScope.id,
3166 },
3167 },
3168 {
3169 type: "run-attempt.updated",
3170 threadId,
3171 runId: queuedRun.id,
3172 nodeId: rootNodeId,
3173 providerInstanceId: queuedRun.providerInstanceId,
3174 occurredAt: now,
3175 payload: {
3176 ...attempt,
3177 providerInstanceId: queuedRun.providerInstanceId,
3178 providerThreadId: queuedProviderThread.id,
3179 },
3180 },
3181 ]
3182 : [];
3183 const detachReason = freshContext
3184 ? FRESH_CONTEXT_DETACH_REASON
3185 : "Provider or model selection changed.";
3186 yield* writeSystemEvents(
3187 [
3188 ...(selectionChanged
3189 ? [
3190 {
3191 type:
3192 queuedRun.providerInstanceId === projection.thread.providerInstanceId
3193 ? ("thread.model-selection-updated" as const)
3194 : ("thread.provider-switched" as const),
3195 threadId,
3196 providerInstanceId: queuedRun.providerInstanceId,
3197 occurredAt: now,
3198 payload: {
3199 ...projection.thread,
3200 providerInstanceId: queuedRun.providerInstanceId,
3201 modelSelection: queuedRun.modelSelection,
3202 updatedAt: now,
3203 },
3204 },
3205 ]
3206 : []),
3207 ...(handoff === null || transferId === null || latestHandoffRun === undefined
3208 ? []
3209 : [
3210 {
3211 type: "context-transfer.created" as const,
3212 threadId,
3213 runId: queuedRun.id,
3214 providerInstanceId: queuedRun.providerInstanceId,
3215 occurredAt: now,
3216 payload: {
3217 id: transferId,
3218 type: "provider_handoff" as const,
3219 sourceThreadId: threadId,
3220 targetThreadId: threadId,
3221 sourcePoint: contextSourcePointForRun(projection, latestHandoffRun),
3222 basePoint:
3223 needsFullContext || targetLastCompletedRun === undefined
3224 ? null
3225 : contextSourcePointForRun(projection, targetLastCompletedRun),
3226 sourceProviderInstanceId: latestHandoffRun.providerInstanceId,
3227 targetProviderInstanceId: queuedRun.providerInstanceId,
3228 targetRunId: queuedRun.id,
3229 status: "consumed" as const,
3230 resolution: {
3231 strategy: needsFullContext
3232 ? ("portable_context" as const)
3233 : ("delta_context" as const),
3234 contextHandoffId: handoff.id,
3235 },
3236 createdBy: queuedMessage.createdBy,
3237 error: null,
3238 createdAt: now,
3239 updatedAt: now,
3240 consumedAt: now,
3241 },
3242 },
3243 {
3244 type: "context-handoff.updated" as const,
3245 threadId,
3246 runId: queuedRun.id,
3247 providerInstanceId: queuedRun.providerInstanceId,
3248 occurredAt: now,
3249 payload: handoff,
3250 },
3251 ]),
3252 ...(legacyImportRecoveryHandoff === null
3253 ? []
3254 : [
3255 {
3256 type: "context-handoff.updated" as const,
3257 threadId,
3258 runId: queuedRun.id,
3259 providerInstanceId: queuedRun.providerInstanceId,
3260 occurredAt: now,
3261 payload: legacyImportRecoveryHandoff,
3262 },
3263 ]),
3264 ...(supersededForkTransfer === undefined
3265 ? []
3266 : [
3267 forkTransferSupersededEvent(
3268 supersededForkTransfer,
3269 queuedRun.id,
3270 queuedRun.providerInstanceId,
3271 now,
3272 ),
3273 ]),
3274 ...(handoffTurnItem === null
3275 ? []
3276 : [
3277 {
3278 type: "turn-item.updated" as const,
3279 threadId,
3280 runId: queuedRun.id,
3281 nodeId: rootNodeId,
3282 providerInstanceId: queuedRun.providerInstanceId,
3283 occurredAt: now,
3284 payload: handoffTurnItem,
3285 },
3286 ]),
3287 ...sessionsToDetach.map((session) => ({
3288 type: "provider-session.detached" as const,
3289 threadId,
3290 driver: session.driver,
3291 providerInstanceId: session.providerInstanceId,
3292 occurredAt: now,
3293 payload: {
3294 providerSessionId: session.id,
3295 detachedAt: now,
3296 reason: detachReason,
3297 },
3298 })),
3299 ...executionIdentityEvents,
3300 {
3301 type: "provider-thread.updated",
3302 threadId,
3303 providerInstanceId: queuedRun.providerInstanceId,
3304 occurredAt: now,
3305 payload: providerThread,
3306 },
3307 {
3308 type: "turn-item.updated",
3309 threadId,
3310 runId: queuedRun.id,
3311 nodeId: rootNodeId,
3312 providerInstanceId: queuedRun.providerInstanceId,
3313 occurredAt: now,
3314 payload: notificationTurnItem(
3315 userTurnItem,
3316 queuedMessage,
3317 yield* notificationTasks(threadId, queuedMessage, projection.subagents),
3318 ),
3319 },
3320 {
3321 type: "run.updated",
3322 threadId,
3323 runId: queuedRun.id,
3324 nodeId: rootNodeId,
3325 providerInstanceId: queuedRun.providerInstanceId,
3326 occurredAt: now,
3327 payload: startingRun,
3328 },
3329 ],
3330 [
3331 ...sessionsToDetach.map((session) => ({
3332 id: `effect:${commandId}:provider-session.detach:${session.id}`,
3333 commandId,
3334 threadId,
3335 request: {
3336 type: "provider-session.detach" as const,
3337 providerSessionId: session.id,
3338 detail: detachReason,
3339 },
3340 })),
3341 {
3342 // A restart can return this run to the queue after its previous start effect was cancelled.
3343 id: `effect:${commandId}:provider-turn.start:${queuedRun.id}:${yield* idAllocator.allocate.event({ threadId, commandId })}`,
3344 commandId,
3345 threadId,
3346 request: { type: "provider-turn.start", runId: queuedRun.id },
3347 },
3348 ],
3349 );
3350 },
3351 (effect, threadId, options?) =>
3352 effect.pipe(
3353 Effect.catch((cause) => failQueuedRunStart(threadId, cause, options?.queuedRunId)),
3354 ),
3355 );
3356
3357 // A provider failure hold leaves buffered wakes unheld, so a second pass starts the first one.
3358 const startNextQueuedRun = (...args: Parameters<typeof startNextQueuedRunOnce>) =>
3359 startNextQueuedRunOnce(...args).pipe(
3360 Effect.flatMap((wakeLeftUnheld) =>
3361 wakeLeftUnheld === true ? startNextQueuedRunOnce(args[0]) : Effect.void,
3362 ),
3363 );
3364
3365 // Under load a resumed provider can time out before taking the prompt; the restart still owes the turn.
3366 const retryContinuationStart = Effect.fn("Orchestrator.retryContinuationStart")(function* (
3367 threadId: ThreadId,
3368 runId: RunId,
3369 ) {
3370 const projection = yield* projectionStore.getThreadRecords(
3371 threadId,
3372 ["runs", "providerTurns", "turnItems"],
3373 { turnItemTypes: ["error"], turnItemRunIds: [runId] },
3374 );
3375 const run = projection.runs.find((candidate) => candidate.id === runId);
3376 if (run === undefined) return;
3377 const delayMs = continuationStartRetryDelayMs(run, projection);
3378 if (delayMs === undefined) return;
3379 const commandId = CommandId.make(`command:restart-continuation-retry:${run.id}`);
3380 // A start retry is not a crash: carry the crash budget forward unchanged.
3381 const previous =
3382 run.restartContinuationOfRunId === undefined
3383 ? undefined
3384 : Option.getOrUndefined(
3385 yield* effectOutbox.get(
3386 `effect:restart-continuation:${run.restartContinuationOfRunId}`,
3387 ),
3388 )?.request;
3389 const processLossCount =
3390 previous?.type === "provider-runtime.continue" ? previous.processLossCount : undefined;
3391 yield* writeSystemEvents(
3392 [],
3393 [
3394 {
3395 id: `effect:restart-continuation:${run.id}`,
3396 commandId,
3397 threadId,
3398 request: {
3399 type: "provider-runtime.continue",
3400 sourceRunId: run.id,
3401 ...(processLossCount === undefined ? {} : { processLossCount }),
3402 },
3403 availableAt: DateTime.add(yield* DateTime.now, { milliseconds: delayMs }),
3404 },
3405 ],
3406 );
3407 });
3408
3409 // A hold waits for a person to review what they queued. Nobody reviews messages from agents,
3410 // the server or scheduled tasks, so a later completed run releases those unless it was stopped.
3411 const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
3412 threadId: ThreadId,
3413 completedRunId: RunId,
3414 ) {
3415 const held = yield* projectionStore.getHeldQueuedRuns(threadId);
3416 if (held.length === 0) return;
3417 const projection = yield* projectionStore.getThreadRecords(
3418 threadId,
3419 ["runs", "messages", "turnItems"],
3420 {
3421 messageIds: held.map((run) => run.userMessageId),
3422 turnItemTypes: ["run_interrupt_request"],
3423 turnItemRunIds: [completedRunId],
3424 },
3425 );
3426 // Only the newest run completing shows the thread moved past the hold; a late or
3427 // bookkeeping update to an older run, overtaken by a failure or Stop, does not.
3428 if (
3429 latestExecutedRun(projection.runs)?.id !== completedRunId ||
3430 projection.turnItems.length > 0 ||
3431 worktreeSetupPending(projection)
3432 )
3433 return;
3434 const { messages } = projection;
3435 const unattended = new Set(
3436 messages
3437 .filter((message) => message.createdBy !== "user" || message.scheduledTaskId !== undefined)
3438 .map((message) => message.id),
3439 );
3440 const now = yield* DateTime.now;
3441 const released = held.filter((run) => unattended.has(run.userMessageId));
3442 if (released.length === 0) return;
3443 yield* writeSystemEvents(
3444 released.map((run) => ({
3445 type: "run.updated" as const,
3446 threadId,
3447 runId: run.id,
3448 providerInstanceId: run.providerInstanceId,
3449 occurredAt: now,
3450 payload: { ...run, queueHeld: false },
3451 })),
3452 );
3453 });
3454
3455 // A steer still pending after its run ended was never read: the provider marks one it read
3456 // completed, and one Stop dropped cancelled. It runs next, in its place among waiting messages.
3457 const requeueUnreadSteers = Effect.fn("Orchestrator.requeueUnreadSteers")(function* (
3458 threadId: ThreadId,
3459 runIds?: ReadonlyArray<RunId>,
3460 ) {
3461 const { turnItems } = yield* projectionStore.getThreadRecords(threadId, ["turnItems"], {
3462 turnItemTypes: ["user_message"],
3463 turnItemStatuses: ["pending"],
3464 ...(runIds === undefined ? {} : { turnItemRunIds: runIds }),
3465 });
3466 const unread = turnItems.filter(
3467 (item) =>
3468 item.type === "user_message" &&
3469 (item.inputIntent === "steer" || item.inputIntent === "promoted_queued_to_steer"),
3470 );
3471 if (unread.length === 0) return;
3472 const projection = yield* projectionStore.getThreadRecords(threadId, ["runs", "messages"], {
3473 messageIds: unread.flatMap((item) => (item.type === "user_message" ? [item.messageId] : [])),
3474 });
3475 const placed = [];
3476 for (const item of unread) {
3477 const run = projection.runs.find((candidate) => candidate.id === item.runId);
3478 const message = projection.messages.find(
3479 (candidate) =>
3480 item.type === "user_message" &&
3481 candidate.id === item.messageId &&
3482 candidate.runId === item.runId,
3483 );
3484 if (run === undefined || message === undefined || isActiveRun(run)) continue;
3485 // Stop drops a steer it reached; say so, as the adapter does for steers it still held.
3486 if (yield* projectionStore.hasRunInterruptRequest(threadId, run.id)) {
3487 const now = yield* DateTime.now;
3488 yield* writeSystemEvents([
3489 {
3490 type: "turn-item.updated",
3491 threadId,
3492 runId: run.id,
3493 ...(item.nodeId === null ? {} : { nodeId: item.nodeId }),
3494 providerInstanceId: run.providerInstanceId,
3495 occurredAt: now,
3496 payload: { ...item, status: "cancelled" as const, completedAt: now, updatedAt: now },
3497 },
3498 {
3499 type: "turn-item.updated",
3500 threadId,
3501 runId: run.id,
3502 ...(item.nodeId === null ? {} : { nodeId: item.nodeId }),
3503 providerInstanceId: run.providerInstanceId,
3504 occurredAt: now,
3505 payload: {
3506 id: idAllocator.derive.runSignalTurnItem({
3507 runId: run.id,
3508 signal: `unread-steer:${message.id}`,
3509 }),
3510 threadId,
3511 runId: run.id,
3512 nodeId: item.nodeId,
3513 providerThreadId: item.providerThreadId ?? null,
3514 providerTurnId: item.providerTurnId ?? null,
3515 nativeItemRef: null,
3516 parentItemId: null,
3517 ordinal: item.ordinal,
3518 type: "system_notice" as const,
3519 status: "completed" as const,
3520 title: "Steer delivery not confirmed",
3521 message: `The agent did not confirm reading your steer before Stop. It may not have been delivered. Resend it if needed: "${message.text}"`,
3522 startedAt: now,
3523 completedAt: now,
3524 updatedAt: now,
3525 },
3526 },
3527 ]);
3528 continue;
3529 }
3530 // The message's first steer effect recorded which runs were queued before it was sent; a
3531 // queued message that was promoted also stays behind only the runs queued before its own.
3532 const firstSteer = Option.getOrUndefined(
3533 yield* effectOutbox.firstSteer(threadId, message.id),
3534 )?.request;
3535 const steer = firstSteer?.type === "provider-turn.steer" ? firstSteer : undefined;
3536 const afterRunOrdinal = Math.max(
3537 0,
3538 Math.min(
3539 steer?.steerFollowUpAfterRunOrdinal ??
3540 Math.max(...projection.runs.map((candidate) => candidate.ordinal)),
3541 ...projection.runs.flatMap((candidate) =>
3542 candidate.userMessageId === message.id ? [candidate.ordinal - 1] : [],
3543 ),
3544 ),
3545 );
3546 placed.push({ item, run, message, steer, afterRunOrdinal });
3547 }
3548 // Each requeue starts the queue once it commits, so earlier places go first.
3549 placed.sort(
3550 (left, right) =>
3551 left.afterRunOrdinal - right.afterRunOrdinal || left.item.ordinal - right.item.ordinal,
3552 );
3553 for (const { run, message, steer, afterRunOrdinal } of placed) {
3554 yield* dispatchWithReceiptEffect({
3555 type: "message.dispatch",
3556 commandId: CommandId.make(`command:unread-steer:${run.id}:${message.id}`),
3557 threadId,
3558 messageId: message.id,
3559 steerFollowUpOfRunId: run.id,
3560 steerFollowUpAfterRunOrdinal: afterRunOrdinal,
3561 modelSelection: steer?.modelSelection ?? run.modelSelection,
3562 text: message.text,
3563 ...(message.context ? { context: message.context } : {}),
3564 attachments: message.attachments,
3565 ...(message.delegatedCompletion === undefined
3566 ? {}
3567 : { delegatedCompletion: message.delegatedCompletion }),
3568 ...(message.notification === undefined ? {} : { notification: message.notification }),
3569 dispatchMode: { type: "queue_after_active" },
3570 createdBy: message.createdBy,
3571 creationSource: message.creationSource,
3572 ...(message.scheduledTaskId === undefined
3573 ? {}
3574 : { scheduledTaskId: message.scheduledTaskId }),
3575 ...(message.senderThreadId === undefined ? {} : { senderThreadId: message.senderThreadId }),
3576 ...(message.linkedSender === undefined ? {} : { linkedSender: message.linkedSender }),
3577 }).pipe(
3578 Effect.catchCause((cause) =>
3579 Effect.logWarning("Failed to requeue an unread steer", {
3580 threadId,
3581 messageId: message.id,
3582 cause,
3583 }),
3584 ),
3585 );
3586 }
3587 });
3588
3589 const resumeQueuedRuns = Effect.gen(function* () {
3590 const threadIds = yield* projectionStore.getRecoveryThreadIds("queued-runs");
3591 let resumed = 0;
3592 for (const threadId of threadIds) {
3593 const resumedThread = yield* Effect.gen(function* () {
3594 if (!(yield* projectionStore.canStartQueuedRun(threadId))) return false;
3595 yield* threadDispatch.withLock(threadId, startNextQueuedRun(threadId));
3596 return true;
3597 }).pipe(
3598 Effect.catch((cause) =>
3599 Effect.logWarning("Failed to resume queued V2 run after recovery", {
3600 threadId,
3601 cause,
3602 }).pipe(Effect.as(false)),
3603 ),
3604 );
3605 if (resumedThread) {
3606 resumed += 1;
3607 }
3608 }
3609 return resumed;
3610 }).pipe(
3611 Effect.mapError(
3612 (cause) =>
3613 new OrchestratorDispatchError({
3614 commandId: CommandId.make("command:system:resume-queued-runs"),
3615 commandType: "message.dispatch",
3616 cause,
3617 }),
3618 ),
3619 );
3620
3621 const dispatchDelegatedTaskCompletionDeliveryResolution = Effect.fn(
3622 "Orchestrator.dispatchDelegatedTaskCompletionDeliveryResolution",
3623 )(function* (
3624 command: Extract<
3625 OrchestrationV2Command,
3626 {
3627 readonly type:
3628 | "delegated_task.completion-delivery.acknowledge"
3629 | "delegated_task.completion-delivery.dispose";
3630 }
3631 >,
3632 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
3633 ) {
3634 const projection = yield* loadProjectionForCommand(command, ["subagents"], {
3635 subagentIds: [command.taskId],
3636 });
3637 const task = projection.subagents.find(
3638 (candidate) => candidate.id === command.taskId && candidate.origin === "app_owned",
3639 );
3640 if (task === undefined) {
3641 return yield* new OrchestratorDispatchError({
3642 commandId: command.commandId,
3643 commandType: command.type,
3644 cause: `Delegated task ${command.taskId} is not an app-owned task of thread ${command.parentThreadId}.`,
3645 });
3646 }
3647 const state =
3648 command.type === "delegated_task.completion-delivery.acknowledge"
3649 ? "acknowledged"
3650 : "disposed";
3651 const now = yield* DateTime.now;
3652 const emitEvent = emit(events, command);
3653 const {
3654 runs: [parentRun],
3655 } = yield* loadProjectionForCommand(command, ["runs"], {
3656 runIds: task.runId === null ? [] : [task.runId],
3657 });
3658 // A late wait can arrive after finalization held the result for its still-live spawning run.
3659 if (isUnobservedAcknowledgement(command)) {
3660 if (
3661 !isTerminalDelegatedTaskStatus(task.status) ||
3662 task.completionDelivery?.state !== "pending"
3663 )
3664 return;
3665 if (parentRun !== undefined && isLiveRun(parentRun)) return;
3666 const records = yield* loadProjectionForCommand(
3667 command,
3668 ["runs", "subagents", "messages", "providerTurns"],
3669 { messageRoles: ["user"] },
3670 );
3671 const plan = yield* planDelegatedCompletionDelivery({
3672 parentProjection: records,
3673 parentRun,
3674 task,
3675 updatedTask: task,
3676 now,
3677 }).pipe(mapDispatchError(command));
3678 if (plan.task.completionDelivery?.state === "pending") return;
3679 yield* emitEvent({
3680 type: "subagent.updated",
3681 threadId: command.parentThreadId,
3682 ...(task.runId === null ? {} : { runId: task.runId }),
3683 nodeId: task.id,
3684 driver: task.driver,
3685 providerInstanceId: task.providerInstanceId,
3686 occurredAt: now,
3687 payload: plan.task,
3688 });
3689 for (const event of delegatedCompletionPlanEvents(
3690 command.parentThreadId,
3691 plan,
3692 projection.thread.providerInstanceId,
3693 now,
3694 )) {
3695 yield* emitEvent(event);
3696 }
3697 return plan.parentRun?.id;
3698 }
3699 // task_status and t3_thread_read use distinct command IDs, so two
3700 // valid observations can race after their read preflight. Re-emit the
3701 // existing task row so the second dispatch is a successful idempotent
3702 // no-op rather than "already acknowledged/disposed" or empty-events.
3703 if (
3704 task.completionDelivery?.state === state ||
3705 (command.type === "delegated_task.completion-delivery.acknowledge" &&
3706 task.completionDelivery?.state === "disposed")
3707 ) {
3708 yield* emitEvent({
3709 type: "subagent.updated",
3710 threadId: command.parentThreadId,
3711 ...(task.runId === null ? {} : { runId: task.runId }),
3712 nodeId: task.id,
3713 driver: task.driver,
3714 providerInstanceId: task.providerInstanceId,
3715 occurredAt: now,
3716 payload: task,
3717 });
3718 return;
3719 }
3720 const updatedTask: OrchestrationV2Subagent = {
3721 ...task,
3722 completionDelivery: {
3723 state,
3724 observedByRunId:
3725 command.type === "delegated_task.completion-delivery.acknowledge"
3726 ? command.observedByRunId
3727 : null,
3728 },
3729 updatedAt: now,
3730 };
3731 yield* emitEvent({
3732 type: "subagent.updated",
3733 threadId: command.parentThreadId,
3734 ...(task.runId === null ? {} : { runId: task.runId }),
3735 nodeId: task.id,
3736 driver: task.driver,
3737 providerInstanceId: task.providerInstanceId,
3738 occurredAt: now,
3739 payload: updatedTask,
3740 });
3741
3742 const cohort = parentRun?.delegatedCompletion;
3743 const delivery = cohort?.delivery;
3744 if (
3745 parentRun === undefined ||
3746 cohort === undefined ||
3747 delivery === null ||
3748 delivery === undefined
3749 ) {
3750 return;
3751 }
3752 if (!delivery.taskIds.includes(task.id)) {
3753 return;
3754 }
3755
3756 const { messages } = yield* loadProjectionForCommand(command, ["messages"], {
3757 messageRoles: ["user"],
3758 messageIds: [delivery.messageId],
3759 });
3760 const deliveryRunId = completionDeliveryMessage({ messages }, delivery)?.runId ?? null;
3761 const { runs } = yield* loadProjectionForCommand(command, ["runs"], {
3762 runIds: deliveryRunId === null ? [] : [deliveryRunId],
3763 });
3764 const deliveryProjection = { messages, runs };
3765 const remainingTaskIds = delivery.taskIds.filter((taskId) => taskId !== task.id);
3766 const deliveryRun = completionDeliveryRun(deliveryProjection, delivery);
3767 const clearDelivery =
3768 remainingTaskIds.length === 0 &&
3769 (deliveryRun === undefined || deliveryRun.status === "queued");
3770 const updatedCohort: OrchestrationV2DelegatedCompletionCohort = {
3771 ...cohort,
3772 delivery: clearDelivery
3773 ? null
3774 : {
3775 ...delivery,
3776 taskIds: remainingTaskIds,
3777 },
3778 };
3779 yield* emitEvent({
3780 type: "run.updated",
3781 threadId: command.parentThreadId,
3782 runId: parentRun.id,
3783 ...(parentRun.rootNodeId === null ? {} : { nodeId: parentRun.rootNodeId }),
3784 providerInstanceId: parentRun.providerInstanceId,
3785 occurredAt: now,
3786 payload: {
3787 ...parentRun,
3788 delegatedCompletion: updatedCohort,
3789 },
3790 });
3791
3792 if (deliveryRun?.status === "queued") {
3793 if (remainingTaskIds.length === 0) {
3794 const rootNodeFilter = {
3795 nodeIds: deliveryRun.rootNodeId === null ? [] : [deliveryRun.rootNodeId],
3796 };
3797 yield* emitQueuedRunCancellation({
3798 command,
3799 events,
3800 // Attempts have no id filter; a wake that never started has no attempt to cancel.
3801 projection:
3802 deliveryRun.activeAttemptId === null
3803 ? {
3804 attempts: [],
3805 ...(yield* loadProjectionForCommand(command, ["nodes"], rootNodeFilter)),
3806 }
3807 : yield* loadProjectionForCommand(command, ["nodes", "attempts"], rootNodeFilter),
3808 run: deliveryRun,
3809 now,
3810 });
3811 return parentRun.id;
3812 }
3813 const message = completionDeliveryMessage(deliveryProjection, delivery);
3814 if (message !== undefined) {
3815 const { subagents: remainingTasks } = yield* loadProjectionForCommand(
3816 command,
3817 ["subagents"],
3818 { subagentIds: remainingTaskIds },
3819 );
3820 yield* emitEvent({
3821 type: "message.updated",
3822 threadId: command.parentThreadId,
3823 runId: deliveryRun.id,
3824 ...(deliveryRun.rootNodeId === null ? {} : { nodeId: deliveryRun.rootNodeId }),
3825 providerInstanceId: deliveryRun.providerInstanceId,
3826 occurredAt: now,
3827 payload: {
3828 ...message,
3829 text: delegatedCompletionWakeDetail(remainingTaskIds, remainingTasks),
3830 delegatedCompletion: {
3831 parentRunId: parentRun.id,
3832 generation: delivery.generation,
3833 taskIds: remainingTaskIds,
3834 },
3835 updatedAt: now,
3836 },
3837 });
3838 }
3839 }
3840 return parentRun.id;
3841 });
3842
3843 const disposeDelegatedCompletionCohort = Effect.fn(
3844 "Orchestrator.disposeDelegatedCompletionCohort",
3845 )(function* (input: {
3846 readonly command: OrchestrationV2ServerCommand;
3847 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
3848 readonly projection: Pick<
3849 OrchestrationV2ThreadProjection,
3850 "runs" | "subagents" | "messages" | "nodes" | "attempts"
3851 >;
3852 readonly parentRunId: RunId;
3853 readonly disposition: "stopped" | "disposed";
3854 readonly now: DateTime.Utc;
3855 readonly cancelQueuedDelivery?: boolean;
3856 // An interrupted or cancelled wake disposes only the results it carried. The cohort stays
3857 // open, and its delivery stays the wake's, so the wake's terminal reserves any sibling's result.
3858 readonly interruptedWakeTaskIds?: ReadonlyArray<NodeId>;
3859 }) {
3860 const parentRun = input.projection.runs.find((candidate) => candidate.id === input.parentRunId);
3861 if (parentRun === undefined) {
3862 return;
3863 }
3864 const cohort = parentRun.delegatedCompletion;
3865 const tasks = input.projection.subagents.filter(
3866 (candidate) => candidate.origin === "app_owned" && candidate.runId === input.parentRunId,
3867 );
3868 if (cohort === undefined && tasks.length === 0) {
3869 return;
3870 }
3871 const emitEvent = emit(input.events, input.command);
3872 const wakeTaskIds = input.interruptedWakeTaskIds;
3873 const keptTaskIds = new Set(
3874 wakeTaskIds !== undefined && cohort?.disposition === "open"
3875 ? tasks
3876 .filter(
3877 (task) =>
3878 !wakeTaskIds.includes(task.id) &&
3879 task.completionDelivery?.state !== "acknowledged" &&
3880 task.completionDelivery?.state !== "delivered" &&
3881 task.completionDelivery?.state !== "disposed",
3882 )
3883 .map((task) => task.id)
3884 : [],
3885 );
3886 const nextCohort =
3887 keptTaskIds.size > 0
3888 ? cohort!
3889 : ({
3890 disposition: cohort?.disposition === "disposed" ? "disposed" : input.disposition,
3891 nextGeneration: cohort?.nextGeneration ?? 1,
3892 delivery: null,
3893 } as const);
3894 yield* emitEvent({
3895 type: "run.updated",
3896 threadId: parentRun.threadId,
3897 runId: parentRun.id,
3898 ...(parentRun.rootNodeId === null ? {} : { nodeId: parentRun.rootNodeId }),
3899 providerInstanceId: parentRun.providerInstanceId,
3900 occurredAt: input.now,
3901 payload: {
3902 ...parentRun,
3903 delegatedCompletion: nextCohort,
3904 },
3905 });
3906 for (const task of tasks) {
3907 if (
3908 keptTaskIds.has(task.id) ||
3909 task.completionDelivery?.state === "acknowledged" ||
3910 task.completionDelivery?.state === "delivered" ||
3911 task.completionDelivery?.state === "disposed"
3912 ) {
3913 continue;
3914 }
3915 yield* emitEvent({
3916 type: "subagent.updated",
3917 threadId: parentRun.threadId,
3918 ...(task.runId === null ? {} : { runId: task.runId }),
3919 nodeId: task.id,
3920 driver: task.driver,
3921 providerInstanceId: task.providerInstanceId,
3922 occurredAt: input.now,
3923 payload: {
3924 ...task,
3925 completionDelivery: {
3926 state: "disposed",
3927 observedByRunId: null,
3928 },
3929 updatedAt: input.now,
3930 },
3931 });
3932 }
3933 const deliveryRun = completionDeliveryRun(input.projection, cohort?.delivery ?? null);
3934 if (
3935 keptTaskIds.size === 0 &&
3936 input.cancelQueuedDelivery !== false &&
3937 deliveryRun?.status === "queued"
3938 ) {
3939 yield* emitQueuedRunCancellation({
3940 command: input.command,
3941 events: input.events,
3942 projection: input.projection,
3943 run: deliveryRun,
3944 now: input.now,
3945 });
3946 }
3947 });
3948
3949 const disposeAllDelegatedCompletionCohorts = (input: {
3950 readonly command: OrchestrationV2Command;
3951 readonly events: Ref.Ref<Array<OrchestrationV2DomainEvent>>;
3952 readonly projection: Pick<
3953 OrchestrationV2ThreadProjection,
3954 "runs" | "subagents" | "messages" | "nodes" | "attempts"
3955 >;
3956 readonly now: DateTime.Utc;
3957 readonly cancelQueuedDelivery?: boolean;
3958 }) =>
3959 Effect.forEach(
3960 Array.from(
3961 new Set([
3962 ...input.projection.runs
3963 .filter((run) => run.delegatedCompletion !== undefined)
3964 .map((run) => run.id),
3965 ...input.projection.subagents
3966 .filter((task) => task.origin === "app_owned" && task.runId !== null)
3967 .map((task) => task.runId!),
3968 ]),
3969 ),
3970 (parentRunId) =>
3971 disposeDelegatedCompletionCohort({
3972 ...input,
3973 parentRunId,
3974 disposition: "disposed",
3975 }),
3976 { concurrency: 1, discard: true },
3977 );
3978
3979 const dispatchThreadCreate = Effect.fn("orchestrationV2.dispatch.threadCreate")(function* (
3980 command: Extract<OrchestrationV2Command, { readonly type: "thread.create" }>,
3981 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
3982 ) {
3983 yield* Effect.annotateCurrentSpan({
3984 "orchestration_v2.command_id": command.commandId,
3985 "orchestration_v2.command_type": command.type,
3986 "orchestration_v2.thread_id": command.threadId,
3987 "orchestration_v2.driver": command.modelSelection.instanceId,
3988 });
3989
3990 // Tombstones reserve their ids too; recreating one would overwrite its history.
3991 const existing = yield* findThread(command.threadId);
3992 if (existing !== null) {
3993 return yield* new OrchestratorThreadAlreadyExistsError({
3994 commandId: command.commandId,
3995 threadId: command.threadId,
3996 });
3997 }
3998
3999 const now = yield* DateTime.now;
4000 const emitEvent = emit(events, command);
4001 const thread: OrchestrationV2AppThread = {
4002 createdBy: command.createdBy,
4003 creationSource: command.creationSource,
4004 id: command.threadId,
4005 projectId: command.projectId,
4006 title: command.title,
4007 providerInstanceId: command.modelSelection.instanceId,
4008 modelSelection: command.modelSelection,
4009 runtimeMode: command.runtimeMode,
4010 interactionMode: command.interactionMode,
4011 branch: command.branch,
4012 worktreePath: command.worktreePath,
4013 activeProviderThreadId: null,
4014 pullRequests: [],
4015 ...(command.linkOrigin === undefined ? {} : { linkOrigin: command.linkOrigin }),
4016 // Only stamped work names a remote parent, so a client cannot claim one.
4017 ...(command.linkOrigin === undefined || command.delegatedFrom === undefined
4018 ? {}
4019 : { delegatedFrom: command.delegatedFrom }),
4020 lineage: {
4021 parentThreadId: null,
4022 relationshipToParent: null,
4023 rootThreadId: command.threadId,
4024 },
4025 forkedFrom: null,
4026 createdAt: now,
4027 updatedAt: now,
4028 archivedAt: null,
4029 settledOverride: null,
4030 settledAt: null,
4031 snoozedUntil: null,
4032 snoozedAt: null,
4033 lastVisitedAt: null,
4034 deletedAt: null,
4035 };
4036
4037 yield* emitEvent({
4038 type: "thread.created",
4039 threadId: command.threadId,
4040 providerInstanceId: command.modelSelection.instanceId,
4041 occurredAt: now,
4042 payload: thread,
4043 });
4044 if (command.importedNativeThread !== undefined) {
4045 yield* emitEvent({
4046 type: "provider-thread.updated",
4047 threadId: command.threadId,
4048 driver: command.importedNativeThread.ref.driver,
4049 providerInstanceId: command.modelSelection.instanceId,
4050 occurredAt: now,
4051 payload: {
4052 id: idAllocator.derive.providerThread({
4053 driver: command.importedNativeThread.ref.driver,
4054 providerInstanceId: command.modelSelection.instanceId,
4055 nativeThreadId: command.importedNativeThread.ref.nativeId,
4056 }),
4057 driver: command.importedNativeThread.ref.driver,
4058 providerInstanceId: command.modelSelection.instanceId,
4059 providerSessionId: null,
4060 appThreadId: command.threadId,
4061 ownerNodeId: null,
4062 nativeThreadRef: command.importedNativeThread.ref,
4063 nativeConversationHeadRef: null,
4064 status: "not_loaded",
4065 firstRunOrdinal: null,
4066 lastRunOrdinal: null,
4067 handoffIds: [],
4068 forkedFrom: null,
4069 contextUsage: null,
4070 nativeMetadata: command.importedNativeThread.metadata ?? null,
4071 createdAt: now,
4072 updatedAt: now,
4073 },
4074 });
4075 }
4076 });
4077
4078 const dispatchThreadVisit = Effect.fn("orchestrationV2.dispatch.threadVisit")(function* (
4079 command: Extract<OrchestrationV2Command, { readonly type: "thread.visit" }>,
4080 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
4081 ) {
4082 const thread = yield* projectionStore
4083 .getThread(command.threadId)
4084 .pipe(
4085 Effect.mapError(
4086 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
4087 ),
4088 );
4089 if (thread.deletedAt !== null) {
4090 return yield* new OrchestratorDispatchError({
4091 commandId: command.commandId,
4092 commandType: command.type,
4093 cause: `Thread ${command.threadId} is deleted.`,
4094 });
4095 }
4096 const visitedAt = DateTime.make(command.visitedAt);
4097 if (Option.isNone(visitedAt)) {
4098 return yield* new OrchestratorDispatchError({
4099 commandId: command.commandId,
4100 commandType: command.type,
4101 cause: `Thread ${command.threadId} visit time ${command.visitedAt} is not a valid timestamp.`,
4102 });
4103 }
4104 const movesForward =
4105 thread.lastVisitedAt === null ||
4106 DateTime.toEpochMillis(visitedAt.value) > DateTime.toEpochMillis(thread.lastVisitedAt);
4107 const occurredAt = yield* DateTime.now;
4108 // Opens are timed by the server clock so devices with skewed clocks agree.
4109 const opens = command.opened
4110 ? recordThreadOpen(
4111 thread.opens ?? null,
4112 DateTime.toEpochMillis(occurredAt),
4113 command.searchQuery,
4114 )
4115 : null;
4116 if (!movesForward && opens === null) return;
4117 // Viewing a thread changes read state only. Loading its transcript, bumping
4118 // updatedAt, or copying the thread into the event makes a read receipt scale with it.
4119 yield* emit(
4120 events,
4121 command,
4122 )({
4123 type: "thread.visit-recorded",
4124 threadId: command.threadId,
4125 providerInstanceId: thread.providerInstanceId,
4126 occurredAt,
4127 payload: {
4128 lastVisitedAt: movesForward ? visitedAt.value : thread.lastVisitedAt,
4129 ...(opens ? { opens } : {}),
4130 },
4131 });
4132 });
4133
4134 // Recheck under the lock after reading the host. Record the watch first so wake events carry it.
4135 const dispatchPullRequestWatchSync = Effect.fn("orchestrationV2.dispatch.pullRequestWatchSync")(
4136 function* (
4137 command: Extract<
4138 OrchestrationV2ServerCommand,
4139 { readonly type: "thread.pull-request-watch.sync" }
4140 >,
4141 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
4142 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
4143 ) {
4144 const thread = yield* projectionStore
4145 .getThread(command.threadId)
4146 .pipe(
4147 Effect.mapError(
4148 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
4149 ),
4150 );
4151 const { others = [], wake, ...primary } = command;
4152 const entries = [
4153 { ...primary, includesRemarks: wake?.includesRemarks },
4154 ...others.map((other) => ({
4155 type: primary.type,
4156 commandId: primary.commandId,
4157 threadId: primary.threadId,
4158 ...other,
4159 })),
4160 ].map((entry) => {
4161 const key = normalizeThreadPullRequestKey(entry);
4162 const link = threadPullRequestsOf(thread).find(
4163 (candidate) =>
4164 candidate.source !== "stack-dismissed" && threadPullRequestKeysEqual(candidate, key),
4165 );
4166 return { entry, key, link };
4167 });
4168 const inactive =
4169 thread.archivedAt !== null ||
4170 thread.settledOverride === "settled" ||
4171 thread.settledAt !== null ||
4172 isProviderNativeSubagentThread(thread);
4173 // An idle thread dispatches immediately, bypassing queued-wake validation.
4174 const outdated = entries.map(
4175 ({ entry, link }) =>
4176 wake?.notification.source.kind === "monitor" &&
4177 entry.watch !== null &&
4178 pullRequestSnapshotEndsWatch(link?.snapshot ?? null, entry.startedAt),
4179 );
4180 if (
4181 entries.some(({ entry, link }) => link?.watch?.startedAt !== entry.startedAt) ||
4182 (wake && inactive) ||
4183 // One wake cannot drop the news of just some of its pull requests; the sender splits it.
4184 (entries.length > 1 && outdated.some(Boolean))
4185 ) {
4186 return yield* new OrchestratorDispatchError({
4187 commandId: command.commandId,
4188 commandType: command.type,
4189 cause: "The pull request watch ended or its thread settled while it was read.",
4190 });
4191 }
4192 for (const { entry } of entries) {
4193 const { includesRemarks: _, ...mutation } = entry;
4194 yield* dispatchThreadMutation(mutation, events, effects);
4195 }
4196 if (wake === undefined || outdated[0]) return;
4197 const { sections, ...wakeMessage } = wake;
4198 const stamps = entries.map(({ entry, key }, index) => ({
4199 ...key,
4200 watchStartedAt: entry.startedAt,
4201 headSha: entry.includesRemarks === true ? null : (entry.watch?.headSha ?? null),
4202 ...(entry.watch === null ? { watchEnded: true } : {}),
4203 ...(entries.length > 1 && sections?.[index] !== undefined
4204 ? { section: sections[index] }
4205 : {}),
4206 }));
4207 yield* dispatchMessage(
4208 {
4209 type: "message.dispatch",
4210 commandId: command.commandId,
4211 threadId: command.threadId,
4212 messageId: wakeMessage.messageId,
4213 text: wakeMessage.text,
4214 notification: {
4215 ...wakeMessage.notification,
4216 source:
4217 wake.notification.source.kind === "monitor"
4218 ? {
4219 kind: "monitor",
4220 pullRequest: stamps[0]!,
4221 ...(stamps.length > 1 ? { otherPullRequests: stamps.slice(1) } : {}),
4222 }
4223 : wake.notification.source,
4224 },
4225 attachments: [],
4226 dispatchMode: { type: "queue_after_active" },
4227 createdBy: "agent",
4228 creationSource: "server",
4229 },
4230 events,
4231 effects,
4232 );
4233 },
4234 );
4235
4236 // Deleted after the share change commits: the files of waiting messages it dismissed or dropped.
4237 const releaseGuestAttachments = (
4238 command: { readonly commandId: CommandId; readonly threadId: ThreadId },
4239 before: OrchestrationV2AppThread["guestShare"],
4240 after: OrchestrationV2AppThread["guestShare"],
4241 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
4242 ) => {
4243 const attachmentIds = releasedGuestAttachmentIds(before ?? null, after ?? null);
4244 return attachmentIds.length === 0
4245 ? Effect.void
4246 : Ref.update(effects, (existing) => [
4247 ...existing,
4248 {
4249 id: `effect:${command.commandId}:guest-attachment.cleanup`,
4250 commandId: command.commandId,
4251 threadId: command.threadId,
4252 request: { type: "attachment.cleanup", attachmentIds: [...attachmentIds] },
4253 } satisfies PendingOrchestrationEffectV2,
4254 ]);
4255 };
4256
4257 const dispatchGuestShare = Effect.fn("orchestrationV2.dispatch.guestShare")(function* (
4258 command: GuestShareCommand,
4259 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
4260 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
4261 ) {
4262 const projection = yield* projectionStore
4263 .getThreadRecords(command.threadId, [])
4264 .pipe(
4265 Effect.mapError(
4266 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
4267 ),
4268 );
4269 const thread = projection.thread;
4270 const now = yield* DateTime.now;
4271 const decided =
4272 thread.deletedAt !== null
4273 ? Result.fail(`Thread ${command.threadId} is deleted.`)
4274 : decideGuestShareCommand({
4275 thread,
4276 command,
4277 now,
4278 });
4279 if (Result.isFailure(decided)) {
4280 return yield* new OrchestratorDispatchError({
4281 commandId: command.commandId,
4282 commandType: command.type,
4283 cause: decided.failure,
4284 });
4285 }
4286 if (decided.success === thread) return;
4287 yield* emit(
4288 events,
4289 command,
4290 )({
4291 type: "thread.guest-share-updated",
4292 threadId: command.threadId,
4293 providerInstanceId: thread.providerInstanceId,
4294 occurredAt: now,
4295 payload: decided.success,
4296 });
4297 yield* releaseGuestAttachments(command, thread.guestShare, decided.success.guestShare, effects);
4298 });
4299
4300 // Emitted before the message so the send and the status change commit together, and so
4301 // `dispatchMessage`, which reads pending events, keeps the updated share.
4302 const emitGuestMessageSent = Effect.fn("orchestrationV2.dispatch.guestMessageSent")(function* (
4303 command: Extract<OrchestrationV2Command, { readonly type: "message.dispatch" }>,
4304 guestMessageId: GuestMessageId,
4305 thread: OrchestrationV2AppThread,
4306 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
4307 ) {
4308 const now = yield* DateTime.now;
4309 const decided = decideGuestMessageSend({ thread, guestMessageId, now });
4310 if (Result.isFailure(decided)) {
4311 return yield* new OrchestratorDispatchError({
4312 commandId: command.commandId,
4313 commandType: command.type,
4314 cause: decided.failure,
4315 });
4316 }
4317 yield* emit(
4318 events,
4319 command,
4320 )({
4321 type: "thread.guest-share-updated",
4322 threadId: command.threadId,
4323 providerInstanceId: thread.providerInstanceId,
4324 occurredAt: now,
4325 payload: decided.success.thread,
4326 });
4327 return decided.success;
4328 });
4329
4330 const dispatchThreadMutation = Effect.fn("orchestrationV2.dispatch.threadMutation")(function* (
4331 command: Extract<
4332 OrchestrationV2ServerCommand,
4333 {
4334 readonly type:
4335 | "thread.archive"
4336 | "thread.unarchive"
4337 | "thread.settle"
4338 | "thread.unsettle"
4339 | "thread.snooze"
4340 | "thread.unsnooze"
4341 | "thread.auto-settle.set"
4342 | "thread.pin"
4343 | "thread.unpin"
4344 | "thread.pin.reorder"
4345 | "thread.tag.set"
4346 | "thread.active.reorder"
4347 | "thread.mark-unread"
4348 | "thread.metadata.update"
4349 | "thread.pull-request.link"
4350 | "thread.pull-request.unlink"
4351 | "thread.pull-request-link.sync"
4352 | "thread.issue.link"
4353 | "thread.issue.unlink"
4354 | "thread.pull-request.watch"
4355 | "thread.pull-request-watch.sync"
4356 | "thread.pull-request-watch.checks-reopen"
4357 | "thread.pull-request.sync"
4358 | "thread.title.regeneration.complete"
4359 | "thread.runtime-mode.set"
4360 | "thread.interaction-mode.set"
4361 | "thread.model-selection.set"
4362 | "provider.switch";
4363 }
4364 >,
4365 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
4366 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
4367 ) {
4368 const thread = yield* projectionStore.getThread(command.threadId).pipe(
4369 Effect.mapError(
4370 (cause) =>
4371 new OrchestratorProjectionError({
4372 threadId: command.threadId,
4373 cause,
4374 }),
4375 ),
4376 );
4377 if (thread.deletedAt !== null) {
4378 return yield* new OrchestratorDispatchError({
4379 commandId: command.commandId,
4380 commandType: command.type,
4381 cause: `Thread ${command.threadId} is deleted.`,
4382 });
4383 }
4384 if (
4385 command.type === "thread.pull-request.watch" &&
4386 command.watching &&
4387 isProviderNativeSubagentThread(thread)
4388 ) {
4389 return yield* new OrchestratorSubagentThreadReadOnlyError({
4390 commandId: command.commandId,
4391 threadId: command.threadId,
4392 });
4393 }
4394 if (
4395 command.type === "thread.pull-request.watch" &&
4396 command.watching &&
4397 (thread.settledOverride === "settled" ||
4398 thread.settledAt !== null ||
4399 thread.archivedAt !== null)
4400 ) {
4401 return yield* new OrchestratorDispatchError({
4402 commandId: command.commandId,
4403 commandType: command.type,
4404 cause: `Thread ${command.threadId} is settled or archived and cannot watch pull requests.`,
4405 });
4406 }
4407 // Child tasks report to their parent, which owns PR monitoring; child watches would cause unrelated wakes.
4408 if (
4409 command.type === "thread.pull-request.watch" &&
4410 command.watching &&
4411 thread.lineage.relationshipToParent === "subagent"
4412 ) {
4413 return yield* new OrchestratorDispatchError({
4414 commandId: command.commandId,
4415 commandType: command.type,
4416 cause: `Thread ${command.threadId} is a subagent; its parent thread watches pull requests.`,
4417 });
4418 }
4419 // Agent watches use the latest executed run to catch calls racing Stop; users can still watch.
4420 if (
4421 command.type === "thread.pull-request.watch" &&
4422 command.watching &&
4423 command.link?.source === "agent"
4424 ) {
4425 const { runs } = yield* projectionStore
4426 .getThreadRecords(command.threadId, ["runs"])
4427 .pipe(mapDispatchError(command));
4428 const latest = latestExecutedRun(runs);
4429 if (latest !== null && (yield* stopReachedRun(command, command.threadId, latest.id))) {
4430 return yield* new OrchestratorDispatchError({
4431 commandId: command.commandId,
4432 commandType: command.type,
4433 cause: `Thread ${command.threadId} was stopped.`,
4434 });
4435 }
4436 }
4437 if (
4438 command.type === "thread.metadata.update" &&
4439 command.expectedWorktreePath !== undefined &&
4440 command.expectedWorktreePath !== thread.worktreePath
4441 ) {
4442 return yield* new OrchestratorDispatchError({
4443 commandId: command.commandId,
4444 commandType: command.type,
4445 cause: `Thread ${command.threadId} worktree changed before the metadata update could be applied.`,
4446 });
4447 }
4448 if (
4449 command.type === "thread.metadata.update" &&
4450 command.expectedBranch !== undefined &&
4451 command.expectedBranch !== thread.branch
4452 ) {
4453 return yield* new OrchestratorDispatchError({
4454 commandId: command.commandId,
4455 commandType: command.type,
4456 cause: `Thread ${command.threadId} branch changed before the metadata update could be applied.`,
4457 });
4458 }
4459 if (command.type === "thread.metadata.update" && command.expectedEmpty === true) {
4460 const records = yield* projectionStore
4461 .getThreadRecords(command.threadId, ["runs"])
4462 .pipe(mapDispatchError(command));
4463 const messageCount = yield* projectionStore
4464 .getMessageCount(command.threadId)
4465 .pipe(mapDispatchError(command));
4466 if (messageCount > 0 || records.runs.length > 0)
4467 return yield* new OrchestratorDispatchError({
4468 commandId: command.commandId,
4469 commandType: command.type,
4470 cause: `Thread ${command.threadId} is no longer empty.`,
4471 });
4472 }
4473 if (command.type === "thread.archive" && thread.archivedAt !== null) {
4474 return yield* new OrchestratorDispatchError({
4475 commandId: command.commandId,
4476 commandType: command.type,
4477 cause: `Thread ${command.threadId} is already archived.`,
4478 });
4479 }
4480 if (command.type === "thread.archive") {
4481 const records = yield* projectionStore
4482 .getThreadRecords(command.threadId, ["subagents"])
4483 .pipe(mapDispatchError(command));
4484 if (hasRunningDelegatedTasks(records.subagents)) {
4485 return yield* new OrchestratorDispatchError({
4486 commandId: command.commandId,
4487 commandType: command.type,
4488 cause: `Thread ${command.threadId} has running delegated tasks. Wait for them or cancel them before archiving.`,
4489 });
4490 }
4491 }
4492 if (command.type === "thread.unarchive" && thread.archivedAt === null) {
4493 return yield* new OrchestratorDispatchError({
4494 commandId: command.commandId,
4495 commandType: command.type,
4496 cause: `Thread ${command.threadId} is not archived.`,
4497 });
4498 }
4499 if (
4500 (command.type === "thread.settle" ||
4501 command.type === "thread.unsettle" ||
4502 command.type === "thread.snooze" ||
4503 command.type === "thread.unsnooze" ||
4504 command.type === "thread.auto-settle.set" ||
4505 command.type === "thread.pin" ||
4506 command.type === "thread.unpin" ||
4507 command.type === "thread.pin.reorder" ||
4508 command.type === "thread.active.reorder" ||
4509 command.type === "thread.pull-request.sync") &&
4510 thread.archivedAt !== null
4511 ) {
4512 return yield* new OrchestratorDispatchError({
4513 commandId: command.commandId,
4514 commandType: command.type,
4515 cause: `Thread ${command.threadId} is archived.`,
4516 });
4517 }
4518 // Reject reorder after unpin so a raced request cannot restore a cleared pin.
4519 if (command.type === "thread.pin.reorder" && thread.pinnedAt == null) {
4520 return yield* new OrchestratorDispatchError({
4521 commandId: command.commandId,
4522 commandType: command.type,
4523 cause: `Thread ${command.threadId} is not pinned and cannot be reordered.`,
4524 });
4525 }
4526 if (
4527 command.type === "thread.active.reorder" &&
4528 (thread.pinnedAt != null || thread.settledOverride === "settled")
4529 ) {
4530 return yield* new OrchestratorDispatchError({
4531 commandId: command.commandId,
4532 commandType: command.type,
4533 cause: `Thread ${command.threadId} is not active and cannot be reordered.`,
4534 });
4535 }
4536 if (command.type === "thread.pull-request.sync") {
4537 const project = yield* projects.get(command.projectId).pipe(mapDispatchError(command));
4538 const currentSequence = yield* eventSink.latestSequence({ threadId: command.threadId }).pipe(
4539 Effect.mapError(
4540 (cause) =>
4541 new OrchestratorProjectionError({
4542 threadId: command.threadId,
4543 cause,
4544 }),
4545 ),
4546 );
4547 if (
4548 Option.isNone(project) ||
4549 project.value.deletedAt !== null ||
4550 project.value.workspaceRoot !== command.expected.workspaceRoot ||
4551 // A replacement rewrites pullRequests, which the expected fields do not capture.
4552 (command.linkedPullRequest !== undefined && currentSequence > command.snapshotSequence) ||
4553 thread.projectId !== command.projectId ||
4554 thread.branch !== command.expected.branch ||
4555 thread.worktreePath !== command.expected.worktreePath ||
4556 !threadPullRequestLinksEqual(
4557 thread.linkedPullRequest ?? null,
4558 command.expected.linkedPullRequest,
4559 ) ||
4560 !threadPullRequestLinksEqual(
4561 thread.branchPullRequest ?? null,
4562 command.expected.branchPullRequest,
4563 )
4564 ) {
4565 return yield* new OrchestratorDispatchError({
4566 commandId: command.commandId,
4567 commandType: command.type,
4568 cause: `Thread ${command.threadId} changed before pull request discovery.`,
4569 });
4570 }
4571 }
4572 if (command.type === "thread.settle") {
4573 const projection = yield* loadProjectionForCommand(command, ["runs", "runtimeRequests"]);
4574 if (
4575 command.expectedLatestRunId !== undefined &&
4576 projection.runs.reduce<OrchestrationV2Run | undefined>(
4577 (latest, run) => (latest === undefined || run.ordinal > latest.ordinal ? run : latest),
4578 undefined,
4579 )?.id !== command.expectedLatestRunId
4580 ) {
4581 return yield* new OrchestratorDispatchError({
4582 commandId: command.commandId,
4583 commandType: command.type,
4584 cause: "The thread received another run before settlement.",
4585 });
4586 }
4587 const queuedMessages = yield* projectionStore
4588 .getMessagesById(
4589 command.threadId,
4590 projection.runs.filter((run) => run.status === "queued").map((run) => run.userMessageId),
4591 )
4592 .pipe(mapDispatchError(command));
4593 const automaticMessageIds = new Set(
4594 [...queuedMessages.values()]
4595 .filter(
4596 (message) =>
4597 message.role === "user" &&
4598 (message.createdBy !== "user" ||
4599 message.scheduledTaskId !== undefined ||
4600 message.notification !== undefined ||
4601 message.delegatedCompletion !== undefined),
4602 )
4603 .map((message) => message.id),
4604 );
4605 const automaticQueuedRuns = projection.runs.filter(
4606 (run) => run.status === "queued" && automaticMessageIds.has(run.userMessageId),
4607 );
4608 const runningRunExists = projection.runs.some((run) =>
4609 ["preparing", "starting", "running", "waiting"].includes(run.status),
4610 );
4611 const queuedMessageExists = projection.runs.some(
4612 (run) => run.status === "queued" && !automaticQueuedRuns.includes(run),
4613 );
4614 const pendingRequests = projection.runtimeRequests.filter(
4615 (request) => request.status === "pending",
4616 );
4617 const blockingRequestExists = pendingRequests.some(
4618 (request) => request.kind !== "user_input" || request.responseCapability.type !== "message",
4619 );
4620 const blocker = blockingRequestExists
4621 ? "is waiting on an approval or question. Answer it before settling."
4622 : runningRunExists
4623 ? "is still running. Stop it before settling."
4624 : queuedMessageExists
4625 ? "has a queued message. Send it or remove it from the queue before settling."
4626 : null;
4627 if (blocker !== null) {
4628 return yield* new OrchestratorDispatchError({
4629 commandId: command.commandId,
4630 commandType: command.type,
4631 cause: `Thread ${command.threadId} ${blocker}`,
4632 });
4633 }
4634
4635 for (const request of pendingRequests) {
4636 yield* dispatchRuntimeRequestRespond(
4637 {
4638 type: "runtime-request.respond",
4639 commandId: command.commandId,
4640 threadId: command.threadId,
4641 requestId: request.id,
4642 decision: "cancel",
4643 },
4644 events,
4645 effects,
4646 );
4647 }
4648 for (const run of automaticQueuedRuns) {
4649 yield* dispatchQueuedRunCancel(
4650 {
4651 type: "queued-run.cancel",
4652 commandId: command.commandId,
4653 threadId: command.threadId,
4654 runId: run.id,
4655 },
4656 events,
4657 "cohort",
4658 );
4659 }
4660 }
4661
4662 const needsProviderState =
4663 command.type === "thread.runtime-mode.set" ||
4664 command.type === "thread.model-selection.set" ||
4665 command.type === "provider.switch" ||
4666 command.type === "thread.archive" ||
4667 command.type === "thread.settle" ||
4668 (command.type === "thread.metadata.update" &&
4669 command.worktreePath !== undefined &&
4670 command.worktreePath !== thread.worktreePath);
4671 const providerContext = needsProviderState
4672 ? yield* projectionStore
4673 .getThreadProviderContext(
4674 command.threadId,
4675 command.type === "thread.model-selection.set" || command.type === "provider.switch"
4676 ? command.modelSelection.instanceId
4677 : undefined,
4678 )
4679 .pipe(mapDispatchError(command))
4680 : null;
4681 const providerSwitchPlan =
4682 command.type === "thread.model-selection.set" || command.type === "provider.switch"
4683 ? yield* Effect.gen(function* () {
4684 yield* providerAdapters.get(command.modelSelection.instanceId).pipe(
4685 Effect.mapError(
4686 (cause) =>
4687 new OrchestratorProviderAdapterError({
4688 commandId: command.commandId,
4689 providerInstanceId: command.modelSelection.instanceId,
4690 cause,
4691 }),
4692 ),
4693 );
4694 const plan = yield* providerSwitchService
4695 .plan({
4696 projection: providerContext!,
4697 targetModelSelection: command.modelSelection,
4698 })
4699 .pipe(mapDispatchError(command));
4700 if (plan.instanceChanged || plan.releaseProviderSessionIds.length > 0) {
4701 const projection = yield* loadProjectionForCommand(command, ["runs"], {
4702 turnItemTypes: [],
4703 });
4704 if (projection.runs.some(isActiveRun)) {
4705 return yield* new OrchestratorProviderSelectionBusyError({
4706 commandId: command.commandId,
4707 threadId: command.threadId,
4708 });
4709 }
4710 }
4711 return plan;
4712 })
4713 : null;
4714
4715 // Archive and settle stop idle background work; serialized commands prevent starts racing detach.
4716 // Settlement rejects active or blocked runs.
4717 const detachSessionIds = new Set(
4718 command.type === "thread.archive" || command.type === "thread.settle"
4719 ? (providerContext?.providerSessions ?? []).map((session) => session.id)
4720 : command.type === "thread.metadata.update" &&
4721 command.worktreePath !== undefined &&
4722 command.worktreePath !== thread.worktreePath
4723 ? (providerContext?.providerSessions ?? []).map((session) => session.id)
4724 : command.type === "thread.runtime-mode.set"
4725 ? (providerContext?.providerSessions ?? [])
4726 .filter(
4727 (session) => !session.capabilities.sessions.supportsRuntimeModeSwitchInSession,
4728 )
4729 .map((session) => session.id)
4730 : (providerSwitchPlan?.releaseProviderSessionIds ?? []),
4731 );
4732 const liveSessions = (providerContext?.providerSessions ?? []).filter(
4733 (session) =>
4734 detachSessionIds.has(session.id) &&
4735 session.status !== "stopped" &&
4736 session.status !== "error",
4737 );
4738 if (
4739 command.type !== "thread.archive" &&
4740 command.type !== "thread.settle" &&
4741 liveSessions.length > 0
4742 ) {
4743 // The provider context holds only the active and target provider threads; a session kept
4744 // busy by an earlier queued hand-off owns neither, so probe every thread of this one.
4745 const { providerThreads } = yield* projectionStore
4746 .getThreadRecords(command.threadId, ["providerThreads"], {
4747 omitSettledSubagentRecords: true,
4748 })
4749 .pipe(mapDispatchError(command));
4750 for (const session of liveSessions) {
4751 if (yield* sessionHasBackgroundWork(session, providerThreads)) {
4752 return yield* new OrchestratorDispatchError({
4753 commandId: command.commandId,
4754 commandType: command.type,
4755 cause:
4756 "The current session is still running background agents or commands, and this change would end them. Wait for them to finish, or press Stop, then try again.",
4757 });
4758 }
4759 }
4760 }
4761
4762 const now = yield* DateTime.now;
4763 if (command.type === "thread.metadata.update" && command.limitRecovery != null) {
4764 const projection = yield* loadProjectionForCommand(
4765 command,
4766 ["runs", "runtimeRequests", "turnItems"],
4767 { turnItemTypes: ["error"] },
4768 );
4769 const run = usageLimitBlockedRun(projection.runs, projection.turnItems, null);
4770 const failure = latestRootProviderFailure(run, projection.turnItems);
4771 const resetMs = Date.parse(command.limitRecovery.resetAt);
4772 if (command.limitRecovery.snooze === true && resetMs <= DateTime.toEpochMillis(now)) {
4773 return yield* new OrchestratorDispatchError({
4774 commandId: command.commandId,
4775 commandType: command.type,
4776 cause: "The reset time has passed. Retry the thread manually.",
4777 });
4778 }
4779 if (thread.settledOverride === "settled") {
4780 return yield* new OrchestratorDispatchError({
4781 commandId: command.commandId,
4782 commandType: command.type,
4783 cause: "The thread is settled. Un-settle it to resume at the reset.",
4784 });
4785 }
4786 if (
4787 !Number.isFinite(resetMs) ||
4788 thread.archivedAt !== null ||
4789 run?.id !== command.limitRecovery.runId ||
4790 failure?.class !== "usage_limit" ||
4791 failure.resetAt !== command.limitRecovery.resetAt ||
4792 resetMs <= DateTime.toEpochMillis(run.completedAt ?? run.requestedAt) ||
4793 projection.runtimeRequests.some((request) => request.status === "pending")
4794 ) {
4795 return yield* new OrchestratorDispatchError({
4796 commandId: command.commandId,
4797 commandType: command.type,
4798 cause: "The provider limit changed before recovery could be configured.",
4799 });
4800 }
4801 }
4802 let restoredUnsettledAt: DateTime.Utc | null | undefined;
4803 if (command.type === "thread.unsettle" && command.restoreUnsettledAt !== undefined) {
4804 const parsed =
4805 command.restoreUnsettledAt === null
4806 ? Option.some(null)
4807 : DateTime.make(command.restoreUnsettledAt);
4808 if (Option.isNone(parsed)) {
4809 return yield* new OrchestratorDispatchError({
4810 commandId: command.commandId,
4811 commandType: command.type,
4812 cause: `Thread ${command.threadId} unsettle anchor ${command.restoreUnsettledAt} is not a valid time.`,
4813 });
4814 }
4815 restoredUnsettledAt = parsed.value;
4816 }
4817 let snoozedUntil: DateTime.Utc | null = null;
4818 if (command.type === "thread.snooze") {
4819 const projection = yield* loadProjectionForCommand(command, ["runs", "runtimeRequests"], {
4820 turnItemTypes: [],
4821 });
4822 const parsedSnoozedUntil = DateTime.make(command.snoozedUntil);
4823 if (
4824 Option.isNone(parsedSnoozedUntil) ||
4825 DateTime.toEpochMillis(parsedSnoozedUntil.value) <= DateTime.toEpochMillis(now)
4826 ) {
4827 return yield* new OrchestratorDispatchError({
4828 commandId: command.commandId,
4829 commandType: command.type,
4830 cause: `Thread ${command.threadId} snooze wake time ${command.snoozedUntil} is not in the future.`,
4831 });
4832 }
4833 if (
4834 projection.runtimeRequests.some(
4835 (request) => request.status === "pending" && request.kind !== "user_input",
4836 )
4837 ) {
4838 return yield* new OrchestratorDispatchError({
4839 commandId: command.commandId,
4840 commandType: command.type,
4841 cause: `Thread ${command.threadId} has a pending approval and cannot be snoozed.`,
4842 });
4843 }
4844 const queuedRuns = projection.runs.filter((run) => run.status === "queued");
4845 const queuedMessages =
4846 queuedRuns.length === 0
4847 ? []
4848 : (yield* loadProjectionForCommand(command, ["messages"], {
4849 messageIds: queuedRuns.map((run) => run.userMessageId),
4850 })).messages;
4851 const automaticMessageIds = new Set(
4852 queuedMessages
4853 .filter(
4854 (message) =>
4855 message.notification !== undefined || message.delegatedCompletion !== undefined,
4856 )
4857 .map((message) => message.id),
4858 );
4859 const userRuns = queuedRuns.filter((run) => !automaticMessageIds.has(run.userMessageId));
4860 // User work waiting behind a usage limit cannot start until the reset or the user acts.
4861 const latest = latestExecutedRun(projection.runs);
4862 const limitRecords =
4863 userRuns.length === 0 || latest?.status !== "failed"
4864 ? null
4865 : yield* loadProjectionForCommand(command, ["turnItems", "providerSessions"], {
4866 turnItemTypes: ["error"],
4867 turnItemRunIds: [latest.id],
4868 });
4869 const heldBehindLimit =
4870 limitRecords !== null &&
4871 userRuns.every((run) =>
4872 usageLimitHoldsQueue({ ...limitRecords, runs: projection.runs }, run.providerInstanceId),
4873 );
4874 if (userRuns.length > 0 && !heldBehindLimit) {
4875 return yield* new OrchestratorDispatchError({
4876 commandId: command.commandId,
4877 commandType: command.type,
4878 cause: `Thread ${command.threadId} has a queued run and cannot be snoozed.`,
4879 });
4880 }
4881 snoozedUntil = parsedSnoozedUntil.value;
4882 }
4883 let markUnreadVisitedAt: DateTime.Utc | null = null;
4884 if (command.type === "thread.mark-unread") {
4885 const shell = yield* projectionStore
4886 .getThreadShell(command.threadId)
4887 .pipe(
4888 Effect.mapError(
4889 (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
4890 ),
4891 );
4892 const latestRunCompletedAt = shell?.latestRunCompletedAt ?? null;
4893 if (latestRunCompletedAt === null) {
4894 return yield* new OrchestratorDispatchError({
4895 commandId: command.commandId,
4896 commandType: command.type,
4897 cause: `Thread ${command.threadId} has no completed run to mark unread.`,
4898 });
4899 }
4900 markUnreadVisitedAt = DateTime.subtract(latestRunCompletedAt, { milliseconds: 1 });
4901 }
4902 const updatedThread: OrchestrationV2AppThread = (() => {
4903 switch (command.type) {
4904 case "thread.archive":
4905 return {
4906 ...thread,
4907 archivedAt: now,
4908 titleRegeneration: null,
4909 guestShare: null,
4910 pullRequests: thread.pullRequests?.map((link) => withPullRequestWatch(link, undefined)),
4911 updatedAt: now,
4912 };
4913 case "thread.unarchive":
4914 return { ...thread, archivedAt: null, updatedAt: now };
4915 case "thread.settle": {
4916 const wasPinned = thread.pinnedAt != null;
4917 const alreadySettled =
4918 thread.settledOverride === "settled" && thread.settledAt !== null && !wasPinned;
4919 return {
4920 ...thread,
4921 settledOverride: "settled",
4922 settledAt: alreadySettled ? thread.settledAt : (command.settledAt ?? now),
4923 // Settled threads refuse recovery, so an armed resume would stay hidden until un-settled.
4924 limitRecovery: thread.limitRecovery
4925 ? { ...thread.limitRecovery, autoResume: false, snooze: false }
4926 : thread.limitRecovery,
4927 pullRequests: thread.pullRequests?.map((link) => withPullRequestWatch(link, undefined)),
4928 unsettledAt: null,
4929 pinnedAt: null,
4930 pinOrderKey: null,
4931 activeOrderKey: null,
4932 snoozedUntil: null,
4933 snoozedAt: null,
4934 updatedAt:
4935 alreadySettled && thread.snoozedUntil == null && thread.snoozedAt == null
4936 ? thread.updatedAt
4937 : now,
4938 };
4939 }
4940 case "thread.unsettle": {
4941 const alreadyPinnedActive = thread.settledOverride === "active";
4942 return {
4943 ...thread,
4944 settledOverride: "active",
4945 settledAt: null,
4946 unsettledAt: alreadyPinnedActive
4947 ? (thread.unsettledAt ?? null)
4948 : restoredUnsettledAt === undefined
4949 ? now
4950 : restoredUnsettledAt,
4951 updatedAt: alreadyPinnedActive ? thread.updatedAt : now,
4952 };
4953 }
4954 case "thread.snooze": {
4955 // Re-snoozing acknowledges the latest wake even at the same time; commandId deduplicates retries.
4956 return {
4957 ...thread,
4958 snoozedUntil,
4959 limitRecovery: thread.limitRecovery ? { ...thread.limitRecovery, snooze: false } : null,
4960 snoozedAt: now,
4961 updatedAt: now,
4962 };
4963 }
4964 case "thread.unsnooze": {
4965 const alreadyAwake = thread.snoozedUntil == null;
4966 return {
4967 ...thread,
4968 snoozedUntil: null,
4969 snoozedAt: null,
4970 lastSnoozeWakeAt: alreadyAwake ? thread.lastSnoozeWakeAt : now,
4971 updatedAt: alreadyAwake ? thread.updatedAt : now,
4972 };
4973 }
4974 case "thread.auto-settle.set": {
4975 const unchanged = command.enabled === (thread.autoSettleDisabledAt == null);
4976 return {
4977 ...thread,
4978 autoSettleDisabledAt: command.enabled ? null : (thread.autoSettleDisabledAt ?? now),
4979 updatedAt: unchanged ? thread.updatedAt : now,
4980 };
4981 }
4982 case "thread.pin": {
4983 const alreadyPinned = thread.pinnedAt != null;
4984 const promotes = thread.settledOverride === "settled" || thread.snoozedUntil != null;
4985 return {
4986 ...thread,
4987 pinnedAt: alreadyPinned ? thread.pinnedAt : now,
4988 ...(alreadyPinned || command.orderKey === undefined
4989 ? {}
4990 : { pinOrderKey: command.orderKey }),
4991 settledOverride:
4992 thread.settledOverride === "settled" ? "active" : thread.settledOverride,
4993 settledAt: thread.settledOverride === "settled" ? null : thread.settledAt,
4994 ...(thread.settledOverride === "settled" ? { unsettledAt: now } : {}),
4995 snoozedUntil: null,
4996 snoozedAt: null,
4997 lastSnoozeWakeAt: thread.snoozedUntil == null ? thread.lastSnoozeWakeAt : now,
4998 updatedAt: alreadyPinned && !promotes ? thread.updatedAt : now,
4999 };
5000 }
Showing first 5,000 of 17,137 lines. View raw