The best way to control Claude Code, Codex and more from any computer or phone

main
ts 2,381 lines 102 KB
Raw
1 import { makeAssistantStreamingFilter } from "./assistantStreaming.ts";
2 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
3 import {
4 isDelegatedAppThread,
5 isOrchestrationV2WorkActive,
6 CommandId,
7 type EventId,
8 type ModelSelection,
9 type NodeId,
10 type OrchestrationV2AppThread,
11 type OrchestrationV2CheckpointScope,
12 type OrchestrationV2DomainEvent,
13 type OrchestrationV2ExecutionNode,
14 type OrchestrationV2ProviderFailure,
15 type OrchestrationV2ProviderThread,
16 type OrchestrationV2ProviderTurn,
17 type OrchestrationV2Run,
18 type OrchestrationV2RunAttempt,
19 type OrchestrationV2Subagent,
20 type OrchestrationV2TurnItem,
21 type ProviderSessionId,
22 type ProviderThreadId,
23 type ProviderTurnId,
24 type RunAttemptId,
25 type ThreadId,
26 type TurnItemId,
27 } from "@t3tools/contracts";
28 import * as Context from "effect/Context";
29 import * as Cause from "effect/Cause";
30 import * as DateTime from "effect/DateTime";
31 import * as Duration from "effect/Duration";
32 import * as Effect from "effect/Effect";
33 import * as Exit from "effect/Exit";
34 import * as Fiber from "effect/Fiber";
35 import * as Layer from "effect/Layer";
36 import * as Queue from "effect/Queue";
37 import * as Ref from "effect/Ref";
38 import * as Schedule from "effect/Schedule";
39 import * as Schema from "effect/Schema";
40 import * as Stream from "effect/Stream";
41
42 import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts";
43 import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts";
44 import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts";
45 import * as ServerSettings from "../serverSettings.ts";
46 import * as CheckpointService from "./CheckpointService.ts";
47 import * as EventSink from "./EventSink.ts";
48 import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
49 import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
50 import { PROVIDER_STOP_REASON_SIGNAL } from "@t3tools/provider-core/server/subagentProjection";
51 import * as ProviderEventIngestor from "./ProviderEventIngestor.ts";
52 import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts";
53 import type { ProjectionStoreV2Error } from "./ProjectionStore.ts";
54 import {
55 knownProviderFailureMessage,
56 makeProviderFailure,
57 makeProviderFailureTurnItem,
58 } from "@t3tools/provider-core/server/failure";
59 import * as RunFinalizationService from "./RunFinalizationService.ts";
60 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
61 import * as TurnItemPositionStore from "./TurnItemPositionStore.ts";
62 import { userFacingDatabaseErrorMessage } from "./UserFacingErrors.ts";
63
64 export interface ProviderEventRoutingState {
65 readonly ownedThreadIds: ReadonlySet<ThreadId>;
66 // After root settlement, new children belong to the live run unless their coordinator still holds this run open.
67 readonly rootTurnEnded: boolean;
68 readonly workflowMemberParents: ReadonlyMap<ThreadId, ThreadId>;
69 readonly ownedProviderThreadIds: ReadonlySet<ProviderThreadId>;
70 readonly ownedProviderTurns: ReadonlyMap<ProviderTurnId, ProviderThreadId>;
71 readonly inheritedBackgroundTurnItems: ReadonlyMap<TurnItemId, OrchestrationV2Run["id"]>;
72 readonly rootProviderTurnId: ProviderTurnId | null;
73 }
74
75 export interface ProviderEventRouteIdentity {
76 readonly threadId: ThreadId;
77 readonly runId: OrchestrationV2Run["id"];
78 readonly attemptId: RunAttemptId;
79 readonly providerThreadId: ProviderThreadId;
80 }
81
82 export interface InheritedBackgroundTurnItemRoute {
83 readonly id: TurnItemId;
84 readonly runId: OrchestrationV2Run["id"];
85 }
86
87 type ProviderTerminalEvent = Extract<
88 ProviderAdapter.ProviderAdapterV2Event,
89 { readonly type: "turn.terminal" }
90 >;
91
92 function isTerminalProviderTurnStatus(status: OrchestrationV2ProviderTurn["status"]): boolean {
93 return (
94 status === "completed" ||
95 status === "interrupted" ||
96 status === "failed" ||
97 status === "cancelled"
98 );
99 }
100
101 function isSettledSubagentStatus(status: OrchestrationV2Subagent["status"]): boolean {
102 return !isOrchestrationV2WorkActive(status);
103 }
104
105 /** A provider-native subagent's turns arrive on its parent's run, but the parent agent started them. */
106 const NATIVE_SUBAGENT_TURN_ORIGIN: ProviderUsageAnalytics.ProviderTurnOrigin = {
107 createdBy: "agent",
108 creationSource: "provider",
109 scheduled: false,
110 subagent: true,
111 };
112
113 const backgroundCapableTurnItemTypes: ReadonlySet<OrchestrationV2TurnItem["type"]> = new Set([
114 "command_execution",
115 "dynamic_tool",
116 "subagent",
117 ]);
118
119 function isSettledTurnItemStatus(status: OrchestrationV2TurnItem["status"]): boolean {
120 return !isOrchestrationV2WorkActive(status);
121 }
122
123 function isSettledRunEligibleForInheritedBackground(status: OrchestrationV2Run["status"]): boolean {
124 return status === "interrupted" || status === "failed" || status === "cancelled";
125 }
126
127 /** Use durable provider-thread lineage; sessions can host multiple threads. */
128 export function selectInheritedBackgroundTurnItems(input: {
129 readonly threadId: ThreadId;
130 readonly currentProviderThreadId: ProviderThreadId;
131 readonly currentRunOrdinal: number;
132 readonly runs: ReadonlyArray<OrchestrationV2Run>;
133 readonly turnItems: ReadonlyArray<OrchestrationV2TurnItem>;
134 }): ReadonlyArray<InheritedBackgroundTurnItemRoute> {
135 const settledPriorRunIds = new Set(
136 input.runs
137 .filter(
138 (run) =>
139 run.threadId === input.threadId &&
140 run.ordinal < input.currentRunOrdinal &&
141 isSettledRunEligibleForInheritedBackground(run.status),
142 )
143 .map((run) => run.id),
144 );
145 return input.turnItems.flatMap((turnItem) =>
146 turnItem.threadId === input.threadId &&
147 turnItem.providerThreadId === input.currentProviderThreadId &&
148 turnItem.runId !== null &&
149 settledPriorRunIds.has(turnItem.runId) &&
150 backgroundCapableTurnItemTypes.has(turnItem.type) &&
151 !isSettledTurnItemStatus(turnItem.status)
152 ? [{ id: turnItem.id, runId: turnItem.runId }]
153 : [],
154 );
155 }
156
157 type SubagentTurnItem = Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>;
158
159 // Tool calls that only a live provider process can end.
160 const processBoundToolCallTypes: ReadonlySet<OrchestrationV2TurnItem["type"]> = new Set([
161 "command_execution",
162 "dynamic_tool",
163 ]);
164
165 type OpenRunOwnedSubagentProjection = {
166 readonly subagents: ReadonlyMap<NodeId, OrchestrationV2Subagent>;
167 readonly turnItems: ReadonlyMap<NodeId, SubagentTurnItem>;
168 readonly childTurnItems: ReadonlyMap<TurnItemId, OrchestrationV2TurnItem>;
169 readonly nodes: ReadonlyMap<NodeId, OrchestrationV2ExecutionNode>;
170 readonly linkedChildThreadIds: ReadonlySet<ThreadId>;
171 };
172
173 export type UnsettledWork = {
174 readonly subagents: ReadonlyArray<OrchestrationV2Subagent>;
175 readonly nodes: ReadonlyArray<OrchestrationV2ExecutionNode>;
176 readonly turnItems: ReadonlyArray<OrchestrationV2TurnItem>;
177 };
178
179 type RunOwnedSubagentTerminalStatus = Extract<
180 OrchestrationV2Subagent["status"],
181 "interrupted" | "failed" | "cancelled"
182 >;
183
184 function isOpenExecutionNodeStatus(status: OrchestrationV2ExecutionNode["status"]): boolean {
185 return status === "pending" || status === "running" || status === "waiting";
186 }
187
188 function isRunOwnedSubagentTerminalStatus(
189 status: ProviderTerminalEvent["status"],
190 ): status is RunOwnedSubagentTerminalStatus {
191 return status === "interrupted" || status === "failed" || status === "cancelled";
192 }
193
194 function streamedThreadId(event: ProviderAdapter.ProviderAdapterV2Event): ThreadId | null {
195 if (event.type === "message.updated") return event.message.threadId;
196 if (event.type === "turn_item.updated") return event.turnItem.threadId;
197 return null;
198 }
199
200 function childTurnsContinue(terminal: ProviderTerminalEvent): boolean {
201 return terminal.status === "failed" && terminal.childTurnsContinue === true;
202 }
203
204 /** Launching runs still ingest running subagents; taking them over would duplicate events. */
205 export function canRouteRelatedSubagent(status: OrchestrationV2Subagent["status"]): boolean {
206 return status === "completed";
207 }
208
209 function emptyOpenRunOwnedSubagentProjection(): OpenRunOwnedSubagentProjection {
210 return {
211 subagents: new Map(),
212 turnItems: new Map(),
213 childTurnItems: new Map(),
214 nodes: new Map(),
215 linkedChildThreadIds: new Set(),
216 };
217 }
218
219 function withLinkedChildThreadId(
220 current: OpenRunOwnedSubagentProjection,
221 childThreadId: ThreadId | null,
222 ): OpenRunOwnedSubagentProjection {
223 if (childThreadId === null || current.linkedChildThreadIds.has(childThreadId)) {
224 return current;
225 }
226 const linkedChildThreadIds = new Set(current.linkedChildThreadIds);
227 linkedChildThreadIds.add(childThreadId);
228 return { ...current, linkedChildThreadIds };
229 }
230
231 export const cascadeTerminalizeRunOwnedSubagents = Effect.fn(
232 "RunExecutionService.cascadeTerminalizeRunOwnedSubagents",
233 )(function* (input: {
234 readonly run: OrchestrationV2Run;
235 readonly open: OpenRunOwnedSubagentProjection;
236 readonly status: RunOwnedSubagentTerminalStatus;
237 readonly completedAt: DateTime.Utc;
238 readonly allocateEventId: () => Effect.Effect<EventId, IdAllocator.IdAllocatorV2AllocationError>;
239 }): Effect.fn.Return<
240 ReadonlyArray<OrchestrationV2DomainEvent>,
241 IdAllocator.IdAllocatorV2AllocationError
242 > {
243 const events: Array<OrchestrationV2DomainEvent> = [];
244 // Prefer lifetime linkage over currently-open rows: subagent/turn-item
245 // snapshots may terminalize before the linked child-thread node settles.
246 const childThreadIds = new Set(input.open.linkedChildThreadIds);
247 for (const item of [...input.open.subagents.values(), ...input.open.turnItems.values()]) {
248 if (item.childThreadId !== null) {
249 childThreadIds.add(item.childThreadId);
250 }
251 }
252 const keys = new Set<NodeId>([
253 ...input.open.subagents.keys(),
254 ...input.open.turnItems.keys(),
255 ...input.open.nodes.keys(),
256 ]);
257 for (const key of keys) {
258 const subagent = input.open.subagents.get(key);
259 if (subagent !== undefined && !isSettledSubagentStatus(subagent.status)) {
260 events.push({
261 id: yield* input.allocateEventId(),
262 type: "subagent.updated",
263 threadId: subagent.threadId,
264 runId: input.run.id,
265 nodeId: subagent.id,
266 driver: subagent.driver,
267 providerInstanceId: subagent.providerInstanceId,
268 occurredAt: input.completedAt,
269 payload: {
270 ...subagent,
271 status: input.status,
272 completedAt: input.completedAt,
273 updatedAt: input.completedAt,
274 },
275 });
276 }
277 const node = input.open.nodes.get(key);
278 if (
279 node !== undefined &&
280 ((node.threadId === input.run.threadId && node.runId === input.run.id) ||
281 childThreadIds.has(node.threadId)) &&
282 isOpenExecutionNodeStatus(node.status)
283 ) {
284 events.push({
285 id: yield* input.allocateEventId(),
286 type: "node.updated",
287 threadId: node.threadId,
288 runId: node.runId ?? input.run.id,
289 nodeId: node.id,
290 providerInstanceId: input.run.providerInstanceId,
291 occurredAt: input.completedAt,
292 payload: {
293 ...node,
294 status: input.status,
295 completedAt: input.completedAt,
296 },
297 });
298 }
299 const turnItem = input.open.turnItems.get(key);
300 if (
301 turnItem !== undefined &&
302 turnItem.runId === input.run.id &&
303 !isSettledTurnItemStatus(turnItem.status)
304 ) {
305 events.push({
306 id: yield* input.allocateEventId(),
307 type: "turn-item.updated",
308 threadId: turnItem.threadId,
309 runId: input.run.id,
310 ...(turnItem.nodeId === null ? {} : { nodeId: turnItem.nodeId }),
311 providerInstanceId: input.run.providerInstanceId,
312 occurredAt: input.completedAt,
313 payload: {
314 ...turnItem,
315 status: input.status,
316 completedAt: input.completedAt,
317 updatedAt: input.completedAt,
318 },
319 });
320 }
321 }
322 for (const turnItem of input.open.childTurnItems.values()) {
323 if (!childThreadIds.has(turnItem.threadId) || isSettledTurnItemStatus(turnItem.status)) {
324 continue;
325 }
326 events.push({
327 id: yield* input.allocateEventId(),
328 type: "turn-item.updated",
329 threadId: turnItem.threadId,
330 runId: turnItem.runId ?? input.run.id,
331 ...(turnItem.nodeId === null ? {} : { nodeId: turnItem.nodeId }),
332 providerInstanceId: input.run.providerInstanceId,
333 occurredAt: input.completedAt,
334 payload: {
335 ...turnItem,
336 ...("streaming" in turnItem ? { streaming: false } : {}),
337 status: input.status,
338 completedAt: input.completedAt,
339 updatedAt: input.completedAt,
340 },
341 });
342 }
343 return events;
344 });
345
346 const interruptLostToolCalls = Effect.fn("RunExecutionService.interruptLostToolCalls")(
347 function* (input: {
348 readonly run: OrchestrationV2Run;
349 readonly toolCalls: Iterable<OrchestrationV2TurnItem>;
350 readonly completedAt: DateTime.Utc;
351 readonly allocateEventId: () => Effect.Effect<
352 EventId,
353 IdAllocator.IdAllocatorV2AllocationError
354 >;
355 }) {
356 const events: Array<OrchestrationV2DomainEvent> = [];
357 for (const toolCall of input.toolCalls) {
358 events.push({
359 id: yield* input.allocateEventId(),
360 type: "turn-item.updated",
361 threadId: toolCall.threadId,
362 runId: toolCall.runId ?? input.run.id,
363 ...(toolCall.nodeId === null ? {} : { nodeId: toolCall.nodeId }),
364 providerInstanceId: input.run.providerInstanceId,
365 occurredAt: input.completedAt,
366 payload: {
367 ...toolCall,
368 status: "interrupted",
369 completedAt: input.completedAt,
370 updatedAt: input.completedAt,
371 },
372 });
373 }
374 return events;
375 },
376 );
377
378 function finalProviderThreadStatus(
379 disposition: ProviderTerminalEvent["threadDisposition"],
380 ): OrchestrationV2ProviderThread["status"] {
381 return disposition === "broken" ? "error" : "idle";
382 }
383
384 export function makeProviderEventRoutingState(input: {
385 readonly identity: ProviderEventRouteIdentity;
386 readonly inheritedBackgroundTurnItems?: ReadonlyArray<InheritedBackgroundTurnItemRoute>;
387 readonly providerTurnId: ProviderTurnId | null;
388 readonly relatedThreadIds?: ReadonlyArray<ThreadId>;
389 readonly relatedProviderThreadIds?: ReadonlyArray<ProviderThreadId>;
390 }): ProviderEventRoutingState {
391 return {
392 ownedThreadIds: new Set([input.identity.threadId, ...(input.relatedThreadIds ?? [])]),
393 rootTurnEnded: false,
394 workflowMemberParents: new Map(),
395 ownedProviderThreadIds: new Set([
396 input.identity.providerThreadId,
397 ...(input.relatedProviderThreadIds ?? []),
398 ]),
399 ownedProviderTurns:
400 input.providerTurnId === null
401 ? new Map()
402 : new Map([[input.providerTurnId, input.identity.providerThreadId]]),
403 inheritedBackgroundTurnItems: new Map(
404 (input.inheritedBackgroundTurnItems ?? []).map((item) => [item.id, item.runId]),
405 ),
406 rootProviderTurnId: input.providerTurnId,
407 };
408 }
409
410 export function routeProviderEvent(
411 event: ProviderAdapter.ProviderAdapterV2Event,
412 input: ProviderEventRouteIdentity,
413 state: ProviderEventRoutingState,
414 ): readonly [boolean, ProviderEventRoutingState] {
415 const ownsThread = (threadId: ThreadId): boolean => state.ownedThreadIds.has(threadId);
416 const ownsChildThread = (threadId: ThreadId): boolean =>
417 threadId !== input.threadId && ownsThread(threadId);
418 const ownsRun = (runId: string | null): boolean => runId === input.runId;
419 // ACP plan ids are session-scoped, so a turn can update a plan an earlier turn of this
420 // thread reported. Only an active root turn adopts those rows, never a lingering run.
421 const adoptsEarlierPlan = (threadId: ThreadId): boolean =>
422 threadId === input.threadId && !state.rootTurnEnded;
423 const addProviderThread = (providerThreadId: ProviderThreadId): ProviderEventRoutingState => ({
424 ...state,
425 ownedProviderThreadIds: new Set([...state.ownedProviderThreadIds, providerThreadId]),
426 });
427 const addProviderTurn = (
428 providerTurnId: ProviderTurnId,
429 providerThreadId: ProviderThreadId,
430 root: boolean,
431 ): ProviderEventRoutingState => ({
432 ...state,
433 ownedProviderTurns: new Map(state.ownedProviderTurns).set(providerTurnId, providerThreadId),
434 rootProviderTurnId: root ? providerTurnId : state.rootProviderTurnId,
435 });
436
437 switch (event.type) {
438 case "provider_session.updated":
439 return [false, state];
440 case "app_thread.created": {
441 if (event.appThread.id === input.threadId) {
442 return [true, state];
443 }
444 const isOwnedSubagent =
445 event.appThread.lineage.relationshipToParent === "subagent" &&
446 event.appThread.lineage.parentThreadId !== null &&
447 ownsThread(event.appThread.lineage.parentThreadId) &&
448 (!state.rootTurnEnded ||
449 state.workflowMemberParents.get(event.appThread.id) ===
450 event.appThread.lineage.parentThreadId);
451 if (!isOwnedSubagent) {
452 return [false, state];
453 }
454 return [
455 true,
456 {
457 ...state,
458 ownedThreadIds: new Set([...state.ownedThreadIds, event.appThread.id]),
459 },
460 ];
461 }
462 case "provider_thread.updated": {
463 const belongs =
464 state.ownedProviderThreadIds.has(event.providerThread.id) ||
465 (event.providerThread.appThreadId !== null && ownsThread(event.providerThread.appThreadId));
466 return belongs ? [true, addProviderThread(event.providerThread.id)] : [false, state];
467 }
468 case "provider_turn.updated": {
469 const isRoot = event.providerTurn.runAttemptId === input.attemptId;
470 const belongs =
471 isRoot ||
472 (event.providerTurn.providerThreadId !== input.providerThreadId &&
473 state.ownedProviderThreadIds.has(event.providerTurn.providerThreadId)) ||
474 state.ownedProviderTurns.has(event.providerTurn.id) ||
475 (event.threadId !== undefined && ownsChildThread(event.threadId));
476 return belongs
477 ? [
478 true,
479 addProviderTurn(event.providerTurn.id, event.providerTurn.providerThreadId, isRoot),
480 ]
481 : [false, state];
482 }
483 case "node.updated": {
484 const belongs =
485 ownsRun(event.node.runId) ||
486 ownsChildThread(event.node.threadId) ||
487 ((event.node.kind === "todo_list" || event.node.kind === "plan") &&
488 adoptsEarlierPlan(event.node.threadId));
489 if (!belongs || event.node.providerThreadId === null) {
490 return [belongs, state];
491 }
492 return [true, addProviderThread(event.node.providerThreadId)];
493 }
494 case "subagent.updated": {
495 const moved = event.subagent;
496 if (
497 moved.origin === "provider_native" &&
498 moved.threadId === input.threadId &&
499 moved.runId !== null &&
500 !ownsRun(moved.runId) &&
501 moved.childThreadId !== null &&
502 ownsChildThread(moved.childThreadId)
503 ) {
504 const ownedThreadIds = new Set(state.ownedThreadIds);
505 ownedThreadIds.delete(moved.childThreadId);
506 const ownedProviderThreadIds = new Set(state.ownedProviderThreadIds);
507 if (moved.providerThreadId !== null) ownedProviderThreadIds.delete(moved.providerThreadId);
508 const ownedProviderTurns = new Map(state.ownedProviderTurns);
509 for (const [turnId, threadId] of ownedProviderTurns) {
510 if (threadId === moved.providerThreadId) ownedProviderTurns.delete(turnId);
511 }
512 return [false, { ...state, ownedThreadIds, ownedProviderThreadIds, ownedProviderTurns }];
513 }
514 const belongs = ownsRun(event.subagent.runId) || ownsChildThread(event.subagent.threadId);
515 const coordinator = event.subagent;
516 // Resumes reattribute child threads to the live run, which may not have observed their creation.
517 const adopted =
518 ownsRun(coordinator.runId) &&
519 !state.rootTurnEnded &&
520 isOrchestrationV2WorkActive(coordinator.status) &&
521 coordinator.origin === "provider_native" &&
522 coordinator.childThreadId !== null &&
523 !ownsThread(coordinator.childThreadId)
524 ? {
525 ...state,
526 ownedThreadIds: new Set([...state.ownedThreadIds, coordinator.childThreadId]),
527 }
528 : state;
529 if (
530 !ownsRun(coordinator.runId) ||
531 coordinator.workflow === undefined ||
532 coordinator.childThreadId === null
533 ) {
534 return [belongs, adopted];
535 }
536 const workflowMemberParents = new Map(state.workflowMemberParents);
537 for (const member of coordinator.workflow.agents) {
538 if (member.childThreadId !== undefined) {
539 workflowMemberParents.set(member.childThreadId, coordinator.childThreadId);
540 }
541 }
542 return [true, { ...adopted, workflowMemberParents }];
543 }
544 case "message.updated":
545 return [ownsRun(event.message.runId) || ownsChildThread(event.message.threadId), state];
546 case "steer.updated":
547 return [ownsRun(event.runId), state];
548 case "turn_item.updated": {
549 if (
550 ownsRun(event.turnItem.runId) ||
551 ownsChildThread(event.turnItem.threadId) ||
552 ((event.turnItem.type === "todo_list" || event.turnItem.type === "proposed_plan") &&
553 adoptsEarlierPlan(event.turnItem.threadId))
554 ) {
555 return [true, state];
556 }
557 const inheritedRunId = state.inheritedBackgroundTurnItems.get(event.turnItem.id);
558 const isInheritedBackgroundItem =
559 event.turnItem.threadId === input.threadId &&
560 event.turnItem.runId !== null &&
561 event.turnItem.runId === inheritedRunId &&
562 backgroundCapableTurnItemTypes.has(event.turnItem.type);
563 if (!isInheritedBackgroundItem) {
564 return [false, state];
565 }
566 if (!isSettledTurnItemStatus(event.turnItem.status)) {
567 return [true, state];
568 }
569 const inheritedBackgroundTurnItems = new Map(state.inheritedBackgroundTurnItems);
570 inheritedBackgroundTurnItems.delete(event.turnItem.id);
571 return [true, { ...state, inheritedBackgroundTurnItems }];
572 }
573 case "plan.updated":
574 return [
575 ownsRun(event.plan.runId) ||
576 ownsChildThread(event.plan.threadId) ||
577 adoptsEarlierPlan(event.plan.threadId),
578 state,
579 ];
580 case "runtime_request.updated":
581 return [
582 (event.threadId !== undefined && ownsChildThread(event.threadId)) ||
583 (event.runtimeRequest.providerTurnId !== null &&
584 state.ownedProviderTurns.has(event.runtimeRequest.providerTurnId)),
585 state,
586 ];
587 case "background_work.updated":
588 return [state.ownedProviderTurns.has(event.providerTurnId), state];
589 case "command_output.delta":
590 case "prompt_suggestion.updated":
591 // The session manager serves these to clients directly; runs never see them.
592 return [false, state];
593 case "turn.terminal":
594 return event.providerTurnId === state.rootProviderTurnId
595 ? [true, { ...state, rootTurnEnded: true }]
596 : [false, state];
597 }
598 }
599
600 export class RunExecutionStartError extends Schema.TaggedError<RunExecutionStartError>()(
601 "RunExecutionStartError",
602 {
603 commandId: CommandId,
604 runId: Schema.String,
605 cause: Schema.optional(Schema.Defect()),
606 },
607 ) {
608 override get message(): string {
609 return `Failed to start orchestration V2 run execution ${this.runId}.`;
610 }
611 }
612
613 const isProviderEventStreamError = Schema.is(ProviderAdapter.ProviderAdapterEventStreamError);
614
615 export class RunExecutionIngestError extends Schema.TaggedError<RunExecutionIngestError>()(
616 "RunExecutionIngestError",
617 {
618 runId: Schema.String,
619 cause: Schema.optional(Schema.Defect()),
620 },
621 ) {
622 override get message(): string {
623 return `Failed while ingesting orchestration V2 run execution ${this.runId}.`;
624 }
625 }
626 const isRunExecutionIngestError = Schema.is(RunExecutionIngestError);
627
628 export const RunExecutionServiceV2Error = Schema.Union([
629 RunExecutionStartError,
630 RunExecutionIngestError,
631 ]);
632 export type RunExecutionServiceV2Error = typeof RunExecutionServiceV2Error.Type;
633
634 export interface RunExecutionServiceV2StartRootRunInput {
635 readonly commandId: CommandId;
636 readonly appThread: OrchestrationV2AppThread;
637 readonly providerSessionId: ProviderSessionId;
638 readonly session: ProviderAdapter.ProviderAdapterV2SessionRuntime;
639 readonly run: OrchestrationV2Run;
640 readonly rootNode: OrchestrationV2ExecutionNode;
641 readonly checkpointScope: OrchestrationV2CheckpointScope;
642 readonly providerThread: OrchestrationV2ProviderThread;
643 readonly attempt: OrchestrationV2RunAttempt;
644 readonly attemptId: RunAttemptId;
645 /** Waits for a baseline started with `forkBaseline`; without it the run captures one itself. */
646 readonly baseline?: Effect.Effect<void>;
647 readonly providerTurnOrdinal: number;
648 readonly nativeThreadHasTurns?: boolean;
649 readonly resume?: boolean;
650 readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect<
651 ReadonlyArray<InheritedBackgroundTurnItemRoute>,
652 unknown
653 >;
654 readonly relatedThreadIds?: ReadonlyArray<ThreadId>;
655 readonly relatedProviderThreadIds?: ReadonlyArray<ProviderThreadId>;
656 readonly shouldStartProviderTurn?: () => Effect.Effect<boolean, ProjectionStoreV2Error>;
657 readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>;
658 readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
659 readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>;
660 readonly hasWorkspaceMoved?: () => Effect.Effect<boolean, never>;
661 /**
662 * Stored rows of a thread's unsettled work. Work a dead runtime left is ended from these, since
663 * Stop may have settled it after the snapshots this run holds.
664 */
665 readonly loadUnsettledWork?: (
666 threadId: ThreadId,
667 nodeIds: ReadonlyArray<NodeId>,
668 ) => Effect.Effect<UnsettledWork, ProjectionStoreV2Error>;
669 readonly message: ProviderAdapter.ProviderAdapterV2TurnMessage;
670 readonly modelSelection: ModelSelection;
671 readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy;
672 }
673
674 type RunBaselineInput = Pick<
675 RunExecutionServiceV2StartRootRunInput,
676 "appThread" | "run" | "checkpointScope"
677 >;
678
679 export interface RunExecutionServiceV2Shape {
680 readonly startRootRun: (
681 input: RunExecutionServiceV2StartRootRunInput,
682 ) => Effect.Effect<void, RunExecutionServiceV2Error>;
683 /**
684 * Starts the run's baseline capture on a child fiber so it overlaps provider session startup.
685 * Pass the returned effect to `startRootRun`, which awaits it before prompting the provider.
686 */
687 readonly forkBaseline: (input: RunBaselineInput) => Effect.Effect<Effect.Effect<void>>;
688 }
689
690 export class RunExecutionServiceV2 extends Context.Service<
691 RunExecutionServiceV2,
692 RunExecutionServiceV2Shape
693 >()("t3/orchestration-v2/RunExecutionService/RunExecutionServiceV2") {}
694
695 export const layer: Layer.Layer<
696 RunExecutionServiceV2,
697 never,
698 | CheckpointService.CheckpointServiceV2
699 | EventSink.EventSinkV2
700 | IdAllocator.IdAllocatorV2
701 | ProviderEventIngestor.ProviderEventIngestorV2
702 | ServerSettings.ServerSettingsService
703 | McpAppModelContext.McpAppModelContext
704 | PreviewAgentActivity.PreviewAgentActivity
705 | TurnItemPositionStore.TurnItemPositionStoreV2
706 | ThreadCommandExecutor.ThreadCommandExecutor
707 > = Layer.effect(
708 RunExecutionServiceV2,
709 Effect.gen(function* () {
710 const scope = yield* Effect.scope;
711 const checkpointService = yield* CheckpointService.CheckpointServiceV2;
712 const eventSink = yield* EventSink.EventSinkV2;
713 const idAllocator = yield* IdAllocator.IdAllocatorV2;
714 const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2;
715 const serverSettings = yield* ServerSettings.ServerSettingsService;
716 const mcpAppModelContext = yield* McpAppModelContext.McpAppModelContext;
717 const previewAgentActivity = yield* PreviewAgentActivity.PreviewAgentActivity;
718 const turnItemPositions = yield* TurnItemPositionStore.TurnItemPositionStoreV2;
719 const finalizationObserver = yield* RunFinalizationService.RunFinalizationObserver;
720 const turnAnalytics = yield* ProviderUsageAnalytics.ProviderTurnAnalytics;
721 const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor;
722
723 const writeFinalRunEvents = Effect.fn("RunExecutionService.writeFinalRunEvents")(
724 function* (input: {
725 readonly run: OrchestrationV2Run;
726 readonly rootNode: OrchestrationV2ExecutionNode;
727 readonly checkpointScope: OrchestrationV2CheckpointScope;
728 readonly providerThread: OrchestrationV2ProviderThread;
729 readonly attempt: OrchestrationV2RunAttempt;
730 readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>;
731 readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
732 readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>;
733 readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
734 readonly openRunOwnedSubagentsTerminal?: ProviderTerminalEvent;
735 // Root turn snapshot written with the run when T3 ends the run itself.
736 readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn;
737 // Stored root tool calls still open when the provider runtime was lost; nothing else ends them.
738 readonly lostToolCalls?: ReadonlyArray<OrchestrationV2TurnItem>;
739 readonly terminal: ProviderTerminalEvent;
740 readonly completedAt?: DateTime.Utc;
741 readonly failureItemPersisted: boolean;
742 readonly refreshAfterTurn: Effect.Effect<void>;
743 readonly writeIfRunCurrent?: {
744 readonly activeAttemptId: RunAttemptId;
745 readonly expectedStatus: OrchestrationV2Run["status"];
746 readonly unlessInterruptRequested?: boolean;
747 };
748 }) {
749 const completedAt = input.completedAt ?? (yield* DateTime.now);
750 const finalizedAttempt: OrchestrationV2RunAttempt = {
751 ...input.attempt,
752 status: input.terminal.status,
753 completedAt,
754 };
755 const shouldFinalizeRun =
756 input.shouldFinalizeRun === undefined ? true : yield* input.shouldFinalizeRun();
757 if (!shouldFinalizeRun) {
758 if (input.terminal.status === "interrupted") {
759 const hasUnpairedRequest =
760 input.hasUnpairedRunInterruptRequest === undefined
761 ? false
762 : yield* input.hasUnpairedRunInterruptRequest();
763 if (hasUnpairedRequest) {
764 const reason =
765 input.loadRunInterruptReason === undefined
766 ? undefined
767 : yield* input.loadRunInterruptReason();
768 yield* eventSink.writeWithEffects({
769 effects: [],
770 events: [
771 {
772 id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }),
773 type: "turn-item.updated" as const,
774 threadId: input.run.threadId,
775 runId: input.run.id,
776 nodeId: input.rootNode.id,
777 providerInstanceId: input.run.providerInstanceId,
778 occurredAt: completedAt,
779 payload: makeInterruptResultTurnItem({
780 idAllocator,
781 run: input.run,
782 rootNode: input.rootNode,
783 providerThread: input.providerThread,
784 reason,
785 completedAt,
786 }),
787 },
788 ],
789 });
790 yield* input.refreshAfterTurn;
791 }
792 }
793 return false;
794 }
795 const allocateEventId = () => idAllocator.allocate.event({ threadId: input.run.threadId });
796 const reason =
797 input.terminal.status === "interrupted"
798 ? ((input.loadRunInterruptReason === undefined
799 ? undefined
800 : yield* input.loadRunInterruptReason()) ?? input.terminal.reason)
801 : undefined;
802 const open = input.openRunOwnedSubagents ?? emptyOpenRunOwnedSubagentProjection();
803 const hasOpenSubagentProjection =
804 open.subagents.size > 0 ||
805 open.turnItems.size > 0 ||
806 open.childTurnItems.size > 0 ||
807 open.nodes.size > 0;
808 const subagentTerminal = input.openRunOwnedSubagentsTerminal ?? input.terminal;
809 const cascadedSubagentEvents =
810 isRunOwnedSubagentTerminalStatus(subagentTerminal.status) &&
811 !childTurnsContinue(subagentTerminal) &&
812 hasOpenSubagentProjection
813 ? yield* cascadeTerminalizeRunOwnedSubagents({
814 run: input.run,
815 open,
816 status: subagentTerminal.status,
817 completedAt,
818 allocateEventId,
819 })
820 : [];
821 const persistedStatus =
822 input.terminal.status === "completed" ? "waiting" : input.terminal.status;
823 // Omit delegatedCompletion; the startup snapshot must not overwrite newer acknowledgements, successors, or Stop barriers.
824 const { delegatedCompletion: _delegatedCompletion, ...runWithoutDelegatedCompletion } =
825 input.run;
826 const finalizedRun: OrchestrationV2Run = {
827 ...runWithoutDelegatedCompletion,
828 status: persistedStatus,
829 completedAt: input.terminal.status === "completed" ? null : completedAt,
830 };
831 const finalizedRootNode: OrchestrationV2ExecutionNode = {
832 ...input.rootNode,
833 status: persistedStatus,
834 completedAt: input.terminal.status === "completed" ? null : completedAt,
835 checkpointScopeId: input.checkpointScope.id,
836 };
837 const finalizedProviderThread: OrchestrationV2ProviderThread = {
838 ...input.providerThread,
839 status: finalProviderThreadStatus(input.terminal.threadDisposition),
840 updatedAt: completedAt,
841 };
842 const runEventId = yield* allocateEventId();
843 const nodeEventId = yield* allocateEventId();
844 const providerThreadEventId = yield* allocateEventId();
845 const checkpointCaptureCommandId = CommandId.make(
846 `command:effect:checkpoint.capture:${input.run.id}`,
847 );
848 // Stopped runs capture rollback points for the next message; enqueue before later starts on this effect lane.
849 const finalization = {
850 effects:
851 input.terminal.status === "completed" ||
852 input.terminal.status === "interrupted" ||
853 input.terminal.status === "cancelled"
854 ? [
855 {
856 id: `effect:checkpoint.capture:${input.run.id}`,
857 commandId: checkpointCaptureCommandId,
858 threadId: input.run.threadId,
859 request: {
860 type: "checkpoint.capture" as const,
861 runId: input.run.id,
862 scopeId: input.checkpointScope.id,
863 },
864 },
865 ]
866 : [],
867 events: [
868 ...cascadedSubagentEvents,
869 ...(yield* interruptLostToolCalls({
870 run: input.run,
871 toolCalls: input.lostToolCalls ?? [],
872 completedAt,
873 allocateEventId,
874 })),
875 ...(input.abandonedProviderTurn === undefined
876 ? []
877 : [
878 {
879 id: yield* allocateEventId(),
880 type: "provider-turn.updated" as const,
881 threadId: input.run.threadId,
882 runId: input.run.id,
883 nodeId: input.rootNode.id,
884 providerInstanceId: input.run.providerInstanceId,
885 occurredAt: completedAt,
886 payload: input.abandonedProviderTurn,
887 },
888 ]),
889 {
890 id: yield* allocateEventId(),
891 type: "run-attempt.updated" as const,
892 threadId: input.run.threadId,
893 runId: input.run.id,
894 nodeId: input.rootNode.id,
895 providerInstanceId: input.run.providerInstanceId,
896 occurredAt: completedAt,
897 payload: finalizedAttempt,
898 },
899 ...(input.terminal.status === "interrupted"
900 ? [
901 {
902 id: yield* allocateEventId(),
903 type: "turn-item.updated" as const,
904 threadId: input.run.threadId,
905 runId: input.run.id,
906 nodeId: input.rootNode.id,
907 providerInstanceId: input.run.providerInstanceId,
908 occurredAt: completedAt,
909 payload: makeInterruptResultTurnItem({
910 idAllocator,
911 run: input.run,
912 rootNode: input.rootNode,
913 providerThread: input.providerThread,
914 reason,
915 completedAt,
916 }),
917 },
918 ]
919 : []),
920 // Codex reports a hook that stops the turn as completed; keep its reason visible.
921 ...(input.terminal.status === "completed" && input.terminal.reason !== undefined
922 ? [
923 {
924 id: yield* allocateEventId(),
925 type: "turn-item.updated" as const,
926 threadId: input.run.threadId,
927 runId: input.run.id,
928 nodeId: input.rootNode.id,
929 providerInstanceId: input.run.providerInstanceId,
930 occurredAt: completedAt,
931 payload: {
932 id: idAllocator.derive.runSignalTurnItem({
933 runId: input.run.id,
934 signal: PROVIDER_STOP_REASON_SIGNAL,
935 }),
936 threadId: input.run.threadId,
937 runId: input.run.id,
938 nodeId: input.rootNode.id,
939 providerThreadId: input.providerThread.id,
940 providerTurnId: input.rootNode.providerTurnId,
941 nativeItemRef: null,
942 parentItemId: null,
943 ordinal: input.run.ordinal * 100 + 98,
944 status: "completed" as const,
945 title: "Stopped by the provider",
946 startedAt: completedAt,
947 completedAt,
948 updatedAt: completedAt,
949 type: "system_notice" as const,
950 level: "warning" as const,
951 message: input.terminal.reason,
952 },
953 },
954 ]
955 : []),
956 ...(input.terminal.status === "failed" && !input.failureItemPersisted
957 ? [
958 {
959 id: yield* allocateEventId(),
960 type: "turn-item.updated" as const,
961 threadId: input.run.threadId,
962 runId: input.run.id,
963 nodeId: input.rootNode.id,
964 providerInstanceId: input.run.providerInstanceId,
965 occurredAt: completedAt,
966 payload: makeProviderFailureTurnItem({
967 driver: input.terminal.driver,
968 threadId: input.run.threadId,
969 runId: input.run.id,
970 nodeId: input.rootNode.id,
971 providerThreadId: input.terminal.providerThreadId,
972 providerTurnId: input.terminal.providerTurnId,
973 itemOrdinal: input.terminal.failureItemOrdinal,
974 failure: input.terminal.failure,
975 occurredAt: completedAt,
976 }),
977 },
978 ]
979 : []),
980 {
981 id: runEventId,
982 type: "run.updated",
983 threadId: input.run.threadId,
984 runId: input.run.id,
985 nodeId: input.rootNode.id,
986 providerInstanceId: input.run.providerInstanceId,
987 occurredAt: completedAt,
988 payload: finalizedRun,
989 },
990 {
991 id: nodeEventId,
992 type: "node.updated",
993 threadId: input.run.threadId,
994 runId: input.run.id,
995 nodeId: input.rootNode.id,
996 providerInstanceId: input.run.providerInstanceId,
997 occurredAt: completedAt,
998 payload: finalizedRootNode,
999 },
1000 {
1001 id: providerThreadEventId,
1002 type: "provider-thread.updated",
1003 threadId: input.run.threadId,
1004 providerInstanceId: input.run.providerInstanceId,
1005 occurredAt: completedAt,
1006 payload: finalizedProviderThread,
1007 },
1008 ],
1009 } satisfies Parameters<typeof eventSink.writeWithEffects>[0];
1010 if (input.writeIfRunCurrent !== undefined) {
1011 const result = yield* eventSink.writeIfRunCurrent({
1012 threadId: input.run.threadId,
1013 runId: input.run.id,
1014 activeAttemptId: input.writeIfRunCurrent.activeAttemptId,
1015 expectedStatus: input.writeIfRunCurrent.expectedStatus,
1016 ...(input.writeIfRunCurrent.unlessInterruptRequested === true
1017 ? {
1018 unlessInterruptRequested: {
1019 requestId: idAllocator.derive.runSignalTurnItem({
1020 runId: input.run.id,
1021 signal: "interrupt-request",
1022 }),
1023 resultId: idAllocator.derive.runSignalTurnItem({
1024 runId: input.run.id,
1025 signal: "interrupt-result",
1026 }),
1027 },
1028 }
1029 : {}),
1030 events: finalization.events,
1031 effects: finalization.effects,
1032 });
1033 if (!result.committed) {
1034 return false;
1035 }
1036 } else {
1037 yield* eventSink.writeWithEffects(finalization);
1038 }
1039 yield* turnAnalytics.runSettled({
1040 runId: input.run.id,
1041 status: input.terminal.status,
1042 completedAt,
1043 childrenRunning: hasOpenSubagentProjection && cascadedSubagentEvents.length === 0,
1044 });
1045 yield* input.refreshAfterTurn;
1046 return true;
1047 },
1048 );
1049
1050 const captureRunBaseline = Effect.fn("RunExecutionService.captureBaseline")(function* (
1051 input: RunBaselineInput,
1052 ) {
1053 // Side chats share the parent's worktree; a snapshot would capture the parent's edits.
1054 if (input.appThread.lineage.sideChat === true) return;
1055 yield* checkpointService
1056 .captureBaseline({
1057 scope: input.checkpointScope,
1058 ordinalWithinScope: Math.max(0, input.run.ordinal - 1),
1059 })
1060 .pipe(
1061 Effect.catchCause((cause) =>
1062 Cause.hasInterruptsOnly(cause)
1063 ? Effect.interrupt
1064 : Effect.logWarning(
1065 "orchestration V2 checkpoint baseline capture failed; starting provider without a baseline",
1066 { runId: input.run.id },
1067 ),
1068 ),
1069 );
1070 });
1071
1072 return RunExecutionServiceV2.of({
1073 forkBaseline: (input) =>
1074 captureRunBaseline(input).pipe(
1075 Effect.forkChild({ startImmediately: true }),
1076 Effect.map(Fiber.join),
1077 ),
1078 startRootRun: Effect.fn("RunExecutionService.startRootRun")(function* (input) {
1079 const rootTurnOrigin: ProviderUsageAnalytics.ProviderTurnOrigin = {
1080 createdBy: input.message.createdBy,
1081 creationSource: input.message.creationSource,
1082 scheduled: input.message.scheduledTaskId !== undefined,
1083 subagent: input.appThread.lineage.relationshipToParent === "subagent",
1084 };
1085 // Startup failure and stream shutdown can report the same attempt.
1086 const refreshAfterTurn = yield* Effect.cached(
1087 finalizationObserver.refreshAfterTurn(input.appThread.projectId).pipe(
1088 Effect.catchCause((cause) =>
1089 Effect.logWarning("failed to refresh pull requests after run termination", {
1090 threadId: input.run.threadId,
1091 runId: input.run.id,
1092 cause,
1093 }),
1094 ),
1095 ),
1096 );
1097 const failedProviderTurnId =
1098 input.attempt.providerTurnId ??
1099 idAllocator.derive.providerTurn({
1100 driver: input.providerThread.driver,
1101 nativeTurnId: `failed:${input.attempt.id}`,
1102 });
1103 const makeFailedTerminalEvent = (
1104 failure: OrchestrationV2ProviderFailure,
1105 failureItemOrdinal: number,
1106 ): ProviderTerminalEvent => ({
1107 type: "turn.terminal",
1108 driver: input.providerThread.driver,
1109 providerThreadId: input.providerThread.id,
1110 providerTurnId: failedProviderTurnId,
1111 runOrdinal: input.run.ordinal,
1112 failureItemOrdinal,
1113 status: "failed",
1114 failure,
1115 threadDisposition: "reusable",
1116 });
1117 const responseStreamingMode = yield* Effect.gen(function* () {
1118 const responseStreamingMode = yield* serverSettings.getSettings.pipe(
1119 Effect.map(
1120 (settings) =>
1121 resolveProjectSettings(settings, input.appThread.projectId).settings
1122 .responseStreamingMode,
1123 ),
1124 );
1125 yield* input.baseline ?? captureRunBaseline(input);
1126 if (
1127 input.shouldStartProviderTurn !== undefined &&
1128 !(yield* input.shouldStartProviderTurn())
1129 ) {
1130 return null;
1131 }
1132 return responseStreamingMode;
1133 }).pipe(
1134 Effect.catchCause((cause) =>
1135 Effect.gen(function* () {
1136 if (Cause.hasInterruptsOnly(cause)) {
1137 return yield* Effect.failCause(cause);
1138 }
1139 yield* Effect.logError("orchestration V2 run preparation failed", {
1140 runId: input.run.id,
1141 cause,
1142 });
1143 yield* writeFinalRunEvents({
1144 run: input.run,
1145 rootNode: input.rootNode,
1146 checkpointScope: input.checkpointScope,
1147 providerThread: input.providerThread,
1148 attempt: input.attempt,
1149 terminal: makeFailedTerminalEvent(
1150 makeProviderFailure({
1151 cause: Cause.squash(cause),
1152 message: "Run preparation failed.",
1153 class: "unknown",
1154 }),
1155 input.providerTurnOrdinal * 100 + 1,
1156 ),
1157 failureItemPersisted: false,
1158 refreshAfterTurn,
1159 writeIfRunCurrent: {
1160 activeAttemptId: input.attemptId,
1161 expectedStatus: "running",
1162 },
1163 });
1164 return null;
1165 }),
1166 ),
1167 Effect.mapError(
1168 (cause) =>
1169 new RunExecutionStartError({
1170 commandId: input.commandId,
1171 runId: input.run.id,
1172 cause,
1173 }),
1174 ),
1175 );
1176 if (responseStreamingMode === null) {
1177 return;
1178 }
1179 const terminalEvent = yield* Ref.make<ProviderTerminalEvent | null>(null);
1180 const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100);
1181 const latestProviderThread = yield* Ref.make(input.providerThread);
1182 const latestRootProviderTurn = yield* Ref.make<OrchestrationV2ProviderTurn | null>(null);
1183 // Later root turns in one attempt are provider continuations, such as Codex goal turns.
1184 let firstRootProviderTurnId: ProviderTurnId | undefined;
1185 const analyticsContext = {
1186 runId: input.run.id,
1187 modelSelection: input.modelSelection,
1188 runtimeMode: input.runtimePolicy.runtimeMode,
1189 interactionMode: input.runtimePolicy.interactionMode,
1190 origin: rootTurnOrigin,
1191 ...(input.run.restartContinuationOfRunId === undefined
1192 ? {}
1193 : { restartContinuation: true }),
1194 } satisfies ProviderUsageAnalytics.ProviderTurnAnalyticsContext;
1195 if (input.attempt.reason === "initial") {
1196 yield* turnAnalytics.runStarted({
1197 runId: input.run.id,
1198 driver: input.providerThread.driver,
1199 context: { ...analyticsContext, segment: "initial" },
1200 });
1201 }
1202 const routeIdentity: ProviderEventRouteIdentity = {
1203 threadId: input.run.threadId,
1204 runId: input.run.id,
1205 attemptId: input.attempt.id,
1206 providerThreadId: input.providerThread.id,
1207 };
1208 const eventSubscription =
1209 input.session.subscribeEvents === undefined
1210 ? { events: input.session.events, close: Effect.void }
1211 : yield* input.session.subscribeEvents;
1212 const inheritedBackgroundTurnItems = yield* (
1213 input.loadInheritedBackgroundTurnItems?.() ?? Effect.succeed([])
1214 ).pipe(
1215 Effect.onError(() => eventSubscription.close),
1216 Effect.mapError(
1217 (cause) =>
1218 new RunExecutionStartError({
1219 commandId: input.commandId,
1220 runId: input.run.id,
1221 cause,
1222 }),
1223 ),
1224 );
1225 const inheritedBackgroundTurnItemsById = new Map(
1226 inheritedBackgroundTurnItems.map((item) => [item.id, item.runId]),
1227 );
1228 const eventRouting = yield* Ref.make<ProviderEventRoutingState>(
1229 makeProviderEventRoutingState({
1230 identity: routeIdentity,
1231 inheritedBackgroundTurnItems,
1232 providerTurnId: input.attempt.providerTurnId,
1233 ...(input.relatedThreadIds === undefined
1234 ? {}
1235 : { relatedThreadIds: input.relatedThreadIds }),
1236 ...(input.relatedProviderThreadIds === undefined
1237 ? {}
1238 : { relatedProviderThreadIds: input.relatedProviderThreadIds }),
1239 }),
1240 );
1241 const rootTerminalSeen = yield* Ref.make(false);
1242 const rootRunFinalized = yield* Ref.make(false);
1243 const providerThreadOwnerLost = yield* Ref.make(false);
1244 const pendingBackgroundProviderTurns = new Set<ProviderTurnId>();
1245 const activeChildProviderTurns = yield* Ref.make<ReadonlySet<ProviderTurnId>>(new Set());
1246 const activeChildSubagents = yield* Ref.make<ReadonlyMap<NodeId, ThreadId>>(new Map());
1247 const activeBackgroundTurnItems = yield* Ref.make<
1248 ReadonlySet<OrchestrationV2TurnItem["id"]>
1249 >(new Set(inheritedBackgroundTurnItemsById.keys()));
1250 const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection());
1251 // Open tool calls on the root thread, to end from their stored rows if the runtime is lost.
1252 const openRootToolCalls = new Set<TurnItemId>();
1253 // The last stored root snapshot still reports background work, by flag or task roster.
1254 let rootSnapshotPendingWork = false;
1255 // Owned child threads whose last snapshot reported pending work, such as a queued native
1256 // continuation. Unlike the root's, these must hold the stream past owner loss.
1257 const childSnapshotPendingWork = new Set<ProviderThreadId>();
1258 let rootTurnOwnershipChecked = false;
1259 const assistantStreaming = makeAssistantStreamingFilter(responseStreamingMode);
1260 // Set when ingestion ends or finalization settles the run's open items, which a later
1261 // streaming snapshot must not reopen.
1262 let idleFlushClosed = false;
1263 const workspaceHandedOff = Effect.suspend(() =>
1264 input.shouldFinalizeRun === undefined || input.hasWorkspaceMoved === undefined
1265 ? Effect.succeed(false)
1266 : input.hasWorkspaceMoved(),
1267 );
1268 const finalizeRootRun = Effect.fn("RunExecutionService.finalizeRootRun")(function* (
1269 terminal: ProviderTerminalEvent,
1270 abandonedProviderTurn?: OrchestrationV2ProviderTurn,
1271 ) {
1272 if (yield* Ref.get(rootRunFinalized)) {
1273 return;
1274 }
1275 const providerThread = yield* Ref.get(latestProviderThread);
1276 const openSubagents = yield* Ref.get(openRunOwnedSubagents);
1277 const writeTerminal = (settledTerminal: ProviderTerminalEvent) =>
1278 writeFinalRunEvents({
1279 run: input.run,
1280 rootNode: input.rootNode,
1281 checkpointScope: input.checkpointScope,
1282 providerThread,
1283 attempt: input.attempt,
1284 ...(input.shouldFinalizeRun === undefined
1285 ? {}
1286 : {
1287 shouldFinalizeRun: input.shouldFinalizeRun,
1288 // Stop may commit between the ownership read and this
1289 // terminal write. Gate the events and checkpoint together.
1290 writeIfRunCurrent: {
1291 activeAttemptId: input.attempt.id,
1292 expectedStatus: "running" as const,
1293 unlessInterruptRequested: settledTerminal !== terminal,
1294 },
1295 }),
1296 ...(input.hasUnpairedRunInterruptRequest === undefined
1297 ? {}
1298 : {
1299 hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest,
1300 }),
1301 ...(input.loadRunInterruptReason === undefined
1302 ? {}
1303 : { loadRunInterruptReason: input.loadRunInterruptReason }),
1304 openRunOwnedSubagents: openSubagents,
1305 openRunOwnedSubagentsTerminal: terminal,
1306 ...(abandonedProviderTurn === undefined ? {} : { abandonedProviderTurn }),
1307 terminal: settledTerminal,
1308 failureItemPersisted: settledTerminal.status === "failed",
1309 refreshAfterTurn,
1310 }).pipe(
1311 Effect.mapError(
1312 (cause) => new RunExecutionIngestError({ runId: input.run.id, cause }),
1313 ),
1314 );
1315 // A workspace handoff detaches the session and stops the turn that asked
1316 // for it: the run completes, while its native subagents still settle
1317 // as interrupted. The completed write refuses in the same transaction
1318 // as a stored Stop request, so a racing Stop stays authoritative.
1319 const handedOff = terminal.status === "interrupted" && (yield* workspaceHandedOff);
1320 if (!handedOff || !(yield* writeTerminal({ ...terminal, status: "completed" }))) {
1321 yield* writeTerminal(terminal);
1322 }
1323 if (isRunOwnedSubagentTerminalStatus(terminal.status) && !childTurnsContinue(terminal)) {
1324 yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection());
1325 idleFlushClosed = true;
1326 }
1327 yield* Ref.set(rootRunFinalized, true);
1328 });
1329 // The stored rows of the work this run still tracks as open on the given threads.
1330 const loadTrackedWork = Effect.fn("RunExecutionService.loadTrackedWork")(function* (
1331 threadIds: Iterable<ThreadId>,
1332 ) {
1333 const open = yield* Ref.get(openRunOwnedSubagents);
1334 const subagents = new Map<NodeId, OrchestrationV2Subagent>();
1335 const nodes = new Map<NodeId, OrchestrationV2ExecutionNode>();
1336 const turnItems = new Map<TurnItemId, OrchestrationV2TurnItem>();
1337 if (input.loadUnsettledWork === undefined) return { subagents, nodes, turnItems };
1338 const trackedItemIds = new Set<TurnItemId>([
1339 ...openRootToolCalls.keys(),
1340 ...open.childTurnItems.keys(),
1341 ...[...open.turnItems.values()].map((item) => item.id),
1342 ]);
1343 for (const threadId of new Set(threadIds)) {
1344 const stored = yield* input.loadUnsettledWork(
1345 threadId,
1346 [...open.nodes.values(), ...open.subagents.values()]
1347 .filter((record) => record.threadId === threadId)
1348 .map((record) => record.id),
1349 );
1350 for (const subagent of stored.subagents) {
1351 if (open.subagents.has(subagent.id) && !isSettledSubagentStatus(subagent.status)) {
1352 subagents.set(subagent.id, subagent);
1353 }
1354 }
1355 for (const node of stored.nodes) {
1356 if (open.nodes.has(node.id) && isOpenExecutionNodeStatus(node.status)) {
1357 nodes.set(node.id, node);
1358 }
1359 }
1360 for (const item of stored.turnItems) {
1361 if (trackedItemIds.has(item.id) && !isSettledTurnItemStatus(item.status)) {
1362 turnItems.set(item.id, item);
1363 }
1364 }
1365 }
1366 return { subagents, nodes, turnItems };
1367 });
1368 const loadLostRootToolCalls = loadTrackedWork([input.run.threadId]).pipe(
1369 Effect.map(({ turnItems }) =>
1370 [...turnItems.values()].filter((item) => openRootToolCalls.has(item.id)),
1371 ),
1372 );
1373 // Subagents and tool calls that outlived the root run die with the provider runtime. Stop
1374 // can settle them meanwhile, so each write holds only while its stored row is unchanged.
1375 const endWorkLostAfterRootRun = Effect.fn("RunExecutionService.endWorkLostAfterRootRun")(
1376 function* (completedAt: DateTime.Utc) {
1377 const allocateEventId = () =>
1378 idAllocator.allocate.event({ threadId: input.run.threadId });
1379 const open = yield* Ref.get(openRunOwnedSubagents);
1380 const stored = yield* loadTrackedWork([
1381 input.run.threadId,
1382 ...[
1383 ...open.subagents.values(),
1384 ...open.nodes.values(),
1385 ...open.childTurnItems.values(),
1386 ].map((record) => record.threadId),
1387 ]);
1388 for (const subagent of stored.subagents.values()) {
1389 const subagentItemId = open.turnItems.get(subagent.id)?.id;
1390 const subagentItem =
1391 subagentItemId === undefined ? undefined : stored.turnItems.get(subagentItemId);
1392 const events = yield* cascadeTerminalizeRunOwnedSubagents({
1393 run: input.run,
1394 open: {
1395 subagents: new Map([[subagent.id, subagent]]),
1396 turnItems: new Map(
1397 subagentItem?.type === "subagent" ? [[subagent.id, subagentItem]] : [],
1398 ),
1399 childTurnItems: new Map(
1400 [...stored.turnItems].filter(
1401 ([id, item]) =>
1402 open.childTurnItems.has(id) && item.threadId === subagent.childThreadId,
1403 ),
1404 ),
1405 nodes: new Map(
1406 [...stored.nodes].filter(
1407 ([id, node]) =>
1408 id === subagent.id || node.threadId === subagent.childThreadId,
1409 ),
1410 ),
1411 linkedChildThreadIds: new Set(
1412 subagent.childThreadId === null ? [] : [subagent.childThreadId],
1413 ),
1414 },
1415 status: "failed",
1416 completedAt,
1417 allocateEventId,
1418 });
1419 yield* eventSink.writeIfSubagentUnchanged({ expected: subagent, events });
1420 }
1421 for (const toolCall of stored.turnItems.values()) {
1422 if (!openRootToolCalls.has(toolCall.id)) continue;
1423 yield* eventSink.writeIfTurnItemUnchanged({
1424 expected: toolCall,
1425 events: yield* interruptLostToolCalls({
1426 run: input.run,
1427 toolCalls: [toolCall],
1428 completedAt,
1429 allocateEventId,
1430 }),
1431 });
1432 }
1433 yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection());
1434 openRootToolCalls.clear();
1435 },
1436 (effect) =>
1437 effect.pipe(
1438 Effect.catchCauseIf(
1439 (cause) => !Cause.hasInterruptsOnly(cause),
1440 (cause) =>
1441 Effect.logWarning(
1442 "orchestration V2 failed to end work lost with the provider runtime",
1443 { runId: input.run.id, cause },
1444 ),
1445 ),
1446 ),
1447 );
1448 // Runs for every provider event, including streamed text updates; avoid a per-event span.
1449 const trackChildLifecycle = Effect.fnUntraced(function* (
1450 event: ProviderAdapter.ProviderAdapterV2Event,
1451 deliverable: boolean,
1452 ) {
1453 const routing = yield* Ref.get(eventRouting);
1454 if (event.type === "provider_turn.updated") {
1455 const isRoot =
1456 event.providerTurn.runAttemptId === input.attempt.id ||
1457 event.providerTurn.id === routing.rootProviderTurnId;
1458 if (!isRoot) {
1459 yield* Ref.update(activeChildProviderTurns, (current) => {
1460 const next = new Set(current);
1461 if (isTerminalProviderTurnStatus(event.providerTurn.status)) {
1462 next.delete(event.providerTurn.id);
1463 } else {
1464 next.add(event.providerTurn.id);
1465 }
1466 return next;
1467 });
1468 }
1469 }
1470 if (event.type === "subagent.updated") {
1471 const belongsToRootRun = event.subagent.runId === input.run.id;
1472 const belongsToOwnedChildThread =
1473 event.subagent.threadId !== input.run.threadId &&
1474 routing.ownedThreadIds.has(event.subagent.threadId);
1475 if (belongsToRootRun || belongsToOwnedChildThread) {
1476 yield* Ref.update(activeChildSubagents, (current) => {
1477 const next = new Map(current);
1478 if (isSettledSubagentStatus(event.subagent.status)) {
1479 next.delete(event.subagent.id);
1480 } else {
1481 next.set(event.subagent.id, event.subagent.threadId);
1482 }
1483 return next;
1484 });
1485 }
1486 // Preserve childThreadId linkage for the root-run lifetime even after the subagent
1487 // row terminalizes, so open child-thread nodes can still be proven linked on a
1488 // later root interrupt. A provider records nested native subagents on the child
1489 // thread, runless; they die with this run too.
1490 if (
1491 belongsToRootRun ||
1492 (belongsToOwnedChildThread &&
1493 event.subagent.runId === null &&
1494 event.subagent.origin === "provider_native")
1495 ) {
1496 yield* Ref.update(openRunOwnedSubagents, (current) => {
1497 const withLink = withLinkedChildThreadId(current, event.subagent.childThreadId);
1498 const subagents = new Map(withLink.subagents);
1499 if (isSettledSubagentStatus(event.subagent.status)) {
1500 subagents.delete(event.subagent.id);
1501 } else {
1502 subagents.set(event.subagent.id, event.subagent);
1503 }
1504 return { ...withLink, subagents };
1505 });
1506 }
1507 }
1508 if (event.type === "node.updated") {
1509 const belongsToRootSubagent =
1510 event.node.kind === "subagent" && event.node.runId === input.run.id;
1511 const belongsToOwnedChildThread =
1512 event.node.threadId !== input.run.threadId &&
1513 routing.ownedThreadIds.has(event.node.threadId);
1514 if (
1515 !belongsToRootSubagent &&
1516 !(
1517 belongsToOwnedChildThread &&
1518 (event.node.runId === null || event.node.runId === input.run.id)
1519 )
1520 ) {
1521 return;
1522 }
1523 yield* Ref.update(openRunOwnedSubagents, (current) => {
1524 const nodes = new Map(current.nodes);
1525 if (isOpenExecutionNodeStatus(event.node.status)) {
1526 nodes.set(event.node.id, event.node);
1527 } else {
1528 nodes.delete(event.node.id);
1529 }
1530 return { ...current, nodes };
1531 });
1532 }
1533 if (event.type === "turn_item.updated") {
1534 const belongsToRootRun = event.turnItem.runId === input.run.id;
1535 const belongsToOwnedChildThread =
1536 event.turnItem.threadId !== input.run.threadId &&
1537 routing.ownedThreadIds.has(event.turnItem.threadId);
1538 const belongsToInheritedBackgroundItem =
1539 event.turnItem.threadId === input.run.threadId &&
1540 event.turnItem.runId !== null &&
1541 inheritedBackgroundTurnItemsById.get(event.turnItem.id) === event.turnItem.runId;
1542 if (
1543 backgroundCapableTurnItemTypes.has(event.turnItem.type) &&
1544 (belongsToRootRun || belongsToOwnedChildThread || belongsToInheritedBackgroundItem)
1545 ) {
1546 yield* Ref.update(activeBackgroundTurnItems, (current) => {
1547 const next = new Set(current);
1548 if (isSettledTurnItemStatus(event.turnItem.status)) {
1549 next.delete(event.turnItem.id);
1550 } else {
1551 next.add(event.turnItem.id);
1552 }
1553 return next;
1554 });
1555 if (
1556 event.turnItem.threadId === input.run.threadId &&
1557 processBoundToolCallTypes.has(event.turnItem.type)
1558 ) {
1559 if (isSettledTurnItemStatus(event.turnItem.status)) {
1560 openRootToolCalls.delete(event.turnItem.id);
1561 } else {
1562 openRootToolCalls.add(event.turnItem.id);
1563 }
1564 }
1565 }
1566 if (
1567 belongsToOwnedChildThread &&
1568 (event.turnItem.runId === null || belongsToRootRun) &&
1569 deliverable
1570 ) {
1571 yield* Ref.update(openRunOwnedSubagents, (current) => {
1572 const childTurnItems = new Map(current.childTurnItems);
1573 if (isSettledTurnItemStatus(event.turnItem.status)) {
1574 childTurnItems.delete(event.turnItem.id);
1575 } else {
1576 childTurnItems.set(event.turnItem.id, event.turnItem);
1577 }
1578 return { ...current, childTurnItems };
1579 });
1580 }
1581 if (belongsToRootRun && event.turnItem.type === "subagent") {
1582 const subagentItem = event.turnItem;
1583 yield* Ref.update(openRunOwnedSubagents, (current) => {
1584 const withLink = withLinkedChildThreadId(current, subagentItem.childThreadId);
1585 const turnItems = new Map(withLink.turnItems);
1586 if (isSettledTurnItemStatus(subagentItem.status)) {
1587 turnItems.delete(subagentItem.subagentId);
1588 } else {
1589 turnItems.set(subagentItem.subagentId, subagentItem);
1590 }
1591 return { ...withLink, turnItems };
1592 });
1593 }
1594 }
1595 });
1596 const agentTurnsSettled = Effect.gen(function* () {
1597 if (!(yield* Ref.get(rootTerminalSeen))) {
1598 return false;
1599 }
1600 const terminal = yield* Ref.get(terminalEvent);
1601 if (
1602 terminal !== null &&
1603 terminal.status !== "completed" &&
1604 !childTurnsContinue(terminal)
1605 ) {
1606 return true;
1607 }
1608 if (pendingBackgroundProviderTurns.size > 0) {
1609 return false;
1610 }
1611 const childProviderTurns = yield* Ref.get(activeChildProviderTurns);
1612 if (childProviderTurns.size > 0) {
1613 return false;
1614 }
1615 const childSubagents = yield* Ref.get(activeChildSubagents);
1616 return childSubagents.size === 0;
1617 });
1618 // Claude background Bash has no turn-item projection. Use only the
1619 // thread-scoped probe: session-wide pending work (siblings, wake
1620 // buffers, session subagents) must not pin this root subscription.
1621 const threadHasPendingBackgroundWork = Effect.gen(function* () {
1622 // The live probe reads adapter state, which can be ahead of the events consumed here.
1623 if (rootSnapshotPendingWork) {
1624 return true;
1625 }
1626 if (input.session.hasPendingBackgroundWorkForThread === undefined) {
1627 return false;
1628 }
1629 return yield* input.session
1630 .hasPendingBackgroundWorkForThread(yield* Ref.get(latestProviderThread))
1631 .pipe(Effect.catchCause(() => Effect.succeed(false)));
1632 });
1633 const shouldStopProviderEventIngestion = Effect.gen(function* () {
1634 if (!(yield* agentTurnsSettled)) {
1635 return false;
1636 }
1637 const terminal = yield* Ref.get(terminalEvent);
1638 if (
1639 terminal !== null &&
1640 terminal.status !== "completed" &&
1641 !childTurnsContinue(terminal)
1642 ) {
1643 // A failed turn can leave background work running in a live provider process.
1644 return (
1645 terminal.status !== "failed" ||
1646 (yield* Ref.get(providerThreadOwnerLost)) ||
1647 !(yield* threadHasPendingBackgroundWork)
1648 );
1649 }
1650 // Newly owned items depend on adapters emitting a non-terminal event before the root
1651 // terminal. Owner loss (a newer run claimed lastRunOrdinal) must not close this stream
1652 // while these sets are non-empty: turn_item.updated writes are not ownership-gated, so
1653 // late completions still land.
1654 const backgroundItems = yield* Ref.get(activeBackgroundTurnItems);
1655 if (backgroundItems.size > 0 || childSnapshotPendingWork.size > 0) {
1656 return false;
1657 }
1658 if (yield* Ref.get(providerThreadOwnerLost)) {
1659 return true;
1660 }
1661 return !(yield* threadHasPendingBackgroundWork);
1662 });
1663 // Held text writes nothing, so its item would take a position only
1664 // when finally written, behind every item that began after it.
1665 const reservedHeldTurnItemIds = new Set<TurnItemId>();
1666 // Ends once the root turn and the children it drives settle. Background commands such
1667 // as a dev server keep this consumer alive afterwards and must not pin preview focus.
1668 const endPreviewActivity = yield* previewAgentActivity.begin(input.run.threadId);
1669 // Per-event routing and lifecycle bookkeeping share the run's tracing boundary.
1670 const routeEvent = Effect.fnUntraced(function* (
1671 event: ProviderAdapter.ProviderAdapterV2Event,
1672 ) {
1673 const previous = yield* Ref.get(eventRouting);
1674 const [accepted, routing] = routeProviderEvent(event, routeIdentity, previous);
1675 yield* Ref.set(eventRouting, routing);
1676 if (
1677 event.type === "subagent.updated" &&
1678 event.subagent.childThreadId !== null &&
1679 previous.ownedThreadIds.has(event.subagent.childThreadId) &&
1680 !routing.ownedThreadIds.has(event.subagent.childThreadId)
1681 ) {
1682 const moved = event.subagent;
1683 const detached = yield* Ref.modify(openRunOwnedSubagents, (current) => {
1684 const subagentIds = new Set([moved.id]);
1685 for (const subagent of current.subagents.values()) {
1686 if (subagent.threadId === moved.childThreadId) subagentIds.add(subagent.id);
1687 }
1688 const childTurnItems = new Map(
1689 [...current.childTurnItems].filter(
1690 ([, item]) => item.threadId !== moved.childThreadId,
1691 ),
1692 );
1693 const turnItems = new Map(current.turnItems);
1694 turnItems.delete(moved.id);
1695 const detachedItems = new Set(
1696 [...current.childTurnItems.values(), ...current.turnItems.values()]
1697 .filter(
1698 (item) => item.threadId === moved.childThreadId || item.nodeId === moved.id,
1699 )
1700 .map((item) => item.id),
1701 );
1702 return [
1703 { itemIds: detachedItems, subagentIds },
1704 {
1705 subagents: new Map([...current.subagents].filter(([id]) => !subagentIds.has(id))),
1706 turnItems,
1707 childTurnItems,
1708 nodes: new Map(
1709 [...current.nodes].filter(
1710 ([id, node]) => id !== moved.id && node.threadId !== moved.childThreadId,
1711 ),
1712 ),
1713 linkedChildThreadIds: new Set(
1714 [...current.linkedChildThreadIds].filter((id) => id !== moved.childThreadId),
1715 ),
1716 },
1717 ];
1718 });
1719 yield* Ref.update(
1720 activeChildSubagents,
1721 (current) =>
1722 new Map(
1723 [...current].filter(
1724 ([id, threadId]) =>
1725 !detached.subagentIds.has(id) && threadId !== moved.childThreadId,
1726 ),
1727 ),
1728 );
1729 yield* Ref.update(
1730 activeChildProviderTurns,
1731 (current) => new Set([...current].filter((id) => routing.ownedProviderTurns.has(id))),
1732 );
1733 for (const id of pendingBackgroundProviderTurns) {
1734 if (!routing.ownedProviderTurns.has(id)) {
1735 pendingBackgroundProviderTurns.delete(id);
1736 }
1737 }
1738 // Routing now rejects the moved thread's snapshots, so none would clear its entry.
1739 for (const id of childSnapshotPendingWork) {
1740 if (!routing.ownedProviderThreadIds.has(id)) childSnapshotPendingWork.delete(id);
1741 }
1742 yield* Ref.update(
1743 activeBackgroundTurnItems,
1744 (current) => new Set([...current].filter((id) => !detached.itemIds.has(id))),
1745 );
1746 }
1747 return { event, accepted };
1748 });
1749 // Per-event ingestion avoids adding a span for every provider snapshot.
1750 const ingestDelivered = Effect.fnUntraced(function* (
1751 event: ProviderAdapter.ProviderAdapterV2Event,
1752 deliveredEvent: ProviderAdapter.ProviderAdapterV2Event,
1753 ) {
1754 const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen);
1755 const isRootProviderThreadUpdate =
1756 event.type === "provider_thread.updated" &&
1757 event.providerThread.id === input.providerThread.id;
1758 const nativeChildTurn =
1759 event.type === "provider_turn.updated" &&
1760 event.providerTurn.providerThreadId !== input.providerThread.id;
1761 if (event.type === "provider_turn.updated" && !nativeChildTurn) {
1762 firstRootProviderTurnId ??= event.providerTurn.id;
1763 }
1764 const storedEvents = yield* providerEventIngestor.ingestNormalized({
1765 analyticsContext: nativeChildTurn
1766 ? {
1767 ...analyticsContext,
1768 origin: NATIVE_SUBAGENT_TURN_ORIGIN,
1769 segment: "native_child",
1770 }
1771 : {
1772 ...analyticsContext,
1773 segment:
1774 event.type === "provider_turn.updated" &&
1775 event.providerTurn.id !== firstRootProviderTurnId
1776 ? "continuation"
1777 : input.attempt.reason,
1778 },
1779 providerSessionId: input.providerSessionId,
1780 providerInstanceId: input.run.providerInstanceId,
1781 threadId: input.run.threadId,
1782 runId: input.run.id,
1783 nodeId: input.rootNode.id,
1784 event: deliveredEvent,
1785 ...(isRootProviderThreadUpdate
1786 ? rootTerminalAlreadySeen
1787 ? {
1788 writeIfProviderThreadOwner: {
1789 providerThreadId: input.providerThread.id,
1790 runId: input.run.id,
1791 activeAttemptId: input.attempt.id,
1792 expectedLastRunOrdinal: input.run.ordinal,
1793 },
1794 }
1795 : {
1796 writeIfRunCurrent: {
1797 runId: input.run.id,
1798 activeAttemptId: input.attempt.id,
1799 expectedStatus: "running" as const,
1800 },
1801 }
1802 : {}),
1803 });
1804 if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEvents.length === 0) {
1805 yield* Ref.set(providerThreadOwnerLost, true);
1806 }
1807 return storedEvents.length;
1808 });
1809 // The timer only wakes ingestion. The flush runs as an ingestion step outside any provider
1810 // batch, so it stays ordered with routing and snapshots, and a write has committed when
1811 // it returns.
1812 const idleFlushTicks = yield* Queue.dropping<{
1813 readonly type: "idle_flush";
1814 readonly at: number;
1815 }>(1);
1816 let idleFlushWake: { readonly at: number; readonly fiber: Fiber.Fiber<void> } | undefined;
1817 const scheduleIdleFlush = Effect.suspend(() => {
1818 const at = assistantStreaming.nextIdleFlushAt();
1819 if (
1820 idleFlushClosed ||
1821 at === null ||
1822 (idleFlushWake !== undefined && idleFlushWake.at <= at)
1823 )
1824 return Effect.void;
1825 const previous = idleFlushWake?.fiber;
1826 return Effect.gen(function* () {
1827 if (previous !== undefined) yield* Fiber.interrupt(previous);
1828 const now = DateTime.toEpochMillis(yield* DateTime.now);
1829 const fiber = yield* Effect.sleep(Duration.millis(Math.max(0, at - now))).pipe(
1830 Effect.andThen(Queue.offer(idleFlushTicks, { type: "idle_flush", at })),
1831 Effect.asVoid,
1832 Effect.forkDetach,
1833 );
1834 idleFlushWake = { at, fiber };
1835 });
1836 });
1837 const closeIdleFlush = Effect.suspend(() => {
1838 idleFlushClosed = true;
1839 const wake = idleFlushWake;
1840 idleFlushWake = undefined;
1841 return wake === undefined ? Effect.void : Fiber.interrupt(wake.fiber);
1842 });
1843 // Per-tick ingestion avoids adding a span for every idle wake.
1844 const flushIdleAssistantText = Effect.fnUntraced(function* (wokeAt: number) {
1845 // A tick means its timer slept until `wokeAt`, but the wall clock can still read up
1846 // to a millisecond earlier; trusting it would keep the finished wake and never re-arm.
1847 const now = Math.max(DateTime.toEpochMillis(yield* DateTime.now), wokeAt);
1848 // A wake due by now has fired; a later deadline needs a new one.
1849 if (idleFlushWake !== undefined && idleFlushWake.at <= now) idleFlushWake = undefined;
1850 if (idleFlushClosed) return;
1851 const rootSettled = yield* Ref.get(rootRunFinalized);
1852 for (const event of assistantStreaming.takeIdleFlushes(now)) {
1853 // Settling the root run closed its messages; only children may still stream.
1854 if (rootSettled && streamedThreadId(event) === input.run.threadId) continue;
1855 // The held text's child may have been handed to another run since it arrived.
1856 if (!routeProviderEvent(event, routeIdentity, yield* Ref.get(eventRouting))[0])
1857 continue;
1858 // A failed write fails ingestion like any provider write.
1859 yield* ingestDelivered(event, event);
1860 yield* trackChildLifecycle(event, true);
1861 }
1862 yield* scheduleIdleFlush;
1863 });
1864 const ingestEvent = Effect.fnUntraced(function* ({
1865 event,
1866 accepted,
1867 }: {
1868 event: ProviderAdapter.ProviderAdapterV2Event;
1869 accepted: boolean;
1870 }) {
1871 if (!accepted) return;
1872 if (
1873 event.type === "provider_turn.updated" &&
1874 event.providerTurn.runAttemptId === input.attempt.id &&
1875 event.providerTurn.providerThreadId === input.providerThread.id
1876 ) {
1877 yield* Ref.set(latestRootProviderTurn, event.providerTurn);
1878 }
1879 let storedEventCount = 0;
1880 const deliveredEvent = assistantStreaming.filter(
1881 event,
1882 DateTime.toEpochMillis(yield* DateTime.now),
1883 );
1884 const startedRootTurn =
1885 event.type === "provider_turn.updated" &&
1886 event.providerTurn.runAttemptId === input.attempt.id &&
1887 event.providerTurn.providerThreadId === input.providerThread.id &&
1888 !isTerminalProviderTurnStatus(event.providerTurn.status) &&
1889 !rootTurnOwnershipChecked
1890 ? event.providerTurn
1891 : undefined;
1892 const shouldStartProviderTurn = input.shouldStartProviderTurn;
1893 if (deliveredEvent && startedRootTurn !== undefined && shouldStartProviderTurn) {
1894 rootTurnOwnershipChecked = true;
1895 // Stop plans under this lock from stored turns. Storing the first turn and rechecking
1896 // the attempt under it means Stop either saw the turn or already settled the run.
1897 const [stored, owned] = yield* threadCommands.withLock(
1898 input.run.threadId,
1899 Effect.all([
1900 ingestDelivered(event, deliveredEvent),
1901 shouldStartProviderTurn().pipe(Effect.orElseSucceed(() => true)),
1902 ]),
1903 );
1904 storedEventCount = stored;
1905 if (!owned) {
1906 yield* Ref.get(latestProviderThread).pipe(
1907 Effect.flatMap((providerThread) =>
1908 input.session.interruptTurn({
1909 providerThread,
1910 providerTurnId: startedRootTurn.id,
1911 }),
1912 ),
1913 Effect.catchCauseIf(
1914 (cause) => !Cause.hasInterruptsOnly(cause),
1915 (cause) =>
1916 Effect.logWarning(
1917 "orchestration V2 failed to interrupt a provider turn that started after Stop",
1918 { runId: input.run.id, cause },
1919 ),
1920 ),
1921 Effect.forkIn(scope),
1922 );
1923 }
1924 } else if (deliveredEvent) {
1925 storedEventCount = yield* ingestDelivered(event, deliveredEvent);
1926 } else if (
1927 event.type === "turn_item.updated" &&
1928 (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") &&
1929 event.turnItem.streaming &&
1930 !reservedHeldTurnItemIds.has(event.turnItem.id)
1931 ) {
1932 yield* turnItemPositions.allocate({
1933 threadId: event.turnItem.threadId,
1934 turnItemId: event.turnItem.id,
1935 runId: event.turnItem.runId,
1936 });
1937 reservedHeldTurnItemIds.add(event.turnItem.id);
1938 }
1939 if (event.type === "provider_thread.updated") {
1940 if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) {
1941 yield* Ref.set(latestProviderThread, event.providerThread);
1942 rootSnapshotPendingWork =
1943 event.pendingBackgroundWork === true ||
1944 (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0;
1945 } else if (event.providerThread.id !== input.providerThread.id) {
1946 if (event.pendingBackgroundWork === true) {
1947 childSnapshotPendingWork.add(event.providerThread.id);
1948 } else if (event.pendingBackgroundWork === false) {
1949 childSnapshotPendingWork.delete(event.providerThread.id);
1950 }
1951 }
1952 }
1953 if (
1954 event.type === "turn_item.updated" &&
1955 event.turnItem.providerTurnId === (yield* Ref.get(eventRouting)).rootProviderTurnId
1956 ) {
1957 yield* Ref.update(latestTurnItemOrdinal, (current) =>
1958 Math.max(current, event.turnItem.ordinal),
1959 );
1960 }
1961 if (event.type === "turn.terminal") {
1962 yield* Ref.set(terminalEvent, event);
1963 yield* Ref.set(rootTerminalSeen, true);
1964 yield* finalizeRootRun(event);
1965 }
1966 if (event.type === "background_work.updated") {
1967 if (event.pending) {
1968 pendingBackgroundProviderTurns.add(event.providerTurnId);
1969 } else {
1970 pendingBackgroundProviderTurns.delete(event.providerTurnId);
1971 }
1972 }
1973 yield* trackChildLifecycle(event, deliveredEvent !== null);
1974 yield* scheduleIdleFlush;
1975 });
1976 const providerEventFiber = yield* eventSink
1977 .consumeProviderEvents(
1978 eventSubscription.events.pipe(
1979 Stream.merge(Stream.fromQueue(idleFlushTicks), { haltStrategy: "left" }),
1980 ),
1981 (event) =>
1982 event.type === "idle_flush"
1983 ? flushIdleAssistantText(event.at)
1984 : Effect.gen(function* () {
1985 yield* ingestEvent(yield* routeEvent(event));
1986 }),
1987 Effect.gen(function* () {
1988 if (yield* agentTurnsSettled) yield* endPreviewActivity;
1989 return yield* shouldStopProviderEventIngestion;
1990 }),
1991 Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)),
1992 { threadId: input.run.threadId, delegated: isDelegatedAppThread(input.appThread) },
1993 )
1994 .pipe(
1995 Effect.ensuring(closeIdleFlush),
1996 Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })),
1997 Effect.flatMap(() =>
1998 Effect.gen(function* () {
1999 const terminal = yield* Ref.get(terminalEvent);
2000 if (terminal === null) {
2001 // A session release interrupts or fails the stream; a clean
2002 // end without a terminal means the run would never settle.
2003 return yield* new RunExecutionIngestError({
2004 runId: input.run.id,
2005 cause: new ProviderAdapter.ProviderAdapterEventStreamError({
2006 driver: input.providerThread.driver,
2007 providerSessionId: input.providerSessionId,
2008 raisedByT3: true,
2009 cause: "Provider event stream ended without a root terminal.",
2010 }),
2011 });
2012 }
2013 yield* finalizeRootRun(terminal);
2014 }),
2015 ),
2016 // Server shutdown interrupts the subscription and leaves the run to
2017 // restart recovery; every other failure fails the run.
2018 Effect.catchCauseIf(
2019 (cause) => !Cause.hasInterruptsOnly(cause),
2020 (cause) =>
2021 Effect.gen(function* () {
2022 const completedAt = yield* DateTime.now;
2023 const routing = yield* Ref.get(eventRouting);
2024 const rootProviderTurn = yield* Ref.get(latestRootProviderTurn);
2025 const squashed = Cause.squash(cause);
2026 // An exclusive session closes when its thread detaches, and no terminal
2027 // follows. The process is gone, so a handoff settles here with no interrupt.
2028 if (
2029 isRunExecutionIngestError(squashed) &&
2030 isProviderEventStreamError(squashed.cause) &&
2031 squashed.cause.detachedThreadId === input.run.threadId &&
2032 (yield* workspaceHandedOff)
2033 ) {
2034 const handedOff = yield* finalizeRootRun(
2035 {
2036 type: "turn.terminal",
2037 driver: input.providerThread.driver,
2038 providerThreadId: input.providerThread.id,
2039 providerTurnId: routing.rootProviderTurnId ?? failedProviderTurnId,
2040 runOrdinal: input.run.ordinal,
2041 status: "interrupted",
2042 failure: null,
2043 threadDisposition: "reusable",
2044 },
2045 rootProviderTurn === null ||
2046 isTerminalProviderTurnStatus(rootProviderTurn.status)
2047 ? undefined
2048 : { ...rootProviderTurn, status: "interrupted", completedAt },
2049 ).pipe(
2050 Effect.as(true),
2051 Effect.catchCauseIf(
2052 (settleCause) => !Cause.hasInterruptsOnly(settleCause),
2053 (settleCause) =>
2054 Effect.logWarning(
2055 "orchestration V2 failed to settle a run after its workspace handoff",
2056 { runId: input.run.id, cause: settleCause },
2057 ).pipe(Effect.as(false)),
2058 ),
2059 );
2060 if (handedOff) {
2061 // Its tool calls, the handoff's own among them, died with that process.
2062 yield* endWorkLostAfterRootRun(completedAt);
2063 return;
2064 }
2065 }
2066 yield* Effect.logWarning("orchestration V2 provider event ingestion failed", {
2067 runId: input.run.id,
2068 cause,
2069 });
2070 // An adapter fails its stream when its runtime is gone. A failed write, or a session T3
2071 // released or saw end, leaves work that can still report or that the releaser settles.
2072 const runtimeLost =
2073 isRunExecutionIngestError(squashed) &&
2074 isProviderEventStreamError(squashed.cause) &&
2075 squashed.cause.raisedByT3 !== true;
2076 if (yield* Ref.get(rootRunFinalized)) {
2077 if (runtimeLost) yield* endWorkLostAfterRootRun(completedAt);
2078 return;
2079 }
2080 const providerThread = yield* Ref.get(latestProviderThread);
2081 // Stop background work too; the failed consumer cannot persist terminal events.
2082 const abandonedProviderTurn =
2083 routing.rootProviderTurnId !== null && !routing.rootTurnEnded
2084 ? yield* input.session
2085 .interruptTurn({
2086 providerThread,
2087 providerTurnId: routing.rootProviderTurnId,
2088 requestRuntimeRestart: true,
2089 })
2090 .pipe(
2091 Effect.as(true),
2092 Effect.catchCauseIf(
2093 (interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
2094 (interruptCause) =>
2095 Effect.logWarning(
2096 "orchestration V2 failed to interrupt turn after ingestion failure",
2097 { runId: input.run.id, cause: interruptCause },
2098 ).pipe(Effect.as(false)),
2099 ),
2100 // A failed interrupt leaves the stored turn running so the next start
2101 // retries it. The snapshot is cached before it is published, so it may
2102 // not be stored: replay a terminal one as-is.
2103 Effect.map((interrupted) =>
2104 !interrupted || rootProviderTurn === null
2105 ? null
2106 : isTerminalProviderTurnStatus(rootProviderTurn.status)
2107 ? rootProviderTurn
2108 : {
2109 ...rootProviderTurn,
2110 status: "interrupted" as const,
2111 completedAt,
2112 },
2113 ),
2114 )
2115 : null;
2116 const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
2117 const openSubagents = yield* Ref.get(openRunOwnedSubagents);
2118 const lostToolCalls = runtimeLost
2119 ? yield* loadLostRootToolCalls.pipe(
2120 Effect.catchCauseIf(
2121 (loadCause) => !Cause.hasInterruptsOnly(loadCause),
2122 (loadCause) =>
2123 Effect.logWarning(
2124 "orchestration V2 failed to read tool calls lost with the provider runtime",
2125 { runId: input.run.id, cause: loadCause },
2126 ).pipe(Effect.as([])),
2127 ),
2128 )
2129 : [];
2130 let nextRetryWarningAt = completedAt;
2131 yield* writeFinalRunEvents({
2132 run: input.run,
2133 rootNode: input.rootNode,
2134 checkpointScope: input.checkpointScope,
2135 providerThread,
2136 attempt: input.attempt,
2137 writeIfRunCurrent: {
2138 activeAttemptId: input.attempt.id,
2139 expectedStatus: "running",
2140 },
2141 openRunOwnedSubagents: openSubagents,
2142 ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }),
2143 lostToolCalls,
2144 completedAt,
2145 terminal: makeFailedTerminalEvent(
2146 makeProviderFailure({
2147 cause: squashed,
2148 message:
2149 knownProviderFailureMessage(squashed) ??
2150 userFacingDatabaseErrorMessage(squashed),
2151 class: "unknown",
2152 }),
2153 latestItemOrdinal + 1,
2154 ),
2155 failureItemPersisted: false,
2156 refreshAfterTurn,
2157 }).pipe(
2158 // Storage failures can be defects; retry both failure channels.
2159 Effect.catchCause((writeCause) =>
2160 Effect.gen(function* () {
2161 if (Cause.hasInterruptsOnly(writeCause)) {
2162 return yield* Effect.failCause(writeCause);
2163 }
2164 const now = yield* DateTime.now;
2165 if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) {
2166 yield* Effect.logWarning(
2167 "orchestration V2 run failure finalization retry",
2168 {
2169 runId: input.run.id,
2170 attemptId: input.attempt.id,
2171 cause: writeCause,
2172 },
2173 );
2174 // Persistent storage failures must not flood the same disk.
2175 nextRetryWarningAt = DateTime.add(now, { minutes: 10 });
2176 }
2177 return yield* Effect.fail(writeCause);
2178 }),
2179 ),
2180 Effect.retry({
2181 schedule: Schedule.exponential("1 second").pipe(
2182 Schedule.modifyDelay(({ duration }) =>
2183 Effect.succeed(Duration.min(duration, Duration.seconds(30))),
2184 ),
2185 ),
2186 }),
2187 // Startup may cancel the consumer and fail its own terminal
2188 // write. Only service shutdown or the atomic gate ends recovery.
2189 Effect.forkIn(scope),
2190 Effect.flatMap(Fiber.join),
2191 );
2192 }),
2193 ),
2194 Effect.ensuring(endPreviewActivity),
2195 Effect.ensuring(eventSubscription.close),
2196 Effect.forkDetach,
2197 );
2198
2199 // A provider turn is a sign that the session is still alive. Keep
2200 // its already-issued MCP credential valid even when the agent goes
2201 // a long time between browser-tool calls.
2202 const touchMcpCredential = McpSessionRegistry.touchActiveMcpThread(input.run.threadId).pipe(
2203 Effect.ignoreCause({
2204 log: true,
2205 message: "Failed to refresh MCP credential liveness.",
2206 }),
2207 );
2208 yield* touchMcpCredential;
2209 const appContext = yield* McpAppModelContext.forProvider(
2210 mcpAppModelContext,
2211 input.run.threadId,
2212 );
2213 const turnInput = {
2214 appThread: input.appThread,
2215 threadId: input.run.threadId,
2216 runId: input.run.id,
2217 runOrdinal: input.run.ordinal,
2218 providerTurnOrdinal: input.providerTurnOrdinal,
2219 ...(input.nativeThreadHasTurns === undefined
2220 ? {}
2221 : { nativeThreadHasTurns: input.nativeThreadHasTurns }),
2222 ...(input.run.restartContinuationOfRunId === undefined
2223 ? {}
2224 : {
2225 restartContinuationOfRunId: input.run.restartContinuationOfRunId,
2226 }),
2227 attemptId: input.attemptId,
2228 rootNodeId: input.rootNode.id,
2229 providerThread: input.providerThread,
2230 message: input.message,
2231 modelSelection: input.modelSelection,
2232 runtimePolicy: input.runtimePolicy,
2233 ...(appContext.length === 0 ? {} : { appContext }),
2234 };
2235 // Check after the context read: ingestion can settle the run while
2236 // it awaits. A failed ownership read fails the start below.
2237 const shouldStart =
2238 input.shouldStartProviderTurn === undefined
2239 ? Exit.succeed(true)
2240 : yield* Effect.exit(input.shouldStartProviderTurn());
2241 if (Exit.isSuccess(shouldStart) && !shouldStart.value) {
2242 yield* Fiber.interrupt(providerEventFiber);
2243 return;
2244 }
2245
2246 const compact =
2247 input.message.attachments.length === 0 &&
2248 input.message.text.trim().toLowerCase() === "/compact";
2249 const { message: _message, ...resumeInput } = turnInput;
2250 const startTurn =
2251 input.resume === true
2252 ? (input.session.resumeTurn?.({ ...resumeInput, messageId: input.message.messageId }) ??
2253 Effect.fail(
2254 new ProviderAdapter.ProviderAdapterTurnStartError({
2255 driver: input.session.driver,
2256 threadId: input.run.threadId,
2257 providerThreadId: input.providerThread.id,
2258 runId: input.run.id,
2259 cause: new ProviderAdapter.ProviderResumeUnsupportedError({
2260 driver: input.session.driver,
2261 providerThreadId: input.providerThread.id,
2262 }),
2263 }),
2264 ))
2265 : compact
2266 ? (input.session.compactThread?.(turnInput) ??
2267 Effect.fail(
2268 new ProviderAdapter.ProviderAdapterTurnStartError({
2269 driver: input.session.driver,
2270 threadId: input.run.threadId,
2271 providerThreadId: input.providerThread.id,
2272 runId: input.run.id,
2273 cause: new ProviderAdapter.ProviderCompactionUnsupportedError({
2274 driver: input.session.driver,
2275 providerThreadId: input.providerThread.id,
2276 }),
2277 }),
2278 ))
2279 : input.session.startTurn(turnInput);
2280 yield* Effect.andThen(shouldStart, startTurn).pipe(
2281 // Wait for acceptance so abandoned startup cannot leave a refresher
2282 // on the already-detached ingestion fiber.
2283 Effect.tap(() =>
2284 Fiber.await(providerEventFiber).pipe(
2285 Effect.raceFirst(touchMcpCredential.pipe(Effect.delay("1 hour"), Effect.forever)),
2286 Effect.forkDetach,
2287 ),
2288 ),
2289 Effect.catchCause((cause) =>
2290 Effect.logError("orchestration V2 provider turn start failed", {
2291 runId: input.run.id,
2292 cause,
2293 }).pipe(
2294 Effect.andThen(Fiber.interrupt(providerEventFiber)),
2295 Effect.andThen(Ref.get(latestProviderThread)),
2296 Effect.flatMap((providerThread) =>
2297 Ref.get(latestTurnItemOrdinal).pipe(
2298 Effect.flatMap((latestItemOrdinal) =>
2299 Ref.get(openRunOwnedSubagents).pipe(
2300 Effect.flatMap((openSubagents) =>
2301 writeFinalRunEvents({
2302 run: input.run,
2303 rootNode: input.rootNode,
2304 checkpointScope: input.checkpointScope,
2305 providerThread,
2306 attempt: input.attempt,
2307 writeIfRunCurrent: {
2308 activeAttemptId: input.attempt.id,
2309 expectedStatus: "running",
2310 },
2311 openRunOwnedSubagents: openSubagents,
2312 terminal: makeFailedTerminalEvent(
2313 makeProviderFailure({
2314 cause: Cause.squash(cause),
2315 class: Exit.isFailure(shouldStart) ? "unknown" : "provider_error",
2316 }),
2317 latestItemOrdinal + 1,
2318 ),
2319 failureItemPersisted: false,
2320 refreshAfterTurn,
2321 }),
2322 ),
2323 ),
2324 ),
2325 ),
2326 ),
2327 Effect.mapError(
2328 (writeCause) =>
2329 new RunExecutionStartError({
2330 commandId: input.commandId,
2331 runId: input.run.id,
2332 cause: { start: cause, write: writeCause },
2333 }),
2334 ),
2335 ),
2336 ),
2337 );
2338 }),
2339 } satisfies RunExecutionServiceV2Shape);
2340 }),
2341 );
2342
2343 /**
2344 * Clients send Stop without a reason; server callers acting for an agent
2345 * always set one, so a missing reason means a person stopped the run.
2346 */
2347 export const USER_STOP_REASON = "Stopped by user";
2348
2349 export function makeInterruptResultTurnItem(input: {
2350 readonly idAllocator: IdAllocator.IdAllocatorV2["Service"];
2351 readonly run: OrchestrationV2Run;
2352 readonly rootNode: OrchestrationV2ExecutionNode;
2353 readonly providerThread: OrchestrationV2ProviderThread;
2354 readonly reason: string | undefined;
2355 readonly completedAt: DateTime.Utc;
2356 }): OrchestrationV2TurnItem {
2357 return {
2358 id: input.idAllocator.derive.runSignalTurnItem({
2359 runId: input.run.id,
2360 signal: "interrupt-result",
2361 }),
2362 threadId: input.run.threadId,
2363 runId: input.run.id,
2364 nodeId: input.rootNode.id,
2365 providerThreadId: input.providerThread.id,
2366 providerTurnId: input.rootNode.providerTurnId,
2367 nativeItemRef: null,
2368 parentItemId: input.idAllocator.derive.runSignalTurnItem({
2369 runId: input.run.id,
2370 signal: "interrupt-request",
2371 }),
2372 ordinal: input.run.ordinal * 100 + 98,
2373 status: "interrupted",
2374 title: "Interrupted",
2375 startedAt: input.completedAt,
2376 completedAt: input.completedAt,
2377 updatedAt: input.completedAt,
2378 type: "run_interrupt_result",
2379 message: input.reason ?? "Stopped by the provider",
2380 };
2381 }