The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { makeAssistantStreamingFilter } from "./assistantStreaming.ts"; |
| 2 | import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; |
| 3 | import { |
| 4 | isDelegatedAppThread, |
| 5 | isOrchestrationV2WorkActive, |
| 6 | CommandId, |
| 7 | type EventId, |
| 8 | type ModelSelection, |
| 9 | type NodeId, |
| 10 | type OrchestrationV2AppThread, |
| 11 | type OrchestrationV2CheckpointScope, |
| 12 | type OrchestrationV2DomainEvent, |
| 13 | type OrchestrationV2ExecutionNode, |
| 14 | type OrchestrationV2ProviderFailure, |
| 15 | type OrchestrationV2ProviderThread, |
| 16 | type OrchestrationV2ProviderTurn, |
| 17 | type OrchestrationV2Run, |
| 18 | type OrchestrationV2RunAttempt, |
| 19 | type OrchestrationV2Subagent, |
| 20 | type OrchestrationV2TurnItem, |
| 21 | type ProviderSessionId, |
| 22 | type ProviderThreadId, |
| 23 | type ProviderTurnId, |
| 24 | type RunAttemptId, |
| 25 | type ThreadId, |
| 26 | type TurnItemId, |
| 27 | } from "@t3tools/contracts"; |
| 28 | import * as Context from "effect/Context"; |
| 29 | import * as Cause from "effect/Cause"; |
| 30 | import * as DateTime from "effect/DateTime"; |
| 31 | import * as Duration from "effect/Duration"; |
| 32 | import * as Effect from "effect/Effect"; |
| 33 | import * as Exit from "effect/Exit"; |
| 34 | import * as Fiber from "effect/Fiber"; |
| 35 | import * as Layer from "effect/Layer"; |
| 36 | import * as Queue from "effect/Queue"; |
| 37 | import * as Ref from "effect/Ref"; |
| 38 | import * as Schedule from "effect/Schedule"; |
| 39 | import * as Schema from "effect/Schema"; |
| 40 | import * as Stream from "effect/Stream"; |
| 41 | |
| 42 | import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; |
| 43 | import * as McpAppModelContext from "../mcpApps/McpAppModelContext.ts"; |
| 44 | import * as PreviewAgentActivity from "../preview/PreviewAgentActivity.ts"; |
| 45 | import * as ServerSettings from "../serverSettings.ts"; |
| 46 | import * as CheckpointService from "./CheckpointService.ts"; |
| 47 | import * as EventSink from "./EventSink.ts"; |
| 48 | import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; |
| 49 | import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; |
| 50 | import { PROVIDER_STOP_REASON_SIGNAL } from "@t3tools/provider-core/server/subagentProjection"; |
| 51 | import * as ProviderEventIngestor from "./ProviderEventIngestor.ts"; |
| 52 | import * as ProviderUsageAnalytics from "../telemetry/ProviderUsageAnalytics.ts"; |
| 53 | import type { ProjectionStoreV2Error } from "./ProjectionStore.ts"; |
| 54 | import { |
| 55 | knownProviderFailureMessage, |
| 56 | makeProviderFailure, |
| 57 | makeProviderFailureTurnItem, |
| 58 | } from "@t3tools/provider-core/server/failure"; |
| 59 | import * as RunFinalizationService from "./RunFinalizationService.ts"; |
| 60 | import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; |
| 61 | import * as TurnItemPositionStore from "./TurnItemPositionStore.ts"; |
| 62 | import { userFacingDatabaseErrorMessage } from "./UserFacingErrors.ts"; |
| 63 | |
| 64 | export interface ProviderEventRoutingState { |
| 65 | readonly ownedThreadIds: ReadonlySet<ThreadId>; |
| 66 | // After root settlement, new children belong to the live run unless their coordinator still holds this run open. |
| 67 | readonly rootTurnEnded: boolean; |
| 68 | readonly workflowMemberParents: ReadonlyMap<ThreadId, ThreadId>; |
| 69 | readonly ownedProviderThreadIds: ReadonlySet<ProviderThreadId>; |
| 70 | readonly ownedProviderTurns: ReadonlyMap<ProviderTurnId, ProviderThreadId>; |
| 71 | readonly inheritedBackgroundTurnItems: ReadonlyMap<TurnItemId, OrchestrationV2Run["id"]>; |
| 72 | readonly rootProviderTurnId: ProviderTurnId | null; |
| 73 | } |
| 74 | |
| 75 | export interface ProviderEventRouteIdentity { |
| 76 | readonly threadId: ThreadId; |
| 77 | readonly runId: OrchestrationV2Run["id"]; |
| 78 | readonly attemptId: RunAttemptId; |
| 79 | readonly providerThreadId: ProviderThreadId; |
| 80 | } |
| 81 | |
| 82 | export interface InheritedBackgroundTurnItemRoute { |
| 83 | readonly id: TurnItemId; |
| 84 | readonly runId: OrchestrationV2Run["id"]; |
| 85 | } |
| 86 | |
| 87 | type ProviderTerminalEvent = Extract< |
| 88 | ProviderAdapter.ProviderAdapterV2Event, |
| 89 | { readonly type: "turn.terminal" } |
| 90 | >; |
| 91 | |
| 92 | function isTerminalProviderTurnStatus(status: OrchestrationV2ProviderTurn["status"]): boolean { |
| 93 | return ( |
| 94 | status === "completed" || |
| 95 | status === "interrupted" || |
| 96 | status === "failed" || |
| 97 | status === "cancelled" |
| 98 | ); |
| 99 | } |
| 100 | |
| 101 | function isSettledSubagentStatus(status: OrchestrationV2Subagent["status"]): boolean { |
| 102 | return !isOrchestrationV2WorkActive(status); |
| 103 | } |
| 104 | |
| 105 | /** A provider-native subagent's turns arrive on its parent's run, but the parent agent started them. */ |
| 106 | const NATIVE_SUBAGENT_TURN_ORIGIN: ProviderUsageAnalytics.ProviderTurnOrigin = { |
| 107 | createdBy: "agent", |
| 108 | creationSource: "provider", |
| 109 | scheduled: false, |
| 110 | subagent: true, |
| 111 | }; |
| 112 | |
| 113 | const backgroundCapableTurnItemTypes: ReadonlySet<OrchestrationV2TurnItem["type"]> = new Set([ |
| 114 | "command_execution", |
| 115 | "dynamic_tool", |
| 116 | "subagent", |
| 117 | ]); |
| 118 | |
| 119 | function isSettledTurnItemStatus(status: OrchestrationV2TurnItem["status"]): boolean { |
| 120 | return !isOrchestrationV2WorkActive(status); |
| 121 | } |
| 122 | |
| 123 | function isSettledRunEligibleForInheritedBackground(status: OrchestrationV2Run["status"]): boolean { |
| 124 | return status === "interrupted" || status === "failed" || status === "cancelled"; |
| 125 | } |
| 126 | |
| 127 | /** Use durable provider-thread lineage; sessions can host multiple threads. */ |
| 128 | export function selectInheritedBackgroundTurnItems(input: { |
| 129 | readonly threadId: ThreadId; |
| 130 | readonly currentProviderThreadId: ProviderThreadId; |
| 131 | readonly currentRunOrdinal: number; |
| 132 | readonly runs: ReadonlyArray<OrchestrationV2Run>; |
| 133 | readonly turnItems: ReadonlyArray<OrchestrationV2TurnItem>; |
| 134 | }): ReadonlyArray<InheritedBackgroundTurnItemRoute> { |
| 135 | const settledPriorRunIds = new Set( |
| 136 | input.runs |
| 137 | .filter( |
| 138 | (run) => |
| 139 | run.threadId === input.threadId && |
| 140 | run.ordinal < input.currentRunOrdinal && |
| 141 | isSettledRunEligibleForInheritedBackground(run.status), |
| 142 | ) |
| 143 | .map((run) => run.id), |
| 144 | ); |
| 145 | return input.turnItems.flatMap((turnItem) => |
| 146 | turnItem.threadId === input.threadId && |
| 147 | turnItem.providerThreadId === input.currentProviderThreadId && |
| 148 | turnItem.runId !== null && |
| 149 | settledPriorRunIds.has(turnItem.runId) && |
| 150 | backgroundCapableTurnItemTypes.has(turnItem.type) && |
| 151 | !isSettledTurnItemStatus(turnItem.status) |
| 152 | ? [{ id: turnItem.id, runId: turnItem.runId }] |
| 153 | : [], |
| 154 | ); |
| 155 | } |
| 156 | |
| 157 | type SubagentTurnItem = Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>; |
| 158 | |
| 159 | // Tool calls that only a live provider process can end. |
| 160 | const processBoundToolCallTypes: ReadonlySet<OrchestrationV2TurnItem["type"]> = new Set([ |
| 161 | "command_execution", |
| 162 | "dynamic_tool", |
| 163 | ]); |
| 164 | |
| 165 | type OpenRunOwnedSubagentProjection = { |
| 166 | readonly subagents: ReadonlyMap<NodeId, OrchestrationV2Subagent>; |
| 167 | readonly turnItems: ReadonlyMap<NodeId, SubagentTurnItem>; |
| 168 | readonly childTurnItems: ReadonlyMap<TurnItemId, OrchestrationV2TurnItem>; |
| 169 | readonly nodes: ReadonlyMap<NodeId, OrchestrationV2ExecutionNode>; |
| 170 | readonly linkedChildThreadIds: ReadonlySet<ThreadId>; |
| 171 | }; |
| 172 | |
| 173 | export type UnsettledWork = { |
| 174 | readonly subagents: ReadonlyArray<OrchestrationV2Subagent>; |
| 175 | readonly nodes: ReadonlyArray<OrchestrationV2ExecutionNode>; |
| 176 | readonly turnItems: ReadonlyArray<OrchestrationV2TurnItem>; |
| 177 | }; |
| 178 | |
| 179 | type RunOwnedSubagentTerminalStatus = Extract< |
| 180 | OrchestrationV2Subagent["status"], |
| 181 | "interrupted" | "failed" | "cancelled" |
| 182 | >; |
| 183 | |
| 184 | function isOpenExecutionNodeStatus(status: OrchestrationV2ExecutionNode["status"]): boolean { |
| 185 | return status === "pending" || status === "running" || status === "waiting"; |
| 186 | } |
| 187 | |
| 188 | function isRunOwnedSubagentTerminalStatus( |
| 189 | status: ProviderTerminalEvent["status"], |
| 190 | ): status is RunOwnedSubagentTerminalStatus { |
| 191 | return status === "interrupted" || status === "failed" || status === "cancelled"; |
| 192 | } |
| 193 | |
| 194 | function streamedThreadId(event: ProviderAdapter.ProviderAdapterV2Event): ThreadId | null { |
| 195 | if (event.type === "message.updated") return event.message.threadId; |
| 196 | if (event.type === "turn_item.updated") return event.turnItem.threadId; |
| 197 | return null; |
| 198 | } |
| 199 | |
| 200 | function childTurnsContinue(terminal: ProviderTerminalEvent): boolean { |
| 201 | return terminal.status === "failed" && terminal.childTurnsContinue === true; |
| 202 | } |
| 203 | |
| 204 | /** Launching runs still ingest running subagents; taking them over would duplicate events. */ |
| 205 | export function canRouteRelatedSubagent(status: OrchestrationV2Subagent["status"]): boolean { |
| 206 | return status === "completed"; |
| 207 | } |
| 208 | |
| 209 | function emptyOpenRunOwnedSubagentProjection(): OpenRunOwnedSubagentProjection { |
| 210 | return { |
| 211 | subagents: new Map(), |
| 212 | turnItems: new Map(), |
| 213 | childTurnItems: new Map(), |
| 214 | nodes: new Map(), |
| 215 | linkedChildThreadIds: new Set(), |
| 216 | }; |
| 217 | } |
| 218 | |
| 219 | function withLinkedChildThreadId( |
| 220 | current: OpenRunOwnedSubagentProjection, |
| 221 | childThreadId: ThreadId | null, |
| 222 | ): OpenRunOwnedSubagentProjection { |
| 223 | if (childThreadId === null || current.linkedChildThreadIds.has(childThreadId)) { |
| 224 | return current; |
| 225 | } |
| 226 | const linkedChildThreadIds = new Set(current.linkedChildThreadIds); |
| 227 | linkedChildThreadIds.add(childThreadId); |
| 228 | return { ...current, linkedChildThreadIds }; |
| 229 | } |
| 230 | |
| 231 | export const cascadeTerminalizeRunOwnedSubagents = Effect.fn( |
| 232 | "RunExecutionService.cascadeTerminalizeRunOwnedSubagents", |
| 233 | )(function* (input: { |
| 234 | readonly run: OrchestrationV2Run; |
| 235 | readonly open: OpenRunOwnedSubagentProjection; |
| 236 | readonly status: RunOwnedSubagentTerminalStatus; |
| 237 | readonly completedAt: DateTime.Utc; |
| 238 | readonly allocateEventId: () => Effect.Effect<EventId, IdAllocator.IdAllocatorV2AllocationError>; |
| 239 | }): Effect.fn.Return< |
| 240 | ReadonlyArray<OrchestrationV2DomainEvent>, |
| 241 | IdAllocator.IdAllocatorV2AllocationError |
| 242 | > { |
| 243 | const events: Array<OrchestrationV2DomainEvent> = []; |
| 244 | // Prefer lifetime linkage over currently-open rows: subagent/turn-item |
| 245 | // snapshots may terminalize before the linked child-thread node settles. |
| 246 | const childThreadIds = new Set(input.open.linkedChildThreadIds); |
| 247 | for (const item of [...input.open.subagents.values(), ...input.open.turnItems.values()]) { |
| 248 | if (item.childThreadId !== null) { |
| 249 | childThreadIds.add(item.childThreadId); |
| 250 | } |
| 251 | } |
| 252 | const keys = new Set<NodeId>([ |
| 253 | ...input.open.subagents.keys(), |
| 254 | ...input.open.turnItems.keys(), |
| 255 | ...input.open.nodes.keys(), |
| 256 | ]); |
| 257 | for (const key of keys) { |
| 258 | const subagent = input.open.subagents.get(key); |
| 259 | if (subagent !== undefined && !isSettledSubagentStatus(subagent.status)) { |
| 260 | events.push({ |
| 261 | id: yield* input.allocateEventId(), |
| 262 | type: "subagent.updated", |
| 263 | threadId: subagent.threadId, |
| 264 | runId: input.run.id, |
| 265 | nodeId: subagent.id, |
| 266 | driver: subagent.driver, |
| 267 | providerInstanceId: subagent.providerInstanceId, |
| 268 | occurredAt: input.completedAt, |
| 269 | payload: { |
| 270 | ...subagent, |
| 271 | status: input.status, |
| 272 | completedAt: input.completedAt, |
| 273 | updatedAt: input.completedAt, |
| 274 | }, |
| 275 | }); |
| 276 | } |
| 277 | const node = input.open.nodes.get(key); |
| 278 | if ( |
| 279 | node !== undefined && |
| 280 | ((node.threadId === input.run.threadId && node.runId === input.run.id) || |
| 281 | childThreadIds.has(node.threadId)) && |
| 282 | isOpenExecutionNodeStatus(node.status) |
| 283 | ) { |
| 284 | events.push({ |
| 285 | id: yield* input.allocateEventId(), |
| 286 | type: "node.updated", |
| 287 | threadId: node.threadId, |
| 288 | runId: node.runId ?? input.run.id, |
| 289 | nodeId: node.id, |
| 290 | providerInstanceId: input.run.providerInstanceId, |
| 291 | occurredAt: input.completedAt, |
| 292 | payload: { |
| 293 | ...node, |
| 294 | status: input.status, |
| 295 | completedAt: input.completedAt, |
| 296 | }, |
| 297 | }); |
| 298 | } |
| 299 | const turnItem = input.open.turnItems.get(key); |
| 300 | if ( |
| 301 | turnItem !== undefined && |
| 302 | turnItem.runId === input.run.id && |
| 303 | !isSettledTurnItemStatus(turnItem.status) |
| 304 | ) { |
| 305 | events.push({ |
| 306 | id: yield* input.allocateEventId(), |
| 307 | type: "turn-item.updated", |
| 308 | threadId: turnItem.threadId, |
| 309 | runId: input.run.id, |
| 310 | ...(turnItem.nodeId === null ? {} : { nodeId: turnItem.nodeId }), |
| 311 | providerInstanceId: input.run.providerInstanceId, |
| 312 | occurredAt: input.completedAt, |
| 313 | payload: { |
| 314 | ...turnItem, |
| 315 | status: input.status, |
| 316 | completedAt: input.completedAt, |
| 317 | updatedAt: input.completedAt, |
| 318 | }, |
| 319 | }); |
| 320 | } |
| 321 | } |
| 322 | for (const turnItem of input.open.childTurnItems.values()) { |
| 323 | if (!childThreadIds.has(turnItem.threadId) || isSettledTurnItemStatus(turnItem.status)) { |
| 324 | continue; |
| 325 | } |
| 326 | events.push({ |
| 327 | id: yield* input.allocateEventId(), |
| 328 | type: "turn-item.updated", |
| 329 | threadId: turnItem.threadId, |
| 330 | runId: turnItem.runId ?? input.run.id, |
| 331 | ...(turnItem.nodeId === null ? {} : { nodeId: turnItem.nodeId }), |
| 332 | providerInstanceId: input.run.providerInstanceId, |
| 333 | occurredAt: input.completedAt, |
| 334 | payload: { |
| 335 | ...turnItem, |
| 336 | ...("streaming" in turnItem ? { streaming: false } : {}), |
| 337 | status: input.status, |
| 338 | completedAt: input.completedAt, |
| 339 | updatedAt: input.completedAt, |
| 340 | }, |
| 341 | }); |
| 342 | } |
| 343 | return events; |
| 344 | }); |
| 345 | |
| 346 | const interruptLostToolCalls = Effect.fn("RunExecutionService.interruptLostToolCalls")( |
| 347 | function* (input: { |
| 348 | readonly run: OrchestrationV2Run; |
| 349 | readonly toolCalls: Iterable<OrchestrationV2TurnItem>; |
| 350 | readonly completedAt: DateTime.Utc; |
| 351 | readonly allocateEventId: () => Effect.Effect< |
| 352 | EventId, |
| 353 | IdAllocator.IdAllocatorV2AllocationError |
| 354 | >; |
| 355 | }) { |
| 356 | const events: Array<OrchestrationV2DomainEvent> = []; |
| 357 | for (const toolCall of input.toolCalls) { |
| 358 | events.push({ |
| 359 | id: yield* input.allocateEventId(), |
| 360 | type: "turn-item.updated", |
| 361 | threadId: toolCall.threadId, |
| 362 | runId: toolCall.runId ?? input.run.id, |
| 363 | ...(toolCall.nodeId === null ? {} : { nodeId: toolCall.nodeId }), |
| 364 | providerInstanceId: input.run.providerInstanceId, |
| 365 | occurredAt: input.completedAt, |
| 366 | payload: { |
| 367 | ...toolCall, |
| 368 | status: "interrupted", |
| 369 | completedAt: input.completedAt, |
| 370 | updatedAt: input.completedAt, |
| 371 | }, |
| 372 | }); |
| 373 | } |
| 374 | return events; |
| 375 | }, |
| 376 | ); |
| 377 | |
| 378 | function finalProviderThreadStatus( |
| 379 | disposition: ProviderTerminalEvent["threadDisposition"], |
| 380 | ): OrchestrationV2ProviderThread["status"] { |
| 381 | return disposition === "broken" ? "error" : "idle"; |
| 382 | } |
| 383 | |
| 384 | export function makeProviderEventRoutingState(input: { |
| 385 | readonly identity: ProviderEventRouteIdentity; |
| 386 | readonly inheritedBackgroundTurnItems?: ReadonlyArray<InheritedBackgroundTurnItemRoute>; |
| 387 | readonly providerTurnId: ProviderTurnId | null; |
| 388 | readonly relatedThreadIds?: ReadonlyArray<ThreadId>; |
| 389 | readonly relatedProviderThreadIds?: ReadonlyArray<ProviderThreadId>; |
| 390 | }): ProviderEventRoutingState { |
| 391 | return { |
| 392 | ownedThreadIds: new Set([input.identity.threadId, ...(input.relatedThreadIds ?? [])]), |
| 393 | rootTurnEnded: false, |
| 394 | workflowMemberParents: new Map(), |
| 395 | ownedProviderThreadIds: new Set([ |
| 396 | input.identity.providerThreadId, |
| 397 | ...(input.relatedProviderThreadIds ?? []), |
| 398 | ]), |
| 399 | ownedProviderTurns: |
| 400 | input.providerTurnId === null |
| 401 | ? new Map() |
| 402 | : new Map([[input.providerTurnId, input.identity.providerThreadId]]), |
| 403 | inheritedBackgroundTurnItems: new Map( |
| 404 | (input.inheritedBackgroundTurnItems ?? []).map((item) => [item.id, item.runId]), |
| 405 | ), |
| 406 | rootProviderTurnId: input.providerTurnId, |
| 407 | }; |
| 408 | } |
| 409 | |
| 410 | export function routeProviderEvent( |
| 411 | event: ProviderAdapter.ProviderAdapterV2Event, |
| 412 | input: ProviderEventRouteIdentity, |
| 413 | state: ProviderEventRoutingState, |
| 414 | ): readonly [boolean, ProviderEventRoutingState] { |
| 415 | const ownsThread = (threadId: ThreadId): boolean => state.ownedThreadIds.has(threadId); |
| 416 | const ownsChildThread = (threadId: ThreadId): boolean => |
| 417 | threadId !== input.threadId && ownsThread(threadId); |
| 418 | const ownsRun = (runId: string | null): boolean => runId === input.runId; |
| 419 | // ACP plan ids are session-scoped, so a turn can update a plan an earlier turn of this |
| 420 | // thread reported. Only an active root turn adopts those rows, never a lingering run. |
| 421 | const adoptsEarlierPlan = (threadId: ThreadId): boolean => |
| 422 | threadId === input.threadId && !state.rootTurnEnded; |
| 423 | const addProviderThread = (providerThreadId: ProviderThreadId): ProviderEventRoutingState => ({ |
| 424 | ...state, |
| 425 | ownedProviderThreadIds: new Set([...state.ownedProviderThreadIds, providerThreadId]), |
| 426 | }); |
| 427 | const addProviderTurn = ( |
| 428 | providerTurnId: ProviderTurnId, |
| 429 | providerThreadId: ProviderThreadId, |
| 430 | root: boolean, |
| 431 | ): ProviderEventRoutingState => ({ |
| 432 | ...state, |
| 433 | ownedProviderTurns: new Map(state.ownedProviderTurns).set(providerTurnId, providerThreadId), |
| 434 | rootProviderTurnId: root ? providerTurnId : state.rootProviderTurnId, |
| 435 | }); |
| 436 | |
| 437 | switch (event.type) { |
| 438 | case "provider_session.updated": |
| 439 | return [false, state]; |
| 440 | case "app_thread.created": { |
| 441 | if (event.appThread.id === input.threadId) { |
| 442 | return [true, state]; |
| 443 | } |
| 444 | const isOwnedSubagent = |
| 445 | event.appThread.lineage.relationshipToParent === "subagent" && |
| 446 | event.appThread.lineage.parentThreadId !== null && |
| 447 | ownsThread(event.appThread.lineage.parentThreadId) && |
| 448 | (!state.rootTurnEnded || |
| 449 | state.workflowMemberParents.get(event.appThread.id) === |
| 450 | event.appThread.lineage.parentThreadId); |
| 451 | if (!isOwnedSubagent) { |
| 452 | return [false, state]; |
| 453 | } |
| 454 | return [ |
| 455 | true, |
| 456 | { |
| 457 | ...state, |
| 458 | ownedThreadIds: new Set([...state.ownedThreadIds, event.appThread.id]), |
| 459 | }, |
| 460 | ]; |
| 461 | } |
| 462 | case "provider_thread.updated": { |
| 463 | const belongs = |
| 464 | state.ownedProviderThreadIds.has(event.providerThread.id) || |
| 465 | (event.providerThread.appThreadId !== null && ownsThread(event.providerThread.appThreadId)); |
| 466 | return belongs ? [true, addProviderThread(event.providerThread.id)] : [false, state]; |
| 467 | } |
| 468 | case "provider_turn.updated": { |
| 469 | const isRoot = event.providerTurn.runAttemptId === input.attemptId; |
| 470 | const belongs = |
| 471 | isRoot || |
| 472 | (event.providerTurn.providerThreadId !== input.providerThreadId && |
| 473 | state.ownedProviderThreadIds.has(event.providerTurn.providerThreadId)) || |
| 474 | state.ownedProviderTurns.has(event.providerTurn.id) || |
| 475 | (event.threadId !== undefined && ownsChildThread(event.threadId)); |
| 476 | return belongs |
| 477 | ? [ |
| 478 | true, |
| 479 | addProviderTurn(event.providerTurn.id, event.providerTurn.providerThreadId, isRoot), |
| 480 | ] |
| 481 | : [false, state]; |
| 482 | } |
| 483 | case "node.updated": { |
| 484 | const belongs = |
| 485 | ownsRun(event.node.runId) || |
| 486 | ownsChildThread(event.node.threadId) || |
| 487 | ((event.node.kind === "todo_list" || event.node.kind === "plan") && |
| 488 | adoptsEarlierPlan(event.node.threadId)); |
| 489 | if (!belongs || event.node.providerThreadId === null) { |
| 490 | return [belongs, state]; |
| 491 | } |
| 492 | return [true, addProviderThread(event.node.providerThreadId)]; |
| 493 | } |
| 494 | case "subagent.updated": { |
| 495 | const moved = event.subagent; |
| 496 | if ( |
| 497 | moved.origin === "provider_native" && |
| 498 | moved.threadId === input.threadId && |
| 499 | moved.runId !== null && |
| 500 | !ownsRun(moved.runId) && |
| 501 | moved.childThreadId !== null && |
| 502 | ownsChildThread(moved.childThreadId) |
| 503 | ) { |
| 504 | const ownedThreadIds = new Set(state.ownedThreadIds); |
| 505 | ownedThreadIds.delete(moved.childThreadId); |
| 506 | const ownedProviderThreadIds = new Set(state.ownedProviderThreadIds); |
| 507 | if (moved.providerThreadId !== null) ownedProviderThreadIds.delete(moved.providerThreadId); |
| 508 | const ownedProviderTurns = new Map(state.ownedProviderTurns); |
| 509 | for (const [turnId, threadId] of ownedProviderTurns) { |
| 510 | if (threadId === moved.providerThreadId) ownedProviderTurns.delete(turnId); |
| 511 | } |
| 512 | return [false, { ...state, ownedThreadIds, ownedProviderThreadIds, ownedProviderTurns }]; |
| 513 | } |
| 514 | const belongs = ownsRun(event.subagent.runId) || ownsChildThread(event.subagent.threadId); |
| 515 | const coordinator = event.subagent; |
| 516 | // Resumes reattribute child threads to the live run, which may not have observed their creation. |
| 517 | const adopted = |
| 518 | ownsRun(coordinator.runId) && |
| 519 | !state.rootTurnEnded && |
| 520 | isOrchestrationV2WorkActive(coordinator.status) && |
| 521 | coordinator.origin === "provider_native" && |
| 522 | coordinator.childThreadId !== null && |
| 523 | !ownsThread(coordinator.childThreadId) |
| 524 | ? { |
| 525 | ...state, |
| 526 | ownedThreadIds: new Set([...state.ownedThreadIds, coordinator.childThreadId]), |
| 527 | } |
| 528 | : state; |
| 529 | if ( |
| 530 | !ownsRun(coordinator.runId) || |
| 531 | coordinator.workflow === undefined || |
| 532 | coordinator.childThreadId === null |
| 533 | ) { |
| 534 | return [belongs, adopted]; |
| 535 | } |
| 536 | const workflowMemberParents = new Map(state.workflowMemberParents); |
| 537 | for (const member of coordinator.workflow.agents) { |
| 538 | if (member.childThreadId !== undefined) { |
| 539 | workflowMemberParents.set(member.childThreadId, coordinator.childThreadId); |
| 540 | } |
| 541 | } |
| 542 | return [true, { ...adopted, workflowMemberParents }]; |
| 543 | } |
| 544 | case "message.updated": |
| 545 | return [ownsRun(event.message.runId) || ownsChildThread(event.message.threadId), state]; |
| 546 | case "steer.updated": |
| 547 | return [ownsRun(event.runId), state]; |
| 548 | case "turn_item.updated": { |
| 549 | if ( |
| 550 | ownsRun(event.turnItem.runId) || |
| 551 | ownsChildThread(event.turnItem.threadId) || |
| 552 | ((event.turnItem.type === "todo_list" || event.turnItem.type === "proposed_plan") && |
| 553 | adoptsEarlierPlan(event.turnItem.threadId)) |
| 554 | ) { |
| 555 | return [true, state]; |
| 556 | } |
| 557 | const inheritedRunId = state.inheritedBackgroundTurnItems.get(event.turnItem.id); |
| 558 | const isInheritedBackgroundItem = |
| 559 | event.turnItem.threadId === input.threadId && |
| 560 | event.turnItem.runId !== null && |
| 561 | event.turnItem.runId === inheritedRunId && |
| 562 | backgroundCapableTurnItemTypes.has(event.turnItem.type); |
| 563 | if (!isInheritedBackgroundItem) { |
| 564 | return [false, state]; |
| 565 | } |
| 566 | if (!isSettledTurnItemStatus(event.turnItem.status)) { |
| 567 | return [true, state]; |
| 568 | } |
| 569 | const inheritedBackgroundTurnItems = new Map(state.inheritedBackgroundTurnItems); |
| 570 | inheritedBackgroundTurnItems.delete(event.turnItem.id); |
| 571 | return [true, { ...state, inheritedBackgroundTurnItems }]; |
| 572 | } |
| 573 | case "plan.updated": |
| 574 | return [ |
| 575 | ownsRun(event.plan.runId) || |
| 576 | ownsChildThread(event.plan.threadId) || |
| 577 | adoptsEarlierPlan(event.plan.threadId), |
| 578 | state, |
| 579 | ]; |
| 580 | case "runtime_request.updated": |
| 581 | return [ |
| 582 | (event.threadId !== undefined && ownsChildThread(event.threadId)) || |
| 583 | (event.runtimeRequest.providerTurnId !== null && |
| 584 | state.ownedProviderTurns.has(event.runtimeRequest.providerTurnId)), |
| 585 | state, |
| 586 | ]; |
| 587 | case "background_work.updated": |
| 588 | return [state.ownedProviderTurns.has(event.providerTurnId), state]; |
| 589 | case "command_output.delta": |
| 590 | case "prompt_suggestion.updated": |
| 591 | // The session manager serves these to clients directly; runs never see them. |
| 592 | return [false, state]; |
| 593 | case "turn.terminal": |
| 594 | return event.providerTurnId === state.rootProviderTurnId |
| 595 | ? [true, { ...state, rootTurnEnded: true }] |
| 596 | : [false, state]; |
| 597 | } |
| 598 | } |
| 599 | |
| 600 | export class RunExecutionStartError extends Schema.TaggedError<RunExecutionStartError>()( |
| 601 | "RunExecutionStartError", |
| 602 | { |
| 603 | commandId: CommandId, |
| 604 | runId: Schema.String, |
| 605 | cause: Schema.optional(Schema.Defect()), |
| 606 | }, |
| 607 | ) { |
| 608 | override get message(): string { |
| 609 | return `Failed to start orchestration V2 run execution ${this.runId}.`; |
| 610 | } |
| 611 | } |
| 612 | |
| 613 | const isProviderEventStreamError = Schema.is(ProviderAdapter.ProviderAdapterEventStreamError); |
| 614 | |
| 615 | export class RunExecutionIngestError extends Schema.TaggedError<RunExecutionIngestError>()( |
| 616 | "RunExecutionIngestError", |
| 617 | { |
| 618 | runId: Schema.String, |
| 619 | cause: Schema.optional(Schema.Defect()), |
| 620 | }, |
| 621 | ) { |
| 622 | override get message(): string { |
| 623 | return `Failed while ingesting orchestration V2 run execution ${this.runId}.`; |
| 624 | } |
| 625 | } |
| 626 | const isRunExecutionIngestError = Schema.is(RunExecutionIngestError); |
| 627 | |
| 628 | export const RunExecutionServiceV2Error = Schema.Union([ |
| 629 | RunExecutionStartError, |
| 630 | RunExecutionIngestError, |
| 631 | ]); |
| 632 | export type RunExecutionServiceV2Error = typeof RunExecutionServiceV2Error.Type; |
| 633 | |
| 634 | export interface RunExecutionServiceV2StartRootRunInput { |
| 635 | readonly commandId: CommandId; |
| 636 | readonly appThread: OrchestrationV2AppThread; |
| 637 | readonly providerSessionId: ProviderSessionId; |
| 638 | readonly session: ProviderAdapter.ProviderAdapterV2SessionRuntime; |
| 639 | readonly run: OrchestrationV2Run; |
| 640 | readonly rootNode: OrchestrationV2ExecutionNode; |
| 641 | readonly checkpointScope: OrchestrationV2CheckpointScope; |
| 642 | readonly providerThread: OrchestrationV2ProviderThread; |
| 643 | readonly attempt: OrchestrationV2RunAttempt; |
| 644 | readonly attemptId: RunAttemptId; |
| 645 | /** Waits for a baseline started with `forkBaseline`; without it the run captures one itself. */ |
| 646 | readonly baseline?: Effect.Effect<void>; |
| 647 | readonly providerTurnOrdinal: number; |
| 648 | readonly nativeThreadHasTurns?: boolean; |
| 649 | readonly resume?: boolean; |
| 650 | readonly loadInheritedBackgroundTurnItems?: () => Effect.Effect< |
| 651 | ReadonlyArray<InheritedBackgroundTurnItemRoute>, |
| 652 | unknown |
| 653 | >; |
| 654 | readonly relatedThreadIds?: ReadonlyArray<ThreadId>; |
| 655 | readonly relatedProviderThreadIds?: ReadonlyArray<ProviderThreadId>; |
| 656 | readonly shouldStartProviderTurn?: () => Effect.Effect<boolean, ProjectionStoreV2Error>; |
| 657 | readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>; |
| 658 | readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>; |
| 659 | readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>; |
| 660 | readonly hasWorkspaceMoved?: () => Effect.Effect<boolean, never>; |
| 661 | /** |
| 662 | * Stored rows of a thread's unsettled work. Work a dead runtime left is ended from these, since |
| 663 | * Stop may have settled it after the snapshots this run holds. |
| 664 | */ |
| 665 | readonly loadUnsettledWork?: ( |
| 666 | threadId: ThreadId, |
| 667 | nodeIds: ReadonlyArray<NodeId>, |
| 668 | ) => Effect.Effect<UnsettledWork, ProjectionStoreV2Error>; |
| 669 | readonly message: ProviderAdapter.ProviderAdapterV2TurnMessage; |
| 670 | readonly modelSelection: ModelSelection; |
| 671 | readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy; |
| 672 | } |
| 673 | |
| 674 | type RunBaselineInput = Pick< |
| 675 | RunExecutionServiceV2StartRootRunInput, |
| 676 | "appThread" | "run" | "checkpointScope" |
| 677 | >; |
| 678 | |
| 679 | export interface RunExecutionServiceV2Shape { |
| 680 | readonly startRootRun: ( |
| 681 | input: RunExecutionServiceV2StartRootRunInput, |
| 682 | ) => Effect.Effect<void, RunExecutionServiceV2Error>; |
| 683 | /** |
| 684 | * Starts the run's baseline capture on a child fiber so it overlaps provider session startup. |
| 685 | * Pass the returned effect to `startRootRun`, which awaits it before prompting the provider. |
| 686 | */ |
| 687 | readonly forkBaseline: (input: RunBaselineInput) => Effect.Effect<Effect.Effect<void>>; |
| 688 | } |
| 689 | |
| 690 | export class RunExecutionServiceV2 extends Context.Service< |
| 691 | RunExecutionServiceV2, |
| 692 | RunExecutionServiceV2Shape |
| 693 | >()("t3/orchestration-v2/RunExecutionService/RunExecutionServiceV2") {} |
| 694 | |
| 695 | export const layer: Layer.Layer< |
| 696 | RunExecutionServiceV2, |
| 697 | never, |
| 698 | | CheckpointService.CheckpointServiceV2 |
| 699 | | EventSink.EventSinkV2 |
| 700 | | IdAllocator.IdAllocatorV2 |
| 701 | | ProviderEventIngestor.ProviderEventIngestorV2 |
| 702 | | ServerSettings.ServerSettingsService |
| 703 | | McpAppModelContext.McpAppModelContext |
| 704 | | PreviewAgentActivity.PreviewAgentActivity |
| 705 | | TurnItemPositionStore.TurnItemPositionStoreV2 |
| 706 | | ThreadCommandExecutor.ThreadCommandExecutor |
| 707 | > = Layer.effect( |
| 708 | RunExecutionServiceV2, |
| 709 | Effect.gen(function* () { |
| 710 | const scope = yield* Effect.scope; |
| 711 | const checkpointService = yield* CheckpointService.CheckpointServiceV2; |
| 712 | const eventSink = yield* EventSink.EventSinkV2; |
| 713 | const idAllocator = yield* IdAllocator.IdAllocatorV2; |
| 714 | const providerEventIngestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; |
| 715 | const serverSettings = yield* ServerSettings.ServerSettingsService; |
| 716 | const mcpAppModelContext = yield* McpAppModelContext.McpAppModelContext; |
| 717 | const previewAgentActivity = yield* PreviewAgentActivity.PreviewAgentActivity; |
| 718 | const turnItemPositions = yield* TurnItemPositionStore.TurnItemPositionStoreV2; |
| 719 | const finalizationObserver = yield* RunFinalizationService.RunFinalizationObserver; |
| 720 | const turnAnalytics = yield* ProviderUsageAnalytics.ProviderTurnAnalytics; |
| 721 | const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor; |
| 722 | |
| 723 | const writeFinalRunEvents = Effect.fn("RunExecutionService.writeFinalRunEvents")( |
| 724 | function* (input: { |
| 725 | readonly run: OrchestrationV2Run; |
| 726 | readonly rootNode: OrchestrationV2ExecutionNode; |
| 727 | readonly checkpointScope: OrchestrationV2CheckpointScope; |
| 728 | readonly providerThread: OrchestrationV2ProviderThread; |
| 729 | readonly attempt: OrchestrationV2RunAttempt; |
| 730 | readonly shouldFinalizeRun?: () => Effect.Effect<boolean, ProjectionStoreV2Error>; |
| 731 | readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>; |
| 732 | readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>; |
| 733 | readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection; |
| 734 | readonly openRunOwnedSubagentsTerminal?: ProviderTerminalEvent; |
| 735 | // Root turn snapshot written with the run when T3 ends the run itself. |
| 736 | readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn; |
| 737 | // Stored root tool calls still open when the provider runtime was lost; nothing else ends them. |
| 738 | readonly lostToolCalls?: ReadonlyArray<OrchestrationV2TurnItem>; |
| 739 | readonly terminal: ProviderTerminalEvent; |
| 740 | readonly completedAt?: DateTime.Utc; |
| 741 | readonly failureItemPersisted: boolean; |
| 742 | readonly refreshAfterTurn: Effect.Effect<void>; |
| 743 | readonly writeIfRunCurrent?: { |
| 744 | readonly activeAttemptId: RunAttemptId; |
| 745 | readonly expectedStatus: OrchestrationV2Run["status"]; |
| 746 | readonly unlessInterruptRequested?: boolean; |
| 747 | }; |
| 748 | }) { |
| 749 | const completedAt = input.completedAt ?? (yield* DateTime.now); |
| 750 | const finalizedAttempt: OrchestrationV2RunAttempt = { |
| 751 | ...input.attempt, |
| 752 | status: input.terminal.status, |
| 753 | completedAt, |
| 754 | }; |
| 755 | const shouldFinalizeRun = |
| 756 | input.shouldFinalizeRun === undefined ? true : yield* input.shouldFinalizeRun(); |
| 757 | if (!shouldFinalizeRun) { |
| 758 | if (input.terminal.status === "interrupted") { |
| 759 | const hasUnpairedRequest = |
| 760 | input.hasUnpairedRunInterruptRequest === undefined |
| 761 | ? false |
| 762 | : yield* input.hasUnpairedRunInterruptRequest(); |
| 763 | if (hasUnpairedRequest) { |
| 764 | const reason = |
| 765 | input.loadRunInterruptReason === undefined |
| 766 | ? undefined |
| 767 | : yield* input.loadRunInterruptReason(); |
| 768 | yield* eventSink.writeWithEffects({ |
| 769 | effects: [], |
| 770 | events: [ |
| 771 | { |
| 772 | id: yield* idAllocator.allocate.event({ threadId: input.run.threadId }), |
| 773 | type: "turn-item.updated" as const, |
| 774 | threadId: input.run.threadId, |
| 775 | runId: input.run.id, |
| 776 | nodeId: input.rootNode.id, |
| 777 | providerInstanceId: input.run.providerInstanceId, |
| 778 | occurredAt: completedAt, |
| 779 | payload: makeInterruptResultTurnItem({ |
| 780 | idAllocator, |
| 781 | run: input.run, |
| 782 | rootNode: input.rootNode, |
| 783 | providerThread: input.providerThread, |
| 784 | reason, |
| 785 | completedAt, |
| 786 | }), |
| 787 | }, |
| 788 | ], |
| 789 | }); |
| 790 | yield* input.refreshAfterTurn; |
| 791 | } |
| 792 | } |
| 793 | return false; |
| 794 | } |
| 795 | const allocateEventId = () => idAllocator.allocate.event({ threadId: input.run.threadId }); |
| 796 | const reason = |
| 797 | input.terminal.status === "interrupted" |
| 798 | ? ((input.loadRunInterruptReason === undefined |
| 799 | ? undefined |
| 800 | : yield* input.loadRunInterruptReason()) ?? input.terminal.reason) |
| 801 | : undefined; |
| 802 | const open = input.openRunOwnedSubagents ?? emptyOpenRunOwnedSubagentProjection(); |
| 803 | const hasOpenSubagentProjection = |
| 804 | open.subagents.size > 0 || |
| 805 | open.turnItems.size > 0 || |
| 806 | open.childTurnItems.size > 0 || |
| 807 | open.nodes.size > 0; |
| 808 | const subagentTerminal = input.openRunOwnedSubagentsTerminal ?? input.terminal; |
| 809 | const cascadedSubagentEvents = |
| 810 | isRunOwnedSubagentTerminalStatus(subagentTerminal.status) && |
| 811 | !childTurnsContinue(subagentTerminal) && |
| 812 | hasOpenSubagentProjection |
| 813 | ? yield* cascadeTerminalizeRunOwnedSubagents({ |
| 814 | run: input.run, |
| 815 | open, |
| 816 | status: subagentTerminal.status, |
| 817 | completedAt, |
| 818 | allocateEventId, |
| 819 | }) |
| 820 | : []; |
| 821 | const persistedStatus = |
| 822 | input.terminal.status === "completed" ? "waiting" : input.terminal.status; |
| 823 | // Omit delegatedCompletion; the startup snapshot must not overwrite newer acknowledgements, successors, or Stop barriers. |
| 824 | const { delegatedCompletion: _delegatedCompletion, ...runWithoutDelegatedCompletion } = |
| 825 | input.run; |
| 826 | const finalizedRun: OrchestrationV2Run = { |
| 827 | ...runWithoutDelegatedCompletion, |
| 828 | status: persistedStatus, |
| 829 | completedAt: input.terminal.status === "completed" ? null : completedAt, |
| 830 | }; |
| 831 | const finalizedRootNode: OrchestrationV2ExecutionNode = { |
| 832 | ...input.rootNode, |
| 833 | status: persistedStatus, |
| 834 | completedAt: input.terminal.status === "completed" ? null : completedAt, |
| 835 | checkpointScopeId: input.checkpointScope.id, |
| 836 | }; |
| 837 | const finalizedProviderThread: OrchestrationV2ProviderThread = { |
| 838 | ...input.providerThread, |
| 839 | status: finalProviderThreadStatus(input.terminal.threadDisposition), |
| 840 | updatedAt: completedAt, |
| 841 | }; |
| 842 | const runEventId = yield* allocateEventId(); |
| 843 | const nodeEventId = yield* allocateEventId(); |
| 844 | const providerThreadEventId = yield* allocateEventId(); |
| 845 | const checkpointCaptureCommandId = CommandId.make( |
| 846 | `command:effect:checkpoint.capture:${input.run.id}`, |
| 847 | ); |
| 848 | // Stopped runs capture rollback points for the next message; enqueue before later starts on this effect lane. |
| 849 | const finalization = { |
| 850 | effects: |
| 851 | input.terminal.status === "completed" || |
| 852 | input.terminal.status === "interrupted" || |
| 853 | input.terminal.status === "cancelled" |
| 854 | ? [ |
| 855 | { |
| 856 | id: `effect:checkpoint.capture:${input.run.id}`, |
| 857 | commandId: checkpointCaptureCommandId, |
| 858 | threadId: input.run.threadId, |
| 859 | request: { |
| 860 | type: "checkpoint.capture" as const, |
| 861 | runId: input.run.id, |
| 862 | scopeId: input.checkpointScope.id, |
| 863 | }, |
| 864 | }, |
| 865 | ] |
| 866 | : [], |
| 867 | events: [ |
| 868 | ...cascadedSubagentEvents, |
| 869 | ...(yield* interruptLostToolCalls({ |
| 870 | run: input.run, |
| 871 | toolCalls: input.lostToolCalls ?? [], |
| 872 | completedAt, |
| 873 | allocateEventId, |
| 874 | })), |
| 875 | ...(input.abandonedProviderTurn === undefined |
| 876 | ? [] |
| 877 | : [ |
| 878 | { |
| 879 | id: yield* allocateEventId(), |
| 880 | type: "provider-turn.updated" as const, |
| 881 | threadId: input.run.threadId, |
| 882 | runId: input.run.id, |
| 883 | nodeId: input.rootNode.id, |
| 884 | providerInstanceId: input.run.providerInstanceId, |
| 885 | occurredAt: completedAt, |
| 886 | payload: input.abandonedProviderTurn, |
| 887 | }, |
| 888 | ]), |
| 889 | { |
| 890 | id: yield* allocateEventId(), |
| 891 | type: "run-attempt.updated" as const, |
| 892 | threadId: input.run.threadId, |
| 893 | runId: input.run.id, |
| 894 | nodeId: input.rootNode.id, |
| 895 | providerInstanceId: input.run.providerInstanceId, |
| 896 | occurredAt: completedAt, |
| 897 | payload: finalizedAttempt, |
| 898 | }, |
| 899 | ...(input.terminal.status === "interrupted" |
| 900 | ? [ |
| 901 | { |
| 902 | id: yield* allocateEventId(), |
| 903 | type: "turn-item.updated" as const, |
| 904 | threadId: input.run.threadId, |
| 905 | runId: input.run.id, |
| 906 | nodeId: input.rootNode.id, |
| 907 | providerInstanceId: input.run.providerInstanceId, |
| 908 | occurredAt: completedAt, |
| 909 | payload: makeInterruptResultTurnItem({ |
| 910 | idAllocator, |
| 911 | run: input.run, |
| 912 | rootNode: input.rootNode, |
| 913 | providerThread: input.providerThread, |
| 914 | reason, |
| 915 | completedAt, |
| 916 | }), |
| 917 | }, |
| 918 | ] |
| 919 | : []), |
| 920 | // Codex reports a hook that stops the turn as completed; keep its reason visible. |
| 921 | ...(input.terminal.status === "completed" && input.terminal.reason !== undefined |
| 922 | ? [ |
| 923 | { |
| 924 | id: yield* allocateEventId(), |
| 925 | type: "turn-item.updated" as const, |
| 926 | threadId: input.run.threadId, |
| 927 | runId: input.run.id, |
| 928 | nodeId: input.rootNode.id, |
| 929 | providerInstanceId: input.run.providerInstanceId, |
| 930 | occurredAt: completedAt, |
| 931 | payload: { |
| 932 | id: idAllocator.derive.runSignalTurnItem({ |
| 933 | runId: input.run.id, |
| 934 | signal: PROVIDER_STOP_REASON_SIGNAL, |
| 935 | }), |
| 936 | threadId: input.run.threadId, |
| 937 | runId: input.run.id, |
| 938 | nodeId: input.rootNode.id, |
| 939 | providerThreadId: input.providerThread.id, |
| 940 | providerTurnId: input.rootNode.providerTurnId, |
| 941 | nativeItemRef: null, |
| 942 | parentItemId: null, |
| 943 | ordinal: input.run.ordinal * 100 + 98, |
| 944 | status: "completed" as const, |
| 945 | title: "Stopped by the provider", |
| 946 | startedAt: completedAt, |
| 947 | completedAt, |
| 948 | updatedAt: completedAt, |
| 949 | type: "system_notice" as const, |
| 950 | level: "warning" as const, |
| 951 | message: input.terminal.reason, |
| 952 | }, |
| 953 | }, |
| 954 | ] |
| 955 | : []), |
| 956 | ...(input.terminal.status === "failed" && !input.failureItemPersisted |
| 957 | ? [ |
| 958 | { |
| 959 | id: yield* allocateEventId(), |
| 960 | type: "turn-item.updated" as const, |
| 961 | threadId: input.run.threadId, |
| 962 | runId: input.run.id, |
| 963 | nodeId: input.rootNode.id, |
| 964 | providerInstanceId: input.run.providerInstanceId, |
| 965 | occurredAt: completedAt, |
| 966 | payload: makeProviderFailureTurnItem({ |
| 967 | driver: input.terminal.driver, |
| 968 | threadId: input.run.threadId, |
| 969 | runId: input.run.id, |
| 970 | nodeId: input.rootNode.id, |
| 971 | providerThreadId: input.terminal.providerThreadId, |
| 972 | providerTurnId: input.terminal.providerTurnId, |
| 973 | itemOrdinal: input.terminal.failureItemOrdinal, |
| 974 | failure: input.terminal.failure, |
| 975 | occurredAt: completedAt, |
| 976 | }), |
| 977 | }, |
| 978 | ] |
| 979 | : []), |
| 980 | { |
| 981 | id: runEventId, |
| 982 | type: "run.updated", |
| 983 | threadId: input.run.threadId, |
| 984 | runId: input.run.id, |
| 985 | nodeId: input.rootNode.id, |
| 986 | providerInstanceId: input.run.providerInstanceId, |
| 987 | occurredAt: completedAt, |
| 988 | payload: finalizedRun, |
| 989 | }, |
| 990 | { |
| 991 | id: nodeEventId, |
| 992 | type: "node.updated", |
| 993 | threadId: input.run.threadId, |
| 994 | runId: input.run.id, |
| 995 | nodeId: input.rootNode.id, |
| 996 | providerInstanceId: input.run.providerInstanceId, |
| 997 | occurredAt: completedAt, |
| 998 | payload: finalizedRootNode, |
| 999 | }, |
| 1000 | { |
| 1001 | id: providerThreadEventId, |
| 1002 | type: "provider-thread.updated", |
| 1003 | threadId: input.run.threadId, |
| 1004 | providerInstanceId: input.run.providerInstanceId, |
| 1005 | occurredAt: completedAt, |
| 1006 | payload: finalizedProviderThread, |
| 1007 | }, |
| 1008 | ], |
| 1009 | } satisfies Parameters<typeof eventSink.writeWithEffects>[0]; |
| 1010 | if (input.writeIfRunCurrent !== undefined) { |
| 1011 | const result = yield* eventSink.writeIfRunCurrent({ |
| 1012 | threadId: input.run.threadId, |
| 1013 | runId: input.run.id, |
| 1014 | activeAttemptId: input.writeIfRunCurrent.activeAttemptId, |
| 1015 | expectedStatus: input.writeIfRunCurrent.expectedStatus, |
| 1016 | ...(input.writeIfRunCurrent.unlessInterruptRequested === true |
| 1017 | ? { |
| 1018 | unlessInterruptRequested: { |
| 1019 | requestId: idAllocator.derive.runSignalTurnItem({ |
| 1020 | runId: input.run.id, |
| 1021 | signal: "interrupt-request", |
| 1022 | }), |
| 1023 | resultId: idAllocator.derive.runSignalTurnItem({ |
| 1024 | runId: input.run.id, |
| 1025 | signal: "interrupt-result", |
| 1026 | }), |
| 1027 | }, |
| 1028 | } |
| 1029 | : {}), |
| 1030 | events: finalization.events, |
| 1031 | effects: finalization.effects, |
| 1032 | }); |
| 1033 | if (!result.committed) { |
| 1034 | return false; |
| 1035 | } |
| 1036 | } else { |
| 1037 | yield* eventSink.writeWithEffects(finalization); |
| 1038 | } |
| 1039 | yield* turnAnalytics.runSettled({ |
| 1040 | runId: input.run.id, |
| 1041 | status: input.terminal.status, |
| 1042 | completedAt, |
| 1043 | childrenRunning: hasOpenSubagentProjection && cascadedSubagentEvents.length === 0, |
| 1044 | }); |
| 1045 | yield* input.refreshAfterTurn; |
| 1046 | return true; |
| 1047 | }, |
| 1048 | ); |
| 1049 | |
| 1050 | const captureRunBaseline = Effect.fn("RunExecutionService.captureBaseline")(function* ( |
| 1051 | input: RunBaselineInput, |
| 1052 | ) { |
| 1053 | // Side chats share the parent's worktree; a snapshot would capture the parent's edits. |
| 1054 | if (input.appThread.lineage.sideChat === true) return; |
| 1055 | yield* checkpointService |
| 1056 | .captureBaseline({ |
| 1057 | scope: input.checkpointScope, |
| 1058 | ordinalWithinScope: Math.max(0, input.run.ordinal - 1), |
| 1059 | }) |
| 1060 | .pipe( |
| 1061 | Effect.catchCause((cause) => |
| 1062 | Cause.hasInterruptsOnly(cause) |
| 1063 | ? Effect.interrupt |
| 1064 | : Effect.logWarning( |
| 1065 | "orchestration V2 checkpoint baseline capture failed; starting provider without a baseline", |
| 1066 | { runId: input.run.id }, |
| 1067 | ), |
| 1068 | ), |
| 1069 | ); |
| 1070 | }); |
| 1071 | |
| 1072 | return RunExecutionServiceV2.of({ |
| 1073 | forkBaseline: (input) => |
| 1074 | captureRunBaseline(input).pipe( |
| 1075 | Effect.forkChild({ startImmediately: true }), |
| 1076 | Effect.map(Fiber.join), |
| 1077 | ), |
| 1078 | startRootRun: Effect.fn("RunExecutionService.startRootRun")(function* (input) { |
| 1079 | const rootTurnOrigin: ProviderUsageAnalytics.ProviderTurnOrigin = { |
| 1080 | createdBy: input.message.createdBy, |
| 1081 | creationSource: input.message.creationSource, |
| 1082 | scheduled: input.message.scheduledTaskId !== undefined, |
| 1083 | subagent: input.appThread.lineage.relationshipToParent === "subagent", |
| 1084 | }; |
| 1085 | // Startup failure and stream shutdown can report the same attempt. |
| 1086 | const refreshAfterTurn = yield* Effect.cached( |
| 1087 | finalizationObserver.refreshAfterTurn(input.appThread.projectId).pipe( |
| 1088 | Effect.catchCause((cause) => |
| 1089 | Effect.logWarning("failed to refresh pull requests after run termination", { |
| 1090 | threadId: input.run.threadId, |
| 1091 | runId: input.run.id, |
| 1092 | cause, |
| 1093 | }), |
| 1094 | ), |
| 1095 | ), |
| 1096 | ); |
| 1097 | const failedProviderTurnId = |
| 1098 | input.attempt.providerTurnId ?? |
| 1099 | idAllocator.derive.providerTurn({ |
| 1100 | driver: input.providerThread.driver, |
| 1101 | nativeTurnId: `failed:${input.attempt.id}`, |
| 1102 | }); |
| 1103 | const makeFailedTerminalEvent = ( |
| 1104 | failure: OrchestrationV2ProviderFailure, |
| 1105 | failureItemOrdinal: number, |
| 1106 | ): ProviderTerminalEvent => ({ |
| 1107 | type: "turn.terminal", |
| 1108 | driver: input.providerThread.driver, |
| 1109 | providerThreadId: input.providerThread.id, |
| 1110 | providerTurnId: failedProviderTurnId, |
| 1111 | runOrdinal: input.run.ordinal, |
| 1112 | failureItemOrdinal, |
| 1113 | status: "failed", |
| 1114 | failure, |
| 1115 | threadDisposition: "reusable", |
| 1116 | }); |
| 1117 | const responseStreamingMode = yield* Effect.gen(function* () { |
| 1118 | const responseStreamingMode = yield* serverSettings.getSettings.pipe( |
| 1119 | Effect.map( |
| 1120 | (settings) => |
| 1121 | resolveProjectSettings(settings, input.appThread.projectId).settings |
| 1122 | .responseStreamingMode, |
| 1123 | ), |
| 1124 | ); |
| 1125 | yield* input.baseline ?? captureRunBaseline(input); |
| 1126 | if ( |
| 1127 | input.shouldStartProviderTurn !== undefined && |
| 1128 | !(yield* input.shouldStartProviderTurn()) |
| 1129 | ) { |
| 1130 | return null; |
| 1131 | } |
| 1132 | return responseStreamingMode; |
| 1133 | }).pipe( |
| 1134 | Effect.catchCause((cause) => |
| 1135 | Effect.gen(function* () { |
| 1136 | if (Cause.hasInterruptsOnly(cause)) { |
| 1137 | return yield* Effect.failCause(cause); |
| 1138 | } |
| 1139 | yield* Effect.logError("orchestration V2 run preparation failed", { |
| 1140 | runId: input.run.id, |
| 1141 | cause, |
| 1142 | }); |
| 1143 | yield* writeFinalRunEvents({ |
| 1144 | run: input.run, |
| 1145 | rootNode: input.rootNode, |
| 1146 | checkpointScope: input.checkpointScope, |
| 1147 | providerThread: input.providerThread, |
| 1148 | attempt: input.attempt, |
| 1149 | terminal: makeFailedTerminalEvent( |
| 1150 | makeProviderFailure({ |
| 1151 | cause: Cause.squash(cause), |
| 1152 | message: "Run preparation failed.", |
| 1153 | class: "unknown", |
| 1154 | }), |
| 1155 | input.providerTurnOrdinal * 100 + 1, |
| 1156 | ), |
| 1157 | failureItemPersisted: false, |
| 1158 | refreshAfterTurn, |
| 1159 | writeIfRunCurrent: { |
| 1160 | activeAttemptId: input.attemptId, |
| 1161 | expectedStatus: "running", |
| 1162 | }, |
| 1163 | }); |
| 1164 | return null; |
| 1165 | }), |
| 1166 | ), |
| 1167 | Effect.mapError( |
| 1168 | (cause) => |
| 1169 | new RunExecutionStartError({ |
| 1170 | commandId: input.commandId, |
| 1171 | runId: input.run.id, |
| 1172 | cause, |
| 1173 | }), |
| 1174 | ), |
| 1175 | ); |
| 1176 | if (responseStreamingMode === null) { |
| 1177 | return; |
| 1178 | } |
| 1179 | const terminalEvent = yield* Ref.make<ProviderTerminalEvent | null>(null); |
| 1180 | const latestTurnItemOrdinal = yield* Ref.make(input.providerTurnOrdinal * 100); |
| 1181 | const latestProviderThread = yield* Ref.make(input.providerThread); |
| 1182 | const latestRootProviderTurn = yield* Ref.make<OrchestrationV2ProviderTurn | null>(null); |
| 1183 | // Later root turns in one attempt are provider continuations, such as Codex goal turns. |
| 1184 | let firstRootProviderTurnId: ProviderTurnId | undefined; |
| 1185 | const analyticsContext = { |
| 1186 | runId: input.run.id, |
| 1187 | modelSelection: input.modelSelection, |
| 1188 | runtimeMode: input.runtimePolicy.runtimeMode, |
| 1189 | interactionMode: input.runtimePolicy.interactionMode, |
| 1190 | origin: rootTurnOrigin, |
| 1191 | ...(input.run.restartContinuationOfRunId === undefined |
| 1192 | ? {} |
| 1193 | : { restartContinuation: true }), |
| 1194 | } satisfies ProviderUsageAnalytics.ProviderTurnAnalyticsContext; |
| 1195 | if (input.attempt.reason === "initial") { |
| 1196 | yield* turnAnalytics.runStarted({ |
| 1197 | runId: input.run.id, |
| 1198 | driver: input.providerThread.driver, |
| 1199 | context: { ...analyticsContext, segment: "initial" }, |
| 1200 | }); |
| 1201 | } |
| 1202 | const routeIdentity: ProviderEventRouteIdentity = { |
| 1203 | threadId: input.run.threadId, |
| 1204 | runId: input.run.id, |
| 1205 | attemptId: input.attempt.id, |
| 1206 | providerThreadId: input.providerThread.id, |
| 1207 | }; |
| 1208 | const eventSubscription = |
| 1209 | input.session.subscribeEvents === undefined |
| 1210 | ? { events: input.session.events, close: Effect.void } |
| 1211 | : yield* input.session.subscribeEvents; |
| 1212 | const inheritedBackgroundTurnItems = yield* ( |
| 1213 | input.loadInheritedBackgroundTurnItems?.() ?? Effect.succeed([]) |
| 1214 | ).pipe( |
| 1215 | Effect.onError(() => eventSubscription.close), |
| 1216 | Effect.mapError( |
| 1217 | (cause) => |
| 1218 | new RunExecutionStartError({ |
| 1219 | commandId: input.commandId, |
| 1220 | runId: input.run.id, |
| 1221 | cause, |
| 1222 | }), |
| 1223 | ), |
| 1224 | ); |
| 1225 | const inheritedBackgroundTurnItemsById = new Map( |
| 1226 | inheritedBackgroundTurnItems.map((item) => [item.id, item.runId]), |
| 1227 | ); |
| 1228 | const eventRouting = yield* Ref.make<ProviderEventRoutingState>( |
| 1229 | makeProviderEventRoutingState({ |
| 1230 | identity: routeIdentity, |
| 1231 | inheritedBackgroundTurnItems, |
| 1232 | providerTurnId: input.attempt.providerTurnId, |
| 1233 | ...(input.relatedThreadIds === undefined |
| 1234 | ? {} |
| 1235 | : { relatedThreadIds: input.relatedThreadIds }), |
| 1236 | ...(input.relatedProviderThreadIds === undefined |
| 1237 | ? {} |
| 1238 | : { relatedProviderThreadIds: input.relatedProviderThreadIds }), |
| 1239 | }), |
| 1240 | ); |
| 1241 | const rootTerminalSeen = yield* Ref.make(false); |
| 1242 | const rootRunFinalized = yield* Ref.make(false); |
| 1243 | const providerThreadOwnerLost = yield* Ref.make(false); |
| 1244 | const pendingBackgroundProviderTurns = new Set<ProviderTurnId>(); |
| 1245 | const activeChildProviderTurns = yield* Ref.make<ReadonlySet<ProviderTurnId>>(new Set()); |
| 1246 | const activeChildSubagents = yield* Ref.make<ReadonlyMap<NodeId, ThreadId>>(new Map()); |
| 1247 | const activeBackgroundTurnItems = yield* Ref.make< |
| 1248 | ReadonlySet<OrchestrationV2TurnItem["id"]> |
| 1249 | >(new Set(inheritedBackgroundTurnItemsById.keys())); |
| 1250 | const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection()); |
| 1251 | // Open tool calls on the root thread, to end from their stored rows if the runtime is lost. |
| 1252 | const openRootToolCalls = new Set<TurnItemId>(); |
| 1253 | // The last stored root snapshot still reports background work, by flag or task roster. |
| 1254 | let rootSnapshotPendingWork = false; |
| 1255 | // Owned child threads whose last snapshot reported pending work, such as a queued native |
| 1256 | // continuation. Unlike the root's, these must hold the stream past owner loss. |
| 1257 | const childSnapshotPendingWork = new Set<ProviderThreadId>(); |
| 1258 | let rootTurnOwnershipChecked = false; |
| 1259 | const assistantStreaming = makeAssistantStreamingFilter(responseStreamingMode); |
| 1260 | // Set when ingestion ends or finalization settles the run's open items, which a later |
| 1261 | // streaming snapshot must not reopen. |
| 1262 | let idleFlushClosed = false; |
| 1263 | const workspaceHandedOff = Effect.suspend(() => |
| 1264 | input.shouldFinalizeRun === undefined || input.hasWorkspaceMoved === undefined |
| 1265 | ? Effect.succeed(false) |
| 1266 | : input.hasWorkspaceMoved(), |
| 1267 | ); |
| 1268 | const finalizeRootRun = Effect.fn("RunExecutionService.finalizeRootRun")(function* ( |
| 1269 | terminal: ProviderTerminalEvent, |
| 1270 | abandonedProviderTurn?: OrchestrationV2ProviderTurn, |
| 1271 | ) { |
| 1272 | if (yield* Ref.get(rootRunFinalized)) { |
| 1273 | return; |
| 1274 | } |
| 1275 | const providerThread = yield* Ref.get(latestProviderThread); |
| 1276 | const openSubagents = yield* Ref.get(openRunOwnedSubagents); |
| 1277 | const writeTerminal = (settledTerminal: ProviderTerminalEvent) => |
| 1278 | writeFinalRunEvents({ |
| 1279 | run: input.run, |
| 1280 | rootNode: input.rootNode, |
| 1281 | checkpointScope: input.checkpointScope, |
| 1282 | providerThread, |
| 1283 | attempt: input.attempt, |
| 1284 | ...(input.shouldFinalizeRun === undefined |
| 1285 | ? {} |
| 1286 | : { |
| 1287 | shouldFinalizeRun: input.shouldFinalizeRun, |
| 1288 | // Stop may commit between the ownership read and this |
| 1289 | // terminal write. Gate the events and checkpoint together. |
| 1290 | writeIfRunCurrent: { |
| 1291 | activeAttemptId: input.attempt.id, |
| 1292 | expectedStatus: "running" as const, |
| 1293 | unlessInterruptRequested: settledTerminal !== terminal, |
| 1294 | }, |
| 1295 | }), |
| 1296 | ...(input.hasUnpairedRunInterruptRequest === undefined |
| 1297 | ? {} |
| 1298 | : { |
| 1299 | hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, |
| 1300 | }), |
| 1301 | ...(input.loadRunInterruptReason === undefined |
| 1302 | ? {} |
| 1303 | : { loadRunInterruptReason: input.loadRunInterruptReason }), |
| 1304 | openRunOwnedSubagents: openSubagents, |
| 1305 | openRunOwnedSubagentsTerminal: terminal, |
| 1306 | ...(abandonedProviderTurn === undefined ? {} : { abandonedProviderTurn }), |
| 1307 | terminal: settledTerminal, |
| 1308 | failureItemPersisted: settledTerminal.status === "failed", |
| 1309 | refreshAfterTurn, |
| 1310 | }).pipe( |
| 1311 | Effect.mapError( |
| 1312 | (cause) => new RunExecutionIngestError({ runId: input.run.id, cause }), |
| 1313 | ), |
| 1314 | ); |
| 1315 | // A workspace handoff detaches the session and stops the turn that asked |
| 1316 | // for it: the run completes, while its native subagents still settle |
| 1317 | // as interrupted. The completed write refuses in the same transaction |
| 1318 | // as a stored Stop request, so a racing Stop stays authoritative. |
| 1319 | const handedOff = terminal.status === "interrupted" && (yield* workspaceHandedOff); |
| 1320 | if (!handedOff || !(yield* writeTerminal({ ...terminal, status: "completed" }))) { |
| 1321 | yield* writeTerminal(terminal); |
| 1322 | } |
| 1323 | if (isRunOwnedSubagentTerminalStatus(terminal.status) && !childTurnsContinue(terminal)) { |
| 1324 | yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); |
| 1325 | idleFlushClosed = true; |
| 1326 | } |
| 1327 | yield* Ref.set(rootRunFinalized, true); |
| 1328 | }); |
| 1329 | // The stored rows of the work this run still tracks as open on the given threads. |
| 1330 | const loadTrackedWork = Effect.fn("RunExecutionService.loadTrackedWork")(function* ( |
| 1331 | threadIds: Iterable<ThreadId>, |
| 1332 | ) { |
| 1333 | const open = yield* Ref.get(openRunOwnedSubagents); |
| 1334 | const subagents = new Map<NodeId, OrchestrationV2Subagent>(); |
| 1335 | const nodes = new Map<NodeId, OrchestrationV2ExecutionNode>(); |
| 1336 | const turnItems = new Map<TurnItemId, OrchestrationV2TurnItem>(); |
| 1337 | if (input.loadUnsettledWork === undefined) return { subagents, nodes, turnItems }; |
| 1338 | const trackedItemIds = new Set<TurnItemId>([ |
| 1339 | ...openRootToolCalls.keys(), |
| 1340 | ...open.childTurnItems.keys(), |
| 1341 | ...[...open.turnItems.values()].map((item) => item.id), |
| 1342 | ]); |
| 1343 | for (const threadId of new Set(threadIds)) { |
| 1344 | const stored = yield* input.loadUnsettledWork( |
| 1345 | threadId, |
| 1346 | [...open.nodes.values(), ...open.subagents.values()] |
| 1347 | .filter((record) => record.threadId === threadId) |
| 1348 | .map((record) => record.id), |
| 1349 | ); |
| 1350 | for (const subagent of stored.subagents) { |
| 1351 | if (open.subagents.has(subagent.id) && !isSettledSubagentStatus(subagent.status)) { |
| 1352 | subagents.set(subagent.id, subagent); |
| 1353 | } |
| 1354 | } |
| 1355 | for (const node of stored.nodes) { |
| 1356 | if (open.nodes.has(node.id) && isOpenExecutionNodeStatus(node.status)) { |
| 1357 | nodes.set(node.id, node); |
| 1358 | } |
| 1359 | } |
| 1360 | for (const item of stored.turnItems) { |
| 1361 | if (trackedItemIds.has(item.id) && !isSettledTurnItemStatus(item.status)) { |
| 1362 | turnItems.set(item.id, item); |
| 1363 | } |
| 1364 | } |
| 1365 | } |
| 1366 | return { subagents, nodes, turnItems }; |
| 1367 | }); |
| 1368 | const loadLostRootToolCalls = loadTrackedWork([input.run.threadId]).pipe( |
| 1369 | Effect.map(({ turnItems }) => |
| 1370 | [...turnItems.values()].filter((item) => openRootToolCalls.has(item.id)), |
| 1371 | ), |
| 1372 | ); |
| 1373 | // Subagents and tool calls that outlived the root run die with the provider runtime. Stop |
| 1374 | // can settle them meanwhile, so each write holds only while its stored row is unchanged. |
| 1375 | const endWorkLostAfterRootRun = Effect.fn("RunExecutionService.endWorkLostAfterRootRun")( |
| 1376 | function* (completedAt: DateTime.Utc) { |
| 1377 | const allocateEventId = () => |
| 1378 | idAllocator.allocate.event({ threadId: input.run.threadId }); |
| 1379 | const open = yield* Ref.get(openRunOwnedSubagents); |
| 1380 | const stored = yield* loadTrackedWork([ |
| 1381 | input.run.threadId, |
| 1382 | ...[ |
| 1383 | ...open.subagents.values(), |
| 1384 | ...open.nodes.values(), |
| 1385 | ...open.childTurnItems.values(), |
| 1386 | ].map((record) => record.threadId), |
| 1387 | ]); |
| 1388 | for (const subagent of stored.subagents.values()) { |
| 1389 | const subagentItemId = open.turnItems.get(subagent.id)?.id; |
| 1390 | const subagentItem = |
| 1391 | subagentItemId === undefined ? undefined : stored.turnItems.get(subagentItemId); |
| 1392 | const events = yield* cascadeTerminalizeRunOwnedSubagents({ |
| 1393 | run: input.run, |
| 1394 | open: { |
| 1395 | subagents: new Map([[subagent.id, subagent]]), |
| 1396 | turnItems: new Map( |
| 1397 | subagentItem?.type === "subagent" ? [[subagent.id, subagentItem]] : [], |
| 1398 | ), |
| 1399 | childTurnItems: new Map( |
| 1400 | [...stored.turnItems].filter( |
| 1401 | ([id, item]) => |
| 1402 | open.childTurnItems.has(id) && item.threadId === subagent.childThreadId, |
| 1403 | ), |
| 1404 | ), |
| 1405 | nodes: new Map( |
| 1406 | [...stored.nodes].filter( |
| 1407 | ([id, node]) => |
| 1408 | id === subagent.id || node.threadId === subagent.childThreadId, |
| 1409 | ), |
| 1410 | ), |
| 1411 | linkedChildThreadIds: new Set( |
| 1412 | subagent.childThreadId === null ? [] : [subagent.childThreadId], |
| 1413 | ), |
| 1414 | }, |
| 1415 | status: "failed", |
| 1416 | completedAt, |
| 1417 | allocateEventId, |
| 1418 | }); |
| 1419 | yield* eventSink.writeIfSubagentUnchanged({ expected: subagent, events }); |
| 1420 | } |
| 1421 | for (const toolCall of stored.turnItems.values()) { |
| 1422 | if (!openRootToolCalls.has(toolCall.id)) continue; |
| 1423 | yield* eventSink.writeIfTurnItemUnchanged({ |
| 1424 | expected: toolCall, |
| 1425 | events: yield* interruptLostToolCalls({ |
| 1426 | run: input.run, |
| 1427 | toolCalls: [toolCall], |
| 1428 | completedAt, |
| 1429 | allocateEventId, |
| 1430 | }), |
| 1431 | }); |
| 1432 | } |
| 1433 | yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); |
| 1434 | openRootToolCalls.clear(); |
| 1435 | }, |
| 1436 | (effect) => |
| 1437 | effect.pipe( |
| 1438 | Effect.catchCauseIf( |
| 1439 | (cause) => !Cause.hasInterruptsOnly(cause), |
| 1440 | (cause) => |
| 1441 | Effect.logWarning( |
| 1442 | "orchestration V2 failed to end work lost with the provider runtime", |
| 1443 | { runId: input.run.id, cause }, |
| 1444 | ), |
| 1445 | ), |
| 1446 | ), |
| 1447 | ); |
| 1448 | // Runs for every provider event, including streamed text updates; avoid a per-event span. |
| 1449 | const trackChildLifecycle = Effect.fnUntraced(function* ( |
| 1450 | event: ProviderAdapter.ProviderAdapterV2Event, |
| 1451 | deliverable: boolean, |
| 1452 | ) { |
| 1453 | const routing = yield* Ref.get(eventRouting); |
| 1454 | if (event.type === "provider_turn.updated") { |
| 1455 | const isRoot = |
| 1456 | event.providerTurn.runAttemptId === input.attempt.id || |
| 1457 | event.providerTurn.id === routing.rootProviderTurnId; |
| 1458 | if (!isRoot) { |
| 1459 | yield* Ref.update(activeChildProviderTurns, (current) => { |
| 1460 | const next = new Set(current); |
| 1461 | if (isTerminalProviderTurnStatus(event.providerTurn.status)) { |
| 1462 | next.delete(event.providerTurn.id); |
| 1463 | } else { |
| 1464 | next.add(event.providerTurn.id); |
| 1465 | } |
| 1466 | return next; |
| 1467 | }); |
| 1468 | } |
| 1469 | } |
| 1470 | if (event.type === "subagent.updated") { |
| 1471 | const belongsToRootRun = event.subagent.runId === input.run.id; |
| 1472 | const belongsToOwnedChildThread = |
| 1473 | event.subagent.threadId !== input.run.threadId && |
| 1474 | routing.ownedThreadIds.has(event.subagent.threadId); |
| 1475 | if (belongsToRootRun || belongsToOwnedChildThread) { |
| 1476 | yield* Ref.update(activeChildSubagents, (current) => { |
| 1477 | const next = new Map(current); |
| 1478 | if (isSettledSubagentStatus(event.subagent.status)) { |
| 1479 | next.delete(event.subagent.id); |
| 1480 | } else { |
| 1481 | next.set(event.subagent.id, event.subagent.threadId); |
| 1482 | } |
| 1483 | return next; |
| 1484 | }); |
| 1485 | } |
| 1486 | // Preserve childThreadId linkage for the root-run lifetime even after the subagent |
| 1487 | // row terminalizes, so open child-thread nodes can still be proven linked on a |
| 1488 | // later root interrupt. A provider records nested native subagents on the child |
| 1489 | // thread, runless; they die with this run too. |
| 1490 | if ( |
| 1491 | belongsToRootRun || |
| 1492 | (belongsToOwnedChildThread && |
| 1493 | event.subagent.runId === null && |
| 1494 | event.subagent.origin === "provider_native") |
| 1495 | ) { |
| 1496 | yield* Ref.update(openRunOwnedSubagents, (current) => { |
| 1497 | const withLink = withLinkedChildThreadId(current, event.subagent.childThreadId); |
| 1498 | const subagents = new Map(withLink.subagents); |
| 1499 | if (isSettledSubagentStatus(event.subagent.status)) { |
| 1500 | subagents.delete(event.subagent.id); |
| 1501 | } else { |
| 1502 | subagents.set(event.subagent.id, event.subagent); |
| 1503 | } |
| 1504 | return { ...withLink, subagents }; |
| 1505 | }); |
| 1506 | } |
| 1507 | } |
| 1508 | if (event.type === "node.updated") { |
| 1509 | const belongsToRootSubagent = |
| 1510 | event.node.kind === "subagent" && event.node.runId === input.run.id; |
| 1511 | const belongsToOwnedChildThread = |
| 1512 | event.node.threadId !== input.run.threadId && |
| 1513 | routing.ownedThreadIds.has(event.node.threadId); |
| 1514 | if ( |
| 1515 | !belongsToRootSubagent && |
| 1516 | !( |
| 1517 | belongsToOwnedChildThread && |
| 1518 | (event.node.runId === null || event.node.runId === input.run.id) |
| 1519 | ) |
| 1520 | ) { |
| 1521 | return; |
| 1522 | } |
| 1523 | yield* Ref.update(openRunOwnedSubagents, (current) => { |
| 1524 | const nodes = new Map(current.nodes); |
| 1525 | if (isOpenExecutionNodeStatus(event.node.status)) { |
| 1526 | nodes.set(event.node.id, event.node); |
| 1527 | } else { |
| 1528 | nodes.delete(event.node.id); |
| 1529 | } |
| 1530 | return { ...current, nodes }; |
| 1531 | }); |
| 1532 | } |
| 1533 | if (event.type === "turn_item.updated") { |
| 1534 | const belongsToRootRun = event.turnItem.runId === input.run.id; |
| 1535 | const belongsToOwnedChildThread = |
| 1536 | event.turnItem.threadId !== input.run.threadId && |
| 1537 | routing.ownedThreadIds.has(event.turnItem.threadId); |
| 1538 | const belongsToInheritedBackgroundItem = |
| 1539 | event.turnItem.threadId === input.run.threadId && |
| 1540 | event.turnItem.runId !== null && |
| 1541 | inheritedBackgroundTurnItemsById.get(event.turnItem.id) === event.turnItem.runId; |
| 1542 | if ( |
| 1543 | backgroundCapableTurnItemTypes.has(event.turnItem.type) && |
| 1544 | (belongsToRootRun || belongsToOwnedChildThread || belongsToInheritedBackgroundItem) |
| 1545 | ) { |
| 1546 | yield* Ref.update(activeBackgroundTurnItems, (current) => { |
| 1547 | const next = new Set(current); |
| 1548 | if (isSettledTurnItemStatus(event.turnItem.status)) { |
| 1549 | next.delete(event.turnItem.id); |
| 1550 | } else { |
| 1551 | next.add(event.turnItem.id); |
| 1552 | } |
| 1553 | return next; |
| 1554 | }); |
| 1555 | if ( |
| 1556 | event.turnItem.threadId === input.run.threadId && |
| 1557 | processBoundToolCallTypes.has(event.turnItem.type) |
| 1558 | ) { |
| 1559 | if (isSettledTurnItemStatus(event.turnItem.status)) { |
| 1560 | openRootToolCalls.delete(event.turnItem.id); |
| 1561 | } else { |
| 1562 | openRootToolCalls.add(event.turnItem.id); |
| 1563 | } |
| 1564 | } |
| 1565 | } |
| 1566 | if ( |
| 1567 | belongsToOwnedChildThread && |
| 1568 | (event.turnItem.runId === null || belongsToRootRun) && |
| 1569 | deliverable |
| 1570 | ) { |
| 1571 | yield* Ref.update(openRunOwnedSubagents, (current) => { |
| 1572 | const childTurnItems = new Map(current.childTurnItems); |
| 1573 | if (isSettledTurnItemStatus(event.turnItem.status)) { |
| 1574 | childTurnItems.delete(event.turnItem.id); |
| 1575 | } else { |
| 1576 | childTurnItems.set(event.turnItem.id, event.turnItem); |
| 1577 | } |
| 1578 | return { ...current, childTurnItems }; |
| 1579 | }); |
| 1580 | } |
| 1581 | if (belongsToRootRun && event.turnItem.type === "subagent") { |
| 1582 | const subagentItem = event.turnItem; |
| 1583 | yield* Ref.update(openRunOwnedSubagents, (current) => { |
| 1584 | const withLink = withLinkedChildThreadId(current, subagentItem.childThreadId); |
| 1585 | const turnItems = new Map(withLink.turnItems); |
| 1586 | if (isSettledTurnItemStatus(subagentItem.status)) { |
| 1587 | turnItems.delete(subagentItem.subagentId); |
| 1588 | } else { |
| 1589 | turnItems.set(subagentItem.subagentId, subagentItem); |
| 1590 | } |
| 1591 | return { ...withLink, turnItems }; |
| 1592 | }); |
| 1593 | } |
| 1594 | } |
| 1595 | }); |
| 1596 | const agentTurnsSettled = Effect.gen(function* () { |
| 1597 | if (!(yield* Ref.get(rootTerminalSeen))) { |
| 1598 | return false; |
| 1599 | } |
| 1600 | const terminal = yield* Ref.get(terminalEvent); |
| 1601 | if ( |
| 1602 | terminal !== null && |
| 1603 | terminal.status !== "completed" && |
| 1604 | !childTurnsContinue(terminal) |
| 1605 | ) { |
| 1606 | return true; |
| 1607 | } |
| 1608 | if (pendingBackgroundProviderTurns.size > 0) { |
| 1609 | return false; |
| 1610 | } |
| 1611 | const childProviderTurns = yield* Ref.get(activeChildProviderTurns); |
| 1612 | if (childProviderTurns.size > 0) { |
| 1613 | return false; |
| 1614 | } |
| 1615 | const childSubagents = yield* Ref.get(activeChildSubagents); |
| 1616 | return childSubagents.size === 0; |
| 1617 | }); |
| 1618 | // Claude background Bash has no turn-item projection. Use only the |
| 1619 | // thread-scoped probe: session-wide pending work (siblings, wake |
| 1620 | // buffers, session subagents) must not pin this root subscription. |
| 1621 | const threadHasPendingBackgroundWork = Effect.gen(function* () { |
| 1622 | // The live probe reads adapter state, which can be ahead of the events consumed here. |
| 1623 | if (rootSnapshotPendingWork) { |
| 1624 | return true; |
| 1625 | } |
| 1626 | if (input.session.hasPendingBackgroundWorkForThread === undefined) { |
| 1627 | return false; |
| 1628 | } |
| 1629 | return yield* input.session |
| 1630 | .hasPendingBackgroundWorkForThread(yield* Ref.get(latestProviderThread)) |
| 1631 | .pipe(Effect.catchCause(() => Effect.succeed(false))); |
| 1632 | }); |
| 1633 | const shouldStopProviderEventIngestion = Effect.gen(function* () { |
| 1634 | if (!(yield* agentTurnsSettled)) { |
| 1635 | return false; |
| 1636 | } |
| 1637 | const terminal = yield* Ref.get(terminalEvent); |
| 1638 | if ( |
| 1639 | terminal !== null && |
| 1640 | terminal.status !== "completed" && |
| 1641 | !childTurnsContinue(terminal) |
| 1642 | ) { |
| 1643 | // A failed turn can leave background work running in a live provider process. |
| 1644 | return ( |
| 1645 | terminal.status !== "failed" || |
| 1646 | (yield* Ref.get(providerThreadOwnerLost)) || |
| 1647 | !(yield* threadHasPendingBackgroundWork) |
| 1648 | ); |
| 1649 | } |
| 1650 | // Newly owned items depend on adapters emitting a non-terminal event before the root |
| 1651 | // terminal. Owner loss (a newer run claimed lastRunOrdinal) must not close this stream |
| 1652 | // while these sets are non-empty: turn_item.updated writes are not ownership-gated, so |
| 1653 | // late completions still land. |
| 1654 | const backgroundItems = yield* Ref.get(activeBackgroundTurnItems); |
| 1655 | if (backgroundItems.size > 0 || childSnapshotPendingWork.size > 0) { |
| 1656 | return false; |
| 1657 | } |
| 1658 | if (yield* Ref.get(providerThreadOwnerLost)) { |
| 1659 | return true; |
| 1660 | } |
| 1661 | return !(yield* threadHasPendingBackgroundWork); |
| 1662 | }); |
| 1663 | // Held text writes nothing, so its item would take a position only |
| 1664 | // when finally written, behind every item that began after it. |
| 1665 | const reservedHeldTurnItemIds = new Set<TurnItemId>(); |
| 1666 | // Ends once the root turn and the children it drives settle. Background commands such |
| 1667 | // as a dev server keep this consumer alive afterwards and must not pin preview focus. |
| 1668 | const endPreviewActivity = yield* previewAgentActivity.begin(input.run.threadId); |
| 1669 | // Per-event routing and lifecycle bookkeeping share the run's tracing boundary. |
| 1670 | const routeEvent = Effect.fnUntraced(function* ( |
| 1671 | event: ProviderAdapter.ProviderAdapterV2Event, |
| 1672 | ) { |
| 1673 | const previous = yield* Ref.get(eventRouting); |
| 1674 | const [accepted, routing] = routeProviderEvent(event, routeIdentity, previous); |
| 1675 | yield* Ref.set(eventRouting, routing); |
| 1676 | if ( |
| 1677 | event.type === "subagent.updated" && |
| 1678 | event.subagent.childThreadId !== null && |
| 1679 | previous.ownedThreadIds.has(event.subagent.childThreadId) && |
| 1680 | !routing.ownedThreadIds.has(event.subagent.childThreadId) |
| 1681 | ) { |
| 1682 | const moved = event.subagent; |
| 1683 | const detached = yield* Ref.modify(openRunOwnedSubagents, (current) => { |
| 1684 | const subagentIds = new Set([moved.id]); |
| 1685 | for (const subagent of current.subagents.values()) { |
| 1686 | if (subagent.threadId === moved.childThreadId) subagentIds.add(subagent.id); |
| 1687 | } |
| 1688 | const childTurnItems = new Map( |
| 1689 | [...current.childTurnItems].filter( |
| 1690 | ([, item]) => item.threadId !== moved.childThreadId, |
| 1691 | ), |
| 1692 | ); |
| 1693 | const turnItems = new Map(current.turnItems); |
| 1694 | turnItems.delete(moved.id); |
| 1695 | const detachedItems = new Set( |
| 1696 | [...current.childTurnItems.values(), ...current.turnItems.values()] |
| 1697 | .filter( |
| 1698 | (item) => item.threadId === moved.childThreadId || item.nodeId === moved.id, |
| 1699 | ) |
| 1700 | .map((item) => item.id), |
| 1701 | ); |
| 1702 | return [ |
| 1703 | { itemIds: detachedItems, subagentIds }, |
| 1704 | { |
| 1705 | subagents: new Map([...current.subagents].filter(([id]) => !subagentIds.has(id))), |
| 1706 | turnItems, |
| 1707 | childTurnItems, |
| 1708 | nodes: new Map( |
| 1709 | [...current.nodes].filter( |
| 1710 | ([id, node]) => id !== moved.id && node.threadId !== moved.childThreadId, |
| 1711 | ), |
| 1712 | ), |
| 1713 | linkedChildThreadIds: new Set( |
| 1714 | [...current.linkedChildThreadIds].filter((id) => id !== moved.childThreadId), |
| 1715 | ), |
| 1716 | }, |
| 1717 | ]; |
| 1718 | }); |
| 1719 | yield* Ref.update( |
| 1720 | activeChildSubagents, |
| 1721 | (current) => |
| 1722 | new Map( |
| 1723 | [...current].filter( |
| 1724 | ([id, threadId]) => |
| 1725 | !detached.subagentIds.has(id) && threadId !== moved.childThreadId, |
| 1726 | ), |
| 1727 | ), |
| 1728 | ); |
| 1729 | yield* Ref.update( |
| 1730 | activeChildProviderTurns, |
| 1731 | (current) => new Set([...current].filter((id) => routing.ownedProviderTurns.has(id))), |
| 1732 | ); |
| 1733 | for (const id of pendingBackgroundProviderTurns) { |
| 1734 | if (!routing.ownedProviderTurns.has(id)) { |
| 1735 | pendingBackgroundProviderTurns.delete(id); |
| 1736 | } |
| 1737 | } |
| 1738 | // Routing now rejects the moved thread's snapshots, so none would clear its entry. |
| 1739 | for (const id of childSnapshotPendingWork) { |
| 1740 | if (!routing.ownedProviderThreadIds.has(id)) childSnapshotPendingWork.delete(id); |
| 1741 | } |
| 1742 | yield* Ref.update( |
| 1743 | activeBackgroundTurnItems, |
| 1744 | (current) => new Set([...current].filter((id) => !detached.itemIds.has(id))), |
| 1745 | ); |
| 1746 | } |
| 1747 | return { event, accepted }; |
| 1748 | }); |
| 1749 | // Per-event ingestion avoids adding a span for every provider snapshot. |
| 1750 | const ingestDelivered = Effect.fnUntraced(function* ( |
| 1751 | event: ProviderAdapter.ProviderAdapterV2Event, |
| 1752 | deliveredEvent: ProviderAdapter.ProviderAdapterV2Event, |
| 1753 | ) { |
| 1754 | const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); |
| 1755 | const isRootProviderThreadUpdate = |
| 1756 | event.type === "provider_thread.updated" && |
| 1757 | event.providerThread.id === input.providerThread.id; |
| 1758 | const nativeChildTurn = |
| 1759 | event.type === "provider_turn.updated" && |
| 1760 | event.providerTurn.providerThreadId !== input.providerThread.id; |
| 1761 | if (event.type === "provider_turn.updated" && !nativeChildTurn) { |
| 1762 | firstRootProviderTurnId ??= event.providerTurn.id; |
| 1763 | } |
| 1764 | const storedEvents = yield* providerEventIngestor.ingestNormalized({ |
| 1765 | analyticsContext: nativeChildTurn |
| 1766 | ? { |
| 1767 | ...analyticsContext, |
| 1768 | origin: NATIVE_SUBAGENT_TURN_ORIGIN, |
| 1769 | segment: "native_child", |
| 1770 | } |
| 1771 | : { |
| 1772 | ...analyticsContext, |
| 1773 | segment: |
| 1774 | event.type === "provider_turn.updated" && |
| 1775 | event.providerTurn.id !== firstRootProviderTurnId |
| 1776 | ? "continuation" |
| 1777 | : input.attempt.reason, |
| 1778 | }, |
| 1779 | providerSessionId: input.providerSessionId, |
| 1780 | providerInstanceId: input.run.providerInstanceId, |
| 1781 | threadId: input.run.threadId, |
| 1782 | runId: input.run.id, |
| 1783 | nodeId: input.rootNode.id, |
| 1784 | event: deliveredEvent, |
| 1785 | ...(isRootProviderThreadUpdate |
| 1786 | ? rootTerminalAlreadySeen |
| 1787 | ? { |
| 1788 | writeIfProviderThreadOwner: { |
| 1789 | providerThreadId: input.providerThread.id, |
| 1790 | runId: input.run.id, |
| 1791 | activeAttemptId: input.attempt.id, |
| 1792 | expectedLastRunOrdinal: input.run.ordinal, |
| 1793 | }, |
| 1794 | } |
| 1795 | : { |
| 1796 | writeIfRunCurrent: { |
| 1797 | runId: input.run.id, |
| 1798 | activeAttemptId: input.attempt.id, |
| 1799 | expectedStatus: "running" as const, |
| 1800 | }, |
| 1801 | } |
| 1802 | : {}), |
| 1803 | }); |
| 1804 | if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEvents.length === 0) { |
| 1805 | yield* Ref.set(providerThreadOwnerLost, true); |
| 1806 | } |
| 1807 | return storedEvents.length; |
| 1808 | }); |
| 1809 | // The timer only wakes ingestion. The flush runs as an ingestion step outside any provider |
| 1810 | // batch, so it stays ordered with routing and snapshots, and a write has committed when |
| 1811 | // it returns. |
| 1812 | const idleFlushTicks = yield* Queue.dropping<{ |
| 1813 | readonly type: "idle_flush"; |
| 1814 | readonly at: number; |
| 1815 | }>(1); |
| 1816 | let idleFlushWake: { readonly at: number; readonly fiber: Fiber.Fiber<void> } | undefined; |
| 1817 | const scheduleIdleFlush = Effect.suspend(() => { |
| 1818 | const at = assistantStreaming.nextIdleFlushAt(); |
| 1819 | if ( |
| 1820 | idleFlushClosed || |
| 1821 | at === null || |
| 1822 | (idleFlushWake !== undefined && idleFlushWake.at <= at) |
| 1823 | ) |
| 1824 | return Effect.void; |
| 1825 | const previous = idleFlushWake?.fiber; |
| 1826 | return Effect.gen(function* () { |
| 1827 | if (previous !== undefined) yield* Fiber.interrupt(previous); |
| 1828 | const now = DateTime.toEpochMillis(yield* DateTime.now); |
| 1829 | const fiber = yield* Effect.sleep(Duration.millis(Math.max(0, at - now))).pipe( |
| 1830 | Effect.andThen(Queue.offer(idleFlushTicks, { type: "idle_flush", at })), |
| 1831 | Effect.asVoid, |
| 1832 | Effect.forkDetach, |
| 1833 | ); |
| 1834 | idleFlushWake = { at, fiber }; |
| 1835 | }); |
| 1836 | }); |
| 1837 | const closeIdleFlush = Effect.suspend(() => { |
| 1838 | idleFlushClosed = true; |
| 1839 | const wake = idleFlushWake; |
| 1840 | idleFlushWake = undefined; |
| 1841 | return wake === undefined ? Effect.void : Fiber.interrupt(wake.fiber); |
| 1842 | }); |
| 1843 | // Per-tick ingestion avoids adding a span for every idle wake. |
| 1844 | const flushIdleAssistantText = Effect.fnUntraced(function* (wokeAt: number) { |
| 1845 | // A tick means its timer slept until `wokeAt`, but the wall clock can still read up |
| 1846 | // to a millisecond earlier; trusting it would keep the finished wake and never re-arm. |
| 1847 | const now = Math.max(DateTime.toEpochMillis(yield* DateTime.now), wokeAt); |
| 1848 | // A wake due by now has fired; a later deadline needs a new one. |
| 1849 | if (idleFlushWake !== undefined && idleFlushWake.at <= now) idleFlushWake = undefined; |
| 1850 | if (idleFlushClosed) return; |
| 1851 | const rootSettled = yield* Ref.get(rootRunFinalized); |
| 1852 | for (const event of assistantStreaming.takeIdleFlushes(now)) { |
| 1853 | // Settling the root run closed its messages; only children may still stream. |
| 1854 | if (rootSettled && streamedThreadId(event) === input.run.threadId) continue; |
| 1855 | // The held text's child may have been handed to another run since it arrived. |
| 1856 | if (!routeProviderEvent(event, routeIdentity, yield* Ref.get(eventRouting))[0]) |
| 1857 | continue; |
| 1858 | // A failed write fails ingestion like any provider write. |
| 1859 | yield* ingestDelivered(event, event); |
| 1860 | yield* trackChildLifecycle(event, true); |
| 1861 | } |
| 1862 | yield* scheduleIdleFlush; |
| 1863 | }); |
| 1864 | const ingestEvent = Effect.fnUntraced(function* ({ |
| 1865 | event, |
| 1866 | accepted, |
| 1867 | }: { |
| 1868 | event: ProviderAdapter.ProviderAdapterV2Event; |
| 1869 | accepted: boolean; |
| 1870 | }) { |
| 1871 | if (!accepted) return; |
| 1872 | if ( |
| 1873 | event.type === "provider_turn.updated" && |
| 1874 | event.providerTurn.runAttemptId === input.attempt.id && |
| 1875 | event.providerTurn.providerThreadId === input.providerThread.id |
| 1876 | ) { |
| 1877 | yield* Ref.set(latestRootProviderTurn, event.providerTurn); |
| 1878 | } |
| 1879 | let storedEventCount = 0; |
| 1880 | const deliveredEvent = assistantStreaming.filter( |
| 1881 | event, |
| 1882 | DateTime.toEpochMillis(yield* DateTime.now), |
| 1883 | ); |
| 1884 | const startedRootTurn = |
| 1885 | event.type === "provider_turn.updated" && |
| 1886 | event.providerTurn.runAttemptId === input.attempt.id && |
| 1887 | event.providerTurn.providerThreadId === input.providerThread.id && |
| 1888 | !isTerminalProviderTurnStatus(event.providerTurn.status) && |
| 1889 | !rootTurnOwnershipChecked |
| 1890 | ? event.providerTurn |
| 1891 | : undefined; |
| 1892 | const shouldStartProviderTurn = input.shouldStartProviderTurn; |
| 1893 | if (deliveredEvent && startedRootTurn !== undefined && shouldStartProviderTurn) { |
| 1894 | rootTurnOwnershipChecked = true; |
| 1895 | // Stop plans under this lock from stored turns. Storing the first turn and rechecking |
| 1896 | // the attempt under it means Stop either saw the turn or already settled the run. |
| 1897 | const [stored, owned] = yield* threadCommands.withLock( |
| 1898 | input.run.threadId, |
| 1899 | Effect.all([ |
| 1900 | ingestDelivered(event, deliveredEvent), |
| 1901 | shouldStartProviderTurn().pipe(Effect.orElseSucceed(() => true)), |
| 1902 | ]), |
| 1903 | ); |
| 1904 | storedEventCount = stored; |
| 1905 | if (!owned) { |
| 1906 | yield* Ref.get(latestProviderThread).pipe( |
| 1907 | Effect.flatMap((providerThread) => |
| 1908 | input.session.interruptTurn({ |
| 1909 | providerThread, |
| 1910 | providerTurnId: startedRootTurn.id, |
| 1911 | }), |
| 1912 | ), |
| 1913 | Effect.catchCauseIf( |
| 1914 | (cause) => !Cause.hasInterruptsOnly(cause), |
| 1915 | (cause) => |
| 1916 | Effect.logWarning( |
| 1917 | "orchestration V2 failed to interrupt a provider turn that started after Stop", |
| 1918 | { runId: input.run.id, cause }, |
| 1919 | ), |
| 1920 | ), |
| 1921 | Effect.forkIn(scope), |
| 1922 | ); |
| 1923 | } |
| 1924 | } else if (deliveredEvent) { |
| 1925 | storedEventCount = yield* ingestDelivered(event, deliveredEvent); |
| 1926 | } else if ( |
| 1927 | event.type === "turn_item.updated" && |
| 1928 | (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") && |
| 1929 | event.turnItem.streaming && |
| 1930 | !reservedHeldTurnItemIds.has(event.turnItem.id) |
| 1931 | ) { |
| 1932 | yield* turnItemPositions.allocate({ |
| 1933 | threadId: event.turnItem.threadId, |
| 1934 | turnItemId: event.turnItem.id, |
| 1935 | runId: event.turnItem.runId, |
| 1936 | }); |
| 1937 | reservedHeldTurnItemIds.add(event.turnItem.id); |
| 1938 | } |
| 1939 | if (event.type === "provider_thread.updated") { |
| 1940 | if (event.providerThread.id === input.providerThread.id && storedEventCount > 0) { |
| 1941 | yield* Ref.set(latestProviderThread, event.providerThread); |
| 1942 | rootSnapshotPendingWork = |
| 1943 | event.pendingBackgroundWork === true || |
| 1944 | (event.providerThread.pendingBackgroundTasks?.length ?? 0) > 0; |
| 1945 | } else if (event.providerThread.id !== input.providerThread.id) { |
| 1946 | if (event.pendingBackgroundWork === true) { |
| 1947 | childSnapshotPendingWork.add(event.providerThread.id); |
| 1948 | } else if (event.pendingBackgroundWork === false) { |
| 1949 | childSnapshotPendingWork.delete(event.providerThread.id); |
| 1950 | } |
| 1951 | } |
| 1952 | } |
| 1953 | if ( |
| 1954 | event.type === "turn_item.updated" && |
| 1955 | event.turnItem.providerTurnId === (yield* Ref.get(eventRouting)).rootProviderTurnId |
| 1956 | ) { |
| 1957 | yield* Ref.update(latestTurnItemOrdinal, (current) => |
| 1958 | Math.max(current, event.turnItem.ordinal), |
| 1959 | ); |
| 1960 | } |
| 1961 | if (event.type === "turn.terminal") { |
| 1962 | yield* Ref.set(terminalEvent, event); |
| 1963 | yield* Ref.set(rootTerminalSeen, true); |
| 1964 | yield* finalizeRootRun(event); |
| 1965 | } |
| 1966 | if (event.type === "background_work.updated") { |
| 1967 | if (event.pending) { |
| 1968 | pendingBackgroundProviderTurns.add(event.providerTurnId); |
| 1969 | } else { |
| 1970 | pendingBackgroundProviderTurns.delete(event.providerTurnId); |
| 1971 | } |
| 1972 | } |
| 1973 | yield* trackChildLifecycle(event, deliveredEvent !== null); |
| 1974 | yield* scheduleIdleFlush; |
| 1975 | }); |
| 1976 | const providerEventFiber = yield* eventSink |
| 1977 | .consumeProviderEvents( |
| 1978 | eventSubscription.events.pipe( |
| 1979 | Stream.merge(Stream.fromQueue(idleFlushTicks), { haltStrategy: "left" }), |
| 1980 | ), |
| 1981 | (event) => |
| 1982 | event.type === "idle_flush" |
| 1983 | ? flushIdleAssistantText(event.at) |
| 1984 | : Effect.gen(function* () { |
| 1985 | yield* ingestEvent(yield* routeEvent(event)); |
| 1986 | }), |
| 1987 | Effect.gen(function* () { |
| 1988 | if (yield* agentTurnsSettled) yield* endPreviewActivity; |
| 1989 | return yield* shouldStopProviderEventIngestion; |
| 1990 | }), |
| 1991 | Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)), |
| 1992 | { threadId: input.run.threadId, delegated: isDelegatedAppThread(input.appThread) }, |
| 1993 | ) |
| 1994 | .pipe( |
| 1995 | Effect.ensuring(closeIdleFlush), |
| 1996 | Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })), |
| 1997 | Effect.flatMap(() => |
| 1998 | Effect.gen(function* () { |
| 1999 | const terminal = yield* Ref.get(terminalEvent); |
| 2000 | if (terminal === null) { |
| 2001 | // A session release interrupts or fails the stream; a clean |
| 2002 | // end without a terminal means the run would never settle. |
| 2003 | return yield* new RunExecutionIngestError({ |
| 2004 | runId: input.run.id, |
| 2005 | cause: new ProviderAdapter.ProviderAdapterEventStreamError({ |
| 2006 | driver: input.providerThread.driver, |
| 2007 | providerSessionId: input.providerSessionId, |
| 2008 | raisedByT3: true, |
| 2009 | cause: "Provider event stream ended without a root terminal.", |
| 2010 | }), |
| 2011 | }); |
| 2012 | } |
| 2013 | yield* finalizeRootRun(terminal); |
| 2014 | }), |
| 2015 | ), |
| 2016 | // Server shutdown interrupts the subscription and leaves the run to |
| 2017 | // restart recovery; every other failure fails the run. |
| 2018 | Effect.catchCauseIf( |
| 2019 | (cause) => !Cause.hasInterruptsOnly(cause), |
| 2020 | (cause) => |
| 2021 | Effect.gen(function* () { |
| 2022 | const completedAt = yield* DateTime.now; |
| 2023 | const routing = yield* Ref.get(eventRouting); |
| 2024 | const rootProviderTurn = yield* Ref.get(latestRootProviderTurn); |
| 2025 | const squashed = Cause.squash(cause); |
| 2026 | // An exclusive session closes when its thread detaches, and no terminal |
| 2027 | // follows. The process is gone, so a handoff settles here with no interrupt. |
| 2028 | if ( |
| 2029 | isRunExecutionIngestError(squashed) && |
| 2030 | isProviderEventStreamError(squashed.cause) && |
| 2031 | squashed.cause.detachedThreadId === input.run.threadId && |
| 2032 | (yield* workspaceHandedOff) |
| 2033 | ) { |
| 2034 | const handedOff = yield* finalizeRootRun( |
| 2035 | { |
| 2036 | type: "turn.terminal", |
| 2037 | driver: input.providerThread.driver, |
| 2038 | providerThreadId: input.providerThread.id, |
| 2039 | providerTurnId: routing.rootProviderTurnId ?? failedProviderTurnId, |
| 2040 | runOrdinal: input.run.ordinal, |
| 2041 | status: "interrupted", |
| 2042 | failure: null, |
| 2043 | threadDisposition: "reusable", |
| 2044 | }, |
| 2045 | rootProviderTurn === null || |
| 2046 | isTerminalProviderTurnStatus(rootProviderTurn.status) |
| 2047 | ? undefined |
| 2048 | : { ...rootProviderTurn, status: "interrupted", completedAt }, |
| 2049 | ).pipe( |
| 2050 | Effect.as(true), |
| 2051 | Effect.catchCauseIf( |
| 2052 | (settleCause) => !Cause.hasInterruptsOnly(settleCause), |
| 2053 | (settleCause) => |
| 2054 | Effect.logWarning( |
| 2055 | "orchestration V2 failed to settle a run after its workspace handoff", |
| 2056 | { runId: input.run.id, cause: settleCause }, |
| 2057 | ).pipe(Effect.as(false)), |
| 2058 | ), |
| 2059 | ); |
| 2060 | if (handedOff) { |
| 2061 | // Its tool calls, the handoff's own among them, died with that process. |
| 2062 | yield* endWorkLostAfterRootRun(completedAt); |
| 2063 | return; |
| 2064 | } |
| 2065 | } |
| 2066 | yield* Effect.logWarning("orchestration V2 provider event ingestion failed", { |
| 2067 | runId: input.run.id, |
| 2068 | cause, |
| 2069 | }); |
| 2070 | // An adapter fails its stream when its runtime is gone. A failed write, or a session T3 |
| 2071 | // released or saw end, leaves work that can still report or that the releaser settles. |
| 2072 | const runtimeLost = |
| 2073 | isRunExecutionIngestError(squashed) && |
| 2074 | isProviderEventStreamError(squashed.cause) && |
| 2075 | squashed.cause.raisedByT3 !== true; |
| 2076 | if (yield* Ref.get(rootRunFinalized)) { |
| 2077 | if (runtimeLost) yield* endWorkLostAfterRootRun(completedAt); |
| 2078 | return; |
| 2079 | } |
| 2080 | const providerThread = yield* Ref.get(latestProviderThread); |
| 2081 | // Stop background work too; the failed consumer cannot persist terminal events. |
| 2082 | const abandonedProviderTurn = |
| 2083 | routing.rootProviderTurnId !== null && !routing.rootTurnEnded |
| 2084 | ? yield* input.session |
| 2085 | .interruptTurn({ |
| 2086 | providerThread, |
| 2087 | providerTurnId: routing.rootProviderTurnId, |
| 2088 | requestRuntimeRestart: true, |
| 2089 | }) |
| 2090 | .pipe( |
| 2091 | Effect.as(true), |
| 2092 | Effect.catchCauseIf( |
| 2093 | (interruptCause) => !Cause.hasInterruptsOnly(interruptCause), |
| 2094 | (interruptCause) => |
| 2095 | Effect.logWarning( |
| 2096 | "orchestration V2 failed to interrupt turn after ingestion failure", |
| 2097 | { runId: input.run.id, cause: interruptCause }, |
| 2098 | ).pipe(Effect.as(false)), |
| 2099 | ), |
| 2100 | // A failed interrupt leaves the stored turn running so the next start |
| 2101 | // retries it. The snapshot is cached before it is published, so it may |
| 2102 | // not be stored: replay a terminal one as-is. |
| 2103 | Effect.map((interrupted) => |
| 2104 | !interrupted || rootProviderTurn === null |
| 2105 | ? null |
| 2106 | : isTerminalProviderTurnStatus(rootProviderTurn.status) |
| 2107 | ? rootProviderTurn |
| 2108 | : { |
| 2109 | ...rootProviderTurn, |
| 2110 | status: "interrupted" as const, |
| 2111 | completedAt, |
| 2112 | }, |
| 2113 | ), |
| 2114 | ) |
| 2115 | : null; |
| 2116 | const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal); |
| 2117 | const openSubagents = yield* Ref.get(openRunOwnedSubagents); |
| 2118 | const lostToolCalls = runtimeLost |
| 2119 | ? yield* loadLostRootToolCalls.pipe( |
| 2120 | Effect.catchCauseIf( |
| 2121 | (loadCause) => !Cause.hasInterruptsOnly(loadCause), |
| 2122 | (loadCause) => |
| 2123 | Effect.logWarning( |
| 2124 | "orchestration V2 failed to read tool calls lost with the provider runtime", |
| 2125 | { runId: input.run.id, cause: loadCause }, |
| 2126 | ).pipe(Effect.as([])), |
| 2127 | ), |
| 2128 | ) |
| 2129 | : []; |
| 2130 | let nextRetryWarningAt = completedAt; |
| 2131 | yield* writeFinalRunEvents({ |
| 2132 | run: input.run, |
| 2133 | rootNode: input.rootNode, |
| 2134 | checkpointScope: input.checkpointScope, |
| 2135 | providerThread, |
| 2136 | attempt: input.attempt, |
| 2137 | writeIfRunCurrent: { |
| 2138 | activeAttemptId: input.attempt.id, |
| 2139 | expectedStatus: "running", |
| 2140 | }, |
| 2141 | openRunOwnedSubagents: openSubagents, |
| 2142 | ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), |
| 2143 | lostToolCalls, |
| 2144 | completedAt, |
| 2145 | terminal: makeFailedTerminalEvent( |
| 2146 | makeProviderFailure({ |
| 2147 | cause: squashed, |
| 2148 | message: |
| 2149 | knownProviderFailureMessage(squashed) ?? |
| 2150 | userFacingDatabaseErrorMessage(squashed), |
| 2151 | class: "unknown", |
| 2152 | }), |
| 2153 | latestItemOrdinal + 1, |
| 2154 | ), |
| 2155 | failureItemPersisted: false, |
| 2156 | refreshAfterTurn, |
| 2157 | }).pipe( |
| 2158 | // Storage failures can be defects; retry both failure channels. |
| 2159 | Effect.catchCause((writeCause) => |
| 2160 | Effect.gen(function* () { |
| 2161 | if (Cause.hasInterruptsOnly(writeCause)) { |
| 2162 | return yield* Effect.failCause(writeCause); |
| 2163 | } |
| 2164 | const now = yield* DateTime.now; |
| 2165 | if (DateTime.isGreaterThanOrEqualTo(now, nextRetryWarningAt)) { |
| 2166 | yield* Effect.logWarning( |
| 2167 | "orchestration V2 run failure finalization retry", |
| 2168 | { |
| 2169 | runId: input.run.id, |
| 2170 | attemptId: input.attempt.id, |
| 2171 | cause: writeCause, |
| 2172 | }, |
| 2173 | ); |
| 2174 | // Persistent storage failures must not flood the same disk. |
| 2175 | nextRetryWarningAt = DateTime.add(now, { minutes: 10 }); |
| 2176 | } |
| 2177 | return yield* Effect.fail(writeCause); |
| 2178 | }), |
| 2179 | ), |
| 2180 | Effect.retry({ |
| 2181 | schedule: Schedule.exponential("1 second").pipe( |
| 2182 | Schedule.modifyDelay(({ duration }) => |
| 2183 | Effect.succeed(Duration.min(duration, Duration.seconds(30))), |
| 2184 | ), |
| 2185 | ), |
| 2186 | }), |
| 2187 | // Startup may cancel the consumer and fail its own terminal |
| 2188 | // write. Only service shutdown or the atomic gate ends recovery. |
| 2189 | Effect.forkIn(scope), |
| 2190 | Effect.flatMap(Fiber.join), |
| 2191 | ); |
| 2192 | }), |
| 2193 | ), |
| 2194 | Effect.ensuring(endPreviewActivity), |
| 2195 | Effect.ensuring(eventSubscription.close), |
| 2196 | Effect.forkDetach, |
| 2197 | ); |
| 2198 | |
| 2199 | // A provider turn is a sign that the session is still alive. Keep |
| 2200 | // its already-issued MCP credential valid even when the agent goes |
| 2201 | // a long time between browser-tool calls. |
| 2202 | const touchMcpCredential = McpSessionRegistry.touchActiveMcpThread(input.run.threadId).pipe( |
| 2203 | Effect.ignoreCause({ |
| 2204 | log: true, |
| 2205 | message: "Failed to refresh MCP credential liveness.", |
| 2206 | }), |
| 2207 | ); |
| 2208 | yield* touchMcpCredential; |
| 2209 | const appContext = yield* McpAppModelContext.forProvider( |
| 2210 | mcpAppModelContext, |
| 2211 | input.run.threadId, |
| 2212 | ); |
| 2213 | const turnInput = { |
| 2214 | appThread: input.appThread, |
| 2215 | threadId: input.run.threadId, |
| 2216 | runId: input.run.id, |
| 2217 | runOrdinal: input.run.ordinal, |
| 2218 | providerTurnOrdinal: input.providerTurnOrdinal, |
| 2219 | ...(input.nativeThreadHasTurns === undefined |
| 2220 | ? {} |
| 2221 | : { nativeThreadHasTurns: input.nativeThreadHasTurns }), |
| 2222 | ...(input.run.restartContinuationOfRunId === undefined |
| 2223 | ? {} |
| 2224 | : { |
| 2225 | restartContinuationOfRunId: input.run.restartContinuationOfRunId, |
| 2226 | }), |
| 2227 | attemptId: input.attemptId, |
| 2228 | rootNodeId: input.rootNode.id, |
| 2229 | providerThread: input.providerThread, |
| 2230 | message: input.message, |
| 2231 | modelSelection: input.modelSelection, |
| 2232 | runtimePolicy: input.runtimePolicy, |
| 2233 | ...(appContext.length === 0 ? {} : { appContext }), |
| 2234 | }; |
| 2235 | // Check after the context read: ingestion can settle the run while |
| 2236 | // it awaits. A failed ownership read fails the start below. |
| 2237 | const shouldStart = |
| 2238 | input.shouldStartProviderTurn === undefined |
| 2239 | ? Exit.succeed(true) |
| 2240 | : yield* Effect.exit(input.shouldStartProviderTurn()); |
| 2241 | if (Exit.isSuccess(shouldStart) && !shouldStart.value) { |
| 2242 | yield* Fiber.interrupt(providerEventFiber); |
| 2243 | return; |
| 2244 | } |
| 2245 | |
| 2246 | const compact = |
| 2247 | input.message.attachments.length === 0 && |
| 2248 | input.message.text.trim().toLowerCase() === "/compact"; |
| 2249 | const { message: _message, ...resumeInput } = turnInput; |
| 2250 | const startTurn = |
| 2251 | input.resume === true |
| 2252 | ? (input.session.resumeTurn?.({ ...resumeInput, messageId: input.message.messageId }) ?? |
| 2253 | Effect.fail( |
| 2254 | new ProviderAdapter.ProviderAdapterTurnStartError({ |
| 2255 | driver: input.session.driver, |
| 2256 | threadId: input.run.threadId, |
| 2257 | providerThreadId: input.providerThread.id, |
| 2258 | runId: input.run.id, |
| 2259 | cause: new ProviderAdapter.ProviderResumeUnsupportedError({ |
| 2260 | driver: input.session.driver, |
| 2261 | providerThreadId: input.providerThread.id, |
| 2262 | }), |
| 2263 | }), |
| 2264 | )) |
| 2265 | : compact |
| 2266 | ? (input.session.compactThread?.(turnInput) ?? |
| 2267 | Effect.fail( |
| 2268 | new ProviderAdapter.ProviderAdapterTurnStartError({ |
| 2269 | driver: input.session.driver, |
| 2270 | threadId: input.run.threadId, |
| 2271 | providerThreadId: input.providerThread.id, |
| 2272 | runId: input.run.id, |
| 2273 | cause: new ProviderAdapter.ProviderCompactionUnsupportedError({ |
| 2274 | driver: input.session.driver, |
| 2275 | providerThreadId: input.providerThread.id, |
| 2276 | }), |
| 2277 | }), |
| 2278 | )) |
| 2279 | : input.session.startTurn(turnInput); |
| 2280 | yield* Effect.andThen(shouldStart, startTurn).pipe( |
| 2281 | // Wait for acceptance so abandoned startup cannot leave a refresher |
| 2282 | // on the already-detached ingestion fiber. |
| 2283 | Effect.tap(() => |
| 2284 | Fiber.await(providerEventFiber).pipe( |
| 2285 | Effect.raceFirst(touchMcpCredential.pipe(Effect.delay("1 hour"), Effect.forever)), |
| 2286 | Effect.forkDetach, |
| 2287 | ), |
| 2288 | ), |
| 2289 | Effect.catchCause((cause) => |
| 2290 | Effect.logError("orchestration V2 provider turn start failed", { |
| 2291 | runId: input.run.id, |
| 2292 | cause, |
| 2293 | }).pipe( |
| 2294 | Effect.andThen(Fiber.interrupt(providerEventFiber)), |
| 2295 | Effect.andThen(Ref.get(latestProviderThread)), |
| 2296 | Effect.flatMap((providerThread) => |
| 2297 | Ref.get(latestTurnItemOrdinal).pipe( |
| 2298 | Effect.flatMap((latestItemOrdinal) => |
| 2299 | Ref.get(openRunOwnedSubagents).pipe( |
| 2300 | Effect.flatMap((openSubagents) => |
| 2301 | writeFinalRunEvents({ |
| 2302 | run: input.run, |
| 2303 | rootNode: input.rootNode, |
| 2304 | checkpointScope: input.checkpointScope, |
| 2305 | providerThread, |
| 2306 | attempt: input.attempt, |
| 2307 | writeIfRunCurrent: { |
| 2308 | activeAttemptId: input.attempt.id, |
| 2309 | expectedStatus: "running", |
| 2310 | }, |
| 2311 | openRunOwnedSubagents: openSubagents, |
| 2312 | terminal: makeFailedTerminalEvent( |
| 2313 | makeProviderFailure({ |
| 2314 | cause: Cause.squash(cause), |
| 2315 | class: Exit.isFailure(shouldStart) ? "unknown" : "provider_error", |
| 2316 | }), |
| 2317 | latestItemOrdinal + 1, |
| 2318 | ), |
| 2319 | failureItemPersisted: false, |
| 2320 | refreshAfterTurn, |
| 2321 | }), |
| 2322 | ), |
| 2323 | ), |
| 2324 | ), |
| 2325 | ), |
| 2326 | ), |
| 2327 | Effect.mapError( |
| 2328 | (writeCause) => |
| 2329 | new RunExecutionStartError({ |
| 2330 | commandId: input.commandId, |
| 2331 | runId: input.run.id, |
| 2332 | cause: { start: cause, write: writeCause }, |
| 2333 | }), |
| 2334 | ), |
| 2335 | ), |
| 2336 | ), |
| 2337 | ); |
| 2338 | }), |
| 2339 | } satisfies RunExecutionServiceV2Shape); |
| 2340 | }), |
| 2341 | ); |
| 2342 | |
| 2343 | /** |
| 2344 | * Clients send Stop without a reason; server callers acting for an agent |
| 2345 | * always set one, so a missing reason means a person stopped the run. |
| 2346 | */ |
| 2347 | export const USER_STOP_REASON = "Stopped by user"; |
| 2348 | |
| 2349 | export function makeInterruptResultTurnItem(input: { |
| 2350 | readonly idAllocator: IdAllocator.IdAllocatorV2["Service"]; |
| 2351 | readonly run: OrchestrationV2Run; |
| 2352 | readonly rootNode: OrchestrationV2ExecutionNode; |
| 2353 | readonly providerThread: OrchestrationV2ProviderThread; |
| 2354 | readonly reason: string | undefined; |
| 2355 | readonly completedAt: DateTime.Utc; |
| 2356 | }): OrchestrationV2TurnItem { |
| 2357 | return { |
| 2358 | id: input.idAllocator.derive.runSignalTurnItem({ |
| 2359 | runId: input.run.id, |
| 2360 | signal: "interrupt-result", |
| 2361 | }), |
| 2362 | threadId: input.run.threadId, |
| 2363 | runId: input.run.id, |
| 2364 | nodeId: input.rootNode.id, |
| 2365 | providerThreadId: input.providerThread.id, |
| 2366 | providerTurnId: input.rootNode.providerTurnId, |
| 2367 | nativeItemRef: null, |
| 2368 | parentItemId: input.idAllocator.derive.runSignalTurnItem({ |
| 2369 | runId: input.run.id, |
| 2370 | signal: "interrupt-request", |
| 2371 | }), |
| 2372 | ordinal: input.run.ordinal * 100 + 98, |
| 2373 | status: "interrupted", |
| 2374 | title: "Interrupted", |
| 2375 | startedAt: input.completedAt, |
| 2376 | completedAt: input.completedAt, |
| 2377 | updatedAt: input.completedAt, |
| 2378 | type: "run_interrupt_result", |
| 2379 | message: input.reason ?? "Stopped by the provider", |
| 2380 | }; |
| 2381 | } |