@devries-cameron20 / t3code-1 / commits / 51cdf23265
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): work a dead provider process was running no longer stays Running (#37325)

Julius Marminge committed Nov 15, 2026 at 11:48 UTC 51cdf232652db8cdcea806f9a9e27d7460868579
7 files changed +451 -18
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
+2 -16
@@ -6354,7 +6354,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
6354 }).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)));
6355 });
6356
6357 - it.effect("Stop ends a background command no Codex process tracks any more", () =>
6357 + it.effect("ends a background command when its Codex process is gone", () =>
6358 Effect.gen(function* () {
6359 const fs = yield* FileSystem.FileSystem;
6360 const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-bg-stale-workspace-" });
@@ -6429,21 +6429,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
6429 yield* worker.drain();
6430 yield* Fiber.join(settled);
6431 yield* worker.drain();
6432 - const before = yield* orchestrator.getThreadShell(threadId);
6433 - assert.deepEqual(
6434 - before?.pendingBackgroundTasks?.map((task) => task.kind),
6435 - ["command"],
6436 - "the thread still shows the command the gone process never finished",
6437 - );
6438 - const run = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
6439 - yield* orchestrator.dispatch({
6440 - type: "run.interrupt",
6441 - commandId: CommandId.make("stop-background-untracked"),
6442 - threadId,
6443 - runId: run.id,
6444 - holdQueue: true,
6445 - });
6446 - yield* worker.drain();
6432 + // regression: #37290 the command stayed running, and the thread waiting, until Stop.
6433 const projection = yield* orchestrator.getThreadProjection(threadId);
6434 assert.deepEqual(
6435 projection.turnItems.flatMap((item) =>
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+8
@@ -3485,6 +3485,14 @@ it.effect.each(["detach", "stream failure", "provider stop", "early close"] as c
3485 : undefined,
3486 release === "detach" ? threadId : undefined,
3487 );
3488 + // A run ends the work it tracks only when the adapter reported its runtime failing.
3489 + assert.equal(
3490 + Exit.isFailure(consumed)
3491 + ? (Cause.squash(consumed.cause) as ProviderAdapter.ProviderAdapterEventStreamError)
3492 + .raisedByT3
3493 + : undefined,
3494 + Exit.isFailure(consumed) ? true : undefined,
3495 + );
3496 yield* interrupts(2);
3497
3498 assert.isTrue(Option.isNone(yield* manager.get(providerSessionId)));
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+3
@@ -667,6 +667,7 @@ export const layerWithOptions = (
667 const error = new ProviderAdapter.ProviderAdapterEventStreamError({
668 driver: entry.runtime.driver,
669 providerSessionId: entry.runtime.providerSessionId,
670 + raisedByT3: true,
671 ...(detachedThreadId === undefined ? {} : { detachedThreadId }),
672 cause: detail,
673 });
@@ -2857,6 +2858,7 @@ export const layerWithOptions = (
2858 new ProviderAdapter.ProviderAdapterEventStreamError({
2859 driver: entry.runtime.driver,
2860 providerSessionId: entry.runtime.providerSessionId,
2861 + raisedByT3: true,
2862 cause,
2863 }),
2864 ),
@@ -2897,6 +2899,7 @@ export const layerWithOptions = (
2899 new ProviderAdapter.ProviderAdapterEventStreamError({
2900 driver: entry.runtime.driver,
2901 providerSessionId: entry.runtime.providerSessionId,
2902 + raisedByT3: true,
2903 cause: "Provider event stream ended unexpectedly.",
2904 }),
2905 );
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+14
@@ -258,6 +258,19 @@ export const layer: Layer.Layer<
258 Effect.map((current) => current.thread.worktreePath !== input.worktreePath),
259 Effect.catchCause(() => Effect.succeed(false)),
260 ),
261 + loadUnsettledWork: (threadId: ThreadId, nodeIds: ReadonlyArray<NodeId>) =>
262 + projectionStore
263 + .getThreadRecords(threadId, ["subagents", "nodes", "turnItems"], {
264 + nodeIds,
265 + subagentIds: nodeIds,
266 + turnItemStatuses: ["pending", "running", "waiting"],
267 + })
268 + .pipe(
269 + Effect.catchTags({
270 + ProjectionStoreThreadNotFoundError: () =>
271 + Effect.succeed({ subagents: [], nodes: [], turnItems: [] }),
272 + }),
273 + ),
274 loadRunInterruptReason: () =>
275 projectionStore
276 .getThreadRecords(input.threadId, ["turnItems"], {
@@ -1851,6 +1864,7 @@ export const layer: Layer.Layer<
1864 hasUnpairedRunInterruptRequest: runControls.hasUnpairedRunInterruptRequest,
1865 loadRunInterruptReason: runControls.loadRunInterruptReason,
1866 hasWorkspaceMoved: runControls.hasWorkspaceMoved,
1867 + loadUnsettledWork: runControls.loadUnsettledWork,
1868 message: {
1869 messageId: message.id,
1870 text: userText,
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+207 -1
@@ -3521,6 +3521,168 @@ it.effect("does not interrupt an ended provider when its terminal write fails",
3521 }),
3522 );
3523
3524 +it.effect("ends a running root command when the provider stream fails mid-turn", () =>
3525 + Effect.gen(function* () {
3526 + const now = yield* DateTime.now;
3527 + const { written, observed } = yield* captureRootRunTermination({
3528 + key: "stream-error-running-command",
3529 + shouldFinalizeRun: () => Effect.succeed(true),
3530 + events: (ids) =>
3531 + Stream.make(
3532 + {
3533 + type: "provider_turn.updated" as const,
3534 + driver,
3535 + providerTurn: {
3536 + id: ids.rootProviderTurnId,
3537 + providerThreadId: ids.providerThreadId,
3538 + nodeId: ids.rootNodeId,
3539 + runAttemptId: ids.attemptId,
3540 + nativeTurnRef: null,
3541 + ordinal: 1,
3542 + status: "running" as const,
3543 + startedAt: now,
3544 + completedAt: null,
3545 + },
3546 + },
3547 + backgroundTurnItemEvent(ids, "command_execution", "running", 1),
3548 + ).pipe(
3549 + Stream.concat(
3550 + Stream.fail(
3551 + new ProviderAdapter.ProviderAdapterEventStreamError({
3552 + driver,
3553 + providerSessionId: ProviderSessionId.make("session:stream-error-running-command"),
3554 + cause: "provider process exited",
3555 + }),
3556 + ),
3557 + ),
3558 + ),
3559 + // The dead process cannot take the interrupt either.
3560 + interruptTurn: (input) =>
3561 + Effect.fail(
3562 + new ProviderAdapter.ProviderAdapterInterruptError({
3563 + driver,
3564 + providerThreadId: input.providerThread.id,
3565 + providerTurnId: input.providerTurnId,
3566 + cause: "provider already exited",
3567 + }),
3568 + ),
3569 + });
3570 + assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3571 + const ids = backgroundScenarioIds("stream-error-running-command");
3572 + const endedCommand = written.find(
3573 + (item) => item.id === ids.itemId && item.status !== "running",
3574 + );
3575 + assert.isDefined(endedCommand);
3576 + }),
3577 +);
3578 +
3579 +it.effect("keeps a running root command open when a write fails under a live provider", () =>
3580 + Effect.gen(function* () {
3581 + const key = "write-error-running-command";
3582 + const ids = backgroundScenarioIds(key);
3583 + const secondItemId = TurnItemId.make(`${ids.itemId}:second`);
3584 + const { written, observed } = yield* captureRootRunTermination({
3585 + key,
3586 + shouldFinalizeRun: () => Effect.succeed(true),
3587 + events: () =>
3588 + Stream.make(
3589 + backgroundTurnItemEvent(ids, "command_execution", "running", 1),
3590 + backgroundTurnItemEvent(ids, "command_execution", "running", 2, secondItemId),
3591 + ),
3592 + ingestNormalized: ({ event }) =>
3593 + event.type === "turn_item.updated" && event.turnItem.id === secondItemId
3594 + ? Effect.fail(
3595 + new ProviderEventIngestor.ProviderEventPublishError({
3596 + providerSessionId: ProviderSessionId.make(`session:${key}`),
3597 + eventCount: 1,
3598 + cause: new Error("SQLITE_FULL: database or disk is full"),
3599 + }),
3600 + )
3601 + : Effect.succeed([]),
3602 + });
3603 + assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3604 + assert.isUndefined(written.find((item) => item.id === ids.itemId));
3605 + }),
3606 +);
3607 +
3608 +it.effect.each([
3609 + { ended: true },
3610 + // T3 ending the stream proves no dead runtime; Stop settles the work it releases.
3611 + { ended: false, raisedByT3: true },
3612 + // A Stop that already settled the subagent keeps its result.
3613 + { ended: false, stopped: true },
3614 +] as const)(
3615 + "ends a subagent that outlived its root when the provider stream fails (%o)",
3616 + ({ ended, raisedByT3, stopped }) =>
3617 + Effect.gen(function* () {
3618 + const providerInstanceId = ProviderInstanceId.make("codex");
3619 + const { finalEvents } = yield* captureRootRunTermination({
3620 + key: "stream-error-outlived-subagent",
3621 + shouldFinalizeRun: () => Effect.succeed(true),
3622 + events: (ids) =>
3623 + Stream.make(
3624 + {
3625 + type: "subagent.updated" as const,
3626 + driver,
3627 + subagent: makeRunOwnedSubagentFixture({
3628 + ids,
3629 + providerInstanceId,
3630 + childThreadId: ids.childThreadId,
3631 + driver,
3632 + status: "running",
3633 + }),
3634 + },
3635 + {
3636 + type: "turn_item.updated" as const,
3637 + driver,
3638 + turnItem: makeRunOwnedSubagentTurnItemFixture({
3639 + ids,
3640 + providerInstanceId,
3641 + childThreadId: ids.childThreadId,
3642 + driver,
3643 + status: "running",
3644 + }),
3645 + },
3646 + rootTerminalEvent(ids, "completed"),
3647 + ).pipe(
3648 + Stream.concat(
3649 + Stream.fail(
3650 + new ProviderAdapter.ProviderAdapterEventStreamError({
3651 + driver,
3652 + providerSessionId: ProviderSessionId.make(
3653 + "session:stream-error-outlived-subagent",
3654 + ),
3655 + ...(raisedByT3 === undefined ? {} : { raisedByT3 }),
3656 + cause: "provider process exited",
3657 + }),
3658 + ),
3659 + ),
3660 + ),
3661 + ...(stopped === true
3662 + ? {
3663 + settleStored: (work) => ({
3664 + ...work,
3665 + subagents: work.subagents.map((row) => ({ ...row, status: "interrupted" })),
3666 + turnItems: work.turnItems.map((row) => ({ ...row, status: "interrupted" })),
3667 + }),
3668 + }
3669 + : {}),
3670 + });
3671 + assert.deepEqual(
3672 + finalEvents.flatMap((event) =>
3673 + event.type === "run.updated" ||
3674 + event.type === "subagent.updated" ||
3675 + event.type === "turn-item.updated"
3676 + ? [`${event.type}:${event.payload.status}`]
3677 + : [],
3678 + ),
3679 + ended
3680 + ? ["run.updated:waiting", "subagent.updated:failed", "turn-item.updated:failed"]
3681 + : ["run.updated:waiting"],
3682 + );
3683 + }),
3684 +);
3685 +
3686 it.effect("does not interrupt a provider when ingestion fails before the root turn is known", () =>
3687 Effect.gen(function* () {
3688 const interrupts = yield* Ref.make(0);
@@ -3783,9 +3945,36 @@ function captureRootRunTermination(input: {
3945 readonly afterStart?: (
3946 closed: Deferred.Deferred<void>,
3947 ) => Effect.Effect<void, never, PreviewAgentActivity.PreviewAgentActivity>;
3948 + /** Changes the stored rows after the provider reported them, as a concurrent Stop does. */
3949 + readonly settleStored?: (
3950 + work: RunExecutionService.UnsettledWork,
3951 + ) => RunExecutionService.UnsettledWork;
3952 }) {
3953 return Effect.gen(function* () {
3954 const ids = backgroundScenarioIds(input.key);
3955 + // The rows the provider's events stored, which the run reads back when its runtime is lost.
3956 + const stored = {
3957 + subagents: new Map<string, OrchestrationV2Subagent>(),
3958 + nodes: new Map<string, OrchestrationV2ExecutionNode>(),
3959 + turnItems: new Map<string, OrchestrationV2TurnItem>(),
3960 + };
3961 + const store = (event: ProviderAdapter.ProviderAdapterV2Event) =>
3962 + Effect.sync(() => {
3963 + if (event.type === "subagent.updated")
3964 + stored.subagents.set(event.subagent.id, event.subagent);
3965 + if (event.type === "node.updated") stored.nodes.set(event.node.id, event.node);
3966 + if (event.type === "turn_item.updated")
3967 + stored.turnItems.set(event.turnItem.id, event.turnItem);
3968 + });
3969 + const loadUnsettledWork = (threadId: ThreadId) =>
3970 + Effect.sync(() => {
3971 + const work = {
3972 + subagents: [...stored.subagents.values()].filter((row) => row.threadId === threadId),
3973 + nodes: [...stored.nodes.values()].filter((row) => row.threadId === threadId),
3974 + turnItems: [...stored.turnItems.values()].filter((row) => row.threadId === threadId),
3975 + };
3976 + return input.settleStored?.(work) ?? work;
3977 + });
3978 const providerInstanceId = ProviderInstanceId.make("codex");
3979 const runningSubagent = makeRunOwnedSubagentFixture({
3980 ids,
@@ -3854,11 +4043,27 @@ function captureRootRunTermination(input: {
4043 yield* captureFinalEvents(payload.events);
4044 return { committed: true, storedEvents: [] };
4045 }),
4046 + // Committed only against the row that was read, as the real conditional writes are.
4047 + writeIfSubagentUnchanged: (payload) =>
4048 + stored.subagents.get(payload.expected.id) === payload.expected
4049 + ? captureFinalEvents(payload.events).pipe(
4050 + Effect.as({ committed: true, storedEvents: [] }),
4051 + )
4052 + : Effect.succeed({ committed: false, storedEvents: [] }),
4053 + writeIfTurnItemUnchanged: (payload) =>
4054 + stored.turnItems.get(payload.expected.id) === payload.expected
4055 + ? captureFinalEvents(payload.events).pipe(
4056 + Effect.as({ committed: true, storedEvents: [] }),
4057 + )
4058 + : Effect.succeed({ committed: false, storedEvents: [] }),
4059 }),
4060 IdAllocator.layer,
4061 ThreadCommandExecutor.layer,
4062 Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
3861 - ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])),
4063 + ingestNormalized: (request) =>
4064 + store(request.event).pipe(
4065 + Effect.andThen(input.ingestNormalized?.(request) ?? Effect.succeed([])),
4066 + ),
4067 }),
4068 ServerSettings.layerTest(),
4069 Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
@@ -3944,6 +4149,7 @@ function captureRootRunTermination(input: {
4149 attemptId: ids.attemptId,
4150 providerTurnOrdinal: 1,
4151 shouldFinalizeRun: input.shouldFinalizeRun,
4152 + loadUnsettledWork,
4153 ...(input.loadRunInterruptReason === undefined
4154 ? {}
4155 : { loadRunInterruptReason: input.loadRunInterruptReason }),
apps/server/src/orchestration-v2/RunExecutionService.ts
+215 -1
@@ -154,6 +154,12 @@ export function selectInheritedBackgroundTurnItems(input: {
154
155 type SubagentTurnItem = Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>;
156
157 +// Tool calls that only a live provider process can end.
158 +const processBoundToolCallTypes: ReadonlySet<OrchestrationV2TurnItem["type"]> = new Set([
159 + "command_execution",
160 + "dynamic_tool",
161 +]);
162 +
163 type OpenRunOwnedSubagentProjection = {
164 readonly subagents: ReadonlyMap<NodeId, OrchestrationV2Subagent>;
165 readonly turnItems: ReadonlyMap<NodeId, SubagentTurnItem>;
@@ -162,6 +168,12 @@ type OpenRunOwnedSubagentProjection = {
168 readonly linkedChildThreadIds: ReadonlySet<ThreadId>;
169 };
170
171 +export type UnsettledWork = {
172 + readonly subagents: ReadonlyArray<OrchestrationV2Subagent>;
173 + readonly nodes: ReadonlyArray<OrchestrationV2ExecutionNode>;
174 + readonly turnItems: ReadonlyArray<OrchestrationV2TurnItem>;
175 +};
176 +
177 type RunOwnedSubagentTerminalStatus = Extract<
178 OrchestrationV2Subagent["status"],
179 "interrupted" | "failed" | "cancelled"
@@ -329,6 +341,38 @@ export const cascadeTerminalizeRunOwnedSubagents = Effect.fn(
341 return events;
342 });
343
344 +const interruptLostToolCalls = Effect.fn("RunExecutionService.interruptLostToolCalls")(
345 + function* (input: {
346 + readonly run: OrchestrationV2Run;
347 + readonly toolCalls: Iterable<OrchestrationV2TurnItem>;
348 + readonly completedAt: DateTime.Utc;
349 + readonly allocateEventId: () => Effect.Effect<
350 + EventId,
351 + IdAllocator.IdAllocatorV2AllocationError
352 + >;
353 + }) {
354 + const events: Array<OrchestrationV2DomainEvent> = [];
355 + for (const toolCall of input.toolCalls) {
356 + events.push({
357 + id: yield* input.allocateEventId(),
358 + type: "turn-item.updated",
359 + threadId: toolCall.threadId,
360 + runId: toolCall.runId ?? input.run.id,
361 + ...(toolCall.nodeId === null ? {} : { nodeId: toolCall.nodeId }),
362 + providerInstanceId: input.run.providerInstanceId,
363 + occurredAt: input.completedAt,
364 + payload: {
365 + ...toolCall,
366 + status: "interrupted",
367 + completedAt: input.completedAt,
368 + updatedAt: input.completedAt,
369 + },
370 + });
371 + }
372 + return events;
373 + },
374 +);
375 +
376 function finalProviderThreadStatus(
377 disposition: ProviderTerminalEvent["threadDisposition"],
378 ): OrchestrationV2ProviderThread["status"] {
@@ -612,6 +656,14 @@ export interface RunExecutionServiceV2StartRootRunInput {
656 readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
657 readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>;
658 readonly hasWorkspaceMoved?: () => Effect.Effect<boolean, never>;
659 + /**
660 + * Stored rows of a thread's unsettled work. Work a dead runtime left is ended from these, since
661 + * Stop may have settled it after the snapshots this run holds.
662 + */
663 + readonly loadUnsettledWork?: (
664 + threadId: ThreadId,
665 + nodeIds: ReadonlyArray<NodeId>,
666 + ) => Effect.Effect<UnsettledWork, ProjectionStoreV2Error>;
667 readonly message: ProviderAdapter.ProviderAdapterV2TurnMessage;
668 readonly modelSelection: ModelSelection;
669 readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy;
@@ -680,6 +732,8 @@ export const layer: Layer.Layer<
732 readonly openRunOwnedSubagentsTerminal?: ProviderTerminalEvent;
733 // Root turn snapshot written with the run when T3 ends the run itself.
734 readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn;
735 + // Stored root tool calls still open when the provider runtime was lost; nothing else ends them.
736 + readonly lostToolCalls?: ReadonlyArray<OrchestrationV2TurnItem>;
737 readonly terminal: ProviderTerminalEvent;
738 readonly completedAt?: DateTime.Utc;
739 readonly failureItemPersisted: boolean;
@@ -810,6 +864,12 @@ export const layer: Layer.Layer<
864 : [],
865 events: [
866 ...cascadedSubagentEvents,
867 + ...(yield* interruptLostToolCalls({
868 + run: input.run,
869 + toolCalls: input.lostToolCalls ?? [],
870 + completedAt,
871 + allocateEventId,
872 + })),
873 ...(input.abandonedProviderTurn === undefined
874 ? []
875 : [
@@ -1150,6 +1210,8 @@ export const layer: Layer.Layer<
1210 ReadonlySet<OrchestrationV2TurnItem["id"]>
1211 >(new Set(inheritedBackgroundTurnItemsById.keys()));
1212 const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection());
1213 + // Open tool calls on the root thread, to end from their stored rows if the runtime is lost.
1214 + const openRootToolCalls = new Set<TurnItemId>();
1215 // The last stored root snapshot's pendingBackgroundWork; adapters that omit it leave it false.
1216 let rootSnapshotPendingWork = false;
1217 // Owned child threads whose last snapshot reported pending work, such as a queued native
@@ -1226,6 +1288,125 @@ export const layer: Layer.Layer<
1288 }
1289 yield* Ref.set(rootRunFinalized, true);
1290 });
1291 + // The stored rows of the work this run still tracks as open on the given threads.
1292 + const loadTrackedWork = Effect.fn("RunExecutionService.loadTrackedWork")(function* (
1293 + threadIds: Iterable<ThreadId>,
1294 + ) {
1295 + const open = yield* Ref.get(openRunOwnedSubagents);
1296 + const subagents = new Map<NodeId, OrchestrationV2Subagent>();
1297 + const nodes = new Map<NodeId, OrchestrationV2ExecutionNode>();
1298 + const turnItems = new Map<TurnItemId, OrchestrationV2TurnItem>();
1299 + if (input.loadUnsettledWork === undefined) return { subagents, nodes, turnItems };
1300 + const trackedItemIds = new Set<TurnItemId>([
1301 + ...openRootToolCalls.keys(),
1302 + ...open.childTurnItems.keys(),
1303 + ...[...open.turnItems.values()].map((item) => item.id),
1304 + ]);
1305 + for (const threadId of new Set(threadIds)) {
1306 + const stored = yield* input.loadUnsettledWork(
1307 + threadId,
1308 + [...open.nodes.values(), ...open.subagents.values()]
1309 + .filter((record) => record.threadId === threadId)
1310 + .map((record) => record.id),
1311 + );
1312 + for (const subagent of stored.subagents) {
1313 + if (open.subagents.has(subagent.id) && !isSettledSubagentStatus(subagent.status)) {
1314 + subagents.set(subagent.id, subagent);
1315 + }
1316 + }
1317 + for (const node of stored.nodes) {
1318 + if (open.nodes.has(node.id) && isOpenExecutionNodeStatus(node.status)) {
1319 + nodes.set(node.id, node);
1320 + }
1321 + }
1322 + for (const item of stored.turnItems) {
1323 + if (trackedItemIds.has(item.id) && !isSettledTurnItemStatus(item.status)) {
1324 + turnItems.set(item.id, item);
1325 + }
1326 + }
1327 + }
1328 + return { subagents, nodes, turnItems };
1329 + });
1330 + const loadLostRootToolCalls = loadTrackedWork([input.run.threadId]).pipe(
1331 + Effect.map(({ turnItems }) =>
1332 + [...turnItems.values()].filter((item) => openRootToolCalls.has(item.id)),
1333 + ),
1334 + );
1335 + // Subagents and tool calls that outlived the root run die with the provider runtime. Stop
1336 + // can settle them meanwhile, so each write holds only while its stored row is unchanged.
1337 + const endWorkLostAfterRootRun = Effect.fn("RunExecutionService.endWorkLostAfterRootRun")(
1338 + function* (completedAt: DateTime.Utc) {
1339 + const allocateEventId = () =>
1340 + idAllocator.allocate.event({ threadId: input.run.threadId });
1341 + const open = yield* Ref.get(openRunOwnedSubagents);
1342 + const stored = yield* loadTrackedWork([
1343 + input.run.threadId,
1344 + ...[
1345 + ...open.subagents.values(),
1346 + ...open.nodes.values(),
1347 + ...open.childTurnItems.values(),
1348 + ].map((record) => record.threadId),
1349 + ]);
1350 + for (const subagent of stored.subagents.values()) {
1351 + const subagentItemId = open.turnItems.get(subagent.id)?.id;
1352 + const subagentItem =
1353 + subagentItemId === undefined ? undefined : stored.turnItems.get(subagentItemId);
1354 + const events = yield* cascadeTerminalizeRunOwnedSubagents({
1355 + run: input.run,
1356 + open: {
1357 + subagents: new Map([[subagent.id, subagent]]),
1358 + turnItems: new Map(
1359 + subagentItem?.type === "subagent" ? [[subagent.id, subagentItem]] : [],
1360 + ),
1361 + childTurnItems: new Map(
1362 + [...stored.turnItems].filter(
1363 + ([id, item]) =>
1364 + open.childTurnItems.has(id) && item.threadId === subagent.childThreadId,
1365 + ),
1366 + ),
1367 + nodes: new Map(
1368 + [...stored.nodes].filter(
1369 + ([id, node]) =>
1370 + id === subagent.id || node.threadId === subagent.childThreadId,
1371 + ),
1372 + ),
1373 + linkedChildThreadIds: new Set(
1374 + subagent.childThreadId === null ? [] : [subagent.childThreadId],
1375 + ),
1376 + },
1377 + status: "failed",
1378 + completedAt,
1379 + allocateEventId,
1380 + });
1381 + yield* eventSink.writeIfSubagentUnchanged({ expected: subagent, events });
1382 + }
1383 + for (const toolCall of stored.turnItems.values()) {
1384 + if (!openRootToolCalls.has(toolCall.id)) continue;
1385 + yield* eventSink.writeIfTurnItemUnchanged({
1386 + expected: toolCall,
1387 + events: yield* interruptLostToolCalls({
1388 + run: input.run,
1389 + toolCalls: [toolCall],
1390 + completedAt,
1391 + allocateEventId,
1392 + }),
1393 + });
1394 + }
1395 + yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection());
1396 + openRootToolCalls.clear();
1397 + },
1398 + (effect) =>
1399 + effect.pipe(
1400 + Effect.catchCauseIf(
1401 + (cause) => !Cause.hasInterruptsOnly(cause),
1402 + (cause) =>
1403 + Effect.logWarning(
1404 + "orchestration V2 failed to end work lost with the provider runtime",
1405 + { runId: input.run.id, cause },
1406 + ),
1407 + ),
1408 + ),
1409 + );
1410 // Runs for every provider event, including streamed text updates; avoid a per-event span.
1411 const trackChildLifecycle = Effect.fnUntraced(function* (
1412 event: ProviderAdapter.ProviderAdapterV2Event,
@@ -1333,6 +1514,16 @@ export const layer: Layer.Layer<
1514 }
1515 return next;
1516 });
1517 + if (
1518 + event.turnItem.threadId === input.run.threadId &&
1519 + processBoundToolCallTypes.has(event.turnItem.type)
1520 + ) {
1521 + if (isSettledTurnItemStatus(event.turnItem.status)) {
1522 + openRootToolCalls.delete(event.turnItem.id);
1523 + } else {
1524 + openRootToolCalls.add(event.turnItem.id);
1525 + }
1526 + }
1527 }
1528 if (
1529 belongsToOwnedChildThread &&
@@ -1769,6 +1960,7 @@ export const layer: Layer.Layer<
1960 cause: new ProviderAdapter.ProviderAdapterEventStreamError({
1961 driver: input.providerThread.driver,
1962 providerSessionId: input.providerSessionId,
1963 + raisedByT3: true,
1964 cause: "Provider event stream ended without a root terminal.",
1965 }),
1966 });
@@ -1826,7 +2018,16 @@ export const layer: Layer.Layer<
2018 runId: input.run.id,
2019 cause,
2020 });
1829 - if (yield* Ref.get(rootRunFinalized)) return;
2021 + // An adapter fails its stream when its runtime is gone. A failed write, or a session T3
2022 + // released or saw end, leaves work that can still report or that the releaser settles.
2023 + const runtimeLost =
2024 + isRunExecutionIngestError(squashed) &&
2025 + isProviderEventStreamError(squashed.cause) &&
2026 + squashed.cause.raisedByT3 !== true;
2027 + if (yield* Ref.get(rootRunFinalized)) {
2028 + if (runtimeLost) yield* endWorkLostAfterRootRun(completedAt);
2029 + return;
2030 + }
2031 const providerThread = yield* Ref.get(latestProviderThread);
2032 // Stop background work too; the failed consumer cannot persist terminal events.
2033 const abandonedProviderTurn =
@@ -1865,6 +2066,18 @@ export const layer: Layer.Layer<
2066 : null;
2067 const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
2068 const openSubagents = yield* Ref.get(openRunOwnedSubagents);
2069 + const lostToolCalls = runtimeLost
2070 + ? yield* loadLostRootToolCalls.pipe(
2071 + Effect.catchCauseIf(
2072 + (loadCause) => !Cause.hasInterruptsOnly(loadCause),
2073 + (loadCause) =>
2074 + Effect.logWarning(
2075 + "orchestration V2 failed to read tool calls lost with the provider runtime",
2076 + { runId: input.run.id, cause: loadCause },
2077 + ).pipe(Effect.as([])),
2078 + ),
2079 + )
2080 + : [];
2081 let nextRetryWarningAt = completedAt;
2082 yield* writeFinalRunEvents({
2083 run: input.run,
@@ -1878,6 +2091,7 @@ export const layer: Layer.Layer<
2091 },
2092 openRunOwnedSubagents: openSubagents,
2093 ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }),
2094 + lostToolCalls,
2095 completedAt,
2096 terminal: makeFailedTerminalEvent(
2097 makeProviderFailure({
packages/provider-core/src/server/ProviderAdapter.ts
+2
@@ -454,6 +454,8 @@ export class ProviderAdapterEventStreamError extends Schema.TaggedError<Provider
454 providerSessionId: ProviderSessionId,
455 /** Set when T3 closed the session because this thread detached from it, not the provider. */
456 detachedThreadId: Schema.optional(ThreadId),
457 + /** Set when T3 raised this itself; unset when the adapter reported its runtime failing. */
458 + raisedByT3: Schema.optional(Schema.Boolean),
459 cause: Schema.optional(Schema.Defect()),
460 },
461 ) {