The best way to control Claude Code, Codex and more from any computer or phone
fix(server): work a dead provider process was running no longer stays Running (#37325)
Julius Marminge committed
Nov 15, 2026 at 11:48 UTC
51cdf232652db8cdcea806f9a9e27d7460868579
7 files changed
+451
-18
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
+2
-16
@@ -6354,7 +6354,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
6354
}).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer)));
6355
});
6356
6357
- it.effect("Stop ends a background command no Codex process tracks any more", () =>
6357
+ it.effect("ends a background command when its Codex process is gone", () =>
6358
Effect.gen(function* () {
6359
const fs = yield* FileSystem.FileSystem;
6360
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-bg-stale-workspace-" });
@@ -6429,21 +6429,7 @@ describe("CodexAdapterV2 post-settle continuation", () => {
6429
yield* worker.drain();
6430
yield* Fiber.join(settled);
6431
yield* worker.drain();
6432
- const before = yield* orchestrator.getThreadShell(threadId);
6433
- assert.deepEqual(
6434
- before?.pendingBackgroundTasks?.map((task) => task.kind),
6435
- ["command"],
6436
- "the thread still shows the command the gone process never finished",
6437
- );
6438
- const run = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!;
6439
- yield* orchestrator.dispatch({
6440
- type: "run.interrupt",
6441
- commandId: CommandId.make("stop-background-untracked"),
6442
- threadId,
6443
- runId: run.id,
6444
- holdQueue: true,
6445
- });
6446
- yield* worker.drain();
6432
+ // regression: #37290 the command stayed running, and the thread waiting, until Stop.
6433
const projection = yield* orchestrator.getThreadProjection(threadId);
6434
assert.deepEqual(
6435
projection.turnItems.flatMap((item) =>
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+8
@@ -3485,6 +3485,14 @@ it.effect.each(["detach", "stream failure", "provider stop", "early close"] as c
3485
: undefined,
3486
release === "detach" ? threadId : undefined,
3487
);
3488
+ // A run ends the work it tracks only when the adapter reported its runtime failing.
3489
+ assert.equal(
3490
+ Exit.isFailure(consumed)
3491
+ ? (Cause.squash(consumed.cause) as ProviderAdapter.ProviderAdapterEventStreamError)
3492
+ .raisedByT3
3493
+ : undefined,
3494
+ Exit.isFailure(consumed) ? true : undefined,
3495
+ );
3496
yield* interrupts(2);
3497
3498
assert.isTrue(Option.isNone(yield* manager.get(providerSessionId)));
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+3
@@ -667,6 +667,7 @@ export const layerWithOptions = (
667
const error = new ProviderAdapter.ProviderAdapterEventStreamError({
668
driver: entry.runtime.driver,
669
providerSessionId: entry.runtime.providerSessionId,
670
+ raisedByT3: true,
671
...(detachedThreadId === undefined ? {} : { detachedThreadId }),
672
cause: detail,
673
});
@@ -2857,6 +2858,7 @@ export const layerWithOptions = (
2858
new ProviderAdapter.ProviderAdapterEventStreamError({
2859
driver: entry.runtime.driver,
2860
providerSessionId: entry.runtime.providerSessionId,
2861
+ raisedByT3: true,
2862
cause,
2863
}),
2864
),
@@ -2897,6 +2899,7 @@ export const layerWithOptions = (
2899
new ProviderAdapter.ProviderAdapterEventStreamError({
2900
driver: entry.runtime.driver,
2901
providerSessionId: entry.runtime.providerSessionId,
2902
+ raisedByT3: true,
2903
cause: "Provider event stream ended unexpectedly.",
2904
}),
2905
);
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+14
@@ -258,6 +258,19 @@ export const layer: Layer.Layer<
258
Effect.map((current) => current.thread.worktreePath !== input.worktreePath),
259
Effect.catchCause(() => Effect.succeed(false)),
260
),
261
+ loadUnsettledWork: (threadId: ThreadId, nodeIds: ReadonlyArray<NodeId>) =>
262
+ projectionStore
263
+ .getThreadRecords(threadId, ["subagents", "nodes", "turnItems"], {
264
+ nodeIds,
265
+ subagentIds: nodeIds,
266
+ turnItemStatuses: ["pending", "running", "waiting"],
267
+ })
268
+ .pipe(
269
+ Effect.catchTags({
270
+ ProjectionStoreThreadNotFoundError: () =>
271
+ Effect.succeed({ subagents: [], nodes: [], turnItems: [] }),
272
+ }),
273
+ ),
274
loadRunInterruptReason: () =>
275
projectionStore
276
.getThreadRecords(input.threadId, ["turnItems"], {
@@ -1851,6 +1864,7 @@ export const layer: Layer.Layer<
1864
hasUnpairedRunInterruptRequest: runControls.hasUnpairedRunInterruptRequest,
1865
loadRunInterruptReason: runControls.loadRunInterruptReason,
1866
hasWorkspaceMoved: runControls.hasWorkspaceMoved,
1867
+ loadUnsettledWork: runControls.loadUnsettledWork,
1868
message: {
1869
messageId: message.id,
1870
text: userText,
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+207
-1
@@ -3521,6 +3521,168 @@ it.effect("does not interrupt an ended provider when its terminal write fails",
3521
}),
3522
);
3523
3524
+it.effect("ends a running root command when the provider stream fails mid-turn", () =>
3525
+ Effect.gen(function* () {
3526
+ const now = yield* DateTime.now;
3527
+ const { written, observed } = yield* captureRootRunTermination({
3528
+ key: "stream-error-running-command",
3529
+ shouldFinalizeRun: () => Effect.succeed(true),
3530
+ events: (ids) =>
3531
+ Stream.make(
3532
+ {
3533
+ type: "provider_turn.updated" as const,
3534
+ driver,
3535
+ providerTurn: {
3536
+ id: ids.rootProviderTurnId,
3537
+ providerThreadId: ids.providerThreadId,
3538
+ nodeId: ids.rootNodeId,
3539
+ runAttemptId: ids.attemptId,
3540
+ nativeTurnRef: null,
3541
+ ordinal: 1,
3542
+ status: "running" as const,
3543
+ startedAt: now,
3544
+ completedAt: null,
3545
+ },
3546
+ },
3547
+ backgroundTurnItemEvent(ids, "command_execution", "running", 1),
3548
+ ).pipe(
3549
+ Stream.concat(
3550
+ Stream.fail(
3551
+ new ProviderAdapter.ProviderAdapterEventStreamError({
3552
+ driver,
3553
+ providerSessionId: ProviderSessionId.make("session:stream-error-running-command"),
3554
+ cause: "provider process exited",
3555
+ }),
3556
+ ),
3557
+ ),
3558
+ ),
3559
+ // The dead process cannot take the interrupt either.
3560
+ interruptTurn: (input) =>
3561
+ Effect.fail(
3562
+ new ProviderAdapter.ProviderAdapterInterruptError({
3563
+ driver,
3564
+ providerThreadId: input.providerThread.id,
3565
+ providerTurnId: input.providerTurnId,
3566
+ cause: "provider already exited",
3567
+ }),
3568
+ ),
3569
+ });
3570
+ assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3571
+ const ids = backgroundScenarioIds("stream-error-running-command");
3572
+ const endedCommand = written.find(
3573
+ (item) => item.id === ids.itemId && item.status !== "running",
3574
+ );
3575
+ assert.isDefined(endedCommand);
3576
+ }),
3577
+);
3578
+
3579
+it.effect("keeps a running root command open when a write fails under a live provider", () =>
3580
+ Effect.gen(function* () {
3581
+ const key = "write-error-running-command";
3582
+ const ids = backgroundScenarioIds(key);
3583
+ const secondItemId = TurnItemId.make(`${ids.itemId}:second`);
3584
+ const { written, observed } = yield* captureRootRunTermination({
3585
+ key,
3586
+ shouldFinalizeRun: () => Effect.succeed(true),
3587
+ events: () =>
3588
+ Stream.make(
3589
+ backgroundTurnItemEvent(ids, "command_execution", "running", 1),
3590
+ backgroundTurnItemEvent(ids, "command_execution", "running", 2, secondItemId),
3591
+ ),
3592
+ ingestNormalized: ({ event }) =>
3593
+ event.type === "turn_item.updated" && event.turnItem.id === secondItemId
3594
+ ? Effect.fail(
3595
+ new ProviderEventIngestor.ProviderEventPublishError({
3596
+ providerSessionId: ProviderSessionId.make(`session:${key}`),
3597
+ eventCount: 1,
3598
+ cause: new Error("SQLITE_FULL: database or disk is full"),
3599
+ }),
3600
+ )
3601
+ : Effect.succeed([]),
3602
+ });
3603
+ assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
3604
+ assert.isUndefined(written.find((item) => item.id === ids.itemId));
3605
+ }),
3606
+);
3607
+
3608
+it.effect.each([
3609
+ { ended: true },
3610
+ // T3 ending the stream proves no dead runtime; Stop settles the work it releases.
3611
+ { ended: false, raisedByT3: true },
3612
+ // A Stop that already settled the subagent keeps its result.
3613
+ { ended: false, stopped: true },
3614
+] as const)(
3615
+ "ends a subagent that outlived its root when the provider stream fails (%o)",
3616
+ ({ ended, raisedByT3, stopped }) =>
3617
+ Effect.gen(function* () {
3618
+ const providerInstanceId = ProviderInstanceId.make("codex");
3619
+ const { finalEvents } = yield* captureRootRunTermination({
3620
+ key: "stream-error-outlived-subagent",
3621
+ shouldFinalizeRun: () => Effect.succeed(true),
3622
+ events: (ids) =>
3623
+ Stream.make(
3624
+ {
3625
+ type: "subagent.updated" as const,
3626
+ driver,
3627
+ subagent: makeRunOwnedSubagentFixture({
3628
+ ids,
3629
+ providerInstanceId,
3630
+ childThreadId: ids.childThreadId,
3631
+ driver,
3632
+ status: "running",
3633
+ }),
3634
+ },
3635
+ {
3636
+ type: "turn_item.updated" as const,
3637
+ driver,
3638
+ turnItem: makeRunOwnedSubagentTurnItemFixture({
3639
+ ids,
3640
+ providerInstanceId,
3641
+ childThreadId: ids.childThreadId,
3642
+ driver,
3643
+ status: "running",
3644
+ }),
3645
+ },
3646
+ rootTerminalEvent(ids, "completed"),
3647
+ ).pipe(
3648
+ Stream.concat(
3649
+ Stream.fail(
3650
+ new ProviderAdapter.ProviderAdapterEventStreamError({
3651
+ driver,
3652
+ providerSessionId: ProviderSessionId.make(
3653
+ "session:stream-error-outlived-subagent",
3654
+ ),
3655
+ ...(raisedByT3 === undefined ? {} : { raisedByT3 }),
3656
+ cause: "provider process exited",
3657
+ }),
3658
+ ),
3659
+ ),
3660
+ ),
3661
+ ...(stopped === true
3662
+ ? {
3663
+ settleStored: (work) => ({
3664
+ ...work,
3665
+ subagents: work.subagents.map((row) => ({ ...row, status: "interrupted" })),
3666
+ turnItems: work.turnItems.map((row) => ({ ...row, status: "interrupted" })),
3667
+ }),
3668
+ }
3669
+ : {}),
3670
+ });
3671
+ assert.deepEqual(
3672
+ finalEvents.flatMap((event) =>
3673
+ event.type === "run.updated" ||
3674
+ event.type === "subagent.updated" ||
3675
+ event.type === "turn-item.updated"
3676
+ ? [`${event.type}:${event.payload.status}`]
3677
+ : [],
3678
+ ),
3679
+ ended
3680
+ ? ["run.updated:waiting", "subagent.updated:failed", "turn-item.updated:failed"]
3681
+ : ["run.updated:waiting"],
3682
+ );
3683
+ }),
3684
+);
3685
+
3686
it.effect("does not interrupt a provider when ingestion fails before the root turn is known", () =>
3687
Effect.gen(function* () {
3688
const interrupts = yield* Ref.make(0);
@@ -3783,9 +3945,36 @@ function captureRootRunTermination(input: {
3945
readonly afterStart?: (
3946
closed: Deferred.Deferred<void>,
3947
) => Effect.Effect<void, never, PreviewAgentActivity.PreviewAgentActivity>;
3948
+ /** Changes the stored rows after the provider reported them, as a concurrent Stop does. */
3949
+ readonly settleStored?: (
3950
+ work: RunExecutionService.UnsettledWork,
3951
+ ) => RunExecutionService.UnsettledWork;
3952
}) {
3953
return Effect.gen(function* () {
3954
const ids = backgroundScenarioIds(input.key);
3955
+ // The rows the provider's events stored, which the run reads back when its runtime is lost.
3956
+ const stored = {
3957
+ subagents: new Map<string, OrchestrationV2Subagent>(),
3958
+ nodes: new Map<string, OrchestrationV2ExecutionNode>(),
3959
+ turnItems: new Map<string, OrchestrationV2TurnItem>(),
3960
+ };
3961
+ const store = (event: ProviderAdapter.ProviderAdapterV2Event) =>
3962
+ Effect.sync(() => {
3963
+ if (event.type === "subagent.updated")
3964
+ stored.subagents.set(event.subagent.id, event.subagent);
3965
+ if (event.type === "node.updated") stored.nodes.set(event.node.id, event.node);
3966
+ if (event.type === "turn_item.updated")
3967
+ stored.turnItems.set(event.turnItem.id, event.turnItem);
3968
+ });
3969
+ const loadUnsettledWork = (threadId: ThreadId) =>
3970
+ Effect.sync(() => {
3971
+ const work = {
3972
+ subagents: [...stored.subagents.values()].filter((row) => row.threadId === threadId),
3973
+ nodes: [...stored.nodes.values()].filter((row) => row.threadId === threadId),
3974
+ turnItems: [...stored.turnItems.values()].filter((row) => row.threadId === threadId),
3975
+ };
3976
+ return input.settleStored?.(work) ?? work;
3977
+ });
3978
const providerInstanceId = ProviderInstanceId.make("codex");
3979
const runningSubagent = makeRunOwnedSubagentFixture({
3980
ids,
@@ -3854,11 +4043,27 @@ function captureRootRunTermination(input: {
4043
yield* captureFinalEvents(payload.events);
4044
return { committed: true, storedEvents: [] };
4045
}),
4046
+ // Committed only against the row that was read, as the real conditional writes are.
4047
+ writeIfSubagentUnchanged: (payload) =>
4048
+ stored.subagents.get(payload.expected.id) === payload.expected
4049
+ ? captureFinalEvents(payload.events).pipe(
4050
+ Effect.as({ committed: true, storedEvents: [] }),
4051
+ )
4052
+ : Effect.succeed({ committed: false, storedEvents: [] }),
4053
+ writeIfTurnItemUnchanged: (payload) =>
4054
+ stored.turnItems.get(payload.expected.id) === payload.expected
4055
+ ? captureFinalEvents(payload.events).pipe(
4056
+ Effect.as({ committed: true, storedEvents: [] }),
4057
+ )
4058
+ : Effect.succeed({ committed: false, storedEvents: [] }),
4059
}),
4060
IdAllocator.layer,
4061
ThreadCommandExecutor.layer,
4062
Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({
3861
- ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])),
4063
+ ingestNormalized: (request) =>
4064
+ store(request.event).pipe(
4065
+ Effect.andThen(input.ingestNormalized?.(request) ?? Effect.succeed([])),
4066
+ ),
4067
}),
4068
ServerSettings.layerTest(),
4069
Layer.succeed(RunFinalizationService.RunFinalizationObserver, {
@@ -3944,6 +4149,7 @@ function captureRootRunTermination(input: {
4149
attemptId: ids.attemptId,
4150
providerTurnOrdinal: 1,
4151
shouldFinalizeRun: input.shouldFinalizeRun,
4152
+ loadUnsettledWork,
4153
...(input.loadRunInterruptReason === undefined
4154
? {}
4155
: { loadRunInterruptReason: input.loadRunInterruptReason }),
apps/server/src/orchestration-v2/RunExecutionService.ts
+215
-1
@@ -154,6 +154,12 @@ export function selectInheritedBackgroundTurnItems(input: {
154
155
type SubagentTurnItem = Extract<OrchestrationV2TurnItem, { readonly type: "subagent" }>;
156
157
+// Tool calls that only a live provider process can end.
158
+const processBoundToolCallTypes: ReadonlySet<OrchestrationV2TurnItem["type"]> = new Set([
159
+ "command_execution",
160
+ "dynamic_tool",
161
+]);
162
+
163
type OpenRunOwnedSubagentProjection = {
164
readonly subagents: ReadonlyMap<NodeId, OrchestrationV2Subagent>;
165
readonly turnItems: ReadonlyMap<NodeId, SubagentTurnItem>;
@@ -162,6 +168,12 @@ type OpenRunOwnedSubagentProjection = {
168
readonly linkedChildThreadIds: ReadonlySet<ThreadId>;
169
};
170
171
+export type UnsettledWork = {
172
+ readonly subagents: ReadonlyArray<OrchestrationV2Subagent>;
173
+ readonly nodes: ReadonlyArray<OrchestrationV2ExecutionNode>;
174
+ readonly turnItems: ReadonlyArray<OrchestrationV2TurnItem>;
175
+};
176
+
177
type RunOwnedSubagentTerminalStatus = Extract<
178
OrchestrationV2Subagent["status"],
179
"interrupted" | "failed" | "cancelled"
@@ -329,6 +341,38 @@ export const cascadeTerminalizeRunOwnedSubagents = Effect.fn(
341
return events;
342
});
343
344
+const interruptLostToolCalls = Effect.fn("RunExecutionService.interruptLostToolCalls")(
345
+ function* (input: {
346
+ readonly run: OrchestrationV2Run;
347
+ readonly toolCalls: Iterable<OrchestrationV2TurnItem>;
348
+ readonly completedAt: DateTime.Utc;
349
+ readonly allocateEventId: () => Effect.Effect<
350
+ EventId,
351
+ IdAllocator.IdAllocatorV2AllocationError
352
+ >;
353
+ }) {
354
+ const events: Array<OrchestrationV2DomainEvent> = [];
355
+ for (const toolCall of input.toolCalls) {
356
+ events.push({
357
+ id: yield* input.allocateEventId(),
358
+ type: "turn-item.updated",
359
+ threadId: toolCall.threadId,
360
+ runId: toolCall.runId ?? input.run.id,
361
+ ...(toolCall.nodeId === null ? {} : { nodeId: toolCall.nodeId }),
362
+ providerInstanceId: input.run.providerInstanceId,
363
+ occurredAt: input.completedAt,
364
+ payload: {
365
+ ...toolCall,
366
+ status: "interrupted",
367
+ completedAt: input.completedAt,
368
+ updatedAt: input.completedAt,
369
+ },
370
+ });
371
+ }
372
+ return events;
373
+ },
374
+);
375
+
376
function finalProviderThreadStatus(
377
disposition: ProviderTerminalEvent["threadDisposition"],
378
): OrchestrationV2ProviderThread["status"] {
@@ -612,6 +656,14 @@ export interface RunExecutionServiceV2StartRootRunInput {
656
readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
657
readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>;
658
readonly hasWorkspaceMoved?: () => Effect.Effect<boolean, never>;
659
+ /**
660
+ * Stored rows of a thread's unsettled work. Work a dead runtime left is ended from these, since
661
+ * Stop may have settled it after the snapshots this run holds.
662
+ */
663
+ readonly loadUnsettledWork?: (
664
+ threadId: ThreadId,
665
+ nodeIds: ReadonlyArray<NodeId>,
666
+ ) => Effect.Effect<UnsettledWork, ProjectionStoreV2Error>;
667
readonly message: ProviderAdapter.ProviderAdapterV2TurnMessage;
668
readonly modelSelection: ModelSelection;
669
readonly runtimePolicy: ProviderAdapter.ProviderAdapterV2RuntimePolicy;
@@ -680,6 +732,8 @@ export const layer: Layer.Layer<
732
readonly openRunOwnedSubagentsTerminal?: ProviderTerminalEvent;
733
// Root turn snapshot written with the run when T3 ends the run itself.
734
readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn;
735
+ // Stored root tool calls still open when the provider runtime was lost; nothing else ends them.
736
+ readonly lostToolCalls?: ReadonlyArray<OrchestrationV2TurnItem>;
737
readonly terminal: ProviderTerminalEvent;
738
readonly completedAt?: DateTime.Utc;
739
readonly failureItemPersisted: boolean;
@@ -810,6 +864,12 @@ export const layer: Layer.Layer<
864
: [],
865
events: [
866
...cascadedSubagentEvents,
867
+ ...(yield* interruptLostToolCalls({
868
+ run: input.run,
869
+ toolCalls: input.lostToolCalls ?? [],
870
+ completedAt,
871
+ allocateEventId,
872
+ })),
873
...(input.abandonedProviderTurn === undefined
874
? []
875
: [
@@ -1150,6 +1210,8 @@ export const layer: Layer.Layer<
1210
ReadonlySet<OrchestrationV2TurnItem["id"]>
1211
>(new Set(inheritedBackgroundTurnItemsById.keys()));
1212
const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection());
1213
+ // Open tool calls on the root thread, to end from their stored rows if the runtime is lost.
1214
+ const openRootToolCalls = new Set<TurnItemId>();
1215
// The last stored root snapshot's pendingBackgroundWork; adapters that omit it leave it false.
1216
let rootSnapshotPendingWork = false;
1217
// Owned child threads whose last snapshot reported pending work, such as a queued native
@@ -1226,6 +1288,125 @@ export const layer: Layer.Layer<
1288
}
1289
yield* Ref.set(rootRunFinalized, true);
1290
});
1291
+ // The stored rows of the work this run still tracks as open on the given threads.
1292
+ const loadTrackedWork = Effect.fn("RunExecutionService.loadTrackedWork")(function* (
1293
+ threadIds: Iterable<ThreadId>,
1294
+ ) {
1295
+ const open = yield* Ref.get(openRunOwnedSubagents);
1296
+ const subagents = new Map<NodeId, OrchestrationV2Subagent>();
1297
+ const nodes = new Map<NodeId, OrchestrationV2ExecutionNode>();
1298
+ const turnItems = new Map<TurnItemId, OrchestrationV2TurnItem>();
1299
+ if (input.loadUnsettledWork === undefined) return { subagents, nodes, turnItems };
1300
+ const trackedItemIds = new Set<TurnItemId>([
1301
+ ...openRootToolCalls.keys(),
1302
+ ...open.childTurnItems.keys(),
1303
+ ...[...open.turnItems.values()].map((item) => item.id),
1304
+ ]);
1305
+ for (const threadId of new Set(threadIds)) {
1306
+ const stored = yield* input.loadUnsettledWork(
1307
+ threadId,
1308
+ [...open.nodes.values(), ...open.subagents.values()]
1309
+ .filter((record) => record.threadId === threadId)
1310
+ .map((record) => record.id),
1311
+ );
1312
+ for (const subagent of stored.subagents) {
1313
+ if (open.subagents.has(subagent.id) && !isSettledSubagentStatus(subagent.status)) {
1314
+ subagents.set(subagent.id, subagent);
1315
+ }
1316
+ }
1317
+ for (const node of stored.nodes) {
1318
+ if (open.nodes.has(node.id) && isOpenExecutionNodeStatus(node.status)) {
1319
+ nodes.set(node.id, node);
1320
+ }
1321
+ }
1322
+ for (const item of stored.turnItems) {
1323
+ if (trackedItemIds.has(item.id) && !isSettledTurnItemStatus(item.status)) {
1324
+ turnItems.set(item.id, item);
1325
+ }
1326
+ }
1327
+ }
1328
+ return { subagents, nodes, turnItems };
1329
+ });
1330
+ const loadLostRootToolCalls = loadTrackedWork([input.run.threadId]).pipe(
1331
+ Effect.map(({ turnItems }) =>
1332
+ [...turnItems.values()].filter((item) => openRootToolCalls.has(item.id)),
1333
+ ),
1334
+ );
1335
+ // Subagents and tool calls that outlived the root run die with the provider runtime. Stop
1336
+ // can settle them meanwhile, so each write holds only while its stored row is unchanged.
1337
+ const endWorkLostAfterRootRun = Effect.fn("RunExecutionService.endWorkLostAfterRootRun")(
1338
+ function* (completedAt: DateTime.Utc) {
1339
+ const allocateEventId = () =>
1340
+ idAllocator.allocate.event({ threadId: input.run.threadId });
1341
+ const open = yield* Ref.get(openRunOwnedSubagents);
1342
+ const stored = yield* loadTrackedWork([
1343
+ input.run.threadId,
1344
+ ...[
1345
+ ...open.subagents.values(),
1346
+ ...open.nodes.values(),
1347
+ ...open.childTurnItems.values(),
1348
+ ].map((record) => record.threadId),
1349
+ ]);
1350
+ for (const subagent of stored.subagents.values()) {
1351
+ const subagentItemId = open.turnItems.get(subagent.id)?.id;
1352
+ const subagentItem =
1353
+ subagentItemId === undefined ? undefined : stored.turnItems.get(subagentItemId);
1354
+ const events = yield* cascadeTerminalizeRunOwnedSubagents({
1355
+ run: input.run,
1356
+ open: {
1357
+ subagents: new Map([[subagent.id, subagent]]),
1358
+ turnItems: new Map(
1359
+ subagentItem?.type === "subagent" ? [[subagent.id, subagentItem]] : [],
1360
+ ),
1361
+ childTurnItems: new Map(
1362
+ [...stored.turnItems].filter(
1363
+ ([id, item]) =>
1364
+ open.childTurnItems.has(id) && item.threadId === subagent.childThreadId,
1365
+ ),
1366
+ ),
1367
+ nodes: new Map(
1368
+ [...stored.nodes].filter(
1369
+ ([id, node]) =>
1370
+ id === subagent.id || node.threadId === subagent.childThreadId,
1371
+ ),
1372
+ ),
1373
+ linkedChildThreadIds: new Set(
1374
+ subagent.childThreadId === null ? [] : [subagent.childThreadId],
1375
+ ),
1376
+ },
1377
+ status: "failed",
1378
+ completedAt,
1379
+ allocateEventId,
1380
+ });
1381
+ yield* eventSink.writeIfSubagentUnchanged({ expected: subagent, events });
1382
+ }
1383
+ for (const toolCall of stored.turnItems.values()) {
1384
+ if (!openRootToolCalls.has(toolCall.id)) continue;
1385
+ yield* eventSink.writeIfTurnItemUnchanged({
1386
+ expected: toolCall,
1387
+ events: yield* interruptLostToolCalls({
1388
+ run: input.run,
1389
+ toolCalls: [toolCall],
1390
+ completedAt,
1391
+ allocateEventId,
1392
+ }),
1393
+ });
1394
+ }
1395
+ yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection());
1396
+ openRootToolCalls.clear();
1397
+ },
1398
+ (effect) =>
1399
+ effect.pipe(
1400
+ Effect.catchCauseIf(
1401
+ (cause) => !Cause.hasInterruptsOnly(cause),
1402
+ (cause) =>
1403
+ Effect.logWarning(
1404
+ "orchestration V2 failed to end work lost with the provider runtime",
1405
+ { runId: input.run.id, cause },
1406
+ ),
1407
+ ),
1408
+ ),
1409
+ );
1410
// Runs for every provider event, including streamed text updates; avoid a per-event span.
1411
const trackChildLifecycle = Effect.fnUntraced(function* (
1412
event: ProviderAdapter.ProviderAdapterV2Event,
@@ -1333,6 +1514,16 @@ export const layer: Layer.Layer<
1514
}
1515
return next;
1516
});
1517
+ if (
1518
+ event.turnItem.threadId === input.run.threadId &&
1519
+ processBoundToolCallTypes.has(event.turnItem.type)
1520
+ ) {
1521
+ if (isSettledTurnItemStatus(event.turnItem.status)) {
1522
+ openRootToolCalls.delete(event.turnItem.id);
1523
+ } else {
1524
+ openRootToolCalls.add(event.turnItem.id);
1525
+ }
1526
+ }
1527
}
1528
if (
1529
belongsToOwnedChildThread &&
@@ -1769,6 +1960,7 @@ export const layer: Layer.Layer<
1960
cause: new ProviderAdapter.ProviderAdapterEventStreamError({
1961
driver: input.providerThread.driver,
1962
providerSessionId: input.providerSessionId,
1963
+ raisedByT3: true,
1964
cause: "Provider event stream ended without a root terminal.",
1965
}),
1966
});
@@ -1826,7 +2018,16 @@ export const layer: Layer.Layer<
2018
runId: input.run.id,
2019
cause,
2020
});
1829
- if (yield* Ref.get(rootRunFinalized)) return;
2021
+ // An adapter fails its stream when its runtime is gone. A failed write, or a session T3
2022
+ // released or saw end, leaves work that can still report or that the releaser settles.
2023
+ const runtimeLost =
2024
+ isRunExecutionIngestError(squashed) &&
2025
+ isProviderEventStreamError(squashed.cause) &&
2026
+ squashed.cause.raisedByT3 !== true;
2027
+ if (yield* Ref.get(rootRunFinalized)) {
2028
+ if (runtimeLost) yield* endWorkLostAfterRootRun(completedAt);
2029
+ return;
2030
+ }
2031
const providerThread = yield* Ref.get(latestProviderThread);
2032
// Stop background work too; the failed consumer cannot persist terminal events.
2033
const abandonedProviderTurn =
@@ -1865,6 +2066,18 @@ export const layer: Layer.Layer<
2066
: null;
2067
const latestItemOrdinal = yield* Ref.get(latestTurnItemOrdinal);
2068
const openSubagents = yield* Ref.get(openRunOwnedSubagents);
2069
+ const lostToolCalls = runtimeLost
2070
+ ? yield* loadLostRootToolCalls.pipe(
2071
+ Effect.catchCauseIf(
2072
+ (loadCause) => !Cause.hasInterruptsOnly(loadCause),
2073
+ (loadCause) =>
2074
+ Effect.logWarning(
2075
+ "orchestration V2 failed to read tool calls lost with the provider runtime",
2076
+ { runId: input.run.id, cause: loadCause },
2077
+ ).pipe(Effect.as([])),
2078
+ ),
2079
+ )
2080
+ : [];
2081
let nextRetryWarningAt = completedAt;
2082
yield* writeFinalRunEvents({
2083
run: input.run,
@@ -1878,6 +2091,7 @@ export const layer: Layer.Layer<
2091
},
2092
openRunOwnedSubagents: openSubagents,
2093
...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }),
2094
+ lostToolCalls,
2095
completedAt,
2096
terminal: makeFailedTerminalEvent(
2097
makeProviderFailure({
packages/provider-core/src/server/ProviderAdapter.ts
+2
@@ -454,6 +454,8 @@ export class ProviderAdapterEventStreamError extends Schema.TaggedError<Provider
454
providerSessionId: ProviderSessionId,
455
/** Set when T3 closed the session because this thread detached from it, not the provider. */
456
detachedThreadId: Schema.optional(ThreadId),
457
+ /** Set when T3 raised this itself; unset when the adapter reported its runtime failing. */
458
+ raisedByT3: Schema.optional(Schema.Boolean),
459
cause: Schema.optional(Schema.Defect()),
460
},
461
) {