The best way to control Claude Code, Codex and more from any computer or phone
fix(server): ended roots no longer leave provider turns running (#22846)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Theo Browne committed
Oct 25, 2026 at 15:20 UTC
725ada6eaa32d4a32c361ee42a1717a3e65568d1
8 files changed
+291
-21
apps/server/src/orchestration-v2/Orchestrator.ts
+2
-2
@@ -9112,7 +9112,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9112
}
9113
childThreadIds.add(childThreadId);
9114
const child = yield* projectionStore
9115
- .getThreadRecords(childThreadId, ["nodes", "subagents", "turnItems"], {
9115
+ .getThreadRecords(childThreadId, ["nodes", "providerTurns", "subagents", "turnItems"], {
9116
// Runless items are the subagent's; any run there is the user's own.
9117
turnItemRunIds: [null],
9118
turnItemStatuses: ["pending", "running", "waiting"],
@@ -9671,7 +9671,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9671
yield* Ref.update(events, (current) => [...current, ...ended]);
9672
if (subagent.childThreadId !== null) {
9673
const child = yield* projectionStore
9674
- .getThreadRecords(subagent.childThreadId, ["nodes", "turnItems"], {
9674
+ .getThreadRecords(subagent.childThreadId, ["nodes", "providerTurns", "turnItems"], {
9675
turnItemRunIds: [null],
9676
turnItemStatuses: ["pending", "running", "waiting"],
9677
})
apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
+100
@@ -12,6 +12,7 @@ import {
12
ProviderInstanceId,
13
ProviderSessionId,
14
ProviderThreadId,
15
+ ProviderTurnId,
16
RunId,
17
RuntimeRequestId,
18
ThreadId,
@@ -966,3 +967,102 @@ it.effect("marks fork descendants unreadable when their source is missing or cor
967
);
968
}).pipe(Effect.provide(layerTest)),
969
);
970
+
971
+it.effect("startup ends the provider turn of a native child root it cancels", () =>
972
+ Effect.gen(function* () {
973
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
974
+ const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
975
+ const now = yield* DateTime.now;
976
+ const parentThreadId = yield* createThread("native-child-root-turn-parent");
977
+ const threadId = yield* createThread("native-child-root-turn");
978
+ const rootNodeId = NodeId.make(`node:${threadId}:thread-root`);
979
+ const providerThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
980
+ const providerTurnId = ProviderTurnId.make(`provider-turn:${threadId}`);
981
+ yield* projections.apply({
982
+ id: EventId.make(`event:${threadId}:root`),
983
+ type: "node.updated",
984
+ threadId,
985
+ nodeId: rootNodeId,
986
+ driver,
987
+ occurredAt: now,
988
+ payload: {
989
+ id: rootNodeId,
990
+ threadId,
991
+ runId: null,
992
+ parentNodeId: null,
993
+ rootNodeId,
994
+ kind: "root_turn",
995
+ status: "running",
996
+ countsForRun: false,
997
+ providerThreadId,
998
+ providerTurnId,
999
+ nativeItemRef: null,
1000
+ runtimeRequestId: null,
1001
+ checkpointScopeId: null,
1002
+ startedAt: now,
1003
+ completedAt: null,
1004
+ },
1005
+ });
1006
+ // A native subagent's turn belongs to no run attempt.
1007
+ yield* projections.apply({
1008
+ id: EventId.make(`event:${threadId}:provider-turn`),
1009
+ type: "provider-turn.updated",
1010
+ threadId,
1011
+ nodeId: rootNodeId,
1012
+ driver,
1013
+ occurredAt: now,
1014
+ payload: {
1015
+ id: providerTurnId,
1016
+ providerThreadId,
1017
+ nodeId: rootNodeId,
1018
+ runAttemptId: null,
1019
+ nativeTurnRef: null,
1020
+ ordinal: 1,
1021
+ status: "running",
1022
+ startedAt: now,
1023
+ completedAt: null,
1024
+ },
1025
+ });
1026
+
1027
+ const subagentId = NodeId.make(`node:${parentThreadId}:subagent`);
1028
+ yield* projections.apply({
1029
+ id: EventId.make(`event:${parentThreadId}:subagent`),
1030
+ type: "subagent.updated",
1031
+ threadId: parentThreadId,
1032
+ occurredAt: now,
1033
+ payload: {
1034
+ id: subagentId,
1035
+ threadId: parentThreadId,
1036
+ runId: null,
1037
+ parentNodeId: NodeId.make(`node:${parentThreadId}:root`),
1038
+ origin: "provider_native",
1039
+ createdBy: "agent",
1040
+ driver,
1041
+ providerInstanceId,
1042
+ providerThreadId,
1043
+ childThreadId: threadId,
1044
+ nativeTaskRef: null,
1045
+ title: null,
1046
+ model: null,
1047
+ prompt: "sleep 300",
1048
+ status: "running",
1049
+ result: null,
1050
+ startedAt: now,
1051
+ completedAt: null,
1052
+ updatedAt: now,
1053
+ },
1054
+ });
1055
+
1056
+ yield* recovery.reconcile("startup");
1057
+
1058
+ const projection = yield* projections.getThreadProjection(threadId);
1059
+ assert.deepEqual(
1060
+ {
1061
+ root: projection.nodes.find((node) => node.id === rootNodeId)?.status,
1062
+ providerTurn: projection.providerTurns.find((turn) => turn.id === providerTurnId)?.status,
1063
+ },
1064
+ { root: "cancelled", providerTurn: "cancelled" },
1065
+ "a root startup recovery cancelled left its provider turn running",
1066
+ );
1067
+ }).pipe(Effect.provide(layerRecoveryTest)),
1068
+);
apps/server/src/orchestration-v2/ProjectionStore.ts
+18
-6
@@ -4605,14 +4605,26 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4605
FROM orchestration_v2_projection_provider_turns AS provider_turn
4606
WHERE provider_turn.thread_id = ${threadId}
4607
AND provider_turn.status IN ('pending', 'starting', 'running', 'waiting')
4608
- AND provider_turn.run_attempt_id IN (
4609
- SELECT attempt_id FROM orchestration_v2_projection_run_attempts
4610
- WHERE thread_id = ${threadId}
4611
- AND run_id IN (
4612
- SELECT run_id FROM orchestration_v2_projection_runs
4608
+ AND (
4609
+ provider_turn.run_attempt_id IN (
4610
+ SELECT attempt_id FROM orchestration_v2_projection_run_attempts
4611
+ WHERE thread_id = ${threadId}
4612
+ AND run_id IN (
4613
+ SELECT run_id FROM orchestration_v2_projection_runs
4614
+ WHERE thread_id = ${threadId}
4615
+ AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
4616
+ )
4617
+ )
4618
+ OR (
4619
+ provider_turn.run_attempt_id IS NULL
4620
+ AND provider_turn.node_id IN (
4621
+ SELECT node_id FROM orchestration_v2_projection_nodes
4622
WHERE thread_id = ${threadId}
4614
- AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
4623
+ AND run_id IS NULL
4624
+ AND kind = 'root_turn'
4625
+ AND status IN ('pending', 'running', 'waiting')
4626
)
4627
+ )
4628
)
4629
ORDER BY provider_turn.provider_thread_id ASC, provider_turn.ordinal ASC
4630
`,
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+101
@@ -11,6 +11,7 @@ import {
11
type OrchestrationV2DomainEvent,
12
type OrchestrationV2ExecutionNode,
13
type OrchestrationV2ProviderThread,
14
+ type OrchestrationV2ProviderTurn,
15
type OrchestrationV2Run,
16
type OrchestrationV2RunAttempt,
17
type OrchestrationV2Subagent,
@@ -2624,6 +2625,106 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c
2625
}),
2626
);
2627
2628
+it.effect.each([
2629
+ { status: "completed", failure: "later-stream", interrupt: "succeeds" },
2630
+ { status: "failed", failure: "later-stream", interrupt: "succeeds" },
2631
+ { status: "completed", failure: "terminal-ingest", interrupt: "succeeds" },
2632
+ { status: "completed", failure: "terminal-ingest", interrupt: "fails" },
2633
+] as const)(
2634
+ "stores a $status root provider turn snapshot after a $failure failure only if the interrupt succeeds (it $interrupt)",
2635
+ ({ status, failure, interrupt }) =>
2636
+ Effect.gen(function* () {
2637
+ const now = yield* DateTime.now;
2638
+ const terminalAt = DateTime.subtract(now, { seconds: 1 });
2639
+ const key = `terminal-snapshot:${status}:${failure}:${interrupt}`;
2640
+ const durableTurns = yield* Ref.make<
2641
+ ReadonlyMap<ProviderTurnId, OrchestrationV2ProviderTurn>
2642
+ >(new Map());
2643
+ const persist = (turn: OrchestrationV2ProviderTurn) =>
2644
+ Ref.update(durableTurns, (current) => new Map(current).set(turn.id, turn));
2645
+ const interrupts = yield* Ref.make<
2646
+ ReadonlyArray<ProviderAdapter.ProviderAdapterV2InterruptInput>
2647
+ >([]);
2648
+ const ids = backgroundScenarioIds(key);
2649
+ const rootTurn = (
2650
+ turnStatus: OrchestrationV2ProviderTurn["status"],
2651
+ completedAt: DateTime.Utc | null,
2652
+ ) => ({
2653
+ type: "provider_turn.updated" as const,
2654
+ driver,
2655
+ providerTurn: {
2656
+ id: ids.rootProviderTurnId,
2657
+ providerThreadId: ids.providerThreadId,
2658
+ nodeId: ids.rootNodeId,
2659
+ runAttemptId: ids.attemptId,
2660
+ nativeTurnRef: null,
2661
+ ordinal: 1,
2662
+ status: turnStatus,
2663
+ startedAt: terminalAt,
2664
+ completedAt,
2665
+ },
2666
+ });
2667
+ const { observed, finalEvents } = yield* captureRootRunTermination({
2668
+ key,
2669
+ shouldFinalizeRun: () => Effect.succeed(true),
2670
+ events: () =>
2671
+ Stream.make(rootTurn("running", null), rootTurn(status, terminalAt)).pipe(
2672
+ Stream.concat(
2673
+ failure === "later-stream"
2674
+ ? Stream.fail(
2675
+ new ProviderAdapter.ProviderAdapterEventStreamError({
2676
+ driver,
2677
+ providerSessionId: ProviderSessionId.make(`session:${key}`),
2678
+ cause: "provider process exited",
2679
+ }),
2680
+ )
2681
+ : Stream.empty,
2682
+ ),
2683
+ ),
2684
+ ingestNormalized: ({ event }) =>
2685
+ event.type !== "provider_turn.updated"
2686
+ ? Effect.succeed([])
2687
+ : failure === "terminal-ingest" && event.providerTurn.status !== "running"
2688
+ ? Effect.fail(
2689
+ new ProviderEventIngestor.ProviderEventPublishError({
2690
+ providerSessionId: ProviderSessionId.make(`session:${key}`),
2691
+ eventCount: 1,
2692
+ cause: new Error("SQLITE_FULL: database or disk is full"),
2693
+ }),
2694
+ )
2695
+ : persist(event.providerTurn).pipe(Effect.as([])),
2696
+ interruptTurn: (input) =>
2697
+ Ref.update(interrupts, (current) => [...current, input]).pipe(
2698
+ Effect.andThen(
2699
+ interrupt === "fails"
2700
+ ? Effect.fail(
2701
+ new ProviderAdapter.ProviderAdapterInterruptError({
2702
+ driver,
2703
+ providerThreadId: input.providerThread.id,
2704
+ providerTurnId: input.providerTurnId,
2705
+ cause: "provider already exited",
2706
+ }),
2707
+ )
2708
+ : Effect.void,
2709
+ ),
2710
+ ),
2711
+ beforeTerminalWrite: (request) =>
2712
+ Effect.forEach(request.events, (event) =>
2713
+ event.type === "provider-turn.updated" ? persist(event.payload) : Effect.void,
2714
+ ).pipe(Effect.as(true)),
2715
+ });
2716
+ const calls = yield* Ref.get(interrupts);
2717
+ assert.equal(calls.length, 1);
2718
+ assert.isTrue(calls[0]?.requestRuntimeRestart);
2719
+ assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2720
+ assert.isTrue(finalEvents.some((event) => event.type === "run.updated"));
2721
+ const durable = (yield* Ref.get(durableTurns)).get(ids.rootProviderTurnId);
2722
+ // A failed interrupt keeps the stored turn running for the next start to retry.
2723
+ assert.equal(durable?.status, interrupt === "fails" ? "running" : status);
2724
+ assert.deepEqual(durable?.completedAt, interrupt === "fails" ? null : terminalAt);
2725
+ }),
2726
+);
2727
+
2728
it.effect("does not interrupt an ended provider when its terminal write fails", () =>
2729
Effect.gen(function* () {
2730
const interrupts = yield* Ref.make(0);
apps/server/src/orchestration-v2/RunExecutionService.ts
+23
-11
@@ -620,7 +620,8 @@ export const layer: Layer.Layer<
620
readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
621
readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>;
622
readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
623
- readonly interruptedProviderTurn?: OrchestrationV2ProviderTurn;
623
+ // Root turn snapshot written with the run when T3 ends the run itself.
624
+ readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn;
625
readonly terminal: ProviderTerminalEvent;
626
readonly completedAt?: DateTime.Utc;
627
readonly failureItemPersisted: boolean;
@@ -747,7 +748,7 @@ export const layer: Layer.Layer<
748
: [],
749
events: [
750
...cascadedSubagentEvents,
750
- ...(input.interruptedProviderTurn === undefined
751
+ ...(input.abandonedProviderTurn === undefined
752
? []
753
: [
754
{
@@ -758,11 +759,7 @@ export const layer: Layer.Layer<
759
nodeId: input.rootNode.id,
760
providerInstanceId: input.run.providerInstanceId,
761
occurredAt: completedAt,
761
- payload: {
762
- ...input.interruptedProviderTurn,
763
- status: "interrupted" as const,
764
- completedAt,
765
- },
762
+ payload: input.abandonedProviderTurn,
763
},
764
]),
765
...(finalizedAttempt === null
@@ -1482,8 +1479,9 @@ export const layer: Layer.Layer<
1479
if (yield* Ref.get(rootRunFinalized)) return;
1480
const providerThread = yield* Ref.get(latestProviderThread);
1481
const routing = yield* Ref.get(eventRouting);
1482
+ const rootProviderTurn = yield* Ref.get(latestRootProviderTurn);
1483
// Stop background work too; the failed consumer cannot persist terminal events.
1486
- const interruptedProviderTurn =
1484
+ const abandonedProviderTurn =
1485
routing.rootProviderTurnId !== null && !routing.rootTurnEnded
1486
? yield* input.session
1487
.interruptTurn({
@@ -1492,14 +1490,28 @@ export const layer: Layer.Layer<
1490
requestRuntimeRestart: true,
1491
})
1492
.pipe(
1495
- Effect.as(yield* Ref.get(latestRootProviderTurn)),
1493
+ Effect.as(true),
1494
Effect.catchCauseIf(
1495
(interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
1496
(interruptCause) =>
1497
Effect.logWarning(
1498
"orchestration V2 failed to interrupt turn after ingestion failure",
1499
{ runId: input.run.id, cause: interruptCause },
1502
- ).pipe(Effect.as(null)),
1500
+ ).pipe(Effect.as(false)),
1501
+ ),
1502
+ // A failed interrupt leaves the stored turn running so the next start
1503
+ // retries it. The snapshot is cached before it is published, so it may
1504
+ // not be stored: replay a terminal one as-is.
1505
+ Effect.map((interrupted) =>
1506
+ !interrupted || rootProviderTurn === null
1507
+ ? null
1508
+ : isTerminalProviderTurnStatus(rootProviderTurn.status)
1509
+ ? rootProviderTurn
1510
+ : {
1511
+ ...rootProviderTurn,
1512
+ status: "interrupted" as const,
1513
+ completedAt,
1514
+ },
1515
),
1516
)
1517
: null;
@@ -1517,7 +1529,7 @@ export const layer: Layer.Layer<
1529
expectedStatus: "running",
1530
},
1531
openRunOwnedSubagents: openSubagents,
1520
- ...(interruptedProviderTurn === null ? {} : { interruptedProviderTurn }),
1532
+ ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }),
1533
completedAt,
1534
terminal: makeFailedTerminalEvent(
1535
makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }),
apps/server/src/orchestration-v2/SubagentProjection.test.ts
+1
@@ -375,6 +375,7 @@ it.effect("ends a runless root turn's items when its node already ended", () =>
375
providerInstanceId: parentProviderInstanceId,
376
projection: {
377
nodes: [root],
378
+ providerTurns: [],
379
turnItems: [item("turn-item:command"), item("turn-item:reasoning")],
380
} as never,
381
status: "cancelled",
apps/server/src/orchestration-v2/ThreadStop.test.ts
+23
@@ -1335,6 +1335,25 @@ it.effect.each(["missing session", "completed", "failed", "cancelled"] as const)
1335
occurredAt: now,
1336
payload: childRoot,
1337
});
1338
+ const childTurnId = ProviderTurnId.make("turn:native-stop-child");
1339
+ yield* projections.apply({
1340
+ id: EventId.make("event:native-child-stop:child-turn"),
1341
+ type: "provider-turn.updated",
1342
+ threadId: childRoot.threadId,
1343
+ nodeId: childRoot.id,
1344
+ occurredAt: now,
1345
+ payload: {
1346
+ id: childTurnId,
1347
+ providerThreadId: ProviderThreadId.make("provider-thread:native-stop-child"),
1348
+ nodeId: childRoot.id,
1349
+ runAttemptId: null,
1350
+ nativeTurnRef: null,
1351
+ ordinal: 1,
1352
+ status: "running",
1353
+ startedAt: now,
1354
+ completedAt: null,
1355
+ },
1356
+ });
1357
const providerThread = before.providerThreads.find(
1358
(thread) => thread.id === run.providerThreadId,
1359
)!;
@@ -1499,6 +1518,10 @@ it.effect.each(["missing session", "completed", "failed", "cancelled"] as const)
1518
);
1519
const child = yield* orchestrator.getThreadProjection(childRoot.threadId);
1520
assert.equal(child.nodes.find((node) => node.id === childRoot.id)?.status, confirmedStatus);
1521
+ assert.equal(
1522
+ child.providerTurns.find((turn) => turn.id === childTurnId)?.status,
1523
+ confirmedStatus,
1524
+ );
1525
assert.isNotNull(child.nodes.find((node) => node.id === childRoot.id)?.completedAt);
1526
assert.equal(settled.subagents.find((task) => task.id === sibling.id)?.status, "running");
1527
assert.deepEqual(yield* threadState(threadId), stateBefore);
packages/provider-core/src/server/subagentProjection.ts
+23
-2
@@ -271,7 +271,7 @@ export function delegatedTaskProgress(projection: {
271
272
function writtenIds(
273
emitted: ReadonlyArray<OrchestrationV2DomainEvent>,
274
- type: "subagent.updated" | "node.updated" | "turn-item.updated",
274
+ type: "subagent.updated" | "node.updated" | "turn-item.updated" | "provider-turn.updated",
275
): ReadonlySet<string> {
276
return new Set(emitted.flatMap((event) => (event.type === type ? [event.payload.id] : [])));
277
}
@@ -359,7 +359,10 @@ export const endRunlessRootTurns = Effect.fn("subagentProjection.endRunlessRootT
359
>(input: {
360
readonly threadId: ThreadId;
361
readonly providerInstanceId: ProviderInstanceId;
362
- readonly projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "turnItems">;
362
+ readonly projection: Pick<
363
+ OrchestrationV2ThreadProjection,
364
+ "nodes" | "providerTurns" | "turnItems"
365
+ >;
366
readonly status: "completed" | "failed" | "interrupted" | "cancelled";
367
readonly completedAt?: DateTime.Utc;
368
readonly now: DateTime.Utc;
@@ -369,6 +372,7 @@ export const endRunlessRootTurns = Effect.fn("subagentProjection.endRunlessRootT
372
const { now, status, completedAt = now } = input;
373
const endedNodeIds = writtenIds(input.emitted, "node.updated");
374
const endedItemIds = writtenIds(input.emitted, "turn-item.updated");
375
+ const endedProviderTurnIds = writtenIds(input.emitted, "provider-turn.updated");
376
const scope = {
377
threadId: input.threadId,
378
providerInstanceId: input.providerInstanceId,
@@ -421,6 +425,23 @@ export const endRunlessRootTurns = Effect.fn("subagentProjection.endRunlessRootT
425
},
426
});
427
}
428
+ for (const providerTurn of input.projection.providerTurns) {
429
+ if (
430
+ providerTurn.nodeId !== node.id ||
431
+ providerTurn.runAttemptId !== null ||
432
+ !isOrchestrationV2WorkActive(providerTurn.status) ||
433
+ endedProviderTurnIds.has(providerTurn.id)
434
+ ) {
435
+ continue;
436
+ }
437
+ events.push({
438
+ ...scope,
439
+ id: yield* input.allocateEventId(),
440
+ type: "provider-turn.updated",
441
+ nodeId: node.id,
442
+ payload: { ...providerTurn, status, completedAt },
443
+ });
444
+ }
445
}
446
return events;
447
});