@devries-cameron20 / t3code Mirror · read-only / commits / 725ada6eaa
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 17:00 UTC

fix(server): ended roots no longer leave provider turns running (#22846)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 25, 2026 at 15:20 UTC 725ada6eaa32d4a32c361ee42a1717a3e65568d1
8 files changed +291 -21
apps/server/src/orchestration-v2/Orchestrator.ts
+2 -2
@@ -9112,7 +9112,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9112 }
9113 childThreadIds.add(childThreadId);
9114 const child = yield* projectionStore
9115 - .getThreadRecords(childThreadId, ["nodes", "subagents", "turnItems"], {
9115 + .getThreadRecords(childThreadId, ["nodes", "providerTurns", "subagents", "turnItems"], {
9116 // Runless items are the subagent's; any run there is the user's own.
9117 turnItemRunIds: [null],
9118 turnItemStatuses: ["pending", "running", "waiting"],
@@ -9671,7 +9671,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
9671 yield* Ref.update(events, (current) => [...current, ...ended]);
9672 if (subagent.childThreadId !== null) {
9673 const child = yield* projectionStore
9674 - .getThreadRecords(subagent.childThreadId, ["nodes", "turnItems"], {
9674 + .getThreadRecords(subagent.childThreadId, ["nodes", "providerTurns", "turnItems"], {
9675 turnItemRunIds: [null],
9676 turnItemStatuses: ["pending", "running", "waiting"],
9677 })
apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
+100
@@ -12,6 +12,7 @@ import {
12 ProviderInstanceId,
13 ProviderSessionId,
14 ProviderThreadId,
15 + ProviderTurnId,
16 RunId,
17 RuntimeRequestId,
18 ThreadId,
@@ -966,3 +967,102 @@ it.effect("marks fork descendants unreadable when their source is missing or cor
967 );
968 }).pipe(Effect.provide(layerTest)),
969 );
970 +
971 +it.effect("startup ends the provider turn of a native child root it cancels", () =>
972 + Effect.gen(function* () {
973 + const projections = yield* ProjectionStore.ProjectionStoreV2;
974 + const recovery = yield* ProviderRuntimeRecovery.ProviderRuntimeRecoveryService;
975 + const now = yield* DateTime.now;
976 + const parentThreadId = yield* createThread("native-child-root-turn-parent");
977 + const threadId = yield* createThread("native-child-root-turn");
978 + const rootNodeId = NodeId.make(`node:${threadId}:thread-root`);
979 + const providerThreadId = ProviderThreadId.make(`provider-thread:${threadId}`);
980 + const providerTurnId = ProviderTurnId.make(`provider-turn:${threadId}`);
981 + yield* projections.apply({
982 + id: EventId.make(`event:${threadId}:root`),
983 + type: "node.updated",
984 + threadId,
985 + nodeId: rootNodeId,
986 + driver,
987 + occurredAt: now,
988 + payload: {
989 + id: rootNodeId,
990 + threadId,
991 + runId: null,
992 + parentNodeId: null,
993 + rootNodeId,
994 + kind: "root_turn",
995 + status: "running",
996 + countsForRun: false,
997 + providerThreadId,
998 + providerTurnId,
999 + nativeItemRef: null,
1000 + runtimeRequestId: null,
1001 + checkpointScopeId: null,
1002 + startedAt: now,
1003 + completedAt: null,
1004 + },
1005 + });
1006 + // A native subagent's turn belongs to no run attempt.
1007 + yield* projections.apply({
1008 + id: EventId.make(`event:${threadId}:provider-turn`),
1009 + type: "provider-turn.updated",
1010 + threadId,
1011 + nodeId: rootNodeId,
1012 + driver,
1013 + occurredAt: now,
1014 + payload: {
1015 + id: providerTurnId,
1016 + providerThreadId,
1017 + nodeId: rootNodeId,
1018 + runAttemptId: null,
1019 + nativeTurnRef: null,
1020 + ordinal: 1,
1021 + status: "running",
1022 + startedAt: now,
1023 + completedAt: null,
1024 + },
1025 + });
1026 +
1027 + const subagentId = NodeId.make(`node:${parentThreadId}:subagent`);
1028 + yield* projections.apply({
1029 + id: EventId.make(`event:${parentThreadId}:subagent`),
1030 + type: "subagent.updated",
1031 + threadId: parentThreadId,
1032 + occurredAt: now,
1033 + payload: {
1034 + id: subagentId,
1035 + threadId: parentThreadId,
1036 + runId: null,
1037 + parentNodeId: NodeId.make(`node:${parentThreadId}:root`),
1038 + origin: "provider_native",
1039 + createdBy: "agent",
1040 + driver,
1041 + providerInstanceId,
1042 + providerThreadId,
1043 + childThreadId: threadId,
1044 + nativeTaskRef: null,
1045 + title: null,
1046 + model: null,
1047 + prompt: "sleep 300",
1048 + status: "running",
1049 + result: null,
1050 + startedAt: now,
1051 + completedAt: null,
1052 + updatedAt: now,
1053 + },
1054 + });
1055 +
1056 + yield* recovery.reconcile("startup");
1057 +
1058 + const projection = yield* projections.getThreadProjection(threadId);
1059 + assert.deepEqual(
1060 + {
1061 + root: projection.nodes.find((node) => node.id === rootNodeId)?.status,
1062 + providerTurn: projection.providerTurns.find((turn) => turn.id === providerTurnId)?.status,
1063 + },
1064 + { root: "cancelled", providerTurn: "cancelled" },
1065 + "a root startup recovery cancelled left its provider turn running",
1066 + );
1067 + }).pipe(Effect.provide(layerRecoveryTest)),
1068 +);
apps/server/src/orchestration-v2/ProjectionStore.ts
+18 -6
@@ -4605,14 +4605,26 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4605 FROM orchestration_v2_projection_provider_turns AS provider_turn
4606 WHERE provider_turn.thread_id = ${threadId}
4607 AND provider_turn.status IN ('pending', 'starting', 'running', 'waiting')
4608 - AND provider_turn.run_attempt_id IN (
4609 - SELECT attempt_id FROM orchestration_v2_projection_run_attempts
4610 - WHERE thread_id = ${threadId}
4611 - AND run_id IN (
4612 - SELECT run_id FROM orchestration_v2_projection_runs
4608 + AND (
4609 + provider_turn.run_attempt_id IN (
4610 + SELECT attempt_id FROM orchestration_v2_projection_run_attempts
4611 + WHERE thread_id = ${threadId}
4612 + AND run_id IN (
4613 + SELECT run_id FROM orchestration_v2_projection_runs
4614 + WHERE thread_id = ${threadId}
4615 + AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
4616 + )
4617 + )
4618 + OR (
4619 + provider_turn.run_attempt_id IS NULL
4620 + AND provider_turn.node_id IN (
4621 + SELECT node_id FROM orchestration_v2_projection_nodes
4622 WHERE thread_id = ${threadId}
4614 - AND status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
4623 + AND run_id IS NULL
4624 + AND kind = 'root_turn'
4625 + AND status IN ('pending', 'running', 'waiting')
4626 )
4627 + )
4628 )
4629 ORDER BY provider_turn.provider_thread_id ASC, provider_turn.ordinal ASC
4630 `,
apps/server/src/orchestration-v2/RunExecutionService.test.ts
+101
@@ -11,6 +11,7 @@ import {
11 type OrchestrationV2DomainEvent,
12 type OrchestrationV2ExecutionNode,
13 type OrchestrationV2ProviderThread,
14 + type OrchestrationV2ProviderTurn,
15 type OrchestrationV2Run,
16 type OrchestrationV2RunAttempt,
17 type OrchestrationV2Subagent,
@@ -2624,6 +2625,106 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c
2625 }),
2626 );
2627
2628 +it.effect.each([
2629 + { status: "completed", failure: "later-stream", interrupt: "succeeds" },
2630 + { status: "failed", failure: "later-stream", interrupt: "succeeds" },
2631 + { status: "completed", failure: "terminal-ingest", interrupt: "succeeds" },
2632 + { status: "completed", failure: "terminal-ingest", interrupt: "fails" },
2633 +] as const)(
2634 + "stores a $status root provider turn snapshot after a $failure failure only if the interrupt succeeds (it $interrupt)",
2635 + ({ status, failure, interrupt }) =>
2636 + Effect.gen(function* () {
2637 + const now = yield* DateTime.now;
2638 + const terminalAt = DateTime.subtract(now, { seconds: 1 });
2639 + const key = `terminal-snapshot:${status}:${failure}:${interrupt}`;
2640 + const durableTurns = yield* Ref.make<
2641 + ReadonlyMap<ProviderTurnId, OrchestrationV2ProviderTurn>
2642 + >(new Map());
2643 + const persist = (turn: OrchestrationV2ProviderTurn) =>
2644 + Ref.update(durableTurns, (current) => new Map(current).set(turn.id, turn));
2645 + const interrupts = yield* Ref.make<
2646 + ReadonlyArray<ProviderAdapter.ProviderAdapterV2InterruptInput>
2647 + >([]);
2648 + const ids = backgroundScenarioIds(key);
2649 + const rootTurn = (
2650 + turnStatus: OrchestrationV2ProviderTurn["status"],
2651 + completedAt: DateTime.Utc | null,
2652 + ) => ({
2653 + type: "provider_turn.updated" as const,
2654 + driver,
2655 + providerTurn: {
2656 + id: ids.rootProviderTurnId,
2657 + providerThreadId: ids.providerThreadId,
2658 + nodeId: ids.rootNodeId,
2659 + runAttemptId: ids.attemptId,
2660 + nativeTurnRef: null,
2661 + ordinal: 1,
2662 + status: turnStatus,
2663 + startedAt: terminalAt,
2664 + completedAt,
2665 + },
2666 + });
2667 + const { observed, finalEvents } = yield* captureRootRunTermination({
2668 + key,
2669 + shouldFinalizeRun: () => Effect.succeed(true),
2670 + events: () =>
2671 + Stream.make(rootTurn("running", null), rootTurn(status, terminalAt)).pipe(
2672 + Stream.concat(
2673 + failure === "later-stream"
2674 + ? Stream.fail(
2675 + new ProviderAdapter.ProviderAdapterEventStreamError({
2676 + driver,
2677 + providerSessionId: ProviderSessionId.make(`session:${key}`),
2678 + cause: "provider process exited",
2679 + }),
2680 + )
2681 + : Stream.empty,
2682 + ),
2683 + ),
2684 + ingestNormalized: ({ event }) =>
2685 + event.type !== "provider_turn.updated"
2686 + ? Effect.succeed([])
2687 + : failure === "terminal-ingest" && event.providerTurn.status !== "running"
2688 + ? Effect.fail(
2689 + new ProviderEventIngestor.ProviderEventPublishError({
2690 + providerSessionId: ProviderSessionId.make(`session:${key}`),
2691 + eventCount: 1,
2692 + cause: new Error("SQLITE_FULL: database or disk is full"),
2693 + }),
2694 + )
2695 + : persist(event.providerTurn).pipe(Effect.as([])),
2696 + interruptTurn: (input) =>
2697 + Ref.update(interrupts, (current) => [...current, input]).pipe(
2698 + Effect.andThen(
2699 + interrupt === "fails"
2700 + ? Effect.fail(
2701 + new ProviderAdapter.ProviderAdapterInterruptError({
2702 + driver,
2703 + providerThreadId: input.providerThread.id,
2704 + providerTurnId: input.providerTurnId,
2705 + cause: "provider already exited",
2706 + }),
2707 + )
2708 + : Effect.void,
2709 + ),
2710 + ),
2711 + beforeTerminalWrite: (request) =>
2712 + Effect.forEach(request.events, (event) =>
2713 + event.type === "provider-turn.updated" ? persist(event.payload) : Effect.void,
2714 + ).pipe(Effect.as(true)),
2715 + });
2716 + const calls = yield* Ref.get(interrupts);
2717 + assert.equal(calls.length, 1);
2718 + assert.isTrue(calls[0]?.requestRuntimeRestart);
2719 + assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]);
2720 + assert.isTrue(finalEvents.some((event) => event.type === "run.updated"));
2721 + const durable = (yield* Ref.get(durableTurns)).get(ids.rootProviderTurnId);
2722 + // A failed interrupt keeps the stored turn running for the next start to retry.
2723 + assert.equal(durable?.status, interrupt === "fails" ? "running" : status);
2724 + assert.deepEqual(durable?.completedAt, interrupt === "fails" ? null : terminalAt);
2725 + }),
2726 +);
2727 +
2728 it.effect("does not interrupt an ended provider when its terminal write fails", () =>
2729 Effect.gen(function* () {
2730 const interrupts = yield* Ref.make(0);
apps/server/src/orchestration-v2/RunExecutionService.ts
+23 -11
@@ -620,7 +620,8 @@ export const layer: Layer.Layer<
620 readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect<boolean, never>;
621 readonly loadRunInterruptReason?: () => Effect.Effect<string | undefined, never>;
622 readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection;
623 - readonly interruptedProviderTurn?: OrchestrationV2ProviderTurn;
623 + // Root turn snapshot written with the run when T3 ends the run itself.
624 + readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn;
625 readonly terminal: ProviderTerminalEvent;
626 readonly completedAt?: DateTime.Utc;
627 readonly failureItemPersisted: boolean;
@@ -747,7 +748,7 @@ export const layer: Layer.Layer<
748 : [],
749 events: [
750 ...cascadedSubagentEvents,
750 - ...(input.interruptedProviderTurn === undefined
751 + ...(input.abandonedProviderTurn === undefined
752 ? []
753 : [
754 {
@@ -758,11 +759,7 @@ export const layer: Layer.Layer<
759 nodeId: input.rootNode.id,
760 providerInstanceId: input.run.providerInstanceId,
761 occurredAt: completedAt,
761 - payload: {
762 - ...input.interruptedProviderTurn,
763 - status: "interrupted" as const,
764 - completedAt,
765 - },
762 + payload: input.abandonedProviderTurn,
763 },
764 ]),
765 ...(finalizedAttempt === null
@@ -1482,8 +1479,9 @@ export const layer: Layer.Layer<
1479 if (yield* Ref.get(rootRunFinalized)) return;
1480 const providerThread = yield* Ref.get(latestProviderThread);
1481 const routing = yield* Ref.get(eventRouting);
1482 + const rootProviderTurn = yield* Ref.get(latestRootProviderTurn);
1483 // Stop background work too; the failed consumer cannot persist terminal events.
1486 - const interruptedProviderTurn =
1484 + const abandonedProviderTurn =
1485 routing.rootProviderTurnId !== null && !routing.rootTurnEnded
1486 ? yield* input.session
1487 .interruptTurn({
@@ -1492,14 +1490,28 @@ export const layer: Layer.Layer<
1490 requestRuntimeRestart: true,
1491 })
1492 .pipe(
1495 - Effect.as(yield* Ref.get(latestRootProviderTurn)),
1493 + Effect.as(true),
1494 Effect.catchCauseIf(
1495 (interruptCause) => !Cause.hasInterruptsOnly(interruptCause),
1496 (interruptCause) =>
1497 Effect.logWarning(
1498 "orchestration V2 failed to interrupt turn after ingestion failure",
1499 { runId: input.run.id, cause: interruptCause },
1502 - ).pipe(Effect.as(null)),
1500 + ).pipe(Effect.as(false)),
1501 + ),
1502 + // A failed interrupt leaves the stored turn running so the next start
1503 + // retries it. The snapshot is cached before it is published, so it may
1504 + // not be stored: replay a terminal one as-is.
1505 + Effect.map((interrupted) =>
1506 + !interrupted || rootProviderTurn === null
1507 + ? null
1508 + : isTerminalProviderTurnStatus(rootProviderTurn.status)
1509 + ? rootProviderTurn
1510 + : {
1511 + ...rootProviderTurn,
1512 + status: "interrupted" as const,
1513 + completedAt,
1514 + },
1515 ),
1516 )
1517 : null;
@@ -1517,7 +1529,7 @@ export const layer: Layer.Layer<
1529 expectedStatus: "running",
1530 },
1531 openRunOwnedSubagents: openSubagents,
1520 - ...(interruptedProviderTurn === null ? {} : { interruptedProviderTurn }),
1532 + ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }),
1533 completedAt,
1534 terminal: makeFailedTerminalEvent(
1535 makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }),
apps/server/src/orchestration-v2/SubagentProjection.test.ts
+1
@@ -375,6 +375,7 @@ it.effect("ends a runless root turn's items when its node already ended", () =>
375 providerInstanceId: parentProviderInstanceId,
376 projection: {
377 nodes: [root],
378 + providerTurns: [],
379 turnItems: [item("turn-item:command"), item("turn-item:reasoning")],
380 } as never,
381 status: "cancelled",
apps/server/src/orchestration-v2/ThreadStop.test.ts
+23
@@ -1335,6 +1335,25 @@ it.effect.each(["missing session", "completed", "failed", "cancelled"] as const)
1335 occurredAt: now,
1336 payload: childRoot,
1337 });
1338 + const childTurnId = ProviderTurnId.make("turn:native-stop-child");
1339 + yield* projections.apply({
1340 + id: EventId.make("event:native-child-stop:child-turn"),
1341 + type: "provider-turn.updated",
1342 + threadId: childRoot.threadId,
1343 + nodeId: childRoot.id,
1344 + occurredAt: now,
1345 + payload: {
1346 + id: childTurnId,
1347 + providerThreadId: ProviderThreadId.make("provider-thread:native-stop-child"),
1348 + nodeId: childRoot.id,
1349 + runAttemptId: null,
1350 + nativeTurnRef: null,
1351 + ordinal: 1,
1352 + status: "running",
1353 + startedAt: now,
1354 + completedAt: null,
1355 + },
1356 + });
1357 const providerThread = before.providerThreads.find(
1358 (thread) => thread.id === run.providerThreadId,
1359 )!;
@@ -1499,6 +1518,10 @@ it.effect.each(["missing session", "completed", "failed", "cancelled"] as const)
1518 );
1519 const child = yield* orchestrator.getThreadProjection(childRoot.threadId);
1520 assert.equal(child.nodes.find((node) => node.id === childRoot.id)?.status, confirmedStatus);
1521 + assert.equal(
1522 + child.providerTurns.find((turn) => turn.id === childTurnId)?.status,
1523 + confirmedStatus,
1524 + );
1525 assert.isNotNull(child.nodes.find((node) => node.id === childRoot.id)?.completedAt);
1526 assert.equal(settled.subagents.find((task) => task.id === sibling.id)?.status, "running");
1527 assert.deepEqual(yield* threadState(threadId), stateBefore);
packages/provider-core/src/server/subagentProjection.ts
+23 -2
@@ -271,7 +271,7 @@ export function delegatedTaskProgress(projection: {
271
272 function writtenIds(
273 emitted: ReadonlyArray<OrchestrationV2DomainEvent>,
274 - type: "subagent.updated" | "node.updated" | "turn-item.updated",
274 + type: "subagent.updated" | "node.updated" | "turn-item.updated" | "provider-turn.updated",
275 ): ReadonlySet<string> {
276 return new Set(emitted.flatMap((event) => (event.type === type ? [event.payload.id] : [])));
277 }
@@ -359,7 +359,10 @@ export const endRunlessRootTurns = Effect.fn("subagentProjection.endRunlessRootT
359 >(input: {
360 readonly threadId: ThreadId;
361 readonly providerInstanceId: ProviderInstanceId;
362 - readonly projection: Pick<OrchestrationV2ThreadProjection, "nodes" | "turnItems">;
362 + readonly projection: Pick<
363 + OrchestrationV2ThreadProjection,
364 + "nodes" | "providerTurns" | "turnItems"
365 + >;
366 readonly status: "completed" | "failed" | "interrupted" | "cancelled";
367 readonly completedAt?: DateTime.Utc;
368 readonly now: DateTime.Utc;
@@ -369,6 +372,7 @@ export const endRunlessRootTurns = Effect.fn("subagentProjection.endRunlessRootT
372 const { now, status, completedAt = now } = input;
373 const endedNodeIds = writtenIds(input.emitted, "node.updated");
374 const endedItemIds = writtenIds(input.emitted, "turn-item.updated");
375 + const endedProviderTurnIds = writtenIds(input.emitted, "provider-turn.updated");
376 const scope = {
377 threadId: input.threadId,
378 providerInstanceId: input.providerInstanceId,
@@ -421,6 +425,23 @@ export const endRunlessRootTurns = Effect.fn("subagentProjection.endRunlessRootT
425 },
426 });
427 }
428 + for (const providerTurn of input.projection.providerTurns) {
429 + if (
430 + providerTurn.nodeId !== node.id ||
431 + providerTurn.runAttemptId !== null ||
432 + !isOrchestrationV2WorkActive(providerTurn.status) ||
433 + endedProviderTurnIds.has(providerTurn.id)
434 + ) {
435 + continue;
436 + }
437 + events.push({
438 + ...scope,
439 + id: yield* input.allocateEventId(),
440 + type: "provider-turn.updated",
441 + nodeId: node.id,
442 + payload: { ...providerTurn, status, completedAt },
443 + });
444 + }
445 }
446 return events;
447 });