@devries-cameron20 / t3code-1 / commits / 7d4f242375
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): turn recovery covers interrupted and cancelled runs' abandoned turns (#34639)

Co-authored-by: Ali-Al1 <Ali-Al1@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Ali-Al1 committed Nov 14, 2026 at 19:32 UTC 7d4f2423754e7d9ff73cb10448ef218f706cc515
2 files changed +124 -22
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+88 -4
@@ -396,12 +396,14 @@ function makeLocalCommandHarness(input: {
396 readonly failReadsAfterRunning?: boolean;
397 readonly recoveryReadFailure?: string;
398 readonly previousTurn?: {
399 - readonly runStatus: "failed" | "completed";
399 + readonly runStatus: "failed" | "completed" | "interrupted";
400 readonly turnStatus: "running" | "completed" | "interrupted";
401 readonly differentThread?: boolean;
402 readonly liveSession?: boolean;
403 readonly interruptFailure?: boolean;
404 readonly interruptCurrentRun?: boolean;
405 + /** Puts the running turn on an attempt a steering restart superseded. */
406 + readonly supersededTurn?: boolean;
407 };
408 /**
409 * A side chat whose context targets this run, an earlier run that failed or was cancelled before
@@ -861,19 +863,57 @@ function makeLocalCommandHarness(input: {
863 if (input.previousTurn !== undefined) {
864 const previousRunId = RunId.make("previous-failed-run");
865 const previousAttemptId = RunAttemptId.make("previous-failed-attempt");
866 + const steeredAttemptId = RunAttemptId.make("previous-steered-attempt");
867 const previousProviderThreadId = input.previousTurn.differentThread
868 ? oldProviderThreadId
869 : providerThreadId;
870 projection = {
871 ...projection,
872 + attempts: [
873 + {
874 + id: previousAttemptId,
875 + runId: previousRunId,
876 + attemptOrdinal: 1,
877 + rootNodeId: NodeId.make("previous-root"),
878 + providerInstanceId: newInstanceId,
879 + providerThreadId: previousProviderThreadId,
880 + providerTurnId: null,
881 + reason: "initial",
882 + status: input.previousTurn.supersededTurn === true ? "superseded" : "interrupted",
883 + startedAt: now,
884 + completedAt: now,
885 + },
886 + ...(input.previousTurn.supersededTurn === true
887 + ? [
888 + {
889 + id: steeredAttemptId,
890 + runId: previousRunId,
891 + attemptOrdinal: 2,
892 + rootNodeId: NodeId.make("previous-steered-root"),
893 + providerInstanceId: newInstanceId,
894 + providerThreadId: previousProviderThreadId,
895 + providerTurnId: null,
896 + reason: "steering_restart" as const,
897 + status: "interrupted" as const,
898 + startedAt: null,
899 + completedAt: now,
900 + },
901 + ]
902 + : []),
903 + ...projection.attempts,
904 + ],
905 runs: [
906 {
907 ...run,
908 id: previousRunId,
909 ordinal: 1,
910 status: input.previousTurn.runStatus,
875 - rootNodeId: NodeId.make("previous-root"),
876 - activeAttemptId: previousAttemptId,
911 + rootNodeId:
912 + input.previousTurn.supersededTurn === true
913 + ? NodeId.make("previous-steered-root")
914 + : NodeId.make("previous-root"),
915 + activeAttemptId:
916 + input.previousTurn.supersededTurn === true ? steeredAttemptId : previousAttemptId,
917 providerThreadId: previousProviderThreadId,
918 completedAt: now,
919 },
@@ -890,6 +930,23 @@ function makeLocalCommandHarness(input: {
930 },
931 ],
932 providerTurns: [
933 + // The steered attempt reached the provider once; only the earlier attempt's
934 + // turn is still running when the run was interrupted.
935 + ...(input.previousTurn.supersededTurn === true
936 + ? [
937 + {
938 + id: ProviderTurnId.make("previous-steered-turn"),
939 + providerThreadId: previousProviderThreadId,
940 + nodeId: NodeId.make("previous-steered-root"),
941 + runAttemptId: steeredAttemptId,
942 + ordinal: 0,
943 + nativeTurnRef: null,
944 + status: "interrupted" as const,
945 + startedAt: now,
946 + completedAt: now,
947 + },
948 + ]
949 + : []),
950 {
951 id: previousTurnId,
952 providerThreadId: previousProviderThreadId,
@@ -905,7 +962,8 @@ function makeLocalCommandHarness(input: {
962 };
963 }
964 let providerBusy =
908 - input.previousTurn?.runStatus === "failed" &&
965 + (input.previousTurn?.runStatus === "failed" ||
966 + input.previousTurn?.runStatus === "interrupted") &&
967 input.previousTurn.turnStatus === "running" &&
968 !input.previousTurn.differentThread &&
969 input.previousTurn.liveSession !== false;
@@ -1529,6 +1587,32 @@ effectIt.effect("recovers an abandoned failed turn before continuing its native
1587 }),
1588 );
1589
1590 +effectIt.effect(
1591 + "recovers a running turn its interrupted run left behind on a superseded attempt",
1592 + () =>
1593 + Effect.gen(function* () {
1594 + const harness = makeLocalCommandHarness({
1595 + text: "Continue",
1596 + previousTurn: { runStatus: "interrupted", turnStatus: "running", supersededTurn: true },
1597 + });
1598 +
1599 + yield* harness.start;
1600 +
1601 + expect(harness.interruptTurn).toHaveBeenCalledOnce();
1602 + // The steered attempt's turn was already interrupted in the fixture; the
1603 + // repair this asserts is the running turn left on the superseded attempt.
1604 + const abandoned = harness
1605 + .projection()
1606 + .providerTurns.find((turn) => turn.id === ProviderTurnId.make("previous-provider-turn"));
1607 + expect(abandoned).toMatchObject({
1608 + status: "interrupted",
1609 + completedAt: expect.anything(),
1610 + });
1611 + expect(harness.projection().runs.at(-1)?.status).toBe("running");
1612 + expect(harness.startRootRun).toHaveBeenCalledOnce();
1613 + }),
1614 +);
1615 +
1616 effectIt.effect("sends new steering text when restarting a manual continuation", () =>
1617 Effect.gen(function* () {
1618 const harness = makeLocalCommandHarness({
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+36 -18
@@ -78,6 +78,14 @@ export class ProviderTurnStartError extends Schema.TaggedError<ProviderTurnStart
78 },
79 ) {}
80
81 +/** Runs that ended without completing: their provider turn may still project as running. */
82 +const NON_COMPLETING_TERMINAL_RUN_STATUSES: ReadonlySet<OrchestrationV2Run["status"]> = new Set([
83 + "failed",
84 + "interrupted",
85 + "cancelled",
86 + "rolled_back",
87 +]);
88 +
89 const isProviderTurnStartError = Schema.is(ProviderTurnStartError);
90 const isProviderAdapterProtocolError = Schema.is(ProviderAdapter.ProviderAdapterProtocolError);
91 const isProviderAdapterOpenSessionError = Schema.is(
@@ -652,22 +660,31 @@ export const layer: Layer.Layer<
660 const existingSessionProjection = projection.providerSessions.find(
661 (candidate) => candidate.id === providerSessionId,
662 );
655 - const failedRuns = new Map(
656 - projection.runs
657 - .filter(
658 - (previous) =>
659 - previous.ordinal < run.ordinal &&
660 - previous.providerThreadId === providerThread.id &&
661 - previous.status === "failed",
662 - )
663 - .map((previous) => [previous.activeAttemptId, previous] as const),
664 - );
663 + // A run that ended without completing can leave its provider turn projected as
664 + // running when the agent ignored Stop or an interrupt timed out. Every attempt
665 + // of such a run owns a possible abandoned turn, including attempts a steering
666 + // restart superseded, and the session cannot be reused until those turns stop.
667 + const abandonedRuns = new Map<string | null, OrchestrationV2Run>();
668 + for (const previous of projection.runs) {
669 + if (
670 + previous.ordinal >= run.ordinal ||
671 + previous.providerThreadId !== providerThread.id ||
672 + !NON_COMPLETING_TERMINAL_RUN_STATUSES.has(previous.status)
673 + ) {
674 + continue;
675 + }
676 + if (previous.activeAttemptId !== null) {
677 + abandonedRuns.set(previous.activeAttemptId, previous);
678 + }
679 + for (const attempt of projection.attempts) {
680 + if (attempt.runId === previous.id) abandonedRuns.set(attempt.id, previous);
681 + }
682 + }
683 const abandonedTurns = projection.providerTurns.flatMap((turn) => {
666 - const previous = failedRuns.get(turn.runAttemptId);
684 + const previous = abandonedRuns.get(turn.runAttemptId);
685 return turn.providerThreadId === providerThread.id &&
686 turn.status === "running" &&
669 - previous !== undefined &&
670 - turn.nodeId === previous.rootNodeId
687 + previous !== undefined
688 ? [{ turn, run: previous }]
689 : [];
690 });
@@ -675,8 +692,9 @@ export const layer: Layer.Layer<
692 if (abandonedTurn !== undefined) {
693 const liveSession = yield* providerSessions.get(providerSessionId);
694 if (Option.isSome(liveSession)) {
678 - // Older versions could fail the run without stopping its provider.
679 - // Recover that exact turn before reusing its native conversation.
695 + // The run already ended without stopping its provider — a failed start, an
696 + // ignored Stop, or an interrupt that never confirmed. Recover that exact
697 + // turn before reusing its native conversation.
698 const recovered = yield* Effect.result(
699 liveSession.value.interruptTurn({
700 providerThread,
@@ -700,15 +718,15 @@ export const layer: Layer.Layer<
718 runId: run.id,
719 activeAttemptId: attempt.id,
720 expectedStatus: "starting",
703 - events: yield* Effect.forEach(abandonedTurns, ({ turn, run: failedRun }) =>
721 + events: yield* Effect.forEach(abandonedTurns, ({ turn, run: abandonedRun }) =>
722 Effect.gen(function* () {
723 return {
724 id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }),
725 type: "provider-turn.updated" as const,
726 threadId: projection.thread.id,
709 - runId: failedRun.id,
727 + runId: abandonedRun.id,
728 nodeId: turn.nodeId,
711 - providerInstanceId: failedRun.providerInstanceId,
729 + providerInstanceId: abandonedRun.providerInstanceId,
730 occurredAt: completedAt,
731 payload: { ...turn, status: "interrupted" as const, completedAt },
732 };