The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 13:30 UTC
fix(server): turn recovery covers interrupted and cancelled runs' abandoned turns (#34639)
Co-authored-by: Ali-Al1 <Ali-Al1@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Ali-Al1 committed
Nov 14, 2026 at 19:32 UTC
7d4f2423754e7d9ff73cb10448ef218f706cc515
2 files changed
+124
-22
apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts
+88
-4
@@ -396,12 +396,14 @@ function makeLocalCommandHarness(input: {
396
readonly failReadsAfterRunning?: boolean;
397
readonly recoveryReadFailure?: string;
398
readonly previousTurn?: {
399
- readonly runStatus: "failed" | "completed";
399
+ readonly runStatus: "failed" | "completed" | "interrupted";
400
readonly turnStatus: "running" | "completed" | "interrupted";
401
readonly differentThread?: boolean;
402
readonly liveSession?: boolean;
403
readonly interruptFailure?: boolean;
404
readonly interruptCurrentRun?: boolean;
405
+ /** Puts the running turn on an attempt a steering restart superseded. */
406
+ readonly supersededTurn?: boolean;
407
};
408
/**
409
* A side chat whose context targets this run, an earlier run that failed or was cancelled before
@@ -861,19 +863,57 @@ function makeLocalCommandHarness(input: {
863
if (input.previousTurn !== undefined) {
864
const previousRunId = RunId.make("previous-failed-run");
865
const previousAttemptId = RunAttemptId.make("previous-failed-attempt");
866
+ const steeredAttemptId = RunAttemptId.make("previous-steered-attempt");
867
const previousProviderThreadId = input.previousTurn.differentThread
868
? oldProviderThreadId
869
: providerThreadId;
870
projection = {
871
...projection,
872
+ attempts: [
873
+ {
874
+ id: previousAttemptId,
875
+ runId: previousRunId,
876
+ attemptOrdinal: 1,
877
+ rootNodeId: NodeId.make("previous-root"),
878
+ providerInstanceId: newInstanceId,
879
+ providerThreadId: previousProviderThreadId,
880
+ providerTurnId: null,
881
+ reason: "initial",
882
+ status: input.previousTurn.supersededTurn === true ? "superseded" : "interrupted",
883
+ startedAt: now,
884
+ completedAt: now,
885
+ },
886
+ ...(input.previousTurn.supersededTurn === true
887
+ ? [
888
+ {
889
+ id: steeredAttemptId,
890
+ runId: previousRunId,
891
+ attemptOrdinal: 2,
892
+ rootNodeId: NodeId.make("previous-steered-root"),
893
+ providerInstanceId: newInstanceId,
894
+ providerThreadId: previousProviderThreadId,
895
+ providerTurnId: null,
896
+ reason: "steering_restart" as const,
897
+ status: "interrupted" as const,
898
+ startedAt: null,
899
+ completedAt: now,
900
+ },
901
+ ]
902
+ : []),
903
+ ...projection.attempts,
904
+ ],
905
runs: [
906
{
907
...run,
908
id: previousRunId,
909
ordinal: 1,
910
status: input.previousTurn.runStatus,
875
- rootNodeId: NodeId.make("previous-root"),
876
- activeAttemptId: previousAttemptId,
911
+ rootNodeId:
912
+ input.previousTurn.supersededTurn === true
913
+ ? NodeId.make("previous-steered-root")
914
+ : NodeId.make("previous-root"),
915
+ activeAttemptId:
916
+ input.previousTurn.supersededTurn === true ? steeredAttemptId : previousAttemptId,
917
providerThreadId: previousProviderThreadId,
918
completedAt: now,
919
},
@@ -890,6 +930,23 @@ function makeLocalCommandHarness(input: {
930
},
931
],
932
providerTurns: [
933
+ // The steered attempt reached the provider once; only the earlier attempt's
934
+ // turn is still running when the run was interrupted.
935
+ ...(input.previousTurn.supersededTurn === true
936
+ ? [
937
+ {
938
+ id: ProviderTurnId.make("previous-steered-turn"),
939
+ providerThreadId: previousProviderThreadId,
940
+ nodeId: NodeId.make("previous-steered-root"),
941
+ runAttemptId: steeredAttemptId,
942
+ ordinal: 0,
943
+ nativeTurnRef: null,
944
+ status: "interrupted" as const,
945
+ startedAt: now,
946
+ completedAt: now,
947
+ },
948
+ ]
949
+ : []),
950
{
951
id: previousTurnId,
952
providerThreadId: previousProviderThreadId,
@@ -905,7 +962,8 @@ function makeLocalCommandHarness(input: {
962
};
963
}
964
let providerBusy =
908
- input.previousTurn?.runStatus === "failed" &&
965
+ (input.previousTurn?.runStatus === "failed" ||
966
+ input.previousTurn?.runStatus === "interrupted") &&
967
input.previousTurn.turnStatus === "running" &&
968
!input.previousTurn.differentThread &&
969
input.previousTurn.liveSession !== false;
@@ -1529,6 +1587,32 @@ effectIt.effect("recovers an abandoned failed turn before continuing its native
1587
}),
1588
);
1589
1590
+effectIt.effect(
1591
+ "recovers a running turn its interrupted run left behind on a superseded attempt",
1592
+ () =>
1593
+ Effect.gen(function* () {
1594
+ const harness = makeLocalCommandHarness({
1595
+ text: "Continue",
1596
+ previousTurn: { runStatus: "interrupted", turnStatus: "running", supersededTurn: true },
1597
+ });
1598
+
1599
+ yield* harness.start;
1600
+
1601
+ expect(harness.interruptTurn).toHaveBeenCalledOnce();
1602
+ // The steered attempt's turn was already interrupted in the fixture; the
1603
+ // repair this asserts is the running turn left on the superseded attempt.
1604
+ const abandoned = harness
1605
+ .projection()
1606
+ .providerTurns.find((turn) => turn.id === ProviderTurnId.make("previous-provider-turn"));
1607
+ expect(abandoned).toMatchObject({
1608
+ status: "interrupted",
1609
+ completedAt: expect.anything(),
1610
+ });
1611
+ expect(harness.projection().runs.at(-1)?.status).toBe("running");
1612
+ expect(harness.startRootRun).toHaveBeenCalledOnce();
1613
+ }),
1614
+);
1615
+
1616
effectIt.effect("sends new steering text when restarting a manual continuation", () =>
1617
Effect.gen(function* () {
1618
const harness = makeLocalCommandHarness({
apps/server/src/orchestration-v2/ProviderTurnStartService.ts
+36
-18
@@ -78,6 +78,14 @@ export class ProviderTurnStartError extends Schema.TaggedError<ProviderTurnStart
78
},
79
) {}
80
81
+/** Runs that ended without completing: their provider turn may still project as running. */
82
+const NON_COMPLETING_TERMINAL_RUN_STATUSES: ReadonlySet<OrchestrationV2Run["status"]> = new Set([
83
+ "failed",
84
+ "interrupted",
85
+ "cancelled",
86
+ "rolled_back",
87
+]);
88
+
89
const isProviderTurnStartError = Schema.is(ProviderTurnStartError);
90
const isProviderAdapterProtocolError = Schema.is(ProviderAdapter.ProviderAdapterProtocolError);
91
const isProviderAdapterOpenSessionError = Schema.is(
@@ -652,22 +660,31 @@ export const layer: Layer.Layer<
660
const existingSessionProjection = projection.providerSessions.find(
661
(candidate) => candidate.id === providerSessionId,
662
);
655
- const failedRuns = new Map(
656
- projection.runs
657
- .filter(
658
- (previous) =>
659
- previous.ordinal < run.ordinal &&
660
- previous.providerThreadId === providerThread.id &&
661
- previous.status === "failed",
662
- )
663
- .map((previous) => [previous.activeAttemptId, previous] as const),
664
- );
663
+ // A run that ended without completing can leave its provider turn projected as
664
+ // running when the agent ignored Stop or an interrupt timed out. Every attempt
665
+ // of such a run owns a possible abandoned turn, including attempts a steering
666
+ // restart superseded, and the session cannot be reused until those turns stop.
667
+ const abandonedRuns = new Map<string | null, OrchestrationV2Run>();
668
+ for (const previous of projection.runs) {
669
+ if (
670
+ previous.ordinal >= run.ordinal ||
671
+ previous.providerThreadId !== providerThread.id ||
672
+ !NON_COMPLETING_TERMINAL_RUN_STATUSES.has(previous.status)
673
+ ) {
674
+ continue;
675
+ }
676
+ if (previous.activeAttemptId !== null) {
677
+ abandonedRuns.set(previous.activeAttemptId, previous);
678
+ }
679
+ for (const attempt of projection.attempts) {
680
+ if (attempt.runId === previous.id) abandonedRuns.set(attempt.id, previous);
681
+ }
682
+ }
683
const abandonedTurns = projection.providerTurns.flatMap((turn) => {
666
- const previous = failedRuns.get(turn.runAttemptId);
684
+ const previous = abandonedRuns.get(turn.runAttemptId);
685
return turn.providerThreadId === providerThread.id &&
686
turn.status === "running" &&
669
- previous !== undefined &&
670
- turn.nodeId === previous.rootNodeId
687
+ previous !== undefined
688
? [{ turn, run: previous }]
689
: [];
690
});
@@ -675,8 +692,9 @@ export const layer: Layer.Layer<
692
if (abandonedTurn !== undefined) {
693
const liveSession = yield* providerSessions.get(providerSessionId);
694
if (Option.isSome(liveSession)) {
678
- // Older versions could fail the run without stopping its provider.
679
- // Recover that exact turn before reusing its native conversation.
695
+ // The run already ended without stopping its provider — a failed start, an
696
+ // ignored Stop, or an interrupt that never confirmed. Recover that exact
697
+ // turn before reusing its native conversation.
698
const recovered = yield* Effect.result(
699
liveSession.value.interruptTurn({
700
providerThread,
@@ -700,15 +718,15 @@ export const layer: Layer.Layer<
718
runId: run.id,
719
activeAttemptId: attempt.id,
720
expectedStatus: "starting",
703
- events: yield* Effect.forEach(abandonedTurns, ({ turn, run: failedRun }) =>
721
+ events: yield* Effect.forEach(abandonedTurns, ({ turn, run: abandonedRun }) =>
722
Effect.gen(function* () {
723
return {
724
id: yield* idAllocator.allocate.event({ threadId: projection.thread.id }),
725
type: "provider-turn.updated" as const,
726
threadId: projection.thread.id,
709
- runId: failedRun.id,
727
+ runId: abandonedRun.id,
728
nodeId: turn.nodeId,
711
- providerInstanceId: failedRun.providerInstanceId,
729
+ providerInstanceId: abandonedRun.providerInstanceId,
730
occurredAt: completedAt,
731
payload: { ...turn, status: "interrupted" as const, completedAt },
732
};