The best way to control Claude Code, Codex and more from any computer or phone
fix(server): preserve the crash recovery budget across restarts (#25603)
Co-authored-by: Tin Zeljar <35663757+Zheruel@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
maria committed
Nov 14, 2026 at 23:47 UTC
a68c367e4ff3d31a4a43493f6b22f83de1c6eb35
11 files changed
+282
-13
apps/server/src/orchestration-v2/EffectOutbox.ts
+12
@@ -35,6 +35,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([
35
Schema.Struct({
36
type: Schema.Literal("provider-runtime.continue"),
37
sourceRunId: RunId,
38
+ processLossCount: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))),
39
}),
40
Schema.Struct({
41
type: Schema.Literal("provider-session.detach"),
@@ -705,6 +706,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
706
const cutoff = DateTime.formatIso(
707
DateTime.subtractDuration(yield* DateTime.now, SETTLED_EFFECT_RETENTION),
708
);
709
+ // Live continuations retain their preceding intent's cumulative crash budget.
710
// Separate batch statements release the write lock between batches.
711
const deleteBatch = sql<{ readonly effect_id: string }>`
712
DELETE FROM orchestration_v2_effect_outbox
@@ -713,6 +715,16 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
715
FROM orchestration_v2_effect_outbox
716
WHERE status IN ('succeeded', 'cancelled')
717
AND completed_at < ${cutoff}
718
+ AND NOT (
719
+ effect_type = 'provider-runtime.continue'
720
+ AND EXISTS (
721
+ SELECT 1 FROM orchestration_v2_projection_runs AS recovery_run
722
+ WHERE recovery_run.thread_id = orchestration_v2_effect_outbox.thread_id
723
+ AND recovery_run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
724
+ AND json_extract(recovery_run.payload_json, '$.restartContinuationOfRunId') =
725
+ json_extract(orchestration_v2_effect_outbox.payload_json, '$.sourceRunId')
726
+ )
727
+ )
728
LIMIT ${PRUNE_BATCH_SIZE}
729
)
730
RETURNING effect_id
apps/server/src/orchestration-v2/EffectWorker.test.ts
+60
@@ -4,6 +4,8 @@ import {
4
CheckpointScopeId,
5
CommandId,
6
NodeId,
7
+ ProjectId,
8
+ ProviderInstanceId,
9
ProviderSessionId,
10
ProviderThreadId,
11
ProviderTurnId,
@@ -12,6 +14,7 @@ import {
14
RuntimeRequestId,
15
ThreadId,
16
VcsProcessExitError,
17
+ type OrchestrationV2ThreadProjection,
18
} from "@t3tools/contracts";
19
import * as Cause from "effect/Cause";
20
import * as DateTime from "effect/DateTime";
@@ -777,3 +780,60 @@ it.effect.each(["confirmed", "unconfirmed", "failed"] as const)(
780
);
781
}),
782
);
783
+
784
+it.effect.each([3, 4])(
785
+ "enforces process-loss count %s after 22 minutes through the executor",
786
+ (processLossCount) =>
787
+ Effect.gen(function* () {
788
+ const now = yield* DateTime.now;
789
+ const instanceId = ProviderInstanceId.make("codex");
790
+ const events = yield* Ref.make<ReadonlyArray<string>>([]);
791
+ const layer = layerExecutorFor({
792
+ events,
793
+ continueAfterRestart: true,
794
+ threads: {
795
+ getThreadRecords: () =>
796
+ Effect.succeed({
797
+ thread: {
798
+ id: threadId,
799
+ projectId: ProjectId.make("project:executor-recovery"),
800
+ providerInstanceId: instanceId,
801
+ archivedAt: null,
802
+ deletedAt: null,
803
+ lineage: { parentThreadId: null, relationshipToParent: null },
804
+ },
805
+ runs: [
806
+ {
807
+ id: runId,
808
+ providerInstanceId: instanceId,
809
+ status: "cancelled",
810
+ requestedAt: DateTime.subtract(now, { minutes: 22 }),
811
+ completedAt: now,
812
+ ordinal: 1,
813
+ },
814
+ ],
815
+ messages: [],
816
+ providerTurns: [],
817
+ runtimeRequests: [],
818
+ turnItems: [],
819
+ attempts: [],
820
+ } as unknown as OrchestrationV2ThreadProjection),
821
+ dispatch: () =>
822
+ Ref.update(events, (entries) => [...entries, "dispatch"]).pipe(Effect.as({} as never)),
823
+ recoverDelegatedTask: () =>
824
+ Ref.update(events, (entries) => [...entries, "recover-child"]),
825
+ },
826
+ });
827
+ yield* Effect.gen(function* () {
828
+ const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
829
+ yield* executor.execute({
830
+ ...restartEffect(now, { type: "detach" }),
831
+ request: { type: "provider-runtime.continue", sourceRunId: runId, processLossCount },
832
+ });
833
+ }).pipe(Effect.provide(layer));
834
+ assert.deepEqual(
835
+ yield* Ref.get(events),
836
+ processLossCount > 3 ? ["recover-child"] : ["dispatch", "recover-child"],
837
+ );
838
+ }),
839
+);
apps/server/src/orchestration-v2/EffectWorker.ts
+7
-1
@@ -160,7 +160,13 @@ export const layerExecutor: Layer.Layer<
160
switch (effect.request.type) {
161
case "provider-runtime.continue": {
162
const sourceRunId = effect.request.sourceRunId;
163
- return continueRestartedRun({ threadId: effect.threadId, sourceRunId }).pipe(
163
+ return continueRestartedRun({
164
+ threadId: effect.threadId,
165
+ sourceRunId,
166
+ ...(effect.request.processLossCount === undefined
167
+ ? {}
168
+ : { processLossCount: effect.request.processLossCount }),
169
+ }).pipe(
170
Effect.provideService(ThreadManagementService.ThreadManagementService, threads),
171
Effect.provideService(ServerSettings.ServerSettingsService, settings),
172
Effect.tapError(() =>
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+29
@@ -2752,6 +2752,34 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
2752
}).pipe(Effect.provide(layerIsolatedOutbox)),
2753
);
2754
2755
+ it.effect.each(["queued", "waiting"])(
2756
+ "keeps a %s continuation's crash budget past outbox retention",
2757
+ (status) =>
2758
+ Effect.gen(function* () {
2759
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
2760
+ const sql = yield* SqlClient.SqlClient;
2761
+ const now = yield* DateTime.now;
2762
+ const old = DateTime.formatIso(DateTime.subtractDuration(now, Duration.days(8)));
2763
+ const timestamp = DateTime.formatIso(now);
2764
+ yield* sql`
2765
+ INSERT INTO orchestration_v2_effect_outbox
2766
+ (effect_id, command_id, thread_id, effect_type, payload_json, status, available_at, created_at, updated_at, completed_at)
2767
+ VALUES ('effect:retained-recovery', 'command:retained-recovery', 'thread:retained-recovery',
2768
+ 'provider-runtime.continue', '{"type":"provider-runtime.continue","sourceRunId":"run:retained-source","processLossCount":3}',
2769
+ 'succeeded', ${old}, ${old}, ${old}, ${old})
2770
+ `;
2771
+ yield* sql`
2772
+ INSERT INTO orchestration_v2_projection_runs
2773
+ (run_id, thread_id, ordinal, provider, status, requested_at, payload_json)
2774
+ VALUES ('run:retained-recovery', 'thread:retained-recovery', 1, 'codex', ${status}, ${timestamp},
2775
+ '{"restartContinuationOfRunId":"run:retained-source"}')
2776
+ `;
2777
+ assert.equal(yield* outbox.pruneSettled, 0);
2778
+ yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'completed' WHERE run_id = 'run:retained-recovery'`;
2779
+ assert.equal(yield* outbox.pruneSettled, 1);
2780
+ }).pipe(Effect.provide(layerIsolatedOutbox)),
2781
+ );
2782
+
2783
it.effect("wakes claimers when cancellation unblocks same-thread work", () =>
2784
Effect.gen(function* () {
2785
const outbox = yield* EffectOutbox.EffectOutboxV2;
@@ -3402,6 +3430,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
3430
assert.deepEqual(effects[0]?.request, {
3431
type: "provider-runtime.continue",
3432
sourceRunId: runId,
3433
+ processLossCount: 0,
3434
});
3435
yield* outbox.cancelUnsettled({
3436
threadId,
apps/server/src/orchestration-v2/Orchestrator.ts
+16
-1
@@ -3250,6 +3250,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3250
const delayMs = continuationStartRetryDelayMs(run, projection);
3251
if (delayMs === undefined) return;
3252
const commandId = CommandId.make(`command:restart-continuation-retry:${run.id}`);
3253
+ // A start retry is not a crash: carry the crash budget forward unchanged.
3254
+ const previous =
3255
+ run.restartContinuationOfRunId === undefined
3256
+ ? undefined
3257
+ : Option.getOrUndefined(
3258
+ yield* effectOutbox.get(
3259
+ `effect:restart-continuation:${run.restartContinuationOfRunId}`,
3260
+ ),
3261
+ )?.request;
3262
+ const processLossCount =
3263
+ previous?.type === "provider-runtime.continue" ? previous.processLossCount : undefined;
3264
yield* writeSystemEvents(
3265
[],
3266
[
@@ -3257,7 +3268,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3268
id: `effect:restart-continuation:${run.id}`,
3269
commandId,
3270
threadId,
3260
- request: { type: "provider-runtime.continue", sourceRunId: run.id },
3271
+ request: {
3272
+ type: "provider-runtime.continue",
3273
+ sourceRunId: run.id,
3274
+ ...(processLossCount === undefined ? {} : { processLossCount }),
3275
+ },
3276
availableAt: DateTime.add(yield* DateTime.now, { milliseconds: delayMs }),
3277
},
3278
],
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+33
-2
@@ -16,6 +16,7 @@ import * as Context from "effect/Context";
16
import * as DateTime from "effect/DateTime";
17
import * as Effect from "effect/Effect";
18
import * as Layer from "effect/Layer";
19
+import * as Option from "effect/Option";
20
import * as Schema from "effect/Schema";
21
22
import * as EffectOutbox from "./EffectOutbox.ts";
@@ -839,13 +840,39 @@ export const make = Effect.gen(function* () {
840
now,
841
)
842
: undefined;
843
+ let processLossCount = 1;
844
+ // Normal upgrades must not exhaust an unfinished turn's crash allowance.
845
+ if (
846
+ continuationRun?.restartContinuationOfRunId !== undefined &&
847
+ continuationRun.status !== "completed" &&
848
+ continuationRun.status !== "waiting"
849
+ ) {
850
+ const previous = yield* outbox
851
+ .get(`effect:restart-continuation:${continuationRun.restartContinuationOfRunId}`)
852
+ .pipe(
853
+ Effect.mapError(
854
+ (cause) =>
855
+ new ProviderRuntimeRecoveryError({
856
+ operation: "reconcile",
857
+ threadId: projection.thread.id,
858
+ cause,
859
+ }),
860
+ ),
861
+ );
862
+ if (Option.isSome(previous) && previous.value.request.type === "provider-runtime.continue")
863
+ processLossCount += previous.value.request.processLossCount ?? 0;
864
+ }
865
const effects: Array<EffectOutbox.PendingOrchestrationEffectV2> = continuationRun
866
? [
867
{
868
id: `effect:restart-continuation:${continuationRun.id}`,
869
commandId,
870
threadId: projection.thread.id,
848
- request: { type: "provider-runtime.continue", sourceRunId: continuationRun.id },
871
+ request: {
872
+ type: "provider-runtime.continue",
873
+ sourceRunId: continuationRun.id,
874
+ processLossCount,
875
+ },
876
},
877
]
878
: [];
@@ -1020,7 +1047,11 @@ export const make = Effect.gen(function* () {
1047
id: `effect:restart-continuation:${run.id}`,
1048
commandId,
1049
threadId,
1023
- request: { type: "provider-runtime.continue", sourceRunId: run.id },
1050
+ request: {
1051
+ type: "provider-runtime.continue",
1052
+ sourceRunId: run.id,
1053
+ processLossCount: 0,
1054
+ },
1055
},
1056
],
1057
});
apps/server/src/orchestration-v2/RestartContinuation.test.ts
+83
-3
@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
20
import * as Deferred from "effect/Deferred";
21
import * as Fiber from "effect/Fiber";
22
import * as Layer from "effect/Layer";
23
+import * as Option from "effect/Option";
24
import * as ServerSettings from "../serverSettings.ts";
25
import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
26
import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts";
@@ -293,6 +294,7 @@ it.effect.each([
294
assert.deepEqual(committed!.effects[0]?.request, {
295
type: "provider-runtime.continue",
296
sourceRunId: runId,
297
+ processLossCount: 1,
298
});
299
}),
300
);
@@ -425,7 +427,7 @@ it.effect.each(["running", "waiting"])(
427
]);
428
assert.deepEqual(
429
committed?.effects.map((effect) => effect.request),
428
- [{ type: "provider-runtime.continue", sourceRunId: runId }],
430
+ [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 1 }],
431
);
432
}),
433
);
@@ -563,6 +565,7 @@ it.effect("prepares no continuation for background work another provider thread
565
assert.deepEqual(writes[0]!.effects[0]?.request, {
566
type: "provider-runtime.continue",
567
sourceRunId: runId,
568
+ processLossCount: 0,
569
});
570
}),
571
);
@@ -668,7 +671,15 @@ it.effect.each([
671
const recorded = [...writes.flatMap((write) => write.effects), ...commits[0]!.effects];
672
assert.deepEqual(
673
recorded.map((effect) => effect.request),
671
- wakes ? [{ type: "provider-runtime.continue", sourceRunId: runId }] : [],
674
+ wakes
675
+ ? [
676
+ {
677
+ type: "provider-runtime.continue",
678
+ sourceRunId: runId,
679
+ processLossCount: trigger === "shutdown" ? 0 : 1,
680
+ },
681
+ ]
682
+ : [],
683
);
684
assert.lengthOf(commits, 1);
685
const events = commits[0]!.events;
@@ -868,6 +879,75 @@ const continuationDispatches = (
879
return dispatches;
880
});
881
882
+it.effect.each([
883
+ [3, "running"],
884
+ [3, "completed"],
885
+ [3, "waiting"],
886
+] as const)("persists recovery after count %s and %s work", ([previousCount, status]) =>
887
+ Effect.gen(function* () {
888
+ const base = makeProjection();
889
+ const source = RunId.make("run:previous-loss");
890
+ const projection = {
891
+ ...base,
892
+ runs: [
893
+ {
894
+ ...base.runs[0]!,
895
+ status,
896
+ restartContinuationOfRunId: source,
897
+ requestedAt: DateTime.subtract(now, { minutes: 22 }),
898
+ },
899
+ ],
900
+ providerTurns: [
901
+ { ...base.providerTurns[0]!, status: status === "running" ? "running" : "completed" },
902
+ ],
903
+ turnItems: status === "running" ? [] : [backgroundSubagent.turnItem],
904
+ } as unknown as OrchestrationV2ThreadProjection;
905
+ const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
906
+ const recovery = yield* ProviderRuntimeRecovery.make.pipe(
907
+ Effect.provide(
908
+ Layer.mergeAll(
909
+ ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
910
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
911
+ getRecoveryThreadIds: () => Effect.succeed([threadId]),
912
+ getRuntimeRecoveryProjection: () => Effect.succeed(projection),
913
+ }),
914
+ Layer.mock(EventSink.EventSinkV2)({
915
+ commitCommand: (input) => {
916
+ commits.push(input);
917
+ return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
918
+ },
919
+ }),
920
+ IdAllocator.layer,
921
+ Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
922
+ Layer.mock(EffectOutbox.EffectOutboxV2)({
923
+ get: (id) => {
924
+ return Effect.succeed(
925
+ id === `effect:restart-continuation:${source}`
926
+ ? Option.some({
927
+ request: {
928
+ type: "provider-runtime.continue",
929
+ sourceRunId: source,
930
+ processLossCount: previousCount,
931
+ },
932
+ } as EffectOutbox.OrchestrationEffectV2)
933
+ : Option.none(),
934
+ );
935
+ },
936
+ listByCommandId: () => Effect.succeed([]),
937
+ reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
938
+ }),
939
+ ),
940
+ ),
941
+ );
942
+ yield* recovery.reconcile("startup");
943
+ assert.deepEqual(commits[0]?.effects[0]?.request, {
944
+ type: "provider-runtime.continue",
945
+ sourceRunId: runId,
946
+ processLossCount: status === "running" ? previousCount + 1 : 1,
947
+ });
948
+ }),
949
+);
950
+
951
const continuationTexts = (projection: OrchestrationV2ThreadProjection) =>
952
continuationDispatches(projection).pipe(
953
Effect.map((dispatches) => dispatches.map((command) => command.text)),
@@ -1165,7 +1245,7 @@ it.effect("prepares later threads' continuations when one thread fails", () =>
1245
yield* recovery.prepareForShutdown;
1246
assert.deepEqual(
1247
writes.map((write) => write.effects[0]?.request),
1168
- [{ type: "provider-runtime.continue", sourceRunId: runId }],
1248
+ [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 0 }],
1249
);
1250
}),
1251
);
apps/server/src/orchestration-v2/RestartContinuation.ts
+10
-2
@@ -183,7 +183,11 @@ export function restartContinuationRun(
183
}
184
185
export const continueRestartedRun = Effect.fn("RestartContinuation.continueRestartedRun")(
186
- function* (input: { readonly threadId: ThreadId; readonly sourceRunId: RunId }) {
186
+ function* (input: {
187
+ readonly threadId: ThreadId;
188
+ readonly sourceRunId: RunId;
189
+ readonly processLossCount?: number;
190
+ }) {
191
const settings = yield* ServerSettings.ServerSettingsService;
192
const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null));
193
if (!enabled) return;
@@ -239,7 +243,11 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
243
if (DateTime.toEpochMillis(run.requestedAt) < loopWindowStart) break;
244
chainLength += 1;
245
}
242
- if (chainLength >= MAX_CONTINUATION_CHAIN) {
246
+ if (
247
+ input.processLossCount === undefined
248
+ ? chainLength >= MAX_CONTINUATION_CHAIN
249
+ : input.processLossCount > MAX_CONTINUATION_CHAIN
250
+ ) {
251
yield* Effect.logWarning("Not continuing a run restarted too many times in a row", {
252
threadId: input.threadId,
253
sourceRunId: input.sourceRunId,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+23
@@ -9254,6 +9254,21 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
9254
});
9255
const cut = (yield* runFor("cut"))!;
9256
yield* write("cut:cancelled", { ...cut, status: "cancelled" });
9257
+ yield* eventSink.writeWithEffects({
9258
+ events: [],
9259
+ effects: [
9260
+ {
9261
+ id: `effect:restart-continuation:${cut.id}`,
9262
+ commandId: CommandId.make(`${threadId}:crash-recovery`),
9263
+ threadId,
9264
+ request: {
9265
+ type: "provider-runtime.continue",
9266
+ sourceRunId: cut.id,
9267
+ processLossCount: 2,
9268
+ },
9269
+ },
9270
+ ],
9271
+ });
9272
yield* orchestrator.dispatch({
9273
type: "message.dispatch",
9274
createdBy: "agent",
@@ -9337,6 +9352,14 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
9352
);
9353
yield* worker.drain();
9354
assert.deepEqual(yield* retriesOf, [], "the retry waits out its backoff");
9355
+ const retryIntent = yield* (yield* EffectOutbox.EffectOutboxV2).get(
9356
+ `effect:restart-continuation:${continuation.id}`,
9357
+ );
9358
+ assert.deepEqual(Option.getOrUndefined(retryIntent)?.request, {
9359
+ type: "provider-runtime.continue",
9360
+ sourceRunId: continuation.id,
9361
+ processLossCount: 2,
9362
+ });
9363
if (stopped) {
9364
yield* orchestrator.dispatch({
9365
type: "thread.stop",
docs/internals/server-updates.md
+5
-2
@@ -130,8 +130,11 @@ and maintenance turns such as `/compact` before dispatching. Intent and delivery
130
refuse a thread that is archived, settled, or still snoozed, a cut turn whose provider
131
turn already ended, and a run waiting on the user: an approval, question, or secret that was pending,
132
or that recovery closed on the cut turn. Message-capable questions stay answerable, so
133
-they do not block. Delivery also stops after three consecutive continuations in one
134
-`restartContinuationOfRunId` chain, so a turn that crashes the server cannot loop. Stable
133
+they do not block. New continuation intents persist a cumulative process-loss count;
134
+delivery stops before the fourth crash recovery of an unfinished turn, regardless
135
+of elapsed time. Planned shutdowns and completed turns reset it. A live continuation
136
+retains its preceding intent past outbox retention so cleanup cannot erase the count.
137
+Legacy intents without a count keep the five-minute chain guard. Stable
138
command and message IDs prevent duplicate submissions after an outbox retry. Codex
139
resumes without adding provider prompt text unless the turn lost background work;
140
other adapters receive the continuation message through their normal turn path.
docs/user/updating.md
+4
-2
@@ -77,8 +77,10 @@ command or monitor the agent said it was waiting on is woken and told that this
77
stopped. Other background shells and monitors, such as a dev server and its
78
watchdog, do not wake it; the thread learns about them with your next message.
79
A thread is not resumed while it waits for your approval, answer, or secret, or
80
-while it is archived, snoozed, or settled. After three automatic resumes in a row,
81
-the thread waits for your next message.
80
+while it is archived, snoozed, or settled. After three consecutive crash recoveries
81
+of an unfinished turn, the thread waits for your next message, even when crashes
82
+are more than five minutes apart. A completed turn or a planned shutdown resets
83
+that limit.
84
85
Updates from the previous orchestration system preserve conversation transcripts but cannot carry
86
every kind of runtime history forward. Read [Threads from older T3 Code versions](./thread-migration.md)