@devries-cameron20 / t3code-1 / commits / a68c367e4f
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): preserve the crash recovery budget across restarts (#25603)

Co-authored-by: Tin Zeljar <35663757+Zheruel@users.noreply.github.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

maria committed Nov 14, 2026 at 23:47 UTC a68c367e4ff3d31a4a43493f6b22f83de1c6eb35
11 files changed +282 -13
apps/server/src/orchestration-v2/EffectOutbox.ts
+12
@@ -35,6 +35,7 @@ export const OrchestrationEffectRequestV2 = Schema.Union([
35 Schema.Struct({
36 type: Schema.Literal("provider-runtime.continue"),
37 sourceRunId: RunId,
38 + processLossCount: Schema.optional(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))),
39 }),
40 Schema.Struct({
41 type: Schema.Literal("provider-session.detach"),
@@ -705,6 +706,7 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
706 const cutoff = DateTime.formatIso(
707 DateTime.subtractDuration(yield* DateTime.now, SETTLED_EFFECT_RETENTION),
708 );
709 + // Live continuations retain their preceding intent's cumulative crash budget.
710 // Separate batch statements release the write lock between batches.
711 const deleteBatch = sql<{ readonly effect_id: string }>`
712 DELETE FROM orchestration_v2_effect_outbox
@@ -713,6 +715,16 @@ export const layer: Layer.Layer<EffectOutboxV2, never, SqlClient.SqlClient> = La
715 FROM orchestration_v2_effect_outbox
716 WHERE status IN ('succeeded', 'cancelled')
717 AND completed_at < ${cutoff}
718 + AND NOT (
719 + effect_type = 'provider-runtime.continue'
720 + AND EXISTS (
721 + SELECT 1 FROM orchestration_v2_projection_runs AS recovery_run
722 + WHERE recovery_run.thread_id = orchestration_v2_effect_outbox.thread_id
723 + AND recovery_run.status IN ('queued', 'preparing', 'starting', 'running', 'waiting')
724 + AND json_extract(recovery_run.payload_json, '$.restartContinuationOfRunId') =
725 + json_extract(orchestration_v2_effect_outbox.payload_json, '$.sourceRunId')
726 + )
727 + )
728 LIMIT ${PRUNE_BATCH_SIZE}
729 )
730 RETURNING effect_id
apps/server/src/orchestration-v2/EffectWorker.test.ts
+60
@@ -4,6 +4,8 @@ import {
4 CheckpointScopeId,
5 CommandId,
6 NodeId,
7 + ProjectId,
8 + ProviderInstanceId,
9 ProviderSessionId,
10 ProviderThreadId,
11 ProviderTurnId,
@@ -12,6 +14,7 @@ import {
14 RuntimeRequestId,
15 ThreadId,
16 VcsProcessExitError,
17 + type OrchestrationV2ThreadProjection,
18 } from "@t3tools/contracts";
19 import * as Cause from "effect/Cause";
20 import * as DateTime from "effect/DateTime";
@@ -777,3 +780,60 @@ it.effect.each(["confirmed", "unconfirmed", "failed"] as const)(
780 );
781 }),
782 );
783 +
784 +it.effect.each([3, 4])(
785 + "enforces process-loss count %s after 22 minutes through the executor",
786 + (processLossCount) =>
787 + Effect.gen(function* () {
788 + const now = yield* DateTime.now;
789 + const instanceId = ProviderInstanceId.make("codex");
790 + const events = yield* Ref.make<ReadonlyArray<string>>([]);
791 + const layer = layerExecutorFor({
792 + events,
793 + continueAfterRestart: true,
794 + threads: {
795 + getThreadRecords: () =>
796 + Effect.succeed({
797 + thread: {
798 + id: threadId,
799 + projectId: ProjectId.make("project:executor-recovery"),
800 + providerInstanceId: instanceId,
801 + archivedAt: null,
802 + deletedAt: null,
803 + lineage: { parentThreadId: null, relationshipToParent: null },
804 + },
805 + runs: [
806 + {
807 + id: runId,
808 + providerInstanceId: instanceId,
809 + status: "cancelled",
810 + requestedAt: DateTime.subtract(now, { minutes: 22 }),
811 + completedAt: now,
812 + ordinal: 1,
813 + },
814 + ],
815 + messages: [],
816 + providerTurns: [],
817 + runtimeRequests: [],
818 + turnItems: [],
819 + attempts: [],
820 + } as unknown as OrchestrationV2ThreadProjection),
821 + dispatch: () =>
822 + Ref.update(events, (entries) => [...entries, "dispatch"]).pipe(Effect.as({} as never)),
823 + recoverDelegatedTask: () =>
824 + Ref.update(events, (entries) => [...entries, "recover-child"]),
825 + },
826 + });
827 + yield* Effect.gen(function* () {
828 + const executor = yield* EffectWorker.OrchestrationEffectExecutorV2;
829 + yield* executor.execute({
830 + ...restartEffect(now, { type: "detach" }),
831 + request: { type: "provider-runtime.continue", sourceRunId: runId, processLossCount },
832 + });
833 + }).pipe(Effect.provide(layer));
834 + assert.deepEqual(
835 + yield* Ref.get(events),
836 + processLossCount > 3 ? ["recover-child"] : ["dispatch", "recover-child"],
837 + );
838 + }),
839 +);
apps/server/src/orchestration-v2/EffectWorker.ts
+7 -1
@@ -160,7 +160,13 @@ export const layerExecutor: Layer.Layer<
160 switch (effect.request.type) {
161 case "provider-runtime.continue": {
162 const sourceRunId = effect.request.sourceRunId;
163 - return continueRestartedRun({ threadId: effect.threadId, sourceRunId }).pipe(
163 + return continueRestartedRun({
164 + threadId: effect.threadId,
165 + sourceRunId,
166 + ...(effect.request.processLossCount === undefined
167 + ? {}
168 + : { processLossCount: effect.request.processLossCount }),
169 + }).pipe(
170 Effect.provideService(ThreadManagementService.ThreadManagementService, threads),
171 Effect.provideService(ServerSettings.ServerSettingsService, settings),
172 Effect.tapError(() =>
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+29
@@ -2752,6 +2752,34 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
2752 }).pipe(Effect.provide(layerIsolatedOutbox)),
2753 );
2754
2755 + it.effect.each(["queued", "waiting"])(
2756 + "keeps a %s continuation's crash budget past outbox retention",
2757 + (status) =>
2758 + Effect.gen(function* () {
2759 + const outbox = yield* EffectOutbox.EffectOutboxV2;
2760 + const sql = yield* SqlClient.SqlClient;
2761 + const now = yield* DateTime.now;
2762 + const old = DateTime.formatIso(DateTime.subtractDuration(now, Duration.days(8)));
2763 + const timestamp = DateTime.formatIso(now);
2764 + yield* sql`
2765 + INSERT INTO orchestration_v2_effect_outbox
2766 + (effect_id, command_id, thread_id, effect_type, payload_json, status, available_at, created_at, updated_at, completed_at)
2767 + VALUES ('effect:retained-recovery', 'command:retained-recovery', 'thread:retained-recovery',
2768 + 'provider-runtime.continue', '{"type":"provider-runtime.continue","sourceRunId":"run:retained-source","processLossCount":3}',
2769 + 'succeeded', ${old}, ${old}, ${old}, ${old})
2770 + `;
2771 + yield* sql`
2772 + INSERT INTO orchestration_v2_projection_runs
2773 + (run_id, thread_id, ordinal, provider, status, requested_at, payload_json)
2774 + VALUES ('run:retained-recovery', 'thread:retained-recovery', 1, 'codex', ${status}, ${timestamp},
2775 + '{"restartContinuationOfRunId":"run:retained-source"}')
2776 + `;
2777 + assert.equal(yield* outbox.pruneSettled, 0);
2778 + yield* sql`UPDATE orchestration_v2_projection_runs SET status = 'completed' WHERE run_id = 'run:retained-recovery'`;
2779 + assert.equal(yield* outbox.pruneSettled, 1);
2780 + }).pipe(Effect.provide(layerIsolatedOutbox)),
2781 + );
2782 +
2783 it.effect("wakes claimers when cancellation unblocks same-thread work", () =>
2784 Effect.gen(function* () {
2785 const outbox = yield* EffectOutbox.EffectOutboxV2;
@@ -3402,6 +3430,7 @@ it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
3430 assert.deepEqual(effects[0]?.request, {
3431 type: "provider-runtime.continue",
3432 sourceRunId: runId,
3433 + processLossCount: 0,
3434 });
3435 yield* outbox.cancelUnsettled({
3436 threadId,
apps/server/src/orchestration-v2/Orchestrator.ts
+16 -1
@@ -3250,6 +3250,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3250 const delayMs = continuationStartRetryDelayMs(run, projection);
3251 if (delayMs === undefined) return;
3252 const commandId = CommandId.make(`command:restart-continuation-retry:${run.id}`);
3253 + // A start retry is not a crash: carry the crash budget forward unchanged.
3254 + const previous =
3255 + run.restartContinuationOfRunId === undefined
3256 + ? undefined
3257 + : Option.getOrUndefined(
3258 + yield* effectOutbox.get(
3259 + `effect:restart-continuation:${run.restartContinuationOfRunId}`,
3260 + ),
3261 + )?.request;
3262 + const processLossCount =
3263 + previous?.type === "provider-runtime.continue" ? previous.processLossCount : undefined;
3264 yield* writeSystemEvents(
3265 [],
3266 [
@@ -3257,7 +3268,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3268 id: `effect:restart-continuation:${run.id}`,
3269 commandId,
3270 threadId,
3260 - request: { type: "provider-runtime.continue", sourceRunId: run.id },
3271 + request: {
3272 + type: "provider-runtime.continue",
3273 + sourceRunId: run.id,
3274 + ...(processLossCount === undefined ? {} : { processLossCount }),
3275 + },
3276 availableAt: DateTime.add(yield* DateTime.now, { milliseconds: delayMs }),
3277 },
3278 ],
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+33 -2
@@ -16,6 +16,7 @@ import * as Context from "effect/Context";
16 import * as DateTime from "effect/DateTime";
17 import * as Effect from "effect/Effect";
18 import * as Layer from "effect/Layer";
19 +import * as Option from "effect/Option";
20 import * as Schema from "effect/Schema";
21
22 import * as EffectOutbox from "./EffectOutbox.ts";
@@ -839,13 +840,39 @@ export const make = Effect.gen(function* () {
840 now,
841 )
842 : undefined;
843 + let processLossCount = 1;
844 + // Normal upgrades must not exhaust an unfinished turn's crash allowance.
845 + if (
846 + continuationRun?.restartContinuationOfRunId !== undefined &&
847 + continuationRun.status !== "completed" &&
848 + continuationRun.status !== "waiting"
849 + ) {
850 + const previous = yield* outbox
851 + .get(`effect:restart-continuation:${continuationRun.restartContinuationOfRunId}`)
852 + .pipe(
853 + Effect.mapError(
854 + (cause) =>
855 + new ProviderRuntimeRecoveryError({
856 + operation: "reconcile",
857 + threadId: projection.thread.id,
858 + cause,
859 + }),
860 + ),
861 + );
862 + if (Option.isSome(previous) && previous.value.request.type === "provider-runtime.continue")
863 + processLossCount += previous.value.request.processLossCount ?? 0;
864 + }
865 const effects: Array<EffectOutbox.PendingOrchestrationEffectV2> = continuationRun
866 ? [
867 {
868 id: `effect:restart-continuation:${continuationRun.id}`,
869 commandId,
870 threadId: projection.thread.id,
848 - request: { type: "provider-runtime.continue", sourceRunId: continuationRun.id },
871 + request: {
872 + type: "provider-runtime.continue",
873 + sourceRunId: continuationRun.id,
874 + processLossCount,
875 + },
876 },
877 ]
878 : [];
@@ -1020,7 +1047,11 @@ export const make = Effect.gen(function* () {
1047 id: `effect:restart-continuation:${run.id}`,
1048 commandId,
1049 threadId,
1023 - request: { type: "provider-runtime.continue", sourceRunId: run.id },
1050 + request: {
1051 + type: "provider-runtime.continue",
1052 + sourceRunId: run.id,
1053 + processLossCount: 0,
1054 + },
1055 },
1056 ],
1057 });
apps/server/src/orchestration-v2/RestartContinuation.test.ts
+83 -3
@@ -20,6 +20,7 @@ import * as Effect from "effect/Effect";
20 import * as Deferred from "effect/Deferred";
21 import * as Fiber from "effect/Fiber";
22 import * as Layer from "effect/Layer";
23 +import * as Option from "effect/Option";
24 import * as ServerSettings from "../serverSettings.ts";
25 import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
26 import { continuationStartRetryDelayMs } from "./RestartBackgroundNote.ts";
@@ -293,6 +294,7 @@ it.effect.each([
294 assert.deepEqual(committed!.effects[0]?.request, {
295 type: "provider-runtime.continue",
296 sourceRunId: runId,
297 + processLossCount: 1,
298 });
299 }),
300 );
@@ -425,7 +427,7 @@ it.effect.each(["running", "waiting"])(
427 ]);
428 assert.deepEqual(
429 committed?.effects.map((effect) => effect.request),
428 - [{ type: "provider-runtime.continue", sourceRunId: runId }],
430 + [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 1 }],
431 );
432 }),
433 );
@@ -563,6 +565,7 @@ it.effect("prepares no continuation for background work another provider thread
565 assert.deepEqual(writes[0]!.effects[0]?.request, {
566 type: "provider-runtime.continue",
567 sourceRunId: runId,
568 + processLossCount: 0,
569 });
570 }),
571 );
@@ -668,7 +671,15 @@ it.effect.each([
671 const recorded = [...writes.flatMap((write) => write.effects), ...commits[0]!.effects];
672 assert.deepEqual(
673 recorded.map((effect) => effect.request),
671 - wakes ? [{ type: "provider-runtime.continue", sourceRunId: runId }] : [],
674 + wakes
675 + ? [
676 + {
677 + type: "provider-runtime.continue",
678 + sourceRunId: runId,
679 + processLossCount: trigger === "shutdown" ? 0 : 1,
680 + },
681 + ]
682 + : [],
683 );
684 assert.lengthOf(commits, 1);
685 const events = commits[0]!.events;
@@ -868,6 +879,75 @@ const continuationDispatches = (
879 return dispatches;
880 });
881
882 +it.effect.each([
883 + [3, "running"],
884 + [3, "completed"],
885 + [3, "waiting"],
886 +] as const)("persists recovery after count %s and %s work", ([previousCount, status]) =>
887 + Effect.gen(function* () {
888 + const base = makeProjection();
889 + const source = RunId.make("run:previous-loss");
890 + const projection = {
891 + ...base,
892 + runs: [
893 + {
894 + ...base.runs[0]!,
895 + status,
896 + restartContinuationOfRunId: source,
897 + requestedAt: DateTime.subtract(now, { minutes: 22 }),
898 + },
899 + ],
900 + providerTurns: [
901 + { ...base.providerTurns[0]!, status: status === "running" ? "running" : "completed" },
902 + ],
903 + turnItems: status === "running" ? [] : [backgroundSubagent.turnItem],
904 + } as unknown as OrchestrationV2ThreadProjection;
905 + const commits: Parameters<EventSink.EventSinkV2["Service"]["commitCommand"]>[0][] = [];
906 + const recovery = yield* ProviderRuntimeRecovery.make.pipe(
907 + Effect.provide(
908 + Layer.mergeAll(
909 + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
910 + Layer.mock(ProjectionStore.ProjectionStoreV2)({
911 + getRecoveryThreadIds: () => Effect.succeed([threadId]),
912 + getRuntimeRecoveryProjection: () => Effect.succeed(projection),
913 + }),
914 + Layer.mock(EventSink.EventSinkV2)({
915 + commitCommand: (input) => {
916 + commits.push(input);
917 + return Effect.succeed({ committed: true, cancelledEffectCount: 0 } as never);
918 + },
919 + }),
920 + IdAllocator.layer,
921 + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
922 + Layer.mock(EffectOutbox.EffectOutboxV2)({
923 + get: (id) => {
924 + return Effect.succeed(
925 + id === `effect:restart-continuation:${source}`
926 + ? Option.some({
927 + request: {
928 + type: "provider-runtime.continue",
929 + sourceRunId: source,
930 + processLossCount: previousCount,
931 + },
932 + } as EffectOutbox.OrchestrationEffectV2)
933 + : Option.none(),
934 + );
935 + },
936 + listByCommandId: () => Effect.succeed([]),
937 + reconcileAfterProcessLoss: Effect.succeed({ requeued: 0, cancelled: 0 }),
938 + }),
939 + ),
940 + ),
941 + );
942 + yield* recovery.reconcile("startup");
943 + assert.deepEqual(commits[0]?.effects[0]?.request, {
944 + type: "provider-runtime.continue",
945 + sourceRunId: runId,
946 + processLossCount: status === "running" ? previousCount + 1 : 1,
947 + });
948 + }),
949 +);
950 +
951 const continuationTexts = (projection: OrchestrationV2ThreadProjection) =>
952 continuationDispatches(projection).pipe(
953 Effect.map((dispatches) => dispatches.map((command) => command.text)),
@@ -1165,7 +1245,7 @@ it.effect("prepares later threads' continuations when one thread fails", () =>
1245 yield* recovery.prepareForShutdown;
1246 assert.deepEqual(
1247 writes.map((write) => write.effects[0]?.request),
1168 - [{ type: "provider-runtime.continue", sourceRunId: runId }],
1248 + [{ type: "provider-runtime.continue", sourceRunId: runId, processLossCount: 0 }],
1249 );
1250 }),
1251 );
apps/server/src/orchestration-v2/RestartContinuation.ts
+10 -2
@@ -183,7 +183,11 @@ export function restartContinuationRun(
183 }
184
185 export const continueRestartedRun = Effect.fn("RestartContinuation.continueRestartedRun")(
186 - function* (input: { readonly threadId: ThreadId; readonly sourceRunId: RunId }) {
186 + function* (input: {
187 + readonly threadId: ThreadId;
188 + readonly sourceRunId: RunId;
189 + readonly processLossCount?: number;
190 + }) {
191 const settings = yield* ServerSettings.ServerSettingsService;
192 const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null));
193 if (!enabled) return;
@@ -239,7 +243,11 @@ export const continueRestartedRun = Effect.fn("RestartContinuation.continueResta
243 if (DateTime.toEpochMillis(run.requestedAt) < loopWindowStart) break;
244 chainLength += 1;
245 }
242 - if (chainLength >= MAX_CONTINUATION_CHAIN) {
246 + if (
247 + input.processLossCount === undefined
248 + ? chainLength >= MAX_CONTINUATION_CHAIN
249 + : input.processLossCount > MAX_CONTINUATION_CHAIN
250 + ) {
251 yield* Effect.logWarning("Not continuing a run restarted too many times in a row", {
252 threadId: input.threadId,
253 sourceRunId: input.sourceRunId,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+23
@@ -9254,6 +9254,21 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
9254 });
9255 const cut = (yield* runFor("cut"))!;
9256 yield* write("cut:cancelled", { ...cut, status: "cancelled" });
9257 + yield* eventSink.writeWithEffects({
9258 + events: [],
9259 + effects: [
9260 + {
9261 + id: `effect:restart-continuation:${cut.id}`,
9262 + commandId: CommandId.make(`${threadId}:crash-recovery`),
9263 + threadId,
9264 + request: {
9265 + type: "provider-runtime.continue",
9266 + sourceRunId: cut.id,
9267 + processLossCount: 2,
9268 + },
9269 + },
9270 + ],
9271 + });
9272 yield* orchestrator.dispatch({
9273 type: "message.dispatch",
9274 createdBy: "agent",
@@ -9337,6 +9352,14 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
9352 );
9353 yield* worker.drain();
9354 assert.deepEqual(yield* retriesOf, [], "the retry waits out its backoff");
9355 + const retryIntent = yield* (yield* EffectOutbox.EffectOutboxV2).get(
9356 + `effect:restart-continuation:${continuation.id}`,
9357 + );
9358 + assert.deepEqual(Option.getOrUndefined(retryIntent)?.request, {
9359 + type: "provider-runtime.continue",
9360 + sourceRunId: continuation.id,
9361 + processLossCount: 2,
9362 + });
9363 if (stopped) {
9364 yield* orchestrator.dispatch({
9365 type: "thread.stop",
docs/internals/server-updates.md
+5 -2
@@ -130,8 +130,11 @@ and maintenance turns such as `/compact` before dispatching. Intent and delivery
130 refuse a thread that is archived, settled, or still snoozed, a cut turn whose provider
131 turn already ended, and a run waiting on the user: an approval, question, or secret that was pending,
132 or that recovery closed on the cut turn. Message-capable questions stay answerable, so
133 -they do not block. Delivery also stops after three consecutive continuations in one
134 -`restartContinuationOfRunId` chain, so a turn that crashes the server cannot loop. Stable
133 +they do not block. New continuation intents persist a cumulative process-loss count;
134 +delivery stops before the fourth crash recovery of an unfinished turn, regardless
135 +of elapsed time. Planned shutdowns and completed turns reset it. A live continuation
136 +retains its preceding intent past outbox retention so cleanup cannot erase the count.
137 +Legacy intents without a count keep the five-minute chain guard. Stable
138 command and message IDs prevent duplicate submissions after an outbox retry. Codex
139 resumes without adding provider prompt text unless the turn lost background work;
140 other adapters receive the continuation message through their normal turn path.
docs/user/updating.md
+4 -2
@@ -77,8 +77,10 @@ command or monitor the agent said it was waiting on is woken and told that this
77 stopped. Other background shells and monitors, such as a dev server and its
78 watchdog, do not wake it; the thread learns about them with your next message.
79 A thread is not resumed while it waits for your approval, answer, or secret, or
80 -while it is archived, snoozed, or settled. After three automatic resumes in a row,
81 -the thread waits for your next message.
80 +while it is archived, snoozed, or settled. After three consecutive crash recoveries
81 +of an unfinished turn, the thread waits for your next message, even when crashes
82 +are more than five minutes apart. A completed turn or a planned shutdown resets
83 +that limit.
84
85 Updates from the previous orchestration system preserve conversation transcripts but cannot carry
86 every kind of runtime history forward. Read [Threads from older T3 Code versions](./thread-migration.md)