The best way to control Claude Code, Codex and more from any computer or phone
fix(server): prepare restart continuations faster on shutdown (#31807)
Yash Singh committed
Nov 5, 2026 at 22:57 UTC
620a37b988a5a0a270ea7f70c0458ba658e9d072
7 files changed
+216
-38
apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
+22
@@ -836,6 +836,28 @@ it.effect("includes shared sessions and provider-owned background rosters in rec
836
new Set(yield* projections.getRecoveryThreadIds("runtime")),
837
new Set([first, second, roster, prepared]),
838
);
839
+ assert.deepEqual(
840
+ new Set(yield* projections.getRecoveryThreadIds("restart-continuations")),
841
+ new Set([first, second, prepared]),
842
+ );
843
+ assert.deepEqual(
844
+ yield* projections.getRecoveryThreadIds("restart-continuations", { include: [] }),
845
+ [],
846
+ );
847
+ assert.deepEqual(
848
+ yield* projections.getRecoveryThreadIds("restart-continuations", {
849
+ exclude: ["project:recovery"],
850
+ }),
851
+ [],
852
+ );
853
+ assert.deepEqual(
854
+ new Set(
855
+ yield* projections.getRecoveryThreadIds("restart-continuations", {
856
+ include: ["project:recovery"],
857
+ }),
858
+ ),
859
+ new Set([first, second, prepared]),
860
+ );
861
const preparedState = yield* projections.getRuntimeRecoveryProjection(prepared);
862
assert.deepEqual(
863
preparedState.providerSessions.map((session) => session.id),
apps/server/src/orchestration-v2/ProjectionStore.ts
+38
-1
@@ -168,6 +168,7 @@ export type ProjectionStoreV2Error = typeof ProjectionStoreV2Error.Type;
168
export type ProjectionRecoveryKind =
169
| "queued-runs"
170
| "runtime"
171
+ | "restart-continuations"
172
| "subagent-results"
173
| "delegated-completions";
174
@@ -353,6 +354,7 @@ export interface ProjectionRecordFilter {
354
readonly subagentIds?: ReadonlyArray<NodeId>;
355
readonly subagentRunIds?: ReadonlyArray<RunId>;
356
readonly nodeIds?: ReadonlyArray<NodeId>;
357
+ readonly turnItemRequestIds?: ReadonlyArray<RuntimeRequestId>;
358
readonly turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>;
359
readonly turnItemStatuses?: ReadonlyArray<OrchestrationV2TurnItem["status"]>;
360
readonly omitSettledLeafNodes?: boolean;
@@ -678,6 +680,10 @@ export interface ProjectionStoreV2Shape {
680
) => Effect.Effect<ReadonlyArray<OrchestrationV2Run>, ProjectionStoreV2Error>;
681
readonly getRecoveryThreadIds: (
682
kind: ProjectionRecoveryKind,
683
+ projects?: {
684
+ readonly include?: ReadonlyArray<string>;
685
+ readonly exclude?: ReadonlyArray<string>;
686
+ },
687
) => Effect.Effect<ReadonlyArray<ThreadId>, ProjectionStoreV2Error>;
688
readonly getUnreadableThreadIds: () => Effect.Effect<
689
ReadonlyArray<ThreadId>,
@@ -3129,6 +3135,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3135
(fields !== undefined && !fields.includes("turnItems")) ||
3136
(window === undefined &&
3137
matchesNothing(
3138
+ filter?.turnItemRequestIds,
3139
filter?.turnItemTypes,
3140
filter?.turnItemStatuses,
3141
filter?.turnItemRunIds,
@@ -3140,6 +3147,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3147
SELECT payload_json
3148
FROM orchestration_v2_projection_turn_items
3149
WHERE thread_id = ${threadId}
3150
+ ${filter?.turnItemRequestIds === undefined ? sql`` : sql`AND json_extract(payload_json, '$.requestId') IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemRequestIds)}))`}
3151
${filter?.turnItemTypes === undefined ? sql`` : sql`AND type IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemTypes)}))`}
3152
${filter?.turnItemStatuses === undefined ? sql`` : sql`AND status IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemStatuses)}))`}
3153
${filter?.turnItemRunId === undefined ? sql`` : sql`AND run_id = ${filter.turnItemRunId}`}
@@ -4120,7 +4128,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4128
);
4129
4130
const getRecoveryThreadIds = Effect.fn("ProjectionStore.getRecoveryThreadIds")(
4123
- function* (kind: ProjectionRecoveryKind) {
4131
+ function* (
4132
+ kind: ProjectionRecoveryKind,
4133
+ projects?: {
4134
+ readonly include?: ReadonlyArray<string>;
4135
+ readonly exclude?: ReadonlyArray<string>;
4136
+ },
4137
+ ) {
4138
const candidates = (() => {
4139
switch (kind) {
4140
case "queued-runs":
@@ -4171,6 +4185,27 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4185
AND type = 'subagent_result'
4186
)))
4187
`;
4188
+ case "restart-continuations":
4189
+ return sql`
4190
+ SELECT bindings.thread_id
4191
+ FROM orchestration_v2_projection_provider_sessions AS sessions
4192
+ JOIN orchestration_v2_projection_provider_session_bindings AS bindings
4193
+ ON bindings.provider_session_id = sessions.provider_session_id
4194
+ WHERE sessions.status NOT IN ('stopped', 'error')
4195
+ UNION
4196
+ SELECT thread_id FROM orchestration_v2_projection_provider_sessions AS sessions
4197
+ WHERE status NOT IN ('stopped', 'error') AND thread_id IS NOT NULL
4198
+ AND NOT EXISTS (
4199
+ SELECT 1 FROM orchestration_v2_projection_provider_session_bindings AS bindings
4200
+ WHERE bindings.provider_session_id = sessions.provider_session_id
4201
+ )
4202
+ UNION
4203
+ SELECT thread_id FROM orchestration_v2_projection_runs
4204
+ WHERE status = 'starting'
4205
+ AND CASE WHEN json_valid(payload_json)
4206
+ THEN json_extract(payload_json, '$.restartContinuationOfRunId') IS NOT NULL
4207
+ ELSE 0 END
4208
+ `;
4209
case "runtime":
4210
return sql`
4211
WITH pending_provider_threads AS MATERIALIZED (
@@ -4257,6 +4292,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4292
SELECT thread_id FROM orchestration_v2_projection_threads
4293
WHERE deleted_at IS NULL
4294
AND thread_id IN (${candidates})
4295
+ ${projects?.include === undefined ? sql`` : sql`AND project_id IN (SELECT value FROM json_each(${encodeIdList(projects.include)}))`}
4296
+ ${projects?.exclude === undefined ? sql`` : sql`AND project_id NOT IN (SELECT value FROM json_each(${encodeIdList(projects.exclude)}))`}
4297
${kind === "queued-runs" ? sql`AND archived_at IS NULL` : sql``}
4298
ORDER BY updated_at ASC, thread_id ASC
4299
`;
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+48
-31
@@ -875,39 +875,56 @@ export const make = Effect.gen(function* () {
875
const prepareForShutdown = Effect.gen(function* () {
876
const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null));
877
if (!enabled) return;
878
- const threadIds = yield* projections.getRecoveryThreadIds("runtime");
879
- for (const threadId of threadIds) {
880
- yield* Effect.gen(function* () {
881
- const projection = yield* projections.getRuntimeRecoveryProjection(threadId);
882
- if (
883
- !resolveProjectSettings(enabled, projection.thread.projectId).settings
884
- .continueThreadsAfterServerUpdate
885
- )
886
- return;
887
- const run = restartContinuationRun(projection, yield* DateTime.now);
888
- if (!run) return;
889
- const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
890
- yield* eventSink.writeWithEffects({
891
- commandId,
892
- events: [],
893
- effects: [
894
- {
895
- id: `effect:restart-continuation:${run.id}`,
896
- commandId,
897
- threadId,
898
- request: { type: "provider-runtime.continue", sourceRunId: run.id },
899
- },
900
- ],
901
- });
902
- }).pipe(
903
- Effect.catchCauseIf(
904
- (cause) => !Cause.hasInterruptsOnly(cause),
905
- (cause) =>
906
- Effect.logWarning("Failed to prepare a restart continuation", { threadId, cause }),
878
+ const enabledProjects = Object.entries(enabled.projectSettingsOverrides)
879
+ .filter(([, overrides]) => overrides.continueThreadsAfterServerUpdate === true)
880
+ .map(([id]) => id);
881
+ if (!enabled.continueThreadsAfterServerUpdate && enabledProjects.length === 0) return;
882
+ const disabledProjects = Object.entries(enabled.projectSettingsOverrides)
883
+ .filter(([, overrides]) => overrides.continueThreadsAfterServerUpdate === false)
884
+ .map(([id]) => id);
885
+ const threadIds = yield* projections.getRecoveryThreadIds(
886
+ "restart-continuations",
887
+ enabled.continueThreadsAfterServerUpdate
888
+ ? { exclude: disabledProjects }
889
+ : { include: enabledProjects },
890
+ );
891
+ yield* Effect.forEach(
892
+ threadIds,
893
+ (threadId) =>
894
+ Effect.gen(function* () {
895
+ const projection = yield* projections.getRuntimeRecoveryProjection(threadId);
896
+ if (
897
+ !resolveProjectSettings(enabled, projection.thread.projectId).settings
898
+ .continueThreadsAfterServerUpdate
899
+ )
900
+ return;
901
+ const run = restartContinuationRun(projection, yield* DateTime.now);
902
+ if (!run) return;
903
+ const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
904
+ yield* eventSink.writeWithEffects({
905
+ commandId,
906
+ events: [],
907
+ effects: [
908
+ {
909
+ id: `effect:restart-continuation:${run.id}`,
910
+ commandId,
911
+ threadId,
912
+ request: { type: "provider-runtime.continue", sourceRunId: run.id },
913
+ },
914
+ ],
915
+ });
916
+ }).pipe(
917
+ Effect.catchCauseIf(
918
+ (cause) => !Cause.hasInterruptsOnly(cause),
919
+ (cause) =>
920
+ Effect.logWarning("Failed to prepare a restart continuation", { threadId, cause }),
921
+ ),
922
),
908
- );
909
- }
923
+ // Leave room in the four-reader pool for other shutdown reads.
924
+ { concurrency: 2, discard: true },
925
+ );
926
}).pipe(
927
+ Effect.withSpan("ProviderRuntimeRecoveryService.prepareForShutdown"),
928
Effect.mapError((cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause })),
929
);
930
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+28
-1
@@ -3113,6 +3113,27 @@ it.effect("ProviderSessionManagerV2 marks pending runtime requests non-live on r
3113
now,
3114
});
3115
yield* eventSink.write({ events: pendingRequest.events });
3116
+ const historical = yield* makePendingRuntimeRequestEvents({
3117
+ idAllocator,
3118
+ threadId,
3119
+ providerSessionId,
3120
+ providerThread,
3121
+ now,
3122
+ });
3123
+ yield* eventSink.write({
3124
+ events: historical.events.map((event) =>
3125
+ event.type === "runtime-request.updated"
3126
+ ? { ...event, payload: { ...event.payload, status: "resolved", resolvedAt: now } }
3127
+ : event,
3128
+ ),
3129
+ });
3130
+ assert.deepEqual(
3131
+ (yield* projectionStore.getThreadRecords(threadId, ["nodes", "turnItems"], {
3132
+ nodeIds: [pendingRequest.nodeId],
3133
+ turnItemRequestIds: [pendingRequest.requestId],
3134
+ })).turnItems.map((item) => item.id),
3135
+ [pendingRequest.events.find((event) => event.type === "turn-item.updated")!.payload.id],
3136
+ );
3137
yield* manager.open({
3138
threadId,
3139
providerSessionId,
@@ -3126,7 +3147,13 @@ it.effect("ProviderSessionManagerV2 marks pending runtime requests non-live on r
3147
});
3148
3149
const projection = yield* projectionStore.getThreadProjection(threadId);
3129
- const request = projection.runtimeRequests.at(-1);
3150
+ const request = projection.runtimeRequests.find(
3151
+ (request) => request.id === pendingRequest.requestId,
3152
+ );
3153
+ assert.equal(
3154
+ projection.runtimeRequests.find((request) => request.id === historical.requestId)?.status,
3155
+ "resolved",
3156
+ );
3157
const requestNode = projection.nodes.find((node) => node.id === request?.nodeId);
3158
const requestTurnItem = projection.turnItems.find(
3159
(item) => item.type === "approval_request" && item.requestId === request?.id,
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+15
-5
@@ -655,8 +655,8 @@ export const layerWithOptions = (
655
for (const threadId of input.entry.attachedThreadIds) {
656
const projection = yield* projectionStore.getThreadRecords(
657
threadId,
658
- ["runtimeRequests", "nodes", "turnItems"],
659
- { turnItemTypes: ["approval_request", "user_input_request"] },
658
+ ["runtimeRequests"],
659
+ { runtimeRequestStatuses: ["pending"] },
660
);
661
const releasedRequests = projection.runtimeRequests.filter(
662
(request) =>
@@ -668,6 +668,16 @@ export const layerWithOptions = (
668
!(input.reason === "server_shutdown" && request.kind === "user_input"),
669
);
670
671
+ if (releasedRequests.length === 0) continue;
672
+ const records = yield* projectionStore.getThreadRecords(
673
+ threadId,
674
+ ["nodes", "turnItems"],
675
+ {
676
+ nodeIds: releasedRequests.map((request) => request.nodeId),
677
+ turnItemRequestIds: releasedRequests.map((request) => request.id),
678
+ turnItemTypes: ["approval_request", "user_input_request"],
679
+ },
680
+ );
681
for (const request of releasedRequests) {
682
events.push({
683
id: yield* idAllocator.allocate.event({
@@ -690,7 +700,7 @@ export const layerWithOptions = (
700
},
701
});
702
693
- const requestNode = projection.nodes.find((node) => node.id === request.nodeId);
703
+ const requestNode = records.nodes.find((node) => node.id === request.nodeId);
704
if (requestNode !== undefined) {
705
events.push({
706
id: yield* idAllocator.allocate.event({
@@ -711,7 +721,7 @@ export const layerWithOptions = (
721
});
722
}
723
714
- const turnItem = projection.turnItems.find(
724
+ const turnItem = records.turnItems.find(
725
(item) =>
726
(item.type === "approval_request" || item.type === "user_input_request") &&
727
item.requestId === request.id,
@@ -2120,7 +2130,7 @@ export const layerWithOptions = (
2130
),
2131
{ concurrency: "unbounded", discard: true },
2132
);
2123
- });
2133
+ }).pipe(Effect.withSpan("ProviderSessionManager.shutdown"));
2134
yield* Effect.addFinalizer(() =>
2135
shutdown.pipe(
2136
Effect.ensuring(closeScopeWithin(sessionScopes, { reason: "server_shutdown" })),
apps/server/src/orchestration-v2/RestartContinuation.test.ts
+64
@@ -17,6 +17,8 @@ import {
17
} from "@t3tools/contracts";
18
import * as DateTime from "effect/DateTime";
19
import * as Effect from "effect/Effect";
20
+import * as Deferred from "effect/Deferred";
21
+import * as Fiber from "effect/Fiber";
22
import * as Layer from "effect/Layer";
23
import * as ServerSettings from "../serverSettings.ts";
24
import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
@@ -1262,3 +1264,65 @@ it.effect.each(["completed", "failed", "interrupted"] as const)(
1264
);
1265
}),
1266
);
1267
+
1268
+it.effect("prepares independent continuations together and waits for durable writes", () =>
1269
+ Effect.gen(function* () {
1270
+ const bothReading = yield* Deferred.make<void>();
1271
+ const bothWriting = yield* Deferred.make<void>();
1272
+ const committed = yield* Deferred.make<void>();
1273
+ let reads = 0;
1274
+ let writes = 0;
1275
+ const recovery = yield* ProviderRuntimeRecovery.make.pipe(
1276
+ Effect.provide(
1277
+ Layer.mergeAll(
1278
+ ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
1279
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
1280
+ getRecoveryThreadIds: () => Effect.succeed([threadId, ThreadId.make("thread:second")]),
1281
+ getRuntimeRecoveryProjection: () =>
1282
+ Effect.gen(function* () {
1283
+ if (++reads === 2) yield* Deferred.succeed(bothReading, undefined);
1284
+ yield* Deferred.await(bothReading);
1285
+ return makeProjection();
1286
+ }),
1287
+ }),
1288
+ Layer.mock(EventSink.EventSinkV2)({
1289
+ writeWithEffects: () =>
1290
+ Effect.gen(function* () {
1291
+ if (++writes === 2) yield* Deferred.succeed(bothWriting, undefined);
1292
+ yield* Deferred.await(committed);
1293
+ return [];
1294
+ }),
1295
+ }),
1296
+ IdAllocator.layer,
1297
+ Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
1298
+ Layer.mock(EffectOutbox.EffectOutboxV2)({}),
1299
+ ),
1300
+ ),
1301
+ );
1302
+ const fiber = yield* recovery.prepareForShutdown.pipe(Effect.forkChild);
1303
+ yield* Deferred.await(bothWriting);
1304
+ assert.equal(writes, 2);
1305
+ assert.isUndefined(fiber.pollUnsafe());
1306
+ yield* Deferred.succeed(committed, undefined);
1307
+ yield* Fiber.join(fiber);
1308
+ }),
1309
+);
1310
+
1311
+it.effect("skips restart discovery when no project enables continuation", () =>
1312
+ Effect.gen(function* () {
1313
+ const recovery = yield* ProviderRuntimeRecovery.make.pipe(
1314
+ Effect.provide(
1315
+ Layer.mergeAll(
1316
+ ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false }),
1317
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({}),
1318
+ Layer.mock(EventSink.EventSinkV2)({}),
1319
+ IdAllocator.layer,
1320
+ Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
1321
+ Layer.mock(EffectOutbox.EffectOutboxV2)({}),
1322
+ ),
1323
+ ),
1324
+ );
1325
+ yield* recovery.prepareForShutdown;
1326
+ yield* recovery.prepareForShutdown;
1327
+ }),
1328
+);
apps/server/src/serverRuntimeStartup.ts
+1
@@ -478,6 +478,7 @@ const make = Effect.fn("serverRuntimeStartup.make")(function* (options?: Startup
478
const reconciliation = yield* providerRuntimeRecovery.reconcile("shutdown");
479
yield* Effect.logInfo("V2 orchestration shutdown reconciliation completed", reconciliation);
480
}).pipe(
481
+ Effect.withSpan("server.shutdown"),
482
Effect.catchCause((cause) =>
483
Effect.logWarning("V2 orchestration shutdown reconciliation failed", {
484
cause: Cause.pretty(cause),