@devries-cameron20 / t3code-1 / commits / 620a37b988
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): prepare restart continuations faster on shutdown (#31807)

Yash Singh committed Nov 5, 2026 at 22:57 UTC 620a37b988a5a0a270ea7f70c0458ba658e9d072
7 files changed +216 -38
apps/server/src/orchestration-v2/ProjectionRecovery.test.ts
+22
@@ -836,6 +836,28 @@ it.effect("includes shared sessions and provider-owned background rosters in rec
836 new Set(yield* projections.getRecoveryThreadIds("runtime")),
837 new Set([first, second, roster, prepared]),
838 );
839 + assert.deepEqual(
840 + new Set(yield* projections.getRecoveryThreadIds("restart-continuations")),
841 + new Set([first, second, prepared]),
842 + );
843 + assert.deepEqual(
844 + yield* projections.getRecoveryThreadIds("restart-continuations", { include: [] }),
845 + [],
846 + );
847 + assert.deepEqual(
848 + yield* projections.getRecoveryThreadIds("restart-continuations", {
849 + exclude: ["project:recovery"],
850 + }),
851 + [],
852 + );
853 + assert.deepEqual(
854 + new Set(
855 + yield* projections.getRecoveryThreadIds("restart-continuations", {
856 + include: ["project:recovery"],
857 + }),
858 + ),
859 + new Set([first, second, prepared]),
860 + );
861 const preparedState = yield* projections.getRuntimeRecoveryProjection(prepared);
862 assert.deepEqual(
863 preparedState.providerSessions.map((session) => session.id),
apps/server/src/orchestration-v2/ProjectionStore.ts
+38 -1
@@ -168,6 +168,7 @@ export type ProjectionStoreV2Error = typeof ProjectionStoreV2Error.Type;
168 export type ProjectionRecoveryKind =
169 | "queued-runs"
170 | "runtime"
171 + | "restart-continuations"
172 | "subagent-results"
173 | "delegated-completions";
174
@@ -353,6 +354,7 @@ export interface ProjectionRecordFilter {
354 readonly subagentIds?: ReadonlyArray<NodeId>;
355 readonly subagentRunIds?: ReadonlyArray<RunId>;
356 readonly nodeIds?: ReadonlyArray<NodeId>;
357 + readonly turnItemRequestIds?: ReadonlyArray<RuntimeRequestId>;
358 readonly turnItemTypes?: ReadonlyArray<OrchestrationV2TurnItem["type"]>;
359 readonly turnItemStatuses?: ReadonlyArray<OrchestrationV2TurnItem["status"]>;
360 readonly omitSettledLeafNodes?: boolean;
@@ -678,6 +680,10 @@ export interface ProjectionStoreV2Shape {
680 ) => Effect.Effect<ReadonlyArray<OrchestrationV2Run>, ProjectionStoreV2Error>;
681 readonly getRecoveryThreadIds: (
682 kind: ProjectionRecoveryKind,
683 + projects?: {
684 + readonly include?: ReadonlyArray<string>;
685 + readonly exclude?: ReadonlyArray<string>;
686 + },
687 ) => Effect.Effect<ReadonlyArray<ThreadId>, ProjectionStoreV2Error>;
688 readonly getUnreadableThreadIds: () => Effect.Effect<
689 ReadonlyArray<ThreadId>,
@@ -3129,6 +3135,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3135 (fields !== undefined && !fields.includes("turnItems")) ||
3136 (window === undefined &&
3137 matchesNothing(
3138 + filter?.turnItemRequestIds,
3139 filter?.turnItemTypes,
3140 filter?.turnItemStatuses,
3141 filter?.turnItemRunIds,
@@ -3140,6 +3147,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3147 SELECT payload_json
3148 FROM orchestration_v2_projection_turn_items
3149 WHERE thread_id = ${threadId}
3150 + ${filter?.turnItemRequestIds === undefined ? sql`` : sql`AND json_extract(payload_json, '$.requestId') IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemRequestIds)}))`}
3151 ${filter?.turnItemTypes === undefined ? sql`` : sql`AND type IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemTypes)}))`}
3152 ${filter?.turnItemStatuses === undefined ? sql`` : sql`AND status IN (SELECT value FROM json_each(${encodeIdList(filter.turnItemStatuses)}))`}
3153 ${filter?.turnItemRunId === undefined ? sql`` : sql`AND run_id = ${filter.turnItemRunId}`}
@@ -4120,7 +4128,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4128 );
4129
4130 const getRecoveryThreadIds = Effect.fn("ProjectionStore.getRecoveryThreadIds")(
4123 - function* (kind: ProjectionRecoveryKind) {
4131 + function* (
4132 + kind: ProjectionRecoveryKind,
4133 + projects?: {
4134 + readonly include?: ReadonlyArray<string>;
4135 + readonly exclude?: ReadonlyArray<string>;
4136 + },
4137 + ) {
4138 const candidates = (() => {
4139 switch (kind) {
4140 case "queued-runs":
@@ -4171,6 +4185,27 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4185 AND type = 'subagent_result'
4186 )))
4187 `;
4188 + case "restart-continuations":
4189 + return sql`
4190 + SELECT bindings.thread_id
4191 + FROM orchestration_v2_projection_provider_sessions AS sessions
4192 + JOIN orchestration_v2_projection_provider_session_bindings AS bindings
4193 + ON bindings.provider_session_id = sessions.provider_session_id
4194 + WHERE sessions.status NOT IN ('stopped', 'error')
4195 + UNION
4196 + SELECT thread_id FROM orchestration_v2_projection_provider_sessions AS sessions
4197 + WHERE status NOT IN ('stopped', 'error') AND thread_id IS NOT NULL
4198 + AND NOT EXISTS (
4199 + SELECT 1 FROM orchestration_v2_projection_provider_session_bindings AS bindings
4200 + WHERE bindings.provider_session_id = sessions.provider_session_id
4201 + )
4202 + UNION
4203 + SELECT thread_id FROM orchestration_v2_projection_runs
4204 + WHERE status = 'starting'
4205 + AND CASE WHEN json_valid(payload_json)
4206 + THEN json_extract(payload_json, '$.restartContinuationOfRunId') IS NOT NULL
4207 + ELSE 0 END
4208 + `;
4209 case "runtime":
4210 return sql`
4211 WITH pending_provider_threads AS MATERIALIZED (
@@ -4257,6 +4292,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
4292 SELECT thread_id FROM orchestration_v2_projection_threads
4293 WHERE deleted_at IS NULL
4294 AND thread_id IN (${candidates})
4295 + ${projects?.include === undefined ? sql`` : sql`AND project_id IN (SELECT value FROM json_each(${encodeIdList(projects.include)}))`}
4296 + ${projects?.exclude === undefined ? sql`` : sql`AND project_id NOT IN (SELECT value FROM json_each(${encodeIdList(projects.exclude)}))`}
4297 ${kind === "queued-runs" ? sql`AND archived_at IS NULL` : sql``}
4298 ORDER BY updated_at ASC, thread_id ASC
4299 `;
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+48 -31
@@ -875,39 +875,56 @@ export const make = Effect.gen(function* () {
875 const prepareForShutdown = Effect.gen(function* () {
876 const enabled = yield* settings.getSettings.pipe(Effect.orElseSucceed(() => null));
877 if (!enabled) return;
878 - const threadIds = yield* projections.getRecoveryThreadIds("runtime");
879 - for (const threadId of threadIds) {
880 - yield* Effect.gen(function* () {
881 - const projection = yield* projections.getRuntimeRecoveryProjection(threadId);
882 - if (
883 - !resolveProjectSettings(enabled, projection.thread.projectId).settings
884 - .continueThreadsAfterServerUpdate
885 - )
886 - return;
887 - const run = restartContinuationRun(projection, yield* DateTime.now);
888 - if (!run) return;
889 - const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
890 - yield* eventSink.writeWithEffects({
891 - commandId,
892 - events: [],
893 - effects: [
894 - {
895 - id: `effect:restart-continuation:${run.id}`,
896 - commandId,
897 - threadId,
898 - request: { type: "provider-runtime.continue", sourceRunId: run.id },
899 - },
900 - ],
901 - });
902 - }).pipe(
903 - Effect.catchCauseIf(
904 - (cause) => !Cause.hasInterruptsOnly(cause),
905 - (cause) =>
906 - Effect.logWarning("Failed to prepare a restart continuation", { threadId, cause }),
878 + const enabledProjects = Object.entries(enabled.projectSettingsOverrides)
879 + .filter(([, overrides]) => overrides.continueThreadsAfterServerUpdate === true)
880 + .map(([id]) => id);
881 + if (!enabled.continueThreadsAfterServerUpdate && enabledProjects.length === 0) return;
882 + const disabledProjects = Object.entries(enabled.projectSettingsOverrides)
883 + .filter(([, overrides]) => overrides.continueThreadsAfterServerUpdate === false)
884 + .map(([id]) => id);
885 + const threadIds = yield* projections.getRecoveryThreadIds(
886 + "restart-continuations",
887 + enabled.continueThreadsAfterServerUpdate
888 + ? { exclude: disabledProjects }
889 + : { include: enabledProjects },
890 + );
891 + yield* Effect.forEach(
892 + threadIds,
893 + (threadId) =>
894 + Effect.gen(function* () {
895 + const projection = yield* projections.getRuntimeRecoveryProjection(threadId);
896 + if (
897 + !resolveProjectSettings(enabled, projection.thread.projectId).settings
898 + .continueThreadsAfterServerUpdate
899 + )
900 + return;
901 + const run = restartContinuationRun(projection, yield* DateTime.now);
902 + if (!run) return;
903 + const commandId = CommandId.make(`command:restart-prepare:${run.id}`);
904 + yield* eventSink.writeWithEffects({
905 + commandId,
906 + events: [],
907 + effects: [
908 + {
909 + id: `effect:restart-continuation:${run.id}`,
910 + commandId,
911 + threadId,
912 + request: { type: "provider-runtime.continue", sourceRunId: run.id },
913 + },
914 + ],
915 + });
916 + }).pipe(
917 + Effect.catchCauseIf(
918 + (cause) => !Cause.hasInterruptsOnly(cause),
919 + (cause) =>
920 + Effect.logWarning("Failed to prepare a restart continuation", { threadId, cause }),
921 + ),
922 ),
908 - );
909 - }
923 + // Leave room in the four-reader pool for other shutdown reads.
924 + { concurrency: 2, discard: true },
925 + );
926 }).pipe(
927 + Effect.withSpan("ProviderRuntimeRecoveryService.prepareForShutdown"),
928 Effect.mapError((cause) => new ProviderRuntimeRecoveryError({ operation: "reconcile", cause })),
929 );
930
apps/server/src/orchestration-v2/ProviderSessionManager.test.ts
+28 -1
@@ -3113,6 +3113,27 @@ it.effect("ProviderSessionManagerV2 marks pending runtime requests non-live on r
3113 now,
3114 });
3115 yield* eventSink.write({ events: pendingRequest.events });
3116 + const historical = yield* makePendingRuntimeRequestEvents({
3117 + idAllocator,
3118 + threadId,
3119 + providerSessionId,
3120 + providerThread,
3121 + now,
3122 + });
3123 + yield* eventSink.write({
3124 + events: historical.events.map((event) =>
3125 + event.type === "runtime-request.updated"
3126 + ? { ...event, payload: { ...event.payload, status: "resolved", resolvedAt: now } }
3127 + : event,
3128 + ),
3129 + });
3130 + assert.deepEqual(
3131 + (yield* projectionStore.getThreadRecords(threadId, ["nodes", "turnItems"], {
3132 + nodeIds: [pendingRequest.nodeId],
3133 + turnItemRequestIds: [pendingRequest.requestId],
3134 + })).turnItems.map((item) => item.id),
3135 + [pendingRequest.events.find((event) => event.type === "turn-item.updated")!.payload.id],
3136 + );
3137 yield* manager.open({
3138 threadId,
3139 providerSessionId,
@@ -3126,7 +3147,13 @@ it.effect("ProviderSessionManagerV2 marks pending runtime requests non-live on r
3147 });
3148
3149 const projection = yield* projectionStore.getThreadProjection(threadId);
3129 - const request = projection.runtimeRequests.at(-1);
3150 + const request = projection.runtimeRequests.find(
3151 + (request) => request.id === pendingRequest.requestId,
3152 + );
3153 + assert.equal(
3154 + projection.runtimeRequests.find((request) => request.id === historical.requestId)?.status,
3155 + "resolved",
3156 + );
3157 const requestNode = projection.nodes.find((node) => node.id === request?.nodeId);
3158 const requestTurnItem = projection.turnItems.find(
3159 (item) => item.type === "approval_request" && item.requestId === request?.id,
apps/server/src/orchestration-v2/ProviderSessionManager.ts
+15 -5
@@ -655,8 +655,8 @@ export const layerWithOptions = (
655 for (const threadId of input.entry.attachedThreadIds) {
656 const projection = yield* projectionStore.getThreadRecords(
657 threadId,
658 - ["runtimeRequests", "nodes", "turnItems"],
659 - { turnItemTypes: ["approval_request", "user_input_request"] },
658 + ["runtimeRequests"],
659 + { runtimeRequestStatuses: ["pending"] },
660 );
661 const releasedRequests = projection.runtimeRequests.filter(
662 (request) =>
@@ -668,6 +668,16 @@ export const layerWithOptions = (
668 !(input.reason === "server_shutdown" && request.kind === "user_input"),
669 );
670
671 + if (releasedRequests.length === 0) continue;
672 + const records = yield* projectionStore.getThreadRecords(
673 + threadId,
674 + ["nodes", "turnItems"],
675 + {
676 + nodeIds: releasedRequests.map((request) => request.nodeId),
677 + turnItemRequestIds: releasedRequests.map((request) => request.id),
678 + turnItemTypes: ["approval_request", "user_input_request"],
679 + },
680 + );
681 for (const request of releasedRequests) {
682 events.push({
683 id: yield* idAllocator.allocate.event({
@@ -690,7 +700,7 @@ export const layerWithOptions = (
700 },
701 });
702
693 - const requestNode = projection.nodes.find((node) => node.id === request.nodeId);
703 + const requestNode = records.nodes.find((node) => node.id === request.nodeId);
704 if (requestNode !== undefined) {
705 events.push({
706 id: yield* idAllocator.allocate.event({
@@ -711,7 +721,7 @@ export const layerWithOptions = (
721 });
722 }
723
714 - const turnItem = projection.turnItems.find(
724 + const turnItem = records.turnItems.find(
725 (item) =>
726 (item.type === "approval_request" || item.type === "user_input_request") &&
727 item.requestId === request.id,
@@ -2120,7 +2130,7 @@ export const layerWithOptions = (
2130 ),
2131 { concurrency: "unbounded", discard: true },
2132 );
2123 - });
2133 + }).pipe(Effect.withSpan("ProviderSessionManager.shutdown"));
2134 yield* Effect.addFinalizer(() =>
2135 shutdown.pipe(
2136 Effect.ensuring(closeScopeWithin(sessionScopes, { reason: "server_shutdown" })),
apps/server/src/orchestration-v2/RestartContinuation.test.ts
+64
@@ -17,6 +17,8 @@ import {
17 } from "@t3tools/contracts";
18 import * as DateTime from "effect/DateTime";
19 import * as Effect from "effect/Effect";
20 +import * as Deferred from "effect/Deferred";
21 +import * as Fiber from "effect/Fiber";
22 import * as Layer from "effect/Layer";
23 import * as ServerSettings from "../serverSettings.ts";
24 import { restartContinuationRun, continueRestartedRun } from "./RestartContinuation.ts";
@@ -1262,3 +1264,65 @@ it.effect.each(["completed", "failed", "interrupted"] as const)(
1264 );
1265 }),
1266 );
1267 +
1268 +it.effect("prepares independent continuations together and waits for durable writes", () =>
1269 + Effect.gen(function* () {
1270 + const bothReading = yield* Deferred.make<void>();
1271 + const bothWriting = yield* Deferred.make<void>();
1272 + const committed = yield* Deferred.make<void>();
1273 + let reads = 0;
1274 + let writes = 0;
1275 + const recovery = yield* ProviderRuntimeRecovery.make.pipe(
1276 + Effect.provide(
1277 + Layer.mergeAll(
1278 + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: true }),
1279 + Layer.mock(ProjectionStore.ProjectionStoreV2)({
1280 + getRecoveryThreadIds: () => Effect.succeed([threadId, ThreadId.make("thread:second")]),
1281 + getRuntimeRecoveryProjection: () =>
1282 + Effect.gen(function* () {
1283 + if (++reads === 2) yield* Deferred.succeed(bothReading, undefined);
1284 + yield* Deferred.await(bothReading);
1285 + return makeProjection();
1286 + }),
1287 + }),
1288 + Layer.mock(EventSink.EventSinkV2)({
1289 + writeWithEffects: () =>
1290 + Effect.gen(function* () {
1291 + if (++writes === 2) yield* Deferred.succeed(bothWriting, undefined);
1292 + yield* Deferred.await(committed);
1293 + return [];
1294 + }),
1295 + }),
1296 + IdAllocator.layer,
1297 + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
1298 + Layer.mock(EffectOutbox.EffectOutboxV2)({}),
1299 + ),
1300 + ),
1301 + );
1302 + const fiber = yield* recovery.prepareForShutdown.pipe(Effect.forkChild);
1303 + yield* Deferred.await(bothWriting);
1304 + assert.equal(writes, 2);
1305 + assert.isUndefined(fiber.pollUnsafe());
1306 + yield* Deferred.succeed(committed, undefined);
1307 + yield* Fiber.join(fiber);
1308 + }),
1309 +);
1310 +
1311 +it.effect("skips restart discovery when no project enables continuation", () =>
1312 + Effect.gen(function* () {
1313 + const recovery = yield* ProviderRuntimeRecovery.make.pipe(
1314 + Effect.provide(
1315 + Layer.mergeAll(
1316 + ServerSettings.layerTest({ continueThreadsAfterServerUpdate: false }),
1317 + Layer.mock(ProjectionStore.ProjectionStoreV2)({}),
1318 + Layer.mock(EventSink.EventSinkV2)({}),
1319 + IdAllocator.layer,
1320 + Layer.mock(EffectWorker.OrchestrationEffectWorkerV2)({}),
1321 + Layer.mock(EffectOutbox.EffectOutboxV2)({}),
1322 + ),
1323 + ),
1324 + );
1325 + yield* recovery.prepareForShutdown;
1326 + yield* recovery.prepareForShutdown;
1327 + }),
1328 +);
apps/server/src/serverRuntimeStartup.ts
+1
@@ -478,6 +478,7 @@ const make = Effect.fn("serverRuntimeStartup.make")(function* (options?: Startup
478 const reconciliation = yield* providerRuntimeRecovery.reconcile("shutdown");
479 yield* Effect.logInfo("V2 orchestration shutdown reconciliation completed", reconciliation);
480 }).pipe(
481 + Effect.withSpan("server.shutdown"),
482 Effect.catchCause((cause) =>
483 Effect.logWarning("V2 orchestration shutdown reconciliation failed", {
484 cause: Cause.pretty(cause),