The best way to control Claude Code, Codex and more from any computer or phone
perf(server): batch shutdown recovery commits (#31854)
Yash Singh committed
Nov 6, 2026 at 00:57 UTC
1bdb6cf1cfb55bff8d007caf6f5a36a4001a84cc
2 files changed
+199
-24
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+153
@@ -563,6 +563,159 @@ it.effect.each(
563
},
564
);
565
566
+it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {
567
+ let commits = 0;
568
+ const tracer = Tracer.make({
569
+ span(options) {
570
+ const span = new Tracer.NativeSpan(options);
571
+ const end = span.end.bind(span);
572
+ span.end = (time, exit) => {
573
+ end(time, exit);
574
+ if (span.name === "COMMIT") commits++;
575
+ };
576
+ return span;
577
+ },
578
+ });
579
+ return Effect.gen(function* () {
580
+ const sink = yield* EventSink.EventSinkV2;
581
+ const events = yield* EventStore.EventStoreV2;
582
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
583
+ const sql = yield* SqlClient.SqlClient;
584
+ const outbox = yield* EffectOutbox.EffectOutboxV2;
585
+ const now = yield* DateTime.now;
586
+ const threads = Array.from({ length: 65 }, (_, i) =>
587
+ ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),
588
+ );
589
+ const sharedSession = {
590
+ id: ProviderSessionId.make("shutdown-shared"),
591
+ driver: providerDriver,
592
+ providerInstanceId,
593
+ status: "ready" as const,
594
+ cwd: process.cwd(),
595
+ model: modelSelection.model,
596
+ capabilities: CodexProviderCapabilitiesV2,
597
+ createdAt: now,
598
+ updatedAt: now,
599
+ lastError: null,
600
+ };
601
+ yield* sink.write({
602
+ events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [
603
+ threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),
604
+ ...(i < 2
605
+ ? [
606
+ {
607
+ id: EventId.make(`shutdown-session:${i}`),
608
+ type: "provider-session.attached" as const,
609
+ threadId,
610
+ occurredAt: now,
611
+ payload: sharedSession,
612
+ },
613
+ ]
614
+ : []),
615
+ {
616
+ id: EventId.make(`shutdown-provider:${i}`),
617
+ type: "provider-thread.updated",
618
+ threadId,
619
+ occurredAt: now,
620
+ payload: {
621
+ id: ProviderThreadId.make(`shutdown-provider:${i}`),
622
+ driver: providerDriver,
623
+ providerInstanceId,
624
+ providerSessionId: i < 2 ? sharedSession.id : null,
625
+ appThreadId: threadId,
626
+ ownerNodeId: null,
627
+ nativeThreadRef: null,
628
+ nativeConversationHeadRef: null,
629
+ status: i === 1 || i === 32 ? "idle" : "active",
630
+ firstRunOrdinal: null,
631
+ lastRunOrdinal: null,
632
+ handoffIds: [],
633
+ forkedFrom: null,
634
+ createdAt: now,
635
+ updatedAt: now,
636
+ },
637
+ },
638
+ ]),
639
+ });
640
+ yield* outbox.enqueue(
641
+ threads.map((threadId, i) => ({
642
+ id: `shutdown-effect:${i}`,
643
+ commandId: CommandId.make(`shutdown-command:${i}`),
644
+ threadId,
645
+ request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },
646
+ })),
647
+ );
648
+ let eventlessNotifications = 0;
649
+ let armed = fail;
650
+ const recovery = yield* ProviderRuntimeRecovery.make.pipe(
651
+ Effect.provide(ServerSettings.layerTest()),
652
+ Effect.provideService(EffectOutbox.EffectOutboxV2, {
653
+ ...outbox,
654
+ signalCancellations: (ids) =>
655
+ Effect.gen(function* () {
656
+ assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));
657
+ eventlessNotifications += ids.length;
658
+ yield* outbox.signalCancellations(ids);
659
+ }),
660
+ }),
661
+ Effect.provideService(
662
+ EffectWorker.OrchestrationEffectWorkerV2,
663
+ EffectWorker.OrchestrationEffectWorkerV2.of({
664
+ awaitWork: Effect.void,
665
+ runRecoveryOnce: Effect.succeed(false),
666
+ runOnce: Effect.succeed(false),
667
+ nextClaimableAt: Effect.succeedNone,
668
+ drain: () => Effect.succeed(0),
669
+ }),
670
+ ),
671
+ Effect.provideService(EventSink.EventSinkV2, {
672
+ ...sink,
673
+ commitCommand: (input) =>
674
+ armed && input.threadId === threads[33]
675
+ ? Effect.die("injected shutdown failure")
676
+ : sink.commitCommand(input),
677
+ }),
678
+ );
679
+ commits = 0;
680
+ const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);
681
+ assert.equal(Exit.isFailure(result), fail);
682
+ assert.equal(commits, fail ? 1 : 3);
683
+ const receipts =
684
+ yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;
685
+ assert.deepEqual(
686
+ receipts.map((r) => r.aggregate_id),
687
+ threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),
688
+ );
689
+ assert.deepEqual(
690
+ yield* projections.getRecoveryThreadIds("runtime"),
691
+ fail ? threads.slice(32) : [],
692
+ );
693
+ assert.equal(eventlessNotifications, fail ? 1 : 2);
694
+ for (const [i] of threads.entries()) {
695
+ const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));
696
+ assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");
697
+ }
698
+ assert.equal(
699
+ (yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(
700
+ (e) =>
701
+ e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,
702
+ ).length,
703
+ 1,
704
+ );
705
+ armed = false;
706
+ yield* recovery.reconcile("shutdown");
707
+ assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
708
+ const highWater = yield* sink.latestSequence();
709
+ yield* recovery.reconcile("shutdown");
710
+ assert.equal(yield* sink.latestSequence(), highWater);
711
+ }).pipe(
712
+ Effect.provide(Layer.fresh(layerTest)),
713
+ Effect.withTracer(tracer),
714
+ Effect.withTracerEnabled(true),
715
+ Effect.provideService(Statement.SpanPropagationEnabled, true),
716
+ );
717
+});
718
+
719
it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
720
it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
721
Effect.scoped(
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+46
-24
@@ -51,6 +51,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRun
51
}
52
}
53
54
+const isProviderRuntimeRecoveryError = Schema.is(ProviderRuntimeRecoveryError);
55
+
56
export interface ProviderRuntimeRecoverySummary {
57
readonly terminalizedRuns: number;
58
readonly stoppedSessions: number;
@@ -762,6 +764,7 @@ export const make = Effect.gen(function* () {
764
(candidate) => candidate.status !== "stopped" && candidate.status !== "error",
765
).length;
766
let retiredEffects: number;
767
+ let cancelledEffectIds: ReadonlyArray<string> = [];
768
if (events.length === 0) {
769
const retiredEffectIds = yield* outbox
770
.cancelUnsettled({
@@ -779,7 +782,7 @@ export const make = Effect.gen(function* () {
782
}),
783
),
784
);
782
- yield* outbox.signalCancellations(retiredEffectIds);
785
+ cancelledEffectIds = retiredEffectIds;
786
retiredEffects = retiredEffectIds.length;
787
} else {
788
const result = yield* eventSink
@@ -812,6 +815,7 @@ export const make = Effect.gen(function* () {
815
stoppedSessions,
816
closedRequests: requests.length,
817
retiredEffects,
818
+ cancelledEffectIds,
819
};
820
},
821
);
@@ -832,31 +836,49 @@ export const make = Effect.gen(function* () {
836
let closedRequests = 0;
837
let retiredEffects = 0;
838
const stoppedThisPass = new Map<string, RecoverySession>();
835
- for (const threadId of threadIds) {
836
- const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(
837
- Effect.mapError(
838
- (cause) =>
839
- new ProviderRuntimeRecoveryError({
840
- operation: "read-projections",
841
- threadId,
842
- cause,
843
- }),
839
+ const grouped = trigger === "shutdown" && threadIds.length > 1;
840
+ const groupSize = grouped ? 32 : 1;
841
+ for (let offset = 0; offset < threadIds.length; offset += groupSize) {
842
+ const cancelledEffectIds: string[] = [];
843
+ // Read and decide on the writer so shared sessions see earlier updates in this group.
844
+ const reconcileGroup = Effect.gen(function* () {
845
+ for (const threadId of threadIds.slice(offset, offset + groupSize)) {
846
+ const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(
847
+ Effect.mapError(
848
+ (cause) =>
849
+ new ProviderRuntimeRecoveryError({
850
+ operation: "read-projections",
851
+ threadId,
852
+ cause,
853
+ }),
854
+ ),
855
+ );
856
+ const enabled =
857
+ continueAfterRestart !== null &&
858
+ resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings
859
+ .continueThreadsAfterServerUpdate;
860
+ const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);
861
+ for (const session of projection.providerSessions) {
862
+ if (session.status !== "stopped" && session.status !== "error") {
863
+ stoppedThisPass.set(session.id, session);
864
+ }
865
+ }
866
+ cancelledEffectIds.push(...result.cancelledEffectIds);
867
+ terminalizedRuns += result.terminalizedRuns;
868
+ stoppedSessions += result.stoppedSessions;
869
+ closedRequests += result.closedRequests;
870
+ retiredEffects += result.retiredEffects;
871
+ }
872
+ });
873
+ yield* (grouped ? eventSink.withProviderBatch(reconcileGroup) : reconcileGroup).pipe(
874
+ Effect.mapError((cause) =>
875
+ isProviderRuntimeRecoveryError(cause)
876
+ ? cause
877
+ : new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),
878
),
879
);
846
- const enabled =
847
- continueAfterRestart !== null &&
848
- resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings
849
- .continueThreadsAfterServerUpdate;
850
- const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);
851
- for (const session of projection.providerSessions) {
852
- if (session.status !== "stopped" && session.status !== "error") {
853
- stoppedThisPass.set(session.id, session);
854
- }
855
- }
856
- terminalizedRuns += result.terminalizedRuns;
857
- stoppedSessions += result.stoppedSessions;
858
- closedRequests += result.closedRequests;
859
- retiredEffects += result.retiredEffects;
880
+ // Eventless outbox cancellations also wait for the group's durable commit.
881
+ if (cancelledEffectIds.length > 0) yield* outbox.signalCancellations(cancelledEffectIds);
882
}
883
const outboxReconciliation = yield* outbox.reconcileAfterProcessLoss.pipe(
884
Effect.mapError(