@devries-cameron20 / t3code-1 / commits / 1bdb6cf1cf
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

perf(server): batch shutdown recovery commits (#31854)

Yash Singh committed Nov 6, 2026 at 00:57 UTC 1bdb6cf1cfb55bff8d007caf6f5a36a4001a84cc
2 files changed +199 -24
apps/server/src/orchestration-v2/FoundationPersistence.test.ts
+153
@@ -563,6 +563,159 @@ it.effect.each(
563 },
564 );
565
566 +it.effect.each([false, true])("groups shutdown commits with rollback=%s", (fail) => {
567 + let commits = 0;
568 + const tracer = Tracer.make({
569 + span(options) {
570 + const span = new Tracer.NativeSpan(options);
571 + const end = span.end.bind(span);
572 + span.end = (time, exit) => {
573 + end(time, exit);
574 + if (span.name === "COMMIT") commits++;
575 + };
576 + return span;
577 + },
578 + });
579 + return Effect.gen(function* () {
580 + const sink = yield* EventSink.EventSinkV2;
581 + const events = yield* EventStore.EventStoreV2;
582 + const projections = yield* ProjectionStore.ProjectionStoreV2;
583 + const sql = yield* SqlClient.SqlClient;
584 + const outbox = yield* EffectOutbox.EffectOutboxV2;
585 + const now = yield* DateTime.now;
586 + const threads = Array.from({ length: 65 }, (_, i) =>
587 + ThreadId.make(`shutdown-group:${String(i).padStart(2, "0")}`),
588 + );
589 + const sharedSession = {
590 + id: ProviderSessionId.make("shutdown-shared"),
591 + driver: providerDriver,
592 + providerInstanceId,
593 + status: "ready" as const,
594 + cwd: process.cwd(),
595 + model: modelSelection.model,
596 + capabilities: CodexProviderCapabilitiesV2,
597 + createdAt: now,
598 + updatedAt: now,
599 + lastError: null,
600 + };
601 + yield* sink.write({
602 + events: threads.flatMap((threadId, i): OrchestrationV2DomainEvent[] => [
603 + threadCreatedEvent({ id: `shutdown-create:${i}`, thread: makeThread(threadId, now), now }),
604 + ...(i < 2
605 + ? [
606 + {
607 + id: EventId.make(`shutdown-session:${i}`),
608 + type: "provider-session.attached" as const,
609 + threadId,
610 + occurredAt: now,
611 + payload: sharedSession,
612 + },
613 + ]
614 + : []),
615 + {
616 + id: EventId.make(`shutdown-provider:${i}`),
617 + type: "provider-thread.updated",
618 + threadId,
619 + occurredAt: now,
620 + payload: {
621 + id: ProviderThreadId.make(`shutdown-provider:${i}`),
622 + driver: providerDriver,
623 + providerInstanceId,
624 + providerSessionId: i < 2 ? sharedSession.id : null,
625 + appThreadId: threadId,
626 + ownerNodeId: null,
627 + nativeThreadRef: null,
628 + nativeConversationHeadRef: null,
629 + status: i === 1 || i === 32 ? "idle" : "active",
630 + firstRunOrdinal: null,
631 + lastRunOrdinal: null,
632 + handoffIds: [],
633 + forkedFrom: null,
634 + createdAt: now,
635 + updatedAt: now,
636 + },
637 + },
638 + ]),
639 + });
640 + yield* outbox.enqueue(
641 + threads.map((threadId, i) => ({
642 + id: `shutdown-effect:${i}`,
643 + commandId: CommandId.make(`shutdown-command:${i}`),
644 + threadId,
645 + request: { type: "provider-turn.start" as const, runId: RunId.make(`shutdown-run:${i}`) },
646 + })),
647 + );
648 + let eventlessNotifications = 0;
649 + let armed = fail;
650 + const recovery = yield* ProviderRuntimeRecovery.make.pipe(
651 + Effect.provide(ServerSettings.layerTest()),
652 + Effect.provideService(EffectOutbox.EffectOutboxV2, {
653 + ...outbox,
654 + signalCancellations: (ids) =>
655 + Effect.gen(function* () {
656 + assert.isTrue(Option.isNone(yield* Effect.serviceOption(sql.transactionService)));
657 + eventlessNotifications += ids.length;
658 + yield* outbox.signalCancellations(ids);
659 + }),
660 + }),
661 + Effect.provideService(
662 + EffectWorker.OrchestrationEffectWorkerV2,
663 + EffectWorker.OrchestrationEffectWorkerV2.of({
664 + awaitWork: Effect.void,
665 + runRecoveryOnce: Effect.succeed(false),
666 + runOnce: Effect.succeed(false),
667 + nextClaimableAt: Effect.succeedNone,
668 + drain: () => Effect.succeed(0),
669 + }),
670 + ),
671 + Effect.provideService(EventSink.EventSinkV2, {
672 + ...sink,
673 + commitCommand: (input) =>
674 + armed && input.threadId === threads[33]
675 + ? Effect.die("injected shutdown failure")
676 + : sink.commitCommand(input),
677 + }),
678 + );
679 + commits = 0;
680 + const result = yield* recovery.reconcile("shutdown").pipe(Effect.exit);
681 + assert.equal(Exit.isFailure(result), fail);
682 + assert.equal(commits, fail ? 1 : 3);
683 + const receipts =
684 + yield* sql`SELECT aggregate_id FROM orchestration_command_receipts WHERE command_type = 'provider-runtime.reconcile' ORDER BY aggregate_id`;
685 + assert.deepEqual(
686 + receipts.map((r) => r.aggregate_id),
687 + threads.slice(0, fail ? 32 : 65).filter((_, i) => i !== 1 && i !== 32),
688 + );
689 + assert.deepEqual(
690 + yield* projections.getRecoveryThreadIds("runtime"),
691 + fail ? threads.slice(32) : [],
692 + );
693 + assert.equal(eventlessNotifications, fail ? 1 : 2);
694 + for (const [i] of threads.entries()) {
695 + const effect = Option.getOrThrow(yield* outbox.get(`shutdown-effect:${i}`));
696 + assert.equal(effect.status, !fail || i < 32 ? "cancelled" : "pending");
697 + }
698 + assert.equal(
699 + (yield* events.read({ afterSequence: 0 }).pipe(Stream.runCollect)).filter(
700 + (e) =>
701 + e.event.type === "provider-session.updated" && e.event.payload.id === sharedSession.id,
702 + ).length,
703 + 1,
704 + );
705 + armed = false;
706 + yield* recovery.reconcile("shutdown");
707 + assert.deepEqual(yield* projections.getRecoveryThreadIds("runtime"), []);
708 + const highWater = yield* sink.latestSequence();
709 + yield* recovery.reconcile("shutdown");
710 + assert.equal(yield* sink.latestSequence(), highWater);
711 + }).pipe(
712 + Effect.provide(Layer.fresh(layerTest)),
713 + Effect.withTracer(tracer),
714 + Effect.withTracerEnabled(true),
715 + Effect.provideService(Statement.SpanPropagationEnabled, true),
716 + );
717 +});
718 +
719 it.layer(layerTest)("orchestration V2 foundation persistence", (it) => {
720 it.effect("projects oversized tool bodies once before replay and live RPC retention", () =>
721 Effect.scoped(
apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts
+46 -24
@@ -51,6 +51,8 @@ export class ProviderRuntimeRecoveryError extends Schema.TaggedError<ProviderRun
51 }
52 }
53
54 +const isProviderRuntimeRecoveryError = Schema.is(ProviderRuntimeRecoveryError);
55 +
56 export interface ProviderRuntimeRecoverySummary {
57 readonly terminalizedRuns: number;
58 readonly stoppedSessions: number;
@@ -762,6 +764,7 @@ export const make = Effect.gen(function* () {
764 (candidate) => candidate.status !== "stopped" && candidate.status !== "error",
765 ).length;
766 let retiredEffects: number;
767 + let cancelledEffectIds: ReadonlyArray<string> = [];
768 if (events.length === 0) {
769 const retiredEffectIds = yield* outbox
770 .cancelUnsettled({
@@ -779,7 +782,7 @@ export const make = Effect.gen(function* () {
782 }),
783 ),
784 );
782 - yield* outbox.signalCancellations(retiredEffectIds);
785 + cancelledEffectIds = retiredEffectIds;
786 retiredEffects = retiredEffectIds.length;
787 } else {
788 const result = yield* eventSink
@@ -812,6 +815,7 @@ export const make = Effect.gen(function* () {
815 stoppedSessions,
816 closedRequests: requests.length,
817 retiredEffects,
818 + cancelledEffectIds,
819 };
820 },
821 );
@@ -832,31 +836,49 @@ export const make = Effect.gen(function* () {
836 let closedRequests = 0;
837 let retiredEffects = 0;
838 const stoppedThisPass = new Map<string, RecoverySession>();
835 - for (const threadId of threadIds) {
836 - const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(
837 - Effect.mapError(
838 - (cause) =>
839 - new ProviderRuntimeRecoveryError({
840 - operation: "read-projections",
841 - threadId,
842 - cause,
843 - }),
839 + const grouped = trigger === "shutdown" && threadIds.length > 1;
840 + const groupSize = grouped ? 32 : 1;
841 + for (let offset = 0; offset < threadIds.length; offset += groupSize) {
842 + const cancelledEffectIds: string[] = [];
843 + // Read and decide on the writer so shared sessions see earlier updates in this group.
844 + const reconcileGroup = Effect.gen(function* () {
845 + for (const threadId of threadIds.slice(offset, offset + groupSize)) {
846 + const projection = yield* projections.getRuntimeRecoveryProjection(threadId).pipe(
847 + Effect.mapError(
848 + (cause) =>
849 + new ProviderRuntimeRecoveryError({
850 + operation: "read-projections",
851 + threadId,
852 + cause,
853 + }),
854 + ),
855 + );
856 + const enabled =
857 + continueAfterRestart !== null &&
858 + resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings
859 + .continueThreadsAfterServerUpdate;
860 + const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);
861 + for (const session of projection.providerSessions) {
862 + if (session.status !== "stopped" && session.status !== "error") {
863 + stoppedThisPass.set(session.id, session);
864 + }
865 + }
866 + cancelledEffectIds.push(...result.cancelledEffectIds);
867 + terminalizedRuns += result.terminalizedRuns;
868 + stoppedSessions += result.stoppedSessions;
869 + closedRequests += result.closedRequests;
870 + retiredEffects += result.retiredEffects;
871 + }
872 + });
873 + yield* (grouped ? eventSink.withProviderBatch(reconcileGroup) : reconcileGroup).pipe(
874 + Effect.mapError((cause) =>
875 + isProviderRuntimeRecoveryError(cause)
876 + ? cause
877 + : new ProviderRuntimeRecoveryError({ operation: "reconcile", cause }),
878 ),
879 );
846 - const enabled =
847 - continueAfterRestart !== null &&
848 - resolveProjectSettings(continueAfterRestart, projection.thread.projectId).settings
849 - .continueThreadsAfterServerUpdate;
850 - const result = yield* reconcileProjection(projection, trigger, enabled, stoppedThisPass);
851 - for (const session of projection.providerSessions) {
852 - if (session.status !== "stopped" && session.status !== "error") {
853 - stoppedThisPass.set(session.id, session);
854 - }
855 - }
856 - terminalizedRuns += result.terminalizedRuns;
857 - stoppedSessions += result.stoppedSessions;
858 - closedRequests += result.closedRequests;
859 - retiredEffects += result.retiredEffects;
880 + // Eventless outbox cancellations also wait for the group's durable commit.
881 + if (cancelledEffectIds.length > 0) yield* outbox.signalCancellations(cancelledEffectIds);
882 }
883 const outboxReconciliation = yield* outbox.reconcileAfterProcessLoss.pipe(
884 Effect.mapError(