@devries-cameron20 / t3code-1 / commits / a530cab2bd
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): a command id reused for two threads at once no longer succeeds for both (#18385)

Co-authored-by: SAPHID <saphid@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>

Alex Southwell committed Nov 6, 2026 at 04:21 UTC a530cab2bdcec09360fb69ec730bc9a1dca5821d
2 files changed +241 -45
apps/server/src/orchestration-v2/Orchestrator.receipt-race.test.ts new
+168
@@ -0,0 +1,168 @@
1 +import { vi } from "vite-plus/test";
2 +import { assert, it } from "@effect/vitest";
3 +import {
4 + CommandId,
5 + ProjectId,
6 + ProviderDriverKind,
7 + ProviderInstanceId,
8 + ThreadId,
9 +} from "@t3tools/contracts";
10 +import * as Cause from "effect/Cause";
11 +import * as Deferred from "effect/Deferred";
12 +import * as Effect from "effect/Effect";
13 +import * as Exit from "effect/Exit";
14 +import * as Layer from "effect/Layer";
15 +import * as Ref from "effect/Ref";
16 +import * as SqlitePersistence from "../persistence/Sqlite.ts";
17 +import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
18 +import * as Orchestrator from "./Orchestrator.ts";
19 +import * as ProjectionStore from "./ProjectionStore.ts";
20 +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
21 +import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
22 +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
23 +
24 +const instanceId = ProviderInstanceId.make("codex");
25 +const modelSelection = { instanceId, model: "gpt-5.1-codex" };
26 +// Holds model-selection planning until both racing commands have read no receipt.
27 +let capabilityGate: Effect.Effect<void> | undefined;
28 +const adapter = {
29 + instanceId,
30 + driver: ProviderDriverKind.make("codex"),
31 + getCapabilities: () =>
32 + (capabilityGate ?? Effect.void).pipe(Effect.as(CodexProviderCapabilitiesV2)),
33 + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }),
34 + openSession: () => Effect.die("No provider process needed for metadata controls"),
35 +} as ProviderAdapter.ProviderAdapterV2["Service"];
36 +const layerDatabase = SqlitePersistence.layerMemory;
37 +const layerTest = Layer.mergeAll(
38 + layerDatabase,
39 + ProjectionStore.layer.pipe(Layer.provide(layerDatabase)),
40 + ProviderReplayHarness.layerWithRegistry(
41 + { name: "receipt-race" },
42 + ProviderAdapterRegistry.layerFromAdapters([adapter]),
43 + { databaseLayer: layerDatabase, runEffectWorker: false },
44 + ),
45 +);
46 +
47 +it.effect("a command id reused concurrently for two threads succeeds for only one", () =>
48 + Effect.gen(function* () {
49 + const orchestrator = yield* Orchestrator.OrchestratorV2;
50 + const projections = yield* ProjectionStore.ProjectionStoreV2;
51 + const threadIds = [ThreadId.make("thread:race-a"), ThreadId.make("thread:race-b")];
52 + for (const threadId of threadIds) {
53 + yield* orchestrator.dispatch({
54 + type: "thread.create",
55 + commandId: CommandId.make(`create:${threadId}`),
56 + threadId,
57 + projectId: ProjectId.make("project:receipt-race"),
58 + title: threadId,
59 + modelSelection,
60 + runtimeMode: "full-access",
61 + interactionMode: "default",
62 + branch: null,
63 + worktreePath: null,
64 + createdBy: "user",
65 + creationSource: "web",
66 + });
67 + }
68 +
69 + const arrived = yield* Ref.make(0);
70 + const bothArrived = yield* Deferred.make<void>();
71 + capabilityGate = Effect.gen(function* () {
72 + if ((yield* Ref.updateAndGet(arrived, (count) => count + 1)) === 2) {
73 + yield* Deferred.succeed(bothArrived, undefined);
74 + }
75 + yield* Deferred.await(bothArrived);
76 + });
77 + const setModel = (threadId: ThreadId) =>
78 + orchestrator.dispatch({
79 + type: "thread.model-selection.set",
80 + commandId: CommandId.make("command:shared"),
81 + threadId,
82 + modelSelection: { ...modelSelection, model: "gpt-6" },
83 + });
84 + const exits = yield* Effect.forEach(threadIds, (threadId) => Effect.exit(setModel(threadId)), {
85 + concurrency: "unbounded",
86 + }).pipe(Effect.ensuring(Effect.sync(() => (capabilityGate = undefined))));
87 +
88 + const winner = exits.findIndex(Exit.isSuccess);
89 + const loser = exits.findIndex(Exit.isFailure);
90 + assert.isAtLeast(winner, 0);
91 + assert.isAtLeast(loser, 0);
92 + const winnerExit = exits[winner]!;
93 + const loserExit = exits[loser]!;
94 + assert.isTrue(Exit.isSuccess(winnerExit) && Exit.isFailure(loserExit));
95 + if (Exit.isFailure(loserExit)) {
96 + const error = Cause.findErrorOption(loserExit.cause);
97 + assert.isTrue(error._tag === "Some");
98 + if (error._tag === "Some") {
99 + assert.equal(error.value._tag, "OrchestratorCommandIdConflictError");
100 + if (error.value._tag === "OrchestratorCommandIdConflictError") {
101 + assert.equal(error.value.receiptThreadId, threadIds[winner]);
102 + assert.equal(error.value.commandThreadId, threadIds[loser]);
103 + }
104 + }
105 + }
106 +
107 + const models = yield* Effect.forEach(threadIds, (threadId) =>
108 + projections.getThreadShell(threadId).pipe(Effect.map((shell) => shell?.modelSelection.model)),
109 + );
110 + assert.equal(models[winner], "gpt-6");
111 + assert.equal(models[loser], "gpt-5.1-codex");
112 +
113 + const noOpArrived = yield* Ref.make(0);
114 + const bothNoOpsArrived = yield* Deferred.make<void>();
115 + const getThreadRecords = projections.getThreadRecords;
116 + using _records = vi.spyOn(projections, "getThreadRecords").mockImplementation((...args) =>
117 + Effect.gen(function* () {
118 + if ((yield* Ref.updateAndGet(noOpArrived, (count) => count + 1)) === 2) {
119 + yield* Deferred.succeed(bothNoOpsArrived, undefined);
120 + }
121 + yield* Deferred.await(bothNoOpsArrived);
122 + return yield* getThreadRecords(...args);
123 + }),
124 + );
125 + const closeShare = (threadId: ThreadId) =>
126 + orchestrator.dispatch({
127 + type: "thread.guest-share.close",
128 + commandId: CommandId.make("command:shared-noop"),
129 + threadId,
130 + });
131 + const noOpExits = yield* Effect.forEach(
132 + threadIds,
133 + (threadId) => Effect.exit(closeShare(threadId)),
134 + {
135 + concurrency: "unbounded",
136 + },
137 + );
138 + const noOpWinner = noOpExits.findIndex(Exit.isSuccess);
139 + const noOpLoser = noOpExits.findIndex(Exit.isFailure);
140 + assert.isAtLeast(noOpWinner, 0);
141 + assert.isAtLeast(noOpLoser, 0);
142 + const noOpFailure = noOpExits[noOpLoser]!;
143 + if (Exit.isFailure(noOpFailure)) {
144 + const error = Cause.findErrorOption(noOpFailure.cause);
145 + assert.isTrue(error._tag === "Some");
146 + if (error._tag === "Some") {
147 + assert.equal(error.value._tag, "OrchestratorCommandIdConflictError");
148 + }
149 + }
150 + const noOpReplay = yield* closeShare(threadIds[noOpWinner]!);
151 + const noOpSuccess = noOpExits[noOpWinner]!;
152 + if (Exit.isSuccess(noOpSuccess)) {
153 + assert.equal(noOpReplay.sequence, noOpSuccess.value.sequence);
154 + assert.deepEqual(noOpReplay.storedEvents, []);
155 + }
156 +
157 + // The winner's own thread still replays its receipt.
158 + const replay = yield* setModel(threadIds[winner]!);
159 + if (Exit.isSuccess(winnerExit)) {
160 + assert.equal(replay.sequence, winnerExit.value.sequence);
161 + assert.isAbove(replay.storedEvents.length, 0);
162 + assert.deepEqual(
163 + replay.storedEvents.map((stored) => stored.sequence),
164 + winnerExit.value.storedEvents.map((stored) => stored.sequence),
165 + );
166 + }
167 + }).pipe(Effect.provide(layerTest)),
168 +);
apps/server/src/orchestration-v2/Orchestrator.ts
+73 -45
@@ -110,7 +110,7 @@ import {
110 isNativeMaintenanceCommand,
111 resolveMessageDispatchIntent,
112 } from "./CommandPolicy.ts";
113 -import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
113 +import { CommandReceiptStoreV2, type CommandReceiptV2 } from "./CommandReceiptStore.ts";
114 import {
115 ContextHandoffServiceV2,
116 readForkHistoryItems,
@@ -13741,6 +13741,54 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13741 };
13742 });
13743
13744 + const replayCommandReceipt = Effect.fn("orchestrationV2.dispatch.replayReceipt")(function* (
13745 + command: OrchestrationV2ServerCommand,
13746 + receipt: CommandReceiptV2,
13747 + ) {
13748 + if (receipt.status === "rejected") {
13749 + return yield* new OrchestratorCommandPreviouslyRejectedError({
13750 + commandId: command.commandId,
13751 + commandType: command.type,
13752 + detail: receipt.error ?? "Previously rejected.",
13753 + });
13754 + }
13755 + const dispatchThreadId = commandThreadId(command);
13756 + // Receipts only prove execution for the thread they were recorded against.
13757 + if (receipt.threadId !== dispatchThreadId) {
13758 + return yield* new OrchestratorCommandIdConflictError({
13759 + commandId: command.commandId,
13760 + commandType: command.type,
13761 + receiptThreadId: receipt.threadId,
13762 + commandThreadId: dispatchThreadId,
13763 + });
13764 + }
13765 + const storedEvents = yield* eventSink.readByCommandId({ commandId: command.commandId }).pipe(
13766 + Stream.runCollect,
13767 + Effect.map((events): ReadonlyArray<OrchestrationV2StoredEvent> => Array.from(events)),
13768 + Effect.mapError(
13769 + (cause) =>
13770 + new OrchestratorDispatchError({
13771 + commandId: command.commandId,
13772 + commandType: command.type,
13773 + cause,
13774 + }),
13775 + ),
13776 + );
13777 + if (command.type === "queue.resume" || command.type === "queued-run.start") {
13778 + yield* mapDispatchError(command)(
13779 + startNextQueuedRun(
13780 + command.threadId,
13781 + command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
13782 + ),
13783 + );
13784 + }
13785 + return {
13786 + sequence: receipt.resultSequence,
13787 + storedEvents,
13788 + replayed: true,
13789 + } satisfies OrchestratorV2DispatchResult;
13790 + });
13791 +
13792 /** `sideChatIds` are the live side chats a `thread.delete` cascades to, read under their locks. */
13793 const dispatchWithReceiptEffect = Effect.fn("orchestrationV2.dispatch.withReceipt")(function* (
13794 command: OrchestrationV2ServerCommand,
@@ -13764,49 +13812,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13812 );
13813
13814 if (Option.isSome(existingReceipt)) {
13767 - const receipt = existingReceipt.value;
13768 - if (receipt.status === "rejected") {
13769 - return yield* new OrchestratorCommandPreviouslyRejectedError({
13770 - commandId: command.commandId,
13771 - commandType: command.type,
13772 - detail: receipt.error ?? "Previously rejected.",
13773 - });
13774 - }
13775 - const dispatchThreadId = commandThreadId(command);
13776 - // Receipts only prove execution for the thread they were recorded against.
13777 - if (receipt.threadId !== dispatchThreadId) {
13778 - return yield* new OrchestratorCommandIdConflictError({
13779 - commandId: command.commandId,
13780 - commandType: command.type,
13781 - receiptThreadId: receipt.threadId,
13782 - commandThreadId: dispatchThreadId,
13783 - });
13784 - }
13785 - const storedEvents = yield* eventSink.readByCommandId({ commandId: command.commandId }).pipe(
13786 - Stream.runCollect,
13787 - Effect.map((events): ReadonlyArray<OrchestrationV2StoredEvent> => Array.from(events)),
13788 - Effect.mapError(
13789 - (cause) =>
13790 - new OrchestratorDispatchError({
13791 - commandId: command.commandId,
13792 - commandType: command.type,
13793 - cause,
13794 - }),
13795 - ),
13796 - );
13797 - if (command.type === "queue.resume" || command.type === "queued-run.start") {
13798 - yield* mapDispatchError(command)(
13799 - startNextQueuedRun(
13800 - command.threadId,
13801 - command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
13802 - ),
13803 - );
13804 - }
13805 - return {
13806 - sequence: receipt.resultSequence,
13807 - storedEvents,
13808 - replayed: true,
13809 - } satisfies OrchestratorV2DispatchResult;
13815 + return yield* replayCommandReceipt(command, existingReceipt.value);
13816 }
13817
13818 // Recheck under the lock; the user may have raised modes since sender preflight.
@@ -13921,10 +13927,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13927 }),
13928 ),
13929 );
13930 + if (!inserted) {
13931 + const receipt = yield* mapDispatchError(command)(
13932 + commandReceipts.getByCommandId(command.commandId),
13933 + );
13934 + if (Option.isNone(receipt)) {
13935 + return yield* new OrchestratorDispatchError({
13936 + commandId: command.commandId,
13937 + commandType: command.type,
13938 + cause: "Conflicting receipt is not a thread command receipt.",
13939 + });
13940 + }
13941 + return yield* replayCommandReceipt(command, receipt.value);
13942 + }
13943 return {
13944 sequence: resultSequence,
13945 storedEvents: [],
13927 - ...(inserted ? {} : { replayed: true as const }),
13946 } satisfies OrchestratorV2DispatchResult;
13947 }
13948 const acceptedAt = plan.events.at(-1)?.occurredAt ?? (yield* DateTime.now);
@@ -13961,6 +13979,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13979 detail: committed.receipt.error ?? "Previously rejected.",
13980 });
13981 }
13982 + // A concurrent command with this id for another thread can commit first,
13983 + // under that thread's lock; its receipt is not this command's result.
13984 + if (committed.receipt.threadId !== commandThreadId(command)) {
13985 + return yield* new OrchestratorCommandIdConflictError({
13986 + commandId: command.commandId,
13987 + commandType: command.type,
13988 + receiptThreadId: committed.receipt.threadId,
13989 + commandThreadId: commandThreadId(command),
13990 + });
13991 + }
13992 if (
13993 command.type === "queue.resume" ||
13994 command.type === "queued-run.start" ||