The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 03:15 UTC
fix(server): a command id reused for two threads at once no longer succeeds for both (#18385)
Co-authored-by: SAPHID <saphid@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: maria-rcks <maria@kuuro.net>
Alex Southwell committed
Nov 6, 2026 at 04:21 UTC
a530cab2bdcec09360fb69ec730bc9a1dca5821d
2 files changed
+241
-45
apps/server/src/orchestration-v2/Orchestrator.receipt-race.test.ts
new
+168
@@ -0,0 +1,168 @@
1
+import { vi } from "vite-plus/test";
2
+import { assert, it } from "@effect/vitest";
3
+import {
4
+ CommandId,
5
+ ProjectId,
6
+ ProviderDriverKind,
7
+ ProviderInstanceId,
8
+ ThreadId,
9
+} from "@t3tools/contracts";
10
+import * as Cause from "effect/Cause";
11
+import * as Deferred from "effect/Deferred";
12
+import * as Effect from "effect/Effect";
13
+import * as Exit from "effect/Exit";
14
+import * as Layer from "effect/Layer";
15
+import * as Ref from "effect/Ref";
16
+import * as SqlitePersistence from "../persistence/Sqlite.ts";
17
+import { CodexProviderCapabilitiesV2 } from "./Adapters/CodexAdapterV2.ts";
18
+import * as Orchestrator from "./Orchestrator.ts";
19
+import * as ProjectionStore from "./ProjectionStore.ts";
20
+import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts";
21
+import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts";
22
+import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
23
+
24
+const instanceId = ProviderInstanceId.make("codex");
25
+const modelSelection = { instanceId, model: "gpt-5.1-codex" };
26
+// Holds model-selection planning until both racing commands have read no receipt.
27
+let capabilityGate: Effect.Effect<void> | undefined;
28
+const adapter = {
29
+ instanceId,
30
+ driver: ProviderDriverKind.make("codex"),
31
+ getCapabilities: () =>
32
+ (capabilityGate ?? Effect.void).pipe(Effect.as(CodexProviderCapabilitiesV2)),
33
+ planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" as const }),
34
+ openSession: () => Effect.die("No provider process needed for metadata controls"),
35
+} as ProviderAdapter.ProviderAdapterV2["Service"];
36
+const layerDatabase = SqlitePersistence.layerMemory;
37
+const layerTest = Layer.mergeAll(
38
+ layerDatabase,
39
+ ProjectionStore.layer.pipe(Layer.provide(layerDatabase)),
40
+ ProviderReplayHarness.layerWithRegistry(
41
+ { name: "receipt-race" },
42
+ ProviderAdapterRegistry.layerFromAdapters([adapter]),
43
+ { databaseLayer: layerDatabase, runEffectWorker: false },
44
+ ),
45
+);
46
+
47
+it.effect("a command id reused concurrently for two threads succeeds for only one", () =>
48
+ Effect.gen(function* () {
49
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
50
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
51
+ const threadIds = [ThreadId.make("thread:race-a"), ThreadId.make("thread:race-b")];
52
+ for (const threadId of threadIds) {
53
+ yield* orchestrator.dispatch({
54
+ type: "thread.create",
55
+ commandId: CommandId.make(`create:${threadId}`),
56
+ threadId,
57
+ projectId: ProjectId.make("project:receipt-race"),
58
+ title: threadId,
59
+ modelSelection,
60
+ runtimeMode: "full-access",
61
+ interactionMode: "default",
62
+ branch: null,
63
+ worktreePath: null,
64
+ createdBy: "user",
65
+ creationSource: "web",
66
+ });
67
+ }
68
+
69
+ const arrived = yield* Ref.make(0);
70
+ const bothArrived = yield* Deferred.make<void>();
71
+ capabilityGate = Effect.gen(function* () {
72
+ if ((yield* Ref.updateAndGet(arrived, (count) => count + 1)) === 2) {
73
+ yield* Deferred.succeed(bothArrived, undefined);
74
+ }
75
+ yield* Deferred.await(bothArrived);
76
+ });
77
+ const setModel = (threadId: ThreadId) =>
78
+ orchestrator.dispatch({
79
+ type: "thread.model-selection.set",
80
+ commandId: CommandId.make("command:shared"),
81
+ threadId,
82
+ modelSelection: { ...modelSelection, model: "gpt-6" },
83
+ });
84
+ const exits = yield* Effect.forEach(threadIds, (threadId) => Effect.exit(setModel(threadId)), {
85
+ concurrency: "unbounded",
86
+ }).pipe(Effect.ensuring(Effect.sync(() => (capabilityGate = undefined))));
87
+
88
+ const winner = exits.findIndex(Exit.isSuccess);
89
+ const loser = exits.findIndex(Exit.isFailure);
90
+ assert.isAtLeast(winner, 0);
91
+ assert.isAtLeast(loser, 0);
92
+ const winnerExit = exits[winner]!;
93
+ const loserExit = exits[loser]!;
94
+ assert.isTrue(Exit.isSuccess(winnerExit) && Exit.isFailure(loserExit));
95
+ if (Exit.isFailure(loserExit)) {
96
+ const error = Cause.findErrorOption(loserExit.cause);
97
+ assert.isTrue(error._tag === "Some");
98
+ if (error._tag === "Some") {
99
+ assert.equal(error.value._tag, "OrchestratorCommandIdConflictError");
100
+ if (error.value._tag === "OrchestratorCommandIdConflictError") {
101
+ assert.equal(error.value.receiptThreadId, threadIds[winner]);
102
+ assert.equal(error.value.commandThreadId, threadIds[loser]);
103
+ }
104
+ }
105
+ }
106
+
107
+ const models = yield* Effect.forEach(threadIds, (threadId) =>
108
+ projections.getThreadShell(threadId).pipe(Effect.map((shell) => shell?.modelSelection.model)),
109
+ );
110
+ assert.equal(models[winner], "gpt-6");
111
+ assert.equal(models[loser], "gpt-5.1-codex");
112
+
113
+ const noOpArrived = yield* Ref.make(0);
114
+ const bothNoOpsArrived = yield* Deferred.make<void>();
115
+ const getThreadRecords = projections.getThreadRecords;
116
+ using _records = vi.spyOn(projections, "getThreadRecords").mockImplementation((...args) =>
117
+ Effect.gen(function* () {
118
+ if ((yield* Ref.updateAndGet(noOpArrived, (count) => count + 1)) === 2) {
119
+ yield* Deferred.succeed(bothNoOpsArrived, undefined);
120
+ }
121
+ yield* Deferred.await(bothNoOpsArrived);
122
+ return yield* getThreadRecords(...args);
123
+ }),
124
+ );
125
+ const closeShare = (threadId: ThreadId) =>
126
+ orchestrator.dispatch({
127
+ type: "thread.guest-share.close",
128
+ commandId: CommandId.make("command:shared-noop"),
129
+ threadId,
130
+ });
131
+ const noOpExits = yield* Effect.forEach(
132
+ threadIds,
133
+ (threadId) => Effect.exit(closeShare(threadId)),
134
+ {
135
+ concurrency: "unbounded",
136
+ },
137
+ );
138
+ const noOpWinner = noOpExits.findIndex(Exit.isSuccess);
139
+ const noOpLoser = noOpExits.findIndex(Exit.isFailure);
140
+ assert.isAtLeast(noOpWinner, 0);
141
+ assert.isAtLeast(noOpLoser, 0);
142
+ const noOpFailure = noOpExits[noOpLoser]!;
143
+ if (Exit.isFailure(noOpFailure)) {
144
+ const error = Cause.findErrorOption(noOpFailure.cause);
145
+ assert.isTrue(error._tag === "Some");
146
+ if (error._tag === "Some") {
147
+ assert.equal(error.value._tag, "OrchestratorCommandIdConflictError");
148
+ }
149
+ }
150
+ const noOpReplay = yield* closeShare(threadIds[noOpWinner]!);
151
+ const noOpSuccess = noOpExits[noOpWinner]!;
152
+ if (Exit.isSuccess(noOpSuccess)) {
153
+ assert.equal(noOpReplay.sequence, noOpSuccess.value.sequence);
154
+ assert.deepEqual(noOpReplay.storedEvents, []);
155
+ }
156
+
157
+ // The winner's own thread still replays its receipt.
158
+ const replay = yield* setModel(threadIds[winner]!);
159
+ if (Exit.isSuccess(winnerExit)) {
160
+ assert.equal(replay.sequence, winnerExit.value.sequence);
161
+ assert.isAbove(replay.storedEvents.length, 0);
162
+ assert.deepEqual(
163
+ replay.storedEvents.map((stored) => stored.sequence),
164
+ winnerExit.value.storedEvents.map((stored) => stored.sequence),
165
+ );
166
+ }
167
+ }).pipe(Effect.provide(layerTest)),
168
+);
apps/server/src/orchestration-v2/Orchestrator.ts
+73
-45
@@ -110,7 +110,7 @@ import {
110
isNativeMaintenanceCommand,
111
resolveMessageDispatchIntent,
112
} from "./CommandPolicy.ts";
113
-import { CommandReceiptStoreV2 } from "./CommandReceiptStore.ts";
113
+import { CommandReceiptStoreV2, type CommandReceiptV2 } from "./CommandReceiptStore.ts";
114
import {
115
ContextHandoffServiceV2,
116
readForkHistoryItems,
@@ -13741,6 +13741,54 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13741
};
13742
});
13743
13744
+ const replayCommandReceipt = Effect.fn("orchestrationV2.dispatch.replayReceipt")(function* (
13745
+ command: OrchestrationV2ServerCommand,
13746
+ receipt: CommandReceiptV2,
13747
+ ) {
13748
+ if (receipt.status === "rejected") {
13749
+ return yield* new OrchestratorCommandPreviouslyRejectedError({
13750
+ commandId: command.commandId,
13751
+ commandType: command.type,
13752
+ detail: receipt.error ?? "Previously rejected.",
13753
+ });
13754
+ }
13755
+ const dispatchThreadId = commandThreadId(command);
13756
+ // Receipts only prove execution for the thread they were recorded against.
13757
+ if (receipt.threadId !== dispatchThreadId) {
13758
+ return yield* new OrchestratorCommandIdConflictError({
13759
+ commandId: command.commandId,
13760
+ commandType: command.type,
13761
+ receiptThreadId: receipt.threadId,
13762
+ commandThreadId: dispatchThreadId,
13763
+ });
13764
+ }
13765
+ const storedEvents = yield* eventSink.readByCommandId({ commandId: command.commandId }).pipe(
13766
+ Stream.runCollect,
13767
+ Effect.map((events): ReadonlyArray<OrchestrationV2StoredEvent> => Array.from(events)),
13768
+ Effect.mapError(
13769
+ (cause) =>
13770
+ new OrchestratorDispatchError({
13771
+ commandId: command.commandId,
13772
+ commandType: command.type,
13773
+ cause,
13774
+ }),
13775
+ ),
13776
+ );
13777
+ if (command.type === "queue.resume" || command.type === "queued-run.start") {
13778
+ yield* mapDispatchError(command)(
13779
+ startNextQueuedRun(
13780
+ command.threadId,
13781
+ command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
13782
+ ),
13783
+ );
13784
+ }
13785
+ return {
13786
+ sequence: receipt.resultSequence,
13787
+ storedEvents,
13788
+ replayed: true,
13789
+ } satisfies OrchestratorV2DispatchResult;
13790
+ });
13791
+
13792
/** `sideChatIds` are the live side chats a `thread.delete` cascades to, read under their locks. */
13793
const dispatchWithReceiptEffect = Effect.fn("orchestrationV2.dispatch.withReceipt")(function* (
13794
command: OrchestrationV2ServerCommand,
@@ -13764,49 +13812,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13812
);
13813
13814
if (Option.isSome(existingReceipt)) {
13767
- const receipt = existingReceipt.value;
13768
- if (receipt.status === "rejected") {
13769
- return yield* new OrchestratorCommandPreviouslyRejectedError({
13770
- commandId: command.commandId,
13771
- commandType: command.type,
13772
- detail: receipt.error ?? "Previously rejected.",
13773
- });
13774
- }
13775
- const dispatchThreadId = commandThreadId(command);
13776
- // Receipts only prove execution for the thread they were recorded against.
13777
- if (receipt.threadId !== dispatchThreadId) {
13778
- return yield* new OrchestratorCommandIdConflictError({
13779
- commandId: command.commandId,
13780
- commandType: command.type,
13781
- receiptThreadId: receipt.threadId,
13782
- commandThreadId: dispatchThreadId,
13783
- });
13784
- }
13785
- const storedEvents = yield* eventSink.readByCommandId({ commandId: command.commandId }).pipe(
13786
- Stream.runCollect,
13787
- Effect.map((events): ReadonlyArray<OrchestrationV2StoredEvent> => Array.from(events)),
13788
- Effect.mapError(
13789
- (cause) =>
13790
- new OrchestratorDispatchError({
13791
- commandId: command.commandId,
13792
- commandType: command.type,
13793
- cause,
13794
- }),
13795
- ),
13796
- );
13797
- if (command.type === "queue.resume" || command.type === "queued-run.start") {
13798
- yield* mapDispatchError(command)(
13799
- startNextQueuedRun(
13800
- command.threadId,
13801
- command.type === "queued-run.start" ? { queuedRunId: command.runId } : undefined,
13802
- ),
13803
- );
13804
- }
13805
- return {
13806
- sequence: receipt.resultSequence,
13807
- storedEvents,
13808
- replayed: true,
13809
- } satisfies OrchestratorV2DispatchResult;
13815
+ return yield* replayCommandReceipt(command, existingReceipt.value);
13816
}
13817
13818
// Recheck under the lock; the user may have raised modes since sender preflight.
@@ -13921,10 +13927,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13927
}),
13928
),
13929
);
13930
+ if (!inserted) {
13931
+ const receipt = yield* mapDispatchError(command)(
13932
+ commandReceipts.getByCommandId(command.commandId),
13933
+ );
13934
+ if (Option.isNone(receipt)) {
13935
+ return yield* new OrchestratorDispatchError({
13936
+ commandId: command.commandId,
13937
+ commandType: command.type,
13938
+ cause: "Conflicting receipt is not a thread command receipt.",
13939
+ });
13940
+ }
13941
+ return yield* replayCommandReceipt(command, receipt.value);
13942
+ }
13943
return {
13944
sequence: resultSequence,
13945
storedEvents: [],
13927
- ...(inserted ? {} : { replayed: true as const }),
13946
} satisfies OrchestratorV2DispatchResult;
13947
}
13948
const acceptedAt = plan.events.at(-1)?.occurredAt ?? (yield* DateTime.now);
@@ -13961,6 +13979,16 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13979
detail: committed.receipt.error ?? "Previously rejected.",
13980
});
13981
}
13982
+ // A concurrent command with this id for another thread can commit first,
13983
+ // under that thread's lock; its receipt is not this command's result.
13984
+ if (committed.receipt.threadId !== commandThreadId(command)) {
13985
+ return yield* new OrchestratorCommandIdConflictError({
13986
+ commandId: command.commandId,
13987
+ commandType: command.type,
13988
+ receiptThreadId: committed.receipt.threadId,
13989
+ commandThreadId: commandThreadId(command),
13990
+ });
13991
+ }
13992
if (
13993
command.type === "queue.resume" ||
13994
command.type === "queued-run.start" ||