The best way to control Claude Code, Codex and more from any computer or phone
fix(server): preserve message order across lineage reads (#24442)
Julius Marminge committed
Nov 11, 2026 at 15:24 UTC
7a64505a21e755498e6b85794124f3754265db38
3 files changed
+113
-15
apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts
+75
@@ -19,7 +19,9 @@ import {
19
TurnItemId,
20
} from "@t3tools/contracts";
21
import * as DateTime from "effect/DateTime";
22
+import * as Deferred from "effect/Deferred";
23
import * as Effect from "effect/Effect";
24
+import * as Fiber from "effect/Fiber";
25
import * as Layer from "effect/Layer";
26
import * as Metric from "effect/Metric";
27
import * as Result from "effect/Result";
@@ -2083,3 +2085,76 @@ it.effect("dispatch records the command outcome and its duration", () =>
2085
assert.strictEqual((yield* durations).count - durationsBefore, 1);
2086
}).pipe(Effect.provide(layerTest)),
2087
);
2088
+
2089
+it.effect("a user message dispatched first stays run 1 when its lineage read is slow", () => {
2090
+ const threadId = ThreadId.make("thread:dispatch-order");
2091
+ // A file database serves projection reads from a reader worker, so the lineage read a user
2092
+ // dispatch makes before taking the thread lock can yield. This gate holds it open instead.
2093
+ const lineageReadEntered = Deferred.makeUnsafe<void>();
2094
+ const lineageReadRelease = Deferred.makeUnsafe<void>();
2095
+ let gateArmed = false;
2096
+ const layerGatedLineageRead = ProviderReplayHarness.layerWithRegistry(
2097
+ { name: "dispatch-order" },
2098
+ ProviderAdapterRegistry.layerFromAdapters([adapter]),
2099
+ {
2100
+ runEffectWorker: false,
2101
+ wrapProjectionStore: (store) => ({
2102
+ ...store,
2103
+ getThread: (id) => {
2104
+ if (!gateArmed || id !== threadId) return store.getThread(id);
2105
+ gateArmed = false;
2106
+ return Deferred.succeed(lineageReadEntered, undefined).pipe(
2107
+ Effect.andThen(Deferred.await(lineageReadRelease)),
2108
+ Effect.andThen(store.getThread(id)),
2109
+ );
2110
+ },
2111
+ }),
2112
+ },
2113
+ );
2114
+ return Effect.gen(function* () {
2115
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
2116
+ yield* orchestrator.dispatch({
2117
+ type: "thread.create",
2118
+ commandId: CommandId.make("create-dispatch-order"),
2119
+ threadId,
2120
+ projectId: ProjectId.make("project:dispatch-order"),
2121
+ title: "Dispatch order",
2122
+ modelSelection,
2123
+ runtimeMode: "full-access",
2124
+ interactionMode: "default",
2125
+ branch: null,
2126
+ worktreePath: null,
2127
+ createdBy: "user",
2128
+ creationSource: "web",
2129
+ });
2130
+ const message = (id: string, createdBy: "user" | "agent") =>
2131
+ orchestrator.dispatch({
2132
+ type: "message.dispatch",
2133
+ commandId: CommandId.make(id),
2134
+ threadId,
2135
+ messageId: MessageId.make(id),
2136
+ text: id,
2137
+ attachments: [],
2138
+ dispatchMode: { type: "defer_start" },
2139
+ createdBy,
2140
+ creationSource: createdBy === "user" ? "web" : "server",
2141
+ });
2142
+
2143
+ gateArmed = true;
2144
+ const user = yield* message("message:user-first", "user").pipe(Effect.forkChild);
2145
+ yield* Deferred.await(lineageReadEntered);
2146
+ // Agent messages skip the lineage read, so this one would take a free thread lock first.
2147
+ const agent = yield* message("message:agent-second", "agent").pipe(
2148
+ Effect.forkChild({ startImmediately: true }),
2149
+ );
2150
+ yield* Deferred.succeed(lineageReadRelease, undefined);
2151
+ yield* Fiber.join(agent);
2152
+ yield* Fiber.join(user);
2153
+
2154
+ const { runs } = yield* orchestrator.getThreadProjection(threadId);
2155
+ assert.deepEqual(
2156
+ runs.toSorted((left, right) => left.ordinal - right.ordinal).map((run) => run.userMessageId),
2157
+ [MessageId.make("message:user-first"), MessageId.make("message:agent-second")],
2158
+ );
2159
+ }).pipe(Effect.provide(layerGatedLineageRead));
2160
+});
apps/server/src/orchestration-v2/Orchestrator.ts
+23
-14
@@ -77,6 +77,7 @@ import {
77
TrimmedNonEmptyString,
78
managedRuntimeModeCeilingMessage,
79
} from "@t3tools/contracts";
80
+import * as KeyedLock from "@t3tools/shared/KeyedLock";
81
import { modelSelectionsEqual } from "@t3tools/shared/model";
82
import { modelSwitchSince } from "./modelSwitch.ts";
83
import { recordThreadOpen } from "@t3tools/shared/paletteSearch";
@@ -1313,6 +1314,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1314
const runtimePolicy = yield* RuntimePolicyV2;
1315
const threadForkService = yield* ThreadForkServiceV2;
1316
const threadDispatch = yield* ThreadCommandExecutor.ThreadCommandExecutor;
1317
+ const messageIntake = yield* KeyedLock.make<ThreadId>();
1318
// Read once: the policy is fixed for the server's life, and dispatch takes it as an input.
1319
const { managedPolicy } = serverSettings;
1320
const runtimeModeCeiling: RuntimeModeCeiling | undefined =
@@ -14674,6 +14676,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14676
} satisfies OrchestratorV2DispatchResult;
14677
}, withCommandMetrics);
14678
14679
+ // Follow-up intake and Stop serialize on the parent before taking the child lock.
14680
+ const withSubagentParentLock = <A, E, R>(threadId: ThreadId, dispatch: Effect.Effect<A, E, R>) =>
14681
+ appOwnedSubagentParentThreadId(threadId).pipe(
14682
+ Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause })),
14683
+ Effect.flatMap((parentThreadId) =>
14684
+ parentThreadId === undefined ? dispatch : threadDispatch.withLock(parentThreadId, dispatch),
14685
+ ),
14686
+ );
14687
+
14688
// dispatchWithReceiptEffect owns the dispatch span; this wrapper selects the thread locks.
14689
const dispatchWithReceipt = Effect.fnUntraced(function* (command: OrchestrationV2ServerCommand) {
14690
const dispatch = threadDispatch.withLock(
@@ -14706,21 +14717,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
14717
),
14718
),
14719
);
14720
+ // Taken before the lineage read, so a message whose read yields keeps its
14721
+ // place ahead of later messages to the same thread.
14722
+ if (command.type === "message.dispatch")
14723
+ return yield* messageIntake.withLock(
14724
+ command.threadId,
14725
+ mayStartSubagentFollowup(command)
14726
+ ? withSubagentParentLock(command.threadId, dispatch)
14727
+ : dispatch,
14728
+ );
14729
// Answering a message-mode question can also start a follow-up.
14710
- if (
14711
- command.type !== "runtime-request.respond" &&
14712
- (command.type !== "message.dispatch" || !mayStartSubagentFollowup(command))
14713
- )
14714
- return yield* dispatch;
14715
- const parentThreadId = yield* appOwnedSubagentParentThreadId(command.threadId).pipe(
14716
- Effect.mapError(
14717
- (cause) => new OrchestratorProjectionError({ threadId: command.threadId, cause }),
14718
- ),
14719
- );
14720
- // Follow-up intake and Stop serialize on the parent before taking the child lock.
14721
- return yield* parentThreadId === undefined
14722
- ? dispatch
14723
- : threadDispatch.withLock(parentThreadId, dispatch);
14730
+ return yield* command.type === "runtime-request.respond"
14731
+ ? withSubagentParentLock(command.threadId, dispatch)
14732
+ : dispatch;
14733
});
14734
14735
const handleRunUpdate = Effect.fn("Orchestrator.handleRunUpdate")(
apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts
+15
-1
@@ -282,6 +282,10 @@ export function layerWithRegistry<Error>(
282
readonly runContinuationWorker?: boolean;
283
readonly recoverOnStartup?: boolean;
284
readonly continueThreadsAfterServerUpdate?: boolean;
285
+ /** Lets a test hold a projection read open to order concurrent dispatches. */
286
+ readonly wrapProjectionStore?: (
287
+ store: ProjectionStore.ProjectionStoreV2Shape,
288
+ ) => ProjectionStore.ProjectionStoreV2Shape;
289
readonly managedPolicy?: ManagedPolicyDocument;
290
} = {},
291
): Layer.Layer<
@@ -322,9 +326,19 @@ export function layerWithRegistry<Error>(
326
),
327
),
328
).pipe(Layer.orDie);
329
+ const wrapProjectionStore = options.wrapProjectionStore;
330
+ const layerProjectionStore =
331
+ wrapProjectionStore === undefined
332
+ ? ProjectionStore.layer
333
+ : Layer.effect(
334
+ ProjectionStore.ProjectionStoreV2,
335
+ Effect.gen(function* () {
336
+ return wrapProjectionStore(yield* ProjectionStore.ProjectionStoreV2);
337
+ }),
338
+ ).pipe(Layer.provide(ProjectionStore.layer));
339
const layerStores = Layer.mergeAll(
340
EventStore.layer,
327
- ProjectionStore.layer,
341
+ layerProjectionStore,
342
ProjectStore.layer,
343
CommandReceiptStore.layer,
344
EffectOutbox.layer,