The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 11:00 UTC
fix(server): one busy thread's lock no longer stalls other threads' queued runs (#37417)
Julius Marminge committed
Nov 15, 2026 at 16:56 UTC
5da89cfcebaa57cd77ae1b2546462f7ac501f42a
2 files changed
+164
-13
apps/server/src/orchestration-v2/Orchestrator.ts
+45
-13
@@ -93,7 +93,9 @@ import {
93
import * as Context from "effect/Context";
94
import * as DateTime from "effect/DateTime";
95
import * as Effect from "effect/Effect";
96
+import * as Deferred from "effect/Deferred";
97
import * as Exit from "effect/Exit";
98
+import * as FiberSet from "effect/FiberSet";
99
import * as FileSystem from "effect/FileSystem";
100
import * as Path from "effect/Path";
101
import * as Layer from "effect/Layer";
@@ -15791,6 +15793,30 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15793
),
15794
);
15795
15796
+ // One thread's reactions run in event order; a reaction waiting on one thread's lock must
15797
+ // not hold up another thread's.
15798
+ const threadReactions = yield* FiberSet.make<void>();
15799
+ const threadReactionTails = new Map<ThreadId, Deferred.Deferred<void>>();
15800
+ const reactInThreadOrder = Effect.fnUntraced(function* <R>(
15801
+ threadId: ThreadId,
15802
+ reaction: Effect.Effect<void, never, R>,
15803
+ ) {
15804
+ const done = yield* Deferred.make<void>();
15805
+ const previous = threadReactionTails.get(threadId);
15806
+ threadReactionTails.set(threadId, done);
15807
+ yield* (
15808
+ previous === undefined ? reaction : Effect.andThen(Deferred.await(previous), reaction)
15809
+ ).pipe(
15810
+ Effect.ensuring(
15811
+ Effect.suspend(() => {
15812
+ if (threadReactionTails.get(threadId) === done) threadReactionTails.delete(threadId);
15813
+ return Deferred.succeed(done, undefined);
15814
+ }),
15815
+ ),
15816
+ FiberSet.run(threadReactions),
15817
+ );
15818
+ });
15819
+
15820
// Projections already contain historical terminals; replaying them delays startup queue promotion with database size.
15821
const terminalEventsAfterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie);
15822
// Filter run updates before buffering so blocked queue promotion does not retain unrelated tool bodies.
@@ -15813,7 +15839,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15839
stored.event.payload.status === "cancelled" ||
15840
stored.event.payload.status === "rolled_back"),
15841
),
15816
- Stream.runForEach(handleRunUpdate),
15842
+ Stream.runForEach((stored) =>
15843
+ reactInThreadOrder(stored.event.threadId, handleRunUpdate(stored)),
15844
+ ),
15845
Effect.forkScoped,
15846
);
15847
@@ -15871,6 +15899,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15899
Effect.catchCause((cause) =>
15900
Effect.logWarning("Failed to deliver deferred steering", { cause }),
15901
),
15902
+ (effect, stored) => reactInThreadOrder(stored.event.threadId, effect),
15903
),
15904
),
15905
Effect.forkScoped,
@@ -15900,18 +15929,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15929
).pipe(
15930
Stream.filter(settlesChildWork),
15931
Stream.runForEach((stored) =>
15903
- Effect.gen(function* () {
15904
- const childThreadId = stored.event.threadId;
15905
- const parentThreadId = yield* appOwnedSubagentParentThreadId(childThreadId);
15906
- if (parentThreadId === undefined) return;
15907
- yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(childThreadId));
15908
- }).pipe(
15909
- Effect.catchCause((cause) =>
15910
- Effect.logWarning("Failed to finalize a delegated child after its work settled", {
15911
- threadId: stored.event.threadId,
15912
- sequence: stored.sequence,
15913
- cause,
15914
- }),
15932
+ reactInThreadOrder(
15933
+ stored.event.threadId,
15934
+ Effect.gen(function* () {
15935
+ const childThreadId = stored.event.threadId;
15936
+ const parentThreadId = yield* appOwnedSubagentParentThreadId(childThreadId);
15937
+ if (parentThreadId === undefined) return;
15938
+ yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(childThreadId));
15939
+ }).pipe(
15940
+ Effect.catchCause((cause) =>
15941
+ Effect.logWarning("Failed to finalize a delegated child after its work settled", {
15942
+ threadId: stored.event.threadId,
15943
+ sequence: stored.sequence,
15944
+ cause,
15945
+ }),
15946
+ ),
15947
),
15948
),
15949
),
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+119
@@ -2048,6 +2048,125 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
2048
}),
2049
);
2050
2051
+ // regression: #37412 one thread's busy lock held up every other thread's run-end reaction
2052
+ it.effect("starts an idle thread's queued run while another thread's lock is busy", () =>
2053
+ Effect.gen(function* () {
2054
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
2055
+ const eventSink = yield* EventSink.EventSinkV2;
2056
+ const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
2057
+ const now = yield* DateTime.now;
2058
+ const startRun = Effect.fnUntraced(function* (threadId: ThreadId) {
2059
+ yield* orchestrator.dispatch({
2060
+ type: "thread.create",
2061
+ createdBy: "user",
2062
+ creationSource: "web",
2063
+ commandId: CommandId.make(`${threadId}:create`),
2064
+ threadId,
2065
+ projectId: ProjectId.make("runtime-run-end-isolation-project"),
2066
+ title: threadId,
2067
+ modelSelection,
2068
+ runtimeMode: "full-access",
2069
+ interactionMode: "default",
2070
+ branch: null,
2071
+ worktreePath: process.cwd(),
2072
+ });
2073
+ yield* orchestrator.dispatch({
2074
+ type: "message.dispatch",
2075
+ createdBy: "user",
2076
+ creationSource: "web",
2077
+ commandId: CommandId.make(`${threadId}:first`),
2078
+ threadId,
2079
+ messageId: MessageId.make(`${threadId}:first`),
2080
+ text: "Start work.",
2081
+ attachments: [],
2082
+ dispatchMode: { type: "start_immediately" },
2083
+ });
2084
+ const run = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
2085
+ yield* eventSink.write({
2086
+ commandId: CommandId.make(`${threadId}:running`),
2087
+ events: [
2088
+ {
2089
+ id: EventId.make(`${threadId}:running-event`),
2090
+ type: "run.updated",
2091
+ threadId,
2092
+ runId: run.id,
2093
+ occurredAt: now,
2094
+ payload: { ...run, status: "running", startedAt: now },
2095
+ },
2096
+ ],
2097
+ });
2098
+ return run;
2099
+ });
2100
+ const endRun = (run: OrchestrationV2Run) =>
2101
+ eventSink.write({
2102
+ commandId: CommandId.make(`${run.threadId}:completed`),
2103
+ events: [
2104
+ {
2105
+ id: EventId.make(`${run.threadId}:completed-event`),
2106
+ type: "run.updated",
2107
+ threadId: run.threadId,
2108
+ runId: run.id,
2109
+ occurredAt: now,
2110
+ payload: { ...run, status: "completed", startedAt: now, completedAt: now },
2111
+ },
2112
+ ],
2113
+ });
2114
+ const busyThreadId = ThreadId.make("runtime-run-end-isolation-busy");
2115
+ const idleThreadId = ThreadId.make("runtime-run-end-isolation-idle");
2116
+ const busyRun = yield* startRun(busyThreadId);
2117
+ const idleRun = yield* startRun(idleThreadId);
2118
+ yield* orchestrator.dispatch({
2119
+ type: "message.dispatch",
2120
+ createdBy: "user",
2121
+ creationSource: "web",
2122
+ commandId: CommandId.make(`${idleThreadId}:queue`),
2123
+ threadId: idleThreadId,
2124
+ messageId: MessageId.make(`${idleThreadId}:queued`),
2125
+ text: "Run this after the current turn.",
2126
+ attachments: [],
2127
+ dispatchMode: { type: "queue_after_active" },
2128
+ });
2129
+ const queuedRun = (yield* orchestrator.getThreadProjection(idleThreadId)).runs.find(
2130
+ (run) => run.status === "queued",
2131
+ );
2132
+ assert.isDefined(queuedRun);
2133
+
2134
+ const order: string[] = [];
2135
+ const queuedRunStarted = yield* orchestrator.streamDomainEvents.pipe(
2136
+ Stream.filter(
2137
+ (event) =>
2138
+ event.type === "run.updated" &&
2139
+ event.payload.id === queuedRun.id &&
2140
+ event.payload.status !== "queued",
2141
+ ),
2142
+ Stream.take(1),
2143
+ Stream.runDrain,
2144
+ Effect.andThen(Effect.sync(() => order.push("idle thread's queued run started"))),
2145
+ Effect.forkScoped,
2146
+ );
2147
+ // Stands in for a long send or delegated bookkeeping holding the busy thread's lock.
2148
+ const lockHeld = yield* Deferred.make<void>();
2149
+ const busyLock = yield* executor
2150
+ .withLock(
2151
+ busyThreadId,
2152
+ Deferred.succeed(lockHeld, undefined).pipe(
2153
+ Effect.andThen(Effect.sleep("1 minute")),
2154
+ Effect.andThen(Effect.sync(() => order.push("busy thread's lock released"))),
2155
+ ),
2156
+ )
2157
+ .pipe(Effect.forkScoped);
2158
+ yield* Deferred.await(lockHeld);
2159
+
2160
+ yield* endRun(busyRun);
2161
+ yield* endRun(idleRun);
2162
+ yield* TestClock.adjust("1 minute");
2163
+ yield* Fiber.join(busyLock);
2164
+ yield* Fiber.join(queuedRunStarted);
2165
+
2166
+ assert.deepEqual(order, ["idle thread's queued run started", "busy thread's lock released"]);
2167
+ }),
2168
+ );
2169
+
2170
const seedPendingQuestion = Effect.fnUntraced(function* (
2171
threadId: ThreadId,
2172
runId: RunId | null,