@devries-cameron20 / t3code-1 / commits / 5da89cfceb
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): one busy thread's lock no longer stalls other threads' queued runs (#37417)

Julius Marminge committed Nov 15, 2026 at 16:56 UTC 5da89cfcebaa57cd77ae1b2546462f7ac501f42a
2 files changed +164 -13
apps/server/src/orchestration-v2/Orchestrator.ts
+45 -13
@@ -93,7 +93,9 @@ import {
93 import * as Context from "effect/Context";
94 import * as DateTime from "effect/DateTime";
95 import * as Effect from "effect/Effect";
96 +import * as Deferred from "effect/Deferred";
97 import * as Exit from "effect/Exit";
98 +import * as FiberSet from "effect/FiberSet";
99 import * as FileSystem from "effect/FileSystem";
100 import * as Path from "effect/Path";
101 import * as Layer from "effect/Layer";
@@ -15791,6 +15793,30 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15793 ),
15794 );
15795
15796 + // One thread's reactions run in event order; a reaction waiting on one thread's lock must
15797 + // not hold up another thread's.
15798 + const threadReactions = yield* FiberSet.make<void>();
15799 + const threadReactionTails = new Map<ThreadId, Deferred.Deferred<void>>();
15800 + const reactInThreadOrder = Effect.fnUntraced(function* <R>(
15801 + threadId: ThreadId,
15802 + reaction: Effect.Effect<void, never, R>,
15803 + ) {
15804 + const done = yield* Deferred.make<void>();
15805 + const previous = threadReactionTails.get(threadId);
15806 + threadReactionTails.set(threadId, done);
15807 + yield* (
15808 + previous === undefined ? reaction : Effect.andThen(Deferred.await(previous), reaction)
15809 + ).pipe(
15810 + Effect.ensuring(
15811 + Effect.suspend(() => {
15812 + if (threadReactionTails.get(threadId) === done) threadReactionTails.delete(threadId);
15813 + return Deferred.succeed(done, undefined);
15814 + }),
15815 + ),
15816 + FiberSet.run(threadReactions),
15817 + );
15818 + });
15819 +
15820 // Projections already contain historical terminals; replaying them delays startup queue promotion with database size.
15821 const terminalEventsAfterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie);
15822 // Filter run updates before buffering so blocked queue promotion does not retain unrelated tool bodies.
@@ -15813,7 +15839,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15839 stored.event.payload.status === "cancelled" ||
15840 stored.event.payload.status === "rolled_back"),
15841 ),
15816 - Stream.runForEach(handleRunUpdate),
15842 + Stream.runForEach((stored) =>
15843 + reactInThreadOrder(stored.event.threadId, handleRunUpdate(stored)),
15844 + ),
15845 Effect.forkScoped,
15846 );
15847
@@ -15871,6 +15899,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15899 Effect.catchCause((cause) =>
15900 Effect.logWarning("Failed to deliver deferred steering", { cause }),
15901 ),
15902 + (effect, stored) => reactInThreadOrder(stored.event.threadId, effect),
15903 ),
15904 ),
15905 Effect.forkScoped,
@@ -15900,18 +15929,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15929 ).pipe(
15930 Stream.filter(settlesChildWork),
15931 Stream.runForEach((stored) =>
15903 - Effect.gen(function* () {
15904 - const childThreadId = stored.event.threadId;
15905 - const parentThreadId = yield* appOwnedSubagentParentThreadId(childThreadId);
15906 - if (parentThreadId === undefined) return;
15907 - yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(childThreadId));
15908 - }).pipe(
15909 - Effect.catchCause((cause) =>
15910 - Effect.logWarning("Failed to finalize a delegated child after its work settled", {
15911 - threadId: stored.event.threadId,
15912 - sequence: stored.sequence,
15913 - cause,
15914 - }),
15932 + reactInThreadOrder(
15933 + stored.event.threadId,
15934 + Effect.gen(function* () {
15935 + const childThreadId = stored.event.threadId;
15936 + const parentThreadId = yield* appOwnedSubagentParentThreadId(childThreadId);
15937 + if (parentThreadId === undefined) return;
15938 + yield* threadDispatch.withLock(parentThreadId, finalizeAppOwnedSubagent(childThreadId));
15939 + }).pipe(
15940 + Effect.catchCause((cause) =>
15941 + Effect.logWarning("Failed to finalize a delegated child after its work settled", {
15942 + threadId: stored.event.threadId,
15943 + sequence: stored.sequence,
15944 + cause,
15945 + }),
15946 + ),
15947 ),
15948 ),
15949 ),
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+119
@@ -2048,6 +2048,125 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
2048 }),
2049 );
2050
2051 + // regression: #37412 one thread's busy lock held up every other thread's run-end reaction
2052 + it.effect("starts an idle thread's queued run while another thread's lock is busy", () =>
2053 + Effect.gen(function* () {
2054 + const orchestrator = yield* Orchestrator.OrchestratorV2;
2055 + const eventSink = yield* EventSink.EventSinkV2;
2056 + const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
2057 + const now = yield* DateTime.now;
2058 + const startRun = Effect.fnUntraced(function* (threadId: ThreadId) {
2059 + yield* orchestrator.dispatch({
2060 + type: "thread.create",
2061 + createdBy: "user",
2062 + creationSource: "web",
2063 + commandId: CommandId.make(`${threadId}:create`),
2064 + threadId,
2065 + projectId: ProjectId.make("runtime-run-end-isolation-project"),
2066 + title: threadId,
2067 + modelSelection,
2068 + runtimeMode: "full-access",
2069 + interactionMode: "default",
2070 + branch: null,
2071 + worktreePath: process.cwd(),
2072 + });
2073 + yield* orchestrator.dispatch({
2074 + type: "message.dispatch",
2075 + createdBy: "user",
2076 + creationSource: "web",
2077 + commandId: CommandId.make(`${threadId}:first`),
2078 + threadId,
2079 + messageId: MessageId.make(`${threadId}:first`),
2080 + text: "Start work.",
2081 + attachments: [],
2082 + dispatchMode: { type: "start_immediately" },
2083 + });
2084 + const run = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!;
2085 + yield* eventSink.write({
2086 + commandId: CommandId.make(`${threadId}:running`),
2087 + events: [
2088 + {
2089 + id: EventId.make(`${threadId}:running-event`),
2090 + type: "run.updated",
2091 + threadId,
2092 + runId: run.id,
2093 + occurredAt: now,
2094 + payload: { ...run, status: "running", startedAt: now },
2095 + },
2096 + ],
2097 + });
2098 + return run;
2099 + });
2100 + const endRun = (run: OrchestrationV2Run) =>
2101 + eventSink.write({
2102 + commandId: CommandId.make(`${run.threadId}:completed`),
2103 + events: [
2104 + {
2105 + id: EventId.make(`${run.threadId}:completed-event`),
2106 + type: "run.updated",
2107 + threadId: run.threadId,
2108 + runId: run.id,
2109 + occurredAt: now,
2110 + payload: { ...run, status: "completed", startedAt: now, completedAt: now },
2111 + },
2112 + ],
2113 + });
2114 + const busyThreadId = ThreadId.make("runtime-run-end-isolation-busy");
2115 + const idleThreadId = ThreadId.make("runtime-run-end-isolation-idle");
2116 + const busyRun = yield* startRun(busyThreadId);
2117 + const idleRun = yield* startRun(idleThreadId);
2118 + yield* orchestrator.dispatch({
2119 + type: "message.dispatch",
2120 + createdBy: "user",
2121 + creationSource: "web",
2122 + commandId: CommandId.make(`${idleThreadId}:queue`),
2123 + threadId: idleThreadId,
2124 + messageId: MessageId.make(`${idleThreadId}:queued`),
2125 + text: "Run this after the current turn.",
2126 + attachments: [],
2127 + dispatchMode: { type: "queue_after_active" },
2128 + });
2129 + const queuedRun = (yield* orchestrator.getThreadProjection(idleThreadId)).runs.find(
2130 + (run) => run.status === "queued",
2131 + );
2132 + assert.isDefined(queuedRun);
2133 +
2134 + const order: string[] = [];
2135 + const queuedRunStarted = yield* orchestrator.streamDomainEvents.pipe(
2136 + Stream.filter(
2137 + (event) =>
2138 + event.type === "run.updated" &&
2139 + event.payload.id === queuedRun.id &&
2140 + event.payload.status !== "queued",
2141 + ),
2142 + Stream.take(1),
2143 + Stream.runDrain,
2144 + Effect.andThen(Effect.sync(() => order.push("idle thread's queued run started"))),
2145 + Effect.forkScoped,
2146 + );
2147 + // Stands in for a long send or delegated bookkeeping holding the busy thread's lock.
2148 + const lockHeld = yield* Deferred.make<void>();
2149 + const busyLock = yield* executor
2150 + .withLock(
2151 + busyThreadId,
2152 + Deferred.succeed(lockHeld, undefined).pipe(
2153 + Effect.andThen(Effect.sleep("1 minute")),
2154 + Effect.andThen(Effect.sync(() => order.push("busy thread's lock released"))),
2155 + ),
2156 + )
2157 + .pipe(Effect.forkScoped);
2158 + yield* Deferred.await(lockHeld);
2159 +
2160 + yield* endRun(busyRun);
2161 + yield* endRun(idleRun);
2162 + yield* TestClock.adjust("1 minute");
2163 + yield* Fiber.join(busyLock);
2164 + yield* Fiber.join(queuedRunStarted);
2165 +
2166 + assert.deepEqual(order, ["idle thread's queued run started", "busy thread's lock released"]);
2167 + }),
2168 + );
2169 +
2170 const seedPendingQuestion = Effect.fnUntraced(function* (
2171 threadId: ThreadId,
2172 runId: RunId | null,