The best way to control Claude Code, Codex and more from any computer or phone
fix(server): a child's question no longer waits behind another coordinator's busy lock (#37470)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Sharad Patel committed
Nov 16, 2026 at 12:25 UTC
ad1b1365999c92c3fc7a054e7f3af84623f74fee
8 files changed
+265
-45
apps/server/src/orchestration-v2/DelegatedTaskQuestionReactor.ts
+41
-3
@@ -14,6 +14,7 @@ import type * as Scope from "effect/Scope";
14
import * as Stream from "effect/Stream";
15
16
import { reportLabel } from "@t3tools/provider-core/server/notification";
17
+import * as KeyedLock from "@t3tools/shared/KeyedLock";
18
19
import { forkParked } from "../serverActivation.ts";
20
import * as Orchestrator from "./Orchestrator.ts";
@@ -227,14 +228,51 @@ const make = Effect.gen(function* () {
228
),
229
);
230
230
- const start: DelegatedTaskQuestionReactor["Service"]["start"] = () =>
231
- forkParked(
232
- Stream.runForEach(engine.streamDomainEvents, handle).pipe(
231
+ // Every wake and cancel goes to the parent, so keying by it keeps a question's wake ahead of
232
+ // its task's cancel while a busy parent's lock delays only that parent's reactions.
233
+ const parentOf = (event: OrchestrationV2DomainEvent) => {
234
+ switch (event.type) {
235
+ case "run.updated":
236
+ return Effect.succeed(event.threadId);
237
+ case "subagent.updated":
238
+ return Effect.succeed(event.payload.threadId);
239
+ case "runtime-request.updated":
240
+ return event.payload.kind === "user_input"
241
+ ? projections
242
+ .getThread(event.threadId)
243
+ .pipe(Effect.map((child) => child.lineage.parentThreadId))
244
+ : Effect.succeed(null);
245
+ default:
246
+ return Effect.succeed(null);
247
+ }
248
+ };
249
+
250
+ const start: DelegatedTaskQuestionReactor["Service"]["start"] = Effect.fn(
251
+ "DelegatedTaskQuestionReactor.start",
252
+ )(function* () {
253
+ const parentReactions = yield* KeyedLock.makeRunner<ThreadId>();
254
+ yield* forkParked(
255
+ Stream.runForEach(engine.streamDomainEvents, (event) =>
256
+ parentOf(event).pipe(
257
+ Effect.flatMap((parentThreadId) =>
258
+ parentThreadId === null
259
+ ? Effect.void
260
+ : parentReactions.run(parentThreadId, handle(event)),
261
+ ),
262
+ Effect.catchCause((cause) =>
263
+ Effect.logWarning("finding the parent of a delegated task question failed", {
264
+ threadId: event.threadId,
265
+ cause,
266
+ }),
267
+ ),
268
+ ),
269
+ ).pipe(
270
Effect.catchCause((cause) =>
271
Effect.logWarning("delegated task question event stream failed", { cause }),
272
),
273
),
274
);
275
+ });
276
277
return { start, handle } satisfies DelegatedTaskQuestionReactor["Service"];
278
});
apps/server/src/orchestration-v2/Orchestrator.ts
+4
-26
@@ -93,9 +93,7 @@ import {
93
import * as Context from "effect/Context";
94
import * as DateTime from "effect/DateTime";
95
import * as Effect from "effect/Effect";
96
-import * as Deferred from "effect/Deferred";
96
import * as Exit from "effect/Exit";
98
-import * as FiberSet from "effect/FiberSet";
97
import * as FileSystem from "effect/FileSystem";
98
import * as Path from "effect/Path";
99
import * as Layer from "effect/Layer";
@@ -15795,27 +15793,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15793
15794
// One thread's reactions run in event order; a reaction waiting on one thread's lock must
15795
// not hold up another thread's.
15798
- const threadReactions = yield* FiberSet.make<void>();
15799
- const threadReactionTails = new Map<ThreadId, Deferred.Deferred<void>>();
15800
- const reactInThreadOrder = Effect.fnUntraced(function* <R>(
15801
- threadId: ThreadId,
15802
- reaction: Effect.Effect<void, never, R>,
15803
- ) {
15804
- const done = yield* Deferred.make<void>();
15805
- const previous = threadReactionTails.get(threadId);
15806
- threadReactionTails.set(threadId, done);
15807
- yield* (
15808
- previous === undefined ? reaction : Effect.andThen(Deferred.await(previous), reaction)
15809
- ).pipe(
15810
- Effect.ensuring(
15811
- Effect.suspend(() => {
15812
- if (threadReactionTails.get(threadId) === done) threadReactionTails.delete(threadId);
15813
- return Deferred.succeed(done, undefined);
15814
- }),
15815
- ),
15816
- FiberSet.run(threadReactions),
15817
- );
15818
- });
15796
+ const threadReactions = yield* KeyedLock.makeRunner<ThreadId>();
15797
15798
// Projections already contain historical terminals; replaying them delays startup queue promotion with database size.
15799
const terminalEventsAfterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie);
@@ -15840,7 +15818,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15818
stored.event.payload.status === "rolled_back"),
15819
),
15820
Stream.runForEach((stored) =>
15843
- reactInThreadOrder(stored.event.threadId, handleRunUpdate(stored)),
15821
+ threadReactions.run(stored.event.threadId, handleRunUpdate(stored)),
15822
),
15823
Effect.forkScoped,
15824
);
@@ -15899,7 +15877,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15877
Effect.catchCause((cause) =>
15878
Effect.logWarning("Failed to deliver deferred steering", { cause }),
15879
),
15902
- (effect, stored) => reactInThreadOrder(stored.event.threadId, effect),
15880
+ (effect, stored) => threadReactions.run(stored.event.threadId, effect),
15881
),
15882
),
15883
Effect.forkScoped,
@@ -15929,7 +15907,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15907
).pipe(
15908
Stream.filter(settlesChildWork),
15909
Stream.runForEach((stored) =>
15932
- reactInThreadOrder(
15910
+ threadReactions.run(
15911
stored.event.threadId,
15912
Effect.gen(function* () {
15913
const childThreadId = stored.event.threadId;
apps/server/src/orchestration-v2/ProviderContinuationService.ts
+2
-4
@@ -65,7 +65,7 @@ export const layer = Layer.effectDiscard(
65
const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
66
const threads = yield* ThreadManagementService.ThreadManagementService;
67
const retryAttempts = yield* Ref.make(new Map<string, number>());
68
- const threadLocks = yield* KeyedLock.make<ThreadId>();
68
+ const continuations = yield* KeyedLock.makeRunner<ThreadId>();
69
70
const clearRetryAttempt = (key: string) =>
71
Ref.update(retryAttempts, (current) => {
@@ -340,9 +340,7 @@ export const layer = Layer.effectDiscard(
340
}),
341
),
342
// A dispatch can wait on its parent's lock, so only its own thread's later requests queue behind it.
343
- (handled) => threadLocks.withLock(request.threadId, handled),
344
- // Starting immediately joins the thread's lock queue before the next take, keeping per-thread order.
345
- Effect.forkScoped({ startImmediately: true }),
343
+ (handled) => continuations.run(request.threadId, handled),
344
),
345
),
346
Effect.forever,
apps/server/src/orchestration-v2/PullRequestSyncReactor.ts
+28
-9
@@ -10,6 +10,7 @@ import {
10
type ThreadPullRequestStack,
11
} from "@t3tools/contracts";
12
import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
13
+import * as KeyedLock from "@t3tools/shared/KeyedLock";
14
import {
15
threadPullRequestKeyOf,
16
normalizeThreadPullRequestKey,
@@ -798,6 +799,19 @@ export const make = Effect.gen(function* () {
799
"PullRequestSyncReactor.start",
800
)(function* () {
801
const events = engine.streamDomainEvents;
802
+ // Linking and reopening dispatch under the thread's lock, so a busy thread delays only its
803
+ // own reactions. Refreshes queue with them, so each thread's reactions keep event order.
804
+ const threadReactions = yield* KeyedLock.makeRunner<ThreadId>();
805
+ const inThreadOrder = <E>(threadId: ThreadId, reaction: Effect.Effect<void, E>) =>
806
+ threadReactions.run(
807
+ threadId,
808
+ // The reactions log their own failures, so this sees only defects and shutdown.
809
+ Effect.catchCause(reaction, (cause) =>
810
+ Cause.hasInterruptsOnly(cause)
811
+ ? Effect.interrupt
812
+ : Effect.logWarning("pull request sync reaction failed", { threadId, cause }),
813
+ ),
814
+ );
815
const stateChanges = Stream.merge(
816
yield* pullRequests.subscribeStateChanges,
817
yield* git.subscribePullRequestStateChanges,
@@ -824,23 +838,28 @@ export const make = Effect.gen(function* () {
838
case "turn-item.updated": {
839
const item = event.payload;
840
if (item.type !== "command_execution") return Effect.void;
827
- const linked = Effect.andThen(
841
+ let reaction = Effect.andThen(
842
linkCreatedChangeRequests(event.threadId, item),
843
reopenWatchedChecks(event.threadId, item),
844
);
831
- if (!isChangeRequestCloseCommand(item.input)) return linked;
832
- const ended = closeCommands.get(event.threadId) ?? new Set<string>();
833
- closeCommands.set(event.threadId, ended);
834
- if (isOrchestrationV2WorkActive(item.status) || ended.has(item.id)) return linked;
835
- ended.add(item.id);
836
- return Effect.andThen(linked, refreshUnmergedLinks(event.threadId));
845
+ const active = isOrchestrationV2WorkActive(item.status);
846
+ if (isChangeRequestCloseCommand(item.input)) {
847
+ const ended = closeCommands.get(event.threadId) ?? new Set<string>();
848
+ closeCommands.set(event.threadId, ended);
849
+ if (!active && !ended.has(item.id)) {
850
+ ended.add(item.id);
851
+ reaction = Effect.andThen(reaction, refreshUnmergedLinks(event.threadId));
852
+ }
853
+ }
854
+ // Every reaction skips a running command, so only an ended one forks.
855
+ return active ? Effect.void : inThreadOrder(event.threadId, reaction);
856
}
857
// Settled threads are not swept, so read their links once to settle with fresh status.
858
case "thread.settled":
840
- return refreshUnmergedLinks(event.threadId);
859
+ return inThreadOrder(event.threadId, refreshUnmergedLinks(event.threadId));
860
case "run.updated":
861
return isTerminalRunStatus(event.payload.status) && closeCommands.delete(event.threadId)
843
- ? refreshUnmergedLinks(event.threadId)
862
+ ? inThreadOrder(event.threadId, refreshUnmergedLinks(event.threadId))
863
: Effect.void;
864
default:
865
return Effect.void;
apps/server/src/orchestration-v2/WorktreeBranchSync.ts
+9
-3
@@ -1,5 +1,6 @@
1
import { CommandId } from "@t3tools/contracts";
2
import { randomUuidV4 } from "@t3tools/provider-core/server/randomUuid";
3
+import * as KeyedLock from "@t3tools/shared/KeyedLock";
4
import * as Effect from "effect/Effect";
5
import * as FileSystem from "effect/FileSystem";
6
import * as Layer from "effect/Layer";
@@ -71,11 +72,16 @@ export const layer = Layer.effectDiscard(
72
Effect.gen(function* () {
73
const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster;
74
const reconcile = yield* makeReconcile;
75
+ // Each update waits on its thread's lock, so a busy thread delays only its own worktree.
76
+ const worktrees = yield* KeyedLock.makeRunner<string>();
77
yield* forkParked(
78
Stream.runForEach(broadcaster.streamBranchChanges, (input) =>
76
- reconcile(input).pipe(
77
- Effect.catchCause((cause) =>
78
- Effect.logWarning("Failed to reconcile worktree branch", { cwd: input.cwd, cause }),
79
+ worktrees.run(
80
+ input.cwd,
81
+ reconcile(input).pipe(
82
+ Effect.catchCause((cause) =>
83
+ Effect.logWarning("Failed to reconcile worktree branch", { cwd: input.cwd, cause }),
84
+ ),
85
),
86
),
87
),
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+123
@@ -99,6 +99,7 @@ import * as TerminalManager from "../terminal/Manager.ts";
99
import * as ResourceCleanupService from "./ResourceCleanupService.ts";
100
import * as ThreadLaunchService from "./ThreadLaunchService.ts";
101
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
102
+import * as DelegatedTaskQuestionReactor from "./DelegatedTaskQuestionReactor.ts";
103
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
104
105
const layerPlatformTest = Layer.merge(
@@ -2167,6 +2168,128 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
2168
}),
2169
);
2170
2171
+ // regression: one coordinator's busy lock held up every other coordinator's question wake
2172
+ it.effect("wakes an idle coordinator for its task's question while another's lock is busy", () =>
2173
+ Effect.gen(function* () {
2174
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
2175
+ const eventSink = yield* EventSink.EventSinkV2;
2176
+ const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
2177
+ const now = yield* DateTime.now;
2178
+ const delegate = Effect.fnUntraced(function* (parentThreadId: ThreadId) {
2179
+ yield* orchestrator.dispatch({
2180
+ type: "thread.create",
2181
+ createdBy: "user",
2182
+ creationSource: "web",
2183
+ commandId: CommandId.make(`${parentThreadId}:create`),
2184
+ threadId: parentThreadId,
2185
+ projectId: ProjectId.make("runtime-question-wake-project"),
2186
+ title: parentThreadId,
2187
+ modelSelection,
2188
+ runtimeMode: "full-access",
2189
+ interactionMode: "default",
2190
+ branch: null,
2191
+ worktreePath: process.cwd(),
2192
+ });
2193
+ yield* orchestrator.dispatch({
2194
+ type: "message.dispatch",
2195
+ createdBy: "user",
2196
+ creationSource: "web",
2197
+ commandId: CommandId.make(`${parentThreadId}:first`),
2198
+ threadId: parentThreadId,
2199
+ messageId: MessageId.make(`${parentThreadId}:first`),
2200
+ text: "Delegate the research.",
2201
+ attachments: [],
2202
+ dispatchMode: { type: "start_immediately" },
2203
+ });
2204
+ const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!;
2205
+ yield* orchestrator.dispatch({
2206
+ type: "delegated_task.request",
2207
+ createdBy: "agent",
2208
+ creationSource: "mcp",
2209
+ commandId: CommandId.make(`${parentThreadId}:delegate`),
2210
+ parentThreadId,
2211
+ parentRunId: parentRun.id,
2212
+ parentNodeId: parentRun.rootNodeId!,
2213
+ task: "Research the sources.",
2214
+ modelSelection,
2215
+ runtimeMode: "full-access",
2216
+ interactionMode: "default",
2217
+ completionWake: "settled_only",
2218
+ });
2219
+ const childThreadId = (yield* orchestrator.getThreadProjection(parentThreadId))
2220
+ .subagents[0]!.childThreadId!;
2221
+ const childRun = (yield* orchestrator.getThreadProjection(childThreadId)).runs[0]!;
2222
+ const requestId = RuntimeRequestId.make(`${childThreadId}:question`);
2223
+ return {
2224
+ wakeMessageId: MessageId.make(
2225
+ `message:delegated-task-question:${parentThreadId}:${requestId}`,
2226
+ ),
2227
+ ask: eventSink.write({
2228
+ events: [
2229
+ {
2230
+ id: EventId.make(`${childThreadId}:question-event`),
2231
+ type: "runtime-request.updated",
2232
+ threadId: childThreadId,
2233
+ runId: childRun.id,
2234
+ occurredAt: now,
2235
+ payload: {
2236
+ id: requestId,
2237
+ nodeId: childRun.rootNodeId!,
2238
+ providerTurnId: null,
2239
+ nativeRequestRef: null,
2240
+ kind: "user_input",
2241
+ status: "pending",
2242
+ responseCapability: { type: "message" },
2243
+ createdAt: now,
2244
+ resolvedAt: null,
2245
+ },
2246
+ },
2247
+ ],
2248
+ }),
2249
+ };
2250
+ });
2251
+ // Started as server.ts starts it, so its wakes come from its own event stream.
2252
+ yield* DelegatedTaskQuestionReactor.DelegatedTaskQuestionReactor.pipe(
2253
+ Effect.flatMap((reactor) => reactor.start()),
2254
+ Effect.provide(DelegatedTaskQuestionReactor.layer),
2255
+ );
2256
+ const busyThreadId = ThreadId.make("runtime-question-wake-busy");
2257
+ const busy = yield* delegate(busyThreadId);
2258
+ const idle = yield* delegate(ThreadId.make("runtime-question-wake-idle"));
2259
+
2260
+ const order: string[] = [];
2261
+ const idleWoken = yield* orchestrator.streamDomainEvents.pipe(
2262
+ Stream.filter(
2263
+ (event) => event.type === "message.updated" && event.payload.id === idle.wakeMessageId,
2264
+ ),
2265
+ Stream.take(1),
2266
+ Stream.runDrain,
2267
+ Effect.andThen(Effect.sync(() => order.push("idle coordinator woken"))),
2268
+ Effect.forkScoped,
2269
+ );
2270
+ // Stands in for a long send or delegated bookkeeping holding the busy coordinator's lock.
2271
+ const lockHeld = yield* Deferred.make<void>();
2272
+ const busyLock = yield* executor
2273
+ .withLock(
2274
+ busyThreadId,
2275
+ Deferred.succeed(lockHeld, undefined).pipe(
2276
+ Effect.andThen(Effect.sleep("1 minute")),
2277
+ Effect.andThen(Effect.sync(() => order.push("busy coordinator's lock released"))),
2278
+ ),
2279
+ )
2280
+ .pipe(Effect.forkScoped);
2281
+ yield* Deferred.await(lockHeld);
2282
+
2283
+ yield* busy.ask;
2284
+ yield* idle.ask;
2285
+ yield* TestClock.adjust("1 minute");
2286
+ yield* Fiber.join(busyLock);
2287
+ yield* Fiber.join(idleWoken);
2288
+
2289
+ assert.deepEqual(order, ["idle coordinator woken", "busy coordinator's lock released"]);
2290
+ }),
2291
+ );
2292
+
2293
const seedPendingQuestion = Effect.fnUntraced(function* (
2294
threadId: ThreadId,
2295
runId: RunId | null,
packages/shared/src/KeyedLock.test.ts
+28
@@ -94,6 +94,34 @@ describe("KeyedLock", () => {
94
}),
95
);
96
97
+ it.effect("runner keeps each key's effects in run order and never holds up another key", () =>
98
+ Effect.gen(function* () {
99
+ const runner = yield* KeyedLock.makeRunner<string>();
100
+ const events = yield* Ref.make<ReadonlyArray<string>>([]);
101
+ const record = (event: string) => Ref.update(events, (current) => [...current, event]);
102
+ const release = yield* Deferred.make<void>();
103
+ const otherRan = yield* Deferred.make<void>();
104
+ const busyRan = yield* Deferred.make<void>();
105
+
106
+ yield* runner.run("busy", Effect.andThen(Deferred.await(release), record("busy:1")));
107
+ yield* runner.run("busy", record("busy:2"));
108
+ yield* runner.run(
109
+ "busy",
110
+ Effect.andThen(record("busy:3"), Deferred.succeed(busyRan, undefined)).pipe(Effect.asVoid),
111
+ );
112
+ yield* runner.run(
113
+ "other",
114
+ Effect.andThen(record("other"), Deferred.succeed(otherRan, undefined)).pipe(Effect.asVoid),
115
+ );
116
+ yield* Deferred.await(otherRan);
117
+ assert.deepStrictEqual(yield* Ref.get(events), ["other"]);
118
+
119
+ yield* Deferred.succeed(release, undefined);
120
+ yield* Deferred.await(busyRan);
121
+ assert.deepStrictEqual(yield* Ref.get(events), ["other", "busy:1", "busy:2", "busy:3"]);
122
+ }),
123
+ );
124
+
125
it.effect("runs the effect in the caller's scope", () =>
126
Effect.gen(function* () {
127
const lock = yield* KeyedLock.make<string>();
packages/shared/src/KeyedLock.ts
+30
@@ -1,5 +1,7 @@
1
import * as Effect from "effect/Effect";
2
+import * as FiberSet from "effect/FiberSet";
3
import * as Ref from "effect/Ref";
4
+import type * as Scope from "effect/Scope";
5
import * as Semaphore from "effect/Semaphore";
6
7
export interface KeyedLock<Key> {
@@ -54,3 +56,31 @@ export const make = Effect.fn("KeyedLock.make")(function* <Key>(): Effect.fn.Ret
56
activeKeys: Effect.map(Ref.get(locks), (current) => Array.from(current.keys())),
57
} satisfies KeyedLock<Key>;
58
});
59
+
60
+export interface KeyedRunner<Key> {
61
+ /** Forks `effect` to run once every effect passed earlier for `key` has finished. */
62
+ readonly run: <R>(
63
+ key: Key,
64
+ effect: Effect.Effect<void, never, R>,
65
+ ) => Effect.Effect<void, never, R>;
66
+}
67
+
68
+/**
69
+ * For a stream consumer whose handler waits on a per-key lock: a busy key delays only its own
70
+ * later effects. The forked fiber starts immediately, so it queues on the key's FIFO semaphore
71
+ * before `run` returns; that keeps each key's effects in `run` order.
72
+ */
73
+export const makeRunner = Effect.fn("KeyedLock.makeRunner")(function* <Key>(): Effect.fn.Return<
74
+ KeyedRunner<Key>,
75
+ never,
76
+ Scope.Scope
77
+> {
78
+ const lock = yield* make<Key>();
79
+ const fibers = yield* FiberSet.make<void, never>();
80
+ return {
81
+ run: (key, effect) =>
82
+ FiberSet.run(fibers, lock.withLock(key, effect), { startImmediately: true }).pipe(
83
+ Effect.asVoid,
84
+ ),
85
+ } satisfies KeyedRunner<Key>;
86
+});