@devries-cameron20 / t3code-1 / commits / ad1b136599
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(server): a child's question no longer waits behind another coordinator's busy lock (#37470)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Sharad Patel committed Nov 16, 2026 at 12:25 UTC ad1b1365999c92c3fc7a054e7f3af84623f74fee
8 files changed +265 -45
apps/server/src/orchestration-v2/DelegatedTaskQuestionReactor.ts
+41 -3
@@ -14,6 +14,7 @@ import type * as Scope from "effect/Scope";
14 import * as Stream from "effect/Stream";
15
16 import { reportLabel } from "@t3tools/provider-core/server/notification";
17 +import * as KeyedLock from "@t3tools/shared/KeyedLock";
18
19 import { forkParked } from "../serverActivation.ts";
20 import * as Orchestrator from "./Orchestrator.ts";
@@ -227,14 +228,51 @@ const make = Effect.gen(function* () {
228 ),
229 );
230
230 - const start: DelegatedTaskQuestionReactor["Service"]["start"] = () =>
231 - forkParked(
232 - Stream.runForEach(engine.streamDomainEvents, handle).pipe(
231 + // Every wake and cancel goes to the parent, so keying by it keeps a question's wake ahead of
232 + // its task's cancel while a busy parent's lock delays only that parent's reactions.
233 + const parentOf = (event: OrchestrationV2DomainEvent) => {
234 + switch (event.type) {
235 + case "run.updated":
236 + return Effect.succeed(event.threadId);
237 + case "subagent.updated":
238 + return Effect.succeed(event.payload.threadId);
239 + case "runtime-request.updated":
240 + return event.payload.kind === "user_input"
241 + ? projections
242 + .getThread(event.threadId)
243 + .pipe(Effect.map((child) => child.lineage.parentThreadId))
244 + : Effect.succeed(null);
245 + default:
246 + return Effect.succeed(null);
247 + }
248 + };
249 +
250 + const start: DelegatedTaskQuestionReactor["Service"]["start"] = Effect.fn(
251 + "DelegatedTaskQuestionReactor.start",
252 + )(function* () {
253 + const parentReactions = yield* KeyedLock.makeRunner<ThreadId>();
254 + yield* forkParked(
255 + Stream.runForEach(engine.streamDomainEvents, (event) =>
256 + parentOf(event).pipe(
257 + Effect.flatMap((parentThreadId) =>
258 + parentThreadId === null
259 + ? Effect.void
260 + : parentReactions.run(parentThreadId, handle(event)),
261 + ),
262 + Effect.catchCause((cause) =>
263 + Effect.logWarning("finding the parent of a delegated task question failed", {
264 + threadId: event.threadId,
265 + cause,
266 + }),
267 + ),
268 + ),
269 + ).pipe(
270 Effect.catchCause((cause) =>
271 Effect.logWarning("delegated task question event stream failed", { cause }),
272 ),
273 ),
274 );
275 + });
276
277 return { start, handle } satisfies DelegatedTaskQuestionReactor["Service"];
278 });
apps/server/src/orchestration-v2/Orchestrator.ts
+4 -26
@@ -93,9 +93,7 @@ import {
93 import * as Context from "effect/Context";
94 import * as DateTime from "effect/DateTime";
95 import * as Effect from "effect/Effect";
96 -import * as Deferred from "effect/Deferred";
96 import * as Exit from "effect/Exit";
98 -import * as FiberSet from "effect/FiberSet";
97 import * as FileSystem from "effect/FileSystem";
98 import * as Path from "effect/Path";
99 import * as Layer from "effect/Layer";
@@ -15795,27 +15793,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15793
15794 // One thread's reactions run in event order; a reaction waiting on one thread's lock must
15795 // not hold up another thread's.
15798 - const threadReactions = yield* FiberSet.make<void>();
15799 - const threadReactionTails = new Map<ThreadId, Deferred.Deferred<void>>();
15800 - const reactInThreadOrder = Effect.fnUntraced(function* <R>(
15801 - threadId: ThreadId,
15802 - reaction: Effect.Effect<void, never, R>,
15803 - ) {
15804 - const done = yield* Deferred.make<void>();
15805 - const previous = threadReactionTails.get(threadId);
15806 - threadReactionTails.set(threadId, done);
15807 - yield* (
15808 - previous === undefined ? reaction : Effect.andThen(Deferred.await(previous), reaction)
15809 - ).pipe(
15810 - Effect.ensuring(
15811 - Effect.suspend(() => {
15812 - if (threadReactionTails.get(threadId) === done) threadReactionTails.delete(threadId);
15813 - return Deferred.succeed(done, undefined);
15814 - }),
15815 - ),
15816 - FiberSet.run(threadReactions),
15817 - );
15818 - });
15796 + const threadReactions = yield* KeyedLock.makeRunner<ThreadId>();
15797
15798 // Projections already contain historical terminals; replaying them delays startup queue promotion with database size.
15799 const terminalEventsAfterSequence = yield* eventSink.latestSequence().pipe(Effect.orDie);
@@ -15840,7 +15818,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15818 stored.event.payload.status === "rolled_back"),
15819 ),
15820 Stream.runForEach((stored) =>
15843 - reactInThreadOrder(stored.event.threadId, handleRunUpdate(stored)),
15821 + threadReactions.run(stored.event.threadId, handleRunUpdate(stored)),
15822 ),
15823 Effect.forkScoped,
15824 );
@@ -15899,7 +15877,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15877 Effect.catchCause((cause) =>
15878 Effect.logWarning("Failed to deliver deferred steering", { cause }),
15879 ),
15902 - (effect, stored) => reactInThreadOrder(stored.event.threadId, effect),
15880 + (effect, stored) => threadReactions.run(stored.event.threadId, effect),
15881 ),
15882 ),
15883 Effect.forkScoped,
@@ -15929,7 +15907,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15907 ).pipe(
15908 Stream.filter(settlesChildWork),
15909 Stream.runForEach((stored) =>
15932 - reactInThreadOrder(
15910 + threadReactions.run(
15911 stored.event.threadId,
15912 Effect.gen(function* () {
15913 const childThreadId = stored.event.threadId;
apps/server/src/orchestration-v2/ProviderContinuationService.ts
+2 -4
@@ -65,7 +65,7 @@ export const layer = Layer.effectDiscard(
65 const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests;
66 const threads = yield* ThreadManagementService.ThreadManagementService;
67 const retryAttempts = yield* Ref.make(new Map<string, number>());
68 - const threadLocks = yield* KeyedLock.make<ThreadId>();
68 + const continuations = yield* KeyedLock.makeRunner<ThreadId>();
69
70 const clearRetryAttempt = (key: string) =>
71 Ref.update(retryAttempts, (current) => {
@@ -340,9 +340,7 @@ export const layer = Layer.effectDiscard(
340 }),
341 ),
342 // A dispatch can wait on its parent's lock, so only its own thread's later requests queue behind it.
343 - (handled) => threadLocks.withLock(request.threadId, handled),
344 - // Starting immediately joins the thread's lock queue before the next take, keeping per-thread order.
345 - Effect.forkScoped({ startImmediately: true }),
343 + (handled) => continuations.run(request.threadId, handled),
344 ),
345 ),
346 Effect.forever,
apps/server/src/orchestration-v2/PullRequestSyncReactor.ts
+28 -9
@@ -10,6 +10,7 @@ import {
10 type ThreadPullRequestStack,
11 } from "@t3tools/contracts";
12 import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
13 +import * as KeyedLock from "@t3tools/shared/KeyedLock";
14 import {
15 threadPullRequestKeyOf,
16 normalizeThreadPullRequestKey,
@@ -798,6 +799,19 @@ export const make = Effect.gen(function* () {
799 "PullRequestSyncReactor.start",
800 )(function* () {
801 const events = engine.streamDomainEvents;
802 + // Linking and reopening dispatch under the thread's lock, so a busy thread delays only its
803 + // own reactions. Refreshes queue with them, so each thread's reactions keep event order.
804 + const threadReactions = yield* KeyedLock.makeRunner<ThreadId>();
805 + const inThreadOrder = <E>(threadId: ThreadId, reaction: Effect.Effect<void, E>) =>
806 + threadReactions.run(
807 + threadId,
808 + // The reactions log their own failures, so this sees only defects and shutdown.
809 + Effect.catchCause(reaction, (cause) =>
810 + Cause.hasInterruptsOnly(cause)
811 + ? Effect.interrupt
812 + : Effect.logWarning("pull request sync reaction failed", { threadId, cause }),
813 + ),
814 + );
815 const stateChanges = Stream.merge(
816 yield* pullRequests.subscribeStateChanges,
817 yield* git.subscribePullRequestStateChanges,
@@ -824,23 +838,28 @@ export const make = Effect.gen(function* () {
838 case "turn-item.updated": {
839 const item = event.payload;
840 if (item.type !== "command_execution") return Effect.void;
827 - const linked = Effect.andThen(
841 + let reaction = Effect.andThen(
842 linkCreatedChangeRequests(event.threadId, item),
843 reopenWatchedChecks(event.threadId, item),
844 );
831 - if (!isChangeRequestCloseCommand(item.input)) return linked;
832 - const ended = closeCommands.get(event.threadId) ?? new Set<string>();
833 - closeCommands.set(event.threadId, ended);
834 - if (isOrchestrationV2WorkActive(item.status) || ended.has(item.id)) return linked;
835 - ended.add(item.id);
836 - return Effect.andThen(linked, refreshUnmergedLinks(event.threadId));
845 + const active = isOrchestrationV2WorkActive(item.status);
846 + if (isChangeRequestCloseCommand(item.input)) {
847 + const ended = closeCommands.get(event.threadId) ?? new Set<string>();
848 + closeCommands.set(event.threadId, ended);
849 + if (!active && !ended.has(item.id)) {
850 + ended.add(item.id);
851 + reaction = Effect.andThen(reaction, refreshUnmergedLinks(event.threadId));
852 + }
853 + }
854 + // Every reaction skips a running command, so only an ended one forks.
855 + return active ? Effect.void : inThreadOrder(event.threadId, reaction);
856 }
857 // Settled threads are not swept, so read their links once to settle with fresh status.
858 case "thread.settled":
840 - return refreshUnmergedLinks(event.threadId);
859 + return inThreadOrder(event.threadId, refreshUnmergedLinks(event.threadId));
860 case "run.updated":
861 return isTerminalRunStatus(event.payload.status) && closeCommands.delete(event.threadId)
843 - ? refreshUnmergedLinks(event.threadId)
862 + ? inThreadOrder(event.threadId, refreshUnmergedLinks(event.threadId))
863 : Effect.void;
864 default:
865 return Effect.void;
apps/server/src/orchestration-v2/WorktreeBranchSync.ts
+9 -3
@@ -1,5 +1,6 @@
1 import { CommandId } from "@t3tools/contracts";
2 import { randomUuidV4 } from "@t3tools/provider-core/server/randomUuid";
3 +import * as KeyedLock from "@t3tools/shared/KeyedLock";
4 import * as Effect from "effect/Effect";
5 import * as FileSystem from "effect/FileSystem";
6 import * as Layer from "effect/Layer";
@@ -71,11 +72,16 @@ export const layer = Layer.effectDiscard(
72 Effect.gen(function* () {
73 const broadcaster = yield* VcsStatusBroadcaster.VcsStatusBroadcaster;
74 const reconcile = yield* makeReconcile;
75 + // Each update waits on its thread's lock, so a busy thread delays only its own worktree.
76 + const worktrees = yield* KeyedLock.makeRunner<string>();
77 yield* forkParked(
78 Stream.runForEach(broadcaster.streamBranchChanges, (input) =>
76 - reconcile(input).pipe(
77 - Effect.catchCause((cause) =>
78 - Effect.logWarning("Failed to reconcile worktree branch", { cwd: input.cwd, cause }),
79 + worktrees.run(
80 + input.cwd,
81 + reconcile(input).pipe(
82 + Effect.catchCause((cause) =>
83 + Effect.logWarning("Failed to reconcile worktree branch", { cwd: input.cwd, cause }),
84 + ),
85 ),
86 ),
87 ),
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+123
@@ -99,6 +99,7 @@ import * as TerminalManager from "../terminal/Manager.ts";
99 import * as ResourceCleanupService from "./ResourceCleanupService.ts";
100 import * as ThreadLaunchService from "./ThreadLaunchService.ts";
101 import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
102 +import * as DelegatedTaskQuestionReactor from "./DelegatedTaskQuestionReactor.ts";
103 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
104
105 const layerPlatformTest = Layer.merge(
@@ -2167,6 +2168,128 @@ it.layer(layerTest)("OrchestrationV2LayerLive", (it) => {
2168 }),
2169 );
2170
2171 + // regression: one coordinator's busy lock held up every other coordinator's question wake
2172 + it.effect("wakes an idle coordinator for its task's question while another's lock is busy", () =>
2173 + Effect.gen(function* () {
2174 + const orchestrator = yield* Orchestrator.OrchestratorV2;
2175 + const eventSink = yield* EventSink.EventSinkV2;
2176 + const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
2177 + const now = yield* DateTime.now;
2178 + const delegate = Effect.fnUntraced(function* (parentThreadId: ThreadId) {
2179 + yield* orchestrator.dispatch({
2180 + type: "thread.create",
2181 + createdBy: "user",
2182 + creationSource: "web",
2183 + commandId: CommandId.make(`${parentThreadId}:create`),
2184 + threadId: parentThreadId,
2185 + projectId: ProjectId.make("runtime-question-wake-project"),
2186 + title: parentThreadId,
2187 + modelSelection,
2188 + runtimeMode: "full-access",
2189 + interactionMode: "default",
2190 + branch: null,
2191 + worktreePath: process.cwd(),
2192 + });
2193 + yield* orchestrator.dispatch({
2194 + type: "message.dispatch",
2195 + createdBy: "user",
2196 + creationSource: "web",
2197 + commandId: CommandId.make(`${parentThreadId}:first`),
2198 + threadId: parentThreadId,
2199 + messageId: MessageId.make(`${parentThreadId}:first`),
2200 + text: "Delegate the research.",
2201 + attachments: [],
2202 + dispatchMode: { type: "start_immediately" },
2203 + });
2204 + const parentRun = (yield* orchestrator.getThreadProjection(parentThreadId)).runs[0]!;
2205 + yield* orchestrator.dispatch({
2206 + type: "delegated_task.request",
2207 + createdBy: "agent",
2208 + creationSource: "mcp",
2209 + commandId: CommandId.make(`${parentThreadId}:delegate`),
2210 + parentThreadId,
2211 + parentRunId: parentRun.id,
2212 + parentNodeId: parentRun.rootNodeId!,
2213 + task: "Research the sources.",
2214 + modelSelection,
2215 + runtimeMode: "full-access",
2216 + interactionMode: "default",
2217 + completionWake: "settled_only",
2218 + });
2219 + const childThreadId = (yield* orchestrator.getThreadProjection(parentThreadId))
2220 + .subagents[0]!.childThreadId!;
2221 + const childRun = (yield* orchestrator.getThreadProjection(childThreadId)).runs[0]!;
2222 + const requestId = RuntimeRequestId.make(`${childThreadId}:question`);
2223 + return {
2224 + wakeMessageId: MessageId.make(
2225 + `message:delegated-task-question:${parentThreadId}:${requestId}`,
2226 + ),
2227 + ask: eventSink.write({
2228 + events: [
2229 + {
2230 + id: EventId.make(`${childThreadId}:question-event`),
2231 + type: "runtime-request.updated",
2232 + threadId: childThreadId,
2233 + runId: childRun.id,
2234 + occurredAt: now,
2235 + payload: {
2236 + id: requestId,
2237 + nodeId: childRun.rootNodeId!,
2238 + providerTurnId: null,
2239 + nativeRequestRef: null,
2240 + kind: "user_input",
2241 + status: "pending",
2242 + responseCapability: { type: "message" },
2243 + createdAt: now,
2244 + resolvedAt: null,
2245 + },
2246 + },
2247 + ],
2248 + }),
2249 + };
2250 + });
2251 + // Started as server.ts starts it, so its wakes come from its own event stream.
2252 + yield* DelegatedTaskQuestionReactor.DelegatedTaskQuestionReactor.pipe(
2253 + Effect.flatMap((reactor) => reactor.start()),
2254 + Effect.provide(DelegatedTaskQuestionReactor.layer),
2255 + );
2256 + const busyThreadId = ThreadId.make("runtime-question-wake-busy");
2257 + const busy = yield* delegate(busyThreadId);
2258 + const idle = yield* delegate(ThreadId.make("runtime-question-wake-idle"));
2259 +
2260 + const order: string[] = [];
2261 + const idleWoken = yield* orchestrator.streamDomainEvents.pipe(
2262 + Stream.filter(
2263 + (event) => event.type === "message.updated" && event.payload.id === idle.wakeMessageId,
2264 + ),
2265 + Stream.take(1),
2266 + Stream.runDrain,
2267 + Effect.andThen(Effect.sync(() => order.push("idle coordinator woken"))),
2268 + Effect.forkScoped,
2269 + );
2270 + // Stands in for a long send or delegated bookkeeping holding the busy coordinator's lock.
2271 + const lockHeld = yield* Deferred.make<void>();
2272 + const busyLock = yield* executor
2273 + .withLock(
2274 + busyThreadId,
2275 + Deferred.succeed(lockHeld, undefined).pipe(
2276 + Effect.andThen(Effect.sleep("1 minute")),
2277 + Effect.andThen(Effect.sync(() => order.push("busy coordinator's lock released"))),
2278 + ),
2279 + )
2280 + .pipe(Effect.forkScoped);
2281 + yield* Deferred.await(lockHeld);
2282 +
2283 + yield* busy.ask;
2284 + yield* idle.ask;
2285 + yield* TestClock.adjust("1 minute");
2286 + yield* Fiber.join(busyLock);
2287 + yield* Fiber.join(idleWoken);
2288 +
2289 + assert.deepEqual(order, ["idle coordinator woken", "busy coordinator's lock released"]);
2290 + }),
2291 + );
2292 +
2293 const seedPendingQuestion = Effect.fnUntraced(function* (
2294 threadId: ThreadId,
2295 runId: RunId | null,
packages/shared/src/KeyedLock.test.ts
+28
@@ -94,6 +94,34 @@ describe("KeyedLock", () => {
94 }),
95 );
96
97 + it.effect("runner keeps each key's effects in run order and never holds up another key", () =>
98 + Effect.gen(function* () {
99 + const runner = yield* KeyedLock.makeRunner<string>();
100 + const events = yield* Ref.make<ReadonlyArray<string>>([]);
101 + const record = (event: string) => Ref.update(events, (current) => [...current, event]);
102 + const release = yield* Deferred.make<void>();
103 + const otherRan = yield* Deferred.make<void>();
104 + const busyRan = yield* Deferred.make<void>();
105 +
106 + yield* runner.run("busy", Effect.andThen(Deferred.await(release), record("busy:1")));
107 + yield* runner.run("busy", record("busy:2"));
108 + yield* runner.run(
109 + "busy",
110 + Effect.andThen(record("busy:3"), Deferred.succeed(busyRan, undefined)).pipe(Effect.asVoid),
111 + );
112 + yield* runner.run(
113 + "other",
114 + Effect.andThen(record("other"), Deferred.succeed(otherRan, undefined)).pipe(Effect.asVoid),
115 + );
116 + yield* Deferred.await(otherRan);
117 + assert.deepStrictEqual(yield* Ref.get(events), ["other"]);
118 +
119 + yield* Deferred.succeed(release, undefined);
120 + yield* Deferred.await(busyRan);
121 + assert.deepStrictEqual(yield* Ref.get(events), ["other", "busy:1", "busy:2", "busy:3"]);
122 + }),
123 + );
124 +
125 it.effect("runs the effect in the caller's scope", () =>
126 Effect.gen(function* () {
127 const lock = yield* KeyedLock.make<string>();
packages/shared/src/KeyedLock.ts
+30
@@ -1,5 +1,7 @@
1 import * as Effect from "effect/Effect";
2 +import * as FiberSet from "effect/FiberSet";
3 import * as Ref from "effect/Ref";
4 +import type * as Scope from "effect/Scope";
5 import * as Semaphore from "effect/Semaphore";
6
7 export interface KeyedLock<Key> {
@@ -54,3 +56,31 @@ export const make = Effect.fn("KeyedLock.make")(function* <Key>(): Effect.fn.Ret
56 activeKeys: Effect.map(Ref.get(locks), (current) => Array.from(current.keys())),
57 } satisfies KeyedLock<Key>;
58 });
59 +
60 +export interface KeyedRunner<Key> {
61 + /** Forks `effect` to run once every effect passed earlier for `key` has finished. */
62 + readonly run: <R>(
63 + key: Key,
64 + effect: Effect.Effect<void, never, R>,
65 + ) => Effect.Effect<void, never, R>;
66 +}
67 +
68 +/**
69 + * For a stream consumer whose handler waits on a per-key lock: a busy key delays only its own
70 + * later effects. The forked fiber starts immediately, so it queues on the key's FIFO semaphore
71 + * before `run` returns; that keeps each key's effects in `run` order.
72 + */
73 +export const makeRunner = Effect.fn("KeyedLock.makeRunner")(function* <Key>(): Effect.fn.Return<
74 + KeyedRunner<Key>,
75 + never,
76 + Scope.Scope
77 +> {
78 + const lock = yield* make<Key>();
79 + const fibers = yield* FiberSet.make<void, never>();
80 + return {
81 + run: (key, effect) =>
82 + FiberSet.run(fibers, lock.withLock(key, effect), { startImmediately: true }).pipe(
83 + Effect.asVoid,
84 + ),
85 + } satisfies KeyedRunner<Key>;
86 +});