The best way to control Claude Code, Codex and more from any computer or phone
fix(server): a send to an idle thread with a queue now waits behind it (#33857)
Julius Marminge committed
Nov 14, 2026 at 19:07 UTC
7a72fe301ca73ae7eb2e2dbe82354095eb358685
2 files changed
+156
-3
apps/server/src/orchestration-v2/Orchestrator.ts
+24
-3
@@ -2405,6 +2405,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2405
readonly failedRunId?: RunId;
2406
readonly endedRunId?: RunId;
2407
readonly queuedRunId?: RunId;
2408
+ /** Treat the latest executed run as the one that just ended, as its run-end handler would. */
2409
+ readonly latestRunEnded?: boolean;
2410
readonly userStarted?: boolean;
2411
},
2412
) {
@@ -2487,12 +2489,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2489
}
2490
// Failed worktree creation would run queued work in the project checkout; wait for the user.
2491
const failedRun = latestExecutedRun(projection.runs);
2492
+ const endedRunId = options?.latestRunEnded === true ? failedRun?.id : options?.endedRunId;
2493
+ const failedRunId =
2494
+ options?.latestRunEnded === true && failedRun?.status === "failed"
2495
+ ? failedRun.id
2496
+ : options?.failedRunId;
2497
const failureClass =
2491
- failedRun?.id === options?.failedRunId
2498
+ failedRun && failedRunId !== undefined && failedRun.id === failedRunId
2499
? latestRootProviderFailure(failedRun, projection.turnItems)?.class
2500
: undefined;
2501
const worktreeMissing =
2495
- failedRun?.id === options?.endedRunId &&
2502
+ endedRunId !== undefined &&
2503
+ failedRun?.id === endedRunId &&
2504
failedRun?.workspacePreparation?.type === "worktree" &&
2505
projection.thread.worktreePath === null;
2506
const settings = yield* serverSettings.getSettings.pipe(Effect.orDie);
@@ -2938,7 +2946,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2946
updatedAt: now,
2947
};
2948
// Resume can clear the hold before the stopped run ends; its Stop marker still makes the drain explicit.
2941
- const endedRunId = options?.endedRunId;
2949
const queueDelivery =
2950
endedRunId === undefined ||
2951
queuedRun.queueHeld === true ||
@@ -15390,6 +15397,20 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
15397
if (shell !== null) yield* refuseAboveDispatchModeLimit(command, threadId, shell);
15398
}
15399
15400
+ // The run-end release can lag behind a new send; start the queue first so the send waits behind it.
15401
+ // A restart continuation resumes the ended run, so it must not be overtaken by the queue.
15402
+ if (
15403
+ command.type === "message.dispatch" &&
15404
+ command.restartContinuationOfRunId === undefined &&
15405
+ (command.dispatchMode.type === "start_immediately" ||
15406
+ command.dispatchMode.type === "queue_after_active" ||
15407
+ command.dispatchMode.type === "defer_start")
15408
+ ) {
15409
+ yield* mapDispatchError(command)(
15410
+ startNextQueuedRun(command.threadId, { latestRunEnded: true }),
15411
+ );
15412
+ }
15413
+
15414
const plan = yield* dispatchOnce(command, sideChatIds).pipe(
15415
Effect.tap((planned) => refuseAboveRuntimeModeCeiling(command, planned.events)),
15416
Effect.filterOrFail(
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+132
@@ -9217,6 +9217,138 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
9217
}),
9218
);
9219
9220
+ it.effect.each([false, true])(
9221
+ "starts the queue before a send that arrives as the active run ends (worktree failed: %s)",
9222
+ (worktreeFailed) =>
9223
+ Effect.gen(function* () {
9224
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
9225
+ const eventSink = yield* EventSink.EventSinkV2;
9226
+ const executor = yield* ThreadCommandExecutor.ThreadCommandExecutor;
9227
+ const threadId = ThreadId.make(`runtime-layer-send-after-run-end-${worktreeFailed}`);
9228
+ const messageId = (key: string) => MessageId.make(`${threadId}:${key}`);
9229
+ yield* orchestrator.dispatch({
9230
+ type: "thread.create",
9231
+ createdBy: "user",
9232
+ creationSource: "web",
9233
+ commandId: CommandId.make(`${threadId}:create`),
9234
+ threadId,
9235
+ projectId: ProjectId.make(`${threadId}:project`),
9236
+ title: "Send after run end",
9237
+ modelSelection,
9238
+ runtimeMode: "full-access",
9239
+ interactionMode: "default",
9240
+ branch: null,
9241
+ worktreePath: process.cwd(),
9242
+ });
9243
+ const dispatch = (key: string, start: boolean) =>
9244
+ orchestrator.dispatch({
9245
+ type: "message.dispatch",
9246
+ createdBy: "agent",
9247
+ creationSource: "server",
9248
+ commandId: CommandId.make(`${threadId}:message:${key}`),
9249
+ threadId,
9250
+ messageId: messageId(key),
9251
+ text: key,
9252
+ attachments: [],
9253
+ modelSelection,
9254
+ dispatchMode: { type: start ? "start_immediately" : "queue_after_active" },
9255
+ });
9256
+ const runFor = (key: string) =>
9257
+ Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) => {
9258
+ const run = runs.find((candidate) => candidate.userMessageId === messageId(key));
9259
+ assert.isDefined(run);
9260
+ return run;
9261
+ });
9262
+ yield* dispatch("active", true);
9263
+ yield* dispatch("queued", false);
9264
+ const active = yield* runFor("active");
9265
+ assert.equal((yield* runFor("queued")).status, "queued");
9266
+
9267
+ // Hold the run-end handler before it takes the thread lock, as a busy handler stream
9268
+ // does, then let it run once and signal when its queue release has finished.
9269
+ const handlerHeld = yield* Deferred.make<void>();
9270
+ const releaseHandler = yield* Deferred.make<void>();
9271
+ const releaseAttempted = yield* Deferred.make<void>();
9272
+ let lockCalls = 0;
9273
+ let armed = false;
9274
+ const withLock = executor.withLock;
9275
+ const observeLock: ThreadCommandExecutor.ThreadCommandExecutor["Service"]["withLock"] = (
9276
+ key,
9277
+ effect,
9278
+ ) => {
9279
+ if (key !== threadId || !armed) return withLock(key, effect);
9280
+ lockCalls += 1;
9281
+ if (lockCalls === 1) {
9282
+ return Deferred.succeed(handlerHeld, undefined).pipe(
9283
+ Effect.andThen(Deferred.await(releaseHandler)),
9284
+ Effect.andThen(withLock(key, effect)),
9285
+ );
9286
+ }
9287
+ if (lockCalls === 3) {
9288
+ return withLock(key, effect).pipe(
9289
+ Effect.ensuring(Deferred.succeed(releaseAttempted, undefined)),
9290
+ );
9291
+ }
9292
+ return withLock(key, effect);
9293
+ };
9294
+ const lockSpy = vi.spyOn(executor, "withLock").mockImplementation(observeLock);
9295
+ yield* Effect.gen(function* () {
9296
+ armed = true;
9297
+ const now = yield* DateTime.now;
9298
+ const { thread } = yield* orchestrator.getThreadProjection(threadId);
9299
+ yield* eventSink.write({
9300
+ events: [
9301
+ // A failed worktree setup leaves the thread without its worktree.
9302
+ ...(worktreeFailed
9303
+ ? [
9304
+ {
9305
+ id: EventId.make(`${threadId}:worktree-missing`),
9306
+ type: "thread.metadata-updated" as const,
9307
+ threadId,
9308
+ occurredAt: now,
9309
+ payload: { ...thread, worktreePath: null },
9310
+ },
9311
+ ]
9312
+ : []),
9313
+ {
9314
+ id: EventId.make(`${threadId}:active:completed`),
9315
+ type: "run.updated",
9316
+ threadId,
9317
+ runId: active.id,
9318
+ ...(active.rootNodeId === null ? {} : { nodeId: active.rootNodeId }),
9319
+ providerInstanceId: active.providerInstanceId,
9320
+ occurredAt: now,
9321
+ payload: worktreeFailed
9322
+ ? {
9323
+ ...active,
9324
+ status: "failed",
9325
+ workspacePreparation: { type: "worktree", baseRef: "main" },
9326
+ startedAt: now,
9327
+ completedAt: now,
9328
+ }
9329
+ : { ...active, status: "completed", startedAt: now, completedAt: now },
9330
+ },
9331
+ ],
9332
+ });
9333
+ yield* Deferred.await(handlerHeld);
9334
+ yield* dispatch("newer", !worktreeFailed);
9335
+ // The thread is idle but has a queue, so the newer send waits behind it.
9336
+ assert.equal((yield* runFor("newer")).status, "queued");
9337
+
9338
+ yield* Deferred.succeed(releaseHandler, undefined);
9339
+ yield* Deferred.await(releaseAttempted);
9340
+ const queued = yield* runFor("queued");
9341
+ if (worktreeFailed) {
9342
+ // Without a worktree the queue stays held instead of running in the project checkout.
9343
+ assert.equal(queued.status, "queued");
9344
+ assert.isTrue(queued.queueHeld);
9345
+ } else {
9346
+ assert.notEqual(queued.status, "queued");
9347
+ }
9348
+ }).pipe(Effect.ensuring(Effect.sync(() => lockSpy.mockRestore())));
9349
+ }),
9350
+ );
9351
+
9352
it.effect.each([false, true])(
9353
"restarts a continuation that failed before its prompt, unless stopped (%s)",
9354
(stopped) =>