@devries-cameron20 / t3code Mirror · read-only / commits / 531cee6f16
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 14:30 UTC

fix(server): Claude background-task wakes start when the queue is held (#17885)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-authored-by: maria-rcks <maria@kuuro.net>

fireboltdude1357 committed Oct 27, 2026 at 19:27 UTC 531cee6f169578183e1b9d63032e5e9b68db9c03
7 files changed +210 -14
apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
+2
@@ -4880,6 +4880,8 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
4880 threadId: context.projectionThreadId,
4881 providerThreadId: context.providerThread.id,
4882 driver: CODEX_PROVIDER,
4883 + // Codex starts a new turn from this text, so nothing is buffered.
4884 + delivery: "message_text",
4885 detail: codexBackgroundCommandDetail(payload.item),
4886 notification: {
4887 ...backgroundWorkNotification([
apps/server/src/orchestration-v2/BackgroundWorkStop.integration.test.ts
+46
@@ -35,6 +35,7 @@ const instanceId = ProviderInstanceId.make("codex");
35 const modelSelection = { instanceId, model: "test-model" };
36
37 const stopEarlierBackgroundWork = ({
38 + wakeAfterStop = false,
39 failedStart = false,
40 stopWithQueue,
41 olderStart = false,
@@ -42,6 +43,7 @@ const stopEarlierBackgroundWork = ({
43 stalledRun,
44 workflowMember = false,
45 }: {
46 + readonly wakeAfterStop?: boolean;
47 readonly failedStart?: boolean;
48 readonly backgroundWorkAfterStop?: "running" | "unknown";
49 readonly stopWithQueue?: "thread.stop" | "run.interrupt";
@@ -192,6 +194,46 @@ const stopEarlierBackgroundWork = ({
194 });
195 yield* worker.drain();
196 yield* Fiber.join(running);
197 + if (wakeAfterStop) {
198 + const send = (id: string, provider = false) =>
199 + orchestrator.dispatch({
200 + type: "message.dispatch",
201 + commandId: CommandId.make(id),
202 + threadId,
203 + messageId: MessageId.make(`message:${id}`),
204 + text: id,
205 + attachments: [],
206 + dispatchMode: { type: "queue_after_active" },
207 + createdBy: provider ? "agent" : "user",
208 + creationSource: provider ? "provider" : "web",
209 + });
210 + yield* send("follow-up");
211 + yield* orchestrator.dispatch({
212 + type: "thread.stop",
213 + commandId: CommandId.make("stop"),
214 + threadId,
215 + });
216 + // Stop is committed, but its provider interrupt has not run yet.
217 + yield* send("Background task completed.", true);
218 + const [stopped, followUp, wake] = (yield* orchestrator.getThreadProjection(threadId))
219 + .runs;
220 + assert.equal(stopped?.status, "running");
221 + assert.isTrue(followUp?.queueHeld);
222 + assert.isTrue(wake?.queueHeld);
223 +
224 + yield* worker.drain();
225 + yield* orchestrator.resumeQueuedRuns;
226 + yield* worker.drain();
227 + const after = yield* orchestrator.getThreadProjection(threadId);
228 + assert.deepEqual(
229 + after.runs.map((run) => run.status),
230 + ["interrupted", "queued", "queued"],
231 + );
232 + assert.isTrue(after.runs.find((run) => run.id === followUp?.id)?.queueHeld);
233 + assert.isTrue(after.runs.find((run) => run.id === wake?.id)?.queueHeld);
234 + assert.lengthOf(started, 1);
235 + return;
236 + }
237 const first = started[0]!;
238 const codexTurn = (yield* orchestrator.getThreadProjection(threadId)).providerTurns[0]!;
239 const devServerId = TurnItemId.make("turn-item:dev-server");
@@ -1016,3 +1058,7 @@ it.effect.each(["missing-session-terminal", "returned-interrupt", "superseded-at
1058 "Stop recovers a stalled run after %s without changing a newer attempt",
1059 (stalledRun) => stopEarlierBackgroundWork({ stalledRun }),
1060 );
1061 +
1062 +it.effect("Stop holds a provider wake that queues before the stopped turn ends", () =>
1063 + stopEarlierBackgroundWork({ wakeAfterStop: true }),
1064 +);
apps/server/src/orchestration-v2/Orchestrator.ts
+16 -8
@@ -729,7 +729,7 @@ function delegatedTaskTerminalStatus(
729 function nextQueuedRun(
730 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "messages">,
731 ): OrchestrationV2Run | undefined {
732 - return queuedRunsInDeliveryOrder(projection)[0];
732 + return queuedRunsInDeliveryOrder(projection).find((run) => run.queueHeld !== true);
733 }
734
735 function followsCurrentConversation(
@@ -1491,8 +1491,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1491 if (
1492 projection.thread.archivedAt !== null ||
1493 projection.thread.deletedAt !== null ||
1494 - projection.runs.some(isBlockingRun) ||
1495 - projection.runs.some((run) => run.status === "queued" && run.queueHeld === true)
1494 + projection.runs.some(isBlockingRun)
1495 ) {
1496 return;
1497 }
@@ -5612,6 +5611,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
5611 }),
5612 ),
5613 );
5614 + const queueIsHeld = projection.runs.some(
5615 + (candidate) => candidate.status === "queued" && candidate.queueHeld === true,
5616 + );
5617 + // Buffered wakes need a T3 run for output the CLI already started producing.
5618 + // Stop still holds wakes that arrive before the active run settles.
5619 + const wakeSkipsHold =
5620 + !worktreeSetupUnresolved &&
5621 + activeRun !== undefined &&
5622 + queueIsHeld &&
5623 + command.createdBy === "agent" &&
5624 + command.creationSource === "provider" &&
5625 + !(yield* stopReachedRun(command, command.threadId, activeRun.id));
5626 const run: OrchestrationV2Run = {
5627 id: runId,
5628 threadId: command.threadId,
@@ -5624,10 +5635,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
5635 activeAttemptId: attemptId,
5636 status: "queued",
5637 ...(worktreeMissingRun !== undefined ||
5627 - (heldRunsToRelease.length === 0 &&
5628 - projection.runs.some(
5629 - (candidate) => candidate.status === "queued" && candidate.queueHeld === true,
5630 - ))
5638 + (heldRunsToRelease.length === 0 && queueIsHeld && !wakeSkipsHold)
5639 ? { queueHeld: true }
5640 : {}),
5641 queuePosition:
@@ -12673,7 +12681,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
12681 });
12682 }
12683 }
12676 - const nextRun = selectedRun ?? nextQueuedRun(projection);
12684 + const nextRun = selectedRun ?? queuedRunsInDeliveryOrder(projection)[0];
12685 if (
12686 usageLimitHoldsQueue(
12687 projection,
apps/server/src/orchestration-v2/ProjectionStore.test.ts
+35
@@ -186,6 +186,41 @@ const addRolledBackRecoveryCandidate = Effect.fn("addRolledBackRecoveryCandidate
186 return threadId;
187 });
188
189 +it.effect.each([
190 + ["memory", ProjectionStore.layerMemory],
191 + ["SQL", layerTest],
192 +] as const)("%s queue readiness skips held runs without bypassing active work", ([, layer]) =>
193 + Effect.gen(function* () {
194 + const store = yield* ProjectionStore.ProjectionStoreV2;
195 + const threadId = yield* addRolledBackRecoveryCandidate("held-queue");
196 + const { runs } = yield* store.getThreadProjection(threadId);
197 + const run = runs[0]!;
198 + const now = yield* DateTime.now;
199 + const held = { ...run, status: "queued" as const, queueHeld: true };
200 + yield* store.apply({
201 + id: EventId.make("held-queue:held"),
202 + type: "run.updated",
203 + threadId,
204 + occurredAt: now,
205 + payload: held,
206 + });
207 + assert.isFalse(yield* store.canStartQueuedRun(threadId));
208 +
209 + const wake = { ...run, id: RunId.make("held-queue:wake"), ordinal: 2 };
210 + for (const status of ["running", "queued", "completed"] as const) {
211 + yield* store.apply({
212 + id: EventId.make(`held-queue:wake:${status}`),
213 + type: "run.updated",
214 + threadId,
215 + occurredAt: now,
216 + payload: { ...wake, status },
217 + });
218 + assert.equal(yield* store.canStartQueuedRun(threadId), status === "queued");
219 + }
220 + assert.deepEqual((yield* store.getThreadProjection(threadId)).runs[0], held);
221 + }).pipe(Effect.provide(layer)),
222 +);
223 +
224 it.effect.each([
225 ["memory", ProjectionStore.layerMemory],
226 ["SQL", layerTest],
apps/server/src/orchestration-v2/ProjectionStore.ts
+5 -6
@@ -5313,11 +5313,11 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5313 SELECT 1 AS ready FROM orchestration_v2_projection_threads AS thread
5314 WHERE thread.thread_id = ${threadId} AND thread.archived_at IS NULL AND thread.deleted_at IS NULL
5315 AND EXISTS (SELECT 1 FROM orchestration_v2_projection_runs
5316 - WHERE thread_id = ${threadId} AND status = 'queued')
5316 + WHERE thread_id = ${threadId} AND status = 'queued'
5317 + AND json_extract(payload_json, '$.queueHeld') IS NOT 1)
5318 AND NOT EXISTS (SELECT 1 FROM orchestration_v2_projection_runs
5319 WHERE thread_id = ${threadId}
5319 - AND (status IN ('preparing', 'starting', 'running', 'waiting')
5320 - OR (status = 'queued' AND json_extract(payload_json, '$.queueHeld') = 1)))`;
5320 + AND status IN ('preparing', 'starting', 'running', 'waiting'))`;
5321 return rows.length > 0;
5322 }),
5323 )
@@ -7604,14 +7604,13 @@ export const layerMemory: Layer.Layer<ProjectionStoreV2> = Layer.effect(
7604 return (
7605 projection.thread.archivedAt === null &&
7606 projection.thread.deletedAt === null &&
7607 - projection.runs.some((run) => run.status === "queued") &&
7607 + projection.runs.some((run) => run.status === "queued" && run.queueHeld !== true) &&
7608 !projection.runs.some(
7609 (run) =>
7610 run.status === "preparing" ||
7611 run.status === "starting" ||
7612 run.status === "running" ||
7613 - run.status === "waiting" ||
7614 - (run.status === "queued" && run.queueHeld === true),
7613 + run.status === "waiting",
7614 )
7615 );
7616 }),
apps/server/src/orchestration-v2/ThreadLaunchService.test.ts
+15
@@ -1320,6 +1320,21 @@ it.effect.each(["failed", "interrupted"] as const)(
1320 createdBy: "user",
1321 creationSource: "web",
1322 });
1323 + yield* threads.dispatch({
1324 + type: "message.dispatch",
1325 + commandId: CommandId.make(`command:launch:late-queued-follow-up-${ending}:wake`),
1326 + threadId: launched.threadId,
1327 + messageId: MessageId.make(`message:launch:late-queued-follow-up-${ending}:wake`),
1328 + text: "Background task completed.",
1329 + attachments: [],
1330 + dispatchMode: { type: "queue_after_active" },
1331 + createdBy: "agent",
1332 + creationSource: "provider",
1333 + });
1334 + const wake = (yield* threads.getThreadProjection(launched.threadId)).runs.at(-1)!;
1335 + assert.equal(wake.status, "queued");
1336 + assert.isTrue(wake.queueHeld);
1337 +
1338 // Nor does the next one, though the run that message started is now the
1339 // latest and carries no workspace preparation.
1340 yield* threads.sendToThread({
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+91
@@ -4904,6 +4904,23 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
4904 threadId,
4905 runId: limited.id,
4906 });
4907 + yield* eventSink.write({
4908 + events: [
4909 + {
4910 + id: EventId.make(`${threadId}:hold-available`),
4911 + type: "run.updated",
4912 + threadId,
4913 + runId: available.id,
4914 + occurredAt: now,
4915 + payload: { ...available, queueHeld: true },
4916 + },
4917 + ],
4918 + });
4919 + yield* orchestrator.dispatch({
4920 + type: "queue.resume",
4921 + commandId: CommandId.make(`${threadId}:resume-available`),
4922 + threadId,
4923 + });
4924 yield* orchestrator.resumeQueuedRuns;
4925 const after = yield* orchestrator.getThreadProjection(threadId);
4926 const started = after.runs.find((run) => run.id === available.id);
@@ -5286,6 +5303,80 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
5303 }),
5304 );
5305
5306 + it.effect("starts a provider wake queued beside a held queue", () =>
5307 + Effect.gen(function* () {
5308 + const orchestrator = yield* Orchestrator.OrchestratorV2;
5309 + const recovery = yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService;
5310 + const events = yield* EventSink.EventSinkV2;
5311 + const threadId = ThreadId.make("queue-hold-provider-wake");
5312 + yield* orchestrator.dispatch({
5313 + type: "thread.create",
5314 + createdBy: "user",
5315 + creationSource: "web",
5316 + commandId: CommandId.make(`${threadId}:create`),
5317 + threadId,
5318 + projectId: ProjectId.make(`${threadId}:project`),
5319 + title: "Wake beside a held queue",
5320 + modelSelection,
5321 + runtimeMode: "full-access",
5322 + interactionMode: "default",
5323 + branch: null,
5324 + worktreePath: process.cwd(),
5325 + });
5326 + const send = (index: number, text: string, source: "web" | "provider" | "server" = "web") =>
5327 + orchestrator.dispatch({
5328 + type: "message.dispatch",
5329 + createdBy: source === "web" ? "user" : "agent",
5330 + creationSource: source,
5331 + commandId: CommandId.make(`${threadId}:message:${index}`),
5332 + threadId,
5333 + messageId: MessageId.make(`${threadId}:message:${index}`),
5334 + text,
5335 + attachments: [],
5336 + modelSelection,
5337 + dispatchMode: { type: index === 0 ? "start_immediately" : "queue_after_active" },
5338 + });
5339 + yield* send(0, "Active");
5340 + yield* send(1, "Queued before the restart");
5341 + yield* recovery.reconcile("startup");
5342 + yield* send(2, "First background task completed.", "provider");
5343 + yield* send(3, "Background task completed.", "provider");
5344 + yield* send(4, "Continue after the command completed.", "server");
5345 + const queued = yield* orchestrator.getThreadProjection(threadId);
5346 + const [held, active, wake] = [queued.runs[1]!, queued.runs[2]!, queued.runs[3]!];
5347 + assert.equal(active.status, "starting");
5348 + assert.isTrue(held.queueHeld);
5349 + assert.equal(wake.status, "queued");
5350 + assert.notEqual(wake.queueHeld, true);
5351 + const textContinuation = queued.runs[4]!;
5352 + assert.isTrue(textContinuation.queueHeld);
5353 + const now = yield* DateTime.now;
5354 + yield* events.write({
5355 + events: [
5356 + {
5357 + id: EventId.make(`${threadId}:active-done`),
5358 + type: "run.updated",
5359 + threadId,
5360 + runId: active.id,
5361 + occurredAt: now,
5362 + payload: { ...active, status: "completed", completedAt: now },
5363 + },
5364 + ],
5365 + });
5366 + yield* orchestrator.resumeQueuedRuns;
5367 + const after = yield* orchestrator.getThreadProjection(threadId);
5368 + assert.equal(after.runs.find((run) => run.id === wake.id)?.status, "starting");
5369 + assert.deepEqual(
5370 + after.runs.find((run) => run.id === held.id),
5371 + held,
5372 + );
5373 + assert.deepEqual(
5374 + after.runs.find((run) => run.id === textContinuation.id),
5375 + textContinuation,
5376 + );
5377 + }),
5378 + );
5379 +
5380 it.effect("edits and removes queued runs", () =>
5381 Effect.gen(function* () {
5382 const orchestrator = yield* Orchestrator.OrchestratorV2;