The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 12:00 UTC
fix(server): held queues release agent and scheduled messages once the thread completes a run (#27221)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Oct 29, 2026 at 18:12 UTC
8ab098a4a8d62335f43cf61b708c2fbb65d4741c
4 files changed
+351
-15
apps/server/src/orchestration-v2/Orchestrator.ts
+73
-13
@@ -741,6 +741,20 @@ function delegatedTaskTerminalStatus(
741
}
742
}
743
744
+/** Failed worktree setup keeps the queue held so queued messages cannot start in the project checkout. */
745
+function worktreeSetupPending(
746
+ projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
747
+): boolean {
748
+ return (
749
+ projection.thread.worktreePath === null &&
750
+ projection.runs.some(
751
+ (run) =>
752
+ (run.status === "failed" || run.status === "interrupted") &&
753
+ run.workspacePreparation?.type === "worktree",
754
+ )
755
+ );
756
+}
757
+
758
function nextQueuedRun(
759
projection: Pick<OrchestrationV2ThreadProjection, "runs" | "messages">,
760
): OrchestrationV2Run | undefined {
@@ -2195,6 +2209,52 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2209
),
2210
);
2211
2212
+ // A hold waits for a person to review what they queued. Nobody reviews messages from agents,
2213
+ // the server or scheduled tasks, so a later completed run releases those unless it was stopped.
2214
+ const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
2215
+ threadId: ThreadId,
2216
+ completedRunId: RunId,
2217
+ ) {
2218
+ const held = yield* projectionStore.getHeldQueuedRuns(threadId);
2219
+ if (held.length === 0) return;
2220
+ const projection = yield* projectionStore.getThreadRecords(
2221
+ threadId,
2222
+ ["runs", "messages", "turnItems"],
2223
+ {
2224
+ messageIds: held.map((run) => run.userMessageId),
2225
+ turnItemTypes: ["run_interrupt_request"],
2226
+ turnItemRunIds: [completedRunId],
2227
+ },
2228
+ );
2229
+ // Only the newest run completing shows the thread moved past the hold; a late or
2230
+ // bookkeeping update to an older run, overtaken by a failure or Stop, does not.
2231
+ if (
2232
+ latestExecutedRun(projection.runs)?.id !== completedRunId ||
2233
+ projection.turnItems.length > 0 ||
2234
+ worktreeSetupPending(projection)
2235
+ )
2236
+ return;
2237
+ const { messages } = projection;
2238
+ const unattended = new Set(
2239
+ messages
2240
+ .filter((message) => message.createdBy !== "user" || message.scheduledTaskId !== undefined)
2241
+ .map((message) => message.id),
2242
+ );
2243
+ const now = yield* DateTime.now;
2244
+ const released = held.filter((run) => unattended.has(run.userMessageId));
2245
+ if (released.length === 0) return;
2246
+ yield* writeSystemEvents(
2247
+ released.map((run) => ({
2248
+ type: "run.updated" as const,
2249
+ threadId,
2250
+ runId: run.id,
2251
+ providerInstanceId: run.providerInstanceId,
2252
+ occurredAt: now,
2253
+ payload: { ...run, queueHeld: false },
2254
+ })),
2255
+ );
2256
+ });
2257
+
2258
const resumeQueuedRuns = Effect.gen(function* () {
2259
const threadIds = yield* projectionStore.getRecoveryThreadIds("queued-runs");
2260
let resumed = 0;
@@ -5539,15 +5599,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
5599
dispatchMode.type === "queue_after_active";
5600
const shouldQueue = queueBehindRun !== undefined && startsOrQueuesRun;
5601
// Client messages resume held queues; server continuations and scheduled tasks do not.
5542
- // Provider failure or usage limits re-hold the queue. Failed worktree setup keeps it held
5543
- // until a worktree exists, so queued messages cannot start in the project checkout.
5544
- const worktreeSetupUnresolved =
5545
- projection.thread.worktreePath === null &&
5546
- projection.runs.some(
5547
- (run) =>
5548
- (run.status === "failed" || run.status === "interrupted") &&
5549
- run.workspacePreparation?.type === "worktree",
5550
- );
5602
+ // Provider failure or usage limits re-hold the queue.
5603
+ const worktreeSetupUnresolved = worktreeSetupPending(projection);
5604
const sentByUserFromClient =
5605
command.createdBy === "user" &&
5606
(command.creationSource === "web" || command.creationSource === "mobile") &&
@@ -13083,10 +13136,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13136
yield* threadDispatch
13137
.withLock(
13138
threadId,
13086
- startNextQueuedRun(threadId, {
13087
- endedRunId: run.id,
13088
- ...(run.status === "failed" ? { failedRunId: run.id } : {}),
13089
- }),
13139
+ (run.status === "completed"
13140
+ ? releaseUnattendedHolds(threadId, run.id)
13141
+ : Effect.void
13142
+ ).pipe(
13143
+ Effect.andThen(
13144
+ startNextQueuedRun(threadId, {
13145
+ endedRunId: run.id,
13146
+ ...(run.status === "failed" ? { failedRunId: run.id } : {}),
13147
+ }),
13148
+ ),
13149
+ ),
13150
)
13151
.pipe(
13152
Effect.catchCause((cause) =>
apps/server/src/orchestration-v2/ProjectionStore.ts
+15
@@ -612,6 +612,9 @@ export interface ProjectionStoreV2Shape {
612
readonly canStartQueuedRun: (
613
threadId: ThreadId,
614
) => Effect.Effect<boolean, ProjectionStoreV2Error>;
615
+ readonly getHeldQueuedRuns: (
616
+ threadId: ThreadId,
617
+ ) => Effect.Effect<ReadonlyArray<OrchestrationV2Run>, ProjectionStoreV2Error>;
618
readonly getRecoveryThreadIds: (
619
kind: ProjectionRecoveryKind,
620
) => Effect.Effect<ReadonlyArray<ThreadId>, ProjectionStoreV2Error>;
@@ -5183,6 +5186,17 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5186
)
5187
.pipe(Effect.mapError(controlReadError(threadId)));
5188
5189
+ const getHeldQueuedRuns: ProjectionStoreV2Shape["getHeldQueuedRuns"] = (threadId) =>
5190
+ sql<PayloadRow>`
5191
+ SELECT payload_json FROM orchestration_v2_projection_runs
5192
+ WHERE thread_id = ${threadId} AND status = 'queued'
5193
+ AND json_extract(payload_json, '$.queueHeld') IS 1
5194
+ ORDER BY ordinal ASC
5195
+ `.pipe(
5196
+ Effect.flatMap(decodeRows(decodeRunPayload, threadId)),
5197
+ Effect.mapError(controlReadError(threadId)),
5198
+ );
5199
+
5200
const getMessageCount: ProjectionStoreV2Shape["getMessageCount"] = (threadId) =>
5201
sql<{
5202
count: number;
@@ -6899,6 +6913,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6913
getCheckpointRollbackContext: readOnReader(getCheckpointRollbackContext),
6914
getRunMessage: readOnReader(getRunMessage),
6915
canStartQueuedRun: readOnReader(canStartQueuedRun),
6916
+ getHeldQueuedRuns: readOnReader(getHeldQueuedRuns),
6917
getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs),
6918
hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest),
6919
isNodeStored: readOnReader(isNodeStored),
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+1
@@ -263,6 +263,7 @@ it.effect(
263
getRunMessage: () => Effect.die("not used"),
264
isNodeStored: () => Effect.die("not used"),
265
canStartQueuedRun: () => Effect.die("not used"),
266
+ getHeldQueuedRuns: () => Effect.die("not used"),
267
getRecoveryThreadIds: () => Effect.die("unused getRecoveryThreadIds"),
268
getUnreadableThreadIds: () => Effect.die("unused getUnreadableThreadIds"),
269
getThreadSnapshot: () => Effect.die("unused getThreadSnapshot"),
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+262
-2
@@ -4768,6 +4768,265 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
4768
}),
4769
);
4770
4771
+ // A thread fed by agent messages, with helpers to end its runs as a provider would.
4772
+ const queueTestThread = (threadId: ThreadId) =>
4773
+ Effect.gen(function* () {
4774
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
4775
+ const eventSink = yield* EventSink.EventSinkV2;
4776
+ yield* orchestrator.dispatch({
4777
+ type: "thread.create",
4778
+ createdBy: "user",
4779
+ creationSource: "web",
4780
+ commandId: CommandId.make(`${threadId}:create`),
4781
+ threadId,
4782
+ projectId: ProjectId.make(`${threadId}:project`),
4783
+ title: "Dispatcher",
4784
+ modelSelection,
4785
+ runtimeMode: "full-access",
4786
+ interactionMode: "default",
4787
+ branch: null,
4788
+ worktreePath: process.cwd(),
4789
+ });
4790
+ const send = (key: string, createdBy: "user" | "agent", start: boolean) =>
4791
+ orchestrator.dispatch({
4792
+ type: "message.dispatch",
4793
+ createdBy,
4794
+ creationSource: createdBy === "user" ? "web" : "server",
4795
+ commandId: CommandId.make(`${threadId}:message:${key}`),
4796
+ threadId,
4797
+ messageId: MessageId.make(`${threadId}:message:${key}`),
4798
+ text: key,
4799
+ attachments: [],
4800
+ modelSelection,
4801
+ dispatchMode: { type: start ? "start_immediately" : "queue_after_active" },
4802
+ });
4803
+ const runFor = (key: string) =>
4804
+ Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) => {
4805
+ const run = runs.find(
4806
+ (candidate) => candidate.userMessageId === MessageId.make(`${threadId}:message:${key}`),
4807
+ );
4808
+ assert.isDefined(run);
4809
+ return run;
4810
+ });
4811
+ const startedRunIds = yield* Queue.unbounded<RunId>();
4812
+ // Run updates in commit order; the run-update listener handles them in this order too.
4813
+ const runUpdates: Array<OrchestrationV2Run> = [];
4814
+ yield* eventSink
4815
+ .stream({ threadId, afterSequence: yield* orchestrator.getThreadEventSequence(threadId) })
4816
+ .pipe(
4817
+ Stream.runForEach((stored) => {
4818
+ if (stored.event.type !== "run.updated") return Effect.void;
4819
+ runUpdates.push(stored.event.payload);
4820
+ return stored.event.payload.status === "starting"
4821
+ ? Queue.offer(startedRunIds, stored.event.payload.id)
4822
+ : Effect.void;
4823
+ }),
4824
+ Effect.forkScoped,
4825
+ );
4826
+ yield* Effect.yieldNow;
4827
+ const releasedAfter = (key: string, completedKey: string) =>
4828
+ Effect.gen(function* () {
4829
+ const [run, completed] = [yield* runFor(key), yield* runFor(completedKey)];
4830
+ const released = runUpdates.findIndex(
4831
+ (update) => update.id === run.id && update.queueHeld === false,
4832
+ );
4833
+ const completion = runUpdates.findIndex(
4834
+ (update) => update.id === completed.id && update.status === "completed",
4835
+ );
4836
+ assert.isAbove(released, completion, `${key} was released before ${completedKey} ended`);
4837
+ });
4838
+ const end = (
4839
+ key: string,
4840
+ status: "completed" | "failed",
4841
+ revision = "",
4842
+ patch: Partial<OrchestrationV2Run> = {},
4843
+ ) =>
4844
+ Effect.gen(function* () {
4845
+ const run = yield* runFor(key);
4846
+ assert.isNotNull(run.rootNodeId);
4847
+ const now = yield* DateTime.now;
4848
+ yield* eventSink.write({
4849
+ events: [
4850
+ ...(status === "failed"
4851
+ ? [
4852
+ {
4853
+ id: EventId.make(`${threadId}:${key}:error`),
4854
+ type: "turn-item.updated" as const,
4855
+ threadId,
4856
+ runId: run.id,
4857
+ nodeId: run.rootNodeId,
4858
+ providerInstanceId: run.providerInstanceId,
4859
+ occurredAt: now,
4860
+ payload: {
4861
+ id: TurnItemId.make(`${threadId}:${key}:error`),
4862
+ type: "error" as const,
4863
+ threadId,
4864
+ runId: run.id,
4865
+ nodeId: run.rootNodeId,
4866
+ providerThreadId: run.providerThreadId,
4867
+ providerTurnId: null,
4868
+ nativeItemRef: null,
4869
+ parentItemId: null,
4870
+ ordinal: 2,
4871
+ status: "failed" as const,
4872
+ title: "Provider failure",
4873
+ startedAt: now,
4874
+ completedAt: now,
4875
+ updatedAt: now,
4876
+ failure: {
4877
+ class: "provider_error" as const,
4878
+ message: "Provider failed.",
4879
+ code: "provider_failed",
4880
+ retryable: null,
4881
+ },
4882
+ },
4883
+ },
4884
+ ]
4885
+ : []),
4886
+ {
4887
+ id: EventId.make(`${threadId}:${key}:${status}${revision}`),
4888
+ type: "run.updated",
4889
+ threadId,
4890
+ runId: run.id,
4891
+ nodeId: run.rootNodeId,
4892
+ providerInstanceId: run.providerInstanceId,
4893
+ occurredAt: now,
4894
+ payload: { ...run, ...patch, status, startedAt: now, completedAt: now },
4895
+ },
4896
+ ],
4897
+ });
4898
+ });
4899
+
4900
+ // The run-update listener places failure holds after the terminal event commits.
4901
+ const awaitHeld = (key: string) =>
4902
+ Effect.gen(function* () {
4903
+ const run = yield* runFor(key);
4904
+ yield* eventSink.stream({ threadId, afterSequence: 0, eventType: "run.updated" }).pipe(
4905
+ Stream.filter(
4906
+ (stored) =>
4907
+ stored.event.type === "run.updated" &&
4908
+ stored.event.payload.id === run.id &&
4909
+ stored.event.payload.queueHeld === true,
4910
+ ),
4911
+ Stream.runHead,
4912
+ );
4913
+ });
4914
+ return { send, runFor, end, startedRunIds, releasedAfter, awaitHeld };
4915
+ });
4916
+
4917
+ it.effect("releases held agent messages once a later run completes, not the user's", () =>
4918
+ Effect.gen(function* () {
4919
+ const eventSink = yield* EventSink.EventSinkV2;
4920
+ const threadId = ThreadId.make("runtime-layer-failed-queue-agent-messages");
4921
+ const { send, runFor, end, startedRunIds, releasedAfter, awaitHeld } =
4922
+ yield* queueTestThread(threadId);
4923
+ yield* send("earlier", "agent", true);
4924
+ yield* end("earlier", "completed");
4925
+ yield* send("active", "agent", true);
4926
+ yield* send("issue", "agent", false);
4927
+ yield* send("user", "user", false);
4928
+ yield* end("active", "failed");
4929
+ yield* awaitHeld("issue");
4930
+ assert.isTrue((yield* runFor("issue")).queueHeld);
4931
+ // Delivery bookkeeping rewrites an older completed run; the thread has not moved on.
4932
+ yield* end("earlier", "completed", ":bookkeeping");
4933
+ // A scheduled tick finds the thread idle and starts past the held queue.
4934
+ yield* send("tick", "agent", true);
4935
+ assert.equal((yield* runFor("tick")).status, "starting");
4936
+ yield* end("tick", "completed");
4937
+ assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("issue")).id);
4938
+ yield* releasedAfter("issue", "tick");
4939
+ const user = yield* runFor("user");
4940
+ assert.equal(user.status, "queued");
4941
+ assert.isTrue(user.queueHeld);
4942
+
4943
+ // A run that finishes despite a stop request does not release what the stop held.
4944
+ yield* send("after-stop", "agent", false);
4945
+ assert.isTrue((yield* runFor("after-stop")).queueHeld);
4946
+ const issue = yield* runFor("issue");
4947
+ assert.isNotNull(issue.rootNodeId);
4948
+ const now = yield* DateTime.now;
4949
+ yield* eventSink.write({
4950
+ events: [
4951
+ {
4952
+ id: EventId.make(`${threadId}:issue:interrupt-request`),
4953
+ type: "turn-item.updated",
4954
+ threadId,
4955
+ runId: issue.id,
4956
+ nodeId: issue.rootNodeId,
4957
+ providerInstanceId: issue.providerInstanceId,
4958
+ occurredAt: now,
4959
+ payload: {
4960
+ id: TurnItemId.make(`${threadId}:issue:interrupt-request`),
4961
+ type: "run_interrupt_request",
4962
+ threadId,
4963
+ runId: issue.id,
4964
+ nodeId: issue.rootNodeId,
4965
+ providerThreadId: issue.providerThreadId,
4966
+ providerTurnId: null,
4967
+ nativeItemRef: null,
4968
+ parentItemId: null,
4969
+ ordinal: 2,
4970
+ status: "completed",
4971
+ title: "Interrupt requested",
4972
+ startedAt: now,
4973
+ completedAt: now,
4974
+ updatedAt: now,
4975
+ message: "Stopped by user",
4976
+ },
4977
+ },
4978
+ ],
4979
+ });
4980
+ yield* end("issue", "completed");
4981
+ // The next completed run releases it, which also shows the listener handled the stopped one.
4982
+ yield* send("probe", "agent", true);
4983
+ assert.equal((yield* runFor("probe")).status, "starting");
4984
+ yield* end("probe", "completed");
4985
+ assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("after-stop")).id);
4986
+ yield* releasedAfter("after-stop", "probe");
4987
+ }),
4988
+ );
4989
+
4990
+ it.effect("keeps held agent messages while the thread's worktree setup is unresolved", () =>
4991
+ Effect.gen(function* () {
4992
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
4993
+ const eventSink = yield* EventSink.EventSinkV2;
4994
+ const dispatcherId = ThreadId.make("runtime-layer-held-queue-worktree");
4995
+ const dispatcher = yield* queueTestThread(dispatcherId);
4996
+ const witness = yield* queueTestThread(ThreadId.make("runtime-layer-held-queue-witness"));
4997
+ yield* dispatcher.send("setup", "agent", true);
4998
+ yield* dispatcher.send("issue", "agent", false);
4999
+ yield* dispatcher.end("setup", "failed", "", {
5000
+ workspacePreparation: { type: "worktree", baseRef: "main" },
5001
+ });
5002
+ yield* dispatcher.awaitHeld("issue");
5003
+ assert.isTrue((yield* dispatcher.runFor("issue")).queueHeld);
5004
+ yield* dispatcher.send("tick", "agent", true);
5005
+ assert.equal((yield* dispatcher.runFor("tick")).status, "starting");
5006
+ yield* witness.send("active", "agent", true);
5007
+ yield* witness.send("queued", "agent", false);
5008
+ // The failed setup never produced the thread's worktree.
5009
+ const { thread } = yield* orchestrator.getThreadProjection(dispatcherId);
5010
+ yield* eventSink.write({
5011
+ events: [
5012
+ {
5013
+ id: EventId.make(`${dispatcherId}:worktree-missing`),
5014
+ type: "thread.metadata-updated",
5015
+ threadId: dispatcherId,
5016
+ occurredAt: yield* DateTime.now,
5017
+ payload: { ...thread, worktreePath: null },
5018
+ },
5019
+ ],
5020
+ });
5021
+ yield* dispatcher.end("tick", "completed");
5022
+ // One listener handles every thread's completions in order, so the witness starting
5023
+ // means the tick's completion was handled too.
5024
+ yield* witness.end("active", "completed");
5025
+ assert.equal(yield* Queue.take(witness.startedRunIds), (yield* witness.runFor("queued")).id);
5026
+ assert.isTrue((yield* dispatcher.runFor("issue")).queueHeld);
5027
+ }),
5028
+ );
5029
+
5030
it.effect("keeps queued user selections and pauses only the limited provider", () =>
5031
Effect.gen(function* () {
5032
const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -5372,9 +5631,10 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
5631
after.runs.find((run) => run.id === held.id),
5632
held,
5633
);
5634
+ // Nobody will resume a server message, so the completed run released it behind the wake.
5635
assert.deepEqual(
5636
after.runs.find((run) => run.id === textContinuation.id),
5377
- textContinuation,
5637
+ { ...textContinuation, queueHeld: false },
5638
);
5639
}),
5640
);
@@ -5714,7 +5974,7 @@ it.layer(layerTest)("snooze queued runs", (it) => {
5974
payload:
5975
run.status === "queued"
5976
? { ...run, queueHeld: true }
5717
- : { ...run, status: "completed" as const, completedAt: now },
5977
+ : { ...run, status: "cancelled" as const, completedAt: now },
5978
})),
5979
effects: [],
5980
});