@devries-cameron20 / t3code Mirror · read-only / commits / 8ab098a4a8
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 12:00 UTC

fix(server): held queues release agent and scheduled messages once the thread completes a run (#27221)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 29, 2026 at 18:12 UTC 8ab098a4a8d62335f43cf61b708c2fbb65d4741c
4 files changed +351 -15
apps/server/src/orchestration-v2/Orchestrator.ts
+73 -13
@@ -741,6 +741,20 @@ function delegatedTaskTerminalStatus(
741 }
742 }
743
744 +/** Failed worktree setup keeps the queue held so queued messages cannot start in the project checkout. */
745 +function worktreeSetupPending(
746 + projection: Pick<OrchestrationV2ThreadProjection, "thread" | "runs">,
747 +): boolean {
748 + return (
749 + projection.thread.worktreePath === null &&
750 + projection.runs.some(
751 + (run) =>
752 + (run.status === "failed" || run.status === "interrupted") &&
753 + run.workspacePreparation?.type === "worktree",
754 + )
755 + );
756 +}
757 +
758 function nextQueuedRun(
759 projection: Pick<OrchestrationV2ThreadProjection, "runs" | "messages">,
760 ): OrchestrationV2Run | undefined {
@@ -2195,6 +2209,52 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2209 ),
2210 );
2211
2212 + // A hold waits for a person to review what they queued. Nobody reviews messages from agents,
2213 + // the server or scheduled tasks, so a later completed run releases those unless it was stopped.
2214 + const releaseUnattendedHolds = Effect.fn("Orchestrator.releaseUnattendedHolds")(function* (
2215 + threadId: ThreadId,
2216 + completedRunId: RunId,
2217 + ) {
2218 + const held = yield* projectionStore.getHeldQueuedRuns(threadId);
2219 + if (held.length === 0) return;
2220 + const projection = yield* projectionStore.getThreadRecords(
2221 + threadId,
2222 + ["runs", "messages", "turnItems"],
2223 + {
2224 + messageIds: held.map((run) => run.userMessageId),
2225 + turnItemTypes: ["run_interrupt_request"],
2226 + turnItemRunIds: [completedRunId],
2227 + },
2228 + );
2229 + // Only the newest run completing shows the thread moved past the hold; a late or
2230 + // bookkeeping update to an older run, overtaken by a failure or Stop, does not.
2231 + if (
2232 + latestExecutedRun(projection.runs)?.id !== completedRunId ||
2233 + projection.turnItems.length > 0 ||
2234 + worktreeSetupPending(projection)
2235 + )
2236 + return;
2237 + const { messages } = projection;
2238 + const unattended = new Set(
2239 + messages
2240 + .filter((message) => message.createdBy !== "user" || message.scheduledTaskId !== undefined)
2241 + .map((message) => message.id),
2242 + );
2243 + const now = yield* DateTime.now;
2244 + const released = held.filter((run) => unattended.has(run.userMessageId));
2245 + if (released.length === 0) return;
2246 + yield* writeSystemEvents(
2247 + released.map((run) => ({
2248 + type: "run.updated" as const,
2249 + threadId,
2250 + runId: run.id,
2251 + providerInstanceId: run.providerInstanceId,
2252 + occurredAt: now,
2253 + payload: { ...run, queueHeld: false },
2254 + })),
2255 + );
2256 + });
2257 +
2258 const resumeQueuedRuns = Effect.gen(function* () {
2259 const threadIds = yield* projectionStore.getRecoveryThreadIds("queued-runs");
2260 let resumed = 0;
@@ -5539,15 +5599,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
5599 dispatchMode.type === "queue_after_active";
5600 const shouldQueue = queueBehindRun !== undefined && startsOrQueuesRun;
5601 // Client messages resume held queues; server continuations and scheduled tasks do not.
5542 - // Provider failure or usage limits re-hold the queue. Failed worktree setup keeps it held
5543 - // until a worktree exists, so queued messages cannot start in the project checkout.
5544 - const worktreeSetupUnresolved =
5545 - projection.thread.worktreePath === null &&
5546 - projection.runs.some(
5547 - (run) =>
5548 - (run.status === "failed" || run.status === "interrupted") &&
5549 - run.workspacePreparation?.type === "worktree",
5550 - );
5602 + // Provider failure or usage limits re-hold the queue.
5603 + const worktreeSetupUnresolved = worktreeSetupPending(projection);
5604 const sentByUserFromClient =
5605 command.createdBy === "user" &&
5606 (command.creationSource === "web" || command.creationSource === "mobile") &&
@@ -13083,10 +13136,17 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
13136 yield* threadDispatch
13137 .withLock(
13138 threadId,
13086 - startNextQueuedRun(threadId, {
13087 - endedRunId: run.id,
13088 - ...(run.status === "failed" ? { failedRunId: run.id } : {}),
13089 - }),
13139 + (run.status === "completed"
13140 + ? releaseUnattendedHolds(threadId, run.id)
13141 + : Effect.void
13142 + ).pipe(
13143 + Effect.andThen(
13144 + startNextQueuedRun(threadId, {
13145 + endedRunId: run.id,
13146 + ...(run.status === "failed" ? { failedRunId: run.id } : {}),
13147 + }),
13148 + ),
13149 + ),
13150 )
13151 .pipe(
13152 Effect.catchCause((cause) =>
apps/server/src/orchestration-v2/ProjectionStore.ts
+15
@@ -612,6 +612,9 @@ export interface ProjectionStoreV2Shape {
612 readonly canStartQueuedRun: (
613 threadId: ThreadId,
614 ) => Effect.Effect<boolean, ProjectionStoreV2Error>;
615 + readonly getHeldQueuedRuns: (
616 + threadId: ThreadId,
617 + ) => Effect.Effect<ReadonlyArray<OrchestrationV2Run>, ProjectionStoreV2Error>;
618 readonly getRecoveryThreadIds: (
619 kind: ProjectionRecoveryKind,
620 ) => Effect.Effect<ReadonlyArray<ThreadId>, ProjectionStoreV2Error>;
@@ -5183,6 +5186,17 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
5186 )
5187 .pipe(Effect.mapError(controlReadError(threadId)));
5188
5189 + const getHeldQueuedRuns: ProjectionStoreV2Shape["getHeldQueuedRuns"] = (threadId) =>
5190 + sql<PayloadRow>`
5191 + SELECT payload_json FROM orchestration_v2_projection_runs
5192 + WHERE thread_id = ${threadId} AND status = 'queued'
5193 + AND json_extract(payload_json, '$.queueHeld') IS 1
5194 + ORDER BY ordinal ASC
5195 + `.pipe(
5196 + Effect.flatMap(decodeRows(decodeRunPayload, threadId)),
5197 + Effect.mapError(controlReadError(threadId)),
5198 + );
5199 +
5200 const getMessageCount: ProjectionStoreV2Shape["getMessageCount"] = (threadId) =>
5201 sql<{
5202 count: number;
@@ -6899,6 +6913,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6913 getCheckpointRollbackContext: readOnReader(getCheckpointRollbackContext),
6914 getRunMessage: readOnReader(getRunMessage),
6915 canStartQueuedRun: readOnReader(canStartQueuedRun),
6916 + getHeldQueuedRuns: readOnReader(getHeldQueuedRuns),
6917 getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs),
6918 hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest),
6919 isNodeStored: readOnReader(isNodeStored),
apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts
+1
@@ -263,6 +263,7 @@ it.effect(
263 getRunMessage: () => Effect.die("not used"),
264 isNodeStored: () => Effect.die("not used"),
265 canStartQueuedRun: () => Effect.die("not used"),
266 + getHeldQueuedRuns: () => Effect.die("not used"),
267 getRecoveryThreadIds: () => Effect.die("unused getRecoveryThreadIds"),
268 getUnreadableThreadIds: () => Effect.die("unused getUnreadableThreadIds"),
269 getThreadSnapshot: () => Effect.die("unused getThreadSnapshot"),
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+262 -2
@@ -4768,6 +4768,265 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
4768 }),
4769 );
4770
4771 + // A thread fed by agent messages, with helpers to end its runs as a provider would.
4772 + const queueTestThread = (threadId: ThreadId) =>
4773 + Effect.gen(function* () {
4774 + const orchestrator = yield* Orchestrator.OrchestratorV2;
4775 + const eventSink = yield* EventSink.EventSinkV2;
4776 + yield* orchestrator.dispatch({
4777 + type: "thread.create",
4778 + createdBy: "user",
4779 + creationSource: "web",
4780 + commandId: CommandId.make(`${threadId}:create`),
4781 + threadId,
4782 + projectId: ProjectId.make(`${threadId}:project`),
4783 + title: "Dispatcher",
4784 + modelSelection,
4785 + runtimeMode: "full-access",
4786 + interactionMode: "default",
4787 + branch: null,
4788 + worktreePath: process.cwd(),
4789 + });
4790 + const send = (key: string, createdBy: "user" | "agent", start: boolean) =>
4791 + orchestrator.dispatch({
4792 + type: "message.dispatch",
4793 + createdBy,
4794 + creationSource: createdBy === "user" ? "web" : "server",
4795 + commandId: CommandId.make(`${threadId}:message:${key}`),
4796 + threadId,
4797 + messageId: MessageId.make(`${threadId}:message:${key}`),
4798 + text: key,
4799 + attachments: [],
4800 + modelSelection,
4801 + dispatchMode: { type: start ? "start_immediately" : "queue_after_active" },
4802 + });
4803 + const runFor = (key: string) =>
4804 + Effect.map(orchestrator.getThreadProjection(threadId), ({ runs }) => {
4805 + const run = runs.find(
4806 + (candidate) => candidate.userMessageId === MessageId.make(`${threadId}:message:${key}`),
4807 + );
4808 + assert.isDefined(run);
4809 + return run;
4810 + });
4811 + const startedRunIds = yield* Queue.unbounded<RunId>();
4812 + // Run updates in commit order; the run-update listener handles them in this order too.
4813 + const runUpdates: Array<OrchestrationV2Run> = [];
4814 + yield* eventSink
4815 + .stream({ threadId, afterSequence: yield* orchestrator.getThreadEventSequence(threadId) })
4816 + .pipe(
4817 + Stream.runForEach((stored) => {
4818 + if (stored.event.type !== "run.updated") return Effect.void;
4819 + runUpdates.push(stored.event.payload);
4820 + return stored.event.payload.status === "starting"
4821 + ? Queue.offer(startedRunIds, stored.event.payload.id)
4822 + : Effect.void;
4823 + }),
4824 + Effect.forkScoped,
4825 + );
4826 + yield* Effect.yieldNow;
4827 + const releasedAfter = (key: string, completedKey: string) =>
4828 + Effect.gen(function* () {
4829 + const [run, completed] = [yield* runFor(key), yield* runFor(completedKey)];
4830 + const released = runUpdates.findIndex(
4831 + (update) => update.id === run.id && update.queueHeld === false,
4832 + );
4833 + const completion = runUpdates.findIndex(
4834 + (update) => update.id === completed.id && update.status === "completed",
4835 + );
4836 + assert.isAbove(released, completion, `${key} was released before ${completedKey} ended`);
4837 + });
4838 + const end = (
4839 + key: string,
4840 + status: "completed" | "failed",
4841 + revision = "",
4842 + patch: Partial<OrchestrationV2Run> = {},
4843 + ) =>
4844 + Effect.gen(function* () {
4845 + const run = yield* runFor(key);
4846 + assert.isNotNull(run.rootNodeId);
4847 + const now = yield* DateTime.now;
4848 + yield* eventSink.write({
4849 + events: [
4850 + ...(status === "failed"
4851 + ? [
4852 + {
4853 + id: EventId.make(`${threadId}:${key}:error`),
4854 + type: "turn-item.updated" as const,
4855 + threadId,
4856 + runId: run.id,
4857 + nodeId: run.rootNodeId,
4858 + providerInstanceId: run.providerInstanceId,
4859 + occurredAt: now,
4860 + payload: {
4861 + id: TurnItemId.make(`${threadId}:${key}:error`),
4862 + type: "error" as const,
4863 + threadId,
4864 + runId: run.id,
4865 + nodeId: run.rootNodeId,
4866 + providerThreadId: run.providerThreadId,
4867 + providerTurnId: null,
4868 + nativeItemRef: null,
4869 + parentItemId: null,
4870 + ordinal: 2,
4871 + status: "failed" as const,
4872 + title: "Provider failure",
4873 + startedAt: now,
4874 + completedAt: now,
4875 + updatedAt: now,
4876 + failure: {
4877 + class: "provider_error" as const,
4878 + message: "Provider failed.",
4879 + code: "provider_failed",
4880 + retryable: null,
4881 + },
4882 + },
4883 + },
4884 + ]
4885 + : []),
4886 + {
4887 + id: EventId.make(`${threadId}:${key}:${status}${revision}`),
4888 + type: "run.updated",
4889 + threadId,
4890 + runId: run.id,
4891 + nodeId: run.rootNodeId,
4892 + providerInstanceId: run.providerInstanceId,
4893 + occurredAt: now,
4894 + payload: { ...run, ...patch, status, startedAt: now, completedAt: now },
4895 + },
4896 + ],
4897 + });
4898 + });
4899 +
4900 + // The run-update listener places failure holds after the terminal event commits.
4901 + const awaitHeld = (key: string) =>
4902 + Effect.gen(function* () {
4903 + const run = yield* runFor(key);
4904 + yield* eventSink.stream({ threadId, afterSequence: 0, eventType: "run.updated" }).pipe(
4905 + Stream.filter(
4906 + (stored) =>
4907 + stored.event.type === "run.updated" &&
4908 + stored.event.payload.id === run.id &&
4909 + stored.event.payload.queueHeld === true,
4910 + ),
4911 + Stream.runHead,
4912 + );
4913 + });
4914 + return { send, runFor, end, startedRunIds, releasedAfter, awaitHeld };
4915 + });
4916 +
4917 + it.effect("releases held agent messages once a later run completes, not the user's", () =>
4918 + Effect.gen(function* () {
4919 + const eventSink = yield* EventSink.EventSinkV2;
4920 + const threadId = ThreadId.make("runtime-layer-failed-queue-agent-messages");
4921 + const { send, runFor, end, startedRunIds, releasedAfter, awaitHeld } =
4922 + yield* queueTestThread(threadId);
4923 + yield* send("earlier", "agent", true);
4924 + yield* end("earlier", "completed");
4925 + yield* send("active", "agent", true);
4926 + yield* send("issue", "agent", false);
4927 + yield* send("user", "user", false);
4928 + yield* end("active", "failed");
4929 + yield* awaitHeld("issue");
4930 + assert.isTrue((yield* runFor("issue")).queueHeld);
4931 + // Delivery bookkeeping rewrites an older completed run; the thread has not moved on.
4932 + yield* end("earlier", "completed", ":bookkeeping");
4933 + // A scheduled tick finds the thread idle and starts past the held queue.
4934 + yield* send("tick", "agent", true);
4935 + assert.equal((yield* runFor("tick")).status, "starting");
4936 + yield* end("tick", "completed");
4937 + assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("issue")).id);
4938 + yield* releasedAfter("issue", "tick");
4939 + const user = yield* runFor("user");
4940 + assert.equal(user.status, "queued");
4941 + assert.isTrue(user.queueHeld);
4942 +
4943 + // A run that finishes despite a stop request does not release what the stop held.
4944 + yield* send("after-stop", "agent", false);
4945 + assert.isTrue((yield* runFor("after-stop")).queueHeld);
4946 + const issue = yield* runFor("issue");
4947 + assert.isNotNull(issue.rootNodeId);
4948 + const now = yield* DateTime.now;
4949 + yield* eventSink.write({
4950 + events: [
4951 + {
4952 + id: EventId.make(`${threadId}:issue:interrupt-request`),
4953 + type: "turn-item.updated",
4954 + threadId,
4955 + runId: issue.id,
4956 + nodeId: issue.rootNodeId,
4957 + providerInstanceId: issue.providerInstanceId,
4958 + occurredAt: now,
4959 + payload: {
4960 + id: TurnItemId.make(`${threadId}:issue:interrupt-request`),
4961 + type: "run_interrupt_request",
4962 + threadId,
4963 + runId: issue.id,
4964 + nodeId: issue.rootNodeId,
4965 + providerThreadId: issue.providerThreadId,
4966 + providerTurnId: null,
4967 + nativeItemRef: null,
4968 + parentItemId: null,
4969 + ordinal: 2,
4970 + status: "completed",
4971 + title: "Interrupt requested",
4972 + startedAt: now,
4973 + completedAt: now,
4974 + updatedAt: now,
4975 + message: "Stopped by user",
4976 + },
4977 + },
4978 + ],
4979 + });
4980 + yield* end("issue", "completed");
4981 + // The next completed run releases it, which also shows the listener handled the stopped one.
4982 + yield* send("probe", "agent", true);
4983 + assert.equal((yield* runFor("probe")).status, "starting");
4984 + yield* end("probe", "completed");
4985 + assert.equal(yield* Queue.take(startedRunIds), (yield* runFor("after-stop")).id);
4986 + yield* releasedAfter("after-stop", "probe");
4987 + }),
4988 + );
4989 +
4990 + it.effect("keeps held agent messages while the thread's worktree setup is unresolved", () =>
4991 + Effect.gen(function* () {
4992 + const orchestrator = yield* Orchestrator.OrchestratorV2;
4993 + const eventSink = yield* EventSink.EventSinkV2;
4994 + const dispatcherId = ThreadId.make("runtime-layer-held-queue-worktree");
4995 + const dispatcher = yield* queueTestThread(dispatcherId);
4996 + const witness = yield* queueTestThread(ThreadId.make("runtime-layer-held-queue-witness"));
4997 + yield* dispatcher.send("setup", "agent", true);
4998 + yield* dispatcher.send("issue", "agent", false);
4999 + yield* dispatcher.end("setup", "failed", "", {
5000 + workspacePreparation: { type: "worktree", baseRef: "main" },
5001 + });
5002 + yield* dispatcher.awaitHeld("issue");
5003 + assert.isTrue((yield* dispatcher.runFor("issue")).queueHeld);
5004 + yield* dispatcher.send("tick", "agent", true);
5005 + assert.equal((yield* dispatcher.runFor("tick")).status, "starting");
5006 + yield* witness.send("active", "agent", true);
5007 + yield* witness.send("queued", "agent", false);
5008 + // The failed setup never produced the thread's worktree.
5009 + const { thread } = yield* orchestrator.getThreadProjection(dispatcherId);
5010 + yield* eventSink.write({
5011 + events: [
5012 + {
5013 + id: EventId.make(`${dispatcherId}:worktree-missing`),
5014 + type: "thread.metadata-updated",
5015 + threadId: dispatcherId,
5016 + occurredAt: yield* DateTime.now,
5017 + payload: { ...thread, worktreePath: null },
5018 + },
5019 + ],
5020 + });
5021 + yield* dispatcher.end("tick", "completed");
5022 + // One listener handles every thread's completions in order, so the witness starting
5023 + // means the tick's completion was handled too.
5024 + yield* witness.end("active", "completed");
5025 + assert.equal(yield* Queue.take(witness.startedRunIds), (yield* witness.runFor("queued")).id);
5026 + assert.isTrue((yield* dispatcher.runFor("issue")).queueHeld);
5027 + }),
5028 + );
5029 +
5030 it.effect("keeps queued user selections and pauses only the limited provider", () =>
5031 Effect.gen(function* () {
5032 const orchestrator = yield* Orchestrator.OrchestratorV2;
@@ -5372,9 +5631,10 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
5631 after.runs.find((run) => run.id === held.id),
5632 held,
5633 );
5634 + // Nobody will resume a server message, so the completed run released it behind the wake.
5635 assert.deepEqual(
5636 after.runs.find((run) => run.id === textContinuation.id),
5377 - textContinuation,
5637 + { ...textContinuation, queueHeld: false },
5638 );
5639 }),
5640 );
@@ -5714,7 +5974,7 @@ it.layer(layerTest)("snooze queued runs", (it) => {
5974 payload:
5975 run.status === "queued"
5976 ? { ...run, queueHeld: true }
5717 - : { ...run, status: "completed" as const, completedAt: now },
5977 + : { ...run, status: "cancelled" as const, completedAt: now },
5978 })),
5979 effects: [],
5980 });