The best way to control Claude Code, Codex and more from any computer or phone
fix(server): drop a finished child's queued question wake (#25680)
Julius Marminge committed
Nov 11, 2026 at 15:17 UTC
86c379d07ad8d94ec2e1742e29e3d3201315e272
3 files changed
+396
-14
apps/server/src/orchestration-v2/DelegatedTaskQuestionReactor.ts
+74
-14
@@ -4,6 +4,8 @@ import {
4
type OrchestrationV2DomainEvent,
5
type OrchestrationV2RuntimeRequest,
6
type OrchestrationV2Subagent,
7
+ type RuntimeRequestId,
8
+ type ThreadId,
9
} from "@t3tools/contracts";
10
import * as Context from "effect/Context";
11
import * as Effect from "effect/Effect";
@@ -58,6 +60,37 @@ const make = Effect.gen(function* () {
60
});
61
});
62
63
+ const cancelQueuedWakes = Effect.fnUntraced(function* (
64
+ parentThreadId: ThreadId,
65
+ requestIds: ReadonlyArray<RuntimeRequestId>,
66
+ ) {
67
+ const parent = yield* projections.getThreadRecords(parentThreadId, ["runs"]);
68
+ for (const requestId of requestIds) {
69
+ const key = `${parentThreadId}:${requestId}`;
70
+ const messageId = MessageId.make(`message:delegated-task-question:${key}`);
71
+ const queued = parent.runs.find(
72
+ (run) => run.status === "queued" && run.userMessageId === messageId,
73
+ );
74
+ if (queued === undefined) continue;
75
+ yield* engine
76
+ .dispatch({
77
+ type: "queued-run.cancel",
78
+ commandId: CommandId.make(`server:delegated-task-question-cancel:${key}`),
79
+ threadId: parentThreadId,
80
+ runId: queued.id,
81
+ })
82
+ .pipe(
83
+ Effect.catchCause((cause) =>
84
+ Effect.logWarning("cancelling a delegated task question wake failed", {
85
+ threadId: parentThreadId,
86
+ requestId,
87
+ cause,
88
+ }),
89
+ ),
90
+ );
91
+ }
92
+ });
93
+
94
// Runs for every domain event; most events return before any work is needed.
95
const handle: DelegatedTaskQuestionReactor["Service"]["handle"] = Effect.fnUntraced(
96
function* (event) {
@@ -100,6 +133,46 @@ const make = Effect.gen(function* () {
133
}
134
return;
135
}
136
+ if (event.type === "subagent.updated") {
137
+ const task = event.payload;
138
+ if (
139
+ task.origin !== "app_owned" ||
140
+ task.childThreadId === null ||
141
+ !["completed", "failed", "cancelled", "interrupted"].includes(task.status)
142
+ ) {
143
+ return;
144
+ }
145
+ // A settled task can no longer use an answer to its still-pending questions.
146
+ const { runtimeRequests } = yield* projections.getThreadRecords(task.childThreadId, [
147
+ "runtimeRequests",
148
+ ]);
149
+ const questions = runtimeRequests.filter((request) => request.kind === "user_input");
150
+ if (questions.length === 0) return;
151
+ const child = yield* projections.getThreadRecords(task.childThreadId, ["runs", "nodes"]);
152
+ const taskRunIds = new Set(
153
+ child.runs
154
+ .filter(
155
+ (run) =>
156
+ run.delegatedTaskId === task.id ||
157
+ (run.delegatedTaskId === undefined &&
158
+ child.thread.forkedFrom?.type === "node" &&
159
+ child.thread.forkedFrom.nodeId === task.id),
160
+ )
161
+ .map((run) => run.id),
162
+ );
163
+ const taskNodeIds = new Set(
164
+ child.nodes
165
+ .filter((node) => node.runId !== null && taskRunIds.has(node.runId))
166
+ .map((node) => node.id),
167
+ );
168
+ yield* cancelQueuedWakes(
169
+ task.threadId,
170
+ questions
171
+ .filter((request) => taskNodeIds.has(request.nodeId))
172
+ .map((request) => request.id),
173
+ );
174
+ return;
175
+ }
176
if (event.type !== "runtime-request.updated" || event.payload.kind !== "user_input") {
177
return;
178
}
@@ -107,21 +180,8 @@ const make = Effect.gen(function* () {
180
const parentThreadId = child.lineage.parentThreadId;
181
if (parentThreadId === null) return;
182
const request = yield* projections.getRuntimeRequest(event.threadId, event.payload.id);
110
- const key = `${parentThreadId}:${event.payload.id}`;
111
- const messageId = MessageId.make(`message:delegated-task-question:${key}`);
183
if (event.payload.status !== "pending" || request?.status !== "pending") {
113
- const parent = yield* projections.getThreadRecords(parentThreadId, ["runs"]);
114
- const queued = parent.runs.find(
115
- (run) => run.status === "queued" && run.userMessageId === messageId,
116
- );
117
- if (queued !== undefined) {
118
- yield* engine.dispatch({
119
- type: "queued-run.cancel",
120
- commandId: CommandId.make(`server:delegated-task-question-cancel:${key}`),
121
- threadId: parentThreadId,
122
- runId: queued.id,
123
- });
124
- }
184
+ yield* cancelQueuedWakes(parentThreadId, [event.payload.id]);
185
return;
186
}
187
const parent = yield* projections.getThreadRecords(parentThreadId, ["subagents", "runs"]);
apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts
+228
@@ -1574,6 +1574,234 @@ it.effect("wakes the parent once when its delegated task asks a question", () =>
1574
}).pipe(Effect.provide(DelegatedTaskQuestionReactor.layer.pipe(Layer.provideMerge(layerTest)))),
1575
);
1576
1577
+it.effect.each([false, true])(
1578
+ "drops a stale question wake (missed terminal event: %s)",
1579
+ (missedEvent) =>
1580
+ Effect.gen(function* () {
1581
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
1582
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
1583
+ const reactor = yield* DelegatedTaskQuestionReactor.DelegatedTaskQuestionReactor;
1584
+ const eventSink = yield* EventSink.EventSinkV2;
1585
+ const parentThreadId = ThreadId.make("thread:stale-question-parent");
1586
+ yield* orchestrator.dispatch({
1587
+ type: "thread.create",
1588
+ commandId: CommandId.make("create-stale-question-parent"),
1589
+ threadId: parentThreadId,
1590
+ projectId: ProjectId.make("project:stale-question"),
1591
+ title: "Parent",
1592
+ modelSelection,
1593
+ runtimeMode: "full-access",
1594
+ interactionMode: "default",
1595
+ branch: null,
1596
+ worktreePath: null,
1597
+ createdBy: "user",
1598
+ creationSource: "web",
1599
+ });
1600
+ yield* orchestrator.dispatch({
1601
+ type: "message.dispatch",
1602
+ commandId: CommandId.make("start-stale-question-parent"),
1603
+ threadId: parentThreadId,
1604
+ messageId: MessageId.make("message:stale-question-parent"),
1605
+ text: "Delegate the proof",
1606
+ attachments: [],
1607
+ dispatchMode: { type: "start_immediately" },
1608
+ createdBy: "user",
1609
+ creationSource: "web",
1610
+ });
1611
+ const parentRun = (yield* projections.getThreadProjection(parentThreadId)).runs[0]!;
1612
+ yield* orchestrator.dispatch({
1613
+ type: "delegated_task.request",
1614
+ commandId: CommandId.make("delegate-stale-question-task"),
1615
+ parentThreadId,
1616
+ parentRunId: parentRun.id,
1617
+ parentNodeId: parentRun.rootNodeId!,
1618
+ task: "Capture the proof",
1619
+ completionWake: "settled_only",
1620
+ modelSelection,
1621
+ runtimeMode: "full-access",
1622
+ interactionMode: "default",
1623
+ createdBy: "agent",
1624
+ creationSource: "mcp",
1625
+ });
1626
+ const task = (yield* projections.getThreadProjection(parentThreadId)).subagents[0]!;
1627
+ const childRun = (yield* projections.getThreadRecords(task.childThreadId!, ["runs"]))
1628
+ .runs[0]!;
1629
+ const now = yield* DateTime.now;
1630
+ const question = {
1631
+ id: EventId.make("event:stale-question"),
1632
+ type: "runtime-request.updated",
1633
+ threadId: task.childThreadId!,
1634
+ runId: childRun.id,
1635
+ occurredAt: now,
1636
+ payload: {
1637
+ id: RuntimeRequestId.make("request:stale-question"),
1638
+ nodeId: childRun.rootNodeId!,
1639
+ providerTurnId: null,
1640
+ nativeRequestRef: null,
1641
+ kind: "user_input",
1642
+ status: "pending",
1643
+ responseCapability: { type: "message" },
1644
+ createdAt: now,
1645
+ resolvedAt: null,
1646
+ },
1647
+ } satisfies OrchestrationV2DomainEvent;
1648
+ yield* eventSink.write({ events: [question] });
1649
+ yield* reactor.handle(question);
1650
+ const wake = (yield* projections.getThreadRecords(parentThreadId, [
1651
+ "messages",
1652
+ ])).messages.find((message) => message.notification !== undefined)!;
1653
+ const wakeRun = () =>
1654
+ projections
1655
+ .getThreadRecords(parentThreadId, ["runs"])
1656
+ .pipe(Effect.map(({ runs }) => runs.find((run) => run.userMessageId === wake.id)));
1657
+ assert.equal((yield* wakeRun())?.status, "queued");
1658
+
1659
+ // The child finishes without ever resolving its message-capability question.
1660
+ const childCompleted = {
1661
+ id: EventId.make("event:stale-question-child-completed"),
1662
+ type: "run.updated",
1663
+ threadId: task.childThreadId!,
1664
+ runId: childRun.id,
1665
+ occurredAt: now,
1666
+ payload: { ...childRun, status: "completed", completedAt: now },
1667
+ } satisfies OrchestrationV2DomainEvent;
1668
+ const taskCompleted = {
1669
+ id: EventId.make("event:stale-question-task-completed"),
1670
+ type: "subagent.updated",
1671
+ threadId: parentThreadId,
1672
+ occurredAt: now,
1673
+ payload: { ...task, status: "completed", result: "Proof captured.", completedAt: now },
1674
+ } satisfies OrchestrationV2DomainEvent;
1675
+ yield* eventSink.write({
1676
+ commandId: CommandId.make("command:runtime-reconcile:stale-question"),
1677
+ events: [childCompleted, taskCompleted],
1678
+ });
1679
+ yield* reactor.handle(childCompleted);
1680
+ if (missedEvent) {
1681
+ // An acknowledged result drops the task from the parent's command projection.
1682
+ yield* orchestrator.dispatch({
1683
+ type: "delegated_task.completion-delivery.acknowledge",
1684
+ commandId: CommandId.make("ack-stale-question-task"),
1685
+ parentThreadId,
1686
+ taskId: task.id,
1687
+ observedByRunId: parentRun.id,
1688
+ });
1689
+ // Persisted terminal state can predate the reactor subscription after restart.
1690
+ yield* projections.apply({
1691
+ id: EventId.make("event:stale-question-parent-finished"),
1692
+ type: "run.updated",
1693
+ threadId: parentThreadId,
1694
+ occurredAt: now,
1695
+ payload: { ...parentRun, status: "completed", completedAt: now },
1696
+ });
1697
+ yield* orchestrator.dispatch({
1698
+ type: "queue.resume",
1699
+ commandId: CommandId.make("resume-stale-question"),
1700
+ threadId: parentThreadId,
1701
+ });
1702
+ } else {
1703
+ yield* reactor.handle(taskCompleted);
1704
+ }
1705
+ assert.equal((yield* wakeRun())?.status, "cancelled");
1706
+ if (missedEvent) {
1707
+ const before = (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.length;
1708
+ const delayedQuestion = {
1709
+ ...question,
1710
+ id: EventId.make("event:delayed-question"),
1711
+ payload: { ...question.payload, id: RuntimeRequestId.make("request:delayed-question") },
1712
+ };
1713
+ yield* eventSink.write({ events: [delayedQuestion] });
1714
+ yield* reactor.handle(delayedQuestion);
1715
+ // A stale delivery is accepted without starting work, not rejected.
1716
+ yield* orchestrator.dispatch({
1717
+ type: "message.dispatch",
1718
+ commandId: CommandId.make("stale-question-redelivery"),
1719
+ threadId: parentThreadId,
1720
+ messageId: MessageId.make("message:stale-question-redelivery"),
1721
+ text: "Answer the delegated question",
1722
+ notification: {
1723
+ source: {
1724
+ kind: "delegated_task",
1725
+ taskIds: [task.id],
1726
+ childThreadId: task.childThreadId!,
1727
+ },
1728
+ outcome: "updated",
1729
+ summary: "Delegated task has a question",
1730
+ },
1731
+ attachments: [],
1732
+ dispatchMode: { type: "queue_after_active" },
1733
+ createdBy: "agent",
1734
+ creationSource: "server",
1735
+ });
1736
+ assert.equal(
1737
+ (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.length,
1738
+ before,
1739
+ );
1740
+ return;
1741
+ }
1742
+
1743
+ yield* orchestrator.dispatch({
1744
+ type: "message.dispatch",
1745
+ commandId: CommandId.make("stale-question-followup"),
1746
+ threadId: task.childThreadId!,
1747
+ messageId: MessageId.make("message:stale-question-followup"),
1748
+ text: "Capture another proof",
1749
+ attachments: [],
1750
+ dispatchMode: { type: "start_immediately" },
1751
+ createdBy: "user",
1752
+ creationSource: "web",
1753
+ });
1754
+ const followup = (yield* projections.getThreadRecords(task.childThreadId!, [
1755
+ "runs",
1756
+ ])).runs.find((run) => run.userMessageId === "message:stale-question-followup")!;
1757
+ const followupQuestion = {
1758
+ ...question,
1759
+ id: EventId.make("event:stale-question-followup"),
1760
+ runId: followup.id,
1761
+ payload: {
1762
+ ...question.payload,
1763
+ id: RuntimeRequestId.make("request:stale-question-followup"),
1764
+ nodeId: followup.rootNodeId!,
1765
+ },
1766
+ };
1767
+ yield* eventSink.write({ events: [followupQuestion] });
1768
+ yield* reactor.handle(followupQuestion);
1769
+ const followupWake = (yield* projections.getThreadRecords(parentThreadId, [
1770
+ "runs",
1771
+ ])).runs.find(
1772
+ (run) =>
1773
+ run.userMessageId ===
1774
+ `message:delegated-task-question:${parentThreadId}:${followupQuestion.payload.id}`,
1775
+ )!;
1776
+ assert.equal(followupWake.status, "queued");
1777
+ yield* reactor.handle(taskCompleted);
1778
+ assert.equal(
1779
+ (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find(
1780
+ (run) => run.id === followupWake.id,
1781
+ )?.status,
1782
+ "queued",
1783
+ );
1784
+ yield* projections.apply({
1785
+ id: EventId.make("event:followup-question-parent-finished"),
1786
+ type: "run.updated",
1787
+ threadId: parentThreadId,
1788
+ occurredAt: now,
1789
+ payload: { ...parentRun, status: "completed", completedAt: now },
1790
+ });
1791
+ yield* orchestrator.dispatch({
1792
+ type: "queue.resume",
1793
+ commandId: CommandId.make("resume-followup-question"),
1794
+ threadId: parentThreadId,
1795
+ });
1796
+ assert.equal(
1797
+ (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find(
1798
+ (run) => run.id === followupWake.id,
1799
+ )?.status,
1800
+ "starting",
1801
+ );
1802
+ }).pipe(Effect.provide(DelegatedTaskQuestionReactor.layer.pipe(Layer.provideMerge(layerTest)))),
1803
+);
1804
+
1805
it.effect.each([false, true])("rejects thread.create for an existing id (deleted: %s)", (deleted) =>
1806
Effect.gen(function* () {
1807
const orchestrator = yield* Orchestrator.OrchestratorV2;
apps/server/src/orchestration-v2/Orchestrator.ts
+94
@@ -1987,6 +1987,85 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1987
]);
1988
});
1989
1990
+ const isStaleQuestionWake = Effect.fn("Orchestrator.isStaleQuestionWake")(function* (
1991
+ projection: Pick<OrchestrationV2ThreadProjection, "thread" | "subagents">,
1992
+ messageId: MessageId,
1993
+ notification: OrchestrationV2ThreadProjection["messages"][number]["notification"],
1994
+ ) {
1995
+ if (notification?.source.kind !== "delegated_task" || notification.outcome !== "updated")
1996
+ return false;
1997
+ const source = notification.source;
1998
+ const threadId = projection.thread.id;
1999
+ const task =
2000
+ projection.subagents.find((candidate) => source.taskIds.includes(candidate.id)) ??
2001
+ // Command projections omit settled tasks whose result was already handed over.
2002
+ (yield* projectionStore
2003
+ .getThreadRecords(threadId, ["subagents"], { subagentIds: source.taskIds })
2004
+ .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))))
2005
+ .subagents[0];
2006
+ // An unknown task gives no evidence the question is obsolete.
2007
+ if (task === undefined) return false;
2008
+ const childThreadId = task.childThreadId;
2009
+ if (
2010
+ childThreadId === null ||
2011
+ ["completed", "failed", "cancelled", "interrupted"].includes(task.status)
2012
+ )
2013
+ return true;
2014
+ const child = yield* projectionStore
2015
+ .getThreadRecords(childThreadId, ["runtimeRequests"])
2016
+ .pipe(
2017
+ Effect.mapError(
2018
+ (cause) => new OrchestratorProjectionError({ threadId: childThreadId, cause }),
2019
+ ),
2020
+ );
2021
+ return !child.runtimeRequests.some(
2022
+ (request) =>
2023
+ request.kind === "user_input" &&
2024
+ request.status === "pending" &&
2025
+ messageId === `message:delegated-task-question:${threadId}:${request.id}`,
2026
+ );
2027
+ });
2028
+
2029
+ const cancelStaleQuestionWakes = Effect.fn("Orchestrator.cancelStaleQuestionWakes")(function* (
2030
+ projection: OrchestrationV2ThreadProjection,
2031
+ ) {
2032
+ const events = yield* Ref.make<Array<OrchestrationV2DomainEvent>>([]);
2033
+ for (const run of projection.runs) {
2034
+ if (run.status !== "queued") continue;
2035
+ const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
2036
+ if (
2037
+ message === undefined ||
2038
+ !(yield* isStaleQuestionWake(projection, message.id, message.notification).pipe(
2039
+ // A child thread read must not fail the parent's unrelated next queued run.
2040
+ Effect.catchTags({
2041
+ OrchestratorProjectionError: (cause) =>
2042
+ Effect.logWarning("Failed to check a queued task question wake", {
2043
+ runId: run.id,
2044
+ cause,
2045
+ }).pipe(Effect.as(false)),
2046
+ }),
2047
+ ))
2048
+ )
2049
+ continue;
2050
+ yield* emitQueuedRunCancellation({
2051
+ command: {
2052
+ type: "queued-run.cancel",
2053
+ commandId: CommandId.make(`command:system:stale-question:${run.id}`),
2054
+ threadId: projection.thread.id,
2055
+ runId: run.id,
2056
+ },
2057
+ events,
2058
+ projection,
2059
+ run,
2060
+ now: yield* DateTime.now,
2061
+ });
2062
+ }
2063
+ const cancellations = yield* Ref.get(events);
2064
+ if (cancellations.length === 0) return false;
2065
+ yield* writeSystemEvents(cancellations);
2066
+ return true;
2067
+ });
2068
+
2069
const startNextQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")(
2070
function* (
2071
threadId: ThreadId,
@@ -2007,6 +2086,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2086
return;
2087
}
2088
2089
+ if (yield* cancelStaleQuestionWakes(projection)) {
2090
+ projection = yield* readCommandProjection(threadId);
2091
+ }
2092
let storedQueuedRun =
2093
options?.queuedRunId === undefined
2094
? nextQueuedRun(projection)
@@ -6085,6 +6167,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6167
effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
6168
) {
6169
let projection = yield* getProjectionWithPendingEvents(command.threadId, events);
6170
+ if (yield* isStaleQuestionWake(projection, command.messageId, command.notification)) {
6171
+ yield* emit(
6172
+ events,
6173
+ command,
6174
+ )({
6175
+ type: "thread.metadata-updated",
6176
+ threadId: command.threadId,
6177
+ occurredAt: yield* DateTime.now,
6178
+ payload: projection.thread,
6179
+ });
6180
+ return;
6181
+ }
6182
if (command.manualContinuationOfRunId !== undefined) {
6183
const source = projection.runs.find((run) => run.id === command.manualContinuationOfRunId);
6184
const limited = latestRootProviderFailure(source ?? null, projection.turnItems);