@devries-cameron20 / t3code Mirror · read-only / commits / 86c379d07a
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 22:15 UTC

fix(server): drop a finished child's queued question wake (#25680)

Julius Marminge committed Nov 11, 2026 at 15:17 UTC 86c379d07ad8d94ec2e1742e29e3d3201315e272
3 files changed +396 -14
apps/server/src/orchestration-v2/DelegatedTaskQuestionReactor.ts
+74 -14
@@ -4,6 +4,8 @@ import {
4 type OrchestrationV2DomainEvent,
5 type OrchestrationV2RuntimeRequest,
6 type OrchestrationV2Subagent,
7 + type RuntimeRequestId,
8 + type ThreadId,
9 } from "@t3tools/contracts";
10 import * as Context from "effect/Context";
11 import * as Effect from "effect/Effect";
@@ -58,6 +60,37 @@ const make = Effect.gen(function* () {
60 });
61 });
62
63 + const cancelQueuedWakes = Effect.fnUntraced(function* (
64 + parentThreadId: ThreadId,
65 + requestIds: ReadonlyArray<RuntimeRequestId>,
66 + ) {
67 + const parent = yield* projections.getThreadRecords(parentThreadId, ["runs"]);
68 + for (const requestId of requestIds) {
69 + const key = `${parentThreadId}:${requestId}`;
70 + const messageId = MessageId.make(`message:delegated-task-question:${key}`);
71 + const queued = parent.runs.find(
72 + (run) => run.status === "queued" && run.userMessageId === messageId,
73 + );
74 + if (queued === undefined) continue;
75 + yield* engine
76 + .dispatch({
77 + type: "queued-run.cancel",
78 + commandId: CommandId.make(`server:delegated-task-question-cancel:${key}`),
79 + threadId: parentThreadId,
80 + runId: queued.id,
81 + })
82 + .pipe(
83 + Effect.catchCause((cause) =>
84 + Effect.logWarning("cancelling a delegated task question wake failed", {
85 + threadId: parentThreadId,
86 + requestId,
87 + cause,
88 + }),
89 + ),
90 + );
91 + }
92 + });
93 +
94 // Runs for every domain event; most events return before any work is needed.
95 const handle: DelegatedTaskQuestionReactor["Service"]["handle"] = Effect.fnUntraced(
96 function* (event) {
@@ -100,6 +133,46 @@ const make = Effect.gen(function* () {
133 }
134 return;
135 }
136 + if (event.type === "subagent.updated") {
137 + const task = event.payload;
138 + if (
139 + task.origin !== "app_owned" ||
140 + task.childThreadId === null ||
141 + !["completed", "failed", "cancelled", "interrupted"].includes(task.status)
142 + ) {
143 + return;
144 + }
145 + // A settled task can no longer use an answer to its still-pending questions.
146 + const { runtimeRequests } = yield* projections.getThreadRecords(task.childThreadId, [
147 + "runtimeRequests",
148 + ]);
149 + const questions = runtimeRequests.filter((request) => request.kind === "user_input");
150 + if (questions.length === 0) return;
151 + const child = yield* projections.getThreadRecords(task.childThreadId, ["runs", "nodes"]);
152 + const taskRunIds = new Set(
153 + child.runs
154 + .filter(
155 + (run) =>
156 + run.delegatedTaskId === task.id ||
157 + (run.delegatedTaskId === undefined &&
158 + child.thread.forkedFrom?.type === "node" &&
159 + child.thread.forkedFrom.nodeId === task.id),
160 + )
161 + .map((run) => run.id),
162 + );
163 + const taskNodeIds = new Set(
164 + child.nodes
165 + .filter((node) => node.runId !== null && taskRunIds.has(node.runId))
166 + .map((node) => node.id),
167 + );
168 + yield* cancelQueuedWakes(
169 + task.threadId,
170 + questions
171 + .filter((request) => taskNodeIds.has(request.nodeId))
172 + .map((request) => request.id),
173 + );
174 + return;
175 + }
176 if (event.type !== "runtime-request.updated" || event.payload.kind !== "user_input") {
177 return;
178 }
@@ -107,21 +180,8 @@ const make = Effect.gen(function* () {
180 const parentThreadId = child.lineage.parentThreadId;
181 if (parentThreadId === null) return;
182 const request = yield* projections.getRuntimeRequest(event.threadId, event.payload.id);
110 - const key = `${parentThreadId}:${event.payload.id}`;
111 - const messageId = MessageId.make(`message:delegated-task-question:${key}`);
183 if (event.payload.status !== "pending" || request?.status !== "pending") {
113 - const parent = yield* projections.getThreadRecords(parentThreadId, ["runs"]);
114 - const queued = parent.runs.find(
115 - (run) => run.status === "queued" && run.userMessageId === messageId,
116 - );
117 - if (queued !== undefined) {
118 - yield* engine.dispatch({
119 - type: "queued-run.cancel",
120 - commandId: CommandId.make(`server:delegated-task-question-cancel:${key}`),
121 - threadId: parentThreadId,
122 - runId: queued.id,
123 - });
124 - }
184 + yield* cancelQueuedWakes(parentThreadId, [event.payload.id]);
185 return;
186 }
187 const parent = yield* projections.getThreadRecords(parentThreadId, ["subagents", "runs"]);
apps/server/src/orchestration-v2/Orchestrator.control-reads.test.ts
+228
@@ -1574,6 +1574,234 @@ it.effect("wakes the parent once when its delegated task asks a question", () =>
1574 }).pipe(Effect.provide(DelegatedTaskQuestionReactor.layer.pipe(Layer.provideMerge(layerTest)))),
1575 );
1576
1577 +it.effect.each([false, true])(
1578 + "drops a stale question wake (missed terminal event: %s)",
1579 + (missedEvent) =>
1580 + Effect.gen(function* () {
1581 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1582 + const projections = yield* ProjectionStore.ProjectionStoreV2;
1583 + const reactor = yield* DelegatedTaskQuestionReactor.DelegatedTaskQuestionReactor;
1584 + const eventSink = yield* EventSink.EventSinkV2;
1585 + const parentThreadId = ThreadId.make("thread:stale-question-parent");
1586 + yield* orchestrator.dispatch({
1587 + type: "thread.create",
1588 + commandId: CommandId.make("create-stale-question-parent"),
1589 + threadId: parentThreadId,
1590 + projectId: ProjectId.make("project:stale-question"),
1591 + title: "Parent",
1592 + modelSelection,
1593 + runtimeMode: "full-access",
1594 + interactionMode: "default",
1595 + branch: null,
1596 + worktreePath: null,
1597 + createdBy: "user",
1598 + creationSource: "web",
1599 + });
1600 + yield* orchestrator.dispatch({
1601 + type: "message.dispatch",
1602 + commandId: CommandId.make("start-stale-question-parent"),
1603 + threadId: parentThreadId,
1604 + messageId: MessageId.make("message:stale-question-parent"),
1605 + text: "Delegate the proof",
1606 + attachments: [],
1607 + dispatchMode: { type: "start_immediately" },
1608 + createdBy: "user",
1609 + creationSource: "web",
1610 + });
1611 + const parentRun = (yield* projections.getThreadProjection(parentThreadId)).runs[0]!;
1612 + yield* orchestrator.dispatch({
1613 + type: "delegated_task.request",
1614 + commandId: CommandId.make("delegate-stale-question-task"),
1615 + parentThreadId,
1616 + parentRunId: parentRun.id,
1617 + parentNodeId: parentRun.rootNodeId!,
1618 + task: "Capture the proof",
1619 + completionWake: "settled_only",
1620 + modelSelection,
1621 + runtimeMode: "full-access",
1622 + interactionMode: "default",
1623 + createdBy: "agent",
1624 + creationSource: "mcp",
1625 + });
1626 + const task = (yield* projections.getThreadProjection(parentThreadId)).subagents[0]!;
1627 + const childRun = (yield* projections.getThreadRecords(task.childThreadId!, ["runs"]))
1628 + .runs[0]!;
1629 + const now = yield* DateTime.now;
1630 + const question = {
1631 + id: EventId.make("event:stale-question"),
1632 + type: "runtime-request.updated",
1633 + threadId: task.childThreadId!,
1634 + runId: childRun.id,
1635 + occurredAt: now,
1636 + payload: {
1637 + id: RuntimeRequestId.make("request:stale-question"),
1638 + nodeId: childRun.rootNodeId!,
1639 + providerTurnId: null,
1640 + nativeRequestRef: null,
1641 + kind: "user_input",
1642 + status: "pending",
1643 + responseCapability: { type: "message" },
1644 + createdAt: now,
1645 + resolvedAt: null,
1646 + },
1647 + } satisfies OrchestrationV2DomainEvent;
1648 + yield* eventSink.write({ events: [question] });
1649 + yield* reactor.handle(question);
1650 + const wake = (yield* projections.getThreadRecords(parentThreadId, [
1651 + "messages",
1652 + ])).messages.find((message) => message.notification !== undefined)!;
1653 + const wakeRun = () =>
1654 + projections
1655 + .getThreadRecords(parentThreadId, ["runs"])
1656 + .pipe(Effect.map(({ runs }) => runs.find((run) => run.userMessageId === wake.id)));
1657 + assert.equal((yield* wakeRun())?.status, "queued");
1658 +
1659 + // The child finishes without ever resolving its message-capability question.
1660 + const childCompleted = {
1661 + id: EventId.make("event:stale-question-child-completed"),
1662 + type: "run.updated",
1663 + threadId: task.childThreadId!,
1664 + runId: childRun.id,
1665 + occurredAt: now,
1666 + payload: { ...childRun, status: "completed", completedAt: now },
1667 + } satisfies OrchestrationV2DomainEvent;
1668 + const taskCompleted = {
1669 + id: EventId.make("event:stale-question-task-completed"),
1670 + type: "subagent.updated",
1671 + threadId: parentThreadId,
1672 + occurredAt: now,
1673 + payload: { ...task, status: "completed", result: "Proof captured.", completedAt: now },
1674 + } satisfies OrchestrationV2DomainEvent;
1675 + yield* eventSink.write({
1676 + commandId: CommandId.make("command:runtime-reconcile:stale-question"),
1677 + events: [childCompleted, taskCompleted],
1678 + });
1679 + yield* reactor.handle(childCompleted);
1680 + if (missedEvent) {
1681 + // An acknowledged result drops the task from the parent's command projection.
1682 + yield* orchestrator.dispatch({
1683 + type: "delegated_task.completion-delivery.acknowledge",
1684 + commandId: CommandId.make("ack-stale-question-task"),
1685 + parentThreadId,
1686 + taskId: task.id,
1687 + observedByRunId: parentRun.id,
1688 + });
1689 + // Persisted terminal state can predate the reactor subscription after restart.
1690 + yield* projections.apply({
1691 + id: EventId.make("event:stale-question-parent-finished"),
1692 + type: "run.updated",
1693 + threadId: parentThreadId,
1694 + occurredAt: now,
1695 + payload: { ...parentRun, status: "completed", completedAt: now },
1696 + });
1697 + yield* orchestrator.dispatch({
1698 + type: "queue.resume",
1699 + commandId: CommandId.make("resume-stale-question"),
1700 + threadId: parentThreadId,
1701 + });
1702 + } else {
1703 + yield* reactor.handle(taskCompleted);
1704 + }
1705 + assert.equal((yield* wakeRun())?.status, "cancelled");
1706 + if (missedEvent) {
1707 + const before = (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.length;
1708 + const delayedQuestion = {
1709 + ...question,
1710 + id: EventId.make("event:delayed-question"),
1711 + payload: { ...question.payload, id: RuntimeRequestId.make("request:delayed-question") },
1712 + };
1713 + yield* eventSink.write({ events: [delayedQuestion] });
1714 + yield* reactor.handle(delayedQuestion);
1715 + // A stale delivery is accepted without starting work, not rejected.
1716 + yield* orchestrator.dispatch({
1717 + type: "message.dispatch",
1718 + commandId: CommandId.make("stale-question-redelivery"),
1719 + threadId: parentThreadId,
1720 + messageId: MessageId.make("message:stale-question-redelivery"),
1721 + text: "Answer the delegated question",
1722 + notification: {
1723 + source: {
1724 + kind: "delegated_task",
1725 + taskIds: [task.id],
1726 + childThreadId: task.childThreadId!,
1727 + },
1728 + outcome: "updated",
1729 + summary: "Delegated task has a question",
1730 + },
1731 + attachments: [],
1732 + dispatchMode: { type: "queue_after_active" },
1733 + createdBy: "agent",
1734 + creationSource: "server",
1735 + });
1736 + assert.equal(
1737 + (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.length,
1738 + before,
1739 + );
1740 + return;
1741 + }
1742 +
1743 + yield* orchestrator.dispatch({
1744 + type: "message.dispatch",
1745 + commandId: CommandId.make("stale-question-followup"),
1746 + threadId: task.childThreadId!,
1747 + messageId: MessageId.make("message:stale-question-followup"),
1748 + text: "Capture another proof",
1749 + attachments: [],
1750 + dispatchMode: { type: "start_immediately" },
1751 + createdBy: "user",
1752 + creationSource: "web",
1753 + });
1754 + const followup = (yield* projections.getThreadRecords(task.childThreadId!, [
1755 + "runs",
1756 + ])).runs.find((run) => run.userMessageId === "message:stale-question-followup")!;
1757 + const followupQuestion = {
1758 + ...question,
1759 + id: EventId.make("event:stale-question-followup"),
1760 + runId: followup.id,
1761 + payload: {
1762 + ...question.payload,
1763 + id: RuntimeRequestId.make("request:stale-question-followup"),
1764 + nodeId: followup.rootNodeId!,
1765 + },
1766 + };
1767 + yield* eventSink.write({ events: [followupQuestion] });
1768 + yield* reactor.handle(followupQuestion);
1769 + const followupWake = (yield* projections.getThreadRecords(parentThreadId, [
1770 + "runs",
1771 + ])).runs.find(
1772 + (run) =>
1773 + run.userMessageId ===
1774 + `message:delegated-task-question:${parentThreadId}:${followupQuestion.payload.id}`,
1775 + )!;
1776 + assert.equal(followupWake.status, "queued");
1777 + yield* reactor.handle(taskCompleted);
1778 + assert.equal(
1779 + (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find(
1780 + (run) => run.id === followupWake.id,
1781 + )?.status,
1782 + "queued",
1783 + );
1784 + yield* projections.apply({
1785 + id: EventId.make("event:followup-question-parent-finished"),
1786 + type: "run.updated",
1787 + threadId: parentThreadId,
1788 + occurredAt: now,
1789 + payload: { ...parentRun, status: "completed", completedAt: now },
1790 + });
1791 + yield* orchestrator.dispatch({
1792 + type: "queue.resume",
1793 + commandId: CommandId.make("resume-followup-question"),
1794 + threadId: parentThreadId,
1795 + });
1796 + assert.equal(
1797 + (yield* projections.getThreadRecords(parentThreadId, ["runs"])).runs.find(
1798 + (run) => run.id === followupWake.id,
1799 + )?.status,
1800 + "starting",
1801 + );
1802 + }).pipe(Effect.provide(DelegatedTaskQuestionReactor.layer.pipe(Layer.provideMerge(layerTest)))),
1803 +);
1804 +
1805 it.effect.each([false, true])("rejects thread.create for an existing id (deleted: %s)", (deleted) =>
1806 Effect.gen(function* () {
1807 const orchestrator = yield* Orchestrator.OrchestratorV2;
apps/server/src/orchestration-v2/Orchestrator.ts
+94
@@ -1987,6 +1987,85 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1987 ]);
1988 });
1989
1990 + const isStaleQuestionWake = Effect.fn("Orchestrator.isStaleQuestionWake")(function* (
1991 + projection: Pick<OrchestrationV2ThreadProjection, "thread" | "subagents">,
1992 + messageId: MessageId,
1993 + notification: OrchestrationV2ThreadProjection["messages"][number]["notification"],
1994 + ) {
1995 + if (notification?.source.kind !== "delegated_task" || notification.outcome !== "updated")
1996 + return false;
1997 + const source = notification.source;
1998 + const threadId = projection.thread.id;
1999 + const task =
2000 + projection.subagents.find((candidate) => source.taskIds.includes(candidate.id)) ??
2001 + // Command projections omit settled tasks whose result was already handed over.
2002 + (yield* projectionStore
2003 + .getThreadRecords(threadId, ["subagents"], { subagentIds: source.taskIds })
2004 + .pipe(Effect.mapError((cause) => new OrchestratorProjectionError({ threadId, cause }))))
2005 + .subagents[0];
2006 + // An unknown task gives no evidence the question is obsolete.
2007 + if (task === undefined) return false;
2008 + const childThreadId = task.childThreadId;
2009 + if (
2010 + childThreadId === null ||
2011 + ["completed", "failed", "cancelled", "interrupted"].includes(task.status)
2012 + )
2013 + return true;
2014 + const child = yield* projectionStore
2015 + .getThreadRecords(childThreadId, ["runtimeRequests"])
2016 + .pipe(
2017 + Effect.mapError(
2018 + (cause) => new OrchestratorProjectionError({ threadId: childThreadId, cause }),
2019 + ),
2020 + );
2021 + return !child.runtimeRequests.some(
2022 + (request) =>
2023 + request.kind === "user_input" &&
2024 + request.status === "pending" &&
2025 + messageId === `message:delegated-task-question:${threadId}:${request.id}`,
2026 + );
2027 + });
2028 +
2029 + const cancelStaleQuestionWakes = Effect.fn("Orchestrator.cancelStaleQuestionWakes")(function* (
2030 + projection: OrchestrationV2ThreadProjection,
2031 + ) {
2032 + const events = yield* Ref.make<Array<OrchestrationV2DomainEvent>>([]);
2033 + for (const run of projection.runs) {
2034 + if (run.status !== "queued") continue;
2035 + const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
2036 + if (
2037 + message === undefined ||
2038 + !(yield* isStaleQuestionWake(projection, message.id, message.notification).pipe(
2039 + // A child thread read must not fail the parent's unrelated next queued run.
2040 + Effect.catchTags({
2041 + OrchestratorProjectionError: (cause) =>
2042 + Effect.logWarning("Failed to check a queued task question wake", {
2043 + runId: run.id,
2044 + cause,
2045 + }).pipe(Effect.as(false)),
2046 + }),
2047 + ))
2048 + )
2049 + continue;
2050 + yield* emitQueuedRunCancellation({
2051 + command: {
2052 + type: "queued-run.cancel",
2053 + commandId: CommandId.make(`command:system:stale-question:${run.id}`),
2054 + threadId: projection.thread.id,
2055 + runId: run.id,
2056 + },
2057 + events,
2058 + projection,
2059 + run,
2060 + now: yield* DateTime.now,
2061 + });
2062 + }
2063 + const cancellations = yield* Ref.get(events);
2064 + if (cancellations.length === 0) return false;
2065 + yield* writeSystemEvents(cancellations);
2066 + return true;
2067 + });
2068 +
2069 const startNextQueuedRun = Effect.fn("Orchestrator.startNextQueuedRun")(
2070 function* (
2071 threadId: ThreadId,
@@ -2007,6 +2086,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2086 return;
2087 }
2088
2089 + if (yield* cancelStaleQuestionWakes(projection)) {
2090 + projection = yield* readCommandProjection(threadId);
2091 + }
2092 let storedQueuedRun =
2093 options?.queuedRunId === undefined
2094 ? nextQueuedRun(projection)
@@ -6085,6 +6167,18 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
6167 effects: Ref.Ref<Array<PendingOrchestrationEffectV2>>,
6168 ) {
6169 let projection = yield* getProjectionWithPendingEvents(command.threadId, events);
6170 + if (yield* isStaleQuestionWake(projection, command.messageId, command.notification)) {
6171 + yield* emit(
6172 + events,
6173 + command,
6174 + )({
6175 + type: "thread.metadata-updated",
6176 + threadId: command.threadId,
6177 + occurredAt: yield* DateTime.now,
6178 + payload: projection.thread,
6179 + });
6180 + return;
6181 + }
6182 if (command.manualContinuationOfRunId !== undefined) {
6183 const source = projection.runs.find((run) => run.id === command.manualContinuationOfRunId);
6184 const limited = latestRootProviderFailure(source ?? null, projection.turnItems);