@devries-cameron20 / t3code Mirror · read-only / commits / 6b96956c30
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 00:45 UTC

fix(server): removing a queued completion message keeps its siblings' wakes (#32815)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Nov 10, 2026 at 05:06 UTC 6b96956c30f1bab808060e2ffd3fa3e481a13685
2 files changed +281 -58
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+270
@@ -1380,6 +1380,145 @@ const seedRestartCancelledChild = (input: {
1380 return { taskId, childThreadId, childRunId };
1381 });
1382
1383 +// Parent spawned two always-wake tasks: the incoming one's completion wake is queued behind a busy
1384 +// run while the sibling's child run is still going.
1385 +const queueCompletionWakeBesideRunningSibling = (key: string) =>
1386 + Effect.gen(function* () {
1387 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1388 + const sink = yield* EventSink.EventSinkV2;
1389 + const now = yield* DateTime.now;
1390 + const threadId = ThreadId.make(`${key}-parent`);
1391 + const originRunId = RunId.make(`${key}-origin`);
1392 + const originRootId = NodeId.make(`${key}-origin-root`);
1393 + const incomingTaskId = NodeId.make(`${key}-incoming`);
1394 + yield* seedParentWithTerminalTask({
1395 + threadId,
1396 + projectId: ProjectId.make(`${key}-project`),
1397 + runId: originRunId,
1398 + rootNodeId: originRootId,
1399 + taskId: incomingTaskId,
1400 + deliveryState: "claimed",
1401 + deliveryTaskIds: [incomingTaskId],
1402 + completionWake: "always",
1403 + now,
1404 + });
1405 + yield* sink.write({
1406 + events: [
1407 + {
1408 + id: EventId.make(`event:${key}-origin-root`),
1409 + type: "node.updated",
1410 + threadId,
1411 + runId: originRunId,
1412 + nodeId: originRootId,
1413 + occurredAt: now,
1414 + payload: {
1415 + id: originRootId,
1416 + threadId,
1417 + runId: originRunId,
1418 + parentNodeId: null,
1419 + rootNodeId: originRootId,
1420 + kind: "root_turn",
1421 + status: "running",
1422 + countsForRun: true,
1423 + providerThreadId: null,
1424 + providerTurnId: null,
1425 + nativeItemRef: null,
1426 + runtimeRequestId: null,
1427 + checkpointScopeId: null,
1428 + startedAt: now,
1429 + completedAt: null,
1430 + },
1431 + },
1432 + ],
1433 + });
1434 + yield* orchestrator.dispatch({
1435 + type: "delegated_task.request",
1436 + createdBy: "agent",
1437 + creationSource: "mcp",
1438 + commandId: CommandId.make(`${key}-sibling`),
1439 + parentThreadId: threadId,
1440 + parentRunId: originRunId,
1441 + parentNodeId: originRootId,
1442 + task: "Keep working",
1443 + modelSelection,
1444 + runtimeMode: "full-access",
1445 + interactionMode: "default",
1446 + completionWake: "always",
1447 + });
1448 + const seeded = yield* orchestrator.getThreadProjection(threadId);
1449 + const origin = seeded.runs.find((run) => run.id === originRunId)!;
1450 + const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!;
1451 + // The coordinator finished its spawning turn but is busy with another run, so the wake queues.
1452 + yield* sink.write({
1453 + events: [
1454 + {
1455 + ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
1456 + payload: { ...origin, status: "completed", completedAt: now },
1457 + },
1458 + runEvent({
1459 + threadId,
1460 + runId: RunId.make(`${key}-busy`),
1461 + ordinal: 2,
1462 + status: "running",
1463 + now,
1464 + }),
1465 + ],
1466 + });
1467 + const delivery = origin.delegatedCompletion!.delivery!;
1468 + yield* orchestrator.dispatch({
1469 + type: "message.dispatch",
1470 + createdBy: "agent",
1471 + creationSource: "server",
1472 + commandId: CommandId.make(`${key}-wake`),
1473 + threadId,
1474 + messageId: delivery.messageId,
1475 + text: "Delegated task completed",
1476 + attachments: [],
1477 + modelSelection,
1478 + dispatchMode: { type: "queue_after_active" },
1479 + delegatedCompletion: {
1480 + parentRunId: originRunId,
1481 + generation: delivery.generation,
1482 + taskIds: delivery.taskIds,
1483 + },
1484 + });
1485 + const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1486 + (run) => run.userMessageId === delivery.messageId,
1487 + )!;
1488 + assert.equal(wake.status, "queued");
1489 + const completeSibling = Effect.gen(function* () {
1490 + const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!;
1491 + const afterSequence = yield* sink.latestSequence();
1492 + yield* sink.write({
1493 + events: [
1494 + {
1495 + ...runEvent({
1496 + threadId: sibling.childThreadId!,
1497 + runId: childRun.id,
1498 + ordinal: childRun.ordinal,
1499 + status: "completed",
1500 + now,
1501 + }),
1502 + payload: { ...childRun, status: "completed", completedAt: now },
1503 + },
1504 + ],
1505 + });
1506 + return afterSequence;
1507 + });
1508 + return {
1509 + orchestrator,
1510 + sink,
1511 + now,
1512 + threadId,
1513 + originRunId,
1514 + incomingTaskId,
1515 + sibling,
1516 + delivery,
1517 + wake,
1518 + completeSibling,
1519 + };
1520 + });
1521 +
1522 it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => {
1523 it.effect.each([
1524 { name: "an interrupt keeps", holdQueue: false, siblingFinishesFirst: false },
@@ -1623,6 +1762,137 @@ it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => {
1762 );
1763 }),
1764 );
1765 +
1766 + it.effect("cancelling a queued completion wake keeps its other siblings", () =>
1767 + Effect.gen(function* () {
1768 + const {
1769 + orchestrator,
1770 + sink,
1771 + threadId,
1772 + originRunId,
1773 + incomingTaskId,
1774 + sibling,
1775 + delivery,
1776 + wake,
1777 + completeSibling,
1778 + } = yield* queueCompletionWakeBesideRunningSibling("sibling-queued-cancel");
1779 + const cancelled = yield* orchestrator.dispatch({
1780 + type: "queued-run.cancel",
1781 + commandId: CommandId.make("sibling-queued-cancel-cancel"),
1782 + threadId,
1783 + runId: wake.id,
1784 + });
1785 + const cancelDelivery = new Map(
1786 + cancelled.storedEvents.flatMap(({ event }) =>
1787 + event.type === "subagent.updated"
1788 + ? [[event.payload.id, event.payload.completionDelivery?.state] as const]
1789 + : [],
1790 + ),
1791 + );
1792 + assert.equal(cancelDelivery.get(incomingTaskId), "disposed");
1793 + assert.notEqual(
1794 + cancelDelivery.get(sibling.id),
1795 + "disposed",
1796 + "cancelling the wake must not dispose a still-running sibling it did not carry",
1797 + );
1798 + const afterCancel = yield* orchestrator.getThreadProjection(threadId);
1799 + assert.equal(
1800 + afterCancel.runs.find((run) => run.id === originRunId)?.delegatedCompletion?.disposition,
1801 + "open",
1802 + );
1803 +
1804 + const afterSequence = yield* completeSibling;
1805 + yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1806 + Stream.filter(
1807 + (stored) =>
1808 + stored.event.type === "subagent.updated" &&
1809 + stored.event.payload.id === sibling.id &&
1810 + stored.event.payload.completionDelivery?.state === "claimed",
1811 + ),
1812 + Stream.take(1),
1813 + Stream.runDrain,
1814 + );
1815 + const successor = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1816 + (run) => run.id === originRunId,
1817 + )?.delegatedCompletion?.delivery;
1818 + assert.deepEqual(successor?.taskIds, [sibling.id]);
1819 + assert.notEqual(successor?.messageId, delivery.messageId);
1820 + }),
1821 + );
1822 +
1823 + it.effect("settling with a queued completion wake disposes its running siblings", () =>
1824 + Effect.gen(function* () {
1825 + const { orchestrator, sink, now, threadId, originRunId, sibling, wake, completeSibling } =
1826 + yield* queueCompletionWakeBesideRunningSibling("sibling-settle");
1827 + const busy = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1828 + (run) => run.status === "running",
1829 + )!;
1830 + yield* sink.write({
1831 + events: [
1832 + {
1833 + ...runEvent({
1834 + threadId,
1835 + runId: busy.id,
1836 + ordinal: busy.ordinal,
1837 + status: "cancelled",
1838 + now,
1839 + }),
1840 + payload: { ...busy, status: "cancelled", completedAt: now },
1841 + },
1842 + {
1843 + ...runEvent({ threadId, runId: wake.id, ordinal: wake.ordinal, status: "queued", now }),
1844 + payload: { ...wake, queueHeld: true },
1845 + },
1846 + ],
1847 + });
1848 +
1849 + const settled = yield* orchestrator.dispatch({
1850 + type: "thread.settle",
1851 + commandId: CommandId.make("sibling-settle-settle"),
1852 + threadId,
1853 + });
1854 + assert.deepEqual(
1855 + settled.storedEvents.flatMap(({ event }) =>
1856 + event.type === "run.updated" && event.payload.id === wake.id
1857 + ? [event.payload.status]
1858 + : [],
1859 + ),
1860 + ["cancelled"],
1861 + );
1862 + const afterSettle = yield* orchestrator.getThreadProjection(threadId);
1863 + assert.equal(
1864 + afterSettle.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
1865 + "disposed",
1866 + );
1867 +
1868 + const afterSequence = yield* completeSibling;
1869 + yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1870 + Stream.filter(
1871 + (stored) =>
1872 + stored.event.type === "subagent.updated" &&
1873 + stored.event.payload.id === sibling.id &&
1874 + stored.event.payload.status === "completed",
1875 + ),
1876 + Stream.take(1),
1877 + Stream.runDrain,
1878 + );
1879 + const afterLateResult = yield* orchestrator.getThreadProjection(threadId);
1880 + assert.equal(
1881 + afterLateResult.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
1882 + "disposed",
1883 + );
1884 + assert.deepEqual(
1885 + afterLateResult.runs.map((run) => run.id),
1886 + afterSettle.runs.map((run) => run.id),
1887 + );
1888 + assert.equal(
1889 + afterLateResult.runs.find((run) => run.id === originRunId)?.delegatedCompletion
1890 + ?.disposition,
1891 + "disposed",
1892 + );
1893 + assert.equal(afterLateResult.thread.settledOverride, "settled");
1894 + }),
1895 + );
1896 });
1897
1898 it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
apps/server/src/orchestration-v2/Orchestrator.ts
+11 -58
@@ -3012,8 +3012,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3012 readonly disposition: "stopped" | "disposed";
3013 readonly now: DateTime.Utc;
3014 readonly cancelQueuedDelivery?: boolean;
3015 - // An interrupted wake disposes only the results it carried. The cohort stays open, and its
3016 - // delivery stays the wake's, so the wake's terminal reserves any sibling's result.
3015 + // An interrupted or cancelled wake disposes only the results it carried. The cohort stays
3016 + // open, and its delivery stays the wake's, so the wake's terminal reserves any sibling's result.
3017 readonly interruptedWakeTaskIds?: ReadonlyArray<NodeId>;
3018 }) {
3019 const parentRun = input.projection.runs.find((candidate) => candidate.id === input.parentRunId);
@@ -3752,6 +3752,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3752 runId: run.id,
3753 },
3754 events,
3755 + "cohort",
3756 );
3757 }
3758 }
@@ -10301,6 +10302,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10302 const dispatchQueuedRunCancel = Effect.fn("Orchestrator.dispatchQueuedRunCancel")(function* (
10303 command: Extract<OrchestrationV2Command, { readonly type: "queued-run.cancel" }>,
10304 events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
10305 + // Settling disposes a cancelled wake's whole cohort, so a sibling's late result cannot unsettle it.
10306 + disposes: "wake" | "cohort" = "wake",
10307 ) {
10308 const projection = yield* loadProjectionForCommand(
10309 command,
@@ -10318,8 +10321,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10321 const queuedMessage = projection.messages.find(
10322 (candidate) => candidate.id === queuedRun.userMessageId,
10323 );
10324 + const now = yield* DateTime.now;
10325 if (queuedMessage?.delegatedCompletion !== undefined) {
10322 - const now = yield* DateTime.now;
10326 yield* disposeDelegatedCompletionCohort({
10327 command,
10328 events,
@@ -10327,64 +10330,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10330 parentRunId: queuedMessage.delegatedCompletion.parentRunId,
10331 disposition: "disposed",
10332 now,
10333 + cancelQueuedDelivery: false,
10334 + ...(disposes === "wake"
10335 + ? { interruptedWakeTaskIds: queuedMessage.delegatedCompletion.taskIds }
10336 + : {}),
10337 });
10331 - return;
10338 }
10333 - const queuedRootNode =
10334 - queuedRun.rootNodeId === null
10335 - ? undefined
10336 - : projection.nodes.find((candidate) => candidate.id === queuedRun.rootNodeId);
10337 - const queuedAttempt =
10338 - queuedRun.activeAttemptId === null
10339 - ? undefined
10340 - : projection.attempts.find((candidate) => candidate.id === queuedRun.activeAttemptId);
10341 -
10342 - const now = yield* DateTime.now;
10339 + yield* emitQueuedRunCancellation({ command, events, projection, run: queuedRun, now });
10340 const emitEvent = emit(events, command);
10344 - yield* emitEvent({
10345 - type: "run.updated",
10346 - threadId: command.threadId,
10347 - runId: queuedRun.id,
10348 - ...(queuedRun.rootNodeId === null ? {} : { nodeId: queuedRun.rootNodeId }),
10349 - providerInstanceId: queuedRun.providerInstanceId,
10350 - occurredAt: now,
10351 - payload: {
10352 - ...queuedRun,
10353 - status: "cancelled",
10354 - queuePosition: null,
10355 - completedAt: now,
10356 - },
10357 - });
10358 - if (queuedAttempt !== undefined && queuedRootNode !== undefined) {
10359 - yield* emitEvent({
10360 - type: "run-attempt.updated",
10361 - threadId: command.threadId,
10362 - runId: queuedRun.id,
10363 - nodeId: queuedRootNode.id,
10364 - providerInstanceId: queuedRun.providerInstanceId,
10365 - occurredAt: now,
10366 - payload: {
10367 - ...queuedAttempt,
10368 - status: "cancelled",
10369 - completedAt: now,
10370 - },
10371 - });
10372 - }
10373 - if (queuedRootNode !== undefined) {
10374 - yield* emitEvent({
10375 - type: "node.updated",
10376 - threadId: command.threadId,
10377 - runId: queuedRun.id,
10378 - nodeId: queuedRootNode.id,
10379 - providerInstanceId: queuedRun.providerInstanceId,
10380 - occurredAt: now,
10381 - payload: {
10382 - ...queuedRootNode,
10383 - status: "cancelled",
10384 - completedAt: now,
10385 - },
10386 - });
10387 - }
10341 const guest = queuedMessage?.context?.guest;
10342 if (guest !== undefined) {
10343 const thread = guestShareAfterCancelledSend(projection.thread, guest, now);