The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 00:45 UTC
fix(server): removing a queued completion message keeps its siblings' wakes (#32815)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Nov 10, 2026 at 05:06 UTC
6b96956c30f1bab808060e2ffd3fa3e481a13685
2 files changed
+281
-58
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+270
@@ -1380,6 +1380,145 @@ const seedRestartCancelledChild = (input: {
1380
return { taskId, childThreadId, childRunId };
1381
});
1382
1383
+// Parent spawned two always-wake tasks: the incoming one's completion wake is queued behind a busy
1384
+// run while the sibling's child run is still going.
1385
+const queueCompletionWakeBesideRunningSibling = (key: string) =>
1386
+ Effect.gen(function* () {
1387
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
1388
+ const sink = yield* EventSink.EventSinkV2;
1389
+ const now = yield* DateTime.now;
1390
+ const threadId = ThreadId.make(`${key}-parent`);
1391
+ const originRunId = RunId.make(`${key}-origin`);
1392
+ const originRootId = NodeId.make(`${key}-origin-root`);
1393
+ const incomingTaskId = NodeId.make(`${key}-incoming`);
1394
+ yield* seedParentWithTerminalTask({
1395
+ threadId,
1396
+ projectId: ProjectId.make(`${key}-project`),
1397
+ runId: originRunId,
1398
+ rootNodeId: originRootId,
1399
+ taskId: incomingTaskId,
1400
+ deliveryState: "claimed",
1401
+ deliveryTaskIds: [incomingTaskId],
1402
+ completionWake: "always",
1403
+ now,
1404
+ });
1405
+ yield* sink.write({
1406
+ events: [
1407
+ {
1408
+ id: EventId.make(`event:${key}-origin-root`),
1409
+ type: "node.updated",
1410
+ threadId,
1411
+ runId: originRunId,
1412
+ nodeId: originRootId,
1413
+ occurredAt: now,
1414
+ payload: {
1415
+ id: originRootId,
1416
+ threadId,
1417
+ runId: originRunId,
1418
+ parentNodeId: null,
1419
+ rootNodeId: originRootId,
1420
+ kind: "root_turn",
1421
+ status: "running",
1422
+ countsForRun: true,
1423
+ providerThreadId: null,
1424
+ providerTurnId: null,
1425
+ nativeItemRef: null,
1426
+ runtimeRequestId: null,
1427
+ checkpointScopeId: null,
1428
+ startedAt: now,
1429
+ completedAt: null,
1430
+ },
1431
+ },
1432
+ ],
1433
+ });
1434
+ yield* orchestrator.dispatch({
1435
+ type: "delegated_task.request",
1436
+ createdBy: "agent",
1437
+ creationSource: "mcp",
1438
+ commandId: CommandId.make(`${key}-sibling`),
1439
+ parentThreadId: threadId,
1440
+ parentRunId: originRunId,
1441
+ parentNodeId: originRootId,
1442
+ task: "Keep working",
1443
+ modelSelection,
1444
+ runtimeMode: "full-access",
1445
+ interactionMode: "default",
1446
+ completionWake: "always",
1447
+ });
1448
+ const seeded = yield* orchestrator.getThreadProjection(threadId);
1449
+ const origin = seeded.runs.find((run) => run.id === originRunId)!;
1450
+ const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!;
1451
+ // The coordinator finished its spawning turn but is busy with another run, so the wake queues.
1452
+ yield* sink.write({
1453
+ events: [
1454
+ {
1455
+ ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
1456
+ payload: { ...origin, status: "completed", completedAt: now },
1457
+ },
1458
+ runEvent({
1459
+ threadId,
1460
+ runId: RunId.make(`${key}-busy`),
1461
+ ordinal: 2,
1462
+ status: "running",
1463
+ now,
1464
+ }),
1465
+ ],
1466
+ });
1467
+ const delivery = origin.delegatedCompletion!.delivery!;
1468
+ yield* orchestrator.dispatch({
1469
+ type: "message.dispatch",
1470
+ createdBy: "agent",
1471
+ creationSource: "server",
1472
+ commandId: CommandId.make(`${key}-wake`),
1473
+ threadId,
1474
+ messageId: delivery.messageId,
1475
+ text: "Delegated task completed",
1476
+ attachments: [],
1477
+ modelSelection,
1478
+ dispatchMode: { type: "queue_after_active" },
1479
+ delegatedCompletion: {
1480
+ parentRunId: originRunId,
1481
+ generation: delivery.generation,
1482
+ taskIds: delivery.taskIds,
1483
+ },
1484
+ });
1485
+ const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1486
+ (run) => run.userMessageId === delivery.messageId,
1487
+ )!;
1488
+ assert.equal(wake.status, "queued");
1489
+ const completeSibling = Effect.gen(function* () {
1490
+ const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!;
1491
+ const afterSequence = yield* sink.latestSequence();
1492
+ yield* sink.write({
1493
+ events: [
1494
+ {
1495
+ ...runEvent({
1496
+ threadId: sibling.childThreadId!,
1497
+ runId: childRun.id,
1498
+ ordinal: childRun.ordinal,
1499
+ status: "completed",
1500
+ now,
1501
+ }),
1502
+ payload: { ...childRun, status: "completed", completedAt: now },
1503
+ },
1504
+ ],
1505
+ });
1506
+ return afterSequence;
1507
+ });
1508
+ return {
1509
+ orchestrator,
1510
+ sink,
1511
+ now,
1512
+ threadId,
1513
+ originRunId,
1514
+ incomingTaskId,
1515
+ sibling,
1516
+ delivery,
1517
+ wake,
1518
+ completeSibling,
1519
+ };
1520
+ });
1521
+
1522
it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => {
1523
it.effect.each([
1524
{ name: "an interrupt keeps", holdQueue: false, siblingFinishesFirst: false },
@@ -1623,6 +1762,137 @@ it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => {
1762
);
1763
}),
1764
);
1765
+
1766
+ it.effect("cancelling a queued completion wake keeps its other siblings", () =>
1767
+ Effect.gen(function* () {
1768
+ const {
1769
+ orchestrator,
1770
+ sink,
1771
+ threadId,
1772
+ originRunId,
1773
+ incomingTaskId,
1774
+ sibling,
1775
+ delivery,
1776
+ wake,
1777
+ completeSibling,
1778
+ } = yield* queueCompletionWakeBesideRunningSibling("sibling-queued-cancel");
1779
+ const cancelled = yield* orchestrator.dispatch({
1780
+ type: "queued-run.cancel",
1781
+ commandId: CommandId.make("sibling-queued-cancel-cancel"),
1782
+ threadId,
1783
+ runId: wake.id,
1784
+ });
1785
+ const cancelDelivery = new Map(
1786
+ cancelled.storedEvents.flatMap(({ event }) =>
1787
+ event.type === "subagent.updated"
1788
+ ? [[event.payload.id, event.payload.completionDelivery?.state] as const]
1789
+ : [],
1790
+ ),
1791
+ );
1792
+ assert.equal(cancelDelivery.get(incomingTaskId), "disposed");
1793
+ assert.notEqual(
1794
+ cancelDelivery.get(sibling.id),
1795
+ "disposed",
1796
+ "cancelling the wake must not dispose a still-running sibling it did not carry",
1797
+ );
1798
+ const afterCancel = yield* orchestrator.getThreadProjection(threadId);
1799
+ assert.equal(
1800
+ afterCancel.runs.find((run) => run.id === originRunId)?.delegatedCompletion?.disposition,
1801
+ "open",
1802
+ );
1803
+
1804
+ const afterSequence = yield* completeSibling;
1805
+ yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1806
+ Stream.filter(
1807
+ (stored) =>
1808
+ stored.event.type === "subagent.updated" &&
1809
+ stored.event.payload.id === sibling.id &&
1810
+ stored.event.payload.completionDelivery?.state === "claimed",
1811
+ ),
1812
+ Stream.take(1),
1813
+ Stream.runDrain,
1814
+ );
1815
+ const successor = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1816
+ (run) => run.id === originRunId,
1817
+ )?.delegatedCompletion?.delivery;
1818
+ assert.deepEqual(successor?.taskIds, [sibling.id]);
1819
+ assert.notEqual(successor?.messageId, delivery.messageId);
1820
+ }),
1821
+ );
1822
+
1823
+ it.effect("settling with a queued completion wake disposes its running siblings", () =>
1824
+ Effect.gen(function* () {
1825
+ const { orchestrator, sink, now, threadId, originRunId, sibling, wake, completeSibling } =
1826
+ yield* queueCompletionWakeBesideRunningSibling("sibling-settle");
1827
+ const busy = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1828
+ (run) => run.status === "running",
1829
+ )!;
1830
+ yield* sink.write({
1831
+ events: [
1832
+ {
1833
+ ...runEvent({
1834
+ threadId,
1835
+ runId: busy.id,
1836
+ ordinal: busy.ordinal,
1837
+ status: "cancelled",
1838
+ now,
1839
+ }),
1840
+ payload: { ...busy, status: "cancelled", completedAt: now },
1841
+ },
1842
+ {
1843
+ ...runEvent({ threadId, runId: wake.id, ordinal: wake.ordinal, status: "queued", now }),
1844
+ payload: { ...wake, queueHeld: true },
1845
+ },
1846
+ ],
1847
+ });
1848
+
1849
+ const settled = yield* orchestrator.dispatch({
1850
+ type: "thread.settle",
1851
+ commandId: CommandId.make("sibling-settle-settle"),
1852
+ threadId,
1853
+ });
1854
+ assert.deepEqual(
1855
+ settled.storedEvents.flatMap(({ event }) =>
1856
+ event.type === "run.updated" && event.payload.id === wake.id
1857
+ ? [event.payload.status]
1858
+ : [],
1859
+ ),
1860
+ ["cancelled"],
1861
+ );
1862
+ const afterSettle = yield* orchestrator.getThreadProjection(threadId);
1863
+ assert.equal(
1864
+ afterSettle.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
1865
+ "disposed",
1866
+ );
1867
+
1868
+ const afterSequence = yield* completeSibling;
1869
+ yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1870
+ Stream.filter(
1871
+ (stored) =>
1872
+ stored.event.type === "subagent.updated" &&
1873
+ stored.event.payload.id === sibling.id &&
1874
+ stored.event.payload.status === "completed",
1875
+ ),
1876
+ Stream.take(1),
1877
+ Stream.runDrain,
1878
+ );
1879
+ const afterLateResult = yield* orchestrator.getThreadProjection(threadId);
1880
+ assert.equal(
1881
+ afterLateResult.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
1882
+ "disposed",
1883
+ );
1884
+ assert.deepEqual(
1885
+ afterLateResult.runs.map((run) => run.id),
1886
+ afterSettle.runs.map((run) => run.id),
1887
+ );
1888
+ assert.equal(
1889
+ afterLateResult.runs.find((run) => run.id === originRunId)?.delegatedCompletion
1890
+ ?.disposition,
1891
+ "disposed",
1892
+ );
1893
+ assert.equal(afterLateResult.thread.settledOverride, "settled");
1894
+ }),
1895
+ );
1896
});
1897
1898
it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
apps/server/src/orchestration-v2/Orchestrator.ts
+11
-58
@@ -3012,8 +3012,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3012
readonly disposition: "stopped" | "disposed";
3013
readonly now: DateTime.Utc;
3014
readonly cancelQueuedDelivery?: boolean;
3015
- // An interrupted wake disposes only the results it carried. The cohort stays open, and its
3016
- // delivery stays the wake's, so the wake's terminal reserves any sibling's result.
3015
+ // An interrupted or cancelled wake disposes only the results it carried. The cohort stays
3016
+ // open, and its delivery stays the wake's, so the wake's terminal reserves any sibling's result.
3017
readonly interruptedWakeTaskIds?: ReadonlyArray<NodeId>;
3018
}) {
3019
const parentRun = input.projection.runs.find((candidate) => candidate.id === input.parentRunId);
@@ -3752,6 +3752,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
3752
runId: run.id,
3753
},
3754
events,
3755
+ "cohort",
3756
);
3757
}
3758
}
@@ -10301,6 +10302,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10302
const dispatchQueuedRunCancel = Effect.fn("Orchestrator.dispatchQueuedRunCancel")(function* (
10303
command: Extract<OrchestrationV2Command, { readonly type: "queued-run.cancel" }>,
10304
events: Ref.Ref<Array<OrchestrationV2DomainEvent>>,
10305
+ // Settling disposes a cancelled wake's whole cohort, so a sibling's late result cannot unsettle it.
10306
+ disposes: "wake" | "cohort" = "wake",
10307
) {
10308
const projection = yield* loadProjectionForCommand(
10309
command,
@@ -10318,8 +10321,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10321
const queuedMessage = projection.messages.find(
10322
(candidate) => candidate.id === queuedRun.userMessageId,
10323
);
10324
+ const now = yield* DateTime.now;
10325
if (queuedMessage?.delegatedCompletion !== undefined) {
10322
- const now = yield* DateTime.now;
10326
yield* disposeDelegatedCompletionCohort({
10327
command,
10328
events,
@@ -10327,64 +10330,14 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
10330
parentRunId: queuedMessage.delegatedCompletion.parentRunId,
10331
disposition: "disposed",
10332
now,
10333
+ cancelQueuedDelivery: false,
10334
+ ...(disposes === "wake"
10335
+ ? { interruptedWakeTaskIds: queuedMessage.delegatedCompletion.taskIds }
10336
+ : {}),
10337
});
10331
- return;
10338
}
10333
- const queuedRootNode =
10334
- queuedRun.rootNodeId === null
10335
- ? undefined
10336
- : projection.nodes.find((candidate) => candidate.id === queuedRun.rootNodeId);
10337
- const queuedAttempt =
10338
- queuedRun.activeAttemptId === null
10339
- ? undefined
10340
- : projection.attempts.find((candidate) => candidate.id === queuedRun.activeAttemptId);
10341
-
10342
- const now = yield* DateTime.now;
10339
+ yield* emitQueuedRunCancellation({ command, events, projection, run: queuedRun, now });
10340
const emitEvent = emit(events, command);
10344
- yield* emitEvent({
10345
- type: "run.updated",
10346
- threadId: command.threadId,
10347
- runId: queuedRun.id,
10348
- ...(queuedRun.rootNodeId === null ? {} : { nodeId: queuedRun.rootNodeId }),
10349
- providerInstanceId: queuedRun.providerInstanceId,
10350
- occurredAt: now,
10351
- payload: {
10352
- ...queuedRun,
10353
- status: "cancelled",
10354
- queuePosition: null,
10355
- completedAt: now,
10356
- },
10357
- });
10358
- if (queuedAttempt !== undefined && queuedRootNode !== undefined) {
10359
- yield* emitEvent({
10360
- type: "run-attempt.updated",
10361
- threadId: command.threadId,
10362
- runId: queuedRun.id,
10363
- nodeId: queuedRootNode.id,
10364
- providerInstanceId: queuedRun.providerInstanceId,
10365
- occurredAt: now,
10366
- payload: {
10367
- ...queuedAttempt,
10368
- status: "cancelled",
10369
- completedAt: now,
10370
- },
10371
- });
10372
- }
10373
- if (queuedRootNode !== undefined) {
10374
- yield* emitEvent({
10375
- type: "node.updated",
10376
- threadId: command.threadId,
10377
- runId: queuedRun.id,
10378
- nodeId: queuedRootNode.id,
10379
- providerInstanceId: queuedRun.providerInstanceId,
10380
- occurredAt: now,
10381
- payload: {
10382
- ...queuedRootNode,
10383
- status: "cancelled",
10384
- completedAt: now,
10385
- },
10386
- });
10387
- }
10341
const guest = queuedMessage?.context?.guest;
10342
if (guest !== undefined) {
10343
const thread = guestShareAfterCancelledSend(projection.thread, guest, now);