The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 04:30 UTC
fix(server): preserve sibling wakeups when interrupting a completion wake (#28204)
Co-authored-by: PR Batch Tester <agent@local.test> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
shivam committed
Nov 5, 2026 at 07:01 UTC
4dff7de3f99677dd886018dbc195062c0768f2c7
2 files changed
+287
-7
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+253
@@ -49,6 +49,8 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
49
import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts";
50
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
51
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
52
+import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
53
+import * as ProviderContinuationService from "./ProviderContinuationService.ts";
54
55
const layerPlatformTest = Layer.merge(
56
Layer.mergeAll(NodeServices.layer, NodeCrypto.layer),
@@ -152,6 +154,12 @@ const layerTest = Layer.mergeAll(
154
Layer.provide(layerPlatformTest),
155
);
156
157
+// ProviderContinuationService consumes the orchestrator's continuation queue, as in production.
158
+const layerContinuationWorkerTest = ProviderContinuationService.layer.pipe(
159
+ Layer.provide(Layer.merge(ProviderContinuationRequests.layer, IdAllocator.layer)),
160
+ Layer.provideMerge(layerTest),
161
+);
162
+
163
const seedParentWithTerminalTask = (input: {
164
readonly threadId: ThreadId;
165
readonly projectId: ProjectId;
@@ -1370,6 +1378,251 @@ const seedRestartCancelledChild = (input: {
1378
return { taskId, childThreadId, childRunId };
1379
});
1380
1381
+it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => {
1382
+ it.effect.each([
1383
+ { name: "an interrupt keeps", holdQueue: false, siblingFinishesFirst: false },
1384
+ { name: "Stop disposes", holdQueue: true, siblingFinishesFirst: false },
1385
+ { name: "an interrupt reserves", holdQueue: false, siblingFinishesFirst: true },
1386
+ ])("$name a completion wake's other siblings", ({ holdQueue, siblingFinishesFirst }) =>
1387
+ Effect.gen(function* () {
1388
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
1389
+ const sink = yield* EventSink.EventSinkV2;
1390
+ const now = yield* DateTime.now;
1391
+ const key = holdQueue
1392
+ ? "sibling-stop"
1393
+ : siblingFinishesFirst
1394
+ ? "sibling-finished-interrupt"
1395
+ : "sibling-interrupt";
1396
+ const threadId = ThreadId.make(`${key}-parent`);
1397
+ const originRunId = RunId.make(`${key}-origin`);
1398
+ const incomingTaskId = NodeId.make(`${key}-incoming`);
1399
+ yield* seedParentWithTerminalTask({
1400
+ threadId,
1401
+ projectId: ProjectId.make(`${key}-project`),
1402
+ runId: originRunId,
1403
+ rootNodeId: NodeId.make(`${key}-origin-root`),
1404
+ taskId: incomingTaskId,
1405
+ deliveryState: "claimed",
1406
+ deliveryTaskIds: [incomingTaskId],
1407
+ completionWake: "always",
1408
+ now,
1409
+ });
1410
+ const originRootId = NodeId.make(`${key}-origin-root`);
1411
+ yield* sink.write({
1412
+ events: [
1413
+ {
1414
+ id: EventId.make(`event:${key}-origin-root`),
1415
+ type: "node.updated",
1416
+ threadId,
1417
+ runId: originRunId,
1418
+ nodeId: originRootId,
1419
+ occurredAt: now,
1420
+ payload: {
1421
+ id: originRootId,
1422
+ threadId,
1423
+ runId: originRunId,
1424
+ parentNodeId: null,
1425
+ rootNodeId: originRootId,
1426
+ kind: "root_turn",
1427
+ status: "running",
1428
+ countsForRun: true,
1429
+ providerThreadId: null,
1430
+ providerTurnId: null,
1431
+ nativeItemRef: null,
1432
+ runtimeRequestId: null,
1433
+ checkpointScopeId: null,
1434
+ startedAt: now,
1435
+ completedAt: null,
1436
+ },
1437
+ },
1438
+ ],
1439
+ });
1440
+ yield* orchestrator.dispatch({
1441
+ type: "delegated_task.request",
1442
+ createdBy: "agent",
1443
+ creationSource: "mcp",
1444
+ commandId: CommandId.make(`${key}-sibling`),
1445
+ parentThreadId: threadId,
1446
+ parentRunId: originRunId,
1447
+ parentNodeId: originRootId,
1448
+ task: "Keep working",
1449
+ modelSelection,
1450
+ runtimeMode: "full-access",
1451
+ interactionMode: "default",
1452
+ completionWake: "always",
1453
+ });
1454
+ const seeded = yield* orchestrator.getThreadProjection(threadId);
1455
+ const origin = seeded.runs.find((run) => run.id === originRunId)!;
1456
+ const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!;
1457
+ yield* sink.write({
1458
+ events: [
1459
+ {
1460
+ ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
1461
+ payload: { ...origin, status: "completed", completedAt: now },
1462
+ },
1463
+ ],
1464
+ });
1465
+ const delivery = origin.delegatedCompletion!.delivery!;
1466
+ yield* orchestrator.dispatch({
1467
+ type: "message.dispatch",
1468
+ createdBy: "agent",
1469
+ creationSource: "server",
1470
+ commandId: CommandId.make(`${key}-wake`),
1471
+ threadId,
1472
+ messageId: delivery.messageId,
1473
+ text: "Delegated task completed",
1474
+ attachments: [],
1475
+ modelSelection,
1476
+ dispatchMode: { type: "queue_after_active" },
1477
+ delegatedCompletion: {
1478
+ parentRunId: originRunId,
1479
+ generation: delivery.generation,
1480
+ taskIds: delivery.taskIds,
1481
+ },
1482
+ });
1483
+ yield* orchestrator.resumeQueuedRuns;
1484
+ const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1485
+ (run) => run.userMessageId === delivery.messageId,
1486
+ )!;
1487
+ const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!;
1488
+ const completeSibling = Effect.gen(function* () {
1489
+ const afterSequence = yield* sink.latestSequence();
1490
+ yield* sink.write({
1491
+ events: [
1492
+ {
1493
+ ...runEvent({
1494
+ threadId: sibling.childThreadId!,
1495
+ runId: childRun.id,
1496
+ ordinal: childRun.ordinal,
1497
+ status: "completed",
1498
+ now,
1499
+ }),
1500
+ payload: { ...childRun, status: "completed", completedAt: now },
1501
+ },
1502
+ ],
1503
+ });
1504
+ return yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1505
+ Stream.filter(
1506
+ (stored) =>
1507
+ stored.event.type === "subagent.updated" &&
1508
+ stored.event.payload.id === sibling.id &&
1509
+ stored.event.payload.status === "completed",
1510
+ ),
1511
+ Stream.take(1),
1512
+ Stream.runDrain,
1513
+ Effect.as(afterSequence),
1514
+ );
1515
+ });
1516
+ if (siblingFinishesFirst) {
1517
+ yield* completeSibling;
1518
+ assert.equal(
1519
+ (yield* orchestrator.getThreadProjection(threadId)).subagents.find(
1520
+ (task) => task.id === sibling.id,
1521
+ )?.completionDelivery?.state,
1522
+ "pending",
1523
+ );
1524
+ }
1525
+ const beforeInterrupt = yield* sink.latestSequence();
1526
+ const interrupted = yield* orchestrator.dispatch({
1527
+ type: "run.interrupt",
1528
+ commandId: CommandId.make(`${key}-interrupt`),
1529
+ threadId,
1530
+ runId: wake.id,
1531
+ ...(holdQueue ? { holdQueue } : {}),
1532
+ });
1533
+ // The wake's terminal may claim the sibling right after commit, so read what the interrupt
1534
+ // itself wrote rather than a later projection.
1535
+ const interruptDelivery = new Map(
1536
+ interrupted.storedEvents.flatMap(({ event }) =>
1537
+ event.type === "subagent.updated"
1538
+ ? [[event.payload.id, event.payload.completionDelivery?.state] as const]
1539
+ : [],
1540
+ ),
1541
+ );
1542
+ const interruptCohort = interrupted.storedEvents.flatMap(({ event }) =>
1543
+ event.type === "run.updated" && event.payload.id === originRunId
1544
+ ? [event.payload.delegatedCompletion]
1545
+ : [],
1546
+ );
1547
+ assert.equal(interruptDelivery.get(incomingTaskId), "disposed");
1548
+ assert.equal(interruptCohort.at(-1)?.disposition, holdQueue ? "stopped" : "open");
1549
+ if (holdQueue) {
1550
+ assert.equal(interruptDelivery.get(sibling.id), "disposed");
1551
+ return;
1552
+ }
1553
+ assert.isFalse(interruptDelivery.has(sibling.id));
1554
+
1555
+ const afterSequence = siblingFinishesFirst ? beforeInterrupt : yield* completeSibling;
1556
+ // The wake's terminal, or the sibling's own completion, reserves the successor.
1557
+ yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1558
+ Stream.filter(
1559
+ (stored) =>
1560
+ stored.event.type === "subagent.updated" &&
1561
+ stored.event.payload.id === sibling.id &&
1562
+ stored.event.payload.completionDelivery?.state === "claimed",
1563
+ ),
1564
+ Stream.take(1),
1565
+ Stream.runDrain,
1566
+ );
1567
+ const woken = yield* orchestrator.getThreadProjection(threadId);
1568
+ const successor = woken.runs.find((run) => run.id === originRunId)?.delegatedCompletion
1569
+ ?.delivery;
1570
+ assert.equal(successor?.generation, delivery.generation + 1);
1571
+ assert.deepEqual(successor?.taskIds, [sibling.id]);
1572
+ assert.notEqual(successor?.messageId, delivery.messageId);
1573
+
1574
+ // ProviderContinuationService consumes the claim's offer and dispatches the successor.
1575
+ yield* sink.stream({ afterSequence, eventType: "message.updated" }).pipe(
1576
+ Stream.filter(
1577
+ (stored) =>
1578
+ stored.event.type === "message.updated" &&
1579
+ stored.event.payload.id === successor!.messageId,
1580
+ ),
1581
+ Stream.take(1),
1582
+ Stream.runDrain,
1583
+ );
1584
+ // Repeated recovery re-offers every open delivery; none may wake the parent twice.
1585
+ yield* orchestrator.resumeQueuedRuns;
1586
+ yield* orchestrator.recoverDelegatedTasks;
1587
+ yield* orchestrator.recoverDelegatedTasks;
1588
+ yield* orchestrator.resumeQueuedRuns;
1589
+ const delivered = yield* orchestrator.getThreadProjection(threadId);
1590
+ const cohort = delivered.runs.find((run) => run.id === originRunId)?.delegatedCompletion;
1591
+ assert.equal(cohort?.disposition, "open");
1592
+ assert.equal(cohort?.nextGeneration, delivery.generation + 2);
1593
+ assert.deepEqual(cohort?.delivery, successor);
1594
+ assert.deepEqual(
1595
+ delivered.messages
1596
+ .filter((message) => message.delegatedCompletion?.parentRunId === originRunId)
1597
+ .map((message) => [message.id, message.delegatedCompletion]),
1598
+ [
1599
+ [
1600
+ delivery.messageId,
1601
+ {
1602
+ parentRunId: originRunId,
1603
+ generation: delivery.generation,
1604
+ taskIds: [incomingTaskId],
1605
+ },
1606
+ ],
1607
+ [
1608
+ successor!.messageId,
1609
+ { parentRunId: originRunId, generation: successor!.generation, taskIds: [sibling.id] },
1610
+ ],
1611
+ ],
1612
+ );
1613
+ const successorRuns = delivered.runs.filter(
1614
+ (run) => run.userMessageId === successor!.messageId,
1615
+ );
1616
+ assert.equal(successorRuns.length, 1);
1617
+ assert.equal(successorRuns[0]?.status, "starting");
1618
+ assert.equal(
1619
+ delivered.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
1620
+ "claimed",
1621
+ );
1622
+ }),
1623
+ );
1624
+});
1625
+
1626
it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
1627
"subagent follow-ups",
1628
(it) => {
apps/server/src/orchestration-v2/Orchestrator.ts
+34
-7
@@ -2700,6 +2700,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2700
readonly disposition: "stopped" | "disposed";
2701
readonly now: DateTime.Utc;
2702
readonly cancelQueuedDelivery?: boolean;
2703
+ // An interrupted wake disposes only the results it carried. The cohort stays open, and its
2704
+ // delivery stays the wake's, so the wake's terminal reserves any sibling's result.
2705
+ readonly interruptedWakeTaskIds?: ReadonlyArray<NodeId>;
2706
}) {
2707
const parentRun = input.projection.runs.find((candidate) => candidate.id === input.parentRunId);
2708
if (parentRun === undefined) {
@@ -2713,12 +2716,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2716
return;
2717
}
2718
const emitEvent = emit(input.events, input.command);
2716
- const nextDisposition = cohort?.disposition === "disposed" ? "disposed" : input.disposition;
2717
- const nextCohort = {
2718
- disposition: nextDisposition,
2719
- nextGeneration: cohort?.nextGeneration ?? 1,
2720
- delivery: null,
2721
- } as const;
2719
+ const wakeTaskIds = input.interruptedWakeTaskIds;
2720
+ const keptTaskIds = new Set(
2721
+ wakeTaskIds !== undefined && cohort?.disposition === "open"
2722
+ ? tasks
2723
+ .filter(
2724
+ (task) =>
2725
+ !wakeTaskIds.includes(task.id) &&
2726
+ task.completionDelivery?.state !== "acknowledged" &&
2727
+ task.completionDelivery?.state !== "delivered" &&
2728
+ task.completionDelivery?.state !== "disposed",
2729
+ )
2730
+ .map((task) => task.id)
2731
+ : [],
2732
+ );
2733
+ const nextCohort =
2734
+ keptTaskIds.size > 0
2735
+ ? cohort!
2736
+ : ({
2737
+ disposition: cohort?.disposition === "disposed" ? "disposed" : input.disposition,
2738
+ nextGeneration: cohort?.nextGeneration ?? 1,
2739
+ delivery: null,
2740
+ } as const);
2741
yield* emitEvent({
2742
type: "run.updated",
2743
threadId: parentRun.threadId,
@@ -2733,6 +2752,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2752
});
2753
for (const task of tasks) {
2754
if (
2755
+ keptTaskIds.has(task.id) ||
2756
task.completionDelivery?.state === "acknowledged" ||
2757
task.completionDelivery?.state === "delivered" ||
2758
task.completionDelivery?.state === "disposed"
@@ -2758,7 +2778,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2778
});
2779
}
2780
const deliveryRun = completionDeliveryRun(input.projection, cohort?.delivery ?? null);
2761
- if (input.cancelQueuedDelivery !== false && deliveryRun?.status === "queued") {
2781
+ if (
2782
+ keptTaskIds.size === 0 &&
2783
+ input.cancelQueuedDelivery !== false &&
2784
+ deliveryRun?.status === "queued"
2785
+ ) {
2786
yield* emitQueuedRunCancellation({
2787
command: input.command,
2788
events: input.events,
@@ -11384,6 +11408,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11408
parentRunId,
11409
disposition: "stopped",
11410
now,
11411
+ ...(parentRunId !== run.id && completionMessage?.delegatedCompletion !== undefined
11412
+ ? { interruptedWakeTaskIds: completionMessage.delegatedCompletion.taskIds }
11413
+ : {}),
11414
});
11415
}
11416
});