@devries-cameron20 / t3code Mirror · read-only / commits / 4dff7de3f9
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 04:30 UTC

fix(server): preserve sibling wakeups when interrupting a completion wake (#28204)

Co-authored-by: PR Batch Tester <agent@local.test> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

shivam committed Nov 5, 2026 at 07:01 UTC 4dff7de3f99677dd886018dbc195062c0768f2c7
2 files changed +287 -7
apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+253
@@ -49,6 +49,8 @@ import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
49 import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts";
50 import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
51 import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
52 +import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator";
53 +import * as ProviderContinuationService from "./ProviderContinuationService.ts";
54
55 const layerPlatformTest = Layer.merge(
56 Layer.mergeAll(NodeServices.layer, NodeCrypto.layer),
@@ -152,6 +154,12 @@ const layerTest = Layer.mergeAll(
154 Layer.provide(layerPlatformTest),
155 );
156
157 +// ProviderContinuationService consumes the orchestrator's continuation queue, as in production.
158 +const layerContinuationWorkerTest = ProviderContinuationService.layer.pipe(
159 + Layer.provide(Layer.merge(ProviderContinuationRequests.layer, IdAllocator.layer)),
160 + Layer.provideMerge(layerTest),
161 +);
162 +
163 const seedParentWithTerminalTask = (input: {
164 readonly threadId: ThreadId;
165 readonly projectId: ProjectId;
@@ -1370,6 +1378,251 @@ const seedRestartCancelledChild = (input: {
1378 return { taskId, childThreadId, childRunId };
1379 });
1380
1381 +it.layer(layerContinuationWorkerTest)("interrupted completion wakes", (it) => {
1382 + it.effect.each([
1383 + { name: "an interrupt keeps", holdQueue: false, siblingFinishesFirst: false },
1384 + { name: "Stop disposes", holdQueue: true, siblingFinishesFirst: false },
1385 + { name: "an interrupt reserves", holdQueue: false, siblingFinishesFirst: true },
1386 + ])("$name a completion wake's other siblings", ({ holdQueue, siblingFinishesFirst }) =>
1387 + Effect.gen(function* () {
1388 + const orchestrator = yield* Orchestrator.OrchestratorV2;
1389 + const sink = yield* EventSink.EventSinkV2;
1390 + const now = yield* DateTime.now;
1391 + const key = holdQueue
1392 + ? "sibling-stop"
1393 + : siblingFinishesFirst
1394 + ? "sibling-finished-interrupt"
1395 + : "sibling-interrupt";
1396 + const threadId = ThreadId.make(`${key}-parent`);
1397 + const originRunId = RunId.make(`${key}-origin`);
1398 + const incomingTaskId = NodeId.make(`${key}-incoming`);
1399 + yield* seedParentWithTerminalTask({
1400 + threadId,
1401 + projectId: ProjectId.make(`${key}-project`),
1402 + runId: originRunId,
1403 + rootNodeId: NodeId.make(`${key}-origin-root`),
1404 + taskId: incomingTaskId,
1405 + deliveryState: "claimed",
1406 + deliveryTaskIds: [incomingTaskId],
1407 + completionWake: "always",
1408 + now,
1409 + });
1410 + const originRootId = NodeId.make(`${key}-origin-root`);
1411 + yield* sink.write({
1412 + events: [
1413 + {
1414 + id: EventId.make(`event:${key}-origin-root`),
1415 + type: "node.updated",
1416 + threadId,
1417 + runId: originRunId,
1418 + nodeId: originRootId,
1419 + occurredAt: now,
1420 + payload: {
1421 + id: originRootId,
1422 + threadId,
1423 + runId: originRunId,
1424 + parentNodeId: null,
1425 + rootNodeId: originRootId,
1426 + kind: "root_turn",
1427 + status: "running",
1428 + countsForRun: true,
1429 + providerThreadId: null,
1430 + providerTurnId: null,
1431 + nativeItemRef: null,
1432 + runtimeRequestId: null,
1433 + checkpointScopeId: null,
1434 + startedAt: now,
1435 + completedAt: null,
1436 + },
1437 + },
1438 + ],
1439 + });
1440 + yield* orchestrator.dispatch({
1441 + type: "delegated_task.request",
1442 + createdBy: "agent",
1443 + creationSource: "mcp",
1444 + commandId: CommandId.make(`${key}-sibling`),
1445 + parentThreadId: threadId,
1446 + parentRunId: originRunId,
1447 + parentNodeId: originRootId,
1448 + task: "Keep working",
1449 + modelSelection,
1450 + runtimeMode: "full-access",
1451 + interactionMode: "default",
1452 + completionWake: "always",
1453 + });
1454 + const seeded = yield* orchestrator.getThreadProjection(threadId);
1455 + const origin = seeded.runs.find((run) => run.id === originRunId)!;
1456 + const sibling = seeded.subagents.find((task) => task.id !== incomingTaskId)!;
1457 + yield* sink.write({
1458 + events: [
1459 + {
1460 + ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }),
1461 + payload: { ...origin, status: "completed", completedAt: now },
1462 + },
1463 + ],
1464 + });
1465 + const delivery = origin.delegatedCompletion!.delivery!;
1466 + yield* orchestrator.dispatch({
1467 + type: "message.dispatch",
1468 + createdBy: "agent",
1469 + creationSource: "server",
1470 + commandId: CommandId.make(`${key}-wake`),
1471 + threadId,
1472 + messageId: delivery.messageId,
1473 + text: "Delegated task completed",
1474 + attachments: [],
1475 + modelSelection,
1476 + dispatchMode: { type: "queue_after_active" },
1477 + delegatedCompletion: {
1478 + parentRunId: originRunId,
1479 + generation: delivery.generation,
1480 + taskIds: delivery.taskIds,
1481 + },
1482 + });
1483 + yield* orchestrator.resumeQueuedRuns;
1484 + const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find(
1485 + (run) => run.userMessageId === delivery.messageId,
1486 + )!;
1487 + const childRun = (yield* orchestrator.getThreadProjection(sibling.childThreadId!)).runs[0]!;
1488 + const completeSibling = Effect.gen(function* () {
1489 + const afterSequence = yield* sink.latestSequence();
1490 + yield* sink.write({
1491 + events: [
1492 + {
1493 + ...runEvent({
1494 + threadId: sibling.childThreadId!,
1495 + runId: childRun.id,
1496 + ordinal: childRun.ordinal,
1497 + status: "completed",
1498 + now,
1499 + }),
1500 + payload: { ...childRun, status: "completed", completedAt: now },
1501 + },
1502 + ],
1503 + });
1504 + return yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1505 + Stream.filter(
1506 + (stored) =>
1507 + stored.event.type === "subagent.updated" &&
1508 + stored.event.payload.id === sibling.id &&
1509 + stored.event.payload.status === "completed",
1510 + ),
1511 + Stream.take(1),
1512 + Stream.runDrain,
1513 + Effect.as(afterSequence),
1514 + );
1515 + });
1516 + if (siblingFinishesFirst) {
1517 + yield* completeSibling;
1518 + assert.equal(
1519 + (yield* orchestrator.getThreadProjection(threadId)).subagents.find(
1520 + (task) => task.id === sibling.id,
1521 + )?.completionDelivery?.state,
1522 + "pending",
1523 + );
1524 + }
1525 + const beforeInterrupt = yield* sink.latestSequence();
1526 + const interrupted = yield* orchestrator.dispatch({
1527 + type: "run.interrupt",
1528 + commandId: CommandId.make(`${key}-interrupt`),
1529 + threadId,
1530 + runId: wake.id,
1531 + ...(holdQueue ? { holdQueue } : {}),
1532 + });
1533 + // The wake's terminal may claim the sibling right after commit, so read what the interrupt
1534 + // itself wrote rather than a later projection.
1535 + const interruptDelivery = new Map(
1536 + interrupted.storedEvents.flatMap(({ event }) =>
1537 + event.type === "subagent.updated"
1538 + ? [[event.payload.id, event.payload.completionDelivery?.state] as const]
1539 + : [],
1540 + ),
1541 + );
1542 + const interruptCohort = interrupted.storedEvents.flatMap(({ event }) =>
1543 + event.type === "run.updated" && event.payload.id === originRunId
1544 + ? [event.payload.delegatedCompletion]
1545 + : [],
1546 + );
1547 + assert.equal(interruptDelivery.get(incomingTaskId), "disposed");
1548 + assert.equal(interruptCohort.at(-1)?.disposition, holdQueue ? "stopped" : "open");
1549 + if (holdQueue) {
1550 + assert.equal(interruptDelivery.get(sibling.id), "disposed");
1551 + return;
1552 + }
1553 + assert.isFalse(interruptDelivery.has(sibling.id));
1554 +
1555 + const afterSequence = siblingFinishesFirst ? beforeInterrupt : yield* completeSibling;
1556 + // The wake's terminal, or the sibling's own completion, reserves the successor.
1557 + yield* sink.stream({ afterSequence, eventType: "subagent.updated" }).pipe(
1558 + Stream.filter(
1559 + (stored) =>
1560 + stored.event.type === "subagent.updated" &&
1561 + stored.event.payload.id === sibling.id &&
1562 + stored.event.payload.completionDelivery?.state === "claimed",
1563 + ),
1564 + Stream.take(1),
1565 + Stream.runDrain,
1566 + );
1567 + const woken = yield* orchestrator.getThreadProjection(threadId);
1568 + const successor = woken.runs.find((run) => run.id === originRunId)?.delegatedCompletion
1569 + ?.delivery;
1570 + assert.equal(successor?.generation, delivery.generation + 1);
1571 + assert.deepEqual(successor?.taskIds, [sibling.id]);
1572 + assert.notEqual(successor?.messageId, delivery.messageId);
1573 +
1574 + // ProviderContinuationService consumes the claim's offer and dispatches the successor.
1575 + yield* sink.stream({ afterSequence, eventType: "message.updated" }).pipe(
1576 + Stream.filter(
1577 + (stored) =>
1578 + stored.event.type === "message.updated" &&
1579 + stored.event.payload.id === successor!.messageId,
1580 + ),
1581 + Stream.take(1),
1582 + Stream.runDrain,
1583 + );
1584 + // Repeated recovery re-offers every open delivery; none may wake the parent twice.
1585 + yield* orchestrator.resumeQueuedRuns;
1586 + yield* orchestrator.recoverDelegatedTasks;
1587 + yield* orchestrator.recoverDelegatedTasks;
1588 + yield* orchestrator.resumeQueuedRuns;
1589 + const delivered = yield* orchestrator.getThreadProjection(threadId);
1590 + const cohort = delivered.runs.find((run) => run.id === originRunId)?.delegatedCompletion;
1591 + assert.equal(cohort?.disposition, "open");
1592 + assert.equal(cohort?.nextGeneration, delivery.generation + 2);
1593 + assert.deepEqual(cohort?.delivery, successor);
1594 + assert.deepEqual(
1595 + delivered.messages
1596 + .filter((message) => message.delegatedCompletion?.parentRunId === originRunId)
1597 + .map((message) => [message.id, message.delegatedCompletion]),
1598 + [
1599 + [
1600 + delivery.messageId,
1601 + {
1602 + parentRunId: originRunId,
1603 + generation: delivery.generation,
1604 + taskIds: [incomingTaskId],
1605 + },
1606 + ],
1607 + [
1608 + successor!.messageId,
1609 + { parentRunId: originRunId, generation: successor!.generation, taskIds: [sibling.id] },
1610 + ],
1611 + ],
1612 + );
1613 + const successorRuns = delivered.runs.filter(
1614 + (run) => run.userMessageId === successor!.messageId,
1615 + );
1616 + assert.equal(successorRuns.length, 1);
1617 + assert.equal(successorRuns[0]?.status, "starting");
1618 + assert.equal(
1619 + delivered.subagents.find((task) => task.id === sibling.id)?.completionDelivery?.state,
1620 + "claimed",
1621 + );
1622 + }),
1623 + );
1624 +});
1625 +
1626 it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
1627 "subagent follow-ups",
1628 (it) => {
apps/server/src/orchestration-v2/Orchestrator.ts
+34 -7
@@ -2700,6 +2700,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2700 readonly disposition: "stopped" | "disposed";
2701 readonly now: DateTime.Utc;
2702 readonly cancelQueuedDelivery?: boolean;
2703 + // An interrupted wake disposes only the results it carried. The cohort stays open, and its
2704 + // delivery stays the wake's, so the wake's terminal reserves any sibling's result.
2705 + readonly interruptedWakeTaskIds?: ReadonlyArray<NodeId>;
2706 }) {
2707 const parentRun = input.projection.runs.find((candidate) => candidate.id === input.parentRunId);
2708 if (parentRun === undefined) {
@@ -2713,12 +2716,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2716 return;
2717 }
2718 const emitEvent = emit(input.events, input.command);
2716 - const nextDisposition = cohort?.disposition === "disposed" ? "disposed" : input.disposition;
2717 - const nextCohort = {
2718 - disposition: nextDisposition,
2719 - nextGeneration: cohort?.nextGeneration ?? 1,
2720 - delivery: null,
2721 - } as const;
2719 + const wakeTaskIds = input.interruptedWakeTaskIds;
2720 + const keptTaskIds = new Set(
2721 + wakeTaskIds !== undefined && cohort?.disposition === "open"
2722 + ? tasks
2723 + .filter(
2724 + (task) =>
2725 + !wakeTaskIds.includes(task.id) &&
2726 + task.completionDelivery?.state !== "acknowledged" &&
2727 + task.completionDelivery?.state !== "delivered" &&
2728 + task.completionDelivery?.state !== "disposed",
2729 + )
2730 + .map((task) => task.id)
2731 + : [],
2732 + );
2733 + const nextCohort =
2734 + keptTaskIds.size > 0
2735 + ? cohort!
2736 + : ({
2737 + disposition: cohort?.disposition === "disposed" ? "disposed" : input.disposition,
2738 + nextGeneration: cohort?.nextGeneration ?? 1,
2739 + delivery: null,
2740 + } as const);
2741 yield* emitEvent({
2742 type: "run.updated",
2743 threadId: parentRun.threadId,
@@ -2733,6 +2752,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2752 });
2753 for (const task of tasks) {
2754 if (
2755 + keptTaskIds.has(task.id) ||
2756 task.completionDelivery?.state === "acknowledged" ||
2757 task.completionDelivery?.state === "delivered" ||
2758 task.completionDelivery?.state === "disposed"
@@ -2758,7 +2778,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2778 });
2779 }
2780 const deliveryRun = completionDeliveryRun(input.projection, cohort?.delivery ?? null);
2761 - if (input.cancelQueuedDelivery !== false && deliveryRun?.status === "queued") {
2781 + if (
2782 + keptTaskIds.size === 0 &&
2783 + input.cancelQueuedDelivery !== false &&
2784 + deliveryRun?.status === "queued"
2785 + ) {
2786 yield* emitQueuedRunCancellation({
2787 command: input.command,
2788 events: input.events,
@@ -11384,6 +11408,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11408 parentRunId,
11409 disposition: "stopped",
11410 now,
11411 + ...(parentRunId !== run.id && completionMessage?.delegatedCompletion !== undefined
11412 + ? { interruptedWakeTaskIds: completionMessage.delegatedCompletion.taskIds }
11413 + : {}),
11414 });
11415 }
11416 });