@devries-cameron20 / t3code Mirror · read-only / commits / f8bcc3cb06
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 22:00 UTC

fix(server): queued PR updates follow the takeover provider (#21314)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Lakshmi Tanmay committed Oct 20, 2026 at 11:02 UTC f8bcc3cb066999e155dcd94d0f39adb9e2496486
3 files changed +671 -68
apps/server/src/orchestration-v2/Orchestrator.ts
+166 -63
@@ -638,6 +638,45 @@ function nextQueuedRun(
638 return queuedRunsInDeliveryOrder(projection)[0];
639 }
640
641 +function followsCurrentConversation(
642 + message: OrchestrationV2ConversationMessage | undefined,
643 +): boolean {
644 + return (
645 + message?.notification?.source.kind === "monitor" && !isBufferedProviderNotification(message)
646 + );
647 +}
648 +
649 +function queuedRunModelSelection(
650 + projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
651 + run: OrchestrationV2Run,
652 +): ModelSelection {
653 + const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
654 + // App-owned monitor updates follow the agent currently handling the thread.
655 + return followsCurrentConversation(message)
656 + ? projection.thread.modelSelection
657 + : run.modelSelection;
658 +}
659 +
660 +function usageLimitHoldsQueue(
661 + projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems" | "providerSessions">,
662 + providerInstanceId: ModelSelection["instanceId"],
663 +): boolean {
664 + let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
665 + for (const session of projection.providerSessions) {
666 + if (
667 + session.providerInstanceId === providerInstanceId &&
668 + (latestSession === undefined ||
669 + DateTime.isGreaterThan(session.updatedAt, latestSession.updatedAt))
670 + ) {
671 + latestSession = session;
672 + }
673 + }
674 + return (
675 + usageLimitBlockedRun(projection.runs, projection.turnItems, latestSession?.lastError ?? null)
676 + ?.providerInstanceId === providerInstanceId
677 + );
678 +}
679 +
680 function latestStableRun(
681 projection: Pick<OrchestrationV2ThreadProjection, "runs">,
682 ): OrchestrationV2Run | null {
@@ -1328,20 +1367,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1367 return;
1368 }
1369
1331 - // The limit already stopped this thread. Starting the queue would send
1332 - // every waiting message and drop it from the queue as each one fails.
1333 - const sessionError =
1334 - projection.providerSessions
1335 - .filter((session) => session.providerInstanceId === projection.thread.providerInstanceId)
1336 - .toSorted(
1337 - (left, right) =>
1338 - DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
1339 - )[0]?.lastError ?? null;
1340 - if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
1370 + const storedQueuedRun = nextQueuedRun(projection);
1371 + if (storedQueuedRun === undefined) {
1372 return;
1373 }
1343 - const queuedRun = nextQueuedRun(projection);
1344 - if (queuedRun === undefined) {
1374 + const queuedMessage = projection.messages.find(
1375 + (candidate) => candidate.id === storedQueuedRun.userMessageId,
1376 + );
1377 + const modelSelection = queuedRunModelSelection(projection, storedQueuedRun);
1378 + const queuedRun = {
1379 + ...storedQueuedRun,
1380 + modelSelection,
1381 + providerInstanceId: modelSelection.instanceId,
1382 + };
1383 + // A limited provider would fail and consume every waiting message.
1384 + if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId)) {
1385 return;
1386 }
1387 // A message queued behind a worktree that was cancelled or never created
@@ -1420,9 +1460,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1460
1461 const rootNode = projection.nodes.find((candidate) => candidate.id === rootNodeId);
1462 const attempt = projection.attempts.find((candidate) => candidate.id === attemptId);
1423 - const queuedMessage = projection.messages.find(
1424 - (candidate) => candidate.id === queuedRun.userMessageId,
1425 - );
1463 const legacyQueuedTurnItem = projection.turnItems.find(
1464 (
1465 candidate,
@@ -1431,7 +1468,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1468 candidate.runId === queuedRun.id &&
1469 candidate.messageId === queuedRun.userMessageId,
1470 );
1434 - const queuedProviderThread = projection.providerThreads.find(
1471 + const storedProviderThread = projection.providerThreads.find(
1472 (candidate) => candidate.id === providerThreadId,
1473 );
1474 const storedCheckpointScope = projection.checkpointScopes.find(
@@ -1441,7 +1478,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1478 rootNode === undefined ||
1479 attempt === undefined ||
1480 queuedMessage === undefined ||
1444 - queuedProviderThread === undefined ||
1481 + storedProviderThread === undefined ||
1482 (rootNode.checkpointScopeId !== null && storedCheckpointScope === undefined)
1483 ) {
1484 return yield* new OrchestratorDispatchError({
@@ -1453,27 +1490,83 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1490
1491 const commandId = CommandId.make(`command:system:start-queued:${queuedRun.id}`);
1492 const now = yield* DateTime.now;
1456 - const selectionChanged = !modelSelectionsEqual(
1457 - projection.thread.modelSelection,
1458 - queuedRun.modelSelection,
1493 + const targetAdapter = yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
1494 + Effect.mapError(
1495 + (cause) =>
1496 + new OrchestratorDispatchError({
1497 + commandId,
1498 + commandType: "message.dispatch",
1499 + cause,
1500 + }),
1501 + ),
1502 );
1460 - const switchPlan = selectionChanged
1461 - ? yield* providerSwitchService
1462 - .plan({ projection, targetModelSelection: queuedRun.modelSelection })
1463 - .pipe(
1464 - Effect.mapError(
1465 - (cause) =>
1466 - new OrchestratorDispatchError({
1467 - commandId,
1468 - commandType: "message.dispatch",
1469 - cause,
1470 - }),
1471 - ),
1472 - )
1473 - : null;
1503 const activeProviderThread = projection.providerThreads.find(
1504 (candidate) => candidate.id === projection.thread.activeProviderThreadId,
1505 );
1506 + const followsCurrent = followsCurrentConversation(queuedMessage);
1507 + const targetProviderThread = followsCurrent
1508 + ? activeProviderThread?.providerInstanceId === queuedRun.providerInstanceId
1509 + ? activeProviderThread
1510 + : rootProviderThreadsForProvider(projection, queuedRun.providerInstanceId)[0]
1511 + : storedProviderThread;
1512 + const queuedProviderThread: OrchestrationV2ProviderThread = targetProviderThread ?? {
1513 + id: idAllocator.derive.providerThread({
1514 + driver: targetAdapter.driver,
1515 + nativeThreadId: `pending:${queuedRun.id}`,
1516 + }),
1517 + driver: targetAdapter.driver,
1518 + providerInstanceId: queuedRun.providerInstanceId,
1519 + providerSessionId: null,
1520 + appThreadId: threadId,
1521 + ownerNodeId: null,
1522 + nativeThreadRef: null,
1523 + nativeConversationHeadRef: null,
1524 + status: "not_loaded",
1525 + firstRunOrdinal: null,
1526 + lastRunOrdinal: null,
1527 + handoffIds: [],
1528 + forkedFrom: null,
1529 + createdAt: now,
1530 + updatedAt: now,
1531 + };
1532 + const selectionChanged = !modelSelectionsEqual(
1533 + projection.thread.modelSelection,
1534 + queuedRun.modelSelection,
1535 + );
1536 + // The picker may already have moved ahead of the native conversation.
1537 + const switchFromActiveProvider =
1538 + followsCurrent &&
1539 + activeProviderThread !== undefined &&
1540 + activeProviderThread.providerInstanceId !== queuedRun.providerInstanceId;
1541 + const switchPlan =
1542 + selectionChanged || switchFromActiveProvider
1543 + ? yield* providerSwitchService
1544 + .plan({
1545 + projection: switchFromActiveProvider
1546 + ? {
1547 + ...projection,
1548 + thread: {
1549 + ...projection.thread,
1550 + modelSelection: {
1551 + ...projection.thread.modelSelection,
1552 + instanceId: activeProviderThread.providerInstanceId,
1553 + },
1554 + },
1555 + }
1556 + : projection,
1557 + targetModelSelection: queuedRun.modelSelection,
1558 + })
1559 + .pipe(
1560 + Effect.mapError(
1561 + (cause) =>
1562 + new OrchestratorDispatchError({
1563 + commandId,
1564 + commandType: "message.dispatch",
1565 + cause,
1566 + }),
1567 + ),
1568 + )
1569 + : null;
1570 const canResumeAcrossInstances =
1571 switchPlan?.instanceChanged === true &&
1572 switchPlan.transition.type === "restart_and_resume" &&
@@ -1488,16 +1581,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1581 nativeMetadata: activeProviderThread.nativeMetadata,
1582 }
1583 : queuedProviderThread;
1491 - const targetAdapter = yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
1492 - Effect.mapError(
1493 - (cause) =>
1494 - new OrchestratorDispatchError({
1495 - commandId,
1496 - commandType: "message.dispatch",
1497 - cause,
1498 - }),
1499 - ),
1500 - );
1584 const targetCapabilities = yield* targetAdapter.getCapabilities().pipe(
1585 Effect.mapError(
1586 (cause) =>
@@ -1690,6 +1773,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1773 };
1774 const startingRun: OrchestrationV2Run = {
1775 ...queuedRun,
1776 + providerThreadId: queuedProviderThread.id,
1777 ordinal,
1778 status: "starting",
1779 queuePosition: null,
@@ -1728,6 +1812,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1812 ? {}
1813 : { senderThreadId: queuedMessage.senderThreadId }),
1814 }),
1815 + providerThreadId: queuedProviderThread.id,
1816 inputIntent: "queued_turn",
1817 startedAt: now,
1818 completedAt: now,
@@ -1774,26 +1859,45 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1859 strategy: activeHandoff.strategy,
1860 summary: activeHandoff.summaryText,
1861 };
1777 - const checkpointEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> =
1778 - storedCheckpointScope === undefined
1862 + const executionIdentityEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> =
1863 + storedCheckpointScope === undefined ||
1864 + storedProviderThread.id !== queuedProviderThread.id ||
1865 + attempt.providerInstanceId !== queuedRun.providerInstanceId
1866 ? [
1867 {
1868 type: "checkpoint-scope.created",
1869 threadId,
1870 runId: queuedRun.id,
1784 - nodeId: rootNode.id,
1871 + nodeId: rootNodeId,
1872 providerInstanceId: queuedRun.providerInstanceId,
1873 occurredAt: now,
1787 - payload: checkpointScope,
1874 + payload: { ...checkpointScope, providerThreadId: queuedProviderThread.id },
1875 },
1876 {
1877 type: "node.updated",
1878 threadId,
1879 runId: queuedRun.id,
1793 - nodeId: rootNode.id,
1880 + nodeId: rootNodeId,
1881 + providerInstanceId: queuedRun.providerInstanceId,
1882 + occurredAt: now,
1883 + payload: {
1884 + ...rootNode,
1885 + providerThreadId: queuedProviderThread.id,
1886 + checkpointScopeId: checkpointScope.id,
1887 + },
1888 + },
1889 + {
1890 + type: "run-attempt.updated",
1891 + threadId,
1892 + runId: queuedRun.id,
1893 + nodeId: rootNodeId,
1894 providerInstanceId: queuedRun.providerInstanceId,
1895 occurredAt: now,
1796 - payload: { ...rootNode, checkpointScopeId: checkpointScope.id },
1896 + payload: {
1897 + ...attempt,
1898 + providerInstanceId: queuedRun.providerInstanceId,
1899 + providerThreadId: queuedProviderThread.id,
1900 + },
1901 },
1902 ]
1903 : [];
@@ -1906,7 +2010,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2010 reason: "Provider or model selection changed.",
2011 },
2012 })),
1909 - ...checkpointEvents,
2013 + ...executionIdentityEvents,
2014 {
2015 type: "provider-thread.updated",
2016 threadId,
@@ -11250,8 +11354,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11354 case "queue.resume": {
11355 const projection = yield* loadProjectionForCommand(
11356 command,
11253 - ["runs", "turnItems", "providerSessions"],
11254 - { turnItemTypes: ["error"] },
11357 + ["runs", "turnItems", "providerSessions", "messages"],
11358 + { turnItemTypes: ["error"], messageRoles: ["user"] },
11359 );
11360 if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) {
11361 return yield* new OrchestratorDispatchError({
@@ -11260,16 +11364,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11364 cause: `Thread ${command.threadId} is not active.`,
11365 });
11366 }
11263 - const sessionError =
11264 - projection.providerSessions
11265 - .filter(
11266 - (session) => session.providerInstanceId === projection.thread.providerInstanceId,
11267 - )
11268 - .toSorted(
11269 - (left, right) =>
11270 - DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
11271 - )[0]?.lastError ?? null;
11272 - if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
11367 + const nextRun = nextQueuedRun(projection);
11368 + if (
11369 + usageLimitHoldsQueue(
11370 + projection,
11371 + nextRun === undefined
11372 + ? projection.thread.providerInstanceId
11373 + : queuedRunModelSelection(projection, nextRun).instanceId,
11374 + )
11375 + ) {
11376 return yield* new OrchestratorDispatchError({
11377 commandId: command.commandId,
11378 commandType: command.type,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+262
@@ -24,6 +24,7 @@ import {
24 PullRequestOperationError,
25 ProviderDriverKind,
26 ProviderInstanceId,
27 + ProviderSessionId,
28 ProviderThreadId,
29 ProviderTurnId,
30 RunId,
@@ -4921,6 +4922,267 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
4922 }),
4923 );
4924
4925 + it.effect("keeps queued user selections and pauses only the limited provider", () =>
4926 + Effect.gen(function* () {
4927 + const orchestrator = yield* Orchestrator.OrchestratorV2;
4928 + const eventSink = yield* EventSink.EventSinkV2;
4929 + const threadId = ThreadId.make("runtime-layer-queue-provider-limit");
4930 + const queuedSelection = {
4931 + instanceId: alternateInstanceId,
4932 + model: "gpt-5.5",
4933 + options: [{ id: "reasoningEffort", value: "high" }],
4934 + };
4935 + yield* orchestrator.dispatch({
4936 + type: "thread.create",
4937 + commandId: CommandId.make(`${threadId}:create`),
4938 + threadId,
4939 + createdBy: "user",
4940 + creationSource: "web",
4941 + projectId: ProjectId.make(`${threadId}:project`),
4942 + title: "Limited queue",
4943 + modelSelection,
4944 + runtimeMode: "full-access",
4945 + interactionMode: "default",
4946 + branch: null,
4947 + worktreePath: process.cwd(),
4948 + });
4949 + const selections = [modelSelection, modelSelection, queuedSelection];
4950 + for (const [index, selection] of selections.entries()) {
4951 + yield* orchestrator.dispatch({
4952 + type: "message.dispatch",
4953 + commandId: CommandId.make(`${threadId}:${index}`),
4954 + threadId,
4955 + messageId: MessageId.make(`${threadId}:${index}`),
4956 + text: `Prompt ${index}`,
4957 + attachments: [],
4958 + createdBy: "user",
4959 + creationSource: "web",
4960 + modelSelection: selection,
4961 + dispatchMode: { type: index === 0 ? "start_immediately" : "queue_after_active" },
4962 + });
4963 + }
4964 + const [active, limited, available] = (yield* orchestrator.getThreadProjection(threadId)).runs;
4965 + assert.isDefined(active);
4966 + assert.isDefined(limited);
4967 + assert.isDefined(available);
4968 + const now = yield* DateTime.now;
4969 + yield* eventSink.write({
4970 + events: [
4971 + {
4972 + id: EventId.make(`${threadId}:error`),
4973 + type: "turn-item.updated",
4974 + threadId,
4975 + runId: active.id,
4976 + nodeId: active.rootNodeId!,
4977 + providerInstanceId: active.providerInstanceId,
4978 + occurredAt: now,
4979 + payload: {
4980 + id: TurnItemId.make(`${threadId}:error`),
4981 + type: "error",
4982 + threadId,
4983 + runId: active.id,
4984 + nodeId: active.rootNodeId!,
4985 + providerThreadId: active.providerThreadId,
4986 + providerTurnId: null,
4987 + nativeItemRef: null,
4988 + parentItemId: null,
4989 + ordinal: 2,
4990 + status: "failed",
4991 + title: "Usage limit",
4992 + startedAt: now,
4993 + completedAt: now,
4994 + updatedAt: now,
4995 + failure: {
4996 + class: "usage_limit",
4997 + message: "Usage limit reached",
4998 + code: "usage_limit",
4999 + retryable: null,
5000 + },
5001 + },
5002 + },
5003 + {
5004 + id: EventId.make(`${threadId}:failed`),
5005 + type: "run.updated",
5006 + threadId,
5007 + runId: active.id,
5008 + providerInstanceId: active.providerInstanceId,
5009 + occurredAt: now,
5010 + payload: { ...active, status: "failed", completedAt: now },
5011 + },
5012 + {
5013 + id: EventId.make(`${threadId}:other-session`),
5014 + type: "provider-session.attached",
5015 + threadId,
5016 + providerInstanceId: alternateInstanceId,
5017 + occurredAt: now,
5018 + payload: {
5019 + id: ProviderSessionId.make(`${threadId}:other-session`),
5020 + driver,
5021 + providerInstanceId: alternateInstanceId,
5022 + status: "error",
5023 + cwd: process.cwd(),
5024 + model: queuedSelection.model,
5025 + capabilities: CodexProviderCapabilitiesV2,
5026 + createdAt: now,
5027 + updatedAt: now,
5028 + lastError: "Unrelated session error on the other provider",
5029 + },
5030 + },
5031 + ],
5032 + });
5033 + yield* orchestrator.dispatch({
5034 + type: "thread.model-selection.set",
5035 + commandId: CommandId.make(`${threadId}:select-other`),
5036 + threadId,
5037 + modelSelection: queuedSelection,
5038 + });
5039 + yield* orchestrator.resumeQueuedRuns;
5040 + const held = yield* orchestrator.getThreadProjection(threadId);
5041 + assert.equal(held.runs.find((run) => run.id === limited.id)?.status, "queued");
5042 + const blocked = yield* orchestrator
5043 + .dispatch({
5044 + type: "queue.resume",
5045 + commandId: CommandId.make(`${threadId}:blocked`),
5046 + threadId,
5047 + })
5048 + .pipe(Effect.flip);
5049 + assert.equal(blocked._tag, "OrchestratorDispatchError");
5050 +
5051 + yield* orchestrator.dispatch({
5052 + type: "thread.model-selection.set",
5053 + commandId: CommandId.make(`${threadId}:select-original`),
5054 + threadId,
5055 + modelSelection,
5056 + });
5057 + yield* orchestrator.dispatch({
5058 + type: "queued-run.cancel",
5059 + commandId: CommandId.make(`${threadId}:cancel-limited`),
5060 + threadId,
5061 + runId: limited.id,
5062 + });
5063 + yield* orchestrator.resumeQueuedRuns;
5064 + const after = yield* orchestrator.getThreadProjection(threadId);
5065 + const started = after.runs.find((run) => run.id === available.id);
5066 + assert.equal(started?.status, "starting");
5067 + assert.deepEqual(started?.modelSelection, queuedSelection);
5068 + assert.deepEqual(after.thread.modelSelection, queuedSelection);
5069 + }),
5070 + );
5071 +
5072 + it.effect.each([
5073 + ["server", modelSelection],
5074 + ["provider", modelSelection],
5075 + ["provider", { ...modelSelection, instanceId: alternateInstanceId }],
5076 + ] as const)(
5077 + "delivers a queued monitor update from %s after conversation selection %j",
5078 + ([creationSource, currentSelection]) =>
5079 + Effect.gen(function* () {
5080 + const orchestrator = yield* Orchestrator.OrchestratorV2;
5081 + const eventSink = yield* EventSink.EventSinkV2;
5082 + const threadId = ThreadId.make(
5083 + `runtime-layer-queue-current-conversation-${creationSource}-${currentSelection.instanceId}`,
5084 + );
5085 + yield* orchestrator.dispatch({
5086 + type: "thread.create",
5087 + commandId: CommandId.make(`${threadId}:create`),
5088 + threadId,
5089 + projectId: ProjectId.make(`${threadId}:project`),
5090 + createdBy: "user",
5091 + creationSource: "web",
5092 + title: "Current conversation",
5093 + modelSelection,
5094 + runtimeMode: "full-access",
5095 + interactionMode: "default",
5096 + branch: null,
5097 + worktreePath: process.cwd(),
5098 + });
5099 + for (const type of ["start_immediately", "queue_after_active"] as const) {
5100 + yield* orchestrator.dispatch({
5101 + type: "message.dispatch",
5102 + commandId: CommandId.make(`${threadId}:${type}`),
5103 + messageId: MessageId.make(`${threadId}:${type}`),
5104 + threadId,
5105 + text: type,
5106 + attachments: [],
5107 + createdBy: "agent",
5108 + creationSource,
5109 + dispatchMode: { type },
5110 + ...(type === "queue_after_active"
5111 + ? {
5112 + notification: {
5113 + source: { kind: "monitor" as const },
5114 + outcome: "updated" as const,
5115 + summary: "PR update",
5116 + },
5117 + }
5118 + : {}),
5119 + });
5120 + }
5121 + const before = yield* orchestrator.getThreadProjection(threadId);
5122 + const [active, queued] = before.runs;
5123 + assert.isDefined(active);
5124 + assert.isDefined(queued);
5125 + const now = yield* DateTime.now;
5126 + const current = {
5127 + ...before.providerThreads[0]!,
5128 + id: ProviderThreadId.make(`${threadId}:current`),
5129 + updatedAt: now,
5130 + };
5131 + yield* orchestrator.dispatch({
5132 + type: "thread.model-selection.set",
5133 + commandId: CommandId.make(`${threadId}:select-current`),
5134 + threadId,
5135 + modelSelection: currentSelection,
5136 + });
5137 + yield* eventSink.write({
5138 + events: [
5139 + {
5140 + id: EventId.make(`${threadId}:completed`),
5141 + type: "run.updated",
5142 + threadId,
5143 + runId: active.id,
5144 + occurredAt: now,
5145 + payload: { ...active, status: "completed", completedAt: now },
5146 + },
5147 + {
5148 + id: EventId.make(`${threadId}:conversation`),
5149 + type: "provider-thread.updated",
5150 + threadId,
5151 + occurredAt: now,
5152 + payload: current,
5153 + },
5154 + ],
5155 + });
5156 + const expectedThreadId =
5157 + creationSource === "provider" ? queued.providerThreadId : current.id;
5158 + yield* orchestrator.resumeQueuedRuns;
5159 + const after = yield* orchestrator.getThreadProjection(threadId);
5160 + const delivered = after.runs.find((run) => run.id === queued.id)!;
5161 + assert.equal(delivered.status, "starting");
5162 + assert.deepEqual(delivered.modelSelection, modelSelection);
5163 + assert.equal(delivered.providerInstanceId, modelSelection.instanceId);
5164 + assert.equal(
5165 + after.attempts.find((attempt) => attempt.id === delivered.activeAttemptId)
5166 + ?.providerInstanceId,
5167 + modelSelection.instanceId,
5168 + );
5169 + assert.equal(delivered.providerThreadId, expectedThreadId);
5170 + assert.equal(after.thread.activeProviderThreadId, expectedThreadId);
5171 + assert.equal(
5172 + after.attempts.find((attempt) => attempt.id === delivered.activeAttemptId)
5173 + ?.providerThreadId,
5174 + expectedThreadId,
5175 + );
5176 + const root = after.nodes.find((node) => node.id === delivered.rootNodeId)!;
5177 + assert.equal(root.providerThreadId, expectedThreadId);
5178 + assert.equal(
5179 + after.checkpointScopes.find((scope) => scope.id === root.checkpointScopeId)
5180 + ?.providerThreadId,
5181 + expectedThreadId,
5182 + );
5183 + }),
5184 + );
5185 +
5186 it.effect("keeps the queue after a user interrupts the active run", () =>
5187 Effect.gen(function* () {
5188 const orchestrator = yield* Orchestrator.OrchestratorV2;
apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts
+243 -5
@@ -81,6 +81,7 @@ interface CapturedTurn {
81 readonly threadId: ThreadId;
82 readonly providerThreadId: ProviderThreadId;
83 readonly text: string;
84 + readonly modelSelection: ModelSelection;
85 readonly attachments: ReadonlyArray<ChatAttachment>;
86 readonly nativeThreadId?: string | null;
87 readonly providerTurnOrdinal?: number;
@@ -112,6 +113,7 @@ function makeTestAdapter(input: {
113 readonly tokenUsageByRunOrdinal?: Readonly<
114 Record<number, Omit<OrchestrationV2ProviderTurnTokenUsage, "updatedAt">>
115 >;
116 + readonly failure?: ReturnType<typeof makeProviderFailure>;
117 readonly failedRunOrdinals?: ReadonlySet<number>;
118 readonly interruptedRunOrdinals?: ReadonlySet<number>;
119 readonly holdRunOrdinal?: number;
@@ -243,6 +245,7 @@ function makeTestAdapter(input: {
245 threadId: turnInput.threadId,
246 providerThreadId: turnInput.providerThread.id,
247 text: turnInput.message.text,
248 + modelSelection: turnInput.modelSelection,
249 attachments: turnInput.message.attachments,
250 nativeThreadId: turnInput.providerThread.nativeThreadRef?.nativeId ?? null,
251 providerTurnOrdinal: turnInput.providerTurnOrdinal,
@@ -330,11 +333,13 @@ function makeTestAdapter(input: {
333 ? {
334 status: terminalStatus,
335 failureItemOrdinal: turnInput.runOrdinal * 100 + 2,
333 - failure: makeProviderFailure({
334 - message: "Simulated provider failure.",
335 - code: "simulated_failure",
336 - class: "provider_error",
337 - }),
336 + failure:
337 + input.failure ??
338 + makeProviderFailure({
339 + message: "Simulated provider failure.",
340 + code: "simulated_failure",
341 + class: "provider_error",
342 + }),
343 }
344 : { status: terminalStatus, failure: null }),
345 threadDisposition: "reusable",
@@ -2229,6 +2234,239 @@ describe("orchestration v2 provider switching", () => {
2234 ),
2235 );
2236
2237 + it.live.each([
2238 + ["automatic", "Claude"],
2239 + ["queue.resume", "Claude"],
2240 + ["automatic", "compatible Codex"],
2241 + ["queue.resume", "compatible Codex"],
2242 + ] as const)(
2243 + "delivers a queued PR update on the current provider through %s (%s)",
2244 + ([delivery, target]) =>
2245 + Effect.scoped(
2246 + Effect.gen(function* () {
2247 + const cwd = yield* checkpointWorkspace(`pr-wake-${delivery}`);
2248 + const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);
2249 + const started = yield* Deferred.make<void>();
2250 + const release = yield* Deferred.make<void>();
2251 + const compatible = target === "compatible Codex";
2252 + const selection = {
2253 + ...(compatible
2254 + ? { ...CODEX_MODEL_SELECTION, instanceId: ProviderInstanceId.make("codex-alternate") }
2255 + : CLAUDE_MODEL_SELECTION),
2256 + options: [{ id: "reasoningEffort", value: "low" }],
2257 + };
2258 + const targetCapabilities = compatible
2259 + ? {
2260 + ...CodexProviderCapabilitiesV2,
2261 + context: {
2262 + ...CodexProviderCapabilitiesV2.context,
2263 + canConsumeHandoffSummaries: false,
2264 + },
2265 + }
2266 + : ClaudeProviderCapabilitiesV2;
2267 + const adapters = [
2268 + makeTestAdapter({
2269 + instanceId: CODEX_MODEL_SELECTION.instanceId,
2270 + driver: CODEX_DRIVER,
2271 + capabilities: CodexProviderCapabilitiesV2,
2272 + modelSelection: CODEX_MODEL_SELECTION,
2273 + responseByRunOrdinal: {},
2274 + capturedTurns,
2275 + holdFirstTurn: started,
2276 + releaseFirstTurn: release,
2277 + failedRunOrdinals: new Set([1]),
2278 + failure: makeProviderFailure({
2279 + message: "Usage limit reached",
2280 + class: "usage_limit",
2281 + code: "usage_limit",
2282 + }),
2283 + }),
2284 + makeTestAdapter({
2285 + instanceId: selection.instanceId,
2286 + driver: compatible ? CODEX_DRIVER : CLAUDE_DRIVER,
2287 + capabilities: targetCapabilities,
2288 + modelSelection: selection,
2289 + responseByRunOrdinal: {},
2290 + capturedTurns,
2291 + }),
2292 + ];
2293 + const layerRegistry = compatible
2294 + ? Layer.succeed(
2295 + ProviderAdapterRegistry.ProviderAdapterRegistryV2,
2296 + ProviderAdapterRegistry.ProviderAdapterRegistryV2.of({
2297 + get: (instanceId) => {
2298 + const adapter = adapters.find(
2299 + (candidate) => candidate.instanceId === instanceId,
2300 + );
2301 + return adapter === undefined
2302 + ? Effect.fail(
2303 + new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({
2304 + instanceId,
2305 + }),
2306 + )
2307 + : Effect.succeed(adapter);
2308 + },
2309 + list: () => Effect.succeed(adapters.map((adapter) => adapter.instanceId)),
2310 + getMetadata: (instanceId) =>
2311 + Effect.succeed({
2312 + driver: CODEX_DRIVER,
2313 + continuationKey: "codex:shared-native-account-history",
2314 + enabled: true,
2315 + capabilities:
2316 + instanceId === selection.instanceId
2317 + ? targetCapabilities
2318 + : CodexProviderCapabilitiesV2,
2319 + }),
2320 + }),
2321 + )
2322 + : ProviderAdapterRegistry.layerFromAdapters(adapters);
2323 + yield* Effect.gen(function* () {
2324 + const orchestrator = yield* Orchestrator.OrchestratorV2;
2325 + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2326 + const dispatch = (text: string, modelSelection: ModelSelection) =>
2327 + orchestrator.dispatch({
2328 + type: "message.dispatch",
2329 + commandId: CommandId.make(text),
2330 + messageId: MessageId.make(text),
2331 + threadId,
2332 + text,
2333 + modelSelection,
2334 + attachments: [],
2335 + createdBy: "user",
2336 + creationSource: "web",
2337 + dispatchMode: { type: "start_immediately" },
2338 + });
2339 + yield* orchestrator.dispatch({
2340 + type: "thread.create",
2341 + commandId: CommandId.make("create"),
2342 + threadId,
2343 + projectId,
2344 + createdBy: "user",
2345 + creationSource: "web",
2346 + title: "PR wake",
2347 + modelSelection: CODEX_MODEL_SELECTION,
2348 + runtimeMode: "full-access",
2349 + interactionMode: "default",
2350 + branch: null,
2351 + worktreePath: cwd,
2352 + });
2353 + yield* dispatch("Original work", CODEX_MODEL_SELECTION);
2354 + yield* Deferred.await(started);
2355 + yield* orchestrator.dispatch({
2356 + type: "message.dispatch",
2357 + commandId: CommandId.make("wake"),
2358 + messageId: MessageId.make("wake"),
2359 + threadId,
2360 + text: "Review the new PR comments",
2361 + attachments: [],
2362 + createdBy: "agent",
2363 + creationSource: "server",
2364 + dispatchMode: { type: "queue_after_active" },
2365 + notification: {
2366 + source: { kind: "monitor" },
2367 + outcome: "updated",
2368 + summary: "New PR comments",
2369 + },
2370 + });
2371 + const [active, wake] = (yield* orchestrator.getThreadProjection(threadId)).runs;
2372 + assert.isDefined(active);
2373 + assert.isDefined(wake);
2374 + assert.equal(wake.status, "queued");
2375 + yield* Deferred.succeed(release, undefined);
2376 + yield* orchestrator.streamStoredEvents.pipe(
2377 + Stream.filter(
2378 + ({ event }) =>
2379 + event.type === "run.updated" &&
2380 + event.runId === active.id &&
2381 + event.payload.status === "failed",
2382 + ),
2383 + Stream.runHead,
2384 + );
2385 + yield* worker.drain();
2386 +
2387 + if (delivery === "automatic") {
2388 + yield* dispatch("Take over the work", selection);
2389 + } else {
2390 + yield* orchestrator.dispatch({
2391 + type: "thread.model-selection.set",
2392 + commandId: CommandId.make("select"),
2393 + threadId,
2394 + modelSelection: selection,
2395 + });
2396 + yield* orchestrator.dispatch({
2397 + type: "queue.resume",
2398 + commandId: CommandId.make("resume"),
2399 + threadId,
2400 + });
2401 + }
2402 + yield* orchestrator.streamStoredEvents.pipe(
2403 + Stream.filter(
2404 + ({ event }) =>
2405 + event.type === "run.updated" &&
2406 + event.runId === wake.id &&
2407 + (event.payload.status === "completed" || event.payload.status === "failed"),
2408 + ),
2409 + Stream.runHead,
2410 + );
2411 + yield* worker.drain();
2412 + const after = yield* orchestrator.getThreadProjection(threadId);
2413 + const turns = yield* Ref.get(capturedTurns);
2414 + assert.lengthOf(turns, delivery === "automatic" ? 3 : 2);
2415 + const executed = turns.at(-1)!;
2416 + assert.equal(executed.driver, compatible ? CODEX_DRIVER : CLAUDE_DRIVER);
2417 + assert.include(executed.text, "Review the new PR comments");
2418 + assert.deepEqual(executed.modelSelection, selection);
2419 + assert.deepEqual(after.thread.modelSelection, selection);
2420 + const delivered = after.runs.find((run) => run.id === wake.id)!;
2421 + assert.equal(delivered.status, "completed");
2422 + assert.deepEqual(delivered.modelSelection, selection);
2423 + assert.equal(delivered.providerInstanceId, selection.instanceId);
2424 + assert.equal(delivered.providerThreadId, executed.providerThreadId);
2425 + assert.equal(after.thread.activeProviderThreadId, delivered.providerThreadId);
2426 + if (delivery === "automatic") {
2427 + assert.equal(executed.providerThreadId, turns[1]!.providerThreadId);
2428 + assert.equal(executed.nativeThreadId, turns[1]!.nativeThreadId);
2429 + }
2430 + if (compatible) {
2431 + assert.equal(executed.nativeThreadId, turns[0]!.nativeThreadId);
2432 + assert.isEmpty(after.contextHandoffs);
2433 + if (delivery === "automatic") {
2434 + assert.equal(delivered.providerThreadId, wake.providerThreadId);
2435 + }
2436 + }
2437 + assert.equal(
2438 + after.providerThreads.find((entry) => entry.id === delivered.providerThreadId)
2439 + ?.providerInstanceId,
2440 + selection.instanceId,
2441 + );
2442 + assert.equal(
2443 + after.turnItems.find(
2444 + (entry) => entry.type === "notification" && entry.runId === wake.id,
2445 + )?.providerThreadId,
2446 + delivered.providerThreadId,
2447 + );
2448 + const attempt = after.attempts.find((entry) => entry.id === delivered.activeAttemptId);
2449 + const root = after.nodes.find((entry) => entry.id === delivered.rootNodeId);
2450 + assert.equal(attempt?.providerInstanceId, selection.instanceId);
2451 + assert.equal(attempt?.providerThreadId, delivered.providerThreadId);
2452 + assert.equal(root?.providerThreadId, delivered.providerThreadId);
2453 + assert.equal(
2454 + after.checkpointScopes.find((entry) => entry.id === root?.checkpointScopeId)
2455 + ?.providerThreadId,
2456 + delivered.providerThreadId,
2457 + );
2458 + }).pipe(
2459 + Effect.provide(
2460 + ProviderReplayHarness.layerWithRegistry(
2461 + { name: `pr-wake-${delivery}`, runtimePolicyOverride: { cwd } },
2462 + layerRegistry,
2463 + ),
2464 + ),
2465 + );
2466 + }),
2467 + ),
2468 + );
2469 +
2470 it.live("fails an unsupported queued handoff and advances to the next queued provider", () =>
2471 Effect.scoped(
2472 Effect.gen(function* () {