The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 22:00 UTC
fix(server): queued PR updates follow the takeover provider (#21314)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Lakshmi Tanmay committed
Oct 20, 2026 at 11:02 UTC
f8bcc3cb066999e155dcd94d0f39adb9e2496486
3 files changed
+671
-68
apps/server/src/orchestration-v2/Orchestrator.ts
+166
-63
@@ -638,6 +638,45 @@ function nextQueuedRun(
638
return queuedRunsInDeliveryOrder(projection)[0];
639
}
640
641
+function followsCurrentConversation(
642
+ message: OrchestrationV2ConversationMessage | undefined,
643
+): boolean {
644
+ return (
645
+ message?.notification?.source.kind === "monitor" && !isBufferedProviderNotification(message)
646
+ );
647
+}
648
+
649
+function queuedRunModelSelection(
650
+ projection: Pick<OrchestrationV2ThreadProjection, "thread" | "messages">,
651
+ run: OrchestrationV2Run,
652
+): ModelSelection {
653
+ const message = projection.messages.find((candidate) => candidate.id === run.userMessageId);
654
+ // App-owned monitor updates follow the agent currently handling the thread.
655
+ return followsCurrentConversation(message)
656
+ ? projection.thread.modelSelection
657
+ : run.modelSelection;
658
+}
659
+
660
+function usageLimitHoldsQueue(
661
+ projection: Pick<OrchestrationV2ThreadProjection, "runs" | "turnItems" | "providerSessions">,
662
+ providerInstanceId: ModelSelection["instanceId"],
663
+): boolean {
664
+ let latestSession: OrchestrationV2ThreadProjection["providerSessions"][number] | undefined;
665
+ for (const session of projection.providerSessions) {
666
+ if (
667
+ session.providerInstanceId === providerInstanceId &&
668
+ (latestSession === undefined ||
669
+ DateTime.isGreaterThan(session.updatedAt, latestSession.updatedAt))
670
+ ) {
671
+ latestSession = session;
672
+ }
673
+ }
674
+ return (
675
+ usageLimitBlockedRun(projection.runs, projection.turnItems, latestSession?.lastError ?? null)
676
+ ?.providerInstanceId === providerInstanceId
677
+ );
678
+}
679
+
680
function latestStableRun(
681
projection: Pick<OrchestrationV2ThreadProjection, "runs">,
682
): OrchestrationV2Run | null {
@@ -1328,20 +1367,21 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1367
return;
1368
}
1369
1331
- // The limit already stopped this thread. Starting the queue would send
1332
- // every waiting message and drop it from the queue as each one fails.
1333
- const sessionError =
1334
- projection.providerSessions
1335
- .filter((session) => session.providerInstanceId === projection.thread.providerInstanceId)
1336
- .toSorted(
1337
- (left, right) =>
1338
- DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
1339
- )[0]?.lastError ?? null;
1340
- if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
1370
+ const storedQueuedRun = nextQueuedRun(projection);
1371
+ if (storedQueuedRun === undefined) {
1372
return;
1373
}
1343
- const queuedRun = nextQueuedRun(projection);
1344
- if (queuedRun === undefined) {
1374
+ const queuedMessage = projection.messages.find(
1375
+ (candidate) => candidate.id === storedQueuedRun.userMessageId,
1376
+ );
1377
+ const modelSelection = queuedRunModelSelection(projection, storedQueuedRun);
1378
+ const queuedRun = {
1379
+ ...storedQueuedRun,
1380
+ modelSelection,
1381
+ providerInstanceId: modelSelection.instanceId,
1382
+ };
1383
+ // A limited provider would fail and consume every waiting message.
1384
+ if (usageLimitHoldsQueue(projection, queuedRun.providerInstanceId)) {
1385
return;
1386
}
1387
// A message queued behind a worktree that was cancelled or never created
@@ -1420,9 +1460,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1460
1461
const rootNode = projection.nodes.find((candidate) => candidate.id === rootNodeId);
1462
const attempt = projection.attempts.find((candidate) => candidate.id === attemptId);
1423
- const queuedMessage = projection.messages.find(
1424
- (candidate) => candidate.id === queuedRun.userMessageId,
1425
- );
1463
const legacyQueuedTurnItem = projection.turnItems.find(
1464
(
1465
candidate,
@@ -1431,7 +1468,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1468
candidate.runId === queuedRun.id &&
1469
candidate.messageId === queuedRun.userMessageId,
1470
);
1434
- const queuedProviderThread = projection.providerThreads.find(
1471
+ const storedProviderThread = projection.providerThreads.find(
1472
(candidate) => candidate.id === providerThreadId,
1473
);
1474
const storedCheckpointScope = projection.checkpointScopes.find(
@@ -1441,7 +1478,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1478
rootNode === undefined ||
1479
attempt === undefined ||
1480
queuedMessage === undefined ||
1444
- queuedProviderThread === undefined ||
1481
+ storedProviderThread === undefined ||
1482
(rootNode.checkpointScopeId !== null && storedCheckpointScope === undefined)
1483
) {
1484
return yield* new OrchestratorDispatchError({
@@ -1453,27 +1490,83 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1490
1491
const commandId = CommandId.make(`command:system:start-queued:${queuedRun.id}`);
1492
const now = yield* DateTime.now;
1456
- const selectionChanged = !modelSelectionsEqual(
1457
- projection.thread.modelSelection,
1458
- queuedRun.modelSelection,
1493
+ const targetAdapter = yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
1494
+ Effect.mapError(
1495
+ (cause) =>
1496
+ new OrchestratorDispatchError({
1497
+ commandId,
1498
+ commandType: "message.dispatch",
1499
+ cause,
1500
+ }),
1501
+ ),
1502
);
1460
- const switchPlan = selectionChanged
1461
- ? yield* providerSwitchService
1462
- .plan({ projection, targetModelSelection: queuedRun.modelSelection })
1463
- .pipe(
1464
- Effect.mapError(
1465
- (cause) =>
1466
- new OrchestratorDispatchError({
1467
- commandId,
1468
- commandType: "message.dispatch",
1469
- cause,
1470
- }),
1471
- ),
1472
- )
1473
- : null;
1503
const activeProviderThread = projection.providerThreads.find(
1504
(candidate) => candidate.id === projection.thread.activeProviderThreadId,
1505
);
1506
+ const followsCurrent = followsCurrentConversation(queuedMessage);
1507
+ const targetProviderThread = followsCurrent
1508
+ ? activeProviderThread?.providerInstanceId === queuedRun.providerInstanceId
1509
+ ? activeProviderThread
1510
+ : rootProviderThreadsForProvider(projection, queuedRun.providerInstanceId)[0]
1511
+ : storedProviderThread;
1512
+ const queuedProviderThread: OrchestrationV2ProviderThread = targetProviderThread ?? {
1513
+ id: idAllocator.derive.providerThread({
1514
+ driver: targetAdapter.driver,
1515
+ nativeThreadId: `pending:${queuedRun.id}`,
1516
+ }),
1517
+ driver: targetAdapter.driver,
1518
+ providerInstanceId: queuedRun.providerInstanceId,
1519
+ providerSessionId: null,
1520
+ appThreadId: threadId,
1521
+ ownerNodeId: null,
1522
+ nativeThreadRef: null,
1523
+ nativeConversationHeadRef: null,
1524
+ status: "not_loaded",
1525
+ firstRunOrdinal: null,
1526
+ lastRunOrdinal: null,
1527
+ handoffIds: [],
1528
+ forkedFrom: null,
1529
+ createdAt: now,
1530
+ updatedAt: now,
1531
+ };
1532
+ const selectionChanged = !modelSelectionsEqual(
1533
+ projection.thread.modelSelection,
1534
+ queuedRun.modelSelection,
1535
+ );
1536
+ // The picker may already have moved ahead of the native conversation.
1537
+ const switchFromActiveProvider =
1538
+ followsCurrent &&
1539
+ activeProviderThread !== undefined &&
1540
+ activeProviderThread.providerInstanceId !== queuedRun.providerInstanceId;
1541
+ const switchPlan =
1542
+ selectionChanged || switchFromActiveProvider
1543
+ ? yield* providerSwitchService
1544
+ .plan({
1545
+ projection: switchFromActiveProvider
1546
+ ? {
1547
+ ...projection,
1548
+ thread: {
1549
+ ...projection.thread,
1550
+ modelSelection: {
1551
+ ...projection.thread.modelSelection,
1552
+ instanceId: activeProviderThread.providerInstanceId,
1553
+ },
1554
+ },
1555
+ }
1556
+ : projection,
1557
+ targetModelSelection: queuedRun.modelSelection,
1558
+ })
1559
+ .pipe(
1560
+ Effect.mapError(
1561
+ (cause) =>
1562
+ new OrchestratorDispatchError({
1563
+ commandId,
1564
+ commandType: "message.dispatch",
1565
+ cause,
1566
+ }),
1567
+ ),
1568
+ )
1569
+ : null;
1570
const canResumeAcrossInstances =
1571
switchPlan?.instanceChanged === true &&
1572
switchPlan.transition.type === "restart_and_resume" &&
@@ -1488,16 +1581,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1581
nativeMetadata: activeProviderThread.nativeMetadata,
1582
}
1583
: queuedProviderThread;
1491
- const targetAdapter = yield* providerAdapters.get(queuedRun.providerInstanceId).pipe(
1492
- Effect.mapError(
1493
- (cause) =>
1494
- new OrchestratorDispatchError({
1495
- commandId,
1496
- commandType: "message.dispatch",
1497
- cause,
1498
- }),
1499
- ),
1500
- );
1584
const targetCapabilities = yield* targetAdapter.getCapabilities().pipe(
1585
Effect.mapError(
1586
(cause) =>
@@ -1690,6 +1773,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1773
};
1774
const startingRun: OrchestrationV2Run = {
1775
...queuedRun,
1776
+ providerThreadId: queuedProviderThread.id,
1777
ordinal,
1778
status: "starting",
1779
queuePosition: null,
@@ -1728,6 +1812,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1812
? {}
1813
: { senderThreadId: queuedMessage.senderThreadId }),
1814
}),
1815
+ providerThreadId: queuedProviderThread.id,
1816
inputIntent: "queued_turn",
1817
startedAt: now,
1818
completedAt: now,
@@ -1774,26 +1859,45 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1859
strategy: activeHandoff.strategy,
1860
summary: activeHandoff.summaryText,
1861
};
1777
- const checkpointEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> =
1778
- storedCheckpointScope === undefined
1862
+ const executionIdentityEvents: ReadonlyArray<Omit<OrchestrationV2DomainEvent, "id">> =
1863
+ storedCheckpointScope === undefined ||
1864
+ storedProviderThread.id !== queuedProviderThread.id ||
1865
+ attempt.providerInstanceId !== queuedRun.providerInstanceId
1866
? [
1867
{
1868
type: "checkpoint-scope.created",
1869
threadId,
1870
runId: queuedRun.id,
1784
- nodeId: rootNode.id,
1871
+ nodeId: rootNodeId,
1872
providerInstanceId: queuedRun.providerInstanceId,
1873
occurredAt: now,
1787
- payload: checkpointScope,
1874
+ payload: { ...checkpointScope, providerThreadId: queuedProviderThread.id },
1875
},
1876
{
1877
type: "node.updated",
1878
threadId,
1879
runId: queuedRun.id,
1793
- nodeId: rootNode.id,
1880
+ nodeId: rootNodeId,
1881
+ providerInstanceId: queuedRun.providerInstanceId,
1882
+ occurredAt: now,
1883
+ payload: {
1884
+ ...rootNode,
1885
+ providerThreadId: queuedProviderThread.id,
1886
+ checkpointScopeId: checkpointScope.id,
1887
+ },
1888
+ },
1889
+ {
1890
+ type: "run-attempt.updated",
1891
+ threadId,
1892
+ runId: queuedRun.id,
1893
+ nodeId: rootNodeId,
1894
providerInstanceId: queuedRun.providerInstanceId,
1895
occurredAt: now,
1796
- payload: { ...rootNode, checkpointScopeId: checkpointScope.id },
1896
+ payload: {
1897
+ ...attempt,
1898
+ providerInstanceId: queuedRun.providerInstanceId,
1899
+ providerThreadId: queuedProviderThread.id,
1900
+ },
1901
},
1902
]
1903
: [];
@@ -1906,7 +2010,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2010
reason: "Provider or model selection changed.",
2011
},
2012
})),
1909
- ...checkpointEvents,
2013
+ ...executionIdentityEvents,
2014
{
2015
type: "provider-thread.updated",
2016
threadId,
@@ -11250,8 +11354,8 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11354
case "queue.resume": {
11355
const projection = yield* loadProjectionForCommand(
11356
command,
11253
- ["runs", "turnItems", "providerSessions"],
11254
- { turnItemTypes: ["error"] },
11357
+ ["runs", "turnItems", "providerSessions", "messages"],
11358
+ { turnItemTypes: ["error"], messageRoles: ["user"] },
11359
);
11360
if (projection.thread.archivedAt !== null || projection.thread.deletedAt !== null) {
11361
return yield* new OrchestratorDispatchError({
@@ -11260,16 +11364,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
11364
cause: `Thread ${command.threadId} is not active.`,
11365
});
11366
}
11263
- const sessionError =
11264
- projection.providerSessions
11265
- .filter(
11266
- (session) => session.providerInstanceId === projection.thread.providerInstanceId,
11267
- )
11268
- .toSorted(
11269
- (left, right) =>
11270
- DateTime.toEpochMillis(right.updatedAt) - DateTime.toEpochMillis(left.updatedAt),
11271
- )[0]?.lastError ?? null;
11272
- if (usageLimitBlockedRun(projection.runs, projection.turnItems, sessionError) !== null) {
11367
+ const nextRun = nextQueuedRun(projection);
11368
+ if (
11369
+ usageLimitHoldsQueue(
11370
+ projection,
11371
+ nextRun === undefined
11372
+ ? projection.thread.providerInstanceId
11373
+ : queuedRunModelSelection(projection, nextRun).instanceId,
11374
+ )
11375
+ ) {
11376
return yield* new OrchestratorDispatchError({
11377
commandId: command.commandId,
11378
commandType: command.type,
apps/server/src/orchestration-v2/runtimeLayer.test.ts
+262
@@ -24,6 +24,7 @@ import {
24
PullRequestOperationError,
25
ProviderDriverKind,
26
ProviderInstanceId,
27
+ ProviderSessionId,
28
ProviderThreadId,
29
ProviderTurnId,
30
RunId,
@@ -4921,6 +4922,267 @@ it.layer(layerTest)("OrchestrationV2LayerLive lifecycle", (it) => {
4922
}),
4923
);
4924
4925
+ it.effect("keeps queued user selections and pauses only the limited provider", () =>
4926
+ Effect.gen(function* () {
4927
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
4928
+ const eventSink = yield* EventSink.EventSinkV2;
4929
+ const threadId = ThreadId.make("runtime-layer-queue-provider-limit");
4930
+ const queuedSelection = {
4931
+ instanceId: alternateInstanceId,
4932
+ model: "gpt-5.5",
4933
+ options: [{ id: "reasoningEffort", value: "high" }],
4934
+ };
4935
+ yield* orchestrator.dispatch({
4936
+ type: "thread.create",
4937
+ commandId: CommandId.make(`${threadId}:create`),
4938
+ threadId,
4939
+ createdBy: "user",
4940
+ creationSource: "web",
4941
+ projectId: ProjectId.make(`${threadId}:project`),
4942
+ title: "Limited queue",
4943
+ modelSelection,
4944
+ runtimeMode: "full-access",
4945
+ interactionMode: "default",
4946
+ branch: null,
4947
+ worktreePath: process.cwd(),
4948
+ });
4949
+ const selections = [modelSelection, modelSelection, queuedSelection];
4950
+ for (const [index, selection] of selections.entries()) {
4951
+ yield* orchestrator.dispatch({
4952
+ type: "message.dispatch",
4953
+ commandId: CommandId.make(`${threadId}:${index}`),
4954
+ threadId,
4955
+ messageId: MessageId.make(`${threadId}:${index}`),
4956
+ text: `Prompt ${index}`,
4957
+ attachments: [],
4958
+ createdBy: "user",
4959
+ creationSource: "web",
4960
+ modelSelection: selection,
4961
+ dispatchMode: { type: index === 0 ? "start_immediately" : "queue_after_active" },
4962
+ });
4963
+ }
4964
+ const [active, limited, available] = (yield* orchestrator.getThreadProjection(threadId)).runs;
4965
+ assert.isDefined(active);
4966
+ assert.isDefined(limited);
4967
+ assert.isDefined(available);
4968
+ const now = yield* DateTime.now;
4969
+ yield* eventSink.write({
4970
+ events: [
4971
+ {
4972
+ id: EventId.make(`${threadId}:error`),
4973
+ type: "turn-item.updated",
4974
+ threadId,
4975
+ runId: active.id,
4976
+ nodeId: active.rootNodeId!,
4977
+ providerInstanceId: active.providerInstanceId,
4978
+ occurredAt: now,
4979
+ payload: {
4980
+ id: TurnItemId.make(`${threadId}:error`),
4981
+ type: "error",
4982
+ threadId,
4983
+ runId: active.id,
4984
+ nodeId: active.rootNodeId!,
4985
+ providerThreadId: active.providerThreadId,
4986
+ providerTurnId: null,
4987
+ nativeItemRef: null,
4988
+ parentItemId: null,
4989
+ ordinal: 2,
4990
+ status: "failed",
4991
+ title: "Usage limit",
4992
+ startedAt: now,
4993
+ completedAt: now,
4994
+ updatedAt: now,
4995
+ failure: {
4996
+ class: "usage_limit",
4997
+ message: "Usage limit reached",
4998
+ code: "usage_limit",
4999
+ retryable: null,
5000
+ },
5001
+ },
5002
+ },
5003
+ {
5004
+ id: EventId.make(`${threadId}:failed`),
5005
+ type: "run.updated",
5006
+ threadId,
5007
+ runId: active.id,
5008
+ providerInstanceId: active.providerInstanceId,
5009
+ occurredAt: now,
5010
+ payload: { ...active, status: "failed", completedAt: now },
5011
+ },
5012
+ {
5013
+ id: EventId.make(`${threadId}:other-session`),
5014
+ type: "provider-session.attached",
5015
+ threadId,
5016
+ providerInstanceId: alternateInstanceId,
5017
+ occurredAt: now,
5018
+ payload: {
5019
+ id: ProviderSessionId.make(`${threadId}:other-session`),
5020
+ driver,
5021
+ providerInstanceId: alternateInstanceId,
5022
+ status: "error",
5023
+ cwd: process.cwd(),
5024
+ model: queuedSelection.model,
5025
+ capabilities: CodexProviderCapabilitiesV2,
5026
+ createdAt: now,
5027
+ updatedAt: now,
5028
+ lastError: "Unrelated session error on the other provider",
5029
+ },
5030
+ },
5031
+ ],
5032
+ });
5033
+ yield* orchestrator.dispatch({
5034
+ type: "thread.model-selection.set",
5035
+ commandId: CommandId.make(`${threadId}:select-other`),
5036
+ threadId,
5037
+ modelSelection: queuedSelection,
5038
+ });
5039
+ yield* orchestrator.resumeQueuedRuns;
5040
+ const held = yield* orchestrator.getThreadProjection(threadId);
5041
+ assert.equal(held.runs.find((run) => run.id === limited.id)?.status, "queued");
5042
+ const blocked = yield* orchestrator
5043
+ .dispatch({
5044
+ type: "queue.resume",
5045
+ commandId: CommandId.make(`${threadId}:blocked`),
5046
+ threadId,
5047
+ })
5048
+ .pipe(Effect.flip);
5049
+ assert.equal(blocked._tag, "OrchestratorDispatchError");
5050
+
5051
+ yield* orchestrator.dispatch({
5052
+ type: "thread.model-selection.set",
5053
+ commandId: CommandId.make(`${threadId}:select-original`),
5054
+ threadId,
5055
+ modelSelection,
5056
+ });
5057
+ yield* orchestrator.dispatch({
5058
+ type: "queued-run.cancel",
5059
+ commandId: CommandId.make(`${threadId}:cancel-limited`),
5060
+ threadId,
5061
+ runId: limited.id,
5062
+ });
5063
+ yield* orchestrator.resumeQueuedRuns;
5064
+ const after = yield* orchestrator.getThreadProjection(threadId);
5065
+ const started = after.runs.find((run) => run.id === available.id);
5066
+ assert.equal(started?.status, "starting");
5067
+ assert.deepEqual(started?.modelSelection, queuedSelection);
5068
+ assert.deepEqual(after.thread.modelSelection, queuedSelection);
5069
+ }),
5070
+ );
5071
+
5072
+ it.effect.each([
5073
+ ["server", modelSelection],
5074
+ ["provider", modelSelection],
5075
+ ["provider", { ...modelSelection, instanceId: alternateInstanceId }],
5076
+ ] as const)(
5077
+ "delivers a queued monitor update from %s after conversation selection %j",
5078
+ ([creationSource, currentSelection]) =>
5079
+ Effect.gen(function* () {
5080
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
5081
+ const eventSink = yield* EventSink.EventSinkV2;
5082
+ const threadId = ThreadId.make(
5083
+ `runtime-layer-queue-current-conversation-${creationSource}-${currentSelection.instanceId}`,
5084
+ );
5085
+ yield* orchestrator.dispatch({
5086
+ type: "thread.create",
5087
+ commandId: CommandId.make(`${threadId}:create`),
5088
+ threadId,
5089
+ projectId: ProjectId.make(`${threadId}:project`),
5090
+ createdBy: "user",
5091
+ creationSource: "web",
5092
+ title: "Current conversation",
5093
+ modelSelection,
5094
+ runtimeMode: "full-access",
5095
+ interactionMode: "default",
5096
+ branch: null,
5097
+ worktreePath: process.cwd(),
5098
+ });
5099
+ for (const type of ["start_immediately", "queue_after_active"] as const) {
5100
+ yield* orchestrator.dispatch({
5101
+ type: "message.dispatch",
5102
+ commandId: CommandId.make(`${threadId}:${type}`),
5103
+ messageId: MessageId.make(`${threadId}:${type}`),
5104
+ threadId,
5105
+ text: type,
5106
+ attachments: [],
5107
+ createdBy: "agent",
5108
+ creationSource,
5109
+ dispatchMode: { type },
5110
+ ...(type === "queue_after_active"
5111
+ ? {
5112
+ notification: {
5113
+ source: { kind: "monitor" as const },
5114
+ outcome: "updated" as const,
5115
+ summary: "PR update",
5116
+ },
5117
+ }
5118
+ : {}),
5119
+ });
5120
+ }
5121
+ const before = yield* orchestrator.getThreadProjection(threadId);
5122
+ const [active, queued] = before.runs;
5123
+ assert.isDefined(active);
5124
+ assert.isDefined(queued);
5125
+ const now = yield* DateTime.now;
5126
+ const current = {
5127
+ ...before.providerThreads[0]!,
5128
+ id: ProviderThreadId.make(`${threadId}:current`),
5129
+ updatedAt: now,
5130
+ };
5131
+ yield* orchestrator.dispatch({
5132
+ type: "thread.model-selection.set",
5133
+ commandId: CommandId.make(`${threadId}:select-current`),
5134
+ threadId,
5135
+ modelSelection: currentSelection,
5136
+ });
5137
+ yield* eventSink.write({
5138
+ events: [
5139
+ {
5140
+ id: EventId.make(`${threadId}:completed`),
5141
+ type: "run.updated",
5142
+ threadId,
5143
+ runId: active.id,
5144
+ occurredAt: now,
5145
+ payload: { ...active, status: "completed", completedAt: now },
5146
+ },
5147
+ {
5148
+ id: EventId.make(`${threadId}:conversation`),
5149
+ type: "provider-thread.updated",
5150
+ threadId,
5151
+ occurredAt: now,
5152
+ payload: current,
5153
+ },
5154
+ ],
5155
+ });
5156
+ const expectedThreadId =
5157
+ creationSource === "provider" ? queued.providerThreadId : current.id;
5158
+ yield* orchestrator.resumeQueuedRuns;
5159
+ const after = yield* orchestrator.getThreadProjection(threadId);
5160
+ const delivered = after.runs.find((run) => run.id === queued.id)!;
5161
+ assert.equal(delivered.status, "starting");
5162
+ assert.deepEqual(delivered.modelSelection, modelSelection);
5163
+ assert.equal(delivered.providerInstanceId, modelSelection.instanceId);
5164
+ assert.equal(
5165
+ after.attempts.find((attempt) => attempt.id === delivered.activeAttemptId)
5166
+ ?.providerInstanceId,
5167
+ modelSelection.instanceId,
5168
+ );
5169
+ assert.equal(delivered.providerThreadId, expectedThreadId);
5170
+ assert.equal(after.thread.activeProviderThreadId, expectedThreadId);
5171
+ assert.equal(
5172
+ after.attempts.find((attempt) => attempt.id === delivered.activeAttemptId)
5173
+ ?.providerThreadId,
5174
+ expectedThreadId,
5175
+ );
5176
+ const root = after.nodes.find((node) => node.id === delivered.rootNodeId)!;
5177
+ assert.equal(root.providerThreadId, expectedThreadId);
5178
+ assert.equal(
5179
+ after.checkpointScopes.find((scope) => scope.id === root.checkpointScopeId)
5180
+ ?.providerThreadId,
5181
+ expectedThreadId,
5182
+ );
5183
+ }),
5184
+ );
5185
+
5186
it.effect("keeps the queue after a user interrupts the active run", () =>
5187
Effect.gen(function* () {
5188
const orchestrator = yield* Orchestrator.OrchestratorV2;
apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts
+243
-5
@@ -81,6 +81,7 @@ interface CapturedTurn {
81
readonly threadId: ThreadId;
82
readonly providerThreadId: ProviderThreadId;
83
readonly text: string;
84
+ readonly modelSelection: ModelSelection;
85
readonly attachments: ReadonlyArray<ChatAttachment>;
86
readonly nativeThreadId?: string | null;
87
readonly providerTurnOrdinal?: number;
@@ -112,6 +113,7 @@ function makeTestAdapter(input: {
113
readonly tokenUsageByRunOrdinal?: Readonly<
114
Record<number, Omit<OrchestrationV2ProviderTurnTokenUsage, "updatedAt">>
115
>;
116
+ readonly failure?: ReturnType<typeof makeProviderFailure>;
117
readonly failedRunOrdinals?: ReadonlySet<number>;
118
readonly interruptedRunOrdinals?: ReadonlySet<number>;
119
readonly holdRunOrdinal?: number;
@@ -243,6 +245,7 @@ function makeTestAdapter(input: {
245
threadId: turnInput.threadId,
246
providerThreadId: turnInput.providerThread.id,
247
text: turnInput.message.text,
248
+ modelSelection: turnInput.modelSelection,
249
attachments: turnInput.message.attachments,
250
nativeThreadId: turnInput.providerThread.nativeThreadRef?.nativeId ?? null,
251
providerTurnOrdinal: turnInput.providerTurnOrdinal,
@@ -330,11 +333,13 @@ function makeTestAdapter(input: {
333
? {
334
status: terminalStatus,
335
failureItemOrdinal: turnInput.runOrdinal * 100 + 2,
333
- failure: makeProviderFailure({
334
- message: "Simulated provider failure.",
335
- code: "simulated_failure",
336
- class: "provider_error",
337
- }),
336
+ failure:
337
+ input.failure ??
338
+ makeProviderFailure({
339
+ message: "Simulated provider failure.",
340
+ code: "simulated_failure",
341
+ class: "provider_error",
342
+ }),
343
}
344
: { status: terminalStatus, failure: null }),
345
threadDisposition: "reusable",
@@ -2229,6 +2234,239 @@ describe("orchestration v2 provider switching", () => {
2234
),
2235
);
2236
2237
+ it.live.each([
2238
+ ["automatic", "Claude"],
2239
+ ["queue.resume", "Claude"],
2240
+ ["automatic", "compatible Codex"],
2241
+ ["queue.resume", "compatible Codex"],
2242
+ ] as const)(
2243
+ "delivers a queued PR update on the current provider through %s (%s)",
2244
+ ([delivery, target]) =>
2245
+ Effect.scoped(
2246
+ Effect.gen(function* () {
2247
+ const cwd = yield* checkpointWorkspace(`pr-wake-${delivery}`);
2248
+ const capturedTurns = yield* Ref.make<ReadonlyArray<CapturedTurn>>([]);
2249
+ const started = yield* Deferred.make<void>();
2250
+ const release = yield* Deferred.make<void>();
2251
+ const compatible = target === "compatible Codex";
2252
+ const selection = {
2253
+ ...(compatible
2254
+ ? { ...CODEX_MODEL_SELECTION, instanceId: ProviderInstanceId.make("codex-alternate") }
2255
+ : CLAUDE_MODEL_SELECTION),
2256
+ options: [{ id: "reasoningEffort", value: "low" }],
2257
+ };
2258
+ const targetCapabilities = compatible
2259
+ ? {
2260
+ ...CodexProviderCapabilitiesV2,
2261
+ context: {
2262
+ ...CodexProviderCapabilitiesV2.context,
2263
+ canConsumeHandoffSummaries: false,
2264
+ },
2265
+ }
2266
+ : ClaudeProviderCapabilitiesV2;
2267
+ const adapters = [
2268
+ makeTestAdapter({
2269
+ instanceId: CODEX_MODEL_SELECTION.instanceId,
2270
+ driver: CODEX_DRIVER,
2271
+ capabilities: CodexProviderCapabilitiesV2,
2272
+ modelSelection: CODEX_MODEL_SELECTION,
2273
+ responseByRunOrdinal: {},
2274
+ capturedTurns,
2275
+ holdFirstTurn: started,
2276
+ releaseFirstTurn: release,
2277
+ failedRunOrdinals: new Set([1]),
2278
+ failure: makeProviderFailure({
2279
+ message: "Usage limit reached",
2280
+ class: "usage_limit",
2281
+ code: "usage_limit",
2282
+ }),
2283
+ }),
2284
+ makeTestAdapter({
2285
+ instanceId: selection.instanceId,
2286
+ driver: compatible ? CODEX_DRIVER : CLAUDE_DRIVER,
2287
+ capabilities: targetCapabilities,
2288
+ modelSelection: selection,
2289
+ responseByRunOrdinal: {},
2290
+ capturedTurns,
2291
+ }),
2292
+ ];
2293
+ const layerRegistry = compatible
2294
+ ? Layer.succeed(
2295
+ ProviderAdapterRegistry.ProviderAdapterRegistryV2,
2296
+ ProviderAdapterRegistry.ProviderAdapterRegistryV2.of({
2297
+ get: (instanceId) => {
2298
+ const adapter = adapters.find(
2299
+ (candidate) => candidate.instanceId === instanceId,
2300
+ );
2301
+ return adapter === undefined
2302
+ ? Effect.fail(
2303
+ new ProviderAdapterRegistry.ProviderAdapterRegistryLookupError({
2304
+ instanceId,
2305
+ }),
2306
+ )
2307
+ : Effect.succeed(adapter);
2308
+ },
2309
+ list: () => Effect.succeed(adapters.map((adapter) => adapter.instanceId)),
2310
+ getMetadata: (instanceId) =>
2311
+ Effect.succeed({
2312
+ driver: CODEX_DRIVER,
2313
+ continuationKey: "codex:shared-native-account-history",
2314
+ enabled: true,
2315
+ capabilities:
2316
+ instanceId === selection.instanceId
2317
+ ? targetCapabilities
2318
+ : CodexProviderCapabilitiesV2,
2319
+ }),
2320
+ }),
2321
+ )
2322
+ : ProviderAdapterRegistry.layerFromAdapters(adapters);
2323
+ yield* Effect.gen(function* () {
2324
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
2325
+ const worker = yield* EffectWorker.OrchestrationEffectWorkerV2;
2326
+ const dispatch = (text: string, modelSelection: ModelSelection) =>
2327
+ orchestrator.dispatch({
2328
+ type: "message.dispatch",
2329
+ commandId: CommandId.make(text),
2330
+ messageId: MessageId.make(text),
2331
+ threadId,
2332
+ text,
2333
+ modelSelection,
2334
+ attachments: [],
2335
+ createdBy: "user",
2336
+ creationSource: "web",
2337
+ dispatchMode: { type: "start_immediately" },
2338
+ });
2339
+ yield* orchestrator.dispatch({
2340
+ type: "thread.create",
2341
+ commandId: CommandId.make("create"),
2342
+ threadId,
2343
+ projectId,
2344
+ createdBy: "user",
2345
+ creationSource: "web",
2346
+ title: "PR wake",
2347
+ modelSelection: CODEX_MODEL_SELECTION,
2348
+ runtimeMode: "full-access",
2349
+ interactionMode: "default",
2350
+ branch: null,
2351
+ worktreePath: cwd,
2352
+ });
2353
+ yield* dispatch("Original work", CODEX_MODEL_SELECTION);
2354
+ yield* Deferred.await(started);
2355
+ yield* orchestrator.dispatch({
2356
+ type: "message.dispatch",
2357
+ commandId: CommandId.make("wake"),
2358
+ messageId: MessageId.make("wake"),
2359
+ threadId,
2360
+ text: "Review the new PR comments",
2361
+ attachments: [],
2362
+ createdBy: "agent",
2363
+ creationSource: "server",
2364
+ dispatchMode: { type: "queue_after_active" },
2365
+ notification: {
2366
+ source: { kind: "monitor" },
2367
+ outcome: "updated",
2368
+ summary: "New PR comments",
2369
+ },
2370
+ });
2371
+ const [active, wake] = (yield* orchestrator.getThreadProjection(threadId)).runs;
2372
+ assert.isDefined(active);
2373
+ assert.isDefined(wake);
2374
+ assert.equal(wake.status, "queued");
2375
+ yield* Deferred.succeed(release, undefined);
2376
+ yield* orchestrator.streamStoredEvents.pipe(
2377
+ Stream.filter(
2378
+ ({ event }) =>
2379
+ event.type === "run.updated" &&
2380
+ event.runId === active.id &&
2381
+ event.payload.status === "failed",
2382
+ ),
2383
+ Stream.runHead,
2384
+ );
2385
+ yield* worker.drain();
2386
+
2387
+ if (delivery === "automatic") {
2388
+ yield* dispatch("Take over the work", selection);
2389
+ } else {
2390
+ yield* orchestrator.dispatch({
2391
+ type: "thread.model-selection.set",
2392
+ commandId: CommandId.make("select"),
2393
+ threadId,
2394
+ modelSelection: selection,
2395
+ });
2396
+ yield* orchestrator.dispatch({
2397
+ type: "queue.resume",
2398
+ commandId: CommandId.make("resume"),
2399
+ threadId,
2400
+ });
2401
+ }
2402
+ yield* orchestrator.streamStoredEvents.pipe(
2403
+ Stream.filter(
2404
+ ({ event }) =>
2405
+ event.type === "run.updated" &&
2406
+ event.runId === wake.id &&
2407
+ (event.payload.status === "completed" || event.payload.status === "failed"),
2408
+ ),
2409
+ Stream.runHead,
2410
+ );
2411
+ yield* worker.drain();
2412
+ const after = yield* orchestrator.getThreadProjection(threadId);
2413
+ const turns = yield* Ref.get(capturedTurns);
2414
+ assert.lengthOf(turns, delivery === "automatic" ? 3 : 2);
2415
+ const executed = turns.at(-1)!;
2416
+ assert.equal(executed.driver, compatible ? CODEX_DRIVER : CLAUDE_DRIVER);
2417
+ assert.include(executed.text, "Review the new PR comments");
2418
+ assert.deepEqual(executed.modelSelection, selection);
2419
+ assert.deepEqual(after.thread.modelSelection, selection);
2420
+ const delivered = after.runs.find((run) => run.id === wake.id)!;
2421
+ assert.equal(delivered.status, "completed");
2422
+ assert.deepEqual(delivered.modelSelection, selection);
2423
+ assert.equal(delivered.providerInstanceId, selection.instanceId);
2424
+ assert.equal(delivered.providerThreadId, executed.providerThreadId);
2425
+ assert.equal(after.thread.activeProviderThreadId, delivered.providerThreadId);
2426
+ if (delivery === "automatic") {
2427
+ assert.equal(executed.providerThreadId, turns[1]!.providerThreadId);
2428
+ assert.equal(executed.nativeThreadId, turns[1]!.nativeThreadId);
2429
+ }
2430
+ if (compatible) {
2431
+ assert.equal(executed.nativeThreadId, turns[0]!.nativeThreadId);
2432
+ assert.isEmpty(after.contextHandoffs);
2433
+ if (delivery === "automatic") {
2434
+ assert.equal(delivered.providerThreadId, wake.providerThreadId);
2435
+ }
2436
+ }
2437
+ assert.equal(
2438
+ after.providerThreads.find((entry) => entry.id === delivered.providerThreadId)
2439
+ ?.providerInstanceId,
2440
+ selection.instanceId,
2441
+ );
2442
+ assert.equal(
2443
+ after.turnItems.find(
2444
+ (entry) => entry.type === "notification" && entry.runId === wake.id,
2445
+ )?.providerThreadId,
2446
+ delivered.providerThreadId,
2447
+ );
2448
+ const attempt = after.attempts.find((entry) => entry.id === delivered.activeAttemptId);
2449
+ const root = after.nodes.find((entry) => entry.id === delivered.rootNodeId);
2450
+ assert.equal(attempt?.providerInstanceId, selection.instanceId);
2451
+ assert.equal(attempt?.providerThreadId, delivered.providerThreadId);
2452
+ assert.equal(root?.providerThreadId, delivered.providerThreadId);
2453
+ assert.equal(
2454
+ after.checkpointScopes.find((entry) => entry.id === root?.checkpointScopeId)
2455
+ ?.providerThreadId,
2456
+ delivered.providerThreadId,
2457
+ );
2458
+ }).pipe(
2459
+ Effect.provide(
2460
+ ProviderReplayHarness.layerWithRegistry(
2461
+ { name: `pr-wake-${delivery}`, runtimePolicyOverride: { cwd } },
2462
+ layerRegistry,
2463
+ ),
2464
+ ),
2465
+ );
2466
+ }),
2467
+ ),
2468
+ );
2469
+
2470
it.live("fails an unsupported queued handoff and advances to the next queued provider", () =>
2471
Effect.scoped(
2472
Effect.gen(function* () {