The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 18:30 UTC
fix(chat): requeued steers no longer look delivered (#23420)
Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>
Alb11747 committed
Nov 12, 2026 at 18:47 UTC
9fb4492329e5847efeabf2b4d4f5c40a58dc883e
12 files changed
+629
-81
apps/server/src/orchestration-v2/Orchestrator.ts
+16
-1
@@ -1973,13 +1973,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1973
const projection = yield* projectionStore.getThreadRecords(
1974
threadId,
1975
["runs", "nodes", "attempts", "providerThreads", "turnItems", "messages"],
1976
- { turnItemTypes: [], messageRoles: ["user"] },
1976
+ { turnItemTypes: ["user_message"], messageRoles: ["user"] },
1977
);
1978
const queuedRun =
1979
queuedRunId === undefined
1980
? nextQueuedRun(projection)
1981
: projection.runs.find((run) => run.id === queuedRunId && run.status === "queued");
1982
if (queuedRun === undefined) return;
1983
+ // A steer that fell back to the queue keeps its user item, hidden while the
1984
+ // run is queued. Republish it before the failure so clients whose bounded
1985
+ // snapshot omitted it can show the prompt beside the error.
1986
+ const retainedUserItems = projection.turnItems.filter(
1987
+ (item) => item.type === "user_message" && item.runId === queuedRun.id,
1988
+ );
1989
const now = yield* DateTime.now;
1990
const rootNode = projection.nodes.find((node) => node.id === queuedRun.rootNodeId);
1991
const attempt = projection.attempts.find((entry) => entry.id === queuedRun.activeAttemptId);
@@ -2005,6 +2011,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2011
class: handoffUnsupported ? "validation_error" : "unknown",
2012
});
2013
yield* writeSystemEvents([
2014
+ ...retainedUserItems.map((item) => ({
2015
+ type: "turn-item.updated" as const,
2016
+ threadId,
2017
+ runId: queuedRun.id,
2018
+ ...(rootNode === undefined ? {} : { nodeId: rootNode.id }),
2019
+ providerInstanceId: queuedRun.providerInstanceId,
2020
+ occurredAt: now,
2021
+ payload: { ...item, status: "failed" as const, completedAt: now, updatedAt: now },
2022
+ })),
2023
...(attempt !== undefined && rootNode !== undefined
2024
? [
2025
{
apps/server/src/orchestration-v2/ProjectionStore.test.ts
+279
-25
@@ -5018,26 +5018,36 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
5018
})}
5019
)
5020
`;
5021
- const cancelledSuffixSnapshot = yield* projectionStore.getThreadSnapshotWindow(threadId, {
5022
- rowLimit: sqlPageLimit,
5023
- });
5024
- const cancelledSuffixPage = buildBoundedThreadProjection({
5025
- policy: { ...THREAD_HISTORY_PAGE_POLICY, maxUserTurns: undefined },
5026
- projection: cancelledSuffixSnapshot.projection,
5027
- snapshotSequence: cancelledSuffixSnapshot.snapshotSequence,
5028
- });
5029
- assert.strictEqual(cancelledSuffixPage.latestLocalTurnOrdinal, 1_202);
5030
- assert.lengthOf(cancelledSuffixPage.projection.visibleTurnItems, 75);
5031
- assert.isFalse(
5032
- cancelledSuffixPage.projection.visibleTurnItems.some((row) =>
5033
- String(row.sourceItemId).includes(":cancelled:"),
5034
- ),
5035
- );
5036
- assert.isTrue(
5037
- cancelledSuffixPage.projection.visibleTurnItems.some(
5038
- (row) => row.sourceItemId === runlessQueuedId,
5039
- ),
5040
- );
5021
+ // A steer that fell back to the queue keeps its queued-turn item while its
5022
+ // run waits. End on cancelled for the checks below.
5023
+ for (const status of ["queued", "cancelled"] as const) {
5024
+ yield* sql`
5025
+ UPDATE orchestration_v2_projection_runs
5026
+ SET status = ${status}, payload_json = json_set(payload_json, '$.status', ${status})
5027
+ WHERE run_id = ${cancelledRunId}
5028
+ `;
5029
+ const cancelledSuffixSnapshot = yield* projectionStore.getThreadSnapshotWindow(threadId, {
5030
+ rowLimit: sqlPageLimit,
5031
+ });
5032
+ const cancelledSuffixPage = buildBoundedThreadProjection({
5033
+ policy: { ...THREAD_HISTORY_PAGE_POLICY, maxUserTurns: undefined },
5034
+ projection: cancelledSuffixSnapshot.projection,
5035
+ snapshotSequence: cancelledSuffixSnapshot.snapshotSequence,
5036
+ });
5037
+ assert.strictEqual(cancelledSuffixPage.latestLocalTurnOrdinal, 1_202);
5038
+ assert.lengthOf(cancelledSuffixPage.projection.visibleTurnItems, 75);
5039
+ assert.isFalse(
5040
+ cancelledSuffixPage.projection.visibleTurnItems.some((row) =>
5041
+ String(row.sourceItemId).includes(":cancelled:"),
5042
+ ),
5043
+ status,
5044
+ );
5045
+ assert.isTrue(
5046
+ cancelledSuffixPage.projection.visibleTurnItems.some(
5047
+ (row) => row.sourceItemId === runlessQueuedId,
5048
+ ),
5049
+ );
5050
+ }
5051
5052
// A steer cancels the queued run before it starts while the steered run keeps writing, so
5053
// the queued message leaves the window but its run is still the shell's latest run.
@@ -5830,6 +5840,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
5840
: [
5841
"rolled back",
5842
"cancelled queued turn",
5843
+ "held queued turn",
5844
"replaced attempt",
5845
"stopped",
5846
"stopped",
@@ -5860,9 +5871,11 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
5871
? "rolled_back"
5872
: kind === "cancelled queued turn"
5873
? "cancelled"
5863
- : kind === "short" && chance(0.5)
5864
- ? "interrupted"
5865
- : "completed";
5874
+ : kind === "held queued turn"
5875
+ ? "queued"
5876
+ : kind === "short" && chance(0.5)
5877
+ ? "interrupted"
5878
+ : "completed";
5879
runs.push({
5880
id: runId,
5881
threadId,
@@ -5941,7 +5954,12 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
5954
? run % 10 !== 0
5955
: kind !== "late wake" && (agentDriven ? run > 1 : chance(0.2));
5956
if (wakes) notification();
5944
- else userMessage(kind === "cancelled queued turn" ? "queued_turn" : "turn_start");
5957
+ else
5958
+ userMessage(
5959
+ kind === "cancelled queued turn" || kind === "held queued turn"
5960
+ ? "queued_turn"
5961
+ : "turn_start",
5962
+ );
5963
// A stopped run never folds, so it is long enough for page ends to fall inside it.
5964
const work =
5965
kind === "late wake"
@@ -6573,7 +6591,13 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
6591
assert.isNotEmpty(countQueries);
6592
for (const countQuery of countQueries) {
6593
for (const detail of yield* planOf(countQuery)) {
6576
- assert.notInclude(detail, "orchestration_v2_projection_turn_items");
6594
+ if (detail.includes("orchestration_v2_projection_turn_items")) {
6595
+ // Only held input is read from the sparse index; transcript history stays unread.
6596
+ assert.include(
6597
+ detail,
6598
+ "SEARCH held USING INDEX orchestration_v2_projection_turn_items_queued_input_idx (run_id=?)",
6599
+ );
6600
+ }
6601
}
6602
}
6603
}),
@@ -10421,6 +10445,236 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
10445
}),
10446
);
10447
10448
+ it.effect("keeps undelivered queued input out of forks taken after a later run", () =>
10449
+ Effect.gen(function* () {
10450
+ const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
10451
+ const now = yield* DateTime.now;
10452
+ const projectId = ProjectId.make("project:projection-fork-held-queue");
10453
+
10454
+ for (const heldStatus of ["queued", "cancelled"] as const) {
10455
+ const prefix = `projection-fork-held-queue:${heldStatus}`;
10456
+ const sourceThreadId = ThreadId.make(`thread:${prefix}:source`);
10457
+ const targetThreadId = ThreadId.make(`thread:${prefix}:target`);
10458
+ const heldRunId = RunId.make(`run:${prefix}:held`);
10459
+ const heldAttemptId = RunAttemptId.make(`attempt:${prefix}:held`);
10460
+ const answeredRunId = RunId.make(`run:${prefix}:answered`);
10461
+
10462
+ for (const [threadId, forkedFrom] of [
10463
+ [sourceThreadId, null],
10464
+ [targetThreadId, { type: "run", threadId: sourceThreadId, runId: answeredRunId }],
10465
+ ] as const) {
10466
+ yield* projectionStore.apply({
10467
+ id: EventId.make(`event:${threadId}:created`),
10468
+ type: "thread.created",
10469
+ threadId,
10470
+ occurredAt: now,
10471
+ payload: {
10472
+ createdBy: "user",
10473
+ creationSource: "web",
10474
+ id: threadId,
10475
+ projectId,
10476
+ title: threadId,
10477
+ providerInstanceId,
10478
+ modelSelection,
10479
+ runtimeMode: "full-access",
10480
+ interactionMode: "default",
10481
+ branch: null,
10482
+ worktreePath: null,
10483
+ activeProviderThreadId: null,
10484
+ lineage: {
10485
+ parentThreadId: forkedFrom === null ? null : sourceThreadId,
10486
+ relationshipToParent: forkedFrom === null ? null : "fork",
10487
+ rootThreadId: sourceThreadId,
10488
+ },
10489
+ forkedFrom,
10490
+ createdAt: now,
10491
+ updatedAt: now,
10492
+ archivedAt: null,
10493
+ settledOverride: null,
10494
+ settledAt: null,
10495
+ lastVisitedAt: null,
10496
+ deletedAt: null,
10497
+ },
10498
+ });
10499
+ }
10500
+
10501
+ const applyRun = (
10502
+ runId: RunId,
10503
+ ordinal: number,
10504
+ status: typeof heldStatus | "completed" | "rolled_back",
10505
+ ) =>
10506
+ projectionStore.apply({
10507
+ id: EventId.make(`event:${runId}:${status}`),
10508
+ type: "run.updated",
10509
+ threadId: sourceThreadId,
10510
+ runId,
10511
+ nodeId: NodeId.make(`node:${runId}`),
10512
+ driver,
10513
+ occurredAt: now,
10514
+ payload: {
10515
+ id: runId,
10516
+ threadId: sourceThreadId,
10517
+ ordinal,
10518
+ providerInstanceId,
10519
+ modelSelection,
10520
+ providerThreadId: null,
10521
+ userMessageId: MessageId.make(`message:${runId}`),
10522
+ rootNodeId: NodeId.make(`node:${runId}`),
10523
+ activeAttemptId: runId === heldRunId ? heldAttemptId : null,
10524
+ status,
10525
+ requestedAt: now,
10526
+ startedAt: runId === heldRunId ? null : now,
10527
+ completedAt: status === "queued" ? null : now,
10528
+ checkpointId: null,
10529
+ contextHandoffId: null,
10530
+ },
10531
+ });
10532
+ const applyUserItem = (input: {
10533
+ readonly threadId: ThreadId;
10534
+ readonly runId: RunId | null;
10535
+ readonly ordinal: number;
10536
+ readonly text: string;
10537
+ readonly inputIntent: "turn_start" | "queued_turn";
10538
+ }) =>
10539
+ projectionStore.apply({
10540
+ id: EventId.make(`event:${input.threadId}:item:${input.ordinal}`),
10541
+ type: "turn-item.updated",
10542
+ threadId: input.threadId,
10543
+ ...(input.runId === null ? {} : { runId: input.runId }),
10544
+ occurredAt: now,
10545
+ payload: {
10546
+ createdBy: "user",
10547
+ creationSource: "web",
10548
+ id: TurnItemId.make(`turn-item:${input.threadId}:${input.ordinal}`),
10549
+ threadId: input.threadId,
10550
+ runId: input.runId,
10551
+ nodeId: null,
10552
+ providerThreadId: null,
10553
+ providerTurnId: null,
10554
+ nativeItemRef: null,
10555
+ parentItemId: null,
10556
+ ordinal: input.ordinal,
10557
+ status: "completed",
10558
+ title: null,
10559
+ startedAt: now,
10560
+ completedAt: now,
10561
+ updatedAt: now,
10562
+ type: "user_message",
10563
+ messageId: MessageId.make(`message:${input.threadId}:${input.ordinal}`),
10564
+ inputIntent: input.inputIntent,
10565
+ text: input.text,
10566
+ attachments: [],
10567
+ },
10568
+ });
10569
+
10570
+ yield* applyRun(heldRunId, 1, heldStatus);
10571
+ yield* projectionStore.apply({
10572
+ id: EventId.make(`event:${heldAttemptId}:created`),
10573
+ type: "run-attempt.created",
10574
+ threadId: sourceThreadId,
10575
+ runId: heldRunId,
10576
+ nodeId: NodeId.make(`node:${heldRunId}`),
10577
+ driver,
10578
+ occurredAt: now,
10579
+ payload: {
10580
+ id: heldAttemptId,
10581
+ runId: heldRunId,
10582
+ attemptOrdinal: 1,
10583
+ rootNodeId: NodeId.make(`node:${heldRunId}`),
10584
+ providerInstanceId,
10585
+ providerThreadId: ProviderThreadId.make(`provider-thread:${prefix}`),
10586
+ providerTurnId: null,
10587
+ reason: "initial",
10588
+ status: heldStatus === "queued" ? "pending" : "cancelled",
10589
+ startedAt: null,
10590
+ completedAt: heldStatus === "queued" ? null : now,
10591
+ },
10592
+ });
10593
+ yield* applyRun(answeredRunId, 2, "completed");
10594
+ yield* applyUserItem({
10595
+ threadId: sourceThreadId,
10596
+ runId: heldRunId,
10597
+ ordinal: 100,
10598
+ text: "held",
10599
+ inputIntent: "queued_turn",
10600
+ });
10601
+ yield* applyUserItem({
10602
+ threadId: sourceThreadId,
10603
+ runId: answeredRunId,
10604
+ ordinal: 200,
10605
+ text: "answered",
10606
+ inputIntent: "turn_start",
10607
+ });
10608
+ yield* applyUserItem({
10609
+ threadId: sourceThreadId,
10610
+ runId: answeredRunId,
10611
+ ordinal: 201,
10612
+ text: "delivered steer",
10613
+ inputIntent: "queued_turn",
10614
+ });
10615
+ // Enough local turns for a turn-limited window to count inherited rows in SQL.
10616
+ for (let index = 0; index < THREAD_HISTORY_PAGE_POLICY.maxUserTurns; index += 1) {
10617
+ yield* applyUserItem({
10618
+ threadId: targetThreadId,
10619
+ runId: null,
10620
+ ordinal: index,
10621
+ text: `local ${index}`,
10622
+ inputIntent: "turn_start",
10623
+ });
10624
+ }
10625
+
10626
+ const assertForkHidesHeldInput = Effect.fn("assertForkHidesHeldInput")(function* () {
10627
+ const target = yield* projectionStore.getThreadProjection(targetThreadId);
10628
+ assert.deepEqual(
10629
+ target.visibleTurnItems
10630
+ .slice(0, 3)
10631
+ .map((row) => (row.item.type === "user_message" ? row.item.text : row.item.type)),
10632
+ ["answered", "delivered steer", "fork"],
10633
+ );
10634
+ const page = yield* projectionStore.getTimelinePage(targetThreadId, {
10635
+ view: "activity",
10636
+ limit: target.visibleTurnItems.length + 1,
10637
+ });
10638
+ assert.deepEqual(page.items, target.visibleTurnItems);
10639
+ const bounded = yield* projectionStore.getThreadSnapshotWindow(targetThreadId, {
10640
+ rowLimit: target.visibleTurnItems.length + 1,
10641
+ });
10642
+ assert.deepEqual(bounded.projection.visibleTurnItems, target.visibleTurnItems);
10643
+ const shellSnapshot = yield* projectionStore.getShellSnapshot();
10644
+ for (const projection of [
10645
+ yield* projectionStore.getThreadProjection(sourceThreadId),
10646
+ target,
10647
+ ]) {
10648
+ for (const shell of [
10649
+ yield* projectionStore.getThreadShell(projection.thread.id),
10650
+ shellSnapshot.threads.find((thread) => thread.id === projection.thread.id),
10651
+ ]) {
10652
+ assert.equal(shell?.visibleItemCount, projection.visibleTurnItems.length);
10653
+ }
10654
+ }
10655
+ const filledLocalPage = yield* projectionStore.getThreadSnapshotWindow(targetThreadId, {
10656
+ rowLimit: target.visibleTurnItems.length,
10657
+ userTurnLimit: THREAD_HISTORY_PAGE_POLICY.maxUserTurns,
10658
+ });
10659
+ assert.deepEqual(
10660
+ filledLocalPage.projection.visibleTurnItems.filter((row) => row.visibility === "local"),
10661
+ target.visibleTurnItems.filter((row) => row.visibility === "local"),
10662
+ );
10663
+ });
10664
+
10665
+ yield* assertForkHidesHeldInput();
10666
+ // Forks keep rolled-back source history, but still not undelivered input.
10667
+ yield* applyRun(answeredRunId, 2, "rolled_back");
10668
+ yield* assertForkHidesHeldInput();
10669
+ // Rollback also takes cancelled runs, and overwrites the status that hid their input.
10670
+ if (heldStatus === "cancelled") {
10671
+ yield* applyRun(heldRunId, 1, "rolled_back");
10672
+ yield* assertForkHidesHeldInput();
10673
+ }
10674
+ }
10675
+ }),
10676
+ );
10677
+
10678
it.effect("hides source turns rewound before the fork from a fork's visible items", () =>
10679
Effect.gen(function* () {
10680
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
apps/server/src/orchestration-v2/ProjectionStore.ts
+113
-45
@@ -78,10 +78,7 @@ import {
78
ProjectId,
79
ProviderInstanceId,
80
} from "@t3tools/contracts";
81
-import {
82
- createOrchestrationV2TurnItemVisibility,
83
- isOrchestrationV2SupersededInterrupt,
84
-} from "@t3tools/shared/orchestrationV2Timeline";
81
+import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline";
82
import {
83
deriveBlockedOnThreadWait,
84
derivePendingBackgroundWork,
@@ -1461,6 +1458,28 @@ function isTurnItemAtOrBeforeRun(input: {
1458
return ordinal !== undefined && ordinal <= input.sourceRunOrdinal;
1459
}
1460
1461
+// Run ordinals enforce the fork cutoff; this keeps later rollbacks from hiding
1462
+// inherited history while retaining cancellation evidence for undelivered input.
1463
+function createInheritedTurnItemVisibility(
1464
+ source: Parameters<typeof createOrchestrationV2TurnItemVisibility>[0],
1465
+) {
1466
+ const cancelledRunIds = new Set(
1467
+ source.attempts
1468
+ .filter((attempt) => attempt.status === "cancelled")
1469
+ .map((attempt) => attempt.runId),
1470
+ );
1471
+ return createOrchestrationV2TurnItemVisibility({
1472
+ ...source,
1473
+ runs: source.runs.flatMap((run) =>
1474
+ run.status !== "rolled_back"
1475
+ ? [run]
1476
+ : cancelledRunIds.has(run.id)
1477
+ ? [{ ...run, status: "cancelled" as const }]
1478
+ : [],
1479
+ ),
1480
+ });
1481
+}
1482
+
1483
// Rollback stamps completedAt; a fork made after the rewind never received that run.
1484
function runRolledBackAt(run: Pick<OrchestrationV2Run, "status" | "completedAt">) {
1485
return run.status === "rolled_back" ? run.completedAt : null;
@@ -1495,6 +1514,11 @@ function visibleTurnItemsThroughRun(input: {
1514
}
1515
1516
const runOrdinalById = forkInheritedRunOrdinals(input.sourceProjection.runs, input.forkCreatedAt);
1517
+ const isVisible = createInheritedTurnItemVisibility({
1518
+ runs: input.sourceProjection.runs,
1519
+ attempts: input.sourceProjection.attempts,
1520
+ items: input.sourceProjection.turnItems,
1521
+ });
1522
const inheritedPrefix = input.sourceProjection.visibleTurnItems
1523
.filter(
1524
(row) => row.item.threadId !== input.sourceProjection.thread.id || row.item.type === "fork",
@@ -1506,23 +1530,16 @@ function visibleTurnItemsThroughRun(input: {
1530
item: row.item,
1531
}));
1532
const localPrefix = inheritedVisibleTurnItemsFromLocalItems(
1509
- input.sourceProjection.turnItems.filter((item) => {
1510
- if (
1511
- isOrchestrationV2SupersededInterrupt({
1512
- item,
1513
- attempts: input.sourceProjection.attempts,
1514
- items: input.sourceProjection.turnItems,
1515
- })
1516
- ) {
1517
- return false;
1518
- }
1519
- return isTurnItemAtOrBeforeRun({
1520
- historyOrigin: input.sourceProjection.thread.historyOrigin,
1521
- itemRunId: item.runId,
1522
- runOrdinalById,
1523
- sourceRunOrdinal: sourceRun.ordinal,
1524
- });
1525
- }),
1533
+ input.sourceProjection.turnItems.filter(
1534
+ (item) =>
1535
+ isVisible(item) &&
1536
+ isTurnItemAtOrBeforeRun({
1537
+ historyOrigin: input.sourceProjection.thread.historyOrigin,
1538
+ itemRunId: item.runId,
1539
+ runOrdinalById,
1540
+ sourceRunOrdinal: sourceRun.ordinal,
1541
+ }),
1542
+ ),
1543
);
1544
1545
return [...inheritedPrefix, ...localPrefix];
@@ -3122,16 +3139,27 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3139
),
3140
9223372036854775807
3141
)
3142
+ -- The fork cutoff below filters rewound runs; undelivered queued input stays hidden.
3143
AND (
3144
${window.requiredRunId ?? null} IS NOT NULL
3127
- OR (
3128
- (run.status IS NULL OR run.status <> 'rolled_back')
3129
- AND NOT (
3130
- item.type = 'user_message'
3131
- AND json_extract(item.payload_json, '$.inputIntent') = 'queued_turn'
3132
- AND run.status IS 'cancelled'
3145
+ OR run.status IS NULL
3146
+ OR run.status <> 'rolled_back'
3147
+ )
3148
+ AND NOT (
3149
+ item.type = 'user_message'
3150
+ AND (
3151
+ COALESCE(run.status IN ('queued', 'cancelled'), 0)
3152
+ OR (
3153
+ run.status = 'rolled_back'
3154
+ AND EXISTS (
3155
+ SELECT 1
3156
+ FROM orchestration_v2_projection_run_attempts AS attempt
3157
+ WHERE attempt.run_id = run.run_id
3158
+ AND attempt.status = 'cancelled'
3159
+ )
3160
)
3161
)
3162
+ AND json_extract(item.payload_json, '$.inputIntent') = 'queued_turn'
3163
)
3164
AND (
3165
${window.requiredRunId ?? null} IS NULL
@@ -6368,18 +6396,18 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6396
let inherited: Array<TimelineIndexRow> = [];
6397
if (sourceRun !== undefined) {
6398
const ordinals = forkInheritedRunOrdinals(source.records.runs, records.thread.createdAt);
6371
- const sourceItems = source.local.map((row) => row.item);
6399
+ const isInheritedVisible = createInheritedTurnItemVisibility({
6400
+ runs: source.records.runs,
6401
+ attempts: source.records.attempts,
6402
+ items: source.local.map((row) => row.item),
6403
+ });
6404
inherited = [
6405
...source.visible.filter(
6406
(row) => row.item.threadId !== fork.threadId || row.item.type === "fork",
6407
),
6408
...source.local.filter(
6409
(row) =>
6378
- !isOrchestrationV2SupersededInterrupt({
6379
- item: row.item,
6380
- attempts: source.records.attempts,
6381
- items: sourceItems,
6382
- }) &&
6410
+ isInheritedVisible(row.item) &&
6411
isTurnItemAtOrBeforeRun({
6412
historyOrigin: source.records.thread.historyOrigin,
6413
itemRunId: row.item.runId,
@@ -6576,13 +6604,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6604
})),
6605
});
6606
// Only these runs can hide rows: every row of a rolled-back run, the queued turn of a
6579
- // cancelled run, and the stop result of a replaced attempt. So the hidden rows are found
6580
- // without reading the thread.
6607
+ // queued or cancelled run, and the stop result of a replaced attempt. So the hidden rows
6608
+ // are found without reading the thread.
6609
const stoppedRuns = new Set(stopRequests.map((row) => row.run_id));
6582
- const runsWithStatus = (status: "rolled_back" | "cancelled") =>
6583
- records.runs.flatMap((run) => (run.status === status ? [String(run.id)] : []));
6610
+ const runsWithStatus = (...statuses: ReadonlyArray<OrchestrationV2Run["status"]>) =>
6611
+ records.runs.flatMap((run) => (statuses.includes(run.status) ? [String(run.id)] : []));
6612
const rolledBack = runsWithStatus("rolled_back");
6585
- const cancelled = runsWithStatus("cancelled");
6613
+ const heldInput = runsWithStatus("queued", "cancelled");
6614
const steered = [...new Set(superseded.map((attempt) => String(attempt.runId)))].filter(
6615
(runId) => !stoppedRuns.has(runId) && !rolledBack.includes(runId),
6616
);
@@ -6607,8 +6635,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6635
INDEXED BY orchestration_v2_projection_turn_items_thread_run_idx
6636
ON item.thread_id = ${threadId} AND item.run_id = wanted.value`,
6637
// The conversation index holds the type and run of these rows, so only the matches are
6610
- // fetched. Through the run index, every row of each cancelled run would be.
6611
- cancelled.length + steered.length === 0
6638
+ // fetched. Through the run index, every row of each such run would be.
6639
+ heldInput.length + steered.length === 0
6640
? Effect.succeed([])
6641
: sql<HiddenCandidate>`
6642
SELECT ${candidateColumns}
@@ -6617,7 +6645,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6645
WHERE item.thread_id = ${threadId}
6646
AND item.type NOT IN ${CONVERSATION_INDEX_EXCLUDED_TYPES}
6647
AND ((item.type = 'user_message'
6620
- AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(cancelled)})))
6648
+ AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(heldInput)})))
6649
OR (item.type = 'run_interrupt_result'
6650
AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(steered)}))))`,
6651
]);
@@ -7706,18 +7734,58 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
7734
WHERE thread_id IN (SELECT value FROM json_each(${encodeIdList(threadIds)}))
7735
`;
7736
7737
+ // Forks do not inherit undelivered queued input, including from cancelled runs
7738
+ // rolled back later. Checking the run first limits sparse-index seeks to those few runs.
7739
const selectShellRunItemCounts = (threadIds: ReadonlyArray<ThreadId>) =>
7740
sql<ShellRunItemCountRow>`
7711
- SELECT thread_id, run_key AS run_id, item_count
7712
- FROM orchestration_v2_projection_turn_item_counts
7713
- WHERE run_key <> '' AND thread_id IN (SELECT value FROM json_each(${encodeIdList(threadIds)}))
7741
+ SELECT per_run.thread_id, per_run.run_key AS run_id,
7742
+ per_run.item_count - CASE
7743
+ WHEN EXISTS (
7744
+ SELECT 1
7745
+ FROM orchestration_v2_projection_runs AS run
7746
+ WHERE run.run_id = per_run.run_key
7747
+ AND (
7748
+ run.status IN ('queued', 'cancelled')
7749
+ OR (
7750
+ run.status = 'rolled_back'
7751
+ AND EXISTS (
7752
+ SELECT 1
7753
+ FROM orchestration_v2_projection_run_attempts AS attempt
7754
+ WHERE attempt.run_id = run.run_id
7755
+ AND attempt.status = 'cancelled'
7756
+ )
7757
+ )
7758
+ )
7759
+ ) THEN (
7760
+ SELECT COUNT(*)
7761
+ FROM orchestration_v2_projection_turn_items AS held
7762
+ INDEXED BY orchestration_v2_projection_turn_items_queued_input_idx
7763
+ WHERE held.run_id = per_run.run_key
7764
+ AND held.type = 'user_message'
7765
+ AND json_extract(held.payload_json, '$.inputIntent') = 'queued_turn'
7766
+ )
7767
+ ELSE 0
7768
+ END AS item_count
7769
+ FROM orchestration_v2_projection_turn_item_counts AS per_run
7770
+ WHERE per_run.run_key <> '' AND per_run.thread_id IN (SELECT value FROM json_each(${encodeIdList(threadIds)}))
7771
`;
7772
7773
const selectShellItemCounts = (threadIds: ReadonlyArray<ThreadId>) =>
7774
sql<ShellItemCountRow>`
7775
SELECT counts.thread_id,
7776
SUM(CASE WHEN counts.run_key = '' OR r.status <> 'rolled_back'
7720
- THEN counts.item_count ELSE 0 END) AS item_count,
7777
+ THEN counts.item_count - CASE
7778
+ WHEN r.status IN ('queued', 'cancelled') THEN (
7779
+ SELECT COUNT(*)
7780
+ FROM orchestration_v2_projection_turn_items held
7781
+ INDEXED BY orchestration_v2_projection_turn_items_queued_input_idx
7782
+ WHERE held.run_id = counts.run_key
7783
+ AND held.type = 'user_message'
7784
+ AND json_extract(held.payload_json, '$.inputIntent') = 'queued_turn'
7785
+ )
7786
+ ELSE 0
7787
+ END
7788
+ ELSE 0 END) AS item_count,
7789
SUM(CASE WHEN counts.run_key = '' THEN counts.item_count ELSE 0 END)
7790
AS runless_item_count
7791
FROM orchestration_v2_projection_turn_item_counts counts
apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts
+15
@@ -379,6 +379,13 @@ it.effect.each(
379
}
380
if (delivery !== null && !question) {
381
const requeued = yield* orchestrator.getThreadProjection(threadId);
382
+ assert.isTrue(requeued.runs.some((run) => run.status === "queued"));
383
+ assert.isFalse(
384
+ requeued.visibleTurnItems.some(
385
+ ({ item }) => item.type === "user_message" && item.messageId === messageId,
386
+ ),
387
+ "a steer that fell back to the queue must not appear delivered",
388
+ );
389
const item = requeued.turnItems.find(
390
(item) => item.type === "user_message" && item.messageId === messageId,
391
);
@@ -462,6 +469,14 @@ it.effect.each(
469
).length,
470
1,
471
);
472
+ if (!mailbox && !question) {
473
+ assert.equal(
474
+ final.visibleTurnItems.filter(
475
+ ({ item }) => item.type === "user_message" && item.messageId === messageId,
476
+ ).length,
477
+ 1,
478
+ );
479
+ }
480
yield* worker.drain();
481
assert.equal(started.length, 2);
482
}).pipe(
apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts
+44
@@ -3938,6 +3938,11 @@ describe("orchestration v2 provider switching", () => {
3938
);
3939
assert.equal(queuedItem?.runId, queued.runs[1]?.id);
3940
assert.equal(queuedItem?.providerThreadId, queued.runs[1]?.providerThreadId);
3941
+ assert.equal(queuedItem?.status, "pending");
3942
+ assert.isNull(queuedItem?.startedAt);
3943
+ assert.isNull(queuedItem?.completedAt);
3944
+ assert.isFalse(queued.visibleTurnItems.some((row) => row.item.id === queuedItem?.id));
3945
+ const queuedSequence = yield* eventSink.latestSequence({ threadId: rejectedThreadId });
3946
yield* eventSink.write({
3947
events: [
3948
{
@@ -3962,6 +3967,40 @@ describe("orchestration v2 provider switching", () => {
3967
Stream.runHead,
3968
);
3969
yield* worker.drain();
3970
+ // Clients whose bounded snapshot omitted the hidden queued item must
3971
+ // receive it while the run is still queued, before it fails.
3972
+ const throughSequence = yield* eventSink.latestSequence({ threadId: rejectedThreadId });
3973
+ const failureEvents = yield* eventSink
3974
+ .stream({ threadId: rejectedThreadId, afterSequence: queuedSequence })
3975
+ .pipe(
3976
+ Stream.takeUntil((stored) => stored.sequence === throughSequence),
3977
+ Stream.map(({ event }) => event),
3978
+ Stream.filter((event) => event.runId === queued.runs[1]!.id),
3979
+ Stream.runCollect,
3980
+ );
3981
+ const republished = failureEvents.findIndex(
3982
+ (event) =>
3983
+ event.type === "turn-item.updated" &&
3984
+ event.payload.type === "user_message" &&
3985
+ event.payload.messageId === rejectedMessageId,
3986
+ );
3987
+ assert.isAtLeast(republished, 0);
3988
+ const restored = failureEvents[republished];
3989
+ assert.equal(restored?.type, "turn-item.updated");
3990
+ if (restored?.type === "turn-item.updated") {
3991
+ assert.deepEqual(restored.payload, {
3992
+ ...queuedItem,
3993
+ status: "failed",
3994
+ completedAt: restored.occurredAt,
3995
+ updatedAt: restored.occurredAt,
3996
+ });
3997
+ }
3998
+ assert.isBelow(
3999
+ republished,
4000
+ failureEvents.findIndex(
4001
+ (event) => event.type === "run.updated" && event.payload.status === "failed",
4002
+ ),
4003
+ );
4004
return yield* orchestrator.getThreadProjection(rejectedThreadId);
4005
}).pipe(
4006
Effect.provide(
@@ -4005,6 +4044,11 @@ describe("orchestration v2 provider switching", () => {
4044
failureItem?.type === "error" ? failureItem.failure.code : null,
4045
"context_handoff_unsupported",
4046
);
4047
+ assert.isTrue(
4048
+ projection.visibleTurnItems.some(
4049
+ ({ item }) => item.type === "user_message" && item.messageId === rejectedMessageId,
4050
+ ),
4051
+ );
4052
assert.deepEqual(
4053
(yield* Ref.get(capturedTurns)).map((turn) => turn.driver),
4054
[CODEX_DRIVER, CODEX_DRIVER],
apps/server/src/persistence/Migrations.ts
+2
@@ -88,6 +88,7 @@ import Migration0082 from "./Migrations/082_TurnItemConversationIndex.ts";
88
import Migration0083 from "./Migrations/083_CommandReadIndexes.ts";
89
import Migration0084 from "./Migrations/084_LastExecutedRunIndex.ts";
90
import Migration0085 from "./Migrations/085_TurnItemCounts.ts";
91
+import Migration0086 from "./Migrations/086_QueuedTurnInputIndex.ts";
92
93
const migrationEntries = [
94
[1, "OrchestrationEvents", Migration0001],
@@ -176,6 +177,7 @@ const migrationEntries = [
177
[83, "CommandReadIndexes", Migration0083],
178
[84, "LastExecutedRunIndex", Migration0084],
179
[85, "TurnItemCounts", Migration0085],
180
+ [86, "QueuedTurnInputIndex", Migration0086],
181
] as const;
182
183
export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
apps/server/src/persistence/Migrations/086_QueuedTurnInputIndex.ts
new
+13
@@ -0,0 +1,13 @@
1
+import * as Effect from "effect/Effect";
2
+import * as SqlClient from "effect/sql/SqlClient";
3
+
4
+export default Effect.gen(function* () {
5
+ const sql = yield* SqlClient.SqlClient;
6
+
7
+ // Shell counts subtract held input without reading a run's tool history or item payloads.
8
+ yield* sql`
9
+ CREATE INDEX orchestration_v2_projection_turn_items_queued_input_idx
10
+ ON orchestration_v2_projection_turn_items(run_id)
11
+ WHERE type = 'user_message' AND json_extract(payload_json, '$.inputIntent') = 'queued_turn'
12
+ `;
13
+});
apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+2
@@ -65,6 +65,7 @@ describe("V2 preview upgrade", () => {
65
[83, "CommandReadIndexes"],
66
[84, "LastExecutedRunIndex"],
67
[85, "TurnItemCounts"],
68
+ [86, "QueuedTurnInputIndex"],
69
]);
70
assert.deepStrictEqual(yield* runMigrations(), []);
71
assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -173,6 +174,7 @@ describe("V2 preview upgrade", () => {
174
[83, "CommandReadIndexes"],
175
[84, "LastExecutedRunIndex"],
176
[85, "TurnItemCounts"],
177
+ [86, "QueuedTurnInputIndex"],
178
]);
179
}).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
180
);
packages/client-runtime/src/state/orchestrationV2Projection.test.ts
+99
@@ -483,6 +483,105 @@ describe("applyOrchestrationV2ProjectionEvent", () => {
483
});
484
});
485
486
+ it("hides a steer that fell back to the queue until its run starts", () => {
487
+ const queuedRunId = RunId.make("run-requeued-steer");
488
+ const queuedRun = { ...run, id: queuedRunId, ordinal: 2, status: "queued" } as const;
489
+ const reply = commandItem("item-reply", "reply", 102);
490
+ const steer: OrchestrationV2TurnItem = {
491
+ id: TurnItemId.make("item-steer"),
492
+ threadId,
493
+ runId,
494
+ nodeId: null,
495
+ providerThreadId: null,
496
+ providerTurnId: null,
497
+ nativeItemRef: null,
498
+ parentItemId: null,
499
+ ordinal: 101,
500
+ status: "completed",
501
+ title: null,
502
+ startedAt: now,
503
+ completedAt: now,
504
+ updatedAt: now,
505
+ type: "user_message",
506
+ messageId: MessageId.make("message-steer"),
507
+ inputIntent: "steer",
508
+ text: "steer",
509
+ attachments: [],
510
+ createdBy: "user",
511
+ creationSource: "web",
512
+ };
513
+ const event = (type: "run.updated" | "turn-item.updated", payload: unknown) =>
514
+ ({
515
+ id: `event-${type}`,
516
+ type,
517
+ threadId,
518
+ occurredAt: now,
519
+ payload,
520
+ }) as OrchestrationV2DomainEvent;
521
+ const apply = (
522
+ projection: OrchestrationV2ThreadProjection | null,
523
+ events: ReadonlyArray<OrchestrationV2DomainEvent>,
524
+ options?: Parameters<typeof applyOrchestrationV2ProjectionEvent>[2],
525
+ ) =>
526
+ events.reduce(
527
+ (next, item) => applyOrchestrationV2ProjectionEvent(next, item, options),
528
+ projection,
529
+ );
530
+ const visibleIds = (projection: OrchestrationV2ThreadProjection | null) =>
531
+ projection?.visibleTurnItems.map((row) => row.item.id);
532
+
533
+ const accepted = apply(emptyProjection, [
534
+ event("run.updated", { ...run, status: "running" }),
535
+ event("turn-item.updated", steer),
536
+ event("turn-item.updated", reply),
537
+ ]);
538
+ expect(visibleIds(accepted)).toEqual([steer.id, reply.id]);
539
+
540
+ const queuedSteer = event("turn-item.updated", {
541
+ ...steer,
542
+ runId: queuedRunId,
543
+ inputIntent: "queued_turn",
544
+ status: "pending",
545
+ startedAt: null,
546
+ completedAt: null,
547
+ });
548
+ const requeued = apply(accepted, [event("run.updated", queuedRun), queuedSteer]);
549
+ expect(visibleIds(requeued)).toEqual([reply.id]);
550
+
551
+ // Servers publish the start's user item and run update in either order.
552
+ const starting = event("run.updated", { ...queuedRun, status: "starting" });
553
+ const dispatchedSteer = event("turn-item.updated", {
554
+ ...steer,
555
+ runId: queuedRunId,
556
+ inputIntent: "queued_turn",
557
+ });
558
+ expect(visibleIds(apply(requeued, [dispatchedSteer, starting]))).toEqual([steer.id, reply.id]);
559
+ expect(visibleIds(apply(requeued, [starting, dispatchedSteer]))).toEqual([steer.id, reply.id]);
560
+ const cancelled = event("run.updated", { ...queuedRun, status: "cancelled" });
561
+ expect(visibleIds(apply(requeued, [cancelled]))).toEqual([reply.id]);
562
+
563
+ // A bounded snapshot taken while queued omits the hidden item, and the
564
+ // newer reply puts it below the watermark.
565
+ const bounded = {
566
+ ...requeued!,
567
+ turnItems: requeued!.turnItems.filter((item) => item.id !== steer.id),
568
+ };
569
+ const partial = { partialTimeline: true, latestLocalTurnOrdinal: reply.ordinal };
570
+ const failed = event("run.updated", { ...queuedRun, status: "failed" });
571
+ const failedSteer = event("turn-item.updated", {
572
+ ...steer,
573
+ runId: queuedRunId,
574
+ inputIntent: "queued_turn",
575
+ status: "failed",
576
+ startedAt: null,
577
+ });
578
+ const retained = apply(bounded, [failedSteer], partial);
579
+ expect(visibleIds(retained)).toEqual([reply.id]);
580
+ expect(visibleIds(apply(retained, [failed], partial))).toEqual([steer.id, reply.id]);
581
+ // Leaving the queue never invents an item the client did not receive.
582
+ expect(visibleIds(apply(bounded, [failed], partial))).toEqual([reply.id]);
583
+ });
584
+
585
it("removes only hidden local items while preserving inherited rows", () => {
586
const inherited = commandItem("item-inherited");
587
const local = commandItem("item-local");
packages/client-runtime/src/state/orchestrationV2Projection.ts
+30
-1
@@ -256,6 +256,29 @@ function upsertVisibleTurnItem(
256
return updated;
257
}
258
259
+// A user item retained for a queued run stays hidden until the run leaves the
260
+// queue, and its update can arrive before or after that run update. Items in
261
+// `turnItems` were loaded or streamed, so they skip the partial-window check;
262
+// items never received stay absent.
263
+function restoreQueuedUserItems(
264
+ projection: OrchestrationV2ThreadProjection,
265
+ runId: OrchestrationV2Run["id"],
266
+): OrchestrationV2ThreadProjection["visibleTurnItems"] {
267
+ let visibleTurnItems = projection.visibleTurnItems;
268
+ for (const item of projection.turnItems) {
269
+ if (item.runId !== runId || item.type !== "user_message" || item.inputIntent !== "queued_turn")
270
+ continue;
271
+ if (!shouldShowLocalTurnItem(projection, item)) continue;
272
+ visibleTurnItems = upsertVisibleTurnItem(
273
+ { ...projection, visibleTurnItems },
274
+ item,
275
+ false,
276
+ null,
277
+ );
278
+ }
279
+ return visibleTurnItems;
280
+}
281
+
282
export function applyOrchestrationV2ProjectionEvent(
283
projection: OrchestrationV2ThreadProjection | null,
284
event: OrchestrationV2DomainEvent,
@@ -301,7 +324,13 @@ export function applyOrchestrationV2ProjectionEvent(
324
case "run.created":
325
case "run.updated": {
326
const next = { ...base, runs: upsertEntity(base.runs, event.payload) };
304
- return { ...next, visibleTurnItems: activeVisibleTurnItems(next) };
327
+ const visible = { ...next, visibleTurnItems: activeVisibleTurnItems(next) };
328
+ const wasQueued = base.runs.some(
329
+ (run) => run.id === event.payload.id && run.status === "queued",
330
+ );
331
+ return wasQueued && event.payload.status !== "queued"
332
+ ? { ...visible, visibleTurnItems: restoreQueuedUserItems(visible, event.payload.id) }
333
+ : visible;
334
}
335
case "run.background-work-cancelled":
336
return {
packages/shared/src/orchestrationV2Timeline.test.ts
+8
-4
@@ -14,7 +14,6 @@ type VisibilityInput = Parameters<typeof createOrchestrationV2TurnItemVisibility
14
15
const ordinaryStatuses = [
16
"preparing",
17
- "queued",
17
"starting",
18
"running",
19
"waiting",
@@ -94,6 +93,7 @@ describe.each([
93
94
it.each([
95
...ordinaryStatuses.map((status) => ({ status, hidden: [] })),
96
+ { status: "queued", hidden: ["queued"] },
97
{ status: "cancelled", hidden: ["queued"] },
98
{
99
status: "rolled_back",
@@ -217,11 +217,15 @@ describe.each([
217
218
describe("indexed timeline visibility", () => {
219
it.each([
220
- ...ordinaryStatuses.flatMap(
220
+ ...[...ordinaryStatuses, "queued" as const].flatMap(
221
(status) =>
222
[
223
- { statuses: ["rolled_back", status], visible: true, ordinaryVisible: true },
224
- { statuses: ["cancelled", status], visible: true, ordinaryVisible: true },
223
+ {
224
+ statuses: ["rolled_back", status],
225
+ visible: status !== "queued",
226
+ ordinaryVisible: true,
227
+ },
228
+ { statuses: ["cancelled", status], visible: status !== "queued", ordinaryVisible: true },
229
{ statuses: [status, "rolled_back"], visible: false, ordinaryVisible: false },
230
{ statuses: [status, "cancelled"], visible: false, ordinaryVisible: true },
231
] as const,
packages/shared/src/orchestrationV2Timeline.ts
+8
-5
@@ -11,7 +11,7 @@ type TimelineTurnItem = Pick<OrchestrationV2TurnItem, "type" | "runId" | "nodeId
11
readonly inputIntent?: OrchestrationV2UserMessageInputIntent;
12
};
13
14
-export function isOrchestrationV2SupersededInterrupt(input: {
14
+function isOrchestrationV2SupersededInterrupt(input: {
15
readonly item: TimelineTurnItem;
16
readonly attempts: ReadonlyArray<TimelineRunAttempt>;
17
readonly items: ReadonlyArray<TimelineTurnItem>;
@@ -52,12 +52,15 @@ export function isOrchestrationV2TurnItemVisible(input: {
52
return false;
53
}
54
55
- // Cancelled queued input never reached the provider and must stay out of the transcript.
55
+ // A steer can return to the queue before delivery. Keep its retained item out
56
+ // of conversation history until its run starts; cancelled queued inputs stay hidden.
57
if (
58
item.type === "user_message" &&
59
item.inputIntent === "queued_turn" &&
60
item.runId !== null &&
60
- input.runs.some((run) => run.id === item.runId && run.status === "cancelled")
61
+ input.runs.some(
62
+ (run) => run.id === item.runId && (run.status === "queued" || run.status === "cancelled"),
63
+ )
64
) {
65
return false;
66
}
@@ -76,7 +79,7 @@ export function createOrchestrationV2TurnItemVisibility(input: {
79
}): (item: TimelineTurnItem) => boolean {
80
const statuses = new Map<TimelineRun["id"], TimelineRun["status"]>();
81
for (const run of input.runs) {
79
- if (run.status === "rolled_back" || run.status === "cancelled")
82
+ if (run.status === "rolled_back" || run.status === "cancelled" || run.status === "queued")
83
statuses.set(run.id, run.status);
84
else statuses.delete(run.id);
85
}
@@ -101,7 +104,7 @@ export function createOrchestrationV2TurnItemVisibility(input: {
104
const status = item.runId === null ? undefined : statuses.get(item.runId);
105
if (status === "rolled_back") return false;
106
if (
104
- status === "cancelled" &&
107
+ (status === "queued" || status === "cancelled") &&
108
item.type === "user_message" &&
109
item.inputIntent === "queued_turn"
110
)