@devries-cameron20 / t3code Mirror · read-only / commits / 9fb4492329
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 18:30 UTC

fix(chat): requeued steers no longer look delivered (#23420)

Co-authored-by: maria-rcks <maria@kuuro.net> Co-authored-by: Julius Marminge <julius0216@outlook.com>

Alb11747 committed Nov 12, 2026 at 18:47 UTC 9fb4492329e5847efeabf2b4d4f5c40a58dc883e
12 files changed +629 -81
apps/server/src/orchestration-v2/Orchestrator.ts
+16 -1
@@ -1973,13 +1973,19 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
1973 const projection = yield* projectionStore.getThreadRecords(
1974 threadId,
1975 ["runs", "nodes", "attempts", "providerThreads", "turnItems", "messages"],
1976 - { turnItemTypes: [], messageRoles: ["user"] },
1976 + { turnItemTypes: ["user_message"], messageRoles: ["user"] },
1977 );
1978 const queuedRun =
1979 queuedRunId === undefined
1980 ? nextQueuedRun(projection)
1981 : projection.runs.find((run) => run.id === queuedRunId && run.status === "queued");
1982 if (queuedRun === undefined) return;
1983 + // A steer that fell back to the queue keeps its user item, hidden while the
1984 + // run is queued. Republish it before the failure so clients whose bounded
1985 + // snapshot omitted it can show the prompt beside the error.
1986 + const retainedUserItems = projection.turnItems.filter(
1987 + (item) => item.type === "user_message" && item.runId === queuedRun.id,
1988 + );
1989 const now = yield* DateTime.now;
1990 const rootNode = projection.nodes.find((node) => node.id === queuedRun.rootNodeId);
1991 const attempt = projection.attempts.find((entry) => entry.id === queuedRun.activeAttemptId);
@@ -2005,6 +2011,15 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
2011 class: handoffUnsupported ? "validation_error" : "unknown",
2012 });
2013 yield* writeSystemEvents([
2014 + ...retainedUserItems.map((item) => ({
2015 + type: "turn-item.updated" as const,
2016 + threadId,
2017 + runId: queuedRun.id,
2018 + ...(rootNode === undefined ? {} : { nodeId: rootNode.id }),
2019 + providerInstanceId: queuedRun.providerInstanceId,
2020 + occurredAt: now,
2021 + payload: { ...item, status: "failed" as const, completedAt: now, updatedAt: now },
2022 + })),
2023 ...(attempt !== undefined && rootNode !== undefined
2024 ? [
2025 {
apps/server/src/orchestration-v2/ProjectionStore.test.ts
+279 -25
@@ -5018,26 +5018,36 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
5018 })}
5019 )
5020 `;
5021 - const cancelledSuffixSnapshot = yield* projectionStore.getThreadSnapshotWindow(threadId, {
5022 - rowLimit: sqlPageLimit,
5023 - });
5024 - const cancelledSuffixPage = buildBoundedThreadProjection({
5025 - policy: { ...THREAD_HISTORY_PAGE_POLICY, maxUserTurns: undefined },
5026 - projection: cancelledSuffixSnapshot.projection,
5027 - snapshotSequence: cancelledSuffixSnapshot.snapshotSequence,
5028 - });
5029 - assert.strictEqual(cancelledSuffixPage.latestLocalTurnOrdinal, 1_202);
5030 - assert.lengthOf(cancelledSuffixPage.projection.visibleTurnItems, 75);
5031 - assert.isFalse(
5032 - cancelledSuffixPage.projection.visibleTurnItems.some((row) =>
5033 - String(row.sourceItemId).includes(":cancelled:"),
5034 - ),
5035 - );
5036 - assert.isTrue(
5037 - cancelledSuffixPage.projection.visibleTurnItems.some(
5038 - (row) => row.sourceItemId === runlessQueuedId,
5039 - ),
5040 - );
5021 + // A steer that fell back to the queue keeps its queued-turn item while its
5022 + // run waits. End on cancelled for the checks below.
5023 + for (const status of ["queued", "cancelled"] as const) {
5024 + yield* sql`
5025 + UPDATE orchestration_v2_projection_runs
5026 + SET status = ${status}, payload_json = json_set(payload_json, '$.status', ${status})
5027 + WHERE run_id = ${cancelledRunId}
5028 + `;
5029 + const cancelledSuffixSnapshot = yield* projectionStore.getThreadSnapshotWindow(threadId, {
5030 + rowLimit: sqlPageLimit,
5031 + });
5032 + const cancelledSuffixPage = buildBoundedThreadProjection({
5033 + policy: { ...THREAD_HISTORY_PAGE_POLICY, maxUserTurns: undefined },
5034 + projection: cancelledSuffixSnapshot.projection,
5035 + snapshotSequence: cancelledSuffixSnapshot.snapshotSequence,
5036 + });
5037 + assert.strictEqual(cancelledSuffixPage.latestLocalTurnOrdinal, 1_202);
5038 + assert.lengthOf(cancelledSuffixPage.projection.visibleTurnItems, 75);
5039 + assert.isFalse(
5040 + cancelledSuffixPage.projection.visibleTurnItems.some((row) =>
5041 + String(row.sourceItemId).includes(":cancelled:"),
5042 + ),
5043 + status,
5044 + );
5045 + assert.isTrue(
5046 + cancelledSuffixPage.projection.visibleTurnItems.some(
5047 + (row) => row.sourceItemId === runlessQueuedId,
5048 + ),
5049 + );
5050 + }
5051
5052 // A steer cancels the queued run before it starts while the steered run keeps writing, so
5053 // the queued message leaves the window but its run is still the shell's latest run.
@@ -5830,6 +5840,7 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
5840 : [
5841 "rolled back",
5842 "cancelled queued turn",
5843 + "held queued turn",
5844 "replaced attempt",
5845 "stopped",
5846 "stopped",
@@ -5860,9 +5871,11 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
5871 ? "rolled_back"
5872 : kind === "cancelled queued turn"
5873 ? "cancelled"
5863 - : kind === "short" && chance(0.5)
5864 - ? "interrupted"
5865 - : "completed";
5874 + : kind === "held queued turn"
5875 + ? "queued"
5876 + : kind === "short" && chance(0.5)
5877 + ? "interrupted"
5878 + : "completed";
5879 runs.push({
5880 id: runId,
5881 threadId,
@@ -5941,7 +5954,12 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
5954 ? run % 10 !== 0
5955 : kind !== "late wake" && (agentDriven ? run > 1 : chance(0.2));
5956 if (wakes) notification();
5944 - else userMessage(kind === "cancelled queued turn" ? "queued_turn" : "turn_start");
5957 + else
5958 + userMessage(
5959 + kind === "cancelled queued turn" || kind === "held queued turn"
5960 + ? "queued_turn"
5961 + : "turn_start",
5962 + );
5963 // A stopped run never folds, so it is long enough for page ends to fall inside it.
5964 const work =
5965 kind === "late wake"
@@ -6573,7 +6591,13 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
6591 assert.isNotEmpty(countQueries);
6592 for (const countQuery of countQueries) {
6593 for (const detail of yield* planOf(countQuery)) {
6576 - assert.notInclude(detail, "orchestration_v2_projection_turn_items");
6594 + if (detail.includes("orchestration_v2_projection_turn_items")) {
6595 + // Only held input is read from the sparse index; transcript history stays unread.
6596 + assert.include(
6597 + detail,
6598 + "SEARCH held USING INDEX orchestration_v2_projection_turn_items_queued_input_idx (run_id=?)",
6599 + );
6600 + }
6601 }
6602 }
6603 }),
@@ -10421,6 +10445,236 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
10445 }),
10446 );
10447
10448 + it.effect("keeps undelivered queued input out of forks taken after a later run", () =>
10449 + Effect.gen(function* () {
10450 + const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
10451 + const now = yield* DateTime.now;
10452 + const projectId = ProjectId.make("project:projection-fork-held-queue");
10453 +
10454 + for (const heldStatus of ["queued", "cancelled"] as const) {
10455 + const prefix = `projection-fork-held-queue:${heldStatus}`;
10456 + const sourceThreadId = ThreadId.make(`thread:${prefix}:source`);
10457 + const targetThreadId = ThreadId.make(`thread:${prefix}:target`);
10458 + const heldRunId = RunId.make(`run:${prefix}:held`);
10459 + const heldAttemptId = RunAttemptId.make(`attempt:${prefix}:held`);
10460 + const answeredRunId = RunId.make(`run:${prefix}:answered`);
10461 +
10462 + for (const [threadId, forkedFrom] of [
10463 + [sourceThreadId, null],
10464 + [targetThreadId, { type: "run", threadId: sourceThreadId, runId: answeredRunId }],
10465 + ] as const) {
10466 + yield* projectionStore.apply({
10467 + id: EventId.make(`event:${threadId}:created`),
10468 + type: "thread.created",
10469 + threadId,
10470 + occurredAt: now,
10471 + payload: {
10472 + createdBy: "user",
10473 + creationSource: "web",
10474 + id: threadId,
10475 + projectId,
10476 + title: threadId,
10477 + providerInstanceId,
10478 + modelSelection,
10479 + runtimeMode: "full-access",
10480 + interactionMode: "default",
10481 + branch: null,
10482 + worktreePath: null,
10483 + activeProviderThreadId: null,
10484 + lineage: {
10485 + parentThreadId: forkedFrom === null ? null : sourceThreadId,
10486 + relationshipToParent: forkedFrom === null ? null : "fork",
10487 + rootThreadId: sourceThreadId,
10488 + },
10489 + forkedFrom,
10490 + createdAt: now,
10491 + updatedAt: now,
10492 + archivedAt: null,
10493 + settledOverride: null,
10494 + settledAt: null,
10495 + lastVisitedAt: null,
10496 + deletedAt: null,
10497 + },
10498 + });
10499 + }
10500 +
10501 + const applyRun = (
10502 + runId: RunId,
10503 + ordinal: number,
10504 + status: typeof heldStatus | "completed" | "rolled_back",
10505 + ) =>
10506 + projectionStore.apply({
10507 + id: EventId.make(`event:${runId}:${status}`),
10508 + type: "run.updated",
10509 + threadId: sourceThreadId,
10510 + runId,
10511 + nodeId: NodeId.make(`node:${runId}`),
10512 + driver,
10513 + occurredAt: now,
10514 + payload: {
10515 + id: runId,
10516 + threadId: sourceThreadId,
10517 + ordinal,
10518 + providerInstanceId,
10519 + modelSelection,
10520 + providerThreadId: null,
10521 + userMessageId: MessageId.make(`message:${runId}`),
10522 + rootNodeId: NodeId.make(`node:${runId}`),
10523 + activeAttemptId: runId === heldRunId ? heldAttemptId : null,
10524 + status,
10525 + requestedAt: now,
10526 + startedAt: runId === heldRunId ? null : now,
10527 + completedAt: status === "queued" ? null : now,
10528 + checkpointId: null,
10529 + contextHandoffId: null,
10530 + },
10531 + });
10532 + const applyUserItem = (input: {
10533 + readonly threadId: ThreadId;
10534 + readonly runId: RunId | null;
10535 + readonly ordinal: number;
10536 + readonly text: string;
10537 + readonly inputIntent: "turn_start" | "queued_turn";
10538 + }) =>
10539 + projectionStore.apply({
10540 + id: EventId.make(`event:${input.threadId}:item:${input.ordinal}`),
10541 + type: "turn-item.updated",
10542 + threadId: input.threadId,
10543 + ...(input.runId === null ? {} : { runId: input.runId }),
10544 + occurredAt: now,
10545 + payload: {
10546 + createdBy: "user",
10547 + creationSource: "web",
10548 + id: TurnItemId.make(`turn-item:${input.threadId}:${input.ordinal}`),
10549 + threadId: input.threadId,
10550 + runId: input.runId,
10551 + nodeId: null,
10552 + providerThreadId: null,
10553 + providerTurnId: null,
10554 + nativeItemRef: null,
10555 + parentItemId: null,
10556 + ordinal: input.ordinal,
10557 + status: "completed",
10558 + title: null,
10559 + startedAt: now,
10560 + completedAt: now,
10561 + updatedAt: now,
10562 + type: "user_message",
10563 + messageId: MessageId.make(`message:${input.threadId}:${input.ordinal}`),
10564 + inputIntent: input.inputIntent,
10565 + text: input.text,
10566 + attachments: [],
10567 + },
10568 + });
10569 +
10570 + yield* applyRun(heldRunId, 1, heldStatus);
10571 + yield* projectionStore.apply({
10572 + id: EventId.make(`event:${heldAttemptId}:created`),
10573 + type: "run-attempt.created",
10574 + threadId: sourceThreadId,
10575 + runId: heldRunId,
10576 + nodeId: NodeId.make(`node:${heldRunId}`),
10577 + driver,
10578 + occurredAt: now,
10579 + payload: {
10580 + id: heldAttemptId,
10581 + runId: heldRunId,
10582 + attemptOrdinal: 1,
10583 + rootNodeId: NodeId.make(`node:${heldRunId}`),
10584 + providerInstanceId,
10585 + providerThreadId: ProviderThreadId.make(`provider-thread:${prefix}`),
10586 + providerTurnId: null,
10587 + reason: "initial",
10588 + status: heldStatus === "queued" ? "pending" : "cancelled",
10589 + startedAt: null,
10590 + completedAt: heldStatus === "queued" ? null : now,
10591 + },
10592 + });
10593 + yield* applyRun(answeredRunId, 2, "completed");
10594 + yield* applyUserItem({
10595 + threadId: sourceThreadId,
10596 + runId: heldRunId,
10597 + ordinal: 100,
10598 + text: "held",
10599 + inputIntent: "queued_turn",
10600 + });
10601 + yield* applyUserItem({
10602 + threadId: sourceThreadId,
10603 + runId: answeredRunId,
10604 + ordinal: 200,
10605 + text: "answered",
10606 + inputIntent: "turn_start",
10607 + });
10608 + yield* applyUserItem({
10609 + threadId: sourceThreadId,
10610 + runId: answeredRunId,
10611 + ordinal: 201,
10612 + text: "delivered steer",
10613 + inputIntent: "queued_turn",
10614 + });
10615 + // Enough local turns for a turn-limited window to count inherited rows in SQL.
10616 + for (let index = 0; index < THREAD_HISTORY_PAGE_POLICY.maxUserTurns; index += 1) {
10617 + yield* applyUserItem({
10618 + threadId: targetThreadId,
10619 + runId: null,
10620 + ordinal: index,
10621 + text: `local ${index}`,
10622 + inputIntent: "turn_start",
10623 + });
10624 + }
10625 +
10626 + const assertForkHidesHeldInput = Effect.fn("assertForkHidesHeldInput")(function* () {
10627 + const target = yield* projectionStore.getThreadProjection(targetThreadId);
10628 + assert.deepEqual(
10629 + target.visibleTurnItems
10630 + .slice(0, 3)
10631 + .map((row) => (row.item.type === "user_message" ? row.item.text : row.item.type)),
10632 + ["answered", "delivered steer", "fork"],
10633 + );
10634 + const page = yield* projectionStore.getTimelinePage(targetThreadId, {
10635 + view: "activity",
10636 + limit: target.visibleTurnItems.length + 1,
10637 + });
10638 + assert.deepEqual(page.items, target.visibleTurnItems);
10639 + const bounded = yield* projectionStore.getThreadSnapshotWindow(targetThreadId, {
10640 + rowLimit: target.visibleTurnItems.length + 1,
10641 + });
10642 + assert.deepEqual(bounded.projection.visibleTurnItems, target.visibleTurnItems);
10643 + const shellSnapshot = yield* projectionStore.getShellSnapshot();
10644 + for (const projection of [
10645 + yield* projectionStore.getThreadProjection(sourceThreadId),
10646 + target,
10647 + ]) {
10648 + for (const shell of [
10649 + yield* projectionStore.getThreadShell(projection.thread.id),
10650 + shellSnapshot.threads.find((thread) => thread.id === projection.thread.id),
10651 + ]) {
10652 + assert.equal(shell?.visibleItemCount, projection.visibleTurnItems.length);
10653 + }
10654 + }
10655 + const filledLocalPage = yield* projectionStore.getThreadSnapshotWindow(targetThreadId, {
10656 + rowLimit: target.visibleTurnItems.length,
10657 + userTurnLimit: THREAD_HISTORY_PAGE_POLICY.maxUserTurns,
10658 + });
10659 + assert.deepEqual(
10660 + filledLocalPage.projection.visibleTurnItems.filter((row) => row.visibility === "local"),
10661 + target.visibleTurnItems.filter((row) => row.visibility === "local"),
10662 + );
10663 + });
10664 +
10665 + yield* assertForkHidesHeldInput();
10666 + // Forks keep rolled-back source history, but still not undelivered input.
10667 + yield* applyRun(answeredRunId, 2, "rolled_back");
10668 + yield* assertForkHidesHeldInput();
10669 + // Rollback also takes cancelled runs, and overwrites the status that hid their input.
10670 + if (heldStatus === "cancelled") {
10671 + yield* applyRun(heldRunId, 1, "rolled_back");
10672 + yield* assertForkHidesHeldInput();
10673 + }
10674 + }
10675 + }),
10676 + );
10677 +
10678 it.effect("hides source turns rewound before the fork from a fork's visible items", () =>
10679 Effect.gen(function* () {
10680 const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
apps/server/src/orchestration-v2/ProjectionStore.ts
+113 -45
@@ -78,10 +78,7 @@ import {
78 ProjectId,
79 ProviderInstanceId,
80 } from "@t3tools/contracts";
81 -import {
82 - createOrchestrationV2TurnItemVisibility,
83 - isOrchestrationV2SupersededInterrupt,
84 -} from "@t3tools/shared/orchestrationV2Timeline";
81 +import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline";
82 import {
83 deriveBlockedOnThreadWait,
84 derivePendingBackgroundWork,
@@ -1461,6 +1458,28 @@ function isTurnItemAtOrBeforeRun(input: {
1458 return ordinal !== undefined && ordinal <= input.sourceRunOrdinal;
1459 }
1460
1461 +// Run ordinals enforce the fork cutoff; this keeps later rollbacks from hiding
1462 +// inherited history while retaining cancellation evidence for undelivered input.
1463 +function createInheritedTurnItemVisibility(
1464 + source: Parameters<typeof createOrchestrationV2TurnItemVisibility>[0],
1465 +) {
1466 + const cancelledRunIds = new Set(
1467 + source.attempts
1468 + .filter((attempt) => attempt.status === "cancelled")
1469 + .map((attempt) => attempt.runId),
1470 + );
1471 + return createOrchestrationV2TurnItemVisibility({
1472 + ...source,
1473 + runs: source.runs.flatMap((run) =>
1474 + run.status !== "rolled_back"
1475 + ? [run]
1476 + : cancelledRunIds.has(run.id)
1477 + ? [{ ...run, status: "cancelled" as const }]
1478 + : [],
1479 + ),
1480 + });
1481 +}
1482 +
1483 // Rollback stamps completedAt; a fork made after the rewind never received that run.
1484 function runRolledBackAt(run: Pick<OrchestrationV2Run, "status" | "completedAt">) {
1485 return run.status === "rolled_back" ? run.completedAt : null;
@@ -1495,6 +1514,11 @@ function visibleTurnItemsThroughRun(input: {
1514 }
1515
1516 const runOrdinalById = forkInheritedRunOrdinals(input.sourceProjection.runs, input.forkCreatedAt);
1517 + const isVisible = createInheritedTurnItemVisibility({
1518 + runs: input.sourceProjection.runs,
1519 + attempts: input.sourceProjection.attempts,
1520 + items: input.sourceProjection.turnItems,
1521 + });
1522 const inheritedPrefix = input.sourceProjection.visibleTurnItems
1523 .filter(
1524 (row) => row.item.threadId !== input.sourceProjection.thread.id || row.item.type === "fork",
@@ -1506,23 +1530,16 @@ function visibleTurnItemsThroughRun(input: {
1530 item: row.item,
1531 }));
1532 const localPrefix = inheritedVisibleTurnItemsFromLocalItems(
1509 - input.sourceProjection.turnItems.filter((item) => {
1510 - if (
1511 - isOrchestrationV2SupersededInterrupt({
1512 - item,
1513 - attempts: input.sourceProjection.attempts,
1514 - items: input.sourceProjection.turnItems,
1515 - })
1516 - ) {
1517 - return false;
1518 - }
1519 - return isTurnItemAtOrBeforeRun({
1520 - historyOrigin: input.sourceProjection.thread.historyOrigin,
1521 - itemRunId: item.runId,
1522 - runOrdinalById,
1523 - sourceRunOrdinal: sourceRun.ordinal,
1524 - });
1525 - }),
1533 + input.sourceProjection.turnItems.filter(
1534 + (item) =>
1535 + isVisible(item) &&
1536 + isTurnItemAtOrBeforeRun({
1537 + historyOrigin: input.sourceProjection.thread.historyOrigin,
1538 + itemRunId: item.runId,
1539 + runOrdinalById,
1540 + sourceRunOrdinal: sourceRun.ordinal,
1541 + }),
1542 + ),
1543 );
1544
1545 return [...inheritedPrefix, ...localPrefix];
@@ -3122,16 +3139,27 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
3139 ),
3140 9223372036854775807
3141 )
3142 + -- The fork cutoff below filters rewound runs; undelivered queued input stays hidden.
3143 AND (
3144 ${window.requiredRunId ?? null} IS NOT NULL
3127 - OR (
3128 - (run.status IS NULL OR run.status <> 'rolled_back')
3129 - AND NOT (
3130 - item.type = 'user_message'
3131 - AND json_extract(item.payload_json, '$.inputIntent') = 'queued_turn'
3132 - AND run.status IS 'cancelled'
3145 + OR run.status IS NULL
3146 + OR run.status <> 'rolled_back'
3147 + )
3148 + AND NOT (
3149 + item.type = 'user_message'
3150 + AND (
3151 + COALESCE(run.status IN ('queued', 'cancelled'), 0)
3152 + OR (
3153 + run.status = 'rolled_back'
3154 + AND EXISTS (
3155 + SELECT 1
3156 + FROM orchestration_v2_projection_run_attempts AS attempt
3157 + WHERE attempt.run_id = run.run_id
3158 + AND attempt.status = 'cancelled'
3159 + )
3160 )
3161 )
3162 + AND json_extract(item.payload_json, '$.inputIntent') = 'queued_turn'
3163 )
3164 AND (
3165 ${window.requiredRunId ?? null} IS NULL
@@ -6368,18 +6396,18 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6396 let inherited: Array<TimelineIndexRow> = [];
6397 if (sourceRun !== undefined) {
6398 const ordinals = forkInheritedRunOrdinals(source.records.runs, records.thread.createdAt);
6371 - const sourceItems = source.local.map((row) => row.item);
6399 + const isInheritedVisible = createInheritedTurnItemVisibility({
6400 + runs: source.records.runs,
6401 + attempts: source.records.attempts,
6402 + items: source.local.map((row) => row.item),
6403 + });
6404 inherited = [
6405 ...source.visible.filter(
6406 (row) => row.item.threadId !== fork.threadId || row.item.type === "fork",
6407 ),
6408 ...source.local.filter(
6409 (row) =>
6378 - !isOrchestrationV2SupersededInterrupt({
6379 - item: row.item,
6380 - attempts: source.records.attempts,
6381 - items: sourceItems,
6382 - }) &&
6410 + isInheritedVisible(row.item) &&
6411 isTurnItemAtOrBeforeRun({
6412 historyOrigin: source.records.thread.historyOrigin,
6413 itemRunId: row.item.runId,
@@ -6576,13 +6604,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6604 })),
6605 });
6606 // Only these runs can hide rows: every row of a rolled-back run, the queued turn of a
6579 - // cancelled run, and the stop result of a replaced attempt. So the hidden rows are found
6580 - // without reading the thread.
6607 + // queued or cancelled run, and the stop result of a replaced attempt. So the hidden rows
6608 + // are found without reading the thread.
6609 const stoppedRuns = new Set(stopRequests.map((row) => row.run_id));
6582 - const runsWithStatus = (status: "rolled_back" | "cancelled") =>
6583 - records.runs.flatMap((run) => (run.status === status ? [String(run.id)] : []));
6610 + const runsWithStatus = (...statuses: ReadonlyArray<OrchestrationV2Run["status"]>) =>
6611 + records.runs.flatMap((run) => (statuses.includes(run.status) ? [String(run.id)] : []));
6612 const rolledBack = runsWithStatus("rolled_back");
6585 - const cancelled = runsWithStatus("cancelled");
6613 + const heldInput = runsWithStatus("queued", "cancelled");
6614 const steered = [...new Set(superseded.map((attempt) => String(attempt.runId)))].filter(
6615 (runId) => !stoppedRuns.has(runId) && !rolledBack.includes(runId),
6616 );
@@ -6607,8 +6635,8 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6635 INDEXED BY orchestration_v2_projection_turn_items_thread_run_idx
6636 ON item.thread_id = ${threadId} AND item.run_id = wanted.value`,
6637 // The conversation index holds the type and run of these rows, so only the matches are
6610 - // fetched. Through the run index, every row of each cancelled run would be.
6611 - cancelled.length + steered.length === 0
6638 + // fetched. Through the run index, every row of each such run would be.
6639 + heldInput.length + steered.length === 0
6640 ? Effect.succeed([])
6641 : sql<HiddenCandidate>`
6642 SELECT ${candidateColumns}
@@ -6617,7 +6645,7 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
6645 WHERE item.thread_id = ${threadId}
6646 AND item.type NOT IN ${CONVERSATION_INDEX_EXCLUDED_TYPES}
6647 AND ((item.type = 'user_message'
6620 - AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(cancelled)})))
6648 + AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(heldInput)})))
6649 OR (item.type = 'run_interrupt_result'
6650 AND item.run_id IN (SELECT value FROM json_each(${encodeIdList(steered)}))))`,
6651 ]);
@@ -7706,18 +7734,58 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
7734 WHERE thread_id IN (SELECT value FROM json_each(${encodeIdList(threadIds)}))
7735 `;
7736
7737 + // Forks do not inherit undelivered queued input, including from cancelled runs
7738 + // rolled back later. Checking the run first limits sparse-index seeks to those few runs.
7739 const selectShellRunItemCounts = (threadIds: ReadonlyArray<ThreadId>) =>
7740 sql<ShellRunItemCountRow>`
7711 - SELECT thread_id, run_key AS run_id, item_count
7712 - FROM orchestration_v2_projection_turn_item_counts
7713 - WHERE run_key <> '' AND thread_id IN (SELECT value FROM json_each(${encodeIdList(threadIds)}))
7741 + SELECT per_run.thread_id, per_run.run_key AS run_id,
7742 + per_run.item_count - CASE
7743 + WHEN EXISTS (
7744 + SELECT 1
7745 + FROM orchestration_v2_projection_runs AS run
7746 + WHERE run.run_id = per_run.run_key
7747 + AND (
7748 + run.status IN ('queued', 'cancelled')
7749 + OR (
7750 + run.status = 'rolled_back'
7751 + AND EXISTS (
7752 + SELECT 1
7753 + FROM orchestration_v2_projection_run_attempts AS attempt
7754 + WHERE attempt.run_id = run.run_id
7755 + AND attempt.status = 'cancelled'
7756 + )
7757 + )
7758 + )
7759 + ) THEN (
7760 + SELECT COUNT(*)
7761 + FROM orchestration_v2_projection_turn_items AS held
7762 + INDEXED BY orchestration_v2_projection_turn_items_queued_input_idx
7763 + WHERE held.run_id = per_run.run_key
7764 + AND held.type = 'user_message'
7765 + AND json_extract(held.payload_json, '$.inputIntent') = 'queued_turn'
7766 + )
7767 + ELSE 0
7768 + END AS item_count
7769 + FROM orchestration_v2_projection_turn_item_counts AS per_run
7770 + WHERE per_run.run_key <> '' AND per_run.thread_id IN (SELECT value FROM json_each(${encodeIdList(threadIds)}))
7771 `;
7772
7773 const selectShellItemCounts = (threadIds: ReadonlyArray<ThreadId>) =>
7774 sql<ShellItemCountRow>`
7775 SELECT counts.thread_id,
7776 SUM(CASE WHEN counts.run_key = '' OR r.status <> 'rolled_back'
7720 - THEN counts.item_count ELSE 0 END) AS item_count,
7777 + THEN counts.item_count - CASE
7778 + WHEN r.status IN ('queued', 'cancelled') THEN (
7779 + SELECT COUNT(*)
7780 + FROM orchestration_v2_projection_turn_items held
7781 + INDEXED BY orchestration_v2_projection_turn_items_queued_input_idx
7782 + WHERE held.run_id = counts.run_key
7783 + AND held.type = 'user_message'
7784 + AND json_extract(held.payload_json, '$.inputIntent') = 'queued_turn'
7785 + )
7786 + ELSE 0
7787 + END
7788 + ELSE 0 END) AS item_count,
7789 SUM(CASE WHEN counts.run_key = '' THEN counts.item_count ELSE 0 END)
7790 AS runless_item_count
7791 FROM orchestration_v2_projection_turn_item_counts counts
apps/server/src/orchestration-v2/SteeringCompletion.integration.test.ts
+15
@@ -379,6 +379,13 @@ it.effect.each(
379 }
380 if (delivery !== null && !question) {
381 const requeued = yield* orchestrator.getThreadProjection(threadId);
382 + assert.isTrue(requeued.runs.some((run) => run.status === "queued"));
383 + assert.isFalse(
384 + requeued.visibleTurnItems.some(
385 + ({ item }) => item.type === "user_message" && item.messageId === messageId,
386 + ),
387 + "a steer that fell back to the queue must not appear delivered",
388 + );
389 const item = requeued.turnItems.find(
390 (item) => item.type === "user_message" && item.messageId === messageId,
391 );
@@ -462,6 +469,14 @@ it.effect.each(
469 ).length,
470 1,
471 );
472 + if (!mailbox && !question) {
473 + assert.equal(
474 + final.visibleTurnItems.filter(
475 + ({ item }) => item.type === "user_message" && item.messageId === messageId,
476 + ).length,
477 + 1,
478 + );
479 + }
480 yield* worker.drain();
481 assert.equal(started.length, 2);
482 }).pipe(
apps/server/src/orchestration-v2/testkit/ProviderSwitch.integration.test.ts
+44
@@ -3938,6 +3938,11 @@ describe("orchestration v2 provider switching", () => {
3938 );
3939 assert.equal(queuedItem?.runId, queued.runs[1]?.id);
3940 assert.equal(queuedItem?.providerThreadId, queued.runs[1]?.providerThreadId);
3941 + assert.equal(queuedItem?.status, "pending");
3942 + assert.isNull(queuedItem?.startedAt);
3943 + assert.isNull(queuedItem?.completedAt);
3944 + assert.isFalse(queued.visibleTurnItems.some((row) => row.item.id === queuedItem?.id));
3945 + const queuedSequence = yield* eventSink.latestSequence({ threadId: rejectedThreadId });
3946 yield* eventSink.write({
3947 events: [
3948 {
@@ -3962,6 +3967,40 @@ describe("orchestration v2 provider switching", () => {
3967 Stream.runHead,
3968 );
3969 yield* worker.drain();
3970 + // Clients whose bounded snapshot omitted the hidden queued item must
3971 + // receive it while the run is still queued, before it fails.
3972 + const throughSequence = yield* eventSink.latestSequence({ threadId: rejectedThreadId });
3973 + const failureEvents = yield* eventSink
3974 + .stream({ threadId: rejectedThreadId, afterSequence: queuedSequence })
3975 + .pipe(
3976 + Stream.takeUntil((stored) => stored.sequence === throughSequence),
3977 + Stream.map(({ event }) => event),
3978 + Stream.filter((event) => event.runId === queued.runs[1]!.id),
3979 + Stream.runCollect,
3980 + );
3981 + const republished = failureEvents.findIndex(
3982 + (event) =>
3983 + event.type === "turn-item.updated" &&
3984 + event.payload.type === "user_message" &&
3985 + event.payload.messageId === rejectedMessageId,
3986 + );
3987 + assert.isAtLeast(republished, 0);
3988 + const restored = failureEvents[republished];
3989 + assert.equal(restored?.type, "turn-item.updated");
3990 + if (restored?.type === "turn-item.updated") {
3991 + assert.deepEqual(restored.payload, {
3992 + ...queuedItem,
3993 + status: "failed",
3994 + completedAt: restored.occurredAt,
3995 + updatedAt: restored.occurredAt,
3996 + });
3997 + }
3998 + assert.isBelow(
3999 + republished,
4000 + failureEvents.findIndex(
4001 + (event) => event.type === "run.updated" && event.payload.status === "failed",
4002 + ),
4003 + );
4004 return yield* orchestrator.getThreadProjection(rejectedThreadId);
4005 }).pipe(
4006 Effect.provide(
@@ -4005,6 +4044,11 @@ describe("orchestration v2 provider switching", () => {
4044 failureItem?.type === "error" ? failureItem.failure.code : null,
4045 "context_handoff_unsupported",
4046 );
4047 + assert.isTrue(
4048 + projection.visibleTurnItems.some(
4049 + ({ item }) => item.type === "user_message" && item.messageId === rejectedMessageId,
4050 + ),
4051 + );
4052 assert.deepEqual(
4053 (yield* Ref.get(capturedTurns)).map((turn) => turn.driver),
4054 [CODEX_DRIVER, CODEX_DRIVER],
apps/server/src/persistence/Migrations.ts
+2
@@ -88,6 +88,7 @@ import Migration0082 from "./Migrations/082_TurnItemConversationIndex.ts";
88 import Migration0083 from "./Migrations/083_CommandReadIndexes.ts";
89 import Migration0084 from "./Migrations/084_LastExecutedRunIndex.ts";
90 import Migration0085 from "./Migrations/085_TurnItemCounts.ts";
91 +import Migration0086 from "./Migrations/086_QueuedTurnInputIndex.ts";
92
93 const migrationEntries = [
94 [1, "OrchestrationEvents", Migration0001],
@@ -176,6 +177,7 @@ const migrationEntries = [
177 [83, "CommandReadIndexes", Migration0083],
178 [84, "LastExecutedRunIndex", Migration0084],
179 [85, "TurnItemCounts", Migration0085],
180 + [86, "QueuedTurnInputIndex", Migration0086],
181 ] as const;
182
183 export const migrationManifest = migrationEntries.map(([id, name]) => [id, name] as const);
apps/server/src/persistence/Migrations/086_QueuedTurnInputIndex.ts new
+13
@@ -0,0 +1,13 @@
1 +import * as Effect from "effect/Effect";
2 +import * as SqlClient from "effect/sql/SqlClient";
3 +
4 +export default Effect.gen(function* () {
5 + const sql = yield* SqlClient.SqlClient;
6 +
7 + // Shell counts subtract held input without reading a run's tool history or item payloads.
8 + yield* sql`
9 + CREATE INDEX orchestration_v2_projection_turn_items_queued_input_idx
10 + ON orchestration_v2_projection_turn_items(run_id)
11 + WHERE type = 'user_message' AND json_extract(payload_json, '$.inputIntent') = 'queued_turn'
12 + `;
13 +});
apps/server/src/persistence/reconcileV2PreviewMigration.test.ts
+2
@@ -65,6 +65,7 @@ describe("V2 preview upgrade", () => {
65 [83, "CommandReadIndexes"],
66 [84, "LastExecutedRunIndex"],
67 [85, "TurnItemCounts"],
68 + [86, "QueuedTurnInputIndex"],
69 ]);
70 assert.deepStrictEqual(yield* runMigrations(), []);
71 assert.deepStrictEqual(yield* sql`SELECT * FROM orchestration_v2_legacy_imports`, imports);
@@ -173,6 +174,7 @@ describe("V2 preview upgrade", () => {
174 [83, "CommandReadIndexes"],
175 [84, "LastExecutedRunIndex"],
176 [85, "TurnItemCounts"],
177 + [86, "QueuedTurnInputIndex"],
178 ]);
179 }).pipe(Effect.provide(NodeSqliteClient.layer({ filename: ":memory:" }))),
180 );
packages/client-runtime/src/state/orchestrationV2Projection.test.ts
+99
@@ -483,6 +483,105 @@ describe("applyOrchestrationV2ProjectionEvent", () => {
483 });
484 });
485
486 + it("hides a steer that fell back to the queue until its run starts", () => {
487 + const queuedRunId = RunId.make("run-requeued-steer");
488 + const queuedRun = { ...run, id: queuedRunId, ordinal: 2, status: "queued" } as const;
489 + const reply = commandItem("item-reply", "reply", 102);
490 + const steer: OrchestrationV2TurnItem = {
491 + id: TurnItemId.make("item-steer"),
492 + threadId,
493 + runId,
494 + nodeId: null,
495 + providerThreadId: null,
496 + providerTurnId: null,
497 + nativeItemRef: null,
498 + parentItemId: null,
499 + ordinal: 101,
500 + status: "completed",
501 + title: null,
502 + startedAt: now,
503 + completedAt: now,
504 + updatedAt: now,
505 + type: "user_message",
506 + messageId: MessageId.make("message-steer"),
507 + inputIntent: "steer",
508 + text: "steer",
509 + attachments: [],
510 + createdBy: "user",
511 + creationSource: "web",
512 + };
513 + const event = (type: "run.updated" | "turn-item.updated", payload: unknown) =>
514 + ({
515 + id: `event-${type}`,
516 + type,
517 + threadId,
518 + occurredAt: now,
519 + payload,
520 + }) as OrchestrationV2DomainEvent;
521 + const apply = (
522 + projection: OrchestrationV2ThreadProjection | null,
523 + events: ReadonlyArray<OrchestrationV2DomainEvent>,
524 + options?: Parameters<typeof applyOrchestrationV2ProjectionEvent>[2],
525 + ) =>
526 + events.reduce(
527 + (next, item) => applyOrchestrationV2ProjectionEvent(next, item, options),
528 + projection,
529 + );
530 + const visibleIds = (projection: OrchestrationV2ThreadProjection | null) =>
531 + projection?.visibleTurnItems.map((row) => row.item.id);
532 +
533 + const accepted = apply(emptyProjection, [
534 + event("run.updated", { ...run, status: "running" }),
535 + event("turn-item.updated", steer),
536 + event("turn-item.updated", reply),
537 + ]);
538 + expect(visibleIds(accepted)).toEqual([steer.id, reply.id]);
539 +
540 + const queuedSteer = event("turn-item.updated", {
541 + ...steer,
542 + runId: queuedRunId,
543 + inputIntent: "queued_turn",
544 + status: "pending",
545 + startedAt: null,
546 + completedAt: null,
547 + });
548 + const requeued = apply(accepted, [event("run.updated", queuedRun), queuedSteer]);
549 + expect(visibleIds(requeued)).toEqual([reply.id]);
550 +
551 + // Servers publish the start's user item and run update in either order.
552 + const starting = event("run.updated", { ...queuedRun, status: "starting" });
553 + const dispatchedSteer = event("turn-item.updated", {
554 + ...steer,
555 + runId: queuedRunId,
556 + inputIntent: "queued_turn",
557 + });
558 + expect(visibleIds(apply(requeued, [dispatchedSteer, starting]))).toEqual([steer.id, reply.id]);
559 + expect(visibleIds(apply(requeued, [starting, dispatchedSteer]))).toEqual([steer.id, reply.id]);
560 + const cancelled = event("run.updated", { ...queuedRun, status: "cancelled" });
561 + expect(visibleIds(apply(requeued, [cancelled]))).toEqual([reply.id]);
562 +
563 + // A bounded snapshot taken while queued omits the hidden item, and the
564 + // newer reply puts it below the watermark.
565 + const bounded = {
566 + ...requeued!,
567 + turnItems: requeued!.turnItems.filter((item) => item.id !== steer.id),
568 + };
569 + const partial = { partialTimeline: true, latestLocalTurnOrdinal: reply.ordinal };
570 + const failed = event("run.updated", { ...queuedRun, status: "failed" });
571 + const failedSteer = event("turn-item.updated", {
572 + ...steer,
573 + runId: queuedRunId,
574 + inputIntent: "queued_turn",
575 + status: "failed",
576 + startedAt: null,
577 + });
578 + const retained = apply(bounded, [failedSteer], partial);
579 + expect(visibleIds(retained)).toEqual([reply.id]);
580 + expect(visibleIds(apply(retained, [failed], partial))).toEqual([steer.id, reply.id]);
581 + // Leaving the queue never invents an item the client did not receive.
582 + expect(visibleIds(apply(bounded, [failed], partial))).toEqual([reply.id]);
583 + });
584 +
585 it("removes only hidden local items while preserving inherited rows", () => {
586 const inherited = commandItem("item-inherited");
587 const local = commandItem("item-local");
packages/client-runtime/src/state/orchestrationV2Projection.ts
+30 -1
@@ -256,6 +256,29 @@ function upsertVisibleTurnItem(
256 return updated;
257 }
258
259 +// A user item retained for a queued run stays hidden until the run leaves the
260 +// queue, and its update can arrive before or after that run update. Items in
261 +// `turnItems` were loaded or streamed, so they skip the partial-window check;
262 +// items never received stay absent.
263 +function restoreQueuedUserItems(
264 + projection: OrchestrationV2ThreadProjection,
265 + runId: OrchestrationV2Run["id"],
266 +): OrchestrationV2ThreadProjection["visibleTurnItems"] {
267 + let visibleTurnItems = projection.visibleTurnItems;
268 + for (const item of projection.turnItems) {
269 + if (item.runId !== runId || item.type !== "user_message" || item.inputIntent !== "queued_turn")
270 + continue;
271 + if (!shouldShowLocalTurnItem(projection, item)) continue;
272 + visibleTurnItems = upsertVisibleTurnItem(
273 + { ...projection, visibleTurnItems },
274 + item,
275 + false,
276 + null,
277 + );
278 + }
279 + return visibleTurnItems;
280 +}
281 +
282 export function applyOrchestrationV2ProjectionEvent(
283 projection: OrchestrationV2ThreadProjection | null,
284 event: OrchestrationV2DomainEvent,
@@ -301,7 +324,13 @@ export function applyOrchestrationV2ProjectionEvent(
324 case "run.created":
325 case "run.updated": {
326 const next = { ...base, runs: upsertEntity(base.runs, event.payload) };
304 - return { ...next, visibleTurnItems: activeVisibleTurnItems(next) };
327 + const visible = { ...next, visibleTurnItems: activeVisibleTurnItems(next) };
328 + const wasQueued = base.runs.some(
329 + (run) => run.id === event.payload.id && run.status === "queued",
330 + );
331 + return wasQueued && event.payload.status !== "queued"
332 + ? { ...visible, visibleTurnItems: restoreQueuedUserItems(visible, event.payload.id) }
333 + : visible;
334 }
335 case "run.background-work-cancelled":
336 return {
packages/shared/src/orchestrationV2Timeline.test.ts
+8 -4
@@ -14,7 +14,6 @@ type VisibilityInput = Parameters<typeof createOrchestrationV2TurnItemVisibility
14
15 const ordinaryStatuses = [
16 "preparing",
17 - "queued",
17 "starting",
18 "running",
19 "waiting",
@@ -94,6 +93,7 @@ describe.each([
93
94 it.each([
95 ...ordinaryStatuses.map((status) => ({ status, hidden: [] })),
96 + { status: "queued", hidden: ["queued"] },
97 { status: "cancelled", hidden: ["queued"] },
98 {
99 status: "rolled_back",
@@ -217,11 +217,15 @@ describe.each([
217
218 describe("indexed timeline visibility", () => {
219 it.each([
220 - ...ordinaryStatuses.flatMap(
220 + ...[...ordinaryStatuses, "queued" as const].flatMap(
221 (status) =>
222 [
223 - { statuses: ["rolled_back", status], visible: true, ordinaryVisible: true },
224 - { statuses: ["cancelled", status], visible: true, ordinaryVisible: true },
223 + {
224 + statuses: ["rolled_back", status],
225 + visible: status !== "queued",
226 + ordinaryVisible: true,
227 + },
228 + { statuses: ["cancelled", status], visible: status !== "queued", ordinaryVisible: true },
229 { statuses: [status, "rolled_back"], visible: false, ordinaryVisible: false },
230 { statuses: [status, "cancelled"], visible: false, ordinaryVisible: true },
231 ] as const,
packages/shared/src/orchestrationV2Timeline.ts
+8 -5
@@ -11,7 +11,7 @@ type TimelineTurnItem = Pick<OrchestrationV2TurnItem, "type" | "runId" | "nodeId
11 readonly inputIntent?: OrchestrationV2UserMessageInputIntent;
12 };
13
14 -export function isOrchestrationV2SupersededInterrupt(input: {
14 +function isOrchestrationV2SupersededInterrupt(input: {
15 readonly item: TimelineTurnItem;
16 readonly attempts: ReadonlyArray<TimelineRunAttempt>;
17 readonly items: ReadonlyArray<TimelineTurnItem>;
@@ -52,12 +52,15 @@ export function isOrchestrationV2TurnItemVisible(input: {
52 return false;
53 }
54
55 - // Cancelled queued input never reached the provider and must stay out of the transcript.
55 + // A steer can return to the queue before delivery. Keep its retained item out
56 + // of conversation history until its run starts; cancelled queued inputs stay hidden.
57 if (
58 item.type === "user_message" &&
59 item.inputIntent === "queued_turn" &&
60 item.runId !== null &&
60 - input.runs.some((run) => run.id === item.runId && run.status === "cancelled")
61 + input.runs.some(
62 + (run) => run.id === item.runId && (run.status === "queued" || run.status === "cancelled"),
63 + )
64 ) {
65 return false;
66 }
@@ -76,7 +79,7 @@ export function createOrchestrationV2TurnItemVisibility(input: {
79 }): (item: TimelineTurnItem) => boolean {
80 const statuses = new Map<TimelineRun["id"], TimelineRun["status"]>();
81 for (const run of input.runs) {
79 - if (run.status === "rolled_back" || run.status === "cancelled")
82 + if (run.status === "rolled_back" || run.status === "cancelled" || run.status === "queued")
83 statuses.set(run.id, run.status);
84 else statuses.delete(run.id);
85 }
@@ -101,7 +104,7 @@ export function createOrchestrationV2TurnItemVisibility(input: {
104 const status = item.runId === null ? undefined : statuses.get(item.runId);
105 if (status === "rolled_back") return false;
106 if (
104 - status === "cancelled" &&
107 + (status === "queued" || status === "cancelled") &&
108 item.type === "user_message" &&
109 item.inputIntent === "queued_turn"
110 )