@devries-cameron20 / t3code-1 / commits / 06bd84623b
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

test(server): wire budget covers stale todo lists in long threads (#34522)

Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>

Theo Browne committed Nov 11, 2026 at 13:23 UTC 06bd84623b86bb9d1b1cae2a20b514e9058648c9
4 files changed +453 -2
apps/server/integration/TransferBudgetReport.integration.ts
+65
@@ -15,6 +15,9 @@ import {
15 LONG_TURN_WORK_ROWS,
16 SHELL_IDLE_SUBAGENTS,
17 SHELL_SETTLED_THREADS,
18 + STALE_TODO_LIVE_ROWS,
19 + STALE_TODO_SETTLED_RUNS,
20 + STALE_TODO_SUBAGENTS_PER_RUN,
21 } from "./TransferBudgetV2Fixture.integration.ts";
22
23 export interface WebSocketCatchUpMeasurement extends WebSocketTransferTotals {
@@ -26,6 +29,15 @@ export interface TimelineHttpMeasurement {
29 readonly rows: number;
30 }
31
32 +const STALE_TODO_RECORD_KINDS = [
33 + "nodes",
34 + "subagents",
35 + "plans",
36 + "providerTurns",
37 + "checkpoints",
38 +] as const;
39 +type StaleTodoRecordCounts = Readonly<Record<(typeof STALE_TODO_RECORD_KINDS)[number], number>>;
40 +
41 export interface TransferBudgetRun {
42 readonly provider: ProviderDriverKind;
43 readonly threadSnapshot: HttpTransferMeasurement;
@@ -46,6 +58,13 @@ export interface TransferBudgetRun {
58 /** The cold-connect shell snapshot after the settled-thread fleet is written. */
59 readonly fleetShell: TimelineHttpMeasurement;
60 readonly fleetShellLegacy: TimelineHttpMeasurement;
61 + /**
62 + * The compact bounded snapshot of the long thread whose finished runs left todo lists open,
63 + * with the count of each record kind that rides along with its rows.
64 + */
65 + readonly staleTodoColdOpen: TimelineHttpMeasurement & {
66 + readonly records: StaleTodoRecordCounts;
67 + };
68 }
69
70 interface ProviderTransferBudget {
@@ -84,6 +103,21 @@ const FLEET_SHELL_BUDGET = {
103 rows: 27,
104 };
105
106 +// About 14% above the observed 75 KB. The window holds the working run only: 2 nodes, 2 subagents,
107 +// 1 plan, 1 provider turn and the last finished turn's checkpoint. When finished runs ride along
108 +// it holds 82 nodes, 82 subagents, 41 plans, 41 provider turns, 40 checkpoints and 740 KB.
109 +const STALE_TODO_BUDGET = {
110 + decodedBytes: 85_000,
111 + rows: 75,
112 + records: {
113 + nodes: 4,
114 + subagents: 4,
115 + plans: 2,
116 + providerTurns: 2,
117 + checkpoints: 2,
118 + } satisfies StaleTodoRecordCounts,
119 +};
120 +
121 function totalWireBytes(run: TransferBudgetRun): number {
122 return run.threadSnapshot.wireBytes + run.measuredTurnWebSocket.wireBytes;
123 }
@@ -245,6 +279,20 @@ export function transferBudgetViolations(runs: ReadonlyArray<TransferBudgetRun>)
279 FLEET_SHELL_BUDGET.decodedBytes,
280 ],
281 ["fleet shell snapshot rows", run.fleetShell.rows, FLEET_SHELL_BUDGET.rows],
282 + [
283 + "stale todo list cold open decoded bytes",
284 + run.staleTodoColdOpen.http.decodedBodyBytes,
285 + STALE_TODO_BUDGET.decodedBytes,
286 + ],
287 + ["stale todo list cold open rows", run.staleTodoColdOpen.rows, STALE_TODO_BUDGET.rows],
288 + ...STALE_TODO_RECORD_KINDS.map(
289 + (kind) =>
290 + [
291 + `stale todo list cold open ${kind}`,
292 + run.staleTodoColdOpen.records[kind],
293 + STALE_TODO_BUDGET.records[kind],
294 + ] as const,
295 + ),
296 ] as const;
297 for (const [metric, observed, maximum] of checks) {
298 if (observed > maximum) {
@@ -264,6 +312,7 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
312 `Scenario: ${TRANSFER_HISTORY_TURN_COUNT} historical turns with ${TRANSFER_HISTORY_TOOLS_PER_TURN} command tools and one retained ${formatBytes(TRANSFER_HISTORY_MCP_RESULT_BYTES)} MCP result each, followed by one measured turn with ${TRANSFER_MEASURED_TOOLS} command tools and a retained ${formatBytes(TRANSFER_MEASURED_MCP_RESULT_BYTES)} MCP result. Synthetic V2 domain events exercise persistence, wire projection, and production HTTP/subscription handlers; this does not measure provider adapter ingestion. Payloads contain no user data.`,
313 `The long-turn rows open a second thread through the compact bounded snapshot web and mobile use, then load one older page. Its last user turn holds ${LONG_TURN_WORK_ROWS} rows of reasoning, commands, file changes, MCP calls, progress messages, checkpoints, and subagents.`,
314 `The fleet shell rows read the cold-connect shell after ${SHELL_SETTLED_THREADS} settled threads and ${SHELL_IDLE_SUBAGENTS} idle subagent children are added. Budgeted reads send the opt-ins current clients send; "older client" rows show the same read without the opt-ins added since row budgets.`,
315 + `The stale todo list rows open a third thread the same way: ${STALE_TODO_SETTLED_RUNS} finished runs, then one run still at work with ${STALE_TODO_LIVE_ROWS} rows. Every run left its todo list open and has ${STALE_TODO_SUBAGENTS_PER_RUN} finished subagents, a provider turn and, once finished, a checkpoint. The record counts must not grow with the number of finished runs.`,
316 "",
317 "| Provider | Total thread wire | Budget | Result |",
318 "| --- | ---: | ---: | --- |",
@@ -352,6 +401,22 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
401 ...legacyRows(run.provider, "long-turn history page", run.longTurnHistoryPageLegacy),
402 ...timelineRows(run.provider, "fleet shell snapshot", run.fleetShell, FLEET_SHELL_BUDGET),
403 ...legacyRows(run.provider, "fleet shell snapshot", run.fleetShellLegacy),
404 + ...timelineRows(
405 + run.provider,
406 + "stale todo list cold open",
407 + run.staleTodoColdOpen,
408 + STALE_TODO_BUDGET,
409 + ),
410 + ...STALE_TODO_RECORD_KINDS.map((kind) =>
411 + row(
412 + run.provider,
413 + "stale todo list cold open",
414 + kind,
415 + run.staleTodoColdOpen.records[kind],
416 + STALE_TODO_BUDGET.records[kind],
417 + String,
418 + ),
419 + ),
420 );
421 }
422
apps/server/integration/TransferBudgetV2Fixture.integration.ts
+350
@@ -1,9 +1,11 @@
1 import {
2 CheckpointId,
3 + CheckpointRef,
4 CheckpointScopeId,
5 EventId,
6 MessageId,
7 NodeId,
8 + PlanId,
9 ProjectId,
10 ProviderInstanceId,
11 ProviderThreadId,
@@ -440,6 +442,354 @@ export function longTurnThreadBatches(
442 ];
443 }
444
445 +export const STALE_TODO_THREAD_ID = ThreadId.make("transfer-budget-stale-todo-lists");
446 +export const STALE_TODO_SETTLED_RUNS = 40;
447 +export const STALE_TODO_SUBAGENTS_PER_RUN = 2;
448 +export const STALE_TODO_LIVE_ROWS = 600;
449 +const staleTodoCheckpointId = (index: number) =>
450 + CheckpointId.make(`checkpoint:thread:${STALE_TODO_THREAD_ID}:ordinal:${index + 1}`);
451 +/** The checkpoint clients read the last finished turn's diff from. */
452 +export const STALE_TODO_LAST_FINISHED_CHECKPOINT_ID = staleTodoCheckpointId(
453 + STALE_TODO_SETTLED_RUNS - 1,
454 +);
455 +const staleTodoRunId = (index: number) =>
456 + RunId.make(`run:thread:${STALE_TODO_THREAD_ID}:ordinal:${index + 1}`);
457 +export const STALE_TODO_LIVE_RUN_ID = staleTodoRunId(STALE_TODO_SETTLED_RUNS);
458 +
459 +/**
460 + * A long thread: many finished runs, then one run still at work with more rows than a window
461 + * holds. Every run leaves its todo list node `running` and its plan `active`, as adapters do,
462 + * and has subagents, a provider turn and (once finished) a checkpoint. One batch per run.
463 + */
464 +export function staleTodoListThreadBatches(
465 + provider: ProviderDriverKind,
466 +): ReadonlyArray<ReadonlyArray<OrchestrationV2DomainEvent>> {
467 + const threadId = STALE_TODO_THREAD_ID;
468 + const providerInstanceId = ProviderInstanceId.make(provider);
469 + const modelSelection = {
470 + instanceId: providerInstanceId,
471 + model: provider === "codex" ? "gpt-5.4" : "claude-sonnet-4-6",
472 + };
473 + const providerThreadId = ProviderThreadId.make(
474 + `provider-thread:provider:${provider}:native-thread:stale-todo-lists`,
475 + );
476 + const scopeId = CheckpointScopeId.make(`checkpoint-scope:thread:${threadId}:root`);
477 + let ordinal = 0;
478 + const text = (run: number, row: number, targetBytes: number) =>
479 + diagnosticOutput({ provider, turnIndex: 200 + run, toolIndex: row, targetBytes });
480 +
481 + const turn = (index: number, live: boolean): OrchestrationV2DomainEvent[] => {
482 + const now = DateTime.makeUnsafe(Date.UTC(2026, 5, 4, 0, index));
483 + const runId = staleTodoRunId(index);
484 + const runKey = encodeURIComponent(runId);
485 + const rootNodeId = NodeId.make(`node:run:${runKey}:root`);
486 + const todoNodeId = NodeId.make(`node:run:${runKey}:todo-list`);
487 + const planId = PlanId.make(`plan:run:${runKey}:todo-list`);
488 + const providerTurnId = ProviderTurnId.make(
489 + `provider-turn:provider:${provider}:native-turn:${encodeURIComponent(`turn:run-attempt:${runKey}:attempt:1`)}`,
490 + );
491 + const status = live ? ("running" as const) : ("completed" as const);
492 + const completedAt = live ? null : now;
493 + const common = {
494 + threadId,
495 + runId,
496 + nodeId: rootNodeId,
497 + driver: provider,
498 + providerInstanceId,
499 + occurredAt: now,
500 + };
501 + const messageId = (role: "user" | "assistant") =>
502 + MessageId.make(`message:provider:${provider}:native-item:msg_01StaleTodo${role}${index}`);
503 + const run = {
504 + id: runId,
505 + threadId,
506 + ordinal: index + 1,
507 + providerInstanceId,
508 + modelSelection,
509 + providerThreadId: null,
510 + userMessageId: messageId("user"),
511 + rootNodeId,
512 + activeAttemptId: null,
513 + status: "running" as const,
514 + requestedAt: now,
515 + startedAt: now,
516 + completedAt: null,
517 + checkpointId: null,
518 + contextHandoffId: null,
519 + };
520 + const base = (row: string, title: string | null, nodeId: NodeId = rootNodeId) => {
521 + ordinal += 1;
522 + const nativeId = `toolu_01StaleTodo${index}x${row}`;
523 + return {
524 + id: TurnItemId.make(`turn-item:provider:${provider}:native-item:${nativeId}`),
525 + threadId,
526 + runId,
527 + nodeId,
528 + providerThreadId,
529 + providerTurnId,
530 + nativeItemRef: { driver: provider, nativeId, strength: "strong" as const },
531 + parentItemId: null,
532 + ordinal,
533 + status: "completed" as const,
534 + title,
535 + startedAt: now,
536 + completedAt: now,
537 + updatedAt: now,
538 + };
539 + };
540 + const events: OrchestrationV2DomainEvent[] = [
541 + { ...common, id: EventId.make(`stale-todo-run-${index}`), type: "run.created", payload: run },
542 + ];
543 + const node = (
544 + id: NodeId,
545 + kind: "root_turn" | "todo_list" | "subagent",
546 + nodeStatus: "running" | "completed",
547 + ) =>
548 + events.push({
549 + ...common,
550 + id: EventId.make(`stale-todo-node-${id}`),
551 + type: "node.updated",
552 + nodeId: id,
553 + payload: {
554 + id,
555 + threadId,
556 + runId,
557 + parentNodeId: kind === "root_turn" ? null : rootNodeId,
558 + rootNodeId,
559 + kind,
560 + status: nodeStatus,
561 + countsForRun: kind === "root_turn",
562 + providerThreadId,
563 + providerTurnId,
564 + nativeItemRef: null,
565 + runtimeRequestId: null,
566 + checkpointScopeId: kind === "root_turn" ? scopeId : null,
567 + startedAt: now,
568 + completedAt: nodeStatus === "running" ? null : now,
569 + },
570 + });
571 + const message = (role: "user" | "assistant", messageText: string) => {
572 + const id = messageId(role);
573 + const createdBy = role === "user" ? ("user" as const) : ("agent" as const);
574 + events.push(
575 + {
576 + ...common,
577 + id: EventId.make(`stale-todo-message-${index}-${role}`),
578 + type: "message.updated",
579 + payload: {
580 + id,
581 + threadId,
582 + runId,
583 + nodeId: rootNodeId,
584 + role,
585 + text: messageText,
586 + attachments: [],
587 + streaming: false,
588 + createdAt: now,
589 + updatedAt: now,
590 + createdBy,
591 + creationSource: "web",
592 + },
593 + },
594 + {
595 + ...common,
596 + id: EventId.make(`stale-todo-message-item-${index}-${role}`),
597 + type: "turn-item.updated",
598 + payload:
599 + role === "user"
600 + ? {
601 + ...base(role, null),
602 + type: "user_message",
603 + createdBy,
604 + creationSource: "web",
605 + messageId: id,
606 + inputIntent: "turn_start",
607 + text: messageText,
608 + attachments: [],
609 + }
610 + : {
611 + ...base(role, null),
612 + type: "assistant_message",
613 + messageId: id,
614 + text: messageText,
615 + streaming: false,
616 + },
617 + },
618 + );
619 + };
620 +
621 + if (index === 0)
622 + events.push({
623 + ...common,
624 + id: EventId.make("stale-todo-scope"),
625 + type: "checkpoint-scope.created",
626 + payload: {
627 + id: scopeId,
628 + threadId,
629 + runId,
630 + nodeId: rootNodeId,
631 + parentScopeId: null,
632 + providerThreadId,
633 + kind: "root_run",
634 + ordinalWithinParent: 0,
635 + advancesAppRunCount: true,
636 + cwd: "/workspace/transfer-budget",
637 + createdAt: now,
638 + },
639 + });
640 + node(rootNodeId, "root_turn", status);
641 + events.push({
642 + ...common,
643 + id: EventId.make(`stale-todo-provider-turn-${index}`),
644 + type: "provider-turn.updated",
645 + payload: {
646 + id: providerTurnId,
647 + providerThreadId,
648 + nodeId: rootNodeId,
649 + runAttemptId: null,
650 + nativeTurnRef: null,
651 + ordinal: index + 1,
652 + status,
653 + startedAt: now,
654 + completedAt,
655 + },
656 + });
657 + message("user", `Continue the migration, step ${index}.`);
658 +
659 + // The list is done but never closed: the node stays running and the plan stays active.
660 + const steps = Array.from({ length: 6 }, (_, step) => ({
661 + id: `step-${step + 1}`,
662 + text: `Migrate module group ${index}.${step + 1} and run its checks`,
663 + status: live && step > 2 ? ("pending" as const) : ("completed" as const),
664 + }));
665 + node(todoNodeId, "todo_list", "running");
666 + events.push(
667 + {
668 + ...common,
669 + id: EventId.make(`stale-todo-plan-${index}`),
670 + type: "plan.updated",
671 + nodeId: todoNodeId,
672 + payload: {
673 + id: planId,
674 + threadId,
675 + runId,
676 + nodeId: todoNodeId,
677 + status: "active",
678 + kind: "todo_list",
679 + steps,
680 + },
681 + },
682 + {
683 + ...common,
684 + id: EventId.make(`stale-todo-plan-item-${index}`),
685 + type: "turn-item.updated",
686 + nodeId: todoNodeId,
687 + payload: { ...base("todo", "Todo list", todoNodeId), type: "todo_list", planId, steps },
688 + },
689 + );
690 +
691 + for (let subagent = 0; subagent < STALE_TODO_SUBAGENTS_PER_RUN; subagent++) {
692 + const subagentId = NodeId.make(`node:run:${runKey}:subagent:${subagent + 1}`);
693 + node(subagentId, "subagent", "completed");
694 + events.push({
695 + ...common,
696 + id: EventId.make(`stale-todo-subagent-${index}-${subagent}`),
697 + type: "subagent.updated",
698 + nodeId: subagentId,
699 + payload: {
700 + id: subagentId,
701 + threadId,
702 + runId,
703 + parentNodeId: rootNodeId,
704 + origin: "app_owned",
705 + createdBy: "agent",
706 + driver: provider,
707 + providerInstanceId,
708 + providerThreadId: null,
709 + childThreadId: null,
710 + nativeTaskRef: null,
711 + prompt: text(index, subagent * 2, 2_048),
712 + title: `Review module group ${index}.${subagent + 1}`,
713 + model: modelSelection.model,
714 + status: "completed",
715 + result: text(index, subagent * 2 + 1, 4_096),
716 + startedAt: now,
717 + completedAt: now,
718 + updatedAt: now,
719 + },
720 + });
721 + }
722 +
723 + const workRows = live ? STALE_TODO_LIVE_ROWS : 6;
724 + for (let row = 0; row < workRows; row++)
725 + events.push({
726 + ...common,
727 + id: EventId.make(`stale-todo-work-${index}-${row}`),
728 + type: "turn-item.updated",
729 + payload:
730 + row % 10 === 0
731 + ? {
732 + ...base(String(row), "Reasoning"),
733 + type: "reasoning",
734 + text: text(index, 10 + row, 1_024),
735 + streaming: false,
736 + }
737 + : {
738 + ...base(String(row), `Run check ${row}`),
739 + type: "command_execution",
740 + input: `vp test run src/migration/module-${index}-${row}.test.ts`,
741 + output: text(index, 10 + row, 2_048),
742 + exitCode: 0,
743 + },
744 + });
745 + if (live) return events;
746 +
747 + message("assistant", text(index, 5, 1_200));
748 + const checkpointId = staleTodoCheckpointId(index);
749 + events.push(
750 + {
751 + ...common,
752 + id: EventId.make(`stale-todo-checkpoint-${index}`),
753 + type: "checkpoint.captured",
754 + payload: {
755 + id: checkpointId,
756 + threadId,
757 + scopeId,
758 + runId,
759 + nodeId: rootNodeId,
760 + parentCheckpointId: index === 0 ? null : staleTodoCheckpointId(index - 1),
761 + ordinalWithinScope: index + 1,
762 + appRunOrdinal: index + 1,
763 + ref: CheckpointRef.make(
764 + `refs/t3/orchestration-v2/checkpoints/stale-todo-lists/ordinal/${index + 1}`,
765 + ),
766 + status: "ready",
767 + files: Array.from({ length: 12 }, (_, file) => ({
768 + path: `src/migration/module-${index}-${file}.ts`,
769 + kind: "modified",
770 + additions: file + 1,
771 + deletions: file,
772 + })),
773 + capturedAt: now,
774 + },
775 + },
776 + {
777 + ...common,
778 + id: EventId.make(`stale-todo-complete-${index}`),
779 + type: "run.updated",
780 + payload: { ...run, status: "completed", completedAt: now, checkpointId },
781 + },
782 + );
783 + return events;
784 + };
785 +
786 + return [
787 + [threadCreated(provider, threadId)],
788 + ...Array.from({ length: STALE_TODO_SETTLED_RUNS }, (_, index) => turn(index, false)),
789 + turn(STALE_TODO_SETTLED_RUNS, true),
790 + ];
791 +}
792 +
793 export const SHELL_SETTLED_THREADS = 100;
794 export const SHELL_IDLE_SUBAGENTS = 40;
795
apps/server/integration/transferBudgetV2.integration.test.ts
+33
@@ -73,6 +73,11 @@ import {
73 SHELL_IDLE_SUBAGENTS,
74 SHELL_SETTLED_THREADS,
75 shellFleetEvents,
76 + STALE_TODO_LAST_FINISHED_CHECKPOINT_ID,
77 + STALE_TODO_LIVE_RUN_ID,
78 + STALE_TODO_SUBAGENTS_PER_RUN,
79 + STALE_TODO_THREAD_ID,
80 + staleTodoListThreadBatches,
81 THREAD_ID,
82 threadCreated,
83 turnEvents,
@@ -427,6 +432,33 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
432 fleetShellLegacy.rows,
433 earlierThreads + SHELL_SETTLED_THREADS + SHELL_IDLE_SUBAGENTS,
434 );
435 +
436 + for (const events of staleTodoListThreadBatches(provider)) yield* sink.write({ events });
437 + const staleTodoHttp = yield* get(
438 + `/api/orchestration/threads/${STALE_TODO_THREAD_ID}/bounded?${CLIENT_BOUNDED_QUERY}`,
439 + );
440 + const staleTodo = (yield* decodeBoundedSnapshot(decoded(staleTodoHttp))).projection;
441 + // Clients show the diff of the last finished turn from this record.
442 + assert.include(
443 + staleTodo.checkpoints.map((checkpoint) => checkpoint.id),
444 + STALE_TODO_LAST_FINISHED_CHECKPOINT_ID,
445 + );
446 + // The caps below also pass when the running turn loses its own tasks.
447 + assert.lengthOf(
448 + staleTodo.subagents.filter((subagent) => subagent.runId === STALE_TODO_LIVE_RUN_ID),
449 + STALE_TODO_SUBAGENTS_PER_RUN,
450 + );
451 + const staleTodoColdOpen = {
452 + http: staleTodoHttp,
453 + rows: staleTodo.visibleTurnItems.length,
454 + records: {
455 + nodes: staleTodo.nodes.length,
456 + subagents: staleTodo.subagents.length,
457 + plans: staleTodo.plans.length,
458 + providerTurns: staleTodo.providerTurns.length,
459 + checkpoints: staleTodo.checkpoints.length,
460 + },
461 + };
462 return {
463 provider,
464 threadSnapshot,
@@ -444,6 +476,7 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
476 longTurnHistoryPage,
477 fleetShellLegacy,
478 fleetShell,
479 + staleTodoColdOpen,
480 } satisfies TransferBudgetRun;
481 }).pipe(
482 Effect.provide(Layer.fresh(layerServices).pipe(Layer.provideMerge(NodeServices.layer))),
docs/internals/performance-regressions.md
+5 -2
@@ -68,8 +68,11 @@ fixtures exclude unrelated settings calls and subscription events. Payload budge
68 these tests so regressions fail locally. The transfer budget integration test
69 (`apps/server/integration/transferBudgetV2.integration.test.ts`) sends the opt-ins current clients
70 send and caps the decoded bytes and rows of the bounded snapshot and one older page for a 900-row
71 -agent turn, and of the cold-connect shell for a sidebar of settled threads and idle subagents. When
72 -clients gain an opt-in, add it to the test's client queries and give a fixture the data it saves.
71 +agent turn, and of the cold-connect shell for a sidebar of settled threads and idle subagents. It
72 +also caps the bytes and the node, subagent, plan, provider turn and checkpoint counts of the
73 +bounded snapshot for a thread of 40 finished runs behind one working run, so finished runs cannot
74 +ride along with a window. When clients gain an opt-in, add it to the test's client queries and give
75 +a fixture the data it saves.
76
77 ## Event store and startup
78