@devries-cameron20 / t3code-1 / commits / 476fc9be66
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

test(server): wire budget measures what clients fetch today, and the PR comment shows every row (#35225)

Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>

Theo Browne committed Nov 12, 2026 at 04:18 UTC 476fc9be6691ab0d81e936f4cc674983cf3913be
9 files changed +623 -152
.github/scripts/thread-transfer-report.cjs
+150 -9
@@ -3,6 +3,7 @@ const path = require("node:path");
3
4 const ARTIFACT_NAME = "thread-transfer-results";
5 const RESULT_FILE = "thread-transfer-result.json";
6 +const SCENARIOS_FILE = "thread-transfer-scenarios.json";
7 const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->";
8 const PROVIDERS = ["codex", "claudeAgent"];
9 const OBSERVED_KEYS = [
@@ -29,6 +30,12 @@ const SCENARIO_KEYS = [
30 "measuredMcpResultBytes",
31 ];
32
33 +const SCENARIO_ROW_KEYS = ["phase", "metric", "unit", "observed", "ceiling"];
34 +// These labels come from the PR and reach a comment the bot posts: no markup, links or mentions.
35 +// GitHub still links plain text such as `GH-26`, so they are rendered as code spans.
36 +const SCENARIO_LABEL = /^[A-Za-z0-9][A-Za-z0-9 ,-]{0,79}$/;
37 +const MAX_SCENARIO_ROWS = 100;
38 +
39 function resultShaMarker(sha) {
40 return `<!-- t3-thread-transfer-result-sha:${sha} -->`;
41 }
@@ -85,15 +92,63 @@ function validateResult(value) {
92 return value;
93 }
94
88 -function readResult(directory) {
95 +function validateScenarios(value) {
96 + assertExactKeys(value, ["schemaVersion", "providers"], "scenarios");
97 + if (value.schemaVersion !== 1) {
98 + throw new Error("scenarios.schemaVersion must be 1");
99 + }
100 + assertExactKeys(value.providers, PROVIDERS, "scenarios.providers");
101 + for (const provider of PROVIDERS) {
102 + const rows = value.providers[provider];
103 + if (!Array.isArray(rows) || rows.length > MAX_SCENARIO_ROWS) {
104 + throw new Error(
105 + `scenarios.providers.${provider} must list at most ${MAX_SCENARIO_ROWS} rows`,
106 + );
107 + }
108 + const seen = new Set();
109 + for (const [index, row] of rows.entries()) {
110 + const label = `scenarios.providers.${provider}[${index}]`;
111 + assertExactKeys(row, SCENARIO_ROW_KEYS, label);
112 + for (const key of ["phase", "metric"]) {
113 + if (typeof row[key] !== "string" || !SCENARIO_LABEL.test(row[key])) {
114 + throw new Error(`${label}.${key} must be a short plain label`);
115 + }
116 + }
117 + if (row.unit !== "bytes" && row.unit !== "count") {
118 + throw new Error(`${label}.unit must be bytes or count`);
119 + }
120 + assertMetric(row.observed, `${label}.observed`);
121 + assertMetric(row.ceiling, `${label}.ceiling`);
122 + const key = scenarioKey(row);
123 + if (seen.has(key)) throw new Error(`${label} repeats an earlier row`);
124 + seen.add(key);
125 + }
126 + }
127 + return value;
128 +}
129 +
130 +function scenarioKey(row) {
131 + return `${row.phase}|${row.metric}`;
132 +}
133 +
134 +function readArtifactFile(directory, name, validate) {
135 if (!directory) return undefined;
90 - const file = path.join(directory, RESULT_FILE);
136 + const file = path.join(directory, name);
137 if (!fs.existsSync(file)) return undefined;
138 const stat = fs.lstatSync(file);
139 if (!stat.isFile() || stat.size > 64 * 1_024) {
94 - throw new Error("thread transfer result must be a regular file smaller than 64 KiB");
140 + throw new Error(`${name} must be a regular file smaller than 64 KiB`);
141 }
96 - return validateResult(JSON.parse(fs.readFileSync(file, "utf8")));
142 + return validate(JSON.parse(fs.readFileSync(file, "utf8")));
143 +}
144 +
145 +function readResult(directory) {
146 + return readArtifactFile(directory, RESULT_FILE, validateResult);
147 +}
148 +
149 +// Older runs upload only the result file, so a run without scenarios is still a valid run.
150 +function readScenarios(directory) {
151 + return readArtifactFile(directory, SCENARIOS_FILE, validateScenarios);
152 }
153
154 function formatBytes(bytes) {
@@ -103,7 +158,7 @@ function formatBytes(bytes) {
158 }
159
160 function formatValue(value, kind) {
106 - return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value);
161 + return kind === "bytes" ? formatBytes(value) : value.toLocaleString("en-US");
162 }
163
164 function formatImpact(current, baseline, kind) {
@@ -136,6 +191,77 @@ const METRICS = [
191 { key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" },
192 ];
193
194 +function providerLabel(provider) {
195 + return provider === "codex" ? "Codex" : "Claude";
196 +}
197 +
198 +/**
199 + * The scenario rows of a run against the same rows on `main`. Rows that fail or differ from
200 + * `main` are listed in the open; the full table is collapsed below them.
201 + */
202 +function renderScenarios(current, baseline, ceilingChanges) {
203 + if (!current) return { lines: [], failed: false };
204 + const header = [
205 + "| Provider | Scenario | Metric | Main baseline | This PR | Impact | PR ceiling | |",
206 + "| --- | --- | --- | ---: | ---: | ---: | ---: | --- |",
207 + ];
208 + const all = [];
209 + const notable = [];
210 + let failed = false;
211 + let withoutBaseline = 0;
212 +
213 + for (const provider of PROVIDERS) {
214 + const baselineRows = new Map(
215 + (baseline?.providers[provider] ?? []).map((row) => [scenarioKey(row), row]),
216 + );
217 + for (const row of current.providers[provider]) {
218 + const before = baselineRows.get(scenarioKey(row));
219 + const beforeObserved = before?.unit === row.unit ? before.observed : undefined;
220 + const pass = row.observed <= row.ceiling;
221 + failed ||= !pass;
222 + if (beforeObserved === undefined) withoutBaseline += 1;
223 + const line = `| ${providerLabel(provider)} | \`${row.phase}\` | \`${row.metric}\` | ${beforeObserved === undefined ? "—" : formatValue(beforeObserved, row.unit)} | ${formatValue(row.observed, row.unit)} | ${formatImpact(row.observed, beforeObserved, row.unit)} | ${formatValue(row.ceiling, row.unit)} | ${pass ? "✅" : "❌"} |`;
224 + all.push(line);
225 + if (!pass || (beforeObserved !== undefined && beforeObserved !== row.observed)) {
226 + notable.push(line);
227 + }
228 + if (beforeObserved !== undefined && before.ceiling !== row.ceiling) {
229 + ceilingChanges.push(
230 + `- ${providerLabel(provider)} \`${row.phase}\` \`${row.metric}\`: ${formatValue(before.ceiling, row.unit)} → ${formatValue(row.ceiling, row.unit)}`,
231 + );
232 + }
233 + }
234 + }
235 +
236 + const compared = all.length - withoutBaseline;
237 + const summary = !baseline
238 + ? `No \`main\` baseline exists yet for these ${all.length} scenario measurements.`
239 + : [
240 + notable.length > 0
241 + ? `${notable.length} of ${all.length} scenario measurements changed or failed.`
242 + : `${compared === all.length ? "All " : ""}${compared} scenario measurements match \`main\`.`,
243 + ...(withoutBaseline > 0 ? [`${withoutBaseline} have no \`main\` baseline.`] : []),
244 + ].join(" ");
245 + return {
246 + failed,
247 + lines: [
248 + "### Other scenarios",
249 + "",
250 + summary,
251 + ...(notable.length > 0 ? ["", ...header, ...notable] : []),
252 + "",
253 + "<details>",
254 + `<summary>All ${all.length} scenario measurements</summary>`,
255 + "",
256 + ...header,
257 + ...all,
258 + "",
259 + "</details>",
260 + "",
261 + ],
262 + };
263 +}
264 +
265 function renderComment(input) {
266 const current = input.current;
267 const baseline = input.baseline;
@@ -154,16 +280,22 @@ function renderComment(input) {
280 const pass = observed <= ceiling;
281 failed ||= !pass;
282 rows.push(
157 - `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
283 + `| ${providerLabel(provider)} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
284 );
285
286 if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) {
287 ceilingChanges.push(
162 - `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
288 + `- ${providerLabel(provider)} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
289 );
290 }
291 }
292 }
293 + const scenarios = renderScenarios(
294 + input.currentScenarios,
295 + input.baselineScenarios,
296 + ceilingChanges,
297 + );
298 + failed ||= scenarios.failed;
299
300 const baselineLink = input.baselineRun
301 ? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})`
@@ -203,6 +335,7 @@ function renderComment(input) {
335 "| --- | --- | ---: | ---: | ---: | ---: | --- |",
336 ...rows,
337 "",
338 + ...scenarios.lines,
339 `Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`,
340 "",
341 "<details>",
@@ -212,7 +345,7 @@ function renderComment(input) {
345 "",
346 ...PROVIDERS.map(
347 (provider) =>
215 - `- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
348 + `- ${providerLabel(provider)} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
349 ),
350 "",
351 "</details>",
@@ -437,7 +570,14 @@ async function publish({ github, context, core }) {
570 url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`,
571 }
572 : undefined;
440 - const body = renderComment({ current, baseline, currentRun, baselineRun });
573 + const body = renderComment({
574 + current,
575 + baseline,
576 + currentScenarios: readScenarios(process.env.PR_RESULT_DIR),
577 + baselineScenarios: readScenarios(process.env.BASELINE_RESULT_DIR),
578 + currentRun,
579 + baselineRun,
580 + });
581 const published = await upsertCommentForCurrentHead(
582 github,
583 context,
@@ -458,4 +598,5 @@ module.exports = {
598 resolve,
599 upsertCommentForCurrentHead,
600 validateResult,
601 + validateScenarios,
602 };
.github/scripts/thread-transfer-report.test.cjs
+79
@@ -2,9 +2,11 @@ const assert = require("node:assert/strict");
2 const test = require("node:test");
3
4 const {
5 + renderComment,
6 resolve,
7 upsertCommentForCurrentHead,
8 validateResult,
9 + validateScenarios,
10 } = require("./thread-transfer-report.cjs");
11
12 function result(overrides = {}) {
@@ -40,6 +42,29 @@ function result(overrides = {}) {
42 };
43 }
44
45 +function scenarios(overrides = {}) {
46 + const rows = [
47 + {
48 + phase: "long turn finished, cold open",
49 + metric: "HTTP decoded",
50 + unit: "bytes",
51 + observed: 390_000,
52 + ceiling: 420_000,
53 + },
54 + {
55 + phase: "idle thread resume",
56 + metric: "WebSocket messages",
57 + unit: "count",
58 + observed: 1,
59 + ceiling: 1,
60 + },
61 + ];
62 + return {
63 + schemaVersion: 1,
64 + providers: { codex: [{ ...rows[0], ...overrides }, rows[1]], claudeAgent: rows },
65 + };
66 +}
67 +
68 test("validates the fixed artifact schema", () => {
69 assert.equal(validateResult(result()).schemaVersion, 1);
70 assert.throws(
@@ -50,6 +75,60 @@ test("validates the fixed artifact schema", () => {
75 () => validateResult(result({ totalWireBytes: "lots" })),
76 /non-negative safe integer/,
77 );
78 +
79 + assert.equal(validateScenarios(scenarios()).schemaVersion, 1);
80 + for (const phase of ["@everyone", "[link](https://example.com)", "a | b", "<img>", ""]) {
81 + assert.throws(() => validateScenarios(scenarios({ phase })), /short plain label/);
82 + }
83 + assert.throws(() => validateScenarios(scenarios({ note: "extra" })), /unexpected fields/);
84 + assert.throws(() => validateScenarios(scenarios({ ceiling: null })), /non-negative safe integer/);
85 + assert.throws(
86 + () =>
87 + validateScenarios(scenarios({ phase: "idle thread resume", metric: "WebSocket messages" })),
88 + /repeats an earlier row/,
89 + );
90 +});
91 +
92 +test("lists scenario rows that changed or failed above the collapsed table", () => {
93 + const run = { sha: "abcdef1234", url: "https://example.com/run", conclusion: "success" };
94 + const render = (currentScenarios, baselineScenarios) =>
95 + renderComment({
96 + current: result(),
97 + baseline: result(),
98 + currentScenarios,
99 + baselineScenarios,
100 + currentRun: run,
101 + baselineRun: { ...run, matchesBase: true },
102 + });
103 + const open = (body) => body.slice(0, body.indexOf("<details>"));
104 +
105 + const unchanged = render(scenarios(), scenarios());
106 + assert.match(unchanged, /✅ Thread transfer remains within every enforced ceiling/);
107 + assert.match(open(unchanged), /All 4 scenario measurements match `main`/);
108 + assert.doesNotMatch(open(unchanged), /long turn finished, cold open/);
109 +
110 + const grown = render(scenarios({ observed: 400_000 }), scenarios());
111 + assert.match(open(grown), /1 of 4 scenario measurements changed or failed/);
112 + assert.match(
113 + open(grown),
114 + /\| Codex \| `long turn finished, cold open` \| `HTTP decoded` \| 380\.9 KiB \| 390\.6 KiB \| \+9\.8 KiB \(\+2\.6%\) \| 410\.2 KiB \| ✅ \|/,
115 + );
116 +
117 + const over = render(scenarios({ observed: 430_000, ceiling: 425_000 }), scenarios());
118 + assert.match(over, /❌ One or more thread transfer ceilings were exceeded/);
119 + assert.match(
120 + over,
121 + /Codex `long turn finished, cold open` `HTTP decoded`: 410\.2 KiB → 415\.0 KiB/,
122 + );
123 +
124 + // A row `main` does not have is not reported as matching it.
125 + const added = render(scenarios({ phase: "GH-26" }), scenarios());
126 + assert.match(open(added), /3 scenario measurements match `main`\. 1 have no `main` baseline\./);
127 + assert.match(added, /\| Codex \| `GH-26` \|/);
128 +
129 + // A main run from before the scenario file has nothing to compare; a PR run from before it has no section.
130 + assert.match(open(render(scenarios(), undefined)), /No `main` baseline exists yet/);
131 + assert.doesNotMatch(render(undefined, scenarios()), /Other scenarios/);
132 });
133
134 test("resolves a fallback PR with a redacted head repo and exact main baseline", async () => {
.github/workflows/ci.yml
+4 -1
@@ -503,6 +503,7 @@ jobs:
503 env:
504 T3CODE_TRANSFER_BUDGET_REPORT_PATH: ${{ runner.temp }}/t3code-transfer-budget.md
505 T3CODE_TRANSFER_BUDGET_RESULT_PATH: ${{ runner.temp }}/thread-transfer-result.json
506 + T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH: ${{ runner.temp }}/thread-transfer-scenarios.json
507 run: >-
508 vp run test:cached -- --shard ${{ matrix.shard }}/${{ strategy.job-total }}
509 --uncached apps/server/integration/transferBudgetV2.integration.test.ts
@@ -543,7 +544,9 @@ jobs:
544 uses: actions/upload-artifact@v7
545 with:
546 name: thread-transfer-results
546 - path: ${{ runner.temp }}/thread-transfer-result.json
547 + path: |
548 + ${{ runner.temp }}/thread-transfer-result.json
549 + ${{ runner.temp }}/thread-transfer-scenarios.json
550 if-no-files-found: ignore
551 retention-days: 30
552
apps/server/integration/TransferBudgetReport.integration.ts
+190 -83
@@ -38,6 +38,11 @@ const STALE_TODO_RECORD_KINDS = [
38 ] as const;
39 type StaleTodoRecordCounts = Readonly<Record<(typeof STALE_TODO_RECORD_KINDS)[number], number>>;
40
41 +/** One new socket, read from the subscribe call until the subscription caught up. */
42 +export interface SocketOpenMeasurement {
43 + readonly totals: WebSocketTransferTotals;
44 +}
45 +
46 export interface TransferBudgetRun {
47 readonly provider: ProviderDriverKind;
48 readonly threadSnapshot: HttpTransferMeasurement;
@@ -49,15 +54,28 @@ export interface TransferBudgetRun {
54 readonly reconnectShell: WebSocketCatchUpMeasurement;
55 readonly measuredTurnSqlStatements: number;
56 readonly reconnectSqlStatements: number;
52 - /** The compact bounded snapshot web and mobile load when the long-turn thread opens. */
53 - readonly longTurnColdOpen: TimelineHttpMeasurement;
54 - readonly longTurnHistoryPage: TimelineHttpMeasurement;
57 + /** A thread resumed from the cursor it held before one ordinary turn. */
58 + readonly shortTurnResume: SocketOpenMeasurement;
59 + /** The long-turn thread while its last turn runs: the bounded snapshot, then one older page. */
60 + readonly longTurnRunningColdOpen: TimelineHttpMeasurement;
61 + readonly longTurnRunningHistoryPage: TimelineHttpMeasurement;
62 /** The same reads as an older client that only sends `compactTurnItems`, to show the savings. */
56 - readonly longTurnColdOpenLegacy: TimelineHttpMeasurement;
57 - readonly longTurnHistoryPageLegacy: TimelineHttpMeasurement;
63 + readonly longTurnRunningColdOpenLegacy: TimelineHttpMeasurement;
64 + readonly longTurnRunningHistoryPageLegacy: TimelineHttpMeasurement;
65 + /** The same thread once the turn finished, over HTTP (web) and over the socket alone (mobile). */
66 + readonly longTurnColdOpen: TimelineHttpMeasurement;
67 + readonly longTurnSocketColdOpen: SocketOpenMeasurement & { readonly rows: number };
68 + /** The first page of the finished turn's deferred work, loaded when its group is expanded. */
69 + readonly longTurnWorkPage: TimelineHttpMeasurement;
70 + /** A thread resumed from the cursor of its own last event. */
71 + readonly idleThreadResume: SocketOpenMeasurement;
72 /** The cold-connect shell snapshot after the settled-thread fleet is written. */
73 readonly fleetShell: TimelineHttpMeasurement;
74 readonly fleetShellLegacy: TimelineHttpMeasurement;
75 + /** The shell subscription a cold connect opens from that snapshot's cursor. */
76 + readonly fleetShellSubscribe: SocketOpenMeasurement;
77 + /** A shell resumed from a cursor too old to replay, by a client that reloads over HTTP. */
78 + readonly staleShellResume: SocketOpenMeasurement;
79 /**
80 * The compact bounded snapshot of the long thread whose finished runs left todo lists open,
81 * with the count of each record kind that rides along with its rows.
@@ -89,14 +107,44 @@ const TRANSFER_BUDGETS: Readonly<Record<string, ProviderTransferBudget>> = {
107 claudeAgent: TRANSFER_BUDGET,
108 };
109
92 -// Decoded bytes, not wire bytes: phones parse and keep the decoded JSON. About 7% above the
93 -// observed 103 KB, so a cold open without body dedupe (117 KB), the ID table, or row budgets
94 -// (all 920 rows) fails. Kept apart from TRANSFER_BUDGET because the PR comment validates its keys.
95 -const LONG_TURN_BUDGET = {
110 +// Decoded bytes, not wire bytes: phones parse and keep the decoded JSON. About 15% above the
111 +// observed 95 KB history page, so a page without the ID table (130 KB) or row budgets fails.
112 +// Kept apart from TRANSFER_BUDGET because the PR comment validates its keys.
113 +const LONG_TURN_PAGE_BUDGET = {
114 decodedBytes: 110_000,
115 rows: 75,
116 };
117
118 +// About 7% above the observed 290 KB. The window holds 75 rows, but every message of a running
119 +// turn rides along with it: 196 KB here. Without body dedupe or the ID table it is 354 KB.
120 +const LONG_TURN_RUNNING_BUDGET = {
121 + decodedBytes: 310_000,
122 + rows: 75,
123 +};
124 +
125 +// About 7% above the observed 393 KB. A finished turn defers its 770 work rows and stays whole:
126 +// 151 rows of messages, checkpoints and subagents, on both transports.
127 +const LONG_TURN_FINISHED_BUDGET = {
128 + decodedBytes: 420_000,
129 + rows: 151,
130 +};
131 +
132 +// About 6% above the observed 193 KB: one older page of work, twice the cold-open row budget.
133 +const LONG_TURN_WORK_PAGE_BUDGET = {
134 + decodedBytes: 205_000,
135 + rows: 150,
136 +};
137 +
138 +const SOCKET_BUDGETS = {
139 + // One ordinary turn replays in 11 KB. With its tool bodies it is over 900 KB.
140 + shortTurnResume: { decodedBytes: 13_000, messages: 2 },
141 + longTurnColdOpen: { decodedBytes: LONG_TURN_FINISHED_BUDGET.decodedBytes, messages: 2 },
142 + // These three send no rows. A snapshot in their place is 393 KB for the thread, 42 KB for the shell.
143 + idleThreadResume: { decodedBytes: 200, messages: 1 },
144 + fleetShellSubscribe: { decodedBytes: 200, messages: 1 },
145 + staleShellResume: { decodedBytes: 400, messages: 2 },
146 +};
147 +
148 // About 30% above the observed 42 KB. Without the shell opt-ins it carries all 142 rows (217 KB).
149 const FLEET_SHELL_BUDGET = {
150 decodedBytes: 55_000,
@@ -211,31 +259,139 @@ function webSocketRows(
259 ];
260 }
261
214 -function timelineRows(
215 - provider: ProviderDriverKind,
262 +interface ScenarioRow {
263 + readonly phase: string;
264 + readonly metric: string;
265 + readonly unit: "bytes" | "count";
266 + readonly observed: number;
267 + /** Absent on figures that only inform: wire bytes, and what an older client receives. */
268 + readonly maximum: number | undefined;
269 +}
270 +
271 +function httpRows(
272 phase: string,
273 measured: TimelineHttpMeasurement,
218 - budget: { readonly decodedBytes: number; readonly rows: number },
219 -): string[] {
274 + budget?: { readonly decodedBytes: number; readonly rows: number },
275 +): ScenarioRow[] {
276 return [
221 - infoRow(provider, phase, "HTTP wire", measured.http.wireBytes),
222 - row(provider, phase, "HTTP decoded", measured.http.decodedBodyBytes, budget.decodedBytes),
223 - row(provider, phase, "rows", measured.rows, budget.rows, String),
277 + {
278 + phase,
279 + metric: "HTTP wire",
280 + unit: "bytes",
281 + observed: measured.http.wireBytes,
282 + maximum: undefined,
283 + },
284 + {
285 + phase,
286 + metric: "HTTP decoded",
287 + unit: "bytes",
288 + observed: measured.http.decodedBodyBytes,
289 + maximum: budget?.decodedBytes,
290 + },
291 + { phase, metric: "rows", unit: "count", observed: measured.rows, maximum: budget?.rows },
292 ];
293 }
294
227 -function legacyRows(
228 - provider: ProviderDriverKind,
295 +function socketRows(
296 phase: string,
230 - measured: TimelineHttpMeasurement,
231 -): string[] {
297 + { totals }: SocketOpenMeasurement,
298 + budget: { readonly decodedBytes: number; readonly messages: number },
299 +): ScenarioRow[] {
300 + return [
301 + {
302 + phase,
303 + metric: "WebSocket wire",
304 + unit: "bytes",
305 + observed: totals.wireBytes,
306 + maximum: undefined,
307 + },
308 + {
309 + phase,
310 + metric: "WebSocket decoded",
311 + unit: "bytes",
312 + observed: totals.decodedBytes,
313 + maximum: budget.decodedBytes,
314 + },
315 + {
316 + phase,
317 + metric: "WebSocket messages",
318 + unit: "count",
319 + observed: totals.messages,
320 + maximum: budget.messages,
321 + },
322 + ];
323 +}
324 +
325 +/**
326 + * Every scenario figure beyond the thread totals. The violations, the report table and the PR
327 + * comment's scenario file all read this list, so a row added here reaches all three. The PR
328 + * comment only accepts letters, digits, spaces, commas and hyphens in `phase` and `metric`.
329 + */
330 +function scenarioRows(run: TransferBudgetRun): ScenarioRow[] {
331 return [
233 - infoRow(provider, `${phase}, older client`, "HTTP wire", measured.http.wireBytes),
234 - infoRow(provider, `${phase}, older client`, "HTTP decoded", measured.http.decodedBodyBytes),
235 - infoRow(provider, `${phase}, older client`, "rows", measured.rows, String),
332 + ...socketRows("short turn resume", run.shortTurnResume, SOCKET_BUDGETS.shortTurnResume),
333 + ...httpRows(
334 + "long turn running, cold open",
335 + run.longTurnRunningColdOpen,
336 + LONG_TURN_RUNNING_BUDGET,
337 + ),
338 + ...httpRows("long turn running, cold open, older client", run.longTurnRunningColdOpenLegacy),
339 + ...httpRows(
340 + "long turn running, history page",
341 + run.longTurnRunningHistoryPage,
342 + LONG_TURN_PAGE_BUDGET,
343 + ),
344 + ...httpRows(
345 + "long turn running, history page, older client",
346 + run.longTurnRunningHistoryPageLegacy,
347 + ),
348 + ...httpRows("long turn finished, cold open", run.longTurnColdOpen, LONG_TURN_FINISHED_BUDGET),
349 + ...socketRows(
350 + "long turn finished, socket cold open",
351 + run.longTurnSocketColdOpen,
352 + SOCKET_BUDGETS.longTurnColdOpen,
353 + ),
354 + {
355 + phase: "long turn finished, socket cold open",
356 + metric: "rows",
357 + unit: "count",
358 + observed: run.longTurnSocketColdOpen.rows,
359 + maximum: LONG_TURN_FINISHED_BUDGET.rows,
360 + },
361 + ...httpRows("long turn finished, work page", run.longTurnWorkPage, LONG_TURN_WORK_PAGE_BUDGET),
362 + ...socketRows("idle thread resume", run.idleThreadResume, SOCKET_BUDGETS.idleThreadResume),
363 + ...httpRows("fleet shell snapshot", run.fleetShell, FLEET_SHELL_BUDGET),
364 + ...httpRows("fleet shell snapshot, older client", run.fleetShellLegacy),
365 + ...socketRows(
366 + "fleet shell subscribe",
367 + run.fleetShellSubscribe,
368 + SOCKET_BUDGETS.fleetShellSubscribe,
369 + ),
370 + ...socketRows("stale shell resume", run.staleShellResume, SOCKET_BUDGETS.staleShellResume),
371 + ...httpRows("stale todo list cold open", run.staleTodoColdOpen, STALE_TODO_BUDGET),
372 + ...STALE_TODO_RECORD_KINDS.map((kind): ScenarioRow => ({
373 + phase: "stale todo list cold open",
374 + metric: kind,
375 + unit: "count",
376 + observed: run.staleTodoColdOpen.records[kind],
377 + maximum: STALE_TODO_BUDGET.records[kind],
378 + })),
379 ];
380 }
381
382 +/** The budgeted scenario figures, for the PR comment to compare with `main`. */
383 +export function formatTransferBudgetScenarios(runs: ReadonlyArray<TransferBudgetRun>): string {
384 + const providers = Object.fromEntries(
385 + runs.map((run) => [
386 + run.provider,
387 + scenarioRows(run).flatMap(({ maximum, ...row }) =>
388 + maximum === undefined ? [] : [{ ...row, ceiling: maximum }],
389 + ),
390 + ]),
391 + );
392 + return `${JSON.stringify({ schemaVersion: 1, providers }, null, 2)}\n`;
393 +}
394 +
395 export function transferBudgetViolations(runs: ReadonlyArray<TransferBudgetRun>): string[] {
396 const violations: string[] = [];
397 for (const run of runs) {
@@ -262,37 +418,8 @@ export function transferBudgetViolations(runs: ReadonlyArray<TransferBudgetRun>)
418 run.measuredTurnWebSocket.messages,
419 budget.measuredTurnWebSocketMessages,
420 ],
265 - [
266 - "long-turn cold open decoded bytes",
267 - run.longTurnColdOpen.http.decodedBodyBytes,
268 - LONG_TURN_BUDGET.decodedBytes,
269 - ],
270 - ["long-turn cold open rows", run.longTurnColdOpen.rows, LONG_TURN_BUDGET.rows],
271 - [
272 - "long-turn history page decoded bytes",
273 - run.longTurnHistoryPage.http.decodedBodyBytes,
274 - LONG_TURN_BUDGET.decodedBytes,
275 - ],
276 - ["long-turn history page rows", run.longTurnHistoryPage.rows, LONG_TURN_BUDGET.rows],
277 - [
278 - "fleet shell snapshot decoded bytes",
279 - run.fleetShell.http.decodedBodyBytes,
280 - FLEET_SHELL_BUDGET.decodedBytes,
281 - ],
282 - ["fleet shell snapshot rows", run.fleetShell.rows, FLEET_SHELL_BUDGET.rows],
283 - [
284 - "stale todo list cold open decoded bytes",
285 - run.staleTodoColdOpen.http.decodedBodyBytes,
286 - STALE_TODO_BUDGET.decodedBytes,
287 - ],
288 - ["stale todo list cold open rows", run.staleTodoColdOpen.rows, STALE_TODO_BUDGET.rows],
289 - ...STALE_TODO_RECORD_KINDS.map(
290 - (kind) =>
291 - [
292 - `stale todo list cold open ${kind}`,
293 - run.staleTodoColdOpen.records[kind],
294 - STALE_TODO_BUDGET.records[kind],
295 - ] as const,
421 + ...scenarioRows(run).flatMap(({ phase, metric, observed, maximum }) =>
422 + maximum === undefined ? [] : [[`${phase} ${metric}`, observed, maximum] as const],
423 ),
424 ] as const;
425 for (const [metric, observed, maximum] of checks) {
@@ -311,8 +438,9 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
438 "Wire values are thread data bytes read from local HTTP and WebSocket sockets. HTTP includes response headers; WebSocket measurement starts after the resumed thread subscription synchronizes. TCP/IP, TLS framing, and the WebSocket upgrade are excluded. WebSocket permessage-deflate is negotiated.",
439 "The measured turn is observed on three sockets at once: one with only the thread subscription (the capped rows), one with only the shell subscription, and a second client holding both. Server egress is the sum of the three. After the turn the second client disconnects and new sockets resubscribe from the cursor it held before the turn, which is the cursor a backgrounded phone would hold. SQL statements are the SQL client's per-statement spans, counted across V2 persistence and the HTTP/WebSocket handlers.",
440 `Scenario: ${TRANSFER_HISTORY_TURN_COUNT} historical turns with ${TRANSFER_HISTORY_TOOLS_PER_TURN} command tools and one retained ${formatBytes(TRANSFER_HISTORY_MCP_RESULT_BYTES)} MCP result each, followed by one measured turn with ${TRANSFER_MEASURED_TOOLS} command tools and a retained ${formatBytes(TRANSFER_MEASURED_MCP_RESULT_BYTES)} MCP result. Synthetic V2 domain events exercise persistence, wire projection, and production HTTP/subscription handlers; this does not measure provider adapter ingestion. Payloads contain no user data.`,
314 - `The long-turn rows open a second thread through the compact bounded snapshot web and mobile use, then load one older page. Its last user turn holds ${LONG_TURN_WORK_ROWS} rows of reasoning, commands, file changes, MCP calls, progress messages, checkpoints, and subagents.`,
315 - `The fleet shell rows read the cold-connect shell after ${SHELL_SETTLED_THREADS} settled threads and ${SHELL_IDLE_SUBAGENTS} idle subagent children are added. Budgeted reads send the opt-ins current clients send; "older client" rows show the same read without the opt-ins added since row budgets.`,
441 + `The long turn rows read a second thread whose last user turn holds ${LONG_TURN_WORK_ROWS} rows of reasoning, commands, file changes, MCP calls, progress messages, checkpoints, and subagents. While that turn runs, web and mobile load the compact bounded snapshot and then one older page. Once it finishes, its work is deferred: the cold open is read over HTTP as web does and over the socket alone as mobile does, then the first page of the deferred work is loaded as an expanded work group does.`,
442 + "The resume rows open a new socket from a cursor the client already holds: the first thread after one more ordinary turn, the long-turn thread with nothing new, and the shell from a cursor too old to replay.",
443 + `The fleet shell rows read the cold-connect shell after ${SHELL_SETTLED_THREADS} settled threads and ${SHELL_IDLE_SUBAGENTS} idle subagent children are added, then subscribe from that snapshot's cursor. Budgeted reads send the opt-ins current clients send; "older client" rows show the same read without the opt-ins added since row budgets.`,
444 `The stale todo list rows open a third thread the same way: ${STALE_TODO_SETTLED_RUNS} finished runs, then one run still at work with ${STALE_TODO_LIVE_ROWS} rows. Every run left its todo list open and has ${STALE_TODO_SUBAGENTS_PER_RUN} finished subagents, a provider turn and, once finished, a checkpoint. The record counts must not grow with the number of finished runs.`,
445 "",
446 "| Provider | Total thread wire | Budget | Result |",
@@ -391,33 +519,12 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
519 run.reconnectShell,
520 ),
521 infoRow(run.provider, "reconnect", "SQL statements", run.reconnectSqlStatements, String),
394 - ...timelineRows(run.provider, "long-turn cold open", run.longTurnColdOpen, LONG_TURN_BUDGET),
395 - ...legacyRows(run.provider, "long-turn cold open", run.longTurnColdOpenLegacy),
396 - ...timelineRows(
397 - run.provider,
398 - "long-turn history page",
399 - run.longTurnHistoryPage,
400 - LONG_TURN_BUDGET,
401 - ),
402 - ...legacyRows(run.provider, "long-turn history page", run.longTurnHistoryPageLegacy),
403 - ...timelineRows(run.provider, "fleet shell snapshot", run.fleetShell, FLEET_SHELL_BUDGET),
404 - ...legacyRows(run.provider, "fleet shell snapshot", run.fleetShellLegacy),
405 - ...timelineRows(
406 - run.provider,
407 - "stale todo list cold open",
408 - run.staleTodoColdOpen,
409 - STALE_TODO_BUDGET,
410 - ),
411 - ...STALE_TODO_RECORD_KINDS.map((kind) =>
412 - row(
413 - run.provider,
414 - "stale todo list cold open",
415 - kind,
416 - run.staleTodoColdOpen.records[kind],
417 - STALE_TODO_BUDGET.records[kind],
418 - String,
419 - ),
420 - ),
522 + ...scenarioRows(run).map(({ phase, metric, unit, observed, maximum }) => {
523 + const format = unit === "bytes" ? formatBytes : String;
524 + return maximum === undefined
525 + ? infoRow(run.provider, phase, metric, observed, format)
526 + : row(run.provider, phase, metric, observed, maximum, format);
527 + }),
528 );
529 }
530
apps/server/integration/TransferBudgetV2Fixture.integration.ts
+7 -5
@@ -217,10 +217,10 @@ export const LONG_TURN_THREAD_ID = ThreadId.make("transfer-budget-long-turn");
217 export const LONG_TURN_WORK_ROWS = 900;
218
219 /**
220 - * Three short user turns, then one agent turn far longer than a cold-open
221 - * window, with the item kinds a long agent run produces. One batch per turn.
222 - * Run, node, provider and item IDs follow the composite shapes the Claude
223 - * adapter stores, and every assistant message has its record, as real threads do.
220 + * Three short user turns, then one agent turn far longer than a cold-open window, with the item
221 + * kinds a long agent run produces. One batch per turn; the last batch only finishes the long turn.
222 + * Run, node, provider and item IDs follow the composite shapes the Claude adapter stores, and
223 + * every assistant message has its record, as real threads do.
224 */
225 export function longTurnThreadBatches(
226 provider: ProviderDriverKind,
@@ -434,12 +434,14 @@ export function longTurnThreadBatches(
434 return events;
435 };
436
437 + const longTurn = turn(3, LONG_TURN_WORK_ROWS);
438 return [
439 [threadCreated(provider, threadId)],
440 turn(0, 4),
441 turn(1, 4),
442 turn(2, 4),
442 - turn(3, LONG_TURN_WORK_ROWS),
443 + longTurn.slice(0, -1),
444 + longTurn.slice(-1),
445 ];
446 }
447
apps/server/integration/transferBudgetV2.integration.test.ts
+171 -43
@@ -23,8 +23,14 @@ import {
23 OrchestrationV2GetThreadProjectionError,
24 OrchestrationV2GetShellSnapshotError,
25 ProviderDriverKind,
26 + type EnvironmentOrchestrationShellSnapshotQuery,
27 + type EnvironmentOrchestrationThreadBoundedSnapshotQuery,
28 + type EnvironmentOrchestrationThreadHistoryQuery,
29 + type OrchestrationV2SubscribeShellInput,
30 + type OrchestrationV2SubscribeThreadInput,
31 type OrchestrationV2ThreadStreamItem,
32 type OrchestrationV2ShellStreamItem,
33 + type ThreadId,
34 } from "@t3tools/contracts";
35 import * as Context from "effect/Context";
36 import * as Config from "effect/Config";
@@ -63,6 +69,7 @@ import { makeSqlStatementCounter } from "./SqlStatementCounter.integration.ts";
69 import {
70 formatTransferBudgetReport,
71 formatTransferBudgetResult,
72 + formatTransferBudgetScenarios,
73 transferBudgetViolations,
74 type TransferBudgetRun,
75 } from "./TransferBudgetReport.integration.ts";
@@ -83,11 +90,41 @@ import {
90 turnEvents,
91 } from "./TransferBudgetV2Fixture.integration.ts";
92
86 -// The opt-ins current web and mobile clients send; keep in sync with client-runtime's
87 -// boundedThreadSnapshotHttp.ts, threadHistoryHttp.ts, shellSnapshotHttp.ts, threads.ts and shell.ts.
88 -const CLIENT_BOUNDED_QUERY = "compactTurnItems=1&dedupeBodies=1&compactIds=1";
89 -const CLIENT_HISTORY_QUERY = "compactIds=1";
90 -const CLIENT_SHELL_QUERY = "omitIdleSubagents=1&omitOlderSettled=1";
93 +// The opt-ins current web and mobile clients send, as client-runtime's boundedThreadSnapshotHttp.ts,
94 +// threadHistoryHttp.ts, shellSnapshotHttp.ts, threads.ts and shell.ts send them. Each is `Required`,
95 +// so an opt-in added to a contract fails typecheck here until this test sends it.
96 +const CLIENT_BOUNDED_OPT_INS = {
97 + compactTurnItems: "1",
98 + dedupeBodies: "1",
99 + compactIds: "1",
100 + deferWork: "1",
101 +} satisfies Required<typeof EnvironmentOrchestrationThreadBoundedSnapshotQuery.Type>;
102 +const CLIENT_HISTORY_OPT_INS = { compactIds: "1", deferWork: "1" } satisfies Required<
103 + Omit<typeof EnvironmentOrchestrationThreadHistoryQuery.Type, "cursor" | "throughEntryId" | "view">
104 +>;
105 +const CLIENT_SHELL_OPT_INS = {
106 + omitIdleSubagents: "1",
107 + omitOlderSettled: "1",
108 +} satisfies Required<typeof EnvironmentOrchestrationShellSnapshotQuery.Type>;
109 +const CLIENT_THREAD_SUBSCRIPTION = {
110 + requestCompletionMarker: true,
111 + acceptBoundedSnapshot: true,
112 + acceptDeferredWork: true,
113 + acceptCompactTurnItems: true,
114 + acceptCompactIds: true,
115 + acceptDedupedBodies: true,
116 +} satisfies Required<Omit<OrchestrationV2SubscribeThreadInput, "threadId" | "afterSequence">>;
117 +const CLIENT_SHELL_SUBSCRIPTION = {
118 + requestCompletionMarker: true,
119 + omitIdleSubagents: true,
120 + omitOlderSettled: true,
121 +} satisfies Required<
122 + Omit<OrchestrationV2SubscribeShellInput, "afterSequence" | "httpSnapshotFallback">
123 +>;
124 +const query = (optIns: Readonly<Record<string, string>>) => new URLSearchParams(optIns).toString();
125 +const CLIENT_BOUNDED_QUERY = query(CLIENT_BOUNDED_OPT_INS);
126 +const CLIENT_HISTORY_QUERY = query(CLIENT_HISTORY_OPT_INS);
127 +const CLIENT_SHELL_QUERY = query(CLIENT_SHELL_OPT_INS);
128
129 const decodeThreadSnapshot = Schema.decodeUnknownEffect(
130 Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadDetailSnapshot)),
@@ -224,30 +261,36 @@ const collectUntil = Effect.fn("transfer.collectUntil")(function* <T extends Str
261 if (done(item)) return items;
262 }
263 });
264 +type Subscription =
265 + | ({ readonly thread: ThreadId } & Pick<OrchestrationV2SubscribeThreadInput, "afterSequence">)
266 + | ({ readonly shell: true } & Pick<
267 + OrchestrationV2SubscribeShellInput,
268 + "afterSequence" | "httpSnapshotFallback"
269 + >);
270 +function subscriptionStream(
271 + client: MeasuredWsClient,
272 + subscription: Subscription,
273 +): Stream.Stream<StreamItem, unknown> {
274 + if ("thread" in subscription) {
275 + const { thread, ...resume } = subscription;
276 + return client.client[ORCHESTRATION_V2_WS_METHODS.subscribeThread]({
277 + ...CLIENT_THREAD_SUBSCRIPTION,
278 + ...resume,
279 + threadId: thread,
280 + });
281 + }
282 + const { shell: _, ...resume } = subscription;
283 + return client.client[ORCHESTRATION_V2_WS_METHODS.subscribeShell]({
284 + ...CLIENT_SHELL_SUBSCRIPTION,
285 + ...resume,
286 + });
287 +}
288 const subscribe = Effect.fn("transfer.subscribe")(function* (
289 client: MeasuredWsClient,
229 - kind: "thread" | "shell",
230 - afterSequence: number,
290 + subscription: Subscription,
291 ) {
292 const queue = yield* Queue.unbounded<StreamItem>();
233 - const stream: Stream.Stream<StreamItem, unknown> =
234 - kind === "thread"
235 - ? client.client[ORCHESTRATION_V2_WS_METHODS.subscribeThread]({
236 - threadId: THREAD_ID,
237 - afterSequence,
238 - requestCompletionMarker: true,
239 - acceptBoundedSnapshot: true,
240 - acceptCompactTurnItems: true,
241 - acceptCompactIds: true,
242 - acceptDedupedBodies: true,
243 - })
244 - : client.client[ORCHESTRATION_V2_WS_METHODS.subscribeShell]({
245 - afterSequence,
246 - requestCompletionMarker: true,
247 - omitIdleSubagents: true,
248 - omitOlderSettled: true,
249 - });
250 - yield* stream.pipe(
293 + yield* subscriptionStream(client, subscription).pipe(
294 Stream.runForEach((item) => Queue.offer(queue, item)),
295 Scope.provide(client.scope),
296 Effect.forkIn(client.scope),
@@ -324,10 +367,11 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
367 const shell = yield* openMeasuredWsClient({ url, cookie: "" });
368 const second = yield* openMeasuredWsClient({ url, cookie: "" });
369 assert.include(thread.recorder.negotiatedExtensions(), "permessage-deflate");
327 - const tq = yield* subscribe(thread, "thread", sequence);
328 - const sq = yield* subscribe(shell, "shell", sequence);
329 - const ttq = yield* subscribe(second, "thread", sequence);
330 - const ssq = yield* subscribe(second, "shell", sequence);
370 + const fromSnapshot = { afterSequence: sequence };
371 + const tq = yield* subscribe(thread, { thread: THREAD_ID, ...fromSnapshot });
372 + const sq = yield* subscribe(shell, { shell: true, ...fromSnapshot });
373 + const ttq = yield* subscribe(second, { thread: THREAD_ID, ...fromSnapshot });
374 + const ssq = yield* subscribe(second, { shell: true, ...fromSnapshot });
375 for (const queue of [tq, sq, ttq, ssq])
376 assert.equal(yield* synchronized(queue), "replay");
377 const threadBefore = thread.recorder.totals();
@@ -376,8 +420,8 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
420 const reconnectSqlStart = counter.count();
421 const resumedThread = yield* openMeasuredWsClient({ url, cookie: "" });
422 const resumedShell = yield* openMeasuredWsClient({ url, cookie: "" });
379 - const rq = yield* subscribe(resumedThread, "thread", sequence);
380 - const rsq = yield* subscribe(resumedShell, "shell", sequence);
423 + const rq = yield* subscribe(resumedThread, { thread: THREAD_ID, ...fromSnapshot });
424 + const rsq = yield* subscribe(resumedShell, { shell: true, ...fromSnapshot });
425 const threadMode = yield* synchronized(rq);
426 const shellMode = yield* synchronized(rsq);
427 const reconnectThread = { ...resumedThread.recorder.totals(), mode: threadMode };
@@ -385,7 +429,6 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
429 const reconnectSqlStatements = counter.count() - reconnectSqlStart;
430 assert.equal(threadMode, "snapshot");
431
388 - for (const events of longTurnThreadBatches(provider)) yield* sink.write({ events });
432 const get = (path: string) =>
433 measureHttpGet({
434 url: `${base}${path}`,
@@ -398,8 +441,9 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
441 Effect.gen(function* () {
442 const http = yield* get(`${longTurnPath}/bounded?${query}`);
443 const snapshot = yield* decodeBoundedSnapshot(decoded(http));
401 - const rows = snapshot.projection.visibleTurnItems.length;
402 - return { http, rows, cursor: snapshot.historyCursor };
444 + const rows = snapshot.projection.visibleTurnItems;
445 + const workAnchor = rows.find((row) => row.item.deferredWork !== undefined);
446 + return { http, rows: rows.length, cursor: snapshot.historyCursor, workAnchor };
447 });
448 const readHistoryPage = (cursor: string, query: string) =>
449 Effect.gen(function* () {
@@ -413,15 +457,75 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
457 const readShell = (query: string) =>
458 Effect.gen(function* () {
459 const http = yield* get(`/api/orchestration/shell?${query}`);
416 - return { http, rows: (yield* decodeShellSnapshot(decoded(http))).threads.length };
460 + const { threads, snapshotSequence } = yield* decodeShellSnapshot(decoded(http));
461 + return { http, rows: threads.length, snapshotSequence };
462 + });
463 + // A new socket, read until the subscription caught up or the server ended it.
464 + const open = (subscription: Subscription) =>
465 + Effect.gen(function* () {
466 + const socket = yield* openMeasuredWsClient({ url, cookie: "" });
467 + const items = yield* subscriptionStream(socket, subscription).pipe(
468 + Stream.takeUntil((item) => item.kind === "synchronized"),
469 + Stream.runCollect,
470 + Scope.provide(socket.scope),
471 + Effect.orDie,
472 + );
473 + const totals = socket.recorder.totals();
474 + yield* socket.close;
475 + return { kinds: items.map((item) => item.kind), items, totals };
476 });
477
419 - const { cursor: _, ...longTurnColdOpenLegacy } =
420 - yield* readColdOpen("compactTurnItems=1");
421 - const { cursor, ...longTurnColdOpen } = yield* readColdOpen(CLIENT_BOUNDED_QUERY);
478 + // A phone that comes back after one ordinary turn replays it, without tool bodies.
479 + const shortTurn = yield* sink.write({
480 + events: turnEvents(provider, TRANSFER_HISTORY_TURN_COUNT + 1, false),
481 + });
482 + const shortTurnResume = yield* open({ thread: THREAD_ID, afterSequence: last.sequence });
483 + assert.notInclude(shortTurnResume.kinds, "snapshot");
484 + // The byte cap below also passes when the resume skips the turn.
485 + assert.isTrue(
486 + shortTurnResume.items.some(
487 + (item) => "sequence" in item && item.sequence === shortTurn.at(-1)!.sequence,
488 + ),
489 + );
490 +
491 + // A turn that is still running cannot defer its work, so it loads in row-budget windows.
492 + const [finishLongTurn, ...longTurnHistory] = longTurnThreadBatches(provider).toReversed();
493 + for (const events of longTurnHistory.toReversed()) yield* sink.write({ events });
494 + const longTurnRunningColdOpenLegacy = yield* readColdOpen("compactTurnItems=1");
495 + const longTurnRunningColdOpen = yield* readColdOpen(CLIENT_BOUNDED_QUERY);
496 + const { cursor } = longTurnRunningColdOpen;
497 if (cursor === null) return yield* Effect.die("Expected older long-turn history");
423 - const longTurnHistoryPageLegacy = yield* readHistoryPage(cursor, "");
424 - const longTurnHistoryPage = yield* readHistoryPage(cursor, CLIENT_HISTORY_QUERY);
498 + const longTurnRunningHistoryPageLegacy = yield* readHistoryPage(cursor, "");
499 + const longTurnRunningHistoryPage = yield* readHistoryPage(cursor, CLIENT_HISTORY_QUERY);
500 +
501 + // Once it finishes, its work stays on the server until a group is expanded.
502 + yield* sink.write({ events: finishLongTurn! });
503 + const longTurnColdOpen = yield* readColdOpen(CLIENT_BOUNDED_QUERY);
504 + const { workAnchor } = longTurnColdOpen;
505 + if (workAnchor?.item.deferredWork === undefined)
506 + return yield* Effect.die("Expected the finished long turn to defer its work");
507 + const longTurnWorkPage = yield* readHistoryPage(
508 + workAnchor.item.deferredWork.cursor,
509 + `view=work&throughEntryId=${encodeURIComponent(workAnchor.sourceItemId)}&compactIds=1`,
510 + );
511 + // Mobile opens a thread over the socket alone.
512 + const socketOpen = yield* open({ thread: LONG_TURN_THREAD_ID });
513 + const socketSnapshot = socketOpen.items.find(
514 + (item) => item.kind === "snapshot" && "projection" in item,
515 + );
516 + if (socketSnapshot?.kind !== "snapshot" || !("projection" in socketSnapshot))
517 + return yield* Effect.die("Expected a bounded socket snapshot");
518 + const longTurnSocketColdOpen = {
519 + ...socketOpen,
520 + rows: socketSnapshot.projection.visibleTurnItems.length,
521 + };
522 + // The row cap below also passes when the socket ignores the deferral and sends a window.
523 + assert.equal(longTurnSocketColdOpen.rows, longTurnColdOpen.rows);
524 + const idleThreadResume = yield* open({
525 + thread: LONG_TURN_THREAD_ID,
526 + afterSequence: socketSnapshot.snapshotSequence,
527 + });
528 + assert.deepEqual(idleThreadResume.kinds, ["synchronized"]);
529
530 // The transfer thread plus the long-turn thread.
531 const earlierThreads = decodedShell.threads.length + 1;
@@ -432,6 +536,19 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
536 fleetShellLegacy.rows,
537 earlierThreads + SHELL_SETTLED_THREADS + SHELL_IDLE_SUBAGENTS,
538 );
539 + // A cold connect subscribes from the snapshot it just loaded.
540 + const fleetShellSubscribe = yield* open({
541 + shell: true,
542 + afterSequence: fleetShell.snapshotSequence,
543 + });
544 + assert.notInclude(fleetShellSubscribe.kinds, "snapshot");
545 + // A cursor too old to replay is told to reload over HTTP, not sent a socket snapshot.
546 + const staleShellResume = yield* open({
547 + shell: true,
548 + afterSequence: sequence,
549 + httpSnapshotFallback: true,
550 + });
551 + assert.deepEqual(staleShellResume.kinds, ["snapshot-required"]);
552
553 for (const events of staleTodoListThreadBatches(provider)) yield* sink.write({ events });
554 const staleTodoHttp = yield* get(
@@ -472,12 +589,19 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
589 reconnectThread,
590 reconnectShell,
591 reconnectSqlStatements,
475 - longTurnColdOpenLegacy,
592 + shortTurnResume,
593 + longTurnRunningColdOpenLegacy,
594 + longTurnRunningColdOpen,
595 + longTurnRunningHistoryPageLegacy,
596 + longTurnRunningHistoryPage,
597 longTurnColdOpen,
477 - longTurnHistoryPageLegacy,
478 - longTurnHistoryPage,
598 + longTurnWorkPage,
599 + longTurnSocketColdOpen,
600 + idleThreadResume,
601 fleetShellLegacy,
602 fleetShell,
603 + fleetShellSubscribe,
604 + staleShellResume,
605 staleTodoColdOpen,
606 } satisfies TransferBudgetRun;
607 }).pipe(
@@ -492,10 +616,14 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
616 const outputPaths = yield* Config.all({
617 report: Config.String("T3CODE_TRANSFER_BUDGET_REPORT_PATH").pipe(Config.withDefault("")),
618 result: Config.String("T3CODE_TRANSFER_BUDGET_RESULT_PATH").pipe(Config.withDefault("")),
619 + scenarios: Config.String("T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH").pipe(
620 + Config.withDefault(""),
621 + ),
622 });
623 for (const [path, contents] of [
624 [outputPaths.report, report],
625 [outputPaths.result, formatTransferBudgetResult(runs)],
626 + [outputPaths.scenarios, formatTransferBudgetScenarios(runs)],
627 ])
628 if (path && contents) yield* fs.writeFileString(path, contents);
629 yield* Effect.log(report);
docs/internals/performance-regressions.md
+18 -8
@@ -19,7 +19,8 @@ The v2 checks pin these invariants:
19 large work groups until expansion. Messages and items outside those groups remain in normal
20 history paging. Required control records and
21 the at-least-one-row rule can exceed the byte target; it is not a hard cap.
22 -- The bounded snapshot does not retain the full duplicate conversation-message table.
22 +- The bounded snapshot does not retain the full duplicate conversation-message table. Messages of
23 + a run that is still in progress are the exception: all of them accompany the window.
24 - Older activity is fetched in bounded HTTP pages and merged without disturbing the live scroll
25 window.
26 - Clients with confirmed history paging opt into socket snapshot fallbacks that use the same
@@ -68,13 +69,22 @@ These are neither compressed WebSocket captures nor measurements of server RSS,
69 fixtures exclude unrelated settings calls and subscription events. Payload budgets belong in
70 these tests so regressions fail locally. The transfer budget integration test
71 (`apps/server/integration/transferBudgetV2.integration.test.ts`) sends the opt-ins current clients
71 -send and caps the decoded bytes and rows of the bounded snapshot and one older page for a 900-row
72 -agent turn, and of the cold-connect shell for a sidebar of settled threads and idle subagents. It
73 -also caps the bytes and the node, subagent, plan, provider turn and checkpoint counts of the
74 -bounded snapshot for a thread of 40 finished runs behind one working run, so finished runs cannot
75 -ride along with a window. The working run has a ready context handoff, so its history cannot ride
76 -along either. When clients gain an opt-in, add it to the test's client queries and give a fixture
77 -the data it saves.
72 +send. Its opt-in objects are typed `Required` against the contracts, so an opt-in added to a
73 +contract fails typecheck until the test sends it; give a fixture the data the opt-in saves. The
74 +test caps decoded bytes and rows for a 900-row agent turn in both states: while the turn runs (the
75 +bounded snapshot and one older page) and after it finishes (the deferred cold open over HTTP and
76 +over the socket alone, and the first page of its deferred work). It caps the socket bytes of three
77 +resumes that must not send a snapshot: a thread after one ordinary turn, an idle thread, and a
78 +shell cursor too old to replay. It caps the cold-connect shell for a sidebar of settled threads and
79 +idle subagents, and the subscription a client opens from that snapshot. It also caps the bytes and
80 +the node, subagent, plan, provider turn and checkpoint counts of the bounded snapshot for a thread
81 +of 40 finished runs behind one working run, so finished runs cannot ride along with a window. The
82 +working run has a ready context handoff, so its history cannot ride along either.
83 +
84 +CI prints every row in the job summary. The pull request comment compares the budgeted rows with
85 +`main`, lists the rows that changed or failed, and names each ceiling a pull request raises. A row
86 +added to `scenarioRows` in `TransferBudgetReport.integration.ts` reaches the pass/fail check, the
87 +summary and the comment.
88
89 The server config has its own budget in `apps/server/src/ws.test.ts`. A client that asks for
90 catalog digests and status deltas gets the provider list of its first config in under 1 KiB, for a
packages/contracts/src/environmentHttp.ts
+3 -3
@@ -682,19 +682,19 @@ const EnvironmentOrchestrationThreadSnapshotParams = Schema.Struct({
682 threadId: ThreadId,
683 });
684
685 -const EnvironmentOrchestrationShellSnapshotQuery = Schema.Struct({
685 +export const EnvironmentOrchestrationShellSnapshotQuery = Schema.Struct({
686 omitIdleSubagents: Schema.optionalKey(Schema.String),
687 omitOlderSettled: Schema.optionalKey(Schema.String),
688 });
689
690 -const EnvironmentOrchestrationThreadBoundedSnapshotQuery = Schema.Struct({
690 +export const EnvironmentOrchestrationThreadBoundedSnapshotQuery = Schema.Struct({
691 deferWork: Schema.optionalKey(Schema.String),
692 compactTurnItems: Schema.optionalKey(Schema.String),
693 dedupeBodies: Schema.optionalKey(Schema.String),
694 compactIds: Schema.optionalKey(Schema.String),
695 });
696
697 -const EnvironmentOrchestrationThreadHistoryQuery = Schema.Struct({
697 +export const EnvironmentOrchestrationThreadHistoryQuery = Schema.Struct({
698 cursor: TrimmedNonEmptyString,
699 throughEntryId: Schema.optional(TrimmedNonEmptyString),
700 view: Schema.optional(Schema.Literals(["conversation", "activity", "work"])),
packages/shared/src/testing/providerEnv.ts
+1
@@ -8,6 +8,7 @@ const KEPT_ENV = new Set([
8 // CI sets these so the transfer budget integration test writes its report for the job summary.
9 "T3CODE_TRANSFER_BUDGET_REPORT_PATH",
10 "T3CODE_TRANSFER_BUDGET_RESULT_PATH",
11 + "T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH",
12 // Opt-in: points the HtmlRender tests at a headless browser, which they skip without.
13 "T3CODE_TEST_HEADLESS_SHELL",
14 ]);