The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 19:45 UTC
test(server): wire budget measures what clients fetch today, and the PR comment shows every row (#35225)
Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
Theo Browne committed
Nov 12, 2026 at 04:18 UTC
476fc9be6691ab0d81e936f4cc674983cf3913be
9 files changed
+623
-152
.github/scripts/thread-transfer-report.cjs
+150
-9
@@ -3,6 +3,7 @@ const path = require("node:path");
3
4
const ARTIFACT_NAME = "thread-transfer-results";
5
const RESULT_FILE = "thread-transfer-result.json";
6
+const SCENARIOS_FILE = "thread-transfer-scenarios.json";
7
const COMMENT_MARKER = "<!-- t3-thread-transfer-report -->";
8
const PROVIDERS = ["codex", "claudeAgent"];
9
const OBSERVED_KEYS = [
@@ -29,6 +30,12 @@ const SCENARIO_KEYS = [
30
"measuredMcpResultBytes",
31
];
32
33
+const SCENARIO_ROW_KEYS = ["phase", "metric", "unit", "observed", "ceiling"];
34
+// These labels come from the PR and reach a comment the bot posts: no markup, links or mentions.
35
+// GitHub still links plain text such as `GH-26`, so they are rendered as code spans.
36
+const SCENARIO_LABEL = /^[A-Za-z0-9][A-Za-z0-9 ,-]{0,79}$/;
37
+const MAX_SCENARIO_ROWS = 100;
38
+
39
function resultShaMarker(sha) {
40
return `<!-- t3-thread-transfer-result-sha:${sha} -->`;
41
}
@@ -85,15 +92,63 @@ function validateResult(value) {
92
return value;
93
}
94
88
-function readResult(directory) {
95
+function validateScenarios(value) {
96
+ assertExactKeys(value, ["schemaVersion", "providers"], "scenarios");
97
+ if (value.schemaVersion !== 1) {
98
+ throw new Error("scenarios.schemaVersion must be 1");
99
+ }
100
+ assertExactKeys(value.providers, PROVIDERS, "scenarios.providers");
101
+ for (const provider of PROVIDERS) {
102
+ const rows = value.providers[provider];
103
+ if (!Array.isArray(rows) || rows.length > MAX_SCENARIO_ROWS) {
104
+ throw new Error(
105
+ `scenarios.providers.${provider} must list at most ${MAX_SCENARIO_ROWS} rows`,
106
+ );
107
+ }
108
+ const seen = new Set();
109
+ for (const [index, row] of rows.entries()) {
110
+ const label = `scenarios.providers.${provider}[${index}]`;
111
+ assertExactKeys(row, SCENARIO_ROW_KEYS, label);
112
+ for (const key of ["phase", "metric"]) {
113
+ if (typeof row[key] !== "string" || !SCENARIO_LABEL.test(row[key])) {
114
+ throw new Error(`${label}.${key} must be a short plain label`);
115
+ }
116
+ }
117
+ if (row.unit !== "bytes" && row.unit !== "count") {
118
+ throw new Error(`${label}.unit must be bytes or count`);
119
+ }
120
+ assertMetric(row.observed, `${label}.observed`);
121
+ assertMetric(row.ceiling, `${label}.ceiling`);
122
+ const key = scenarioKey(row);
123
+ if (seen.has(key)) throw new Error(`${label} repeats an earlier row`);
124
+ seen.add(key);
125
+ }
126
+ }
127
+ return value;
128
+}
129
+
130
+function scenarioKey(row) {
131
+ return `${row.phase}|${row.metric}`;
132
+}
133
+
134
+function readArtifactFile(directory, name, validate) {
135
if (!directory) return undefined;
90
- const file = path.join(directory, RESULT_FILE);
136
+ const file = path.join(directory, name);
137
if (!fs.existsSync(file)) return undefined;
138
const stat = fs.lstatSync(file);
139
if (!stat.isFile() || stat.size > 64 * 1_024) {
94
- throw new Error("thread transfer result must be a regular file smaller than 64 KiB");
140
+ throw new Error(`${name} must be a regular file smaller than 64 KiB`);
141
}
96
- return validateResult(JSON.parse(fs.readFileSync(file, "utf8")));
142
+ return validate(JSON.parse(fs.readFileSync(file, "utf8")));
143
+}
144
+
145
+function readResult(directory) {
146
+ return readArtifactFile(directory, RESULT_FILE, validateResult);
147
+}
148
+
149
+// Older runs upload only the result file, so a run without scenarios is still a valid run.
150
+function readScenarios(directory) {
151
+ return readArtifactFile(directory, SCENARIOS_FILE, validateScenarios);
152
}
153
154
function formatBytes(bytes) {
@@ -103,7 +158,7 @@ function formatBytes(bytes) {
158
}
159
160
function formatValue(value, kind) {
106
- return kind === "messages" ? value.toLocaleString("en-US") : formatBytes(value);
161
+ return kind === "bytes" ? formatBytes(value) : value.toLocaleString("en-US");
162
}
163
164
function formatImpact(current, baseline, kind) {
@@ -136,6 +191,77 @@ const METRICS = [
191
{ key: "measuredTurnWebSocketMessages", label: "Live turn messages", kind: "messages" },
192
];
193
194
+function providerLabel(provider) {
195
+ return provider === "codex" ? "Codex" : "Claude";
196
+}
197
+
198
+/**
199
+ * The scenario rows of a run against the same rows on `main`. Rows that fail or differ from
200
+ * `main` are listed in the open; the full table is collapsed below them.
201
+ */
202
+function renderScenarios(current, baseline, ceilingChanges) {
203
+ if (!current) return { lines: [], failed: false };
204
+ const header = [
205
+ "| Provider | Scenario | Metric | Main baseline | This PR | Impact | PR ceiling | |",
206
+ "| --- | --- | --- | ---: | ---: | ---: | ---: | --- |",
207
+ ];
208
+ const all = [];
209
+ const notable = [];
210
+ let failed = false;
211
+ let withoutBaseline = 0;
212
+
213
+ for (const provider of PROVIDERS) {
214
+ const baselineRows = new Map(
215
+ (baseline?.providers[provider] ?? []).map((row) => [scenarioKey(row), row]),
216
+ );
217
+ for (const row of current.providers[provider]) {
218
+ const before = baselineRows.get(scenarioKey(row));
219
+ const beforeObserved = before?.unit === row.unit ? before.observed : undefined;
220
+ const pass = row.observed <= row.ceiling;
221
+ failed ||= !pass;
222
+ if (beforeObserved === undefined) withoutBaseline += 1;
223
+ const line = `| ${providerLabel(provider)} | \`${row.phase}\` | \`${row.metric}\` | ${beforeObserved === undefined ? "—" : formatValue(beforeObserved, row.unit)} | ${formatValue(row.observed, row.unit)} | ${formatImpact(row.observed, beforeObserved, row.unit)} | ${formatValue(row.ceiling, row.unit)} | ${pass ? "✅" : "❌"} |`;
224
+ all.push(line);
225
+ if (!pass || (beforeObserved !== undefined && beforeObserved !== row.observed)) {
226
+ notable.push(line);
227
+ }
228
+ if (beforeObserved !== undefined && before.ceiling !== row.ceiling) {
229
+ ceilingChanges.push(
230
+ `- ${providerLabel(provider)} \`${row.phase}\` \`${row.metric}\`: ${formatValue(before.ceiling, row.unit)} → ${formatValue(row.ceiling, row.unit)}`,
231
+ );
232
+ }
233
+ }
234
+ }
235
+
236
+ const compared = all.length - withoutBaseline;
237
+ const summary = !baseline
238
+ ? `No \`main\` baseline exists yet for these ${all.length} scenario measurements.`
239
+ : [
240
+ notable.length > 0
241
+ ? `${notable.length} of ${all.length} scenario measurements changed or failed.`
242
+ : `${compared === all.length ? "All " : ""}${compared} scenario measurements match \`main\`.`,
243
+ ...(withoutBaseline > 0 ? [`${withoutBaseline} have no \`main\` baseline.`] : []),
244
+ ].join(" ");
245
+ return {
246
+ failed,
247
+ lines: [
248
+ "### Other scenarios",
249
+ "",
250
+ summary,
251
+ ...(notable.length > 0 ? ["", ...header, ...notable] : []),
252
+ "",
253
+ "<details>",
254
+ `<summary>All ${all.length} scenario measurements</summary>`,
255
+ "",
256
+ ...header,
257
+ ...all,
258
+ "",
259
+ "</details>",
260
+ "",
261
+ ],
262
+ };
263
+}
264
+
265
function renderComment(input) {
266
const current = input.current;
267
const baseline = input.baseline;
@@ -154,16 +280,22 @@ function renderComment(input) {
280
const pass = observed <= ceiling;
281
failed ||= !pass;
282
rows.push(
157
- `| ${provider === "codex" ? "Codex" : "Claude"} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
283
+ `| ${providerLabel(provider)} | ${metric.label} | ${baselineObserved === undefined ? "—" : formatValue(baselineObserved, metric.kind)} | ${formatValue(observed, metric.kind)} | ${formatImpact(observed, baselineObserved, metric.kind)} | ${formatValue(ceiling, metric.kind)} | ${pass ? "✅" : "❌"} |`,
284
);
285
286
if (comparable && baseline.providers[provider].ceiling[metric.key] !== ceiling) {
287
ceilingChanges.push(
162
- `- ${provider === "codex" ? "Codex" : "Claude"} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
288
+ `- ${providerLabel(provider)} ${metric.label}: ${formatValue(baseline.providers[provider].ceiling[metric.key], metric.kind)} → ${formatValue(ceiling, metric.kind)}`,
289
);
290
}
291
}
292
}
293
+ const scenarios = renderScenarios(
294
+ input.currentScenarios,
295
+ input.baselineScenarios,
296
+ ceilingChanges,
297
+ );
298
+ failed ||= scenarios.failed;
299
300
const baselineLink = input.baselineRun
301
? `[\`${input.baselineRun.sha.slice(0, 7)}\`](${input.baselineRun.url})`
@@ -203,6 +335,7 @@ function renderComment(input) {
335
"| --- | --- | ---: | ---: | ---: | ---: | --- |",
336
...rows,
337
"",
338
+ ...scenarios.lines,
339
`Baseline: ${baselineLink} · PR result: ${currentLink} · Source CI: ${input.currentRun.conclusion}`,
340
"",
341
"<details>",
@@ -212,7 +345,7 @@ function renderComment(input) {
345
"",
346
...PROVIDERS.map(
347
(provider) =>
215
- `- ${provider === "codex" ? "Codex" : "Claude"} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
348
+ `- ${providerLabel(provider)} decoded thread snapshot: ${formatBytes(current.providers[provider].observed.threadSnapshotDecodedBytes)}`,
349
),
350
"",
351
"</details>",
@@ -437,7 +570,14 @@ async function publish({ github, context, core }) {
570
url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.BASELINE_RUN_ID}`,
571
}
572
: undefined;
440
- const body = renderComment({ current, baseline, currentRun, baselineRun });
573
+ const body = renderComment({
574
+ current,
575
+ baseline,
576
+ currentScenarios: readScenarios(process.env.PR_RESULT_DIR),
577
+ baselineScenarios: readScenarios(process.env.BASELINE_RESULT_DIR),
578
+ currentRun,
579
+ baselineRun,
580
+ });
581
const published = await upsertCommentForCurrentHead(
582
github,
583
context,
@@ -458,4 +598,5 @@ module.exports = {
598
resolve,
599
upsertCommentForCurrentHead,
600
validateResult,
601
+ validateScenarios,
602
};
.github/scripts/thread-transfer-report.test.cjs
+79
@@ -2,9 +2,11 @@ const assert = require("node:assert/strict");
2
const test = require("node:test");
3
4
const {
5
+ renderComment,
6
resolve,
7
upsertCommentForCurrentHead,
8
validateResult,
9
+ validateScenarios,
10
} = require("./thread-transfer-report.cjs");
11
12
function result(overrides = {}) {
@@ -40,6 +42,29 @@ function result(overrides = {}) {
42
};
43
}
44
45
+function scenarios(overrides = {}) {
46
+ const rows = [
47
+ {
48
+ phase: "long turn finished, cold open",
49
+ metric: "HTTP decoded",
50
+ unit: "bytes",
51
+ observed: 390_000,
52
+ ceiling: 420_000,
53
+ },
54
+ {
55
+ phase: "idle thread resume",
56
+ metric: "WebSocket messages",
57
+ unit: "count",
58
+ observed: 1,
59
+ ceiling: 1,
60
+ },
61
+ ];
62
+ return {
63
+ schemaVersion: 1,
64
+ providers: { codex: [{ ...rows[0], ...overrides }, rows[1]], claudeAgent: rows },
65
+ };
66
+}
67
+
68
test("validates the fixed artifact schema", () => {
69
assert.equal(validateResult(result()).schemaVersion, 1);
70
assert.throws(
@@ -50,6 +75,60 @@ test("validates the fixed artifact schema", () => {
75
() => validateResult(result({ totalWireBytes: "lots" })),
76
/non-negative safe integer/,
77
);
78
+
79
+ assert.equal(validateScenarios(scenarios()).schemaVersion, 1);
80
+ for (const phase of ["@everyone", "[link](https://example.com)", "a | b", "<img>", ""]) {
81
+ assert.throws(() => validateScenarios(scenarios({ phase })), /short plain label/);
82
+ }
83
+ assert.throws(() => validateScenarios(scenarios({ note: "extra" })), /unexpected fields/);
84
+ assert.throws(() => validateScenarios(scenarios({ ceiling: null })), /non-negative safe integer/);
85
+ assert.throws(
86
+ () =>
87
+ validateScenarios(scenarios({ phase: "idle thread resume", metric: "WebSocket messages" })),
88
+ /repeats an earlier row/,
89
+ );
90
+});
91
+
92
+test("lists scenario rows that changed or failed above the collapsed table", () => {
93
+ const run = { sha: "abcdef1234", url: "https://example.com/run", conclusion: "success" };
94
+ const render = (currentScenarios, baselineScenarios) =>
95
+ renderComment({
96
+ current: result(),
97
+ baseline: result(),
98
+ currentScenarios,
99
+ baselineScenarios,
100
+ currentRun: run,
101
+ baselineRun: { ...run, matchesBase: true },
102
+ });
103
+ const open = (body) => body.slice(0, body.indexOf("<details>"));
104
+
105
+ const unchanged = render(scenarios(), scenarios());
106
+ assert.match(unchanged, /✅ Thread transfer remains within every enforced ceiling/);
107
+ assert.match(open(unchanged), /All 4 scenario measurements match `main`/);
108
+ assert.doesNotMatch(open(unchanged), /long turn finished, cold open/);
109
+
110
+ const grown = render(scenarios({ observed: 400_000 }), scenarios());
111
+ assert.match(open(grown), /1 of 4 scenario measurements changed or failed/);
112
+ assert.match(
113
+ open(grown),
114
+ /\| Codex \| `long turn finished, cold open` \| `HTTP decoded` \| 380\.9 KiB \| 390\.6 KiB \| \+9\.8 KiB \(\+2\.6%\) \| 410\.2 KiB \| ✅ \|/,
115
+ );
116
+
117
+ const over = render(scenarios({ observed: 430_000, ceiling: 425_000 }), scenarios());
118
+ assert.match(over, /❌ One or more thread transfer ceilings were exceeded/);
119
+ assert.match(
120
+ over,
121
+ /Codex `long turn finished, cold open` `HTTP decoded`: 410\.2 KiB → 415\.0 KiB/,
122
+ );
123
+
124
+ // A row `main` does not have is not reported as matching it.
125
+ const added = render(scenarios({ phase: "GH-26" }), scenarios());
126
+ assert.match(open(added), /3 scenario measurements match `main`\. 1 have no `main` baseline\./);
127
+ assert.match(added, /\| Codex \| `GH-26` \|/);
128
+
129
+ // A main run from before the scenario file has nothing to compare; a PR run from before it has no section.
130
+ assert.match(open(render(scenarios(), undefined)), /No `main` baseline exists yet/);
131
+ assert.doesNotMatch(render(undefined, scenarios()), /Other scenarios/);
132
});
133
134
test("resolves a fallback PR with a redacted head repo and exact main baseline", async () => {
.github/workflows/ci.yml
+4
-1
@@ -503,6 +503,7 @@ jobs:
503
env:
504
T3CODE_TRANSFER_BUDGET_REPORT_PATH: ${{ runner.temp }}/t3code-transfer-budget.md
505
T3CODE_TRANSFER_BUDGET_RESULT_PATH: ${{ runner.temp }}/thread-transfer-result.json
506
+ T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH: ${{ runner.temp }}/thread-transfer-scenarios.json
507
run: >-
508
vp run test:cached -- --shard ${{ matrix.shard }}/${{ strategy.job-total }}
509
--uncached apps/server/integration/transferBudgetV2.integration.test.ts
@@ -543,7 +544,9 @@ jobs:
544
uses: actions/upload-artifact@v7
545
with:
546
name: thread-transfer-results
546
- path: ${{ runner.temp }}/thread-transfer-result.json
547
+ path: |
548
+ ${{ runner.temp }}/thread-transfer-result.json
549
+ ${{ runner.temp }}/thread-transfer-scenarios.json
550
if-no-files-found: ignore
551
retention-days: 30
552
apps/server/integration/TransferBudgetReport.integration.ts
+190
-83
@@ -38,6 +38,11 @@ const STALE_TODO_RECORD_KINDS = [
38
] as const;
39
type StaleTodoRecordCounts = Readonly<Record<(typeof STALE_TODO_RECORD_KINDS)[number], number>>;
40
41
+/** One new socket, read from the subscribe call until the subscription caught up. */
42
+export interface SocketOpenMeasurement {
43
+ readonly totals: WebSocketTransferTotals;
44
+}
45
+
46
export interface TransferBudgetRun {
47
readonly provider: ProviderDriverKind;
48
readonly threadSnapshot: HttpTransferMeasurement;
@@ -49,15 +54,28 @@ export interface TransferBudgetRun {
54
readonly reconnectShell: WebSocketCatchUpMeasurement;
55
readonly measuredTurnSqlStatements: number;
56
readonly reconnectSqlStatements: number;
52
- /** The compact bounded snapshot web and mobile load when the long-turn thread opens. */
53
- readonly longTurnColdOpen: TimelineHttpMeasurement;
54
- readonly longTurnHistoryPage: TimelineHttpMeasurement;
57
+ /** A thread resumed from the cursor it held before one ordinary turn. */
58
+ readonly shortTurnResume: SocketOpenMeasurement;
59
+ /** The long-turn thread while its last turn runs: the bounded snapshot, then one older page. */
60
+ readonly longTurnRunningColdOpen: TimelineHttpMeasurement;
61
+ readonly longTurnRunningHistoryPage: TimelineHttpMeasurement;
62
/** The same reads as an older client that only sends `compactTurnItems`, to show the savings. */
56
- readonly longTurnColdOpenLegacy: TimelineHttpMeasurement;
57
- readonly longTurnHistoryPageLegacy: TimelineHttpMeasurement;
63
+ readonly longTurnRunningColdOpenLegacy: TimelineHttpMeasurement;
64
+ readonly longTurnRunningHistoryPageLegacy: TimelineHttpMeasurement;
65
+ /** The same thread once the turn finished, over HTTP (web) and over the socket alone (mobile). */
66
+ readonly longTurnColdOpen: TimelineHttpMeasurement;
67
+ readonly longTurnSocketColdOpen: SocketOpenMeasurement & { readonly rows: number };
68
+ /** The first page of the finished turn's deferred work, loaded when its group is expanded. */
69
+ readonly longTurnWorkPage: TimelineHttpMeasurement;
70
+ /** A thread resumed from the cursor of its own last event. */
71
+ readonly idleThreadResume: SocketOpenMeasurement;
72
/** The cold-connect shell snapshot after the settled-thread fleet is written. */
73
readonly fleetShell: TimelineHttpMeasurement;
74
readonly fleetShellLegacy: TimelineHttpMeasurement;
75
+ /** The shell subscription a cold connect opens from that snapshot's cursor. */
76
+ readonly fleetShellSubscribe: SocketOpenMeasurement;
77
+ /** A shell resumed from a cursor too old to replay, by a client that reloads over HTTP. */
78
+ readonly staleShellResume: SocketOpenMeasurement;
79
/**
80
* The compact bounded snapshot of the long thread whose finished runs left todo lists open,
81
* with the count of each record kind that rides along with its rows.
@@ -89,14 +107,44 @@ const TRANSFER_BUDGETS: Readonly<Record<string, ProviderTransferBudget>> = {
107
claudeAgent: TRANSFER_BUDGET,
108
};
109
92
-// Decoded bytes, not wire bytes: phones parse and keep the decoded JSON. About 7% above the
93
-// observed 103 KB, so a cold open without body dedupe (117 KB), the ID table, or row budgets
94
-// (all 920 rows) fails. Kept apart from TRANSFER_BUDGET because the PR comment validates its keys.
95
-const LONG_TURN_BUDGET = {
110
+// Decoded bytes, not wire bytes: phones parse and keep the decoded JSON. About 15% above the
111
+// observed 95 KB history page, so a page without the ID table (130 KB) or row budgets fails.
112
+// Kept apart from TRANSFER_BUDGET because the PR comment validates its keys.
113
+const LONG_TURN_PAGE_BUDGET = {
114
decodedBytes: 110_000,
115
rows: 75,
116
};
117
118
+// About 7% above the observed 290 KB. The window holds 75 rows, but every message of a running
119
+// turn rides along with it: 196 KB here. Without body dedupe or the ID table it is 354 KB.
120
+const LONG_TURN_RUNNING_BUDGET = {
121
+ decodedBytes: 310_000,
122
+ rows: 75,
123
+};
124
+
125
+// About 7% above the observed 393 KB. A finished turn defers its 770 work rows and stays whole:
126
+// 151 rows of messages, checkpoints and subagents, on both transports.
127
+const LONG_TURN_FINISHED_BUDGET = {
128
+ decodedBytes: 420_000,
129
+ rows: 151,
130
+};
131
+
132
+// About 6% above the observed 193 KB: one older page of work, twice the cold-open row budget.
133
+const LONG_TURN_WORK_PAGE_BUDGET = {
134
+ decodedBytes: 205_000,
135
+ rows: 150,
136
+};
137
+
138
+const SOCKET_BUDGETS = {
139
+ // One ordinary turn replays in 11 KB. With its tool bodies it is over 900 KB.
140
+ shortTurnResume: { decodedBytes: 13_000, messages: 2 },
141
+ longTurnColdOpen: { decodedBytes: LONG_TURN_FINISHED_BUDGET.decodedBytes, messages: 2 },
142
+ // These three send no rows. A snapshot in their place is 393 KB for the thread, 42 KB for the shell.
143
+ idleThreadResume: { decodedBytes: 200, messages: 1 },
144
+ fleetShellSubscribe: { decodedBytes: 200, messages: 1 },
145
+ staleShellResume: { decodedBytes: 400, messages: 2 },
146
+};
147
+
148
// About 30% above the observed 42 KB. Without the shell opt-ins it carries all 142 rows (217 KB).
149
const FLEET_SHELL_BUDGET = {
150
decodedBytes: 55_000,
@@ -211,31 +259,139 @@ function webSocketRows(
259
];
260
}
261
214
-function timelineRows(
215
- provider: ProviderDriverKind,
262
+interface ScenarioRow {
263
+ readonly phase: string;
264
+ readonly metric: string;
265
+ readonly unit: "bytes" | "count";
266
+ readonly observed: number;
267
+ /** Absent on figures that only inform: wire bytes, and what an older client receives. */
268
+ readonly maximum: number | undefined;
269
+}
270
+
271
+function httpRows(
272
phase: string,
273
measured: TimelineHttpMeasurement,
218
- budget: { readonly decodedBytes: number; readonly rows: number },
219
-): string[] {
274
+ budget?: { readonly decodedBytes: number; readonly rows: number },
275
+): ScenarioRow[] {
276
return [
221
- infoRow(provider, phase, "HTTP wire", measured.http.wireBytes),
222
- row(provider, phase, "HTTP decoded", measured.http.decodedBodyBytes, budget.decodedBytes),
223
- row(provider, phase, "rows", measured.rows, budget.rows, String),
277
+ {
278
+ phase,
279
+ metric: "HTTP wire",
280
+ unit: "bytes",
281
+ observed: measured.http.wireBytes,
282
+ maximum: undefined,
283
+ },
284
+ {
285
+ phase,
286
+ metric: "HTTP decoded",
287
+ unit: "bytes",
288
+ observed: measured.http.decodedBodyBytes,
289
+ maximum: budget?.decodedBytes,
290
+ },
291
+ { phase, metric: "rows", unit: "count", observed: measured.rows, maximum: budget?.rows },
292
];
293
}
294
227
-function legacyRows(
228
- provider: ProviderDriverKind,
295
+function socketRows(
296
phase: string,
230
- measured: TimelineHttpMeasurement,
231
-): string[] {
297
+ { totals }: SocketOpenMeasurement,
298
+ budget: { readonly decodedBytes: number; readonly messages: number },
299
+): ScenarioRow[] {
300
+ return [
301
+ {
302
+ phase,
303
+ metric: "WebSocket wire",
304
+ unit: "bytes",
305
+ observed: totals.wireBytes,
306
+ maximum: undefined,
307
+ },
308
+ {
309
+ phase,
310
+ metric: "WebSocket decoded",
311
+ unit: "bytes",
312
+ observed: totals.decodedBytes,
313
+ maximum: budget.decodedBytes,
314
+ },
315
+ {
316
+ phase,
317
+ metric: "WebSocket messages",
318
+ unit: "count",
319
+ observed: totals.messages,
320
+ maximum: budget.messages,
321
+ },
322
+ ];
323
+}
324
+
325
+/**
326
+ * Every scenario figure beyond the thread totals. The violations, the report table and the PR
327
+ * comment's scenario file all read this list, so a row added here reaches all three. The PR
328
+ * comment only accepts letters, digits, spaces, commas and hyphens in `phase` and `metric`.
329
+ */
330
+function scenarioRows(run: TransferBudgetRun): ScenarioRow[] {
331
return [
233
- infoRow(provider, `${phase}, older client`, "HTTP wire", measured.http.wireBytes),
234
- infoRow(provider, `${phase}, older client`, "HTTP decoded", measured.http.decodedBodyBytes),
235
- infoRow(provider, `${phase}, older client`, "rows", measured.rows, String),
332
+ ...socketRows("short turn resume", run.shortTurnResume, SOCKET_BUDGETS.shortTurnResume),
333
+ ...httpRows(
334
+ "long turn running, cold open",
335
+ run.longTurnRunningColdOpen,
336
+ LONG_TURN_RUNNING_BUDGET,
337
+ ),
338
+ ...httpRows("long turn running, cold open, older client", run.longTurnRunningColdOpenLegacy),
339
+ ...httpRows(
340
+ "long turn running, history page",
341
+ run.longTurnRunningHistoryPage,
342
+ LONG_TURN_PAGE_BUDGET,
343
+ ),
344
+ ...httpRows(
345
+ "long turn running, history page, older client",
346
+ run.longTurnRunningHistoryPageLegacy,
347
+ ),
348
+ ...httpRows("long turn finished, cold open", run.longTurnColdOpen, LONG_TURN_FINISHED_BUDGET),
349
+ ...socketRows(
350
+ "long turn finished, socket cold open",
351
+ run.longTurnSocketColdOpen,
352
+ SOCKET_BUDGETS.longTurnColdOpen,
353
+ ),
354
+ {
355
+ phase: "long turn finished, socket cold open",
356
+ metric: "rows",
357
+ unit: "count",
358
+ observed: run.longTurnSocketColdOpen.rows,
359
+ maximum: LONG_TURN_FINISHED_BUDGET.rows,
360
+ },
361
+ ...httpRows("long turn finished, work page", run.longTurnWorkPage, LONG_TURN_WORK_PAGE_BUDGET),
362
+ ...socketRows("idle thread resume", run.idleThreadResume, SOCKET_BUDGETS.idleThreadResume),
363
+ ...httpRows("fleet shell snapshot", run.fleetShell, FLEET_SHELL_BUDGET),
364
+ ...httpRows("fleet shell snapshot, older client", run.fleetShellLegacy),
365
+ ...socketRows(
366
+ "fleet shell subscribe",
367
+ run.fleetShellSubscribe,
368
+ SOCKET_BUDGETS.fleetShellSubscribe,
369
+ ),
370
+ ...socketRows("stale shell resume", run.staleShellResume, SOCKET_BUDGETS.staleShellResume),
371
+ ...httpRows("stale todo list cold open", run.staleTodoColdOpen, STALE_TODO_BUDGET),
372
+ ...STALE_TODO_RECORD_KINDS.map((kind): ScenarioRow => ({
373
+ phase: "stale todo list cold open",
374
+ metric: kind,
375
+ unit: "count",
376
+ observed: run.staleTodoColdOpen.records[kind],
377
+ maximum: STALE_TODO_BUDGET.records[kind],
378
+ })),
379
];
380
}
381
382
+/** The budgeted scenario figures, for the PR comment to compare with `main`. */
383
+export function formatTransferBudgetScenarios(runs: ReadonlyArray<TransferBudgetRun>): string {
384
+ const providers = Object.fromEntries(
385
+ runs.map((run) => [
386
+ run.provider,
387
+ scenarioRows(run).flatMap(({ maximum, ...row }) =>
388
+ maximum === undefined ? [] : [{ ...row, ceiling: maximum }],
389
+ ),
390
+ ]),
391
+ );
392
+ return `${JSON.stringify({ schemaVersion: 1, providers }, null, 2)}\n`;
393
+}
394
+
395
export function transferBudgetViolations(runs: ReadonlyArray<TransferBudgetRun>): string[] {
396
const violations: string[] = [];
397
for (const run of runs) {
@@ -262,37 +418,8 @@ export function transferBudgetViolations(runs: ReadonlyArray<TransferBudgetRun>)
418
run.measuredTurnWebSocket.messages,
419
budget.measuredTurnWebSocketMessages,
420
],
265
- [
266
- "long-turn cold open decoded bytes",
267
- run.longTurnColdOpen.http.decodedBodyBytes,
268
- LONG_TURN_BUDGET.decodedBytes,
269
- ],
270
- ["long-turn cold open rows", run.longTurnColdOpen.rows, LONG_TURN_BUDGET.rows],
271
- [
272
- "long-turn history page decoded bytes",
273
- run.longTurnHistoryPage.http.decodedBodyBytes,
274
- LONG_TURN_BUDGET.decodedBytes,
275
- ],
276
- ["long-turn history page rows", run.longTurnHistoryPage.rows, LONG_TURN_BUDGET.rows],
277
- [
278
- "fleet shell snapshot decoded bytes",
279
- run.fleetShell.http.decodedBodyBytes,
280
- FLEET_SHELL_BUDGET.decodedBytes,
281
- ],
282
- ["fleet shell snapshot rows", run.fleetShell.rows, FLEET_SHELL_BUDGET.rows],
283
- [
284
- "stale todo list cold open decoded bytes",
285
- run.staleTodoColdOpen.http.decodedBodyBytes,
286
- STALE_TODO_BUDGET.decodedBytes,
287
- ],
288
- ["stale todo list cold open rows", run.staleTodoColdOpen.rows, STALE_TODO_BUDGET.rows],
289
- ...STALE_TODO_RECORD_KINDS.map(
290
- (kind) =>
291
- [
292
- `stale todo list cold open ${kind}`,
293
- run.staleTodoColdOpen.records[kind],
294
- STALE_TODO_BUDGET.records[kind],
295
- ] as const,
421
+ ...scenarioRows(run).flatMap(({ phase, metric, observed, maximum }) =>
422
+ maximum === undefined ? [] : [[`${phase} ${metric}`, observed, maximum] as const],
423
),
424
] as const;
425
for (const [metric, observed, maximum] of checks) {
@@ -311,8 +438,9 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
438
"Wire values are thread data bytes read from local HTTP and WebSocket sockets. HTTP includes response headers; WebSocket measurement starts after the resumed thread subscription synchronizes. TCP/IP, TLS framing, and the WebSocket upgrade are excluded. WebSocket permessage-deflate is negotiated.",
439
"The measured turn is observed on three sockets at once: one with only the thread subscription (the capped rows), one with only the shell subscription, and a second client holding both. Server egress is the sum of the three. After the turn the second client disconnects and new sockets resubscribe from the cursor it held before the turn, which is the cursor a backgrounded phone would hold. SQL statements are the SQL client's per-statement spans, counted across V2 persistence and the HTTP/WebSocket handlers.",
440
`Scenario: ${TRANSFER_HISTORY_TURN_COUNT} historical turns with ${TRANSFER_HISTORY_TOOLS_PER_TURN} command tools and one retained ${formatBytes(TRANSFER_HISTORY_MCP_RESULT_BYTES)} MCP result each, followed by one measured turn with ${TRANSFER_MEASURED_TOOLS} command tools and a retained ${formatBytes(TRANSFER_MEASURED_MCP_RESULT_BYTES)} MCP result. Synthetic V2 domain events exercise persistence, wire projection, and production HTTP/subscription handlers; this does not measure provider adapter ingestion. Payloads contain no user data.`,
314
- `The long-turn rows open a second thread through the compact bounded snapshot web and mobile use, then load one older page. Its last user turn holds ${LONG_TURN_WORK_ROWS} rows of reasoning, commands, file changes, MCP calls, progress messages, checkpoints, and subagents.`,
315
- `The fleet shell rows read the cold-connect shell after ${SHELL_SETTLED_THREADS} settled threads and ${SHELL_IDLE_SUBAGENTS} idle subagent children are added. Budgeted reads send the opt-ins current clients send; "older client" rows show the same read without the opt-ins added since row budgets.`,
441
+ `The long turn rows read a second thread whose last user turn holds ${LONG_TURN_WORK_ROWS} rows of reasoning, commands, file changes, MCP calls, progress messages, checkpoints, and subagents. While that turn runs, web and mobile load the compact bounded snapshot and then one older page. Once it finishes, its work is deferred: the cold open is read over HTTP as web does and over the socket alone as mobile does, then the first page of the deferred work is loaded as an expanded work group does.`,
442
+ "The resume rows open a new socket from a cursor the client already holds: the first thread after one more ordinary turn, the long-turn thread with nothing new, and the shell from a cursor too old to replay.",
443
+ `The fleet shell rows read the cold-connect shell after ${SHELL_SETTLED_THREADS} settled threads and ${SHELL_IDLE_SUBAGENTS} idle subagent children are added, then subscribe from that snapshot's cursor. Budgeted reads send the opt-ins current clients send; "older client" rows show the same read without the opt-ins added since row budgets.`,
444
`The stale todo list rows open a third thread the same way: ${STALE_TODO_SETTLED_RUNS} finished runs, then one run still at work with ${STALE_TODO_LIVE_ROWS} rows. Every run left its todo list open and has ${STALE_TODO_SUBAGENTS_PER_RUN} finished subagents, a provider turn and, once finished, a checkpoint. The record counts must not grow with the number of finished runs.`,
445
"",
446
"| Provider | Total thread wire | Budget | Result |",
@@ -391,33 +519,12 @@ export function formatTransferBudgetReport(runs: ReadonlyArray<TransferBudgetRun
519
run.reconnectShell,
520
),
521
infoRow(run.provider, "reconnect", "SQL statements", run.reconnectSqlStatements, String),
394
- ...timelineRows(run.provider, "long-turn cold open", run.longTurnColdOpen, LONG_TURN_BUDGET),
395
- ...legacyRows(run.provider, "long-turn cold open", run.longTurnColdOpenLegacy),
396
- ...timelineRows(
397
- run.provider,
398
- "long-turn history page",
399
- run.longTurnHistoryPage,
400
- LONG_TURN_BUDGET,
401
- ),
402
- ...legacyRows(run.provider, "long-turn history page", run.longTurnHistoryPageLegacy),
403
- ...timelineRows(run.provider, "fleet shell snapshot", run.fleetShell, FLEET_SHELL_BUDGET),
404
- ...legacyRows(run.provider, "fleet shell snapshot", run.fleetShellLegacy),
405
- ...timelineRows(
406
- run.provider,
407
- "stale todo list cold open",
408
- run.staleTodoColdOpen,
409
- STALE_TODO_BUDGET,
410
- ),
411
- ...STALE_TODO_RECORD_KINDS.map((kind) =>
412
- row(
413
- run.provider,
414
- "stale todo list cold open",
415
- kind,
416
- run.staleTodoColdOpen.records[kind],
417
- STALE_TODO_BUDGET.records[kind],
418
- String,
419
- ),
420
- ),
522
+ ...scenarioRows(run).map(({ phase, metric, unit, observed, maximum }) => {
523
+ const format = unit === "bytes" ? formatBytes : String;
524
+ return maximum === undefined
525
+ ? infoRow(run.provider, phase, metric, observed, format)
526
+ : row(run.provider, phase, metric, observed, maximum, format);
527
+ }),
528
);
529
}
530
apps/server/integration/TransferBudgetV2Fixture.integration.ts
+7
-5
@@ -217,10 +217,10 @@ export const LONG_TURN_THREAD_ID = ThreadId.make("transfer-budget-long-turn");
217
export const LONG_TURN_WORK_ROWS = 900;
218
219
/**
220
- * Three short user turns, then one agent turn far longer than a cold-open
221
- * window, with the item kinds a long agent run produces. One batch per turn.
222
- * Run, node, provider and item IDs follow the composite shapes the Claude
223
- * adapter stores, and every assistant message has its record, as real threads do.
220
+ * Three short user turns, then one agent turn far longer than a cold-open window, with the item
221
+ * kinds a long agent run produces. One batch per turn; the last batch only finishes the long turn.
222
+ * Run, node, provider and item IDs follow the composite shapes the Claude adapter stores, and
223
+ * every assistant message has its record, as real threads do.
224
*/
225
export function longTurnThreadBatches(
226
provider: ProviderDriverKind,
@@ -434,12 +434,14 @@ export function longTurnThreadBatches(
434
return events;
435
};
436
437
+ const longTurn = turn(3, LONG_TURN_WORK_ROWS);
438
return [
439
[threadCreated(provider, threadId)],
440
turn(0, 4),
441
turn(1, 4),
442
turn(2, 4),
442
- turn(3, LONG_TURN_WORK_ROWS),
443
+ longTurn.slice(0, -1),
444
+ longTurn.slice(-1),
445
];
446
}
447
apps/server/integration/transferBudgetV2.integration.test.ts
+171
-43
@@ -23,8 +23,14 @@ import {
23
OrchestrationV2GetThreadProjectionError,
24
OrchestrationV2GetShellSnapshotError,
25
ProviderDriverKind,
26
+ type EnvironmentOrchestrationShellSnapshotQuery,
27
+ type EnvironmentOrchestrationThreadBoundedSnapshotQuery,
28
+ type EnvironmentOrchestrationThreadHistoryQuery,
29
+ type OrchestrationV2SubscribeShellInput,
30
+ type OrchestrationV2SubscribeThreadInput,
31
type OrchestrationV2ThreadStreamItem,
32
type OrchestrationV2ShellStreamItem,
33
+ type ThreadId,
34
} from "@t3tools/contracts";
35
import * as Context from "effect/Context";
36
import * as Config from "effect/Config";
@@ -63,6 +69,7 @@ import { makeSqlStatementCounter } from "./SqlStatementCounter.integration.ts";
69
import {
70
formatTransferBudgetReport,
71
formatTransferBudgetResult,
72
+ formatTransferBudgetScenarios,
73
transferBudgetViolations,
74
type TransferBudgetRun,
75
} from "./TransferBudgetReport.integration.ts";
@@ -83,11 +90,41 @@ import {
90
turnEvents,
91
} from "./TransferBudgetV2Fixture.integration.ts";
92
86
-// The opt-ins current web and mobile clients send; keep in sync with client-runtime's
87
-// boundedThreadSnapshotHttp.ts, threadHistoryHttp.ts, shellSnapshotHttp.ts, threads.ts and shell.ts.
88
-const CLIENT_BOUNDED_QUERY = "compactTurnItems=1&dedupeBodies=1&compactIds=1";
89
-const CLIENT_HISTORY_QUERY = "compactIds=1";
90
-const CLIENT_SHELL_QUERY = "omitIdleSubagents=1&omitOlderSettled=1";
93
+// The opt-ins current web and mobile clients send, as client-runtime's boundedThreadSnapshotHttp.ts,
94
+// threadHistoryHttp.ts, shellSnapshotHttp.ts, threads.ts and shell.ts send them. Each is `Required`,
95
+// so an opt-in added to a contract fails typecheck here until this test sends it.
96
+const CLIENT_BOUNDED_OPT_INS = {
97
+ compactTurnItems: "1",
98
+ dedupeBodies: "1",
99
+ compactIds: "1",
100
+ deferWork: "1",
101
+} satisfies Required<typeof EnvironmentOrchestrationThreadBoundedSnapshotQuery.Type>;
102
+const CLIENT_HISTORY_OPT_INS = { compactIds: "1", deferWork: "1" } satisfies Required<
103
+ Omit<typeof EnvironmentOrchestrationThreadHistoryQuery.Type, "cursor" | "throughEntryId" | "view">
104
+>;
105
+const CLIENT_SHELL_OPT_INS = {
106
+ omitIdleSubagents: "1",
107
+ omitOlderSettled: "1",
108
+} satisfies Required<typeof EnvironmentOrchestrationShellSnapshotQuery.Type>;
109
+const CLIENT_THREAD_SUBSCRIPTION = {
110
+ requestCompletionMarker: true,
111
+ acceptBoundedSnapshot: true,
112
+ acceptDeferredWork: true,
113
+ acceptCompactTurnItems: true,
114
+ acceptCompactIds: true,
115
+ acceptDedupedBodies: true,
116
+} satisfies Required<Omit<OrchestrationV2SubscribeThreadInput, "threadId" | "afterSequence">>;
117
+const CLIENT_SHELL_SUBSCRIPTION = {
118
+ requestCompletionMarker: true,
119
+ omitIdleSubagents: true,
120
+ omitOlderSettled: true,
121
+} satisfies Required<
122
+ Omit<OrchestrationV2SubscribeShellInput, "afterSequence" | "httpSnapshotFallback">
123
+>;
124
+const query = (optIns: Readonly<Record<string, string>>) => new URLSearchParams(optIns).toString();
125
+const CLIENT_BOUNDED_QUERY = query(CLIENT_BOUNDED_OPT_INS);
126
+const CLIENT_HISTORY_QUERY = query(CLIENT_HISTORY_OPT_INS);
127
+const CLIENT_SHELL_QUERY = query(CLIENT_SHELL_OPT_INS);
128
129
const decodeThreadSnapshot = Schema.decodeUnknownEffect(
130
Schema.fromJsonString(Schema.toCodecJson(OrchestrationV2ThreadDetailSnapshot)),
@@ -224,30 +261,36 @@ const collectUntil = Effect.fn("transfer.collectUntil")(function* <T extends Str
261
if (done(item)) return items;
262
}
263
});
264
+type Subscription =
265
+ | ({ readonly thread: ThreadId } & Pick<OrchestrationV2SubscribeThreadInput, "afterSequence">)
266
+ | ({ readonly shell: true } & Pick<
267
+ OrchestrationV2SubscribeShellInput,
268
+ "afterSequence" | "httpSnapshotFallback"
269
+ >);
270
+function subscriptionStream(
271
+ client: MeasuredWsClient,
272
+ subscription: Subscription,
273
+): Stream.Stream<StreamItem, unknown> {
274
+ if ("thread" in subscription) {
275
+ const { thread, ...resume } = subscription;
276
+ return client.client[ORCHESTRATION_V2_WS_METHODS.subscribeThread]({
277
+ ...CLIENT_THREAD_SUBSCRIPTION,
278
+ ...resume,
279
+ threadId: thread,
280
+ });
281
+ }
282
+ const { shell: _, ...resume } = subscription;
283
+ return client.client[ORCHESTRATION_V2_WS_METHODS.subscribeShell]({
284
+ ...CLIENT_SHELL_SUBSCRIPTION,
285
+ ...resume,
286
+ });
287
+}
288
const subscribe = Effect.fn("transfer.subscribe")(function* (
289
client: MeasuredWsClient,
229
- kind: "thread" | "shell",
230
- afterSequence: number,
290
+ subscription: Subscription,
291
) {
292
const queue = yield* Queue.unbounded<StreamItem>();
233
- const stream: Stream.Stream<StreamItem, unknown> =
234
- kind === "thread"
235
- ? client.client[ORCHESTRATION_V2_WS_METHODS.subscribeThread]({
236
- threadId: THREAD_ID,
237
- afterSequence,
238
- requestCompletionMarker: true,
239
- acceptBoundedSnapshot: true,
240
- acceptCompactTurnItems: true,
241
- acceptCompactIds: true,
242
- acceptDedupedBodies: true,
243
- })
244
- : client.client[ORCHESTRATION_V2_WS_METHODS.subscribeShell]({
245
- afterSequence,
246
- requestCompletionMarker: true,
247
- omitIdleSubagents: true,
248
- omitOlderSettled: true,
249
- });
250
- yield* stream.pipe(
293
+ yield* subscriptionStream(client, subscription).pipe(
294
Stream.runForEach((item) => Queue.offer(queue, item)),
295
Scope.provide(client.scope),
296
Effect.forkIn(client.scope),
@@ -324,10 +367,11 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
367
const shell = yield* openMeasuredWsClient({ url, cookie: "" });
368
const second = yield* openMeasuredWsClient({ url, cookie: "" });
369
assert.include(thread.recorder.negotiatedExtensions(), "permessage-deflate");
327
- const tq = yield* subscribe(thread, "thread", sequence);
328
- const sq = yield* subscribe(shell, "shell", sequence);
329
- const ttq = yield* subscribe(second, "thread", sequence);
330
- const ssq = yield* subscribe(second, "shell", sequence);
370
+ const fromSnapshot = { afterSequence: sequence };
371
+ const tq = yield* subscribe(thread, { thread: THREAD_ID, ...fromSnapshot });
372
+ const sq = yield* subscribe(shell, { shell: true, ...fromSnapshot });
373
+ const ttq = yield* subscribe(second, { thread: THREAD_ID, ...fromSnapshot });
374
+ const ssq = yield* subscribe(second, { shell: true, ...fromSnapshot });
375
for (const queue of [tq, sq, ttq, ssq])
376
assert.equal(yield* synchronized(queue), "replay");
377
const threadBefore = thread.recorder.totals();
@@ -376,8 +420,8 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
420
const reconnectSqlStart = counter.count();
421
const resumedThread = yield* openMeasuredWsClient({ url, cookie: "" });
422
const resumedShell = yield* openMeasuredWsClient({ url, cookie: "" });
379
- const rq = yield* subscribe(resumedThread, "thread", sequence);
380
- const rsq = yield* subscribe(resumedShell, "shell", sequence);
423
+ const rq = yield* subscribe(resumedThread, { thread: THREAD_ID, ...fromSnapshot });
424
+ const rsq = yield* subscribe(resumedShell, { shell: true, ...fromSnapshot });
425
const threadMode = yield* synchronized(rq);
426
const shellMode = yield* synchronized(rsq);
427
const reconnectThread = { ...resumedThread.recorder.totals(), mode: threadMode };
@@ -385,7 +429,6 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
429
const reconnectSqlStatements = counter.count() - reconnectSqlStart;
430
assert.equal(threadMode, "snapshot");
431
388
- for (const events of longTurnThreadBatches(provider)) yield* sink.write({ events });
432
const get = (path: string) =>
433
measureHttpGet({
434
url: `${base}${path}`,
@@ -398,8 +441,9 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
441
Effect.gen(function* () {
442
const http = yield* get(`${longTurnPath}/bounded?${query}`);
443
const snapshot = yield* decodeBoundedSnapshot(decoded(http));
401
- const rows = snapshot.projection.visibleTurnItems.length;
402
- return { http, rows, cursor: snapshot.historyCursor };
444
+ const rows = snapshot.projection.visibleTurnItems;
445
+ const workAnchor = rows.find((row) => row.item.deferredWork !== undefined);
446
+ return { http, rows: rows.length, cursor: snapshot.historyCursor, workAnchor };
447
});
448
const readHistoryPage = (cursor: string, query: string) =>
449
Effect.gen(function* () {
@@ -413,15 +457,75 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
457
const readShell = (query: string) =>
458
Effect.gen(function* () {
459
const http = yield* get(`/api/orchestration/shell?${query}`);
416
- return { http, rows: (yield* decodeShellSnapshot(decoded(http))).threads.length };
460
+ const { threads, snapshotSequence } = yield* decodeShellSnapshot(decoded(http));
461
+ return { http, rows: threads.length, snapshotSequence };
462
+ });
463
+ // A new socket, read until the subscription caught up or the server ended it.
464
+ const open = (subscription: Subscription) =>
465
+ Effect.gen(function* () {
466
+ const socket = yield* openMeasuredWsClient({ url, cookie: "" });
467
+ const items = yield* subscriptionStream(socket, subscription).pipe(
468
+ Stream.takeUntil((item) => item.kind === "synchronized"),
469
+ Stream.runCollect,
470
+ Scope.provide(socket.scope),
471
+ Effect.orDie,
472
+ );
473
+ const totals = socket.recorder.totals();
474
+ yield* socket.close;
475
+ return { kinds: items.map((item) => item.kind), items, totals };
476
});
477
419
- const { cursor: _, ...longTurnColdOpenLegacy } =
420
- yield* readColdOpen("compactTurnItems=1");
421
- const { cursor, ...longTurnColdOpen } = yield* readColdOpen(CLIENT_BOUNDED_QUERY);
478
+ // A phone that comes back after one ordinary turn replays it, without tool bodies.
479
+ const shortTurn = yield* sink.write({
480
+ events: turnEvents(provider, TRANSFER_HISTORY_TURN_COUNT + 1, false),
481
+ });
482
+ const shortTurnResume = yield* open({ thread: THREAD_ID, afterSequence: last.sequence });
483
+ assert.notInclude(shortTurnResume.kinds, "snapshot");
484
+ // The byte cap below also passes when the resume skips the turn.
485
+ assert.isTrue(
486
+ shortTurnResume.items.some(
487
+ (item) => "sequence" in item && item.sequence === shortTurn.at(-1)!.sequence,
488
+ ),
489
+ );
490
+
491
+ // A turn that is still running cannot defer its work, so it loads in row-budget windows.
492
+ const [finishLongTurn, ...longTurnHistory] = longTurnThreadBatches(provider).toReversed();
493
+ for (const events of longTurnHistory.toReversed()) yield* sink.write({ events });
494
+ const longTurnRunningColdOpenLegacy = yield* readColdOpen("compactTurnItems=1");
495
+ const longTurnRunningColdOpen = yield* readColdOpen(CLIENT_BOUNDED_QUERY);
496
+ const { cursor } = longTurnRunningColdOpen;
497
if (cursor === null) return yield* Effect.die("Expected older long-turn history");
423
- const longTurnHistoryPageLegacy = yield* readHistoryPage(cursor, "");
424
- const longTurnHistoryPage = yield* readHistoryPage(cursor, CLIENT_HISTORY_QUERY);
498
+ const longTurnRunningHistoryPageLegacy = yield* readHistoryPage(cursor, "");
499
+ const longTurnRunningHistoryPage = yield* readHistoryPage(cursor, CLIENT_HISTORY_QUERY);
500
+
501
+ // Once it finishes, its work stays on the server until a group is expanded.
502
+ yield* sink.write({ events: finishLongTurn! });
503
+ const longTurnColdOpen = yield* readColdOpen(CLIENT_BOUNDED_QUERY);
504
+ const { workAnchor } = longTurnColdOpen;
505
+ if (workAnchor?.item.deferredWork === undefined)
506
+ return yield* Effect.die("Expected the finished long turn to defer its work");
507
+ const longTurnWorkPage = yield* readHistoryPage(
508
+ workAnchor.item.deferredWork.cursor,
509
+ `view=work&throughEntryId=${encodeURIComponent(workAnchor.sourceItemId)}&compactIds=1`,
510
+ );
511
+ // Mobile opens a thread over the socket alone.
512
+ const socketOpen = yield* open({ thread: LONG_TURN_THREAD_ID });
513
+ const socketSnapshot = socketOpen.items.find(
514
+ (item) => item.kind === "snapshot" && "projection" in item,
515
+ );
516
+ if (socketSnapshot?.kind !== "snapshot" || !("projection" in socketSnapshot))
517
+ return yield* Effect.die("Expected a bounded socket snapshot");
518
+ const longTurnSocketColdOpen = {
519
+ ...socketOpen,
520
+ rows: socketSnapshot.projection.visibleTurnItems.length,
521
+ };
522
+ // The row cap below also passes when the socket ignores the deferral and sends a window.
523
+ assert.equal(longTurnSocketColdOpen.rows, longTurnColdOpen.rows);
524
+ const idleThreadResume = yield* open({
525
+ thread: LONG_TURN_THREAD_ID,
526
+ afterSequence: socketSnapshot.snapshotSequence,
527
+ });
528
+ assert.deepEqual(idleThreadResume.kinds, ["synchronized"]);
529
530
// The transfer thread plus the long-turn thread.
531
const earlierThreads = decodedShell.threads.length + 1;
@@ -432,6 +536,19 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
536
fleetShellLegacy.rows,
537
earlierThreads + SHELL_SETTLED_THREADS + SHELL_IDLE_SUBAGENTS,
538
);
539
+ // A cold connect subscribes from the snapshot it just loaded.
540
+ const fleetShellSubscribe = yield* open({
541
+ shell: true,
542
+ afterSequence: fleetShell.snapshotSequence,
543
+ });
544
+ assert.notInclude(fleetShellSubscribe.kinds, "snapshot");
545
+ // A cursor too old to replay is told to reload over HTTP, not sent a socket snapshot.
546
+ const staleShellResume = yield* open({
547
+ shell: true,
548
+ afterSequence: sequence,
549
+ httpSnapshotFallback: true,
550
+ });
551
+ assert.deepEqual(staleShellResume.kinds, ["snapshot-required"]);
552
553
for (const events of staleTodoListThreadBatches(provider)) yield* sink.write({ events });
554
const staleTodoHttp = yield* get(
@@ -472,12 +589,19 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
589
reconnectThread,
590
reconnectShell,
591
reconnectSqlStatements,
475
- longTurnColdOpenLegacy,
592
+ shortTurnResume,
593
+ longTurnRunningColdOpenLegacy,
594
+ longTurnRunningColdOpen,
595
+ longTurnRunningHistoryPageLegacy,
596
+ longTurnRunningHistoryPage,
597
longTurnColdOpen,
477
- longTurnHistoryPageLegacy,
478
- longTurnHistoryPage,
598
+ longTurnWorkPage,
599
+ longTurnSocketColdOpen,
600
+ idleThreadResume,
601
fleetShellLegacy,
602
fleetShell,
603
+ fleetShellSubscribe,
604
+ staleShellResume,
605
staleTodoColdOpen,
606
} satisfies TransferBudgetRun;
607
}).pipe(
@@ -492,10 +616,14 @@ it.live("reports V2 thread HTTP and WebSocket transfer budgets", () =>
616
const outputPaths = yield* Config.all({
617
report: Config.String("T3CODE_TRANSFER_BUDGET_REPORT_PATH").pipe(Config.withDefault("")),
618
result: Config.String("T3CODE_TRANSFER_BUDGET_RESULT_PATH").pipe(Config.withDefault("")),
619
+ scenarios: Config.String("T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH").pipe(
620
+ Config.withDefault(""),
621
+ ),
622
});
623
for (const [path, contents] of [
624
[outputPaths.report, report],
625
[outputPaths.result, formatTransferBudgetResult(runs)],
626
+ [outputPaths.scenarios, formatTransferBudgetScenarios(runs)],
627
])
628
if (path && contents) yield* fs.writeFileString(path, contents);
629
yield* Effect.log(report);
docs/internals/performance-regressions.md
+18
-8
@@ -19,7 +19,8 @@ The v2 checks pin these invariants:
19
large work groups until expansion. Messages and items outside those groups remain in normal
20
history paging. Required control records and
21
the at-least-one-row rule can exceed the byte target; it is not a hard cap.
22
-- The bounded snapshot does not retain the full duplicate conversation-message table.
22
+- The bounded snapshot does not retain the full duplicate conversation-message table. Messages of
23
+ a run that is still in progress are the exception: all of them accompany the window.
24
- Older activity is fetched in bounded HTTP pages and merged without disturbing the live scroll
25
window.
26
- Clients with confirmed history paging opt into socket snapshot fallbacks that use the same
@@ -68,13 +69,22 @@ These are neither compressed WebSocket captures nor measurements of server RSS,
69
fixtures exclude unrelated settings calls and subscription events. Payload budgets belong in
70
these tests so regressions fail locally. The transfer budget integration test
71
(`apps/server/integration/transferBudgetV2.integration.test.ts`) sends the opt-ins current clients
71
-send and caps the decoded bytes and rows of the bounded snapshot and one older page for a 900-row
72
-agent turn, and of the cold-connect shell for a sidebar of settled threads and idle subagents. It
73
-also caps the bytes and the node, subagent, plan, provider turn and checkpoint counts of the
74
-bounded snapshot for a thread of 40 finished runs behind one working run, so finished runs cannot
75
-ride along with a window. The working run has a ready context handoff, so its history cannot ride
76
-along either. When clients gain an opt-in, add it to the test's client queries and give a fixture
77
-the data it saves.
72
+send. Its opt-in objects are typed `Required` against the contracts, so an opt-in added to a
73
+contract fails typecheck until the test sends it; give a fixture the data the opt-in saves. The
74
+test caps decoded bytes and rows for a 900-row agent turn in both states: while the turn runs (the
75
+bounded snapshot and one older page) and after it finishes (the deferred cold open over HTTP and
76
+over the socket alone, and the first page of its deferred work). It caps the socket bytes of three
77
+resumes that must not send a snapshot: a thread after one ordinary turn, an idle thread, and a
78
+shell cursor too old to replay. It caps the cold-connect shell for a sidebar of settled threads and
79
+idle subagents, and the subscription a client opens from that snapshot. It also caps the bytes and
80
+the node, subagent, plan, provider turn and checkpoint counts of the bounded snapshot for a thread
81
+of 40 finished runs behind one working run, so finished runs cannot ride along with a window. The
82
+working run has a ready context handoff, so its history cannot ride along either.
83
+
84
+CI prints every row in the job summary. The pull request comment compares the budgeted rows with
85
+`main`, lists the rows that changed or failed, and names each ceiling a pull request raises. A row
86
+added to `scenarioRows` in `TransferBudgetReport.integration.ts` reaches the pass/fail check, the
87
+summary and the comment.
88
89
The server config has its own budget in `apps/server/src/ws.test.ts`. A client that asks for
90
catalog digests and status deltas gets the provider list of its first config in under 1 KiB, for a
packages/contracts/src/environmentHttp.ts
+3
-3
@@ -682,19 +682,19 @@ const EnvironmentOrchestrationThreadSnapshotParams = Schema.Struct({
682
threadId: ThreadId,
683
});
684
685
-const EnvironmentOrchestrationShellSnapshotQuery = Schema.Struct({
685
+export const EnvironmentOrchestrationShellSnapshotQuery = Schema.Struct({
686
omitIdleSubagents: Schema.optionalKey(Schema.String),
687
omitOlderSettled: Schema.optionalKey(Schema.String),
688
});
689
690
-const EnvironmentOrchestrationThreadBoundedSnapshotQuery = Schema.Struct({
690
+export const EnvironmentOrchestrationThreadBoundedSnapshotQuery = Schema.Struct({
691
deferWork: Schema.optionalKey(Schema.String),
692
compactTurnItems: Schema.optionalKey(Schema.String),
693
dedupeBodies: Schema.optionalKey(Schema.String),
694
compactIds: Schema.optionalKey(Schema.String),
695
});
696
697
-const EnvironmentOrchestrationThreadHistoryQuery = Schema.Struct({
697
+export const EnvironmentOrchestrationThreadHistoryQuery = Schema.Struct({
698
cursor: TrimmedNonEmptyString,
699
throughEntryId: Schema.optional(TrimmedNonEmptyString),
700
view: Schema.optional(Schema.Literals(["conversation", "activity", "work"])),
packages/shared/src/testing/providerEnv.ts
+1
@@ -8,6 +8,7 @@ const KEPT_ENV = new Set([
8
// CI sets these so the transfer budget integration test writes its report for the job summary.
9
"T3CODE_TRANSFER_BUDGET_REPORT_PATH",
10
"T3CODE_TRANSFER_BUDGET_RESULT_PATH",
11
+ "T3CODE_TRANSFER_BUDGET_SCENARIOS_PATH",
12
// Opt-in: points the HtmlRender tests at a headless browser, which they skip without.
13
"T3CODE_TEST_HEADLESS_SHELL",
14
]);