The best way to control Claude Code, Codex and more from any computer or phone
ci: flag pull requests that grow the test suite, and browser tests that need no browser (#34559)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Nov 11, 2026 at 14:57 UTC
3fd7b75e25eaf031feeb30062195109704bf2455
7 files changed
+210
-2
.github/scripts/test-delta.cjs
new
+78
@@ -0,0 +1,78 @@
1
+// Counts the test cases a pull request adds and removes, from a zero-context diff, so PR Size can
2
+// say when a change grows the suite. Counting declarations keeps the number deterministic, unlike
3
+// CI timings, which move with runner load.
4
+const MARKER = "<!-- test-delta -->";
5
+const NET_CASES_TO_REPORT = 20;
6
+const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$/;
7
+// `<` admits type arguments, as in `it.each<Row>(...)`, which may span lines.
8
+const TEST_CASE = /^\s*(?:it|test)(?:\.\w+)*\s*[(<]/;
9
+const isBrowserTest = (path) => path.includes(".browser.test.");
10
+
11
+function kindOf(path) {
12
+ if (isBrowserTest(path)) return "browser";
13
+ if (path.startsWith("apps/server/") || path.includes(".integration.")) return "server";
14
+ return "unit";
15
+}
16
+
17
+/** @param {string} diff output of `git diff -U0 base...head` */
18
+function countTestDelta(diff) {
19
+ const kinds = { browser: 0, server: 0, unit: 0 };
20
+ const newBrowserFiles = [];
21
+ let path = null;
22
+ // A browser test file is new when it is added, or renamed from a file that was not one.
23
+ let file = null;
24
+ const flush = () => {
25
+ if (file?.isNew && isBrowserTest(file.target)) newBrowserFiles.push(file.target);
26
+ };
27
+ for (const line of diff.split("\n")) {
28
+ if (line.startsWith("diff --git ")) {
29
+ flush();
30
+ path = null;
31
+ file = { target: line.replace(/^diff --git a\/\S+ b\//, ""), isNew: false };
32
+ continue;
33
+ }
34
+ if (line.startsWith("new file mode") && file) file.isNew = true;
35
+ if (line.startsWith("rename from ") && file) file.isNew = !isBrowserTest(line.slice(12));
36
+ if (line.startsWith("--- ")) {
37
+ const source = line.slice(4).replace(/^a\//, "");
38
+ path = TEST_FILE.test(source) ? source : null;
39
+ continue;
40
+ }
41
+ if (line.startsWith("+++ ")) {
42
+ const target = line.slice(4).replace(/^b\//, "");
43
+ // A deleted file's target is /dev/null; keep its old path to count the removed cases.
44
+ if (target !== "/dev/null") path = TEST_FILE.test(target) ? target : null;
45
+ continue;
46
+ }
47
+ if (line.startsWith("new file mode") || !path) continue;
48
+ const sign = line[0] === "+" ? 1 : line[0] === "-" ? -1 : 0;
49
+ if (sign !== 0 && TEST_CASE.test(line.slice(1))) kinds[kindOf(path)] += sign;
50
+ }
51
+ flush();
52
+ return { kinds, net: kinds.browser + kinds.server + kinds.unit, newBrowserFiles };
53
+}
54
+
55
+/** Returns the comment body, or null when the change does not grow the suite enough to mention. */
56
+function render(delta) {
57
+ if (delta.net < NET_CASES_TO_REPORT && delta.newBrowserFiles.length === 0) return null;
58
+ const signed = (n) => (n > 0 ? `+${n}` : String(n));
59
+ const lines = [
60
+ MARKER,
61
+ `This PR changes the test suite by **${signed(delta.net)} test cases** ` +
62
+ `(browser ${signed(delta.kinds.browser)}, unit ${signed(delta.kinds.unit)}, server ${signed(delta.kinds.server)}).`,
63
+ ];
64
+ if (delta.newBrowserFiles.length > 0) {
65
+ lines.push(
66
+ "",
67
+ `New browser test files: ${delta.newBrowserFiles.map((file) => `\`${file}\``).join(", ")}.`,
68
+ "Each browser test file costs CI seconds of module loading before its first test runs; a unit test costs milliseconds.",
69
+ );
70
+ }
71
+ lines.push(
72
+ "",
73
+ "Keep tests that would fail for a real reason, and test logic below the UI where it can be. See [Testing](https://github.com/pingdotgg/t3code/blob/main/docs/internals/testing.md).",
74
+ );
75
+ return lines.join("\n");
76
+}
77
+
78
+module.exports = { MARKER, countTestDelta, render };
.github/scripts/test-delta.test.cjs
new
+59
@@ -0,0 +1,59 @@
1
+const assert = require("node:assert/strict");
2
+const test = require("node:test");
3
+
4
+const { countTestDelta, render } = require("./test-delta.cjs");
5
+
6
+const diff = `diff --git a/apps/web/src/Panel.browser.test.tsx b/apps/web/src/Panel.browser.test.tsx
7
+new file mode 100644
8
+--- /dev/null
9
++++ b/apps/web/src/Panel.browser.test.tsx
10
+@@ -0,0 +1,4 @@
11
++it("opens", async () => {});
12
++ it.each<{ n: number }>([{ n: 1 }])("renders $n", async () => {});
13
++// it("is a comment, still counted only when it starts the line")
14
+diff --git a/apps/server/src/thing.test.ts b/apps/server/src/thing.test.ts
15
+--- a/apps/server/src/thing.test.ts
16
++++ b/apps/server/src/thing.test.ts
17
+@@ -3,2 +3,1 @@
18
+-it.effect("old", () => {});
19
+-test("older", () => {});
20
++ it.effect("new", () => {});
21
+diff --git a/packages/shared/src/gone.test.ts b/packages/shared/src/gone.test.ts
22
+deleted file mode 100644
23
+--- a/packages/shared/src/gone.test.ts
24
++++ /dev/null
25
+@@ -1,1 +0,0 @@
26
+-it("goes", () => {});
27
+diff --git a/apps/web/src/Menu.test.tsx b/apps/web/src/Menu.browser.test.tsx
28
+similarity index 100%
29
+rename from apps/web/src/Menu.test.tsx
30
+rename to apps/web/src/Menu.browser.test.tsx
31
+diff --git a/apps/web/src/Old.browser.test.tsx b/apps/web/src/New.browser.test.tsx
32
+similarity index 100%
33
+rename from apps/web/src/Old.browser.test.tsx
34
+rename to apps/web/src/New.browser.test.tsx
35
+diff --git a/apps/web/src/Panel.tsx b/apps/web/src/Panel.tsx
36
+--- a/apps/web/src/Panel.tsx
37
++++ b/apps/web/src/Panel.tsx
38
+@@ -1 +1 @@
39
++it("is not a test file", () => {});
40
+`;
41
+
42
+test("counts cases by kind, including deleted files, and finds new or renamed browser files", () => {
43
+ assert.deepEqual(countTestDelta(diff), {
44
+ kinds: { browser: 2, server: -1, unit: -1 },
45
+ net: 0,
46
+ newBrowserFiles: ["apps/web/src/Panel.browser.test.tsx", "apps/web/src/Menu.browser.test.tsx"],
47
+ });
48
+});
49
+
50
+test("reports only growth worth a reviewer's attention", () => {
51
+ const quiet = { kinds: { browser: 0, server: 3, unit: 5 }, net: 8, newBrowserFiles: [] };
52
+ assert.equal(render(quiet), null);
53
+ const growth = { kinds: { browser: 4, server: 10, unit: 12 }, net: 26, newBrowserFiles: [] };
54
+ assert.match(render(growth), /\*\*\+26 test cases\*\* \(browser \+4, unit \+12, server \+10\)/);
55
+ assert.match(
56
+ render(countTestDelta(diff)),
57
+ /New browser test files: `apps\/web\/src\/Panel\.browser\.test\.tsx`/,
58
+ );
59
+});
.github/workflows/ci.yml
+1
-1
@@ -326,7 +326,7 @@ jobs:
326
run: python3 -B .github/scripts/stage-preview-bundle.test.py
327
328
- name: Test release checks
329
- run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs
329
+ run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs .github/scripts/test-delta.test.cjs
330
331
- name: Finish installing browser secret helper build libraries
332
timeout-minutes: 5
.github/workflows/pr-size.yml
+52
@@ -293,3 +293,55 @@ jobs:
293
core.info(
294
`PR #${issueNumber}: ${nonTestChangedLines} non-test lines, ${testChangedLines} test lines, ${changedLines} effective lines -> ${nextLabelName} (${classification})`,
295
);
296
+ # Reads the script from the base checkout above; the PR's own commits are only diffed.
297
+ - name: Report test suite growth
298
+ uses: actions/github-script@v8
299
+ with:
300
+ script: |
301
+ const { execFileSync } = require("node:child_process");
302
+ const { MARKER, countTestDelta, render } = require(
303
+ `${process.env.GITHUB_WORKSPACE}/.github/scripts/test-delta.cjs`,
304
+ );
305
+
306
+ const issueNumber = context.payload.pull_request.number;
307
+ const diff = execFileSync(
308
+ "git",
309
+ ["diff", "-U0", `${context.payload.pull_request.base.sha}...refs/remotes/pr-size/${issueNumber}`],
310
+ { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 },
311
+ );
312
+ const delta = countTestDelta(diff);
313
+ const body = render(delta);
314
+ core.info(`PR #${issueNumber}: test cases ${JSON.stringify(delta.kinds)}, net ${delta.net}`);
315
+
316
+ const comments = await github.paginate(github.rest.issues.listComments, {
317
+ owner: context.repo.owner,
318
+ repo: context.repo.repo,
319
+ issue_number: issueNumber,
320
+ per_page: 100,
321
+ });
322
+ const existing = comments.find(
323
+ (comment) => comment.user?.type === "Bot" && comment.body?.startsWith(MARKER),
324
+ );
325
+ if (body && existing) {
326
+ if (existing.body !== body) {
327
+ await github.rest.issues.updateComment({
328
+ owner: context.repo.owner,
329
+ repo: context.repo.repo,
330
+ comment_id: existing.id,
331
+ body,
332
+ });
333
+ }
334
+ } else if (body) {
335
+ await github.rest.issues.createComment({
336
+ owner: context.repo.owner,
337
+ repo: context.repo.repo,
338
+ issue_number: issueNumber,
339
+ body,
340
+ });
341
+ } else if (existing) {
342
+ await github.rest.issues.deleteComment({
343
+ owner: context.repo.owner,
344
+ repo: context.repo.repo,
345
+ comment_id: existing.id,
346
+ });
347
+ }
AGENTS.md
+1
@@ -105,6 +105,7 @@ An empty database is a bad test. Seed your worktree's `.t3` with a copy of real
105
- Test behavior that can break, such as orchestration and event-sourcing, provider protocol translation, auth and permission guards, persistence and migrations, external-format parsing, race fixes, and the logic a bug fix changed. Leave out tests for what types already guarantee, trivial logic, copy text, smoke runs, mock-call mirrors, callback wiring, and props rendered to static markup.
106
- Extend before you add. When an existing test or table covers the behavior, add a row or an assertion there. Write a new `it` only when no case fits.
107
- Test at one layer: the lowest one that fails on `main`. Do not repeat the case in a wrapper test, at each call site, or in web and mobile copies of the same logic.
108
+- Write a browser test only for behavior that needs a real browser: layout and scrolling, focus and pointer input, real workers, or a component wired to its hooks. Logic a component calls gets a unit test beside it (`*.logic.test.ts`). A browser test file spends seconds in CI loading its module graph before its first test runs; a unit test takes milliseconds.
109
- [Testing](docs/internals/testing.md) carries these same test rules, plus review guidance, for review bots. Change both together.
110
- **Do not run repo-wide checks.** No `vp check`, no `vp run -r test`, no `vp run -r typecheck` unless I ask. CI owns the full suite.
111
- Backend behavior changes ship with focused tests for that behavior.
docs/internals/testing.md
+2
-1
@@ -7,7 +7,8 @@ Rules for writing tests, and for asking for them in review. AGENTS.md carries th
7
- Test behavior that can break, such as orchestration and event-sourcing, provider protocol translation, auth and permission guards, persistence and migrations, external-format parsing, race fixes, and the logic a bug fix changed. Leave out tests for what types already guarantee, trivial logic, copy text, smoke runs, mock-call mirrors, callback wiring, and props rendered to static markup.
8
- Extend before you add. When an existing test or table covers the behavior, add a row or an assertion there. Write a new `it` only when no case fits.
9
- Test at one layer: the lowest one that fails on `main`. Do not repeat the case in a wrapper test, at each call site, or in web and mobile copies of the same logic.
10
-- In review, ask for a missing test only when it would catch a real regression. Do not ask for one assertion per constant, label, or `switch` case. Flag a new test that breaks the rules above, and name the fix: fold it into the existing case or table, delete it, or keep one layer.
10
+- Write a browser test only for behavior that needs a real browser: layout and scrolling, focus and pointer input, real workers, or a component wired to its hooks. Logic a component calls gets a unit test beside it (`*.logic.test.ts`). A browser test file spends seconds in CI loading its module graph before its first test runs; a unit test takes milliseconds.
11
+- In review, ask for a missing test only when it would catch a real regression. Do not ask for one assertion per constant, label, or `switch` case. Flag a new test that breaks the rules above, and name the fix: fold it into the existing case or table, delete it, keep one layer, or move a browser test whose assertions would pass without rendering into a unit test.
12
- Backend behavior changes ship with focused tests for that behavior.
13
- Proof for a bug fix depends on the bug. Logic, data, state, and protocol bugs get a failing test first (the fewest that fail on `main` for that reason, in an existing case of the module's test file where one fits), then the fix. Purely visual bugs (spacing, colour, alignment, a layout shift) get before/after captures as their proof and no test: do not add UI tests for nitpicks, and do not ask for one in review when a capture already shows it. A server-only fix with no visible surface ships the test plus a short runtime transcript.
14
evals/test-signal/cases.ts
+17
@@ -133,6 +133,23 @@ export const cases: ReadonlyArray<Case> = [
133
'\n it("returns a boolean", () => {\n expect(typeof isMarkdownFileLinkLabel("a.ts", "/repo/a.ts")).toBe("boolean");\n });\n',
134
targets: [{ path: "packages/shared/src/markdownLinks.test.ts", title: "returns a boolean" }],
135
},
136
+ {
137
+ id: "logic-in-browser-test",
138
+ kind: "bad",
139
+ source: "Synthetic: a browser test that never renders, for a pure function in a .logic.ts file",
140
+ type: "synthetic",
141
+ rev: base,
142
+ path: "apps/web/src/components/chat/OpenInPicker.browser.test.tsx",
143
+ after: ' expect(await spoken()).toBe("Opened in VS Code");\n });\n});\n',
144
+ insert:
145
+ '\ndescribe("open in picker visibility", () => {\n it("shows the picker for the primary environment or a remote that can open links", async () => {\n const { shouldShowOpenInPicker } = await import("./OpenInPicker.logic");\n const show = (overrides: Partial<Parameters<typeof shouldShowOpenInPicker>[0]>) =>\n shouldShowOpenInPicker({\n activeProjectName: "app",\n activeThreadEnvironmentId: selected,\n primaryEnvironmentId: primary,\n remoteOpenMode: "local-exec",\n ...overrides,\n });\n expect(show({ activeProjectName: undefined })).toBe(false);\n expect(show({ activeThreadEnvironmentId: primary })).toBe(true);\n expect(show({})).toBe(false);\n expect(show({ remoteOpenMode: "remote-links" })).toBe(true);\n });\n});\n',
146
+ targets: [
147
+ {
148
+ path: "apps/web/src/components/chat/OpenInPicker.browser.test.tsx",
149
+ title: "shows the picker for the primary environment or a remote that can open links",
150
+ },
151
+ ],
152
+ },
153
{
154
id: "deadlock-race-fix",
155
kind: "guard",