@devries-cameron20 / t3code Mirror · read-only / commits / 3fd7b75e25
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 22:15 UTC

ci: flag pull requests that grow the test suite, and browser tests that need no browser (#34559)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Nov 11, 2026 at 14:57 UTC 3fd7b75e25eaf031feeb30062195109704bf2455
7 files changed +210 -2
.github/scripts/test-delta.cjs new
+78
@@ -0,0 +1,78 @@
1 +// Counts the test cases a pull request adds and removes, from a zero-context diff, so PR Size can
2 +// say when a change grows the suite. Counting declarations keeps the number deterministic, unlike
3 +// CI timings, which move with runner load.
4 +const MARKER = "<!-- test-delta -->";
5 +const NET_CASES_TO_REPORT = 20;
6 +const TEST_FILE = /\.(?:test|spec)\.[cm]?[jt]sx?$/;
7 +// `<` admits type arguments, as in `it.each<Row>(...)`, which may span lines.
8 +const TEST_CASE = /^\s*(?:it|test)(?:\.\w+)*\s*[(<]/;
9 +const isBrowserTest = (path) => path.includes(".browser.test.");
10 +
11 +function kindOf(path) {
12 + if (isBrowserTest(path)) return "browser";
13 + if (path.startsWith("apps/server/") || path.includes(".integration.")) return "server";
14 + return "unit";
15 +}
16 +
17 +/** @param {string} diff output of `git diff -U0 base...head` */
18 +function countTestDelta(diff) {
19 + const kinds = { browser: 0, server: 0, unit: 0 };
20 + const newBrowserFiles = [];
21 + let path = null;
22 + // A browser test file is new when it is added, or renamed from a file that was not one.
23 + let file = null;
24 + const flush = () => {
25 + if (file?.isNew && isBrowserTest(file.target)) newBrowserFiles.push(file.target);
26 + };
27 + for (const line of diff.split("\n")) {
28 + if (line.startsWith("diff --git ")) {
29 + flush();
30 + path = null;
31 + file = { target: line.replace(/^diff --git a\/\S+ b\//, ""), isNew: false };
32 + continue;
33 + }
34 + if (line.startsWith("new file mode") && file) file.isNew = true;
35 + if (line.startsWith("rename from ") && file) file.isNew = !isBrowserTest(line.slice(12));
36 + if (line.startsWith("--- ")) {
37 + const source = line.slice(4).replace(/^a\//, "");
38 + path = TEST_FILE.test(source) ? source : null;
39 + continue;
40 + }
41 + if (line.startsWith("+++ ")) {
42 + const target = line.slice(4).replace(/^b\//, "");
43 + // A deleted file's target is /dev/null; keep its old path to count the removed cases.
44 + if (target !== "/dev/null") path = TEST_FILE.test(target) ? target : null;
45 + continue;
46 + }
47 + if (line.startsWith("new file mode") || !path) continue;
48 + const sign = line[0] === "+" ? 1 : line[0] === "-" ? -1 : 0;
49 + if (sign !== 0 && TEST_CASE.test(line.slice(1))) kinds[kindOf(path)] += sign;
50 + }
51 + flush();
52 + return { kinds, net: kinds.browser + kinds.server + kinds.unit, newBrowserFiles };
53 +}
54 +
55 +/** Returns the comment body, or null when the change does not grow the suite enough to mention. */
56 +function render(delta) {
57 + if (delta.net < NET_CASES_TO_REPORT && delta.newBrowserFiles.length === 0) return null;
58 + const signed = (n) => (n > 0 ? `+${n}` : String(n));
59 + const lines = [
60 + MARKER,
61 + `This PR changes the test suite by **${signed(delta.net)} test cases** ` +
62 + `(browser ${signed(delta.kinds.browser)}, unit ${signed(delta.kinds.unit)}, server ${signed(delta.kinds.server)}).`,
63 + ];
64 + if (delta.newBrowserFiles.length > 0) {
65 + lines.push(
66 + "",
67 + `New browser test files: ${delta.newBrowserFiles.map((file) => `\`${file}\``).join(", ")}.`,
68 + "Each browser test file costs CI seconds of module loading before its first test runs; a unit test costs milliseconds.",
69 + );
70 + }
71 + lines.push(
72 + "",
73 + "Keep tests that would fail for a real reason, and test logic below the UI where it can be. See [Testing](https://github.com/pingdotgg/t3code/blob/main/docs/internals/testing.md).",
74 + );
75 + return lines.join("\n");
76 +}
77 +
78 +module.exports = { MARKER, countTestDelta, render };
.github/scripts/test-delta.test.cjs new
+59
@@ -0,0 +1,59 @@
1 +const assert = require("node:assert/strict");
2 +const test = require("node:test");
3 +
4 +const { countTestDelta, render } = require("./test-delta.cjs");
5 +
6 +const diff = `diff --git a/apps/web/src/Panel.browser.test.tsx b/apps/web/src/Panel.browser.test.tsx
7 +new file mode 100644
8 +--- /dev/null
9 ++++ b/apps/web/src/Panel.browser.test.tsx
10 +@@ -0,0 +1,4 @@
11 ++it("opens", async () => {});
12 ++ it.each<{ n: number }>([{ n: 1 }])("renders $n", async () => {});
13 ++// it("is a comment, still counted only when it starts the line")
14 +diff --git a/apps/server/src/thing.test.ts b/apps/server/src/thing.test.ts
15 +--- a/apps/server/src/thing.test.ts
16 ++++ b/apps/server/src/thing.test.ts
17 +@@ -3,2 +3,1 @@
18 +-it.effect("old", () => {});
19 +-test("older", () => {});
20 ++ it.effect("new", () => {});
21 +diff --git a/packages/shared/src/gone.test.ts b/packages/shared/src/gone.test.ts
22 +deleted file mode 100644
23 +--- a/packages/shared/src/gone.test.ts
24 ++++ /dev/null
25 +@@ -1,1 +0,0 @@
26 +-it("goes", () => {});
27 +diff --git a/apps/web/src/Menu.test.tsx b/apps/web/src/Menu.browser.test.tsx
28 +similarity index 100%
29 +rename from apps/web/src/Menu.test.tsx
30 +rename to apps/web/src/Menu.browser.test.tsx
31 +diff --git a/apps/web/src/Old.browser.test.tsx b/apps/web/src/New.browser.test.tsx
32 +similarity index 100%
33 +rename from apps/web/src/Old.browser.test.tsx
34 +rename to apps/web/src/New.browser.test.tsx
35 +diff --git a/apps/web/src/Panel.tsx b/apps/web/src/Panel.tsx
36 +--- a/apps/web/src/Panel.tsx
37 ++++ b/apps/web/src/Panel.tsx
38 +@@ -1 +1 @@
39 ++it("is not a test file", () => {});
40 +`;
41 +
42 +test("counts cases by kind, including deleted files, and finds new or renamed browser files", () => {
43 + assert.deepEqual(countTestDelta(diff), {
44 + kinds: { browser: 2, server: -1, unit: -1 },
45 + net: 0,
46 + newBrowserFiles: ["apps/web/src/Panel.browser.test.tsx", "apps/web/src/Menu.browser.test.tsx"],
47 + });
48 +});
49 +
50 +test("reports only growth worth a reviewer's attention", () => {
51 + const quiet = { kinds: { browser: 0, server: 3, unit: 5 }, net: 8, newBrowserFiles: [] };
52 + assert.equal(render(quiet), null);
53 + const growth = { kinds: { browser: 4, server: 10, unit: 12 }, net: 26, newBrowserFiles: [] };
54 + assert.match(render(growth), /\*\*\+26 test cases\*\* \(browser \+4, unit \+12, server \+10\)/);
55 + assert.match(
56 + render(countTestDelta(diff)),
57 + /New browser test files: `apps\/web\/src\/Panel\.browser\.test\.tsx`/,
58 + );
59 +});
.github/workflows/ci.yml
+1 -1
@@ -326,7 +326,7 @@ jobs:
326 run: python3 -B .github/scripts/stage-preview-bundle.test.py
327
328 - name: Test release checks
329 - run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs
329 + run: node --test .github/scripts/check-nightly-release.test.cjs .github/scripts/vercel-deployment-output.test.cjs .github/scripts/mobile-preview-comment.test.cjs .github/scripts/test-delta.test.cjs
330
331 - name: Finish installing browser secret helper build libraries
332 timeout-minutes: 5
.github/workflows/pr-size.yml
+52
@@ -293,3 +293,55 @@ jobs:
293 core.info(
294 `PR #${issueNumber}: ${nonTestChangedLines} non-test lines, ${testChangedLines} test lines, ${changedLines} effective lines -> ${nextLabelName} (${classification})`,
295 );
296 + # Reads the script from the base checkout above; the PR's own commits are only diffed.
297 + - name: Report test suite growth
298 + uses: actions/github-script@v8
299 + with:
300 + script: |
301 + const { execFileSync } = require("node:child_process");
302 + const { MARKER, countTestDelta, render } = require(
303 + `${process.env.GITHUB_WORKSPACE}/.github/scripts/test-delta.cjs`,
304 + );
305 +
306 + const issueNumber = context.payload.pull_request.number;
307 + const diff = execFileSync(
308 + "git",
309 + ["diff", "-U0", `${context.payload.pull_request.base.sha}...refs/remotes/pr-size/${issueNumber}`],
310 + { encoding: "utf8", maxBuffer: 256 * 1024 * 1024 },
311 + );
312 + const delta = countTestDelta(diff);
313 + const body = render(delta);
314 + core.info(`PR #${issueNumber}: test cases ${JSON.stringify(delta.kinds)}, net ${delta.net}`);
315 +
316 + const comments = await github.paginate(github.rest.issues.listComments, {
317 + owner: context.repo.owner,
318 + repo: context.repo.repo,
319 + issue_number: issueNumber,
320 + per_page: 100,
321 + });
322 + const existing = comments.find(
323 + (comment) => comment.user?.type === "Bot" && comment.body?.startsWith(MARKER),
324 + );
325 + if (body && existing) {
326 + if (existing.body !== body) {
327 + await github.rest.issues.updateComment({
328 + owner: context.repo.owner,
329 + repo: context.repo.repo,
330 + comment_id: existing.id,
331 + body,
332 + });
333 + }
334 + } else if (body) {
335 + await github.rest.issues.createComment({
336 + owner: context.repo.owner,
337 + repo: context.repo.repo,
338 + issue_number: issueNumber,
339 + body,
340 + });
341 + } else if (existing) {
342 + await github.rest.issues.deleteComment({
343 + owner: context.repo.owner,
344 + repo: context.repo.repo,
345 + comment_id: existing.id,
346 + });
347 + }
AGENTS.md
+1
@@ -105,6 +105,7 @@ An empty database is a bad test. Seed your worktree's `.t3` with a copy of real
105 - Test behavior that can break, such as orchestration and event-sourcing, provider protocol translation, auth and permission guards, persistence and migrations, external-format parsing, race fixes, and the logic a bug fix changed. Leave out tests for what types already guarantee, trivial logic, copy text, smoke runs, mock-call mirrors, callback wiring, and props rendered to static markup.
106 - Extend before you add. When an existing test or table covers the behavior, add a row or an assertion there. Write a new `it` only when no case fits.
107 - Test at one layer: the lowest one that fails on `main`. Do not repeat the case in a wrapper test, at each call site, or in web and mobile copies of the same logic.
108 +- Write a browser test only for behavior that needs a real browser: layout and scrolling, focus and pointer input, real workers, or a component wired to its hooks. Logic a component calls gets a unit test beside it (`*.logic.test.ts`). A browser test file spends seconds in CI loading its module graph before its first test runs; a unit test takes milliseconds.
109 - [Testing](docs/internals/testing.md) carries these same test rules, plus review guidance, for review bots. Change both together.
110 - **Do not run repo-wide checks.** No `vp check`, no `vp run -r test`, no `vp run -r typecheck` unless I ask. CI owns the full suite.
111 - Backend behavior changes ship with focused tests for that behavior.
docs/internals/testing.md
+2 -1
@@ -7,7 +7,8 @@ Rules for writing tests, and for asking for them in review. AGENTS.md carries th
7 - Test behavior that can break, such as orchestration and event-sourcing, provider protocol translation, auth and permission guards, persistence and migrations, external-format parsing, race fixes, and the logic a bug fix changed. Leave out tests for what types already guarantee, trivial logic, copy text, smoke runs, mock-call mirrors, callback wiring, and props rendered to static markup.
8 - Extend before you add. When an existing test or table covers the behavior, add a row or an assertion there. Write a new `it` only when no case fits.
9 - Test at one layer: the lowest one that fails on `main`. Do not repeat the case in a wrapper test, at each call site, or in web and mobile copies of the same logic.
10 -- In review, ask for a missing test only when it would catch a real regression. Do not ask for one assertion per constant, label, or `switch` case. Flag a new test that breaks the rules above, and name the fix: fold it into the existing case or table, delete it, or keep one layer.
10 +- Write a browser test only for behavior that needs a real browser: layout and scrolling, focus and pointer input, real workers, or a component wired to its hooks. Logic a component calls gets a unit test beside it (`*.logic.test.ts`). A browser test file spends seconds in CI loading its module graph before its first test runs; a unit test takes milliseconds.
11 +- In review, ask for a missing test only when it would catch a real regression. Do not ask for one assertion per constant, label, or `switch` case. Flag a new test that breaks the rules above, and name the fix: fold it into the existing case or table, delete it, keep one layer, or move a browser test whose assertions would pass without rendering into a unit test.
12 - Backend behavior changes ship with focused tests for that behavior.
13 - Proof for a bug fix depends on the bug. Logic, data, state, and protocol bugs get a failing test first (the fewest that fail on `main` for that reason, in an existing case of the module's test file where one fits), then the fix. Purely visual bugs (spacing, colour, alignment, a layout shift) get before/after captures as their proof and no test: do not add UI tests for nitpicks, and do not ask for one in review when a capture already shows it. A server-only fix with no visible surface ships the test plus a short runtime transcript.
14
evals/test-signal/cases.ts
+17
@@ -133,6 +133,23 @@ export const cases: ReadonlyArray<Case> = [
133 '\n it("returns a boolean", () => {\n expect(typeof isMarkdownFileLinkLabel("a.ts", "/repo/a.ts")).toBe("boolean");\n });\n',
134 targets: [{ path: "packages/shared/src/markdownLinks.test.ts", title: "returns a boolean" }],
135 },
136 + {
137 + id: "logic-in-browser-test",
138 + kind: "bad",
139 + source: "Synthetic: a browser test that never renders, for a pure function in a .logic.ts file",
140 + type: "synthetic",
141 + rev: base,
142 + path: "apps/web/src/components/chat/OpenInPicker.browser.test.tsx",
143 + after: ' expect(await spoken()).toBe("Opened in VS Code");\n });\n});\n',
144 + insert:
145 + '\ndescribe("open in picker visibility", () => {\n it("shows the picker for the primary environment or a remote that can open links", async () => {\n const { shouldShowOpenInPicker } = await import("./OpenInPicker.logic");\n const show = (overrides: Partial<Parameters<typeof shouldShowOpenInPicker>[0]>) =>\n shouldShowOpenInPicker({\n activeProjectName: "app",\n activeThreadEnvironmentId: selected,\n primaryEnvironmentId: primary,\n remoteOpenMode: "local-exec",\n ...overrides,\n });\n expect(show({ activeProjectName: undefined })).toBe(false);\n expect(show({ activeThreadEnvironmentId: primary })).toBe(true);\n expect(show({})).toBe(false);\n expect(show({ remoteOpenMode: "remote-links" })).toBe(true);\n });\n});\n',
146 + targets: [
147 + {
148 + path: "apps/web/src/components/chat/OpenInPicker.browser.test.tsx",
149 + title: "shows the picker for the primary environment or a remote that can open links",
150 + },
151 + ],
152 + },
153 {
154 id: "deadlock-race-fix",
155 kind: "guard",