The best way to control Claude Code, Codex and more from any computer or phone

main
ts 159 lines 7.03 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off globalConsole:off - a synchronous wrapper around one vitest project.
2 import * as NodeChildProcess from "node:child_process";
3 import * as NodeFS from "node:fs";
4 import * as NodeOS from "node:os";
5 import * as NodePath from "node:path";
6
7 import { vitePlusCommand } from "./check-changed-command.ts";
8
9 const HELP = `Usage:
10 node scripts/render-metrics.ts run [--out <file>] [--runs <n>]
11 node scripts/render-metrics.ts compare <before.json> <after.json> [--top <n>]
12
13 run Runs the web render scenarios (apps/web/test/renderScenarios.renders.tsx,
14 vitest project "renders") on this checkout and writes one JSON object of
15 { scenario: { commits, renders, components } }. With --runs above 1 (default 3)
16 it fails unless every run produced identical counts.
17 compare Prints a markdown table of commit and render deltas per scenario, plus the
18 components whose counts changed most (default top 10).
19
20 commits are React commits that rendered at least one component (null where frame
21 timing splits them); renders are function and class component renders, counted from the
22 fibers React worked on. Every scenario waits for the tree to go quiet before and after
23 its counted window, so layout follow-ups are included.
24
25 Comparing two refs needs both checkouts, since the scenarios run each tree's code:
26 node scripts/render-metrics.ts run --out ../main-renders.json # on main
27 node scripts/render-metrics.ts run --out ../branch-renders.json # on the branch
28 node scripts/render-metrics.ts compare ../main-renders.json ../branch-renders.json
29 A second checkout comes from \`git worktree add ../<name> main\`; copy the harness files
30 in if that tree predates them.`;
31
32 export interface ScenarioReport {
33 /** null where a scenario's commit count depends on frame timing. */
34 readonly commits: number | null;
35 readonly renders: number;
36 readonly components: Readonly<Record<string, number>>;
37 }
38 export type RenderMetrics = Readonly<Record<string, ScenarioReport>>;
39
40 const root = NodePath.resolve(import.meta.dirname, "..");
41
42 export function runOnce(): RenderMetrics {
43 const directory = NodeFS.mkdtempSync(NodePath.join(NodeOS.tmpdir(), "t3-renders-"));
44 try {
45 const [command, commandArgs] = vitePlusCommand(["test", "run", "--project", "renders"]);
46 const result = NodeChildProcess.spawnSync(command, commandArgs, {
47 cwd: NodePath.join(root, "apps/web"),
48 env: { ...process.env, T3CODE_RENDER_REPORT_DIR: directory },
49 stdio: ["ignore", "inherit", "inherit"],
50 });
51 if (result.status !== 0) throw new Error(`render scenarios failed (exit ${result.status})`);
52 const files = NodeFS.readdirSync(directory)
53 .filter((file) => file.endsWith(".json"))
54 .toSorted();
55 return Object.fromEntries(
56 files.map((file) => [
57 file.replace(/\.json$/, ""),
58 JSON.parse(NodeFS.readFileSync(NodePath.join(directory, file), "utf8")) as ScenarioReport,
59 ]),
60 );
61 } finally {
62 NodeFS.rmSync(directory, { recursive: true, force: true });
63 }
64 }
65
66 /** Scenarios whose counts differ between runs, by name. */
67 export function nondeterministic(runs: ReadonlyArray<RenderMetrics>): string[] {
68 const [first, ...rest] = runs;
69 if (!first) return [];
70 return Object.keys(first).filter((scenario) =>
71 rest.some((run) => JSON.stringify(run[scenario]) !== JSON.stringify(first[scenario])),
72 );
73 }
74
75 const signed = (value: number) => (value > 0 ? `+${value}` : `${value}`);
76 const change = (before: number | null, after: number | null) =>
77 before === null || after === null
78 ? "n/a"
79 : before === after
80 ? `${after}`
81 : `${before} → ${after} (${signed(after - before)}${before ? `, ${signed(Math.round(((after - before) / before) * 100))}%` : ""})`;
82
83 export function renderComparison(before: RenderMetrics, after: RenderMetrics, top = 10): string {
84 const scenarios = [...new Set([...Object.keys(before), ...Object.keys(after)])].toSorted();
85 const lines = [
86 "### Render scenarios",
87 "",
88 "| scenario | commits | renders |",
89 "| :-- | --: | --: |",
90 ];
91 const empty: ScenarioReport = { commits: 0, renders: 0, components: {} };
92 for (const scenario of scenarios) {
93 const a = before[scenario] ?? empty;
94 const b = after[scenario] ?? empty;
95 lines.push(
96 `| ${scenario} | ${change(a.commits, b.commits)} | ${change(a.renders, b.renders)} |`,
97 );
98 }
99 for (const scenario of scenarios) {
100 const a = before[scenario]?.components ?? {};
101 const b = after[scenario]?.components ?? {};
102 const moved = [...new Set([...Object.keys(a), ...Object.keys(b)])]
103 .map((name) => [name, a[name] ?? 0, b[name] ?? 0] as const)
104 .filter(([, x, y]) => x !== y)
105 .sort(([n, x, y], [m, p, q]) => Math.abs(q - p) - Math.abs(y - x) || n.localeCompare(m))
106 .slice(0, top);
107 if (moved.length === 0) continue;
108 lines.push("", `<details><summary>${scenario}: components that changed most</summary>`, "");
109 lines.push("| component | renders |", "| :-- | --: |");
110 for (const [name, x, y] of moved) lines.push(`| \`${name}\` | ${change(x, y)} |`);
111 lines.push("", "</details>");
112 }
113 return lines.join("\n");
114 }
115
116 function option(args: ReadonlyArray<string>, name: string): string | undefined {
117 const index = args.indexOf(name);
118 return index === -1 ? undefined : args[index + 1];
119 }
120
121 if (import.meta.main) {
122 const [command, ...args] = process.argv.slice(2);
123 if (command === "run") {
124 const runs = Number(option(args, "--runs") ?? 3);
125 const results = Array.from({ length: runs }, runOnce);
126 const unstable = nondeterministic(results);
127 if (unstable.length) {
128 for (const scenario of unstable) {
129 const counts = results.map((run) => `${run[scenario]?.commits}/${run[scenario]?.renders}`);
130 console.error(`${scenario} differed between runs (commits/renders): ${counts.join(", ")}`);
131 const first = results[0]?.[scenario]?.components ?? {};
132 for (const [index, run] of results.entries()) {
133 const other = run[scenario]?.components ?? {};
134 const names = [...new Set([...Object.keys(first), ...Object.keys(other)])];
135 const moved = names.filter((name) => first[name] !== other[name]);
136 if (moved.length) {
137 const detail = moved.map((name) => `${name} ${first[name] ?? 0}→${other[name] ?? 0}`);
138 console.error(` run ${index + 1} vs run 1: ${detail.join(", ")}`);
139 }
140 }
141 }
142 process.exitCode = 1;
143 } else {
144 const json = JSON.stringify(results[0], null, 2);
145 const out = option(args, "--out");
146 if (out) NodeFS.writeFileSync(out, json + "\n");
147 else console.log(json);
148 console.error(`${runs} run(s) produced identical counts.`);
149 }
150 } else if (command === "compare" && args[0] && args[1]) {
151 const read = (path: string) => JSON.parse(NodeFS.readFileSync(path, "utf8")) as RenderMetrics;
152 console.log(
153 renderComparison(read(args[0]), read(args[1]), Number(option(args, "--top") ?? 10)),
154 );
155 } else {
156 console.log(HELP);
157 if (command !== "--help" && command !== "-h" && command !== undefined) process.exitCode = 1;
158 }
159 }