@devries-cameron20 / t3code-1 / commits / 66ee628e08
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

perf(scripts): check:changed finishes in about 10 s (#23581)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Theo Browne committed Oct 23, 2026 at 07:09 UTC 66ee628e0875b2b735b199f42e6151e347833637
4 files changed +314 -91
docs/operations/development.md
+5
@@ -160,6 +160,11 @@ files to packages with [ci-changes.cjs](../../.github/scripts/ci-changes.cjs), a
160 compares against the merge base with `origin/main`, or with a ref you pass. CI still runs the full
161 test suites and the jobs the summary lists, so a local pass does not promise a green CI run.
162
163 +The steps run side by side, so a run takes about as long as its slowest step, usually knip's
164 +exports pass at about 10 s. When the files are exactly those of the last passing run, it prints that
165 +run's summary instead of running again. Any change to a file, the install, or a generated file runs
166 +the checks again.
167 +
168 While you iterate, run single checks:
169
170 ```sh
knip.jsonc
+3 -1
@@ -32,7 +32,9 @@
32 ],
33 "ignoreDependencies": ["esbuild"],
34 // The `t3` npm package's files: copied into the package, never imported.
35 - "ignore": ["npm-launcher/**"],
35 + // knip-schemas.test.ts writes a throwaway project to .knip-test-*, which a
36 + // knip run beside it in check:changed must not report.
37 + "ignore": ["npm-launcher/**", ".knip-test-*/**"],
38 },
39 "apps/server": {
40 // Vite+ pack entries and the launcher used by installed background services.
scripts/check-changed.ts
+154 -25
@@ -10,7 +10,10 @@
10 //
11 // Uncommitted and untracked files count. Affected packages come from
12 // .github/scripts/ci-changes.cjs, the same mapping CI uses to gate its jobs.
13 +// The steps run side by side; each prints its output when it finishes. A
14 +// rerun on exactly the files of the last passing run prints that run's summary.
15 import * as NodeChildProcess from "node:child_process";
16 +import * as NodeCrypto from "node:crypto";
17 import * as NodeFS from "node:fs";
18 import * as NodeOS from "node:os";
19 import * as NodePath from "node:path";
@@ -19,8 +22,13 @@ import { decide, readWorkspace } from "../.github/scripts/ci-changes.cjs";
22 const root = NodePath.resolve(import.meta.dirname, "..");
23 const CODE = /\.[cm]?[jt]sx?$/;
24
25 +// quotePath=false keeps non-ASCII paths as real paths instead of quoted escapes.
26 const git = (...args: Array<string>) =>
23 - NodeChildProcess.execFileSync("git", args, { cwd: root, encoding: "utf8" })
27 + NodeChildProcess.execFileSync("git", ["-c", "core.quotePath=false", ...args], {
28 + cwd: root,
29 + encoding: "utf8",
30 + maxBuffer: 64 * 1024 * 1024,
31 + })
32 .split("\n")
33 .filter(Boolean);
34
@@ -46,6 +54,75 @@ if (files.length === 0) {
54 console.log(`check:changed: no changes since ${base.slice(0, 12)}.`);
55 process.exit(0);
56 }
57 +console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`);
58 +
59 +// The checked tree is the merge base plus the diff plus the untracked files.
60 +// Ignored generated sources, .astro types, generated declarations such as
61 +// apps/mobile/expo-env.d.ts, env files, and the install state can change results
62 +// too, so they are part of the key. Other ignored folders (iOS Pods, dist, the
63 +// Electron runtime) are large and no check reads them.
64 +// Computed again after a passing run, so edits made during the run are never
65 +// cached as passing.
66 +const inputKey = () => {
67 + const hash = NodeCrypto.createHash("sha256");
68 + // CI changes test behavior (vite.config.ts rejects .only under CI).
69 + hash.update(`${process.version}\0${base}\0${process.env.CI ?? ""}\0`);
70 + hash.update(
71 + NodeChildProcess.execFileSync(
72 + "git",
73 + ["diff", "--binary", "--no-renames", "--no-ext-diff", "--no-textconv", base, "--"],
74 + { cwd: root, maxBuffer: 1024 * 1024 * 1024 },
75 + ),
76 + );
77 + const ignored = git(
78 + "ls-files",
79 + "--others",
80 + "--ignored",
81 + "--exclude-standard",
82 + "--",
83 + ":(glob)**/.generated/**",
84 + ":(glob)**/.astro/**",
85 + ":(glob)**/*.d.ts",
86 + ":(glob)**/.env*",
87 + ":(exclude,glob)**/node_modules/**",
88 + );
89 + for (const file of [
90 + // Raw bytes of changed files too: .gitattributes normalizes line endings in
91 + // the diff, but the formatter reads the file as it is on disk.
92 + ...git("diff", "--name-only", "--no-renames", base, "--"),
93 + ...git("ls-files", "--others", "--exclude-standard"),
94 + ...ignored,
95 + "node_modules/.modules.yaml",
96 + "node_modules/.pnpm/lock.yaml",
97 + ]) {
98 + hash.update(`\0${file}\0`);
99 + const path = NodePath.join(root, file);
100 + // Untracked nested repositories are listed as directories.
101 + if (NodeFS.statSync(path, { throwIfNoEntry: false })?.isFile()) {
102 + hash.update(NodeFS.readFileSync(path));
103 + }
104 + }
105 + return hash.digest("hex");
106 +};
107 +const key = inputKey();
108 +const cachePath = NodePath.join(root, "node_modules/.cache/check-changed.json");
109 +// A missing or unreadable cache is a miss, never a failure.
110 +const readCache = () => {
111 + try {
112 + return JSON.parse(NodeFS.readFileSync(cachePath, "utf8")) as {
113 + key: string;
114 + summary: Array<string>;
115 + };
116 + } catch {
117 + return undefined;
118 + }
119 +};
120 +const cached = readCache();
121 +if (cached?.key === key) {
122 + console.log("Nothing changed since the last passing run.\n");
123 + for (const line of cached.summary) console.log(line);
124 + process.exit(0);
125 +}
126
127 const present = files.filter((file) => NodeFS.existsSync(NodePath.join(root, file)));
128 const code = present.filter((file) => CODE.test(file));
@@ -66,12 +143,24 @@ for (const file of code) {
143 }
144 }
145
69 -// knip:check takes about 35 s and cannot be scoped to a package. Any code
70 -// change can leave a file, export, or dependency unused (removing the last
71 -// import of a helper does), so it runs whenever code or a manifest changes.
146 +// knip:check cannot be scoped to a package. Any code change can leave a file,
147 +// export, or dependency unused (removing the last import of a helper does), so
148 +// it runs whenever code or a manifest changes.
149 const runKnip = files.some(
150 (file) => CODE.test(file) || NodePath.basename(file) === "package.json" || file === "knip.jsonc",
151 );
152 +// CI runs knip:check as one script of `knip ... && knip ...` passes. The passes
153 +// do not depend on each other, so they run side by side here.
154 +const knipCheck: string = JSON.parse(
155 + NodeFS.readFileSync(NodePath.join(root, "package.json"), "utf8"),
156 +).scripts["knip:check"];
157 +const knipPasses = knipCheck.split(" && ").map((command, index, all) => {
158 + const [bin, ...args] = command.trim().split(/\s+/);
159 + if (bin !== "knip" || /["'$\\|;&<>]/.test(command)) {
160 + throw new Error(`check:changed cannot split knip:check: ${command}`);
161 + }
162 + return { name: `knip ${index + 1}/${all.length}`, args: ["exec", "knip", ...args] };
163 +});
164
165 // A version bump can strand a patch in pnpm-workspace.yaml: no resolved
166 // version uses it and pnpm rejects the install with ERR_PNPM_UNUSED_PATCH.
@@ -153,14 +242,24 @@ const steps = [
242 args: ["fmt", "--check", "--no-error-on-unmatched-pattern", ...present],
243 },
244 code.length > 0 && { name: "lint", args: ["lint", ...code] },
156 - runKnip && { name: "knip", args: ["run", "knip:check"] },
245 + ...(runKnip ? knipPasses : []),
246 ...graphql.map((dir) => ({
247 name: `graphql (${dir})`,
248 args: ["run", "--filter", `./${dir}`, "generate", "--check"],
249 })),
250 affected.length > 0 && {
251 name: `typecheck (${affected.length} packages)`,
163 - args: ["run", "--cache", "--concurrency-limit", "4", ...filters(affected), "typecheck"],
252 + // `tsc --noEmit` writes nothing another package reads, so dependency order
253 + // only made dependents wait: --parallel cut a contracts change from 13 s to 9 s.
254 + args: [
255 + "run",
256 + "--cache",
257 + "--parallel",
258 + "--concurrency-limit",
259 + "4",
260 + ...filters(affected),
261 + "typecheck",
262 + ],
263 },
264 // From the repo root, the root vite.config.ts hands each file to its package's config.
265 tests.size > 0 && {
@@ -169,23 +268,36 @@ const steps = [
268 },
269 ].filter((step) => step !== false);
270
172 -console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`);
173 -const results = steps.map((step) => {
174 - console.log(`\n> vp ${step.name}`);
175 - const started = performance.now();
176 - const status =
177 - "args" in step
178 - ? NodeChildProcess.spawnSync("vp", step.args, { cwd: root, stdio: "inherit" }).status
179 - : step.run()
180 - ? 0
181 - : 1;
182 - const seconds = ((performance.now() - started) / 1000).toFixed(1);
183 - return { name: step.name, ok: status === 0, seconds };
184 -});
271 +// Output is piped, so keep the colors a terminal would get.
272 +const env = process.stdout.isTTY ? { ...process.env, FORCE_COLOR: "1" } : process.env;
273 +const results = await Promise.all(
274 + steps.map(
275 + (step) =>
276 + new Promise<{ name: string; ok: boolean; seconds: string }>((resolve) => {
277 + const started = performance.now();
278 + const finish = (ok: boolean, output?: Buffer) => {
279 + const seconds = ((performance.now() - started) / 1000).toFixed(1);
280 + console.log(`\n> vp ${step.name} ${ok ? "pass" : "FAIL"} ${seconds}s`);
281 + if (output) process.stdout.write(output);
282 + resolve({ name: step.name, ok, seconds });
283 + };
284 + // In-process steps print their own output. The patch check is first and
285 + // takes about a second, so the commands start right after it.
286 + if ("run" in step) return finish(step.run());
287 + const output: Array<Buffer> = [];
288 + const child = NodeChildProcess.spawn("vp", step.args, { cwd: root, env });
289 + child.stdout.on("data", (chunk: Buffer) => output.push(chunk));
290 + child.stderr.on("data", (chunk: Buffer) => output.push(chunk));
291 + // A failed spawn emits "error", then "close" with a nonzero code.
292 + child.on("error", (error) => output.push(Buffer.from(`${error.message}\n`)));
293 + child.on("close", (status) => finish(status === 0, Buffer.concat(output)));
294 + }),
295 + ),
296 +);
297
186 -console.log("\ncheck:changed summary");
298 +const summary = ["check:changed summary"];
299 for (const result of results) {
188 - console.log(` ${result.ok ? "pass" : "FAIL"} ${result.name} ${result.seconds}s`);
300 + summary.push(` ${result.ok ? "pass" : "FAIL"} ${result.name} ${result.seconds}s`);
301 }
302 const ciOnly = Object.entries({
303 "full test suites": outputs.test || outputs.test_web || outputs.test_server,
@@ -194,7 +306,24 @@ const ciOnly = Object.entries({
306 "mobile native lint": outputs.mobile_native,
307 "release smoke": outputs.release_smoke,
308 }).flatMap(([job, runs]) => (runs ? [job] : []));
197 -if (ciOnly.length > 0) console.log(` CI also runs: ${ciOnly.join(", ")}`);
198 -if (!runPatchCheck) console.log(" unused patch check skipped: no dependency files changed");
199 -if (!runKnip) console.log(" knip skipped: no code or manifest changed");
200 -if (results.some((result) => !result.ok)) process.exit(1);
309 +if (ciOnly.length > 0) summary.push(` CI also runs: ${ciOnly.join(", ")}`);
310 +if (!runPatchCheck) summary.push(" unused patch check skipped: no dependency files changed");
311 +if (!runKnip) summary.push(" knip skipped: no code or manifest changed");
312 +console.log(`\n${summary.join("\n")}`);
313 +// exitCode, not exit(): piped stdout is asynchronous on macOS, and exit()
314 +// would cut off a failing step's output.
315 +if (results.some((result) => !result.ok)) {
316 + process.exitCode = 1;
317 +} else if (inputKey() !== key) {
318 + console.log(" pass not cached: files changed during the run");
319 +} else {
320 + try {
321 + // Write then rename, so a concurrent run never reads half a file.
322 + const temp = `${cachePath}.${process.pid}`;
323 + NodeFS.mkdirSync(NodePath.dirname(cachePath), { recursive: true });
324 + NodeFS.writeFileSync(temp, JSON.stringify({ key, summary }));
325 + NodeFS.renameSync(temp, cachePath);
326 + } catch (error) {
327 + console.warn(` pass not cached: ${error instanceof Error ? error.message : String(error)}`);
328 + }
329 +}
scripts/knip-schemas.ts
+152 -65
@@ -1,14 +1,117 @@
1 // @effect-diagnostics nodeBuiltinImport:off - Knip and the TypeScript compiler host use synchronous Node paths.
2 +import * as NodeOS from "node:os";
3 import * as NodePath from "node:path";
4 +import * as NodeWorkerThreads from "node:worker_threads";
5 import type { Preprocessor } from "knip";
6 // Knip needs the legacy compiler API, which TypeScript 7 no longer exports.
7 import ts from "typescript-legacy";
8
9 const schemaTypeId = "~effect/Schema/Schema";
10
9 -const preprocess: Preprocessor = (options) => {
11 +// One tsconfig's files, each with the export names its issues mention. Only
12 +// those names are checked: inferring the type of every export cost seconds.
13 +interface Project {
14 + readonly cwd: string;
15 + readonly configPath: string | undefined;
16 + readonly files: ReadonlyArray<readonly [string, ReadonlyArray<string>]>;
17 +}
18 +
19 +// Each tsconfig needs its own program. The programs share parsed and bound
20 +// files (Effect's and Node's declarations) through one registry, as the
21 +// language service does. Without it, building 17 programs took about 13 s.
22 +const registry = ts.createDocumentRegistry();
23 +
24 +// Returns each file with those of its names that are Effect schemas.
25 +function findSchemas({ cwd, configPath, files }: Project) {
26 + const config = configPath
27 + ? ts.getParsedCommandLineOfConfigFile(
28 + configPath,
29 + {},
30 + {
31 + ...ts.sys,
32 + onUnRecoverableConfigFileDiagnostic: (diagnostic) => {
33 + throw new Error(ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n"));
34 + },
35 + },
36 + )
37 + : undefined;
38 + if (config?.errors.length) {
39 + throw new Error(
40 + config.errors
41 + .map((error) => ts.flattenDiagnosticMessageText(error.messageText, "\n"))
42 + .join("\n"),
43 + );
44 + }
45 + const options: ts.CompilerOptions = {
46 + module: ts.ModuleKind.NodeNext,
47 + allowJs: true,
48 + ...config?.options,
49 + noEmit: true,
50 + };
51 + const host = ts.createCompilerHost(options);
52 + const key = registry.getKeyForCompilationSettings(options);
53 + host.getSourceFile = (fileName, languageVersionOrOptions) => {
54 + const text = host.readFile(fileName);
55 + if (text === undefined) return undefined;
56 + return registry.acquireDocumentWithKey(
57 + fileName,
58 + ts.sys.resolvePath(fileName) as ts.Path,
59 + options,
60 + key,
61 + ts.ScriptSnapshot.fromString(text),
62 + // Files do not change during one run.
63 + "1",
64 + undefined,
65 + languageVersionOrOptions,
66 + );
67 + };
68 + const program = ts.createProgram(
69 + files.map(([file]) => NodePath.resolve(cwd, file)),
70 + options,
71 + host,
72 + );
73 + const checker = program.getTypeChecker();
74 + return files.flatMap(([filePath, names]) => {
75 + const source = program.getSourceFile(NodePath.resolve(cwd, filePath));
76 + const module = source && checker.getSymbolAtLocation(source);
77 + if (!source || !module) return [];
78 + const schemas = checker
79 + .getExportsOfModule(module)
80 + .filter((symbol) => names.includes(symbol.name))
81 + .flatMap((symbol) => {
82 + const exported =
83 + symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol;
84 + const target =
85 + exported.flags & ts.SymbolFlags.Value
86 + ? exported
87 + : checker.resolveName(symbol.name, source, ts.SymbolFlags.Value, false);
88 + if (!target) return [];
89 + const type = checker.getTypeOfSymbolAtLocation(target, target.valueDeclaration ?? source);
90 + const marker = type.getProperty(schemaTypeId);
91 + if (!marker) return [];
92 + const markerType = checker.getTypeOfSymbolAtLocation(marker, source);
93 + return markerType.isStringLiteral() && markerType.value === schemaTypeId
94 + ? [symbol.name]
95 + : [];
96 + });
97 + return [[filePath, schemas] as const];
98 + });
99 +}
100 +
101 +// The preprocessor loads this file again as each worker.
102 +if (!NodeWorkerThreads.isMainThread) {
103 + const projects: ReadonlyArray<Project> = NodeWorkerThreads.workerData;
104 + NodeWorkerThreads.parentPort?.postMessage(projects.flatMap(findSchemas));
105 +}
106 +
107 +// Three workers measured fastest. Each one parses Effect again, so more
108 +// workers cost more than they save.
109 +const WORKERS = Math.min(3, NodeOS.availableParallelism());
110 +
111 +// Drops unused-export issues whose every symbol is an Effect schema.
112 +const preprocess = async (options: Parameters<Preprocessor>[0]) => {
113 const categories = ["exports", "nsExports", "duplicates"] as const;
11 - const projects = new Map<string | undefined, Set<string>>();
114 + const projects = new Map<string | undefined, Map<string, Set<string>>>();
115 for (const category of categories) {
116 for (const [filePath, issues] of Object.entries(options.issues[category])) {
117 if (Object.keys(issues).length === 0) continue;
@@ -16,76 +119,60 @@ const preprocess: Preprocessor = (options) => {
119 NodePath.dirname(NodePath.resolve(options.cwd, filePath)),
120 ts.sys.fileExists,
121 );
19 - const files = projects.get(configPath) ?? new Set<string>();
20 - files.add(filePath);
122 + const files = projects.get(configPath) ?? new Map<string, Set<string>>();
123 + const names = files.get(filePath) ?? new Set<string>();
124 + for (const issue of Object.values(issues)) {
125 + for (const { symbol } of issue.symbols ?? [{ symbol: issue.symbol }]) names.add(symbol);
126 + }
127 + files.set(filePath, names);
128 projects.set(configPath, files);
129 }
130 }
131
25 - for (const [configPath, files] of projects) {
26 - const config = configPath
27 - ? ts.getParsedCommandLineOfConfigFile(
28 - configPath,
29 - {},
30 - {
31 - ...ts.sys,
32 - onUnRecoverableConfigFileDiagnostic: (diagnostic) => {
33 - throw new Error(ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n"));
34 - },
35 - },
36 - )
37 - : undefined;
38 - if (config?.errors.length) {
39 - throw new Error(
40 - config.errors
41 - .map((error) => ts.flattenDiagnosticMessageText(error.messageText, "\n"))
42 - .join("\n"),
43 - );
44 - }
45 - const program = ts.createProgram(
46 - [...files].map((file) => NodePath.resolve(options.cwd, file)),
47 - {
48 - module: ts.ModuleKind.NodeNext,
49 - allowJs: true,
50 - ...config?.options,
51 - noEmit: true,
52 - },
53 - );
54 - const checker = program.getTypeChecker();
55 - for (const filePath of files) {
56 - const source = program.getSourceFile(NodePath.resolve(options.cwd, filePath));
57 - const module = source && checker.getSymbolAtLocation(source);
58 - if (!source || !module) continue;
59 - const schemas = new Set(
60 - checker.getExportsOfModule(module).flatMap((symbol) => {
61 - const exported =
62 - symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol;
63 - const target =
64 - exported.flags & ts.SymbolFlags.Value
65 - ? exported
66 - : checker.resolveName(symbol.name, source, ts.SymbolFlags.Value, false);
67 - if (!target) return [];
68 - const type = checker.getTypeOfSymbolAtLocation(target, target.valueDeclaration ?? source);
69 - const marker = type.getProperty(schemaTypeId);
70 - if (!marker) return [];
71 - const markerType = checker.getTypeOfSymbolAtLocation(marker, source);
72 - return markerType.isStringLiteral() && markerType.value === schemaTypeId
73 - ? [symbol.name]
74 - : [];
132 + // Deal the projects out largest first so the workers get similar shares.
133 + const shares = Array.from({ length: WORKERS }, (): Array<Project> => []);
134 + [...projects]
135 + .sort(([, a], [, b]) => b.size - a.size)
136 + .forEach(([configPath, files], index) => {
137 + shares[index % WORKERS]!.push({
138 + cwd: options.cwd,
139 + configPath,
140 + files: [...files].map(([file, names]) => [file, [...names]]),
141 + });
142 + });
143 + const workers = shares
144 + .filter((share) => share.length > 0)
145 + .map((share) => new NodeWorkerThreads.Worker(new URL(import.meta.url), { workerData: share }));
146 + const results = await Promise.all(
147 + workers.map(
148 + (worker) =>
149 + new Promise<ReturnType<typeof findSchemas>>((resolve, reject) => {
150 + worker.once("message", resolve);
151 + worker.once("error", reject);
152 + worker.once("exit", (code) => {
153 + if (code !== 0) reject(new Error(`knip-schemas worker exited with code ${code}`));
154 + });
155 }),
76 - );
77 - for (const category of categories) {
78 - const issues = options.issues[category][filePath];
79 - if (!issues) continue;
80 - for (const [key, issue] of Object.entries(issues)) {
81 - const symbols = issue.symbols ?? [{ symbol: issue.symbol }];
82 - if (symbols.length > 0 && symbols.every(({ symbol }) => schemas.has(symbol))) {
83 - delete issues[key];
84 - options.counters[category]--;
85 - }
156 + ),
157 + ).catch(async (error: unknown) => {
158 + // knip only sets an exit code on a failed preprocessor, so stop the
159 + // siblings or the failed run waits for their work.
160 + await Promise.allSettled(workers.map((worker) => worker.terminate()));
161 + throw error;
162 + });
163 +
164 + for (const [filePath, names] of results.flat()) {
165 + for (const category of categories) {
166 + const issues = options.issues[category][filePath];
167 + if (!issues) continue;
168 + for (const [key, issue] of Object.entries(issues)) {
169 + const symbols = issue.symbols ?? [{ symbol: issue.symbol }];
170 + if (symbols.length > 0 && symbols.every(({ symbol }) => names.includes(symbol))) {
171 + delete issues[key];
172 + options.counters[category]--;
173 }
87 - if (Object.keys(issues).length === 0) delete options.issues[category][filePath];
174 }
175 + if (Object.keys(issues).length === 0) delete options.issues[category][filePath];
176 }
177 }
178 return options;