The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 18:15 UTC
perf(scripts): check:changed finishes in about 10 s (#23581)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Theo Browne committed
Oct 23, 2026 at 07:09 UTC
66ee628e0875b2b735b199f42e6151e347833637
4 files changed
+314
-91
docs/operations/development.md
+5
@@ -160,6 +160,11 @@ files to packages with [ci-changes.cjs](../../.github/scripts/ci-changes.cjs), a
160
compares against the merge base with `origin/main`, or with a ref you pass. CI still runs the full
161
test suites and the jobs the summary lists, so a local pass does not promise a green CI run.
162
163
+The steps run side by side, so a run takes about as long as its slowest step, usually knip's
164
+exports pass at about 10 s. When the files are exactly those of the last passing run, it prints that
165
+run's summary instead of running again. Any change to a file, the install, or a generated file runs
166
+the checks again.
167
+
168
While you iterate, run single checks:
169
170
```sh
knip.jsonc
+3
-1
@@ -32,7 +32,9 @@
32
],
33
"ignoreDependencies": ["esbuild"],
34
// The `t3` npm package's files: copied into the package, never imported.
35
- "ignore": ["npm-launcher/**"],
35
+ // knip-schemas.test.ts writes a throwaway project to .knip-test-*, which a
36
+ // knip run beside it in check:changed must not report.
37
+ "ignore": ["npm-launcher/**", ".knip-test-*/**"],
38
},
39
"apps/server": {
40
// Vite+ pack entries and the launcher used by installed background services.
scripts/check-changed.ts
+154
-25
@@ -10,7 +10,10 @@
10
//
11
// Uncommitted and untracked files count. Affected packages come from
12
// .github/scripts/ci-changes.cjs, the same mapping CI uses to gate its jobs.
13
+// The steps run side by side; each prints its output when it finishes. A
14
+// rerun on exactly the files of the last passing run prints that run's summary.
15
import * as NodeChildProcess from "node:child_process";
16
+import * as NodeCrypto from "node:crypto";
17
import * as NodeFS from "node:fs";
18
import * as NodeOS from "node:os";
19
import * as NodePath from "node:path";
@@ -19,8 +22,13 @@ import { decide, readWorkspace } from "../.github/scripts/ci-changes.cjs";
22
const root = NodePath.resolve(import.meta.dirname, "..");
23
const CODE = /\.[cm]?[jt]sx?$/;
24
25
+// quotePath=false keeps non-ASCII paths as real paths instead of quoted escapes.
26
const git = (...args: Array<string>) =>
23
- NodeChildProcess.execFileSync("git", args, { cwd: root, encoding: "utf8" })
27
+ NodeChildProcess.execFileSync("git", ["-c", "core.quotePath=false", ...args], {
28
+ cwd: root,
29
+ encoding: "utf8",
30
+ maxBuffer: 64 * 1024 * 1024,
31
+ })
32
.split("\n")
33
.filter(Boolean);
34
@@ -46,6 +54,75 @@ if (files.length === 0) {
54
console.log(`check:changed: no changes since ${base.slice(0, 12)}.`);
55
process.exit(0);
56
}
57
+console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`);
58
+
59
+// The checked tree is the merge base plus the diff plus the untracked files.
60
+// Ignored generated sources, .astro types, generated declarations such as
61
+// apps/mobile/expo-env.d.ts, env files, and the install state can change results
62
+// too, so they are part of the key. Other ignored folders (iOS Pods, dist, the
63
+// Electron runtime) are large and no check reads them.
64
+// Computed again after a passing run, so edits made during the run are never
65
+// cached as passing.
66
+const inputKey = () => {
67
+ const hash = NodeCrypto.createHash("sha256");
68
+ // CI changes test behavior (vite.config.ts rejects .only under CI).
69
+ hash.update(`${process.version}\0${base}\0${process.env.CI ?? ""}\0`);
70
+ hash.update(
71
+ NodeChildProcess.execFileSync(
72
+ "git",
73
+ ["diff", "--binary", "--no-renames", "--no-ext-diff", "--no-textconv", base, "--"],
74
+ { cwd: root, maxBuffer: 1024 * 1024 * 1024 },
75
+ ),
76
+ );
77
+ const ignored = git(
78
+ "ls-files",
79
+ "--others",
80
+ "--ignored",
81
+ "--exclude-standard",
82
+ "--",
83
+ ":(glob)**/.generated/**",
84
+ ":(glob)**/.astro/**",
85
+ ":(glob)**/*.d.ts",
86
+ ":(glob)**/.env*",
87
+ ":(exclude,glob)**/node_modules/**",
88
+ );
89
+ for (const file of [
90
+ // Raw bytes of changed files too: .gitattributes normalizes line endings in
91
+ // the diff, but the formatter reads the file as it is on disk.
92
+ ...git("diff", "--name-only", "--no-renames", base, "--"),
93
+ ...git("ls-files", "--others", "--exclude-standard"),
94
+ ...ignored,
95
+ "node_modules/.modules.yaml",
96
+ "node_modules/.pnpm/lock.yaml",
97
+ ]) {
98
+ hash.update(`\0${file}\0`);
99
+ const path = NodePath.join(root, file);
100
+ // Untracked nested repositories are listed as directories.
101
+ if (NodeFS.statSync(path, { throwIfNoEntry: false })?.isFile()) {
102
+ hash.update(NodeFS.readFileSync(path));
103
+ }
104
+ }
105
+ return hash.digest("hex");
106
+};
107
+const key = inputKey();
108
+const cachePath = NodePath.join(root, "node_modules/.cache/check-changed.json");
109
+// A missing or unreadable cache is a miss, never a failure.
110
+const readCache = () => {
111
+ try {
112
+ return JSON.parse(NodeFS.readFileSync(cachePath, "utf8")) as {
113
+ key: string;
114
+ summary: Array<string>;
115
+ };
116
+ } catch {
117
+ return undefined;
118
+ }
119
+};
120
+const cached = readCache();
121
+if (cached?.key === key) {
122
+ console.log("Nothing changed since the last passing run.\n");
123
+ for (const line of cached.summary) console.log(line);
124
+ process.exit(0);
125
+}
126
127
const present = files.filter((file) => NodeFS.existsSync(NodePath.join(root, file)));
128
const code = present.filter((file) => CODE.test(file));
@@ -66,12 +143,24 @@ for (const file of code) {
143
}
144
}
145
69
-// knip:check takes about 35 s and cannot be scoped to a package. Any code
70
-// change can leave a file, export, or dependency unused (removing the last
71
-// import of a helper does), so it runs whenever code or a manifest changes.
146
+// knip:check cannot be scoped to a package. Any code change can leave a file,
147
+// export, or dependency unused (removing the last import of a helper does), so
148
+// it runs whenever code or a manifest changes.
149
const runKnip = files.some(
150
(file) => CODE.test(file) || NodePath.basename(file) === "package.json" || file === "knip.jsonc",
151
);
152
+// CI runs knip:check as one script of `knip ... && knip ...` passes. The passes
153
+// do not depend on each other, so they run side by side here.
154
+const knipCheck: string = JSON.parse(
155
+ NodeFS.readFileSync(NodePath.join(root, "package.json"), "utf8"),
156
+).scripts["knip:check"];
157
+const knipPasses = knipCheck.split(" && ").map((command, index, all) => {
158
+ const [bin, ...args] = command.trim().split(/\s+/);
159
+ if (bin !== "knip" || /["'$\\|;&<>]/.test(command)) {
160
+ throw new Error(`check:changed cannot split knip:check: ${command}`);
161
+ }
162
+ return { name: `knip ${index + 1}/${all.length}`, args: ["exec", "knip", ...args] };
163
+});
164
165
// A version bump can strand a patch in pnpm-workspace.yaml: no resolved
166
// version uses it and pnpm rejects the install with ERR_PNPM_UNUSED_PATCH.
@@ -153,14 +242,24 @@ const steps = [
242
args: ["fmt", "--check", "--no-error-on-unmatched-pattern", ...present],
243
},
244
code.length > 0 && { name: "lint", args: ["lint", ...code] },
156
- runKnip && { name: "knip", args: ["run", "knip:check"] },
245
+ ...(runKnip ? knipPasses : []),
246
...graphql.map((dir) => ({
247
name: `graphql (${dir})`,
248
args: ["run", "--filter", `./${dir}`, "generate", "--check"],
249
})),
250
affected.length > 0 && {
251
name: `typecheck (${affected.length} packages)`,
163
- args: ["run", "--cache", "--concurrency-limit", "4", ...filters(affected), "typecheck"],
252
+ // `tsc --noEmit` writes nothing another package reads, so dependency order
253
+ // only made dependents wait: --parallel cut a contracts change from 13 s to 9 s.
254
+ args: [
255
+ "run",
256
+ "--cache",
257
+ "--parallel",
258
+ "--concurrency-limit",
259
+ "4",
260
+ ...filters(affected),
261
+ "typecheck",
262
+ ],
263
},
264
// From the repo root, the root vite.config.ts hands each file to its package's config.
265
tests.size > 0 && {
@@ -169,23 +268,36 @@ const steps = [
268
},
269
].filter((step) => step !== false);
270
172
-console.log(`check:changed: ${files.length} files since ${base.slice(0, 12)}`);
173
-const results = steps.map((step) => {
174
- console.log(`\n> vp ${step.name}`);
175
- const started = performance.now();
176
- const status =
177
- "args" in step
178
- ? NodeChildProcess.spawnSync("vp", step.args, { cwd: root, stdio: "inherit" }).status
179
- : step.run()
180
- ? 0
181
- : 1;
182
- const seconds = ((performance.now() - started) / 1000).toFixed(1);
183
- return { name: step.name, ok: status === 0, seconds };
184
-});
271
+// Output is piped, so keep the colors a terminal would get.
272
+const env = process.stdout.isTTY ? { ...process.env, FORCE_COLOR: "1" } : process.env;
273
+const results = await Promise.all(
274
+ steps.map(
275
+ (step) =>
276
+ new Promise<{ name: string; ok: boolean; seconds: string }>((resolve) => {
277
+ const started = performance.now();
278
+ const finish = (ok: boolean, output?: Buffer) => {
279
+ const seconds = ((performance.now() - started) / 1000).toFixed(1);
280
+ console.log(`\n> vp ${step.name} ${ok ? "pass" : "FAIL"} ${seconds}s`);
281
+ if (output) process.stdout.write(output);
282
+ resolve({ name: step.name, ok, seconds });
283
+ };
284
+ // In-process steps print their own output. The patch check is first and
285
+ // takes about a second, so the commands start right after it.
286
+ if ("run" in step) return finish(step.run());
287
+ const output: Array<Buffer> = [];
288
+ const child = NodeChildProcess.spawn("vp", step.args, { cwd: root, env });
289
+ child.stdout.on("data", (chunk: Buffer) => output.push(chunk));
290
+ child.stderr.on("data", (chunk: Buffer) => output.push(chunk));
291
+ // A failed spawn emits "error", then "close" with a nonzero code.
292
+ child.on("error", (error) => output.push(Buffer.from(`${error.message}\n`)));
293
+ child.on("close", (status) => finish(status === 0, Buffer.concat(output)));
294
+ }),
295
+ ),
296
+);
297
186
-console.log("\ncheck:changed summary");
298
+const summary = ["check:changed summary"];
299
for (const result of results) {
188
- console.log(` ${result.ok ? "pass" : "FAIL"} ${result.name} ${result.seconds}s`);
300
+ summary.push(` ${result.ok ? "pass" : "FAIL"} ${result.name} ${result.seconds}s`);
301
}
302
const ciOnly = Object.entries({
303
"full test suites": outputs.test || outputs.test_web || outputs.test_server,
@@ -194,7 +306,24 @@ const ciOnly = Object.entries({
306
"mobile native lint": outputs.mobile_native,
307
"release smoke": outputs.release_smoke,
308
}).flatMap(([job, runs]) => (runs ? [job] : []));
197
-if (ciOnly.length > 0) console.log(` CI also runs: ${ciOnly.join(", ")}`);
198
-if (!runPatchCheck) console.log(" unused patch check skipped: no dependency files changed");
199
-if (!runKnip) console.log(" knip skipped: no code or manifest changed");
200
-if (results.some((result) => !result.ok)) process.exit(1);
309
+if (ciOnly.length > 0) summary.push(` CI also runs: ${ciOnly.join(", ")}`);
310
+if (!runPatchCheck) summary.push(" unused patch check skipped: no dependency files changed");
311
+if (!runKnip) summary.push(" knip skipped: no code or manifest changed");
312
+console.log(`\n${summary.join("\n")}`);
313
+// exitCode, not exit(): piped stdout is asynchronous on macOS, and exit()
314
+// would cut off a failing step's output.
315
+if (results.some((result) => !result.ok)) {
316
+ process.exitCode = 1;
317
+} else if (inputKey() !== key) {
318
+ console.log(" pass not cached: files changed during the run");
319
+} else {
320
+ try {
321
+ // Write then rename, so a concurrent run never reads half a file.
322
+ const temp = `${cachePath}.${process.pid}`;
323
+ NodeFS.mkdirSync(NodePath.dirname(cachePath), { recursive: true });
324
+ NodeFS.writeFileSync(temp, JSON.stringify({ key, summary }));
325
+ NodeFS.renameSync(temp, cachePath);
326
+ } catch (error) {
327
+ console.warn(` pass not cached: ${error instanceof Error ? error.message : String(error)}`);
328
+ }
329
+}
scripts/knip-schemas.ts
+152
-65
@@ -1,14 +1,117 @@
1
// @effect-diagnostics nodeBuiltinImport:off - Knip and the TypeScript compiler host use synchronous Node paths.
2
+import * as NodeOS from "node:os";
3
import * as NodePath from "node:path";
4
+import * as NodeWorkerThreads from "node:worker_threads";
5
import type { Preprocessor } from "knip";
6
// Knip needs the legacy compiler API, which TypeScript 7 no longer exports.
7
import ts from "typescript-legacy";
8
9
const schemaTypeId = "~effect/Schema/Schema";
10
9
-const preprocess: Preprocessor = (options) => {
11
+// One tsconfig's files, each with the export names its issues mention. Only
12
+// those names are checked: inferring the type of every export cost seconds.
13
+interface Project {
14
+ readonly cwd: string;
15
+ readonly configPath: string | undefined;
16
+ readonly files: ReadonlyArray<readonly [string, ReadonlyArray<string>]>;
17
+}
18
+
19
+// Each tsconfig needs its own program. The programs share parsed and bound
20
+// files (Effect's and Node's declarations) through one registry, as the
21
+// language service does. Without it, building 17 programs took about 13 s.
22
+const registry = ts.createDocumentRegistry();
23
+
24
+// Returns each file with those of its names that are Effect schemas.
25
+function findSchemas({ cwd, configPath, files }: Project) {
26
+ const config = configPath
27
+ ? ts.getParsedCommandLineOfConfigFile(
28
+ configPath,
29
+ {},
30
+ {
31
+ ...ts.sys,
32
+ onUnRecoverableConfigFileDiagnostic: (diagnostic) => {
33
+ throw new Error(ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n"));
34
+ },
35
+ },
36
+ )
37
+ : undefined;
38
+ if (config?.errors.length) {
39
+ throw new Error(
40
+ config.errors
41
+ .map((error) => ts.flattenDiagnosticMessageText(error.messageText, "\n"))
42
+ .join("\n"),
43
+ );
44
+ }
45
+ const options: ts.CompilerOptions = {
46
+ module: ts.ModuleKind.NodeNext,
47
+ allowJs: true,
48
+ ...config?.options,
49
+ noEmit: true,
50
+ };
51
+ const host = ts.createCompilerHost(options);
52
+ const key = registry.getKeyForCompilationSettings(options);
53
+ host.getSourceFile = (fileName, languageVersionOrOptions) => {
54
+ const text = host.readFile(fileName);
55
+ if (text === undefined) return undefined;
56
+ return registry.acquireDocumentWithKey(
57
+ fileName,
58
+ ts.sys.resolvePath(fileName) as ts.Path,
59
+ options,
60
+ key,
61
+ ts.ScriptSnapshot.fromString(text),
62
+ // Files do not change during one run.
63
+ "1",
64
+ undefined,
65
+ languageVersionOrOptions,
66
+ );
67
+ };
68
+ const program = ts.createProgram(
69
+ files.map(([file]) => NodePath.resolve(cwd, file)),
70
+ options,
71
+ host,
72
+ );
73
+ const checker = program.getTypeChecker();
74
+ return files.flatMap(([filePath, names]) => {
75
+ const source = program.getSourceFile(NodePath.resolve(cwd, filePath));
76
+ const module = source && checker.getSymbolAtLocation(source);
77
+ if (!source || !module) return [];
78
+ const schemas = checker
79
+ .getExportsOfModule(module)
80
+ .filter((symbol) => names.includes(symbol.name))
81
+ .flatMap((symbol) => {
82
+ const exported =
83
+ symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol;
84
+ const target =
85
+ exported.flags & ts.SymbolFlags.Value
86
+ ? exported
87
+ : checker.resolveName(symbol.name, source, ts.SymbolFlags.Value, false);
88
+ if (!target) return [];
89
+ const type = checker.getTypeOfSymbolAtLocation(target, target.valueDeclaration ?? source);
90
+ const marker = type.getProperty(schemaTypeId);
91
+ if (!marker) return [];
92
+ const markerType = checker.getTypeOfSymbolAtLocation(marker, source);
93
+ return markerType.isStringLiteral() && markerType.value === schemaTypeId
94
+ ? [symbol.name]
95
+ : [];
96
+ });
97
+ return [[filePath, schemas] as const];
98
+ });
99
+}
100
+
101
+// The preprocessor loads this file again as each worker.
102
+if (!NodeWorkerThreads.isMainThread) {
103
+ const projects: ReadonlyArray<Project> = NodeWorkerThreads.workerData;
104
+ NodeWorkerThreads.parentPort?.postMessage(projects.flatMap(findSchemas));
105
+}
106
+
107
+// Three workers measured fastest. Each one parses Effect again, so more
108
+// workers cost more than they save.
109
+const WORKERS = Math.min(3, NodeOS.availableParallelism());
110
+
111
+// Drops unused-export issues whose every symbol is an Effect schema.
112
+const preprocess = async (options: Parameters<Preprocessor>[0]) => {
113
const categories = ["exports", "nsExports", "duplicates"] as const;
11
- const projects = new Map<string | undefined, Set<string>>();
114
+ const projects = new Map<string | undefined, Map<string, Set<string>>>();
115
for (const category of categories) {
116
for (const [filePath, issues] of Object.entries(options.issues[category])) {
117
if (Object.keys(issues).length === 0) continue;
@@ -16,76 +119,60 @@ const preprocess: Preprocessor = (options) => {
119
NodePath.dirname(NodePath.resolve(options.cwd, filePath)),
120
ts.sys.fileExists,
121
);
19
- const files = projects.get(configPath) ?? new Set<string>();
20
- files.add(filePath);
122
+ const files = projects.get(configPath) ?? new Map<string, Set<string>>();
123
+ const names = files.get(filePath) ?? new Set<string>();
124
+ for (const issue of Object.values(issues)) {
125
+ for (const { symbol } of issue.symbols ?? [{ symbol: issue.symbol }]) names.add(symbol);
126
+ }
127
+ files.set(filePath, names);
128
projects.set(configPath, files);
129
}
130
}
131
25
- for (const [configPath, files] of projects) {
26
- const config = configPath
27
- ? ts.getParsedCommandLineOfConfigFile(
28
- configPath,
29
- {},
30
- {
31
- ...ts.sys,
32
- onUnRecoverableConfigFileDiagnostic: (diagnostic) => {
33
- throw new Error(ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n"));
34
- },
35
- },
36
- )
37
- : undefined;
38
- if (config?.errors.length) {
39
- throw new Error(
40
- config.errors
41
- .map((error) => ts.flattenDiagnosticMessageText(error.messageText, "\n"))
42
- .join("\n"),
43
- );
44
- }
45
- const program = ts.createProgram(
46
- [...files].map((file) => NodePath.resolve(options.cwd, file)),
47
- {
48
- module: ts.ModuleKind.NodeNext,
49
- allowJs: true,
50
- ...config?.options,
51
- noEmit: true,
52
- },
53
- );
54
- const checker = program.getTypeChecker();
55
- for (const filePath of files) {
56
- const source = program.getSourceFile(NodePath.resolve(options.cwd, filePath));
57
- const module = source && checker.getSymbolAtLocation(source);
58
- if (!source || !module) continue;
59
- const schemas = new Set(
60
- checker.getExportsOfModule(module).flatMap((symbol) => {
61
- const exported =
62
- symbol.flags & ts.SymbolFlags.Alias ? checker.getAliasedSymbol(symbol) : symbol;
63
- const target =
64
- exported.flags & ts.SymbolFlags.Value
65
- ? exported
66
- : checker.resolveName(symbol.name, source, ts.SymbolFlags.Value, false);
67
- if (!target) return [];
68
- const type = checker.getTypeOfSymbolAtLocation(target, target.valueDeclaration ?? source);
69
- const marker = type.getProperty(schemaTypeId);
70
- if (!marker) return [];
71
- const markerType = checker.getTypeOfSymbolAtLocation(marker, source);
72
- return markerType.isStringLiteral() && markerType.value === schemaTypeId
73
- ? [symbol.name]
74
- : [];
132
+ // Deal the projects out largest first so the workers get similar shares.
133
+ const shares = Array.from({ length: WORKERS }, (): Array<Project> => []);
134
+ [...projects]
135
+ .sort(([, a], [, b]) => b.size - a.size)
136
+ .forEach(([configPath, files], index) => {
137
+ shares[index % WORKERS]!.push({
138
+ cwd: options.cwd,
139
+ configPath,
140
+ files: [...files].map(([file, names]) => [file, [...names]]),
141
+ });
142
+ });
143
+ const workers = shares
144
+ .filter((share) => share.length > 0)
145
+ .map((share) => new NodeWorkerThreads.Worker(new URL(import.meta.url), { workerData: share }));
146
+ const results = await Promise.all(
147
+ workers.map(
148
+ (worker) =>
149
+ new Promise<ReturnType<typeof findSchemas>>((resolve, reject) => {
150
+ worker.once("message", resolve);
151
+ worker.once("error", reject);
152
+ worker.once("exit", (code) => {
153
+ if (code !== 0) reject(new Error(`knip-schemas worker exited with code ${code}`));
154
+ });
155
}),
76
- );
77
- for (const category of categories) {
78
- const issues = options.issues[category][filePath];
79
- if (!issues) continue;
80
- for (const [key, issue] of Object.entries(issues)) {
81
- const symbols = issue.symbols ?? [{ symbol: issue.symbol }];
82
- if (symbols.length > 0 && symbols.every(({ symbol }) => schemas.has(symbol))) {
83
- delete issues[key];
84
- options.counters[category]--;
85
- }
156
+ ),
157
+ ).catch(async (error: unknown) => {
158
+ // knip only sets an exit code on a failed preprocessor, so stop the
159
+ // siblings or the failed run waits for their work.
160
+ await Promise.allSettled(workers.map((worker) => worker.terminate()));
161
+ throw error;
162
+ });
163
+
164
+ for (const [filePath, names] of results.flat()) {
165
+ for (const category of categories) {
166
+ const issues = options.issues[category][filePath];
167
+ if (!issues) continue;
168
+ for (const [key, issue] of Object.entries(issues)) {
169
+ const symbols = issue.symbols ?? [{ symbol: issue.symbol }];
170
+ if (symbols.length > 0 && symbols.every(({ symbol }) => names.includes(symbol))) {
171
+ delete issues[key];
172
+ options.counters[category]--;
173
}
87
- if (Object.keys(issues).length === 0) delete options.issues[category][filePath];
174
}
175
+ if (Object.keys(issues).length === 0) delete options.issues[category][filePath];
176
}
177
}
178
return options;