@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 382 lines 15.2 KB
Raw
1 #!/usr/bin/env node
2 // Runs one scenario with `e2e run` on a base and a head commit and says whether it fails on
3 // base and passes on head. See docs/operations/scenarios.md.
4 import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
5 import * as NodeServices from "@effect/platform-node/NodeServices";
6 import * as HostProcess from "@t3tools/shared/HostProcess";
7 import { fromJsonStringPretty } from "@t3tools/shared/schemaJson";
8 import * as Console from "effect/Console";
9 import * as Effect from "effect/Effect";
10 import * as FileSystem from "effect/FileSystem";
11 import * as Option from "effect/Option";
12 import * as Path from "effect/Path";
13 import * as Schema from "effect/Schema";
14 import { Argument, Command, Flag } from "effect/cli";
15
16 import { run, ScenarioError } from "./scenarios/shell.ts";
17
18 /** The parts of e2e's report.json (schema/report-v1.schema.json) a proof reads. */
19 const E2EReport = Schema.Struct({
20 run: Schema.Struct({
21 results: Schema.Array(
22 Schema.Struct({
23 titlePath: Schema.Array(Schema.String),
24 attempts: Schema.Array(
25 Schema.Struct({
26 status: Schema.optional(Schema.String),
27 steps: Schema.Array(
28 Schema.Struct({
29 api: Schema.String,
30 label: Schema.optional(Schema.String),
31 status: Schema.String,
32 }),
33 ),
34 error: Schema.optional(Schema.Struct({ code: Schema.String, message: Schema.String })),
35 artifacts: Schema.optional(
36 Schema.Array(
37 Schema.Struct({ kind: Schema.String, path: Schema.optional(Schema.String) }),
38 ),
39 ),
40 }),
41 ),
42 }),
43 ),
44 }),
45 });
46 type E2EReport = typeof E2EReport.Type;
47 const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(E2EReport));
48 const encodeProof = Schema.encodeEffect(fromJsonStringPretty(Schema.Unknown));
49
50 export interface SideResult {
51 readonly sha: string;
52 /** `error`: the scenario could not run, which says nothing about the behavior. */
53 readonly status: "passed" | "failed" | "error";
54 readonly title: string;
55 readonly message?: string;
56 /**
57 * For `error`, what stopped: `app` when the app did not start or no test ran, `scenario` when
58 * the app ran and the scenario stopped before its assertion, such as on a control it did not find.
59 */
60 readonly stopped?: "app" | "scenario";
61 readonly steps: ReadonlyArray<{ readonly step: string; readonly status: string }>;
62 /** Capture paths under the run's results directory, in the order the scenario took them. */
63 readonly captures: ReadonlyArray<string>;
64 readonly videos: ReadonlyArray<string>;
65 }
66
67 /** Only a failed expectation shows the bug. A missing control or a dead app shows nothing. */
68 export function readSide(
69 sha: string,
70 resultsDir: string,
71 report: E2EReport | undefined,
72 ): SideResult {
73 // The report also lists the tests the run did not select, with no attempts.
74 const executed = report?.run.results.filter((candidate) => candidate.attempts.length > 0) ?? [];
75 // One scenario is one test: with several, a pass in the first could hide a failure in another.
76 const result = executed.length === 1 ? executed[0] : undefined;
77 const attempt = result?.attempts.at(-1);
78 if (result === undefined || attempt === undefined) {
79 const message =
80 executed.length > 1
81 ? `the scenario file ran ${executed.length} tests; a proof needs exactly one`
82 : "e2e wrote no report: the app did not start, or no test was selected";
83 return {
84 sha,
85 status: "error",
86 stopped: "app",
87 title: "",
88 message,
89 steps: [],
90 captures: [],
91 videos: [],
92 };
93 }
94 const files = (kind: string) =>
95 (attempt.artifacts ?? []).flatMap((artifact) =>
96 artifact.kind === kind && artifact.path !== undefined
97 ? [`${resultsDir}/${artifact.path}`]
98 : [],
99 );
100 const error = attempt.error;
101 // e2e exits 0 for a skipped test too, and a test that did not run proves nothing.
102 const status =
103 attempt.status === "passed"
104 ? "passed"
105 : error?.code === "ASSERTION_FAILED"
106 ? "failed"
107 : "error";
108 return {
109 sha,
110 status,
111 // A test that ended with an error of its own got as far as the app.
112 ...(status === "error" ? { stopped: error === undefined ? "app" : "scenario" } : {}),
113 title: result.titlePath.join(" > "),
114 ...(error === undefined
115 ? attempt.status === "passed"
116 ? {}
117 : { message: `the test ended ${attempt.status ?? "without a status"}` }
118 : { message: `${error.code}: ${error.message}` }),
119 steps: attempt.steps.map((step) => ({
120 // The sign-in step's label holds the run's pairing token.
121 step: `${step.api} ${step.label?.includes("token=") ? "<pairing link>" : (step.label ?? "")}`,
122 status: step.status,
123 })),
124 captures: files("screenshot"),
125 videos: files("video"),
126 };
127 }
128
129 const short = (sha: string) => sha.slice(0, 10);
130 const captureLabel = (path: string) =>
131 (path.split("/").at(-1) ?? "")
132 .replace(/\.png$/, "")
133 .replace(/^\d+-/, "")
134 .replaceAll("-", " ");
135
136 /** `link` maps a capture path to what the Markdown should point at, such as an uploaded URL. */
137 export function buildProof(base: SideResult, head: SideResult, link: (path: string) => string) {
138 const baseSha = `\`${short(base.sha)}\``;
139 const headSha = `\`${short(head.sha)}\``;
140 const broken = base.status === "error" ? base : head.status === "error" ? head : undefined;
141 const verdict =
142 broken !== undefined
143 ? `Not proven: the scenario could not run on ${broken === base ? `base ${baseSha}` : `head ${headSha}`} (${broken.message ?? "unknown error"}).${broken.stopped === "scenario" ? " The app ran and the scenario stopped before its assertion: fix the scenario and run it again." : ""}`
144 : base.status === "passed"
145 ? head.status === "passed"
146 ? `Not proven: the scenario passes on base ${baseSha} too, so it does not reproduce the bug.`
147 : `Not proven: the scenario passes on base ${baseSha} and fails on head ${headSha}, so head breaks it.`
148 : head.status === "passed"
149 ? `Proven: the scenario fails on base ${baseSha} and passes on head ${headSha}.`
150 : `Not proven: the scenario fails on base ${baseSha} and still fails on head ${headSha}.`;
151
152 const labels = [...new Set([...base.captures, ...head.captures].map(captureLabel))];
153 const cell = (side: string, result: SideResult, label: string) => {
154 const capture = result.captures.find((path) => captureLabel(path) === label);
155 return capture === undefined
156 ? ""
157 : `![${side} ${short(result.sha)}: ${label}](${link(capture)})`;
158 };
159 const outcome = (result: SideResult) =>
160 result.status === "passed"
161 ? "✅ passed"
162 : `${result.status === "failed" ? "❌ failed" : "⚠️ could not run"}<br>${(result.message ?? "").replaceAll("\n", "<br>").replaceAll("|", "\\|").replaceAll("<br><br>", "<br>")}`;
163 const markdown = [
164 `### Scenario: ${head.title || base.title}`,
165 "",
166 `**${verdict}**`,
167 "",
168 `| | Base ${baseSha} | Head ${headSha} |`,
169 "| --- | --- | --- |",
170 `| Result | ${outcome(base)} | ${outcome(head)} |`,
171 ...labels.map(
172 (label) => `| ${label} | ${cell("Base", base, label)} | ${cell("Head", head, label)} |`,
173 ),
174 "",
175 ].join("\n");
176 return {
177 scenario: head.title || base.title,
178 failsOnBase: base.status === "failed",
179 passesOnHead: head.status === "passed",
180 /** Why there is no verdict on the behavior: the app did not run, or the scenario stopped. */
181 couldNotRun: broken?.stopped ?? null,
182 baseSha: base.sha,
183 headSha: head.sha,
184 verdict,
185 steps: { base: base.steps, head: head.steps },
186 captures: [...base.captures, ...head.captures],
187 videos: [...base.videos, ...head.videos],
188 markdown,
189 base,
190 head,
191 };
192 }
193
194 const roots = Effect.gen(function* () {
195 const path = yield* Path.Path;
196 const scripts = yield* path.fromFileUrl(new URL(".", import.meta.url));
197 return { scripts, repo: path.dirname(path.resolve(scripts)) };
198 });
199
200 const git = Effect.fn("prove.git")(function* (args: ReadonlyArray<string>) {
201 return (yield* run("git", args, { cwd: (yield* roots).repo })).stdout.trim();
202 });
203
204 /** Runs `use` with a checkout of `sha`: this one when it already is that commit, else a worktree. */
205 const withCheckout = Effect.fn("prove.withCheckout")(function* <A, E, R>(
206 sha: string,
207 use: (appDir: string) => Effect.Effect<A, E, R>,
208 ) {
209 const { repo } = yield* roots;
210 const clean = (yield* git(["status", "--porcelain", "--untracked-files=no"])).length === 0;
211 if (clean && (yield* git(["rev-parse", "HEAD"])) === sha) return yield* use(repo);
212 // Beside this checkout, not in the temp directory, so tooling resolves as it does here.
213 const worktree = `${repo}-scenario-${short(sha)}`;
214 // A run that was killed left its worktree behind, and git refuses to add over it.
215 const discard = git(["worktree", "remove", "--force", worktree]).pipe(
216 Effect.ignore,
217 Effect.andThen(git(["worktree", "prune"])),
218 Effect.ignore,
219 );
220 return yield* Effect.acquireUseRelease(
221 discard.pipe(
222 Effect.andThen(git(["worktree", "add", "--quiet", "--detach", "--force", worktree, sha])),
223 ),
224 () => run("vp", ["i"], { cwd: worktree, echo: true }).pipe(Effect.andThen(use(worktree))),
225 () => discard,
226 );
227 });
228
229 /** Runs the scenario with `e2e run` against the app at `sha` and reads the report it wrote. */
230 const runSide = Effect.fn("prove.runSide")(function* (
231 side: "base" | "head",
232 rev: string,
233 scenarioPath: string,
234 target: string,
235 ) {
236 const fs = yield* FileSystem.FileSystem;
237 const path = yield* Path.Path;
238 const { scripts } = yield* roots;
239 const sha = yield* git(["rev-parse", "--verify", `${rev}^{commit}`]);
240 const outDir = path.join(scripts, ".e2e/prove", side);
241 yield* fs.remove(outDir, { recursive: true, force: true });
242 const environment = yield* HostProcess.Environment;
243 yield* withCheckout(sha, (appDir) =>
244 // The scenario and e2e come from this checkout; only the app comes from `appDir`.
245 run(
246 path.join(scripts, "node_modules/.bin/e2e"),
247 ["run", scenarioPath, "--target", target, "--output", path.relative(scripts, outDir)],
248 {
249 cwd: scripts,
250 env: { ...environment, T3_SCENARIO_APP_DIR: appDir, E2E_TELEMETRY_DISABLED: "1" },
251 // The report says how the run went; e2e's exit code only repeats it.
252 allowExit: [0, 1, 2, 3, 4],
253 echo: true,
254 },
255 ),
256 );
257 const report = yield* fs.readFileString(path.join(outDir, "report.json")).pipe(
258 Effect.flatMap(decodeReport),
259 Effect.orElseSucceed(() => undefined),
260 );
261 const result = readSide(sha, path.join(outDir, "results"), report);
262 // Written as each side ends, so a proof whose second side is cut short still has its first.
263 yield* fs
264 .writeFileString(
265 path.join(scripts, ".e2e/prove", `${side}.json`),
266 `${yield* encodeProof(result)}\n`,
267 )
268 .pipe(Effect.ignore);
269 return result;
270 });
271
272 /** The one target a scenario runs on: an iOS scenario imports `t3-mobile.ts`, a web one does not. */
273 const targetOf = Effect.fn("prove.targetOf")(function* (scenarioPath: string) {
274 const fs = yield* FileSystem.FileSystem;
275 const source = yield* fs.readFileString(scenarioPath).pipe(Effect.orElseSucceed(() => ""));
276 return source.includes("t3-mobile") ? "ios" : "web";
277 });
278
279 /** Uploads the captures and maps each to its URL; a failed upload leaves the map empty. */
280 const upload = Effect.fn("prove.upload")(function* (
281 uploadCommand: string,
282 captures: ReadonlyArray<string>,
283 ) {
284 const [command = "", ...args] = uploadCommand.split(/\s+/).filter(Boolean);
285 if (captures.length === 0) return new Map<string, string>();
286 return yield* run(command, [...args, ...captures]).pipe(
287 Effect.map((result) => result.stdout.trim().split("\n")),
288 // Lines pair with captures by position, so a different count would mislabel every image.
289 Effect.filterOrFail(
290 (urls) => urls.length === captures.length,
291 () => new ScenarioError({ message: "The upload command printed an unexpected line count." }),
292 ),
293 Effect.map((urls) => new Map(captures.map((capture, index) => [capture, urls[index] ?? ""]))),
294 // Two finished runs are worth more than their links: the proof then names local files.
295 Effect.catch((error) =>
296 Console.error(`Upload failed, linking local captures: ${error.message}`).pipe(
297 Effect.as(new Map<string, string>()),
298 ),
299 ),
300 );
301 });
302
303 const prove = Effect.fn("prove")(function* (input: {
304 readonly scenario: string;
305 readonly base: string;
306 readonly head: string;
307 readonly uploadCommand: Option.Option<string>;
308 readonly target: Option.Option<string>;
309 }) {
310 const fs = yield* FileSystem.FileSystem;
311 const path = yield* Path.Path;
312 const { scripts } = yield* roots;
313 const proveDir = path.join(scripts, ".e2e/prove");
314 const scenarioPath = path.resolve(yield* HostProcess.WorkingDirectory, input.scenario);
315 // A proof left by an earlier run must not be read as this one's.
316 yield* fs.remove(proveDir, { recursive: true, force: true });
317 yield* fs.makeDirectory(proveDir, { recursive: true });
318 // Only that target's app starts: a web scenario needs no Metro and no device.
319 const target = Option.getOrElse(input.target, () => "") || (yield* targetOf(scenarioPath));
320 const base = yield* runSide("base", input.base, scenarioPath, target);
321 const head = yield* runSide("head", input.head, scenarioPath, target);
322 const links = Option.isSome(input.uploadCommand)
323 ? yield* upload(input.uploadCommand.value, [...base.captures, ...head.captures])
324 : new Map<string, string>();
325 const proof = buildProof(
326 base,
327 head,
328 (capture) => links.get(capture) ?? path.relative(proveDir, capture),
329 );
330 const json = yield* encodeProof(proof);
331 yield* fs.writeFileString(path.join(proveDir, "proof.md"), proof.markdown);
332 yield* fs.writeFileString(path.join(proveDir, "proof.json"), `${json}\n`);
333 yield* Console.log(json);
334 return proof.failsOnBase && proof.passesOnHead
335 ? 0
336 : proof.couldNotRun === "scenario"
337 ? 3
338 : proof.couldNotRun === "app"
339 ? 2
340 : 1;
341 });
342
343 const command = Command.make(
344 "prove-scenario",
345 {
346 scenario: Argument.String("scenario").pipe(
347 Argument.withDescription("Scenario file, such as scripts/scenarios/name.e2e.ts."),
348 ),
349 base: Flag.String("base").pipe(Flag.withDescription("Commit the bug shows on.")),
350 head: Flag.String("head").pipe(
351 Flag.withDescription("Commit with the fix."),
352 Flag.withDefault("HEAD"),
353 ),
354 uploadCommand: Flag.String("upload-command").pipe(
355 Flag.withDescription("Command that takes capture paths and prints one URL per line."),
356 Flag.optional,
357 ),
358 target: Flag.String("target").pipe(
359 Flag.withDescription(
360 "Target to run on: web or ios. Default: ios when the scenario imports t3-mobile.ts, else web.",
361 ),
362 Flag.optional,
363 ),
364 },
365 (input) =>
366 prove(input).pipe(
367 // A bad revision or a failed install is "could not run" (2), not "did not prove" (1).
368 Effect.catch((error) => Console.error(String(error)).pipe(Effect.as(2))),
369 Effect.flatMap((exitCode) =>
370 Effect.sync(() => {
371 process.exitCode = exitCode;
372 }),
373 ),
374 ),
375 );
376
377 if (import.meta.main) {
378 Command.run(command, { version: "0.0.0" }).pipe(
379 Effect.provide(NodeServices.layer),
380 NodeRuntime.runMain,
381 );
382 }