The best way to control Claude Code, Codex and more from any computer or phone
| 1 | #!/usr/bin/env node |
| 2 | // Runs one scenario with `e2e run` on a base and a head commit and says whether it fails on |
| 3 | // base and passes on head. See docs/operations/scenarios.md. |
| 4 | import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; |
| 5 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 6 | import * as HostProcess from "@t3tools/shared/HostProcess"; |
| 7 | import { fromJsonStringPretty } from "@t3tools/shared/schemaJson"; |
| 8 | import * as Console from "effect/Console"; |
| 9 | import * as Effect from "effect/Effect"; |
| 10 | import * as FileSystem from "effect/FileSystem"; |
| 11 | import * as Option from "effect/Option"; |
| 12 | import * as Path from "effect/Path"; |
| 13 | import * as Schema from "effect/Schema"; |
| 14 | import { Argument, Command, Flag } from "effect/cli"; |
| 15 | |
| 16 | import { run, ScenarioError } from "./scenarios/shell.ts"; |
| 17 | |
| 18 | /** The parts of e2e's report.json (schema/report-v1.schema.json) a proof reads. */ |
| 19 | const E2EReport = Schema.Struct({ |
| 20 | run: Schema.Struct({ |
| 21 | results: Schema.Array( |
| 22 | Schema.Struct({ |
| 23 | titlePath: Schema.Array(Schema.String), |
| 24 | attempts: Schema.Array( |
| 25 | Schema.Struct({ |
| 26 | status: Schema.optional(Schema.String), |
| 27 | steps: Schema.Array( |
| 28 | Schema.Struct({ |
| 29 | api: Schema.String, |
| 30 | label: Schema.optional(Schema.String), |
| 31 | status: Schema.String, |
| 32 | }), |
| 33 | ), |
| 34 | error: Schema.optional(Schema.Struct({ code: Schema.String, message: Schema.String })), |
| 35 | artifacts: Schema.optional( |
| 36 | Schema.Array( |
| 37 | Schema.Struct({ kind: Schema.String, path: Schema.optional(Schema.String) }), |
| 38 | ), |
| 39 | ), |
| 40 | }), |
| 41 | ), |
| 42 | }), |
| 43 | ), |
| 44 | }), |
| 45 | }); |
| 46 | type E2EReport = typeof E2EReport.Type; |
| 47 | const decodeReport = Schema.decodeUnknownEffect(Schema.fromJsonString(E2EReport)); |
| 48 | const encodeProof = Schema.encodeEffect(fromJsonStringPretty(Schema.Unknown)); |
| 49 | |
| 50 | export interface SideResult { |
| 51 | readonly sha: string; |
| 52 | /** `error`: the scenario could not run, which says nothing about the behavior. */ |
| 53 | readonly status: "passed" | "failed" | "error"; |
| 54 | readonly title: string; |
| 55 | readonly message?: string; |
| 56 | /** |
| 57 | * For `error`, what stopped: `app` when the app did not start or no test ran, `scenario` when |
| 58 | * the app ran and the scenario stopped before its assertion, such as on a control it did not find. |
| 59 | */ |
| 60 | readonly stopped?: "app" | "scenario"; |
| 61 | readonly steps: ReadonlyArray<{ readonly step: string; readonly status: string }>; |
| 62 | /** Capture paths under the run's results directory, in the order the scenario took them. */ |
| 63 | readonly captures: ReadonlyArray<string>; |
| 64 | readonly videos: ReadonlyArray<string>; |
| 65 | } |
| 66 | |
| 67 | /** Only a failed expectation shows the bug. A missing control or a dead app shows nothing. */ |
| 68 | export function readSide( |
| 69 | sha: string, |
| 70 | resultsDir: string, |
| 71 | report: E2EReport | undefined, |
| 72 | ): SideResult { |
| 73 | // The report also lists the tests the run did not select, with no attempts. |
| 74 | const executed = report?.run.results.filter((candidate) => candidate.attempts.length > 0) ?? []; |
| 75 | // One scenario is one test: with several, a pass in the first could hide a failure in another. |
| 76 | const result = executed.length === 1 ? executed[0] : undefined; |
| 77 | const attempt = result?.attempts.at(-1); |
| 78 | if (result === undefined || attempt === undefined) { |
| 79 | const message = |
| 80 | executed.length > 1 |
| 81 | ? `the scenario file ran ${executed.length} tests; a proof needs exactly one` |
| 82 | : "e2e wrote no report: the app did not start, or no test was selected"; |
| 83 | return { |
| 84 | sha, |
| 85 | status: "error", |
| 86 | stopped: "app", |
| 87 | title: "", |
| 88 | message, |
| 89 | steps: [], |
| 90 | captures: [], |
| 91 | videos: [], |
| 92 | }; |
| 93 | } |
| 94 | const files = (kind: string) => |
| 95 | (attempt.artifacts ?? []).flatMap((artifact) => |
| 96 | artifact.kind === kind && artifact.path !== undefined |
| 97 | ? [`${resultsDir}/${artifact.path}`] |
| 98 | : [], |
| 99 | ); |
| 100 | const error = attempt.error; |
| 101 | // e2e exits 0 for a skipped test too, and a test that did not run proves nothing. |
| 102 | const status = |
| 103 | attempt.status === "passed" |
| 104 | ? "passed" |
| 105 | : error?.code === "ASSERTION_FAILED" |
| 106 | ? "failed" |
| 107 | : "error"; |
| 108 | return { |
| 109 | sha, |
| 110 | status, |
| 111 | // A test that ended with an error of its own got as far as the app. |
| 112 | ...(status === "error" ? { stopped: error === undefined ? "app" : "scenario" } : {}), |
| 113 | title: result.titlePath.join(" > "), |
| 114 | ...(error === undefined |
| 115 | ? attempt.status === "passed" |
| 116 | ? {} |
| 117 | : { message: `the test ended ${attempt.status ?? "without a status"}` } |
| 118 | : { message: `${error.code}: ${error.message}` }), |
| 119 | steps: attempt.steps.map((step) => ({ |
| 120 | // The sign-in step's label holds the run's pairing token. |
| 121 | step: `${step.api} ${step.label?.includes("token=") ? "<pairing link>" : (step.label ?? "")}`, |
| 122 | status: step.status, |
| 123 | })), |
| 124 | captures: files("screenshot"), |
| 125 | videos: files("video"), |
| 126 | }; |
| 127 | } |
| 128 | |
| 129 | const short = (sha: string) => sha.slice(0, 10); |
| 130 | const captureLabel = (path: string) => |
| 131 | (path.split("/").at(-1) ?? "") |
| 132 | .replace(/\.png$/, "") |
| 133 | .replace(/^\d+-/, "") |
| 134 | .replaceAll("-", " "); |
| 135 | |
| 136 | /** `link` maps a capture path to what the Markdown should point at, such as an uploaded URL. */ |
| 137 | export function buildProof(base: SideResult, head: SideResult, link: (path: string) => string) { |
| 138 | const baseSha = `\`${short(base.sha)}\``; |
| 139 | const headSha = `\`${short(head.sha)}\``; |
| 140 | const broken = base.status === "error" ? base : head.status === "error" ? head : undefined; |
| 141 | const verdict = |
| 142 | broken !== undefined |
| 143 | ? `Not proven: the scenario could not run on ${broken === base ? `base ${baseSha}` : `head ${headSha}`} (${broken.message ?? "unknown error"}).${broken.stopped === "scenario" ? " The app ran and the scenario stopped before its assertion: fix the scenario and run it again." : ""}` |
| 144 | : base.status === "passed" |
| 145 | ? head.status === "passed" |
| 146 | ? `Not proven: the scenario passes on base ${baseSha} too, so it does not reproduce the bug.` |
| 147 | : `Not proven: the scenario passes on base ${baseSha} and fails on head ${headSha}, so head breaks it.` |
| 148 | : head.status === "passed" |
| 149 | ? `Proven: the scenario fails on base ${baseSha} and passes on head ${headSha}.` |
| 150 | : `Not proven: the scenario fails on base ${baseSha} and still fails on head ${headSha}.`; |
| 151 | |
| 152 | const labels = [...new Set([...base.captures, ...head.captures].map(captureLabel))]; |
| 153 | const cell = (side: string, result: SideResult, label: string) => { |
| 154 | const capture = result.captures.find((path) => captureLabel(path) === label); |
| 155 | return capture === undefined |
| 156 | ? "" |
| 157 | : `})`; |
| 158 | }; |
| 159 | const outcome = (result: SideResult) => |
| 160 | result.status === "passed" |
| 161 | ? "✅ passed" |
| 162 | : `${result.status === "failed" ? "❌ failed" : "⚠️ could not run"}<br>${(result.message ?? "").replaceAll("\n", "<br>").replaceAll("|", "\\|").replaceAll("<br><br>", "<br>")}`; |
| 163 | const markdown = [ |
| 164 | `### Scenario: ${head.title || base.title}`, |
| 165 | "", |
| 166 | `**${verdict}**`, |
| 167 | "", |
| 168 | `| | Base ${baseSha} | Head ${headSha} |`, |
| 169 | "| --- | --- | --- |", |
| 170 | `| Result | ${outcome(base)} | ${outcome(head)} |`, |
| 171 | ...labels.map( |
| 172 | (label) => `| ${label} | ${cell("Base", base, label)} | ${cell("Head", head, label)} |`, |
| 173 | ), |
| 174 | "", |
| 175 | ].join("\n"); |
| 176 | return { |
| 177 | scenario: head.title || base.title, |
| 178 | failsOnBase: base.status === "failed", |
| 179 | passesOnHead: head.status === "passed", |
| 180 | /** Why there is no verdict on the behavior: the app did not run, or the scenario stopped. */ |
| 181 | couldNotRun: broken?.stopped ?? null, |
| 182 | baseSha: base.sha, |
| 183 | headSha: head.sha, |
| 184 | verdict, |
| 185 | steps: { base: base.steps, head: head.steps }, |
| 186 | captures: [...base.captures, ...head.captures], |
| 187 | videos: [...base.videos, ...head.videos], |
| 188 | markdown, |
| 189 | base, |
| 190 | head, |
| 191 | }; |
| 192 | } |
| 193 | |
| 194 | const roots = Effect.gen(function* () { |
| 195 | const path = yield* Path.Path; |
| 196 | const scripts = yield* path.fromFileUrl(new URL(".", import.meta.url)); |
| 197 | return { scripts, repo: path.dirname(path.resolve(scripts)) }; |
| 198 | }); |
| 199 | |
| 200 | const git = Effect.fn("prove.git")(function* (args: ReadonlyArray<string>) { |
| 201 | return (yield* run("git", args, { cwd: (yield* roots).repo })).stdout.trim(); |
| 202 | }); |
| 203 | |
| 204 | /** Runs `use` with a checkout of `sha`: this one when it already is that commit, else a worktree. */ |
| 205 | const withCheckout = Effect.fn("prove.withCheckout")(function* <A, E, R>( |
| 206 | sha: string, |
| 207 | use: (appDir: string) => Effect.Effect<A, E, R>, |
| 208 | ) { |
| 209 | const { repo } = yield* roots; |
| 210 | const clean = (yield* git(["status", "--porcelain", "--untracked-files=no"])).length === 0; |
| 211 | if (clean && (yield* git(["rev-parse", "HEAD"])) === sha) return yield* use(repo); |
| 212 | // Beside this checkout, not in the temp directory, so tooling resolves as it does here. |
| 213 | const worktree = `${repo}-scenario-${short(sha)}`; |
| 214 | // A run that was killed left its worktree behind, and git refuses to add over it. |
| 215 | const discard = git(["worktree", "remove", "--force", worktree]).pipe( |
| 216 | Effect.ignore, |
| 217 | Effect.andThen(git(["worktree", "prune"])), |
| 218 | Effect.ignore, |
| 219 | ); |
| 220 | return yield* Effect.acquireUseRelease( |
| 221 | discard.pipe( |
| 222 | Effect.andThen(git(["worktree", "add", "--quiet", "--detach", "--force", worktree, sha])), |
| 223 | ), |
| 224 | () => run("vp", ["i"], { cwd: worktree, echo: true }).pipe(Effect.andThen(use(worktree))), |
| 225 | () => discard, |
| 226 | ); |
| 227 | }); |
| 228 | |
| 229 | /** Runs the scenario with `e2e run` against the app at `sha` and reads the report it wrote. */ |
| 230 | const runSide = Effect.fn("prove.runSide")(function* ( |
| 231 | side: "base" | "head", |
| 232 | rev: string, |
| 233 | scenarioPath: string, |
| 234 | target: string, |
| 235 | ) { |
| 236 | const fs = yield* FileSystem.FileSystem; |
| 237 | const path = yield* Path.Path; |
| 238 | const { scripts } = yield* roots; |
| 239 | const sha = yield* git(["rev-parse", "--verify", `${rev}^{commit}`]); |
| 240 | const outDir = path.join(scripts, ".e2e/prove", side); |
| 241 | yield* fs.remove(outDir, { recursive: true, force: true }); |
| 242 | const environment = yield* HostProcess.Environment; |
| 243 | yield* withCheckout(sha, (appDir) => |
| 244 | // The scenario and e2e come from this checkout; only the app comes from `appDir`. |
| 245 | run( |
| 246 | path.join(scripts, "node_modules/.bin/e2e"), |
| 247 | ["run", scenarioPath, "--target", target, "--output", path.relative(scripts, outDir)], |
| 248 | { |
| 249 | cwd: scripts, |
| 250 | env: { ...environment, T3_SCENARIO_APP_DIR: appDir, E2E_TELEMETRY_DISABLED: "1" }, |
| 251 | // The report says how the run went; e2e's exit code only repeats it. |
| 252 | allowExit: [0, 1, 2, 3, 4], |
| 253 | echo: true, |
| 254 | }, |
| 255 | ), |
| 256 | ); |
| 257 | const report = yield* fs.readFileString(path.join(outDir, "report.json")).pipe( |
| 258 | Effect.flatMap(decodeReport), |
| 259 | Effect.orElseSucceed(() => undefined), |
| 260 | ); |
| 261 | const result = readSide(sha, path.join(outDir, "results"), report); |
| 262 | // Written as each side ends, so a proof whose second side is cut short still has its first. |
| 263 | yield* fs |
| 264 | .writeFileString( |
| 265 | path.join(scripts, ".e2e/prove", `${side}.json`), |
| 266 | `${yield* encodeProof(result)}\n`, |
| 267 | ) |
| 268 | .pipe(Effect.ignore); |
| 269 | return result; |
| 270 | }); |
| 271 | |
| 272 | /** The one target a scenario runs on: an iOS scenario imports `t3-mobile.ts`, a web one does not. */ |
| 273 | const targetOf = Effect.fn("prove.targetOf")(function* (scenarioPath: string) { |
| 274 | const fs = yield* FileSystem.FileSystem; |
| 275 | const source = yield* fs.readFileString(scenarioPath).pipe(Effect.orElseSucceed(() => "")); |
| 276 | return source.includes("t3-mobile") ? "ios" : "web"; |
| 277 | }); |
| 278 | |
| 279 | /** Uploads the captures and maps each to its URL; a failed upload leaves the map empty. */ |
| 280 | const upload = Effect.fn("prove.upload")(function* ( |
| 281 | uploadCommand: string, |
| 282 | captures: ReadonlyArray<string>, |
| 283 | ) { |
| 284 | const [command = "", ...args] = uploadCommand.split(/\s+/).filter(Boolean); |
| 285 | if (captures.length === 0) return new Map<string, string>(); |
| 286 | return yield* run(command, [...args, ...captures]).pipe( |
| 287 | Effect.map((result) => result.stdout.trim().split("\n")), |
| 288 | // Lines pair with captures by position, so a different count would mislabel every image. |
| 289 | Effect.filterOrFail( |
| 290 | (urls) => urls.length === captures.length, |
| 291 | () => new ScenarioError({ message: "The upload command printed an unexpected line count." }), |
| 292 | ), |
| 293 | Effect.map((urls) => new Map(captures.map((capture, index) => [capture, urls[index] ?? ""]))), |
| 294 | // Two finished runs are worth more than their links: the proof then names local files. |
| 295 | Effect.catch((error) => |
| 296 | Console.error(`Upload failed, linking local captures: ${error.message}`).pipe( |
| 297 | Effect.as(new Map<string, string>()), |
| 298 | ), |
| 299 | ), |
| 300 | ); |
| 301 | }); |
| 302 | |
| 303 | const prove = Effect.fn("prove")(function* (input: { |
| 304 | readonly scenario: string; |
| 305 | readonly base: string; |
| 306 | readonly head: string; |
| 307 | readonly uploadCommand: Option.Option<string>; |
| 308 | readonly target: Option.Option<string>; |
| 309 | }) { |
| 310 | const fs = yield* FileSystem.FileSystem; |
| 311 | const path = yield* Path.Path; |
| 312 | const { scripts } = yield* roots; |
| 313 | const proveDir = path.join(scripts, ".e2e/prove"); |
| 314 | const scenarioPath = path.resolve(yield* HostProcess.WorkingDirectory, input.scenario); |
| 315 | // A proof left by an earlier run must not be read as this one's. |
| 316 | yield* fs.remove(proveDir, { recursive: true, force: true }); |
| 317 | yield* fs.makeDirectory(proveDir, { recursive: true }); |
| 318 | // Only that target's app starts: a web scenario needs no Metro and no device. |
| 319 | const target = Option.getOrElse(input.target, () => "") || (yield* targetOf(scenarioPath)); |
| 320 | const base = yield* runSide("base", input.base, scenarioPath, target); |
| 321 | const head = yield* runSide("head", input.head, scenarioPath, target); |
| 322 | const links = Option.isSome(input.uploadCommand) |
| 323 | ? yield* upload(input.uploadCommand.value, [...base.captures, ...head.captures]) |
| 324 | : new Map<string, string>(); |
| 325 | const proof = buildProof( |
| 326 | base, |
| 327 | head, |
| 328 | (capture) => links.get(capture) ?? path.relative(proveDir, capture), |
| 329 | ); |
| 330 | const json = yield* encodeProof(proof); |
| 331 | yield* fs.writeFileString(path.join(proveDir, "proof.md"), proof.markdown); |
| 332 | yield* fs.writeFileString(path.join(proveDir, "proof.json"), `${json}\n`); |
| 333 | yield* Console.log(json); |
| 334 | return proof.failsOnBase && proof.passesOnHead |
| 335 | ? 0 |
| 336 | : proof.couldNotRun === "scenario" |
| 337 | ? 3 |
| 338 | : proof.couldNotRun === "app" |
| 339 | ? 2 |
| 340 | : 1; |
| 341 | }); |
| 342 | |
| 343 | const command = Command.make( |
| 344 | "prove-scenario", |
| 345 | { |
| 346 | scenario: Argument.String("scenario").pipe( |
| 347 | Argument.withDescription("Scenario file, such as scripts/scenarios/name.e2e.ts."), |
| 348 | ), |
| 349 | base: Flag.String("base").pipe(Flag.withDescription("Commit the bug shows on.")), |
| 350 | head: Flag.String("head").pipe( |
| 351 | Flag.withDescription("Commit with the fix."), |
| 352 | Flag.withDefault("HEAD"), |
| 353 | ), |
| 354 | uploadCommand: Flag.String("upload-command").pipe( |
| 355 | Flag.withDescription("Command that takes capture paths and prints one URL per line."), |
| 356 | Flag.optional, |
| 357 | ), |
| 358 | target: Flag.String("target").pipe( |
| 359 | Flag.withDescription( |
| 360 | "Target to run on: web or ios. Default: ios when the scenario imports t3-mobile.ts, else web.", |
| 361 | ), |
| 362 | Flag.optional, |
| 363 | ), |
| 364 | }, |
| 365 | (input) => |
| 366 | prove(input).pipe( |
| 367 | // A bad revision or a failed install is "could not run" (2), not "did not prove" (1). |
| 368 | Effect.catch((error) => Console.error(String(error)).pipe(Effect.as(2))), |
| 369 | Effect.flatMap((exitCode) => |
| 370 | Effect.sync(() => { |
| 371 | process.exitCode = exitCode; |
| 372 | }), |
| 373 | ), |
| 374 | ), |
| 375 | ); |
| 376 | |
| 377 | if (import.meta.main) { |
| 378 | Command.run(command, { version: "0.0.0" }).pipe( |
| 379 | Effect.provide(NodeServices.layer), |
| 380 | NodeRuntime.runMain, |
| 381 | ); |
| 382 | } |