@devries-cameron20 / t3code-1 / commits / 60f70d98cd
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(scripts): scenarios reach the screen and say why they could not run (#36054)

Julius Marminge committed Nov 12, 2026 at 23:15 UTC 60f70d98cdeb2c692db649b9ade3adf74dd62bf9
6 files changed +196 -29
docs/operations/scenarios.md
+47 -6
@@ -41,8 +41,10 @@ adds one fixture for what e2e cannot know about T3 Code:
41
42 - `t3.project(title, files)` creates a git repository and adds it as a project.
43 - `t3.thread({ project, title, messages })` starts a thread and plays one replayed turn per
44 - message before returning, so the thread has real history. `model` picks the recording.
45 -- `t3.open(path)` opens the app, already signed in.
44 + message before returning, so the thread has real history. `model` picks the recording. It
45 + returns the thread, which `t3.open` takes.
46 +- `t3.open(path)` opens the app, already signed in, at a path. `t3.open(thread)` opens it on a
47 + thread.
48
49 State is created through the app's own CLI, so it is real events and works on a base commit
50 whose schema differs. New threads default to the replay agent's `showcase` recording: type a
@@ -61,6 +63,30 @@ could not run.
63 Keep `agent.act` and `agent.assert` out of scenarios: they call a model, and no model is
64 configured.
65
66 +### Getting to the screen
67 +
68 +Most scenarios that could not run stopped on the way to the screen the bug is on, not on the
69 +bug. Three rules avoid that:
70 +
71 +- **Open by link, not by sidebar.** `await t3.open(thread)` lands on the thread
72 + ([`thread-opens-by-link.e2e.ts`](../../scripts/scenarios/thread-opens-by-link.e2e.ts)), and
73 + `await t3.open("/settings/general")` on a settings page. A sidebar row is a button that covers
74 + its own title, so a tap on `getByText("Earlier work")` is intercepted. When the sidebar is what
75 + the scenario is about, target the row's button: `screen.getByRole("button", "Earlier work")`.
76 +- **Let `t3.open` do the navigating.** It waits until the pairing page has signed in before it
77 + goes anywhere. A second `app.open` straight after the pairing link cancels the sign-in and
78 + shows the pairing page again.
79 +- **Seed before opening.** Create projects and threads first, then call `t3.open`. With no
80 + project the app opens its first-run page instead of the one the scenario expects.
81 +
82 +The app starts with only `PATH`, `HOME` and the temp directory in its environment, whatever the
83 +shell that ran the scenario has set. Name what it needs in `T3_SCENARIO_INHERIT_ENV`: exact
84 +names or prefixes ending in `*`, separated by commas.
85 +
86 +```sh
87 +T3_SCENARIO_INHERIT_ENV='SHELL,MY_FIXTURE_*' vp run scenarios:prove scripts/scenarios/name.e2e.ts --base main
88 +```
89 +
90 ## Run
91
92 ```sh
@@ -102,12 +128,27 @@ commits and compares the two reports. It prints, and writes to `scripts/.e2e/pro
128 its commit, ready for a pull request description. `--upload-command` names a command that takes
129 the capture paths and prints one URL per line. `--head` defaults to `HEAD`.
130
105 -It exits 0 only when the scenario fails on base and passes on head, 1 when it ran and did not
106 -prove the fix, and 2 when it could not run. A scenario that passes on base does not reproduce the
107 -bug, and the verdict says so.
131 +It exits 0 only when the scenario fails on base and passes on head, and 1 when it ran and did not
132 +prove the fix. A scenario that passes on base does not reproduce the bug, and the verdict says so.
133 +
134 +Two more codes mean there is no verdict on the behavior, and `couldNotRun` in the proof says
135 +which:
136 +
137 +- **3, `scenario`:** the app ran and the scenario stopped before its assertion, usually on a
138 + control it did not find. Read that side's `message` and `steps`, fix the scenario, and run it
139 + again.
140 +- **2, `app`:** the app did not start or no test ran: a bad revision, a failed install, a port in
141 + use. `scripts/.e2e/app.log` has the app's output.
142 +
143 +Each side's result is also written as it ends, to `scripts/.e2e/prove/base.json` and
144 +`head.json`, so a run that is cut short keeps the side it finished.
145 +
146 +It runs the scenario on one target: iOS when the file imports `t3-mobile.ts`, web otherwise.
147 +`--target web` or `--target ios` overrides that.
148
149 The scenario and e2e always come from your checkout. Only the app comes from the commit under
110 -test: a temporary worktree beside your checkout, installed with `vp i` and removed afterwards. A
150 +test: a temporary worktree beside your checkout, installed with `vp i` and removed afterwards (one
151 +left by a run that was killed is replaced). A
152 commit that is your clean `HEAD` runs in place. So a new scenario can prove a fix against a base
153 that predates it.
154
scripts/e2e.config.ts
+8
@@ -31,6 +31,12 @@ const resolved = await Effect.gen(function* () {
31 appDir: path.resolve(scripts, env.T3_SCENARIO_APP_DIR ?? ".."),
32 home,
33 startApp: path.join(scripts, "scenarios/start-app.ts"),
34 + // e2e gives the app only PATH, HOME and the temp directory. A scenario that needs more names
35 + // it here: exact names or prefixes ending in `*`, separated by commas.
36 + inheritEnv: (env.T3_SCENARIO_INHERIT_ENV ?? "")
37 + .split(",")
38 + .map((name) => name.trim())
39 + .filter((name) => name !== ""),
40 mobile: {
41 host: host || "127.0.0.1",
42 // A mobile target has no `app.url`, so e2e cannot pick these ports.
@@ -143,6 +149,7 @@ export default {
149 executable: resolved.node,
150 args: [startApp, "web", T3_HOME, LOOPBACK, "{port}"],
151 cwd: APP_DIR,
152 + inheritEnv: resolved.inheritEnv,
153 startupTimeout: 300_000,
154 log: ".e2e/app.log",
155 },
@@ -170,6 +177,7 @@ export default {
177 MOBILE.host,
178 ],
179 cwd: APP_DIR,
180 + inheritEnv: resolved.inheritEnv,
181 startupTimeout: 300_000,
182 log: ".e2e/mobile-app.log",
183 },
scripts/prove-scenario.test.ts
+14
@@ -91,6 +91,20 @@ describe("prove-scenario", () => {
91 expect(proof.verdict).toMatch(row.verdict);
92 });
93
94 + it("tells a scenario that stopped from an app that did not start", () => {
95 + const side = (report: Parameters<typeof readSide>[2]) =>
96 + readSide("aaaaaaaaaaaa", "/out/base/results", report);
97 + const head = readSide("bbbbbbbbbbbb", "/out/head/results", passed);
98 + // A control the scenario did not find: the app ran, so the scenario is what needs fixing.
99 + const stopped = buildProof(side(brokenScenario), head, (path) => path);
100 + expect(stopped.couldNotRun).toBe("scenario");
101 + expect(stopped.verdict).toMatch(/fix the scenario/);
102 + const noApp = buildProof(side(undefined), head, (path) => path);
103 + expect(noApp.couldNotRun).toBe("app");
104 + expect(noApp.verdict).not.toMatch(/fix the scenario/);
105 + expect(buildProof(side(failed), head, (path) => path).couldNotRun).toBeNull();
106 + });
107 +
108 it("labels captures with their side and commit and keeps the pairing token out", () => {
109 const proof = buildProof(
110 readSide("aaaaaaaaaaaa", "/out/base/results", failed),
scripts/prove-scenario.ts
+74 -17
@@ -53,6 +53,11 @@ export interface SideResult {
53 readonly status: "passed" | "failed" | "error";
54 readonly title: string;
55 readonly message?: string;
56 + /**
57 + * For `error`, what stopped: `app` when the app did not start or no test ran, `scenario` when
58 + * the app ran and the scenario stopped before its assertion, such as on a control it did not find.
59 + */
60 + readonly stopped?: "app" | "scenario";
61 readonly steps: ReadonlyArray<{ readonly step: string; readonly status: string }>;
62 /** Capture paths under the run's results directory, in the order the scenario took them. */
63 readonly captures: ReadonlyArray<string>;
@@ -75,7 +80,16 @@ export function readSide(
80 executed.length > 1
81 ? `the scenario file ran ${executed.length} tests; a proof needs exactly one`
82 : "e2e wrote no report: the app did not start, or no test was selected";
78 - return { sha, status: "error", title: "", message, steps: [], captures: [], videos: [] };
83 + return {
84 + sha,
85 + status: "error",
86 + stopped: "app",
87 + title: "",
88 + message,
89 + steps: [],
90 + captures: [],
91 + videos: [],
92 + };
93 }
94 const files = (kind: string) =>
95 (attempt.artifacts ?? []).flatMap((artifact) =>
@@ -84,15 +98,18 @@ export function readSide(
98 : [],
99 );
100 const error = attempt.error;
101 + // e2e exits 0 for a skipped test too, and a test that did not run proves nothing.
102 + const status =
103 + attempt.status === "passed"
104 + ? "passed"
105 + : error?.code === "ASSERTION_FAILED"
106 + ? "failed"
107 + : "error";
108 return {
109 sha,
89 - // e2e exits 0 for a skipped test too, and a test that did not run proves nothing.
90 - status:
91 - attempt.status === "passed"
92 - ? "passed"
93 - : error?.code === "ASSERTION_FAILED"
94 - ? "failed"
95 - : "error",
110 + status,
111 + // A test that ended with an error of its own got as far as the app.
112 + ...(status === "error" ? { stopped: error === undefined ? "app" : "scenario" } : {}),
113 title: result.titlePath.join(" > "),
114 ...(error === undefined
115 ? attempt.status === "passed"
@@ -123,7 +140,7 @@ export function buildProof(base: SideResult, head: SideResult, link: (path: stri
140 const broken = base.status === "error" ? base : head.status === "error" ? head : undefined;
141 const verdict =
142 broken !== undefined
126 - ? `Not proven: the scenario could not run on ${broken === base ? `base ${baseSha}` : `head ${headSha}`} (${broken.message ?? "unknown error"}).`
143 + ? `Not proven: the scenario could not run on ${broken === base ? `base ${baseSha}` : `head ${headSha}`} (${broken.message ?? "unknown error"}).${broken.stopped === "scenario" ? " The app ran and the scenario stopped before its assertion: fix the scenario and run it again." : ""}`
144 : base.status === "passed"
145 ? head.status === "passed"
146 ? `Not proven: the scenario passes on base ${baseSha} too, so it does not reproduce the bug.`
@@ -160,6 +177,8 @@ export function buildProof(base: SideResult, head: SideResult, link: (path: stri
177 scenario: head.title || base.title,
178 failsOnBase: base.status === "failed",
179 passesOnHead: head.status === "passed",
180 + /** Why there is no verdict on the behavior: the app did not run, or the scenario stopped. */
181 + couldNotRun: broken?.stopped ?? null,
182 baseSha: base.sha,
183 headSha: head.sha,
184 verdict,
@@ -192,10 +211,18 @@ const withCheckout = Effect.fn("prove.withCheckout")(function* <A, E, R>(
211 if (clean && (yield* git(["rev-parse", "HEAD"])) === sha) return yield* use(repo);
212 // Beside this checkout, not in the temp directory, so tooling resolves as it does here.
213 const worktree = `${repo}-scenario-${short(sha)}`;
214 + // A run that was killed left its worktree behind, and git refuses to add over it.
215 + const discard = git(["worktree", "remove", "--force", worktree]).pipe(
216 + Effect.ignore,
217 + Effect.andThen(git(["worktree", "prune"])),
218 + Effect.ignore,
219 + );
220 return yield* Effect.acquireUseRelease(
196 - git(["worktree", "add", "--quiet", "--detach", "--force", worktree, sha]),
221 + discard.pipe(
222 + Effect.andThen(git(["worktree", "add", "--quiet", "--detach", "--force", worktree, sha])),
223 + ),
224 () => run("vp", ["i"], { cwd: worktree, echo: true }).pipe(Effect.andThen(use(worktree))),
198 - () => git(["worktree", "remove", "--force", worktree]).pipe(Effect.ignore),
225 + () => discard,
226 );
227 });
228
@@ -204,6 +231,7 @@ const runSide = Effect.fn("prove.runSide")(function* (
231 side: "base" | "head",
232 rev: string,
233 scenarioPath: string,
234 + target: string,
235 ) {
236 const fs = yield* FileSystem.FileSystem;
237 const path = yield* Path.Path;
@@ -216,7 +244,7 @@ const runSide = Effect.fn("prove.runSide")(function* (
244 // The scenario and e2e come from this checkout; only the app comes from `appDir`.
245 run(
246 path.join(scripts, "node_modules/.bin/e2e"),
219 - ["run", scenarioPath, "--output", path.relative(scripts, outDir)],
247 + ["run", scenarioPath, "--target", target, "--output", path.relative(scripts, outDir)],
248 {
249 cwd: scripts,
250 env: { ...environment, T3_SCENARIO_APP_DIR: appDir, E2E_TELEMETRY_DISABLED: "1" },
@@ -230,7 +258,22 @@ const runSide = Effect.fn("prove.runSide")(function* (
258 Effect.flatMap(decodeReport),
259 Effect.orElseSucceed(() => undefined),
260 );
233 - return readSide(sha, path.join(outDir, "results"), report);
261 + const result = readSide(sha, path.join(outDir, "results"), report);
262 + // Written as each side ends, so a proof whose second side is cut short still has its first.
263 + yield* fs
264 + .writeFileString(
265 + path.join(scripts, ".e2e/prove", `${side}.json`),
266 + `${yield* encodeProof(result)}\n`,
267 + )
268 + .pipe(Effect.ignore);
269 + return result;
270 +});
271 +
272 +/** The one target a scenario runs on: an iOS scenario imports `t3-mobile.ts`, a web one does not. */
273 +const targetOf = Effect.fn("prove.targetOf")(function* (scenarioPath: string) {
274 + const fs = yield* FileSystem.FileSystem;
275 + const source = yield* fs.readFileString(scenarioPath).pipe(Effect.orElseSucceed(() => ""));
276 + return source.includes("t3-mobile") ? "ios" : "web";
277 });
278
279 /** Uploads the captures and maps each to its URL; a failed upload leaves the map empty. */
@@ -262,6 +305,7 @@ const prove = Effect.fn("prove")(function* (input: {
305 readonly base: string;
306 readonly head: string;
307 readonly uploadCommand: Option.Option<string>;
308 + readonly target: Option.Option<string>;
309 }) {
310 const fs = yield* FileSystem.FileSystem;
311 const path = yield* Path.Path;
@@ -271,8 +315,10 @@ const prove = Effect.fn("prove")(function* (input: {
315 // A proof left by an earlier run must not be read as this one's.
316 yield* fs.remove(proveDir, { recursive: true, force: true });
317 yield* fs.makeDirectory(proveDir, { recursive: true });
274 - const base = yield* runSide("base", input.base, scenarioPath);
275 - const head = yield* runSide("head", input.head, scenarioPath);
318 + // Only that target's app starts: a web scenario needs no Metro and no device.
319 + const target = Option.getOrElse(input.target, () => "") || (yield* targetOf(scenarioPath));
320 + const base = yield* runSide("base", input.base, scenarioPath, target);
321 + const head = yield* runSide("head", input.head, scenarioPath, target);
322 const links = Option.isSome(input.uploadCommand)
323 ? yield* upload(input.uploadCommand.value, [...base.captures, ...head.captures])
324 : new Map<string, string>();
@@ -285,8 +331,13 @@ const prove = Effect.fn("prove")(function* (input: {
331 yield* fs.writeFileString(path.join(proveDir, "proof.md"), proof.markdown);
332 yield* fs.writeFileString(path.join(proveDir, "proof.json"), `${json}\n`);
333 yield* Console.log(json);
288 - const couldNotRun = base.status === "error" || head.status === "error";
289 - return proof.failsOnBase && proof.passesOnHead ? 0 : couldNotRun ? 2 : 1;
334 + return proof.failsOnBase && proof.passesOnHead
335 + ? 0
336 + : proof.couldNotRun === "scenario"
337 + ? 3
338 + : proof.couldNotRun === "app"
339 + ? 2
340 + : 1;
341 });
342
343 const command = Command.make(
@@ -304,6 +355,12 @@ const command = Command.make(
355 Flag.withDescription("Command that takes capture paths and prints one URL per line."),
356 Flag.optional,
357 ),
358 + target: Flag.String("target").pipe(
359 + Flag.withDescription(
360 + "Target to run on: web or ios. Default: ios when the scenario imports t3-mobile.ts, else web.",
361 + ),
362 + Flag.optional,
363 + ),
364 },
365 (input) =>
366 prove(input).pipe(
scripts/scenarios/t3.ts
+33 -6
@@ -2,11 +2,13 @@
2 // app. `t3-mobile.ts` is the same for the iOS app.
3 import * as NodeServices from "@effect/platform-node/NodeServices";
4 import { test as base } from "@e2e-dev/web";
5 +import { expect } from "e2e";
6 import * as HostProcess from "@t3tools/shared/HostProcess";
7 import * as Effect from "effect/Effect";
8 import * as FileSystem from "effect/FileSystem";
9 import * as Path from "effect/Path";
10 import * as Schema from "effect/Schema";
11 +import { FetchHttpClient, HttpClient, HttpClientResponse } from "effect/http";
12
13 import { APP_DIR, T3_HOME } from "../e2e.config.ts";
14 import { run } from "./shell.ts";
@@ -23,12 +25,19 @@ export interface T3State {
25 readonly title: string;
26 readonly messages: ReadonlyArray<string>;
27 readonly model?: string;
26 - }) => Promise<void>;
28 + }) => Promise<T3Thread>;
29 +}
30 +
31 +export interface T3Thread {
32 + readonly threadId: string;
33 }
34
35 export interface T3 extends T3State {
30 - /** Opens the app, signed in, at `path`. */
31 - readonly open: (path?: string) => Promise<void>;
36 + /**
37 + * Opens the app, signed in, at a path or on a thread `t3.thread` returned. Opening the thread
38 + * by its link is the reliable way to reach it: a sidebar row covers its own title text.
39 + */
40 + readonly open: (target?: string | T3Thread) => Promise<void>;
41 }
42
43 /** e2e's fixtures are promise functions; this is the one place an Effect becomes a promise. */
@@ -99,6 +108,7 @@ export function makeState(home: string, env: NodeJS.ProcessEnv = {}) {
108 const started = yield* turn(["start", input.project, first, "--title", input.title, ...replay]);
109 const { threadId } = yield* decodeLaunched(started.stdout);
110 for (const message of rest) yield* turn(["send", threadId, message]);
111 + return { threadId } satisfies T3Thread;
112 });
113
114 return {
@@ -122,14 +132,31 @@ const readPairingPath = Effect.gen(function* () {
132 return yield* fs.readFileString(path.join(T3_HOME, "pairing-path"));
133 });
134
135 +/** The id every thread link starts with, asked of the server the app talks to. */
136 +const environmentId = (baseUrl: string) =>
137 + HttpClient.get(`${baseUrl}/.well-known/t3/environment`).pipe(
138 + Effect.flatMap(HttpClientResponse.filterStatusOk),
139 + Effect.flatMap(
140 + HttpClientResponse.schemaBodyJson(Schema.Struct({ environmentId: Schema.String })),
141 + ),
142 + Effect.map((environment) => environment.environmentId),
143 + Effect.provide(FetchHttpClient.layer),
144 + );
145 +
146 export const test = base.extend<{ t3: T3 }>({
126 - t3: async ({ app }, use) => {
147 + t3: async ({ app, browser }, use) => {
148 const { state, cleanup } = makeState(T3_HOME);
149 await use({
150 ...state,
130 - open: async (path = "/") => {
151 + open: async (target = "/") => {
152 await app.open(await runPromise(readPairingPath));
132 - if (path !== "/") await app.open(path);
153 + // The pairing page signs in and then leaves itself. Going anywhere before it has done
154 + // so cancels the sign-in, and the next page is the pairing page again.
155 + await expect(browser).not.toHaveURL(/\/pair(?:[?#]|$)/, { timeout: 60_000 });
156 + if (typeof target !== "string") {
157 + const environment = await runPromise(environmentId(app.baseUrl ?? ""));
158 + await app.open(`/${environment}/${target.threadId}`);
159 + } else if (target !== "/") await app.open(target);
160 },
161 });
162 await runPromise(cleanup);
scripts/scenarios/thread-opens-by-link.e2e.ts new
+20
@@ -0,0 +1,20 @@
1 +import { expect } from "e2e";
2 +
3 +import { test, WEB } from "./t3.ts";
4 +
5 +test(
6 + "A seeded thread opens by its link and shows its history",
7 + WEB,
8 + async ({ t3, app, screen }) => {
9 + const project = await t3.project("demo");
10 + const thread = await t3.thread({
11 + project,
12 + title: "Earlier work",
13 + messages: ["first", "second"],
14 + });
15 + await t3.open(thread);
16 +
17 + await expect(screen.getByText("second fixture turn complete", { exact: false })).toBeVisible();
18 + await app.screenshot("thread");
19 + },
20 +);