The best way to control Claude Code, Codex and more from any computer or phone
fix(scripts): scenarios reach the screen and say why they could not run (#36054)
Julius Marminge committed
Nov 12, 2026 at 23:15 UTC
60f70d98cdeb2c692db649b9ade3adf74dd62bf9
6 files changed
+196
-29
docs/operations/scenarios.md
+47
-6
@@ -41,8 +41,10 @@ adds one fixture for what e2e cannot know about T3 Code:
41
42
- `t3.project(title, files)` creates a git repository and adds it as a project.
43
- `t3.thread({ project, title, messages })` starts a thread and plays one replayed turn per
44
- message before returning, so the thread has real history. `model` picks the recording.
45
-- `t3.open(path)` opens the app, already signed in.
44
+ message before returning, so the thread has real history. `model` picks the recording. It
45
+ returns the thread, which `t3.open` takes.
46
+- `t3.open(path)` opens the app, already signed in, at a path. `t3.open(thread)` opens it on a
47
+ thread.
48
49
State is created through the app's own CLI, so it is real events and works on a base commit
50
whose schema differs. New threads default to the replay agent's `showcase` recording: type a
@@ -61,6 +63,30 @@ could not run.
63
Keep `agent.act` and `agent.assert` out of scenarios: they call a model, and no model is
64
configured.
65
66
+### Getting to the screen
67
+
68
+Most scenarios that could not run stopped on the way to the screen the bug is on, not on the
69
+bug. Three rules avoid that:
70
+
71
+- **Open by link, not by sidebar.** `await t3.open(thread)` lands on the thread
72
+ ([`thread-opens-by-link.e2e.ts`](../../scripts/scenarios/thread-opens-by-link.e2e.ts)), and
73
+ `await t3.open("/settings/general")` on a settings page. A sidebar row is a button that covers
74
+ its own title, so a tap on `getByText("Earlier work")` is intercepted. When the sidebar is what
75
+ the scenario is about, target the row's button: `screen.getByRole("button", "Earlier work")`.
76
+- **Let `t3.open` do the navigating.** It waits until the pairing page has signed in before it
77
+ goes anywhere. A second `app.open` straight after the pairing link cancels the sign-in and
78
+ shows the pairing page again.
79
+- **Seed before opening.** Create projects and threads first, then call `t3.open`. With no
80
+ project the app opens its first-run page instead of the one the scenario expects.
81
+
82
+The app starts with only `PATH`, `HOME` and the temp directory in its environment, whatever the
83
+shell that ran the scenario has set. Name what it needs in `T3_SCENARIO_INHERIT_ENV`: exact
84
+names or prefixes ending in `*`, separated by commas.
85
+
86
+```sh
87
+T3_SCENARIO_INHERIT_ENV='SHELL,MY_FIXTURE_*' vp run scenarios:prove scripts/scenarios/name.e2e.ts --base main
88
+```
89
+
90
## Run
91
92
```sh
@@ -102,12 +128,27 @@ commits and compares the two reports. It prints, and writes to `scripts/.e2e/pro
128
its commit, ready for a pull request description. `--upload-command` names a command that takes
129
the capture paths and prints one URL per line. `--head` defaults to `HEAD`.
130
105
-It exits 0 only when the scenario fails on base and passes on head, 1 when it ran and did not
106
-prove the fix, and 2 when it could not run. A scenario that passes on base does not reproduce the
107
-bug, and the verdict says so.
131
+It exits 0 only when the scenario fails on base and passes on head, and 1 when it ran and did not
132
+prove the fix. A scenario that passes on base does not reproduce the bug, and the verdict says so.
133
+
134
+Two more codes mean there is no verdict on the behavior, and `couldNotRun` in the proof says
135
+which:
136
+
137
+- **3, `scenario`:** the app ran and the scenario stopped before its assertion, usually on a
138
+ control it did not find. Read that side's `message` and `steps`, fix the scenario, and run it
139
+ again.
140
+- **2, `app`:** the app did not start or no test ran: a bad revision, a failed install, a port in
141
+ use. `scripts/.e2e/app.log` has the app's output.
142
+
143
+Each side's result is also written as it ends, to `scripts/.e2e/prove/base.json` and
144
+`head.json`, so a run that is cut short keeps the side it finished.
145
+
146
+It runs the scenario on one target: iOS when the file imports `t3-mobile.ts`, web otherwise.
147
+`--target web` or `--target ios` overrides that.
148
149
The scenario and e2e always come from your checkout. Only the app comes from the commit under
110
-test: a temporary worktree beside your checkout, installed with `vp i` and removed afterwards. A
150
+test: a temporary worktree beside your checkout, installed with `vp i` and removed afterwards (one
151
+left by a run that was killed is replaced). A
152
commit that is your clean `HEAD` runs in place. So a new scenario can prove a fix against a base
153
that predates it.
154
scripts/e2e.config.ts
+8
@@ -31,6 +31,12 @@ const resolved = await Effect.gen(function* () {
31
appDir: path.resolve(scripts, env.T3_SCENARIO_APP_DIR ?? ".."),
32
home,
33
startApp: path.join(scripts, "scenarios/start-app.ts"),
34
+ // e2e gives the app only PATH, HOME and the temp directory. A scenario that needs more names
35
+ // it here: exact names or prefixes ending in `*`, separated by commas.
36
+ inheritEnv: (env.T3_SCENARIO_INHERIT_ENV ?? "")
37
+ .split(",")
38
+ .map((name) => name.trim())
39
+ .filter((name) => name !== ""),
40
mobile: {
41
host: host || "127.0.0.1",
42
// A mobile target has no `app.url`, so e2e cannot pick these ports.
@@ -143,6 +149,7 @@ export default {
149
executable: resolved.node,
150
args: [startApp, "web", T3_HOME, LOOPBACK, "{port}"],
151
cwd: APP_DIR,
152
+ inheritEnv: resolved.inheritEnv,
153
startupTimeout: 300_000,
154
log: ".e2e/app.log",
155
},
@@ -170,6 +177,7 @@ export default {
177
MOBILE.host,
178
],
179
cwd: APP_DIR,
180
+ inheritEnv: resolved.inheritEnv,
181
startupTimeout: 300_000,
182
log: ".e2e/mobile-app.log",
183
},
scripts/prove-scenario.test.ts
+14
@@ -91,6 +91,20 @@ describe("prove-scenario", () => {
91
expect(proof.verdict).toMatch(row.verdict);
92
});
93
94
+ it("tells a scenario that stopped from an app that did not start", () => {
95
+ const side = (report: Parameters<typeof readSide>[2]) =>
96
+ readSide("aaaaaaaaaaaa", "/out/base/results", report);
97
+ const head = readSide("bbbbbbbbbbbb", "/out/head/results", passed);
98
+ // A control the scenario did not find: the app ran, so the scenario is what needs fixing.
99
+ const stopped = buildProof(side(brokenScenario), head, (path) => path);
100
+ expect(stopped.couldNotRun).toBe("scenario");
101
+ expect(stopped.verdict).toMatch(/fix the scenario/);
102
+ const noApp = buildProof(side(undefined), head, (path) => path);
103
+ expect(noApp.couldNotRun).toBe("app");
104
+ expect(noApp.verdict).not.toMatch(/fix the scenario/);
105
+ expect(buildProof(side(failed), head, (path) => path).couldNotRun).toBeNull();
106
+ });
107
+
108
it("labels captures with their side and commit and keeps the pairing token out", () => {
109
const proof = buildProof(
110
readSide("aaaaaaaaaaaa", "/out/base/results", failed),
scripts/prove-scenario.ts
+74
-17
@@ -53,6 +53,11 @@ export interface SideResult {
53
readonly status: "passed" | "failed" | "error";
54
readonly title: string;
55
readonly message?: string;
56
+ /**
57
+ * For `error`, what stopped: `app` when the app did not start or no test ran, `scenario` when
58
+ * the app ran and the scenario stopped before its assertion, such as on a control it did not find.
59
+ */
60
+ readonly stopped?: "app" | "scenario";
61
readonly steps: ReadonlyArray<{ readonly step: string; readonly status: string }>;
62
/** Capture paths under the run's results directory, in the order the scenario took them. */
63
readonly captures: ReadonlyArray<string>;
@@ -75,7 +80,16 @@ export function readSide(
80
executed.length > 1
81
? `the scenario file ran ${executed.length} tests; a proof needs exactly one`
82
: "e2e wrote no report: the app did not start, or no test was selected";
78
- return { sha, status: "error", title: "", message, steps: [], captures: [], videos: [] };
83
+ return {
84
+ sha,
85
+ status: "error",
86
+ stopped: "app",
87
+ title: "",
88
+ message,
89
+ steps: [],
90
+ captures: [],
91
+ videos: [],
92
+ };
93
}
94
const files = (kind: string) =>
95
(attempt.artifacts ?? []).flatMap((artifact) =>
@@ -84,15 +98,18 @@ export function readSide(
98
: [],
99
);
100
const error = attempt.error;
101
+ // e2e exits 0 for a skipped test too, and a test that did not run proves nothing.
102
+ const status =
103
+ attempt.status === "passed"
104
+ ? "passed"
105
+ : error?.code === "ASSERTION_FAILED"
106
+ ? "failed"
107
+ : "error";
108
return {
109
sha,
89
- // e2e exits 0 for a skipped test too, and a test that did not run proves nothing.
90
- status:
91
- attempt.status === "passed"
92
- ? "passed"
93
- : error?.code === "ASSERTION_FAILED"
94
- ? "failed"
95
- : "error",
110
+ status,
111
+ // A test that ended with an error of its own got as far as the app.
112
+ ...(status === "error" ? { stopped: error === undefined ? "app" : "scenario" } : {}),
113
title: result.titlePath.join(" > "),
114
...(error === undefined
115
? attempt.status === "passed"
@@ -123,7 +140,7 @@ export function buildProof(base: SideResult, head: SideResult, link: (path: stri
140
const broken = base.status === "error" ? base : head.status === "error" ? head : undefined;
141
const verdict =
142
broken !== undefined
126
- ? `Not proven: the scenario could not run on ${broken === base ? `base ${baseSha}` : `head ${headSha}`} (${broken.message ?? "unknown error"}).`
143
+ ? `Not proven: the scenario could not run on ${broken === base ? `base ${baseSha}` : `head ${headSha}`} (${broken.message ?? "unknown error"}).${broken.stopped === "scenario" ? " The app ran and the scenario stopped before its assertion: fix the scenario and run it again." : ""}`
144
: base.status === "passed"
145
? head.status === "passed"
146
? `Not proven: the scenario passes on base ${baseSha} too, so it does not reproduce the bug.`
@@ -160,6 +177,8 @@ export function buildProof(base: SideResult, head: SideResult, link: (path: stri
177
scenario: head.title || base.title,
178
failsOnBase: base.status === "failed",
179
passesOnHead: head.status === "passed",
180
+ /** Why there is no verdict on the behavior: the app did not run, or the scenario stopped. */
181
+ couldNotRun: broken?.stopped ?? null,
182
baseSha: base.sha,
183
headSha: head.sha,
184
verdict,
@@ -192,10 +211,18 @@ const withCheckout = Effect.fn("prove.withCheckout")(function* <A, E, R>(
211
if (clean && (yield* git(["rev-parse", "HEAD"])) === sha) return yield* use(repo);
212
// Beside this checkout, not in the temp directory, so tooling resolves as it does here.
213
const worktree = `${repo}-scenario-${short(sha)}`;
214
+ // A run that was killed left its worktree behind, and git refuses to add over it.
215
+ const discard = git(["worktree", "remove", "--force", worktree]).pipe(
216
+ Effect.ignore,
217
+ Effect.andThen(git(["worktree", "prune"])),
218
+ Effect.ignore,
219
+ );
220
return yield* Effect.acquireUseRelease(
196
- git(["worktree", "add", "--quiet", "--detach", "--force", worktree, sha]),
221
+ discard.pipe(
222
+ Effect.andThen(git(["worktree", "add", "--quiet", "--detach", "--force", worktree, sha])),
223
+ ),
224
() => run("vp", ["i"], { cwd: worktree, echo: true }).pipe(Effect.andThen(use(worktree))),
198
- () => git(["worktree", "remove", "--force", worktree]).pipe(Effect.ignore),
225
+ () => discard,
226
);
227
});
228
@@ -204,6 +231,7 @@ const runSide = Effect.fn("prove.runSide")(function* (
231
side: "base" | "head",
232
rev: string,
233
scenarioPath: string,
234
+ target: string,
235
) {
236
const fs = yield* FileSystem.FileSystem;
237
const path = yield* Path.Path;
@@ -216,7 +244,7 @@ const runSide = Effect.fn("prove.runSide")(function* (
244
// The scenario and e2e come from this checkout; only the app comes from `appDir`.
245
run(
246
path.join(scripts, "node_modules/.bin/e2e"),
219
- ["run", scenarioPath, "--output", path.relative(scripts, outDir)],
247
+ ["run", scenarioPath, "--target", target, "--output", path.relative(scripts, outDir)],
248
{
249
cwd: scripts,
250
env: { ...environment, T3_SCENARIO_APP_DIR: appDir, E2E_TELEMETRY_DISABLED: "1" },
@@ -230,7 +258,22 @@ const runSide = Effect.fn("prove.runSide")(function* (
258
Effect.flatMap(decodeReport),
259
Effect.orElseSucceed(() => undefined),
260
);
233
- return readSide(sha, path.join(outDir, "results"), report);
261
+ const result = readSide(sha, path.join(outDir, "results"), report);
262
+ // Written as each side ends, so a proof whose second side is cut short still has its first.
263
+ yield* fs
264
+ .writeFileString(
265
+ path.join(scripts, ".e2e/prove", `${side}.json`),
266
+ `${yield* encodeProof(result)}\n`,
267
+ )
268
+ .pipe(Effect.ignore);
269
+ return result;
270
+});
271
+
272
+/** The one target a scenario runs on: an iOS scenario imports `t3-mobile.ts`, a web one does not. */
273
+const targetOf = Effect.fn("prove.targetOf")(function* (scenarioPath: string) {
274
+ const fs = yield* FileSystem.FileSystem;
275
+ const source = yield* fs.readFileString(scenarioPath).pipe(Effect.orElseSucceed(() => ""));
276
+ return source.includes("t3-mobile") ? "ios" : "web";
277
});
278
279
/** Uploads the captures and maps each to its URL; a failed upload leaves the map empty. */
@@ -262,6 +305,7 @@ const prove = Effect.fn("prove")(function* (input: {
305
readonly base: string;
306
readonly head: string;
307
readonly uploadCommand: Option.Option<string>;
308
+ readonly target: Option.Option<string>;
309
}) {
310
const fs = yield* FileSystem.FileSystem;
311
const path = yield* Path.Path;
@@ -271,8 +315,10 @@ const prove = Effect.fn("prove")(function* (input: {
315
// A proof left by an earlier run must not be read as this one's.
316
yield* fs.remove(proveDir, { recursive: true, force: true });
317
yield* fs.makeDirectory(proveDir, { recursive: true });
274
- const base = yield* runSide("base", input.base, scenarioPath);
275
- const head = yield* runSide("head", input.head, scenarioPath);
318
+ // Only that target's app starts: a web scenario needs no Metro and no device.
319
+ const target = Option.getOrElse(input.target, () => "") || (yield* targetOf(scenarioPath));
320
+ const base = yield* runSide("base", input.base, scenarioPath, target);
321
+ const head = yield* runSide("head", input.head, scenarioPath, target);
322
const links = Option.isSome(input.uploadCommand)
323
? yield* upload(input.uploadCommand.value, [...base.captures, ...head.captures])
324
: new Map<string, string>();
@@ -285,8 +331,13 @@ const prove = Effect.fn("prove")(function* (input: {
331
yield* fs.writeFileString(path.join(proveDir, "proof.md"), proof.markdown);
332
yield* fs.writeFileString(path.join(proveDir, "proof.json"), `${json}\n`);
333
yield* Console.log(json);
288
- const couldNotRun = base.status === "error" || head.status === "error";
289
- return proof.failsOnBase && proof.passesOnHead ? 0 : couldNotRun ? 2 : 1;
334
+ return proof.failsOnBase && proof.passesOnHead
335
+ ? 0
336
+ : proof.couldNotRun === "scenario"
337
+ ? 3
338
+ : proof.couldNotRun === "app"
339
+ ? 2
340
+ : 1;
341
});
342
343
const command = Command.make(
@@ -304,6 +355,12 @@ const command = Command.make(
355
Flag.withDescription("Command that takes capture paths and prints one URL per line."),
356
Flag.optional,
357
),
358
+ target: Flag.String("target").pipe(
359
+ Flag.withDescription(
360
+ "Target to run on: web or ios. Default: ios when the scenario imports t3-mobile.ts, else web.",
361
+ ),
362
+ Flag.optional,
363
+ ),
364
},
365
(input) =>
366
prove(input).pipe(
scripts/scenarios/t3.ts
+33
-6
@@ -2,11 +2,13 @@
2
// app. `t3-mobile.ts` is the same for the iOS app.
3
import * as NodeServices from "@effect/platform-node/NodeServices";
4
import { test as base } from "@e2e-dev/web";
5
+import { expect } from "e2e";
6
import * as HostProcess from "@t3tools/shared/HostProcess";
7
import * as Effect from "effect/Effect";
8
import * as FileSystem from "effect/FileSystem";
9
import * as Path from "effect/Path";
10
import * as Schema from "effect/Schema";
11
+import { FetchHttpClient, HttpClient, HttpClientResponse } from "effect/http";
12
13
import { APP_DIR, T3_HOME } from "../e2e.config.ts";
14
import { run } from "./shell.ts";
@@ -23,12 +25,19 @@ export interface T3State {
25
readonly title: string;
26
readonly messages: ReadonlyArray<string>;
27
readonly model?: string;
26
- }) => Promise<void>;
28
+ }) => Promise<T3Thread>;
29
+}
30
+
31
+export interface T3Thread {
32
+ readonly threadId: string;
33
}
34
35
export interface T3 extends T3State {
30
- /** Opens the app, signed in, at `path`. */
31
- readonly open: (path?: string) => Promise<void>;
36
+ /**
37
+ * Opens the app, signed in, at a path or on a thread `t3.thread` returned. Opening the thread
38
+ * by its link is the reliable way to reach it: a sidebar row covers its own title text.
39
+ */
40
+ readonly open: (target?: string | T3Thread) => Promise<void>;
41
}
42
43
/** e2e's fixtures are promise functions; this is the one place an Effect becomes a promise. */
@@ -99,6 +108,7 @@ export function makeState(home: string, env: NodeJS.ProcessEnv = {}) {
108
const started = yield* turn(["start", input.project, first, "--title", input.title, ...replay]);
109
const { threadId } = yield* decodeLaunched(started.stdout);
110
for (const message of rest) yield* turn(["send", threadId, message]);
111
+ return { threadId } satisfies T3Thread;
112
});
113
114
return {
@@ -122,14 +132,31 @@ const readPairingPath = Effect.gen(function* () {
132
return yield* fs.readFileString(path.join(T3_HOME, "pairing-path"));
133
});
134
135
+/** The id every thread link starts with, asked of the server the app talks to. */
136
+const environmentId = (baseUrl: string) =>
137
+ HttpClient.get(`${baseUrl}/.well-known/t3/environment`).pipe(
138
+ Effect.flatMap(HttpClientResponse.filterStatusOk),
139
+ Effect.flatMap(
140
+ HttpClientResponse.schemaBodyJson(Schema.Struct({ environmentId: Schema.String })),
141
+ ),
142
+ Effect.map((environment) => environment.environmentId),
143
+ Effect.provide(FetchHttpClient.layer),
144
+ );
145
+
146
export const test = base.extend<{ t3: T3 }>({
126
- t3: async ({ app }, use) => {
147
+ t3: async ({ app, browser }, use) => {
148
const { state, cleanup } = makeState(T3_HOME);
149
await use({
150
...state,
130
- open: async (path = "/") => {
151
+ open: async (target = "/") => {
152
await app.open(await runPromise(readPairingPath));
132
- if (path !== "/") await app.open(path);
153
+ // The pairing page signs in and then leaves itself. Going anywhere before it has done
154
+ // so cancels the sign-in, and the next page is the pairing page again.
155
+ await expect(browser).not.toHaveURL(/\/pair(?:[?#]|$)/, { timeout: 60_000 });
156
+ if (typeof target !== "string") {
157
+ const environment = await runPromise(environmentId(app.baseUrl ?? ""));
158
+ await app.open(`/${environment}/${target.threadId}`);
159
+ } else if (target !== "/") await app.open(target);
160
},
161
});
162
await runPromise(cleanup);
scripts/scenarios/thread-opens-by-link.e2e.ts
new
+20
@@ -0,0 +1,20 @@
1
+import { expect } from "e2e";
2
+
3
+import { test, WEB } from "./t3.ts";
4
+
5
+test(
6
+ "A seeded thread opens by its link and shows its history",
7
+ WEB,
8
+ async ({ t3, app, screen }) => {
9
+ const project = await t3.project("demo");
10
+ const thread = await t3.thread({
11
+ project,
12
+ title: "Earlier work",
13
+ messages: ["first", "second"],
14
+ });
15
+ await t3.open(thread);
16
+
17
+ await expect(screen.getByText("second fixture turn complete", { exact: false })).toBeVisible();
18
+ await app.screenshot("thread");
19
+ },
20
+);