@devries-cameron20 / t3code-1 / commits / d171a88fc4
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

feat(mobile): run the chrome e2e across poses on the device pool

`vp run mobile:e2e` now leases iOS and Android devices from the Simlock pool (SIMLOCK_AGENT_DEVICE_URL + AGENT_DEVICE_DAEMON_AUTH_TOKEN) instead of booting local simulators, and walks each of 12 poses (phone sizes, iPads, a fold, a tablet, dark mode, largest text) through the app: thread list, a thread, scrolling, typing, back, settings, then rotation or folding. Every screen gets generic layout checks that need no per-screen code: app in front, no error screen or launch dialog, no fixed control cut off by the screen edge, no sibling controls stacked, nothing under a system bar, no input under the keyboard. Transitions keep the video check for tabs lagging the screen, now measured against the recording's own first and last frames. Afterwards a headless Claude reviews each pose's screenshots for what the checks cannot see, and the findings land in the same report grid. `mobile:install --session` now has the session's host download the dev client from the shared cache with the read-only key, instead of uploading it through the gateway (13 s instead of 10+ minutes for an APK). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 26, 2026 at 01:19 UTC d171a88fc4fa00b1ca7ce6af7854e55cc3cc88ec
11 files changed +1507 -713
scripts/mobile-e2e.ts
+214 -702
@@ -1,5 +1,5 @@
1 #!/usr/bin/env node
2 -// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off - Host-side simulator automation uses Node subprocess and timing APIs directly.
2 +// @effect-diagnostics nodeBuiltinImport:off globalDate:off - Host-side device automation uses Node subprocess and filesystem APIs directly.
3
4 import * as NodeChildProcess from "node:child_process";
5 import * as NodeFSP from "node:fs/promises";
@@ -8,630 +8,74 @@ import * as NodePath from "node:path";
8 import * as NodeProcess from "node:process";
9 import * as NodeURL from "node:url";
10
11 -import { startShowcaseEnvironments, stopProcess, waitForPort } from "./mobile-showcase.ts";
11 +import { Device, type Pool, run } from "./mobile-e2e/device.ts";
12 +import { runJourney, type StepResult } from "./mobile-e2e/journey.ts";
13 +import { POSES, type Pose } from "./mobile-e2e/poses.ts";
14 +import { failedChecks, poseFailed, type PoseResult, renderReport } from "./mobile-e2e/report.ts";
15 +import { reviewPose } from "./mobile-e2e/review.ts";
16 +import {
17 + reserveAvailablePort,
18 + startShowcaseEnvironments,
19 + stopProcess,
20 + waitForPort,
21 +} from "./mobile-showcase.ts";
22 import { SHOWCASE_ENVIRONMENTS } from "./mobile-showcase-environment.ts";
23
24 const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), "..");
15 -const AGENT_DEVICE = NodePath.join(
16 - REPO_ROOT,
17 - "scripts/node_modules/agent-device/bin/agent-device.mjs",
18 -);
19 -const BUNDLE_ID = "com.t3tools.t3code.dev";
20 -const METRO_PORT = 8081;
21 -const AGENT_DEVICE_VERSION = (
22 - JSON.parse(
23 - await NodeFSP.readFile(
24 - NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/package.json"),
25 - "utf8",
26 - ),
27 - ) as { version: string }
28 -).version;
29 -// Persistent so the XCTest runner agent-device builds per simulator survives between runs;
30 -// per version because a daemon refuses clients of another version.
31 -const AGENT_DEVICE_STATE = NodePath.join(
32 - NodeOS.homedir(),
33 - ".cache/t3code/mobile-e2e",
34 - `agent-device-${AGENT_DEVICE_VERSION}`,
35 -);
25
37 -export interface E2eDevice {
38 - readonly id: string;
39 - readonly platform: "ios";
40 - readonly deviceType: string;
41 - readonly runtime: string;
42 -}
43 -
44 -const ios = (id: string, deviceType: string): E2eDevice => ({
45 - id,
46 - platform: "ios",
47 - deviceType: `com.apple.CoreSimulator.SimDeviceType.${deviceType}`,
48 - runtime: "com.apple.CoreSimulator.SimRuntime.iOS-27-0",
49 -});
50 -
51 -export const DEVICES: ReadonlyArray<E2eDevice> = [
52 - ios("iphone-se", "iPhone-SE-3rd-generation"),
53 - ios("iphone-13-mini", "iPhone-13-mini"),
54 - ios("iphone-17", "iPhone-17"),
55 - ios("iphone-17-pro", "iPhone-17-Pro"),
56 - ios("iphone-17-pro-max", "iPhone-17-Pro-Max"),
57 - ios("iphone-air", "iPhone-Air"),
58 -];
59 -
60 -interface Rect {
61 - readonly x: number;
62 - readonly y: number;
63 - readonly width: number;
64 - readonly height: number;
65 -}
66 -
67 -export interface SnapshotNode {
68 - readonly type?: string;
69 - readonly role?: string;
70 - readonly identifier?: string;
71 - readonly label?: string;
72 - readonly rect?: Rect;
73 -}
74 -
75 -export interface Snapshot {
76 - readonly nodes: ReadonlyArray<SnapshotNode>;
77 - readonly viewport: { readonly width: number; readonly height: number };
78 -}
79 -
80 -export interface Check {
81 - readonly name: string;
82 - readonly pass: boolean;
83 - readonly detail: string;
84 -}
85 -
86 -interface StepResult {
87 - readonly name: string;
88 - readonly checks: ReadonlyArray<Check>;
89 - readonly screenshot?: string | undefined;
90 - readonly video?: string | undefined;
91 - readonly contactSheet?: string | undefined;
92 - readonly lagFrame?: string | undefined;
93 - readonly samples: number;
94 -}
95 -
96 -interface DeviceResult {
97 - readonly device: E2eDevice;
98 - readonly label: string;
99 - readonly steps: ReadonlyArray<StepResult>;
100 - readonly error?: string;
101 -}
102 -
103 -const onScreen = (node: SnapshotNode, snapshot: Snapshot) =>
104 - node.rect !== undefined &&
105 - node.rect.width > 0 &&
106 - node.rect.height > 0 &&
107 - node.rect.x < snapshot.viewport.width &&
108 - node.rect.y < snapshot.viewport.height &&
109 - node.rect.x + node.rect.width > 0 &&
110 - node.rect.y + node.rect.height > 0;
111 -
112 -const find = (snapshot: Snapshot, match: (node: SnapshotNode) => boolean) =>
113 - snapshot.nodes.filter((node) => match(node) && onScreen(node, snapshot));
114 -
115 -export const chrome = (snapshot: Snapshot) => ({
116 - tabBar: find(snapshot, (node) => node.type === "TabBar")[0],
117 - navigationBar: find(snapshot, (node) => node.type === "NavigationBar")[0],
118 - backButton: find(snapshot, (node) => node.identifier === "BackButton")[0],
119 - send: find(snapshot, (node) => node.type === "Button" && node.label === "Send")[0],
120 - newChat: find(snapshot, (node) => node.type === "Button" && node.label === "New chat")[0],
121 - // Thread rows are full-width context-menu wrappers; the filter pills are narrow ones.
122 - threadRows: find(
123 - snapshot,
124 - (node) =>
125 - node.role === "MenuView" &&
126 - Boolean(node.label) &&
127 - (node.rect?.width ?? 0) >= snapshot.viewport.width * 0.9,
128 - ),
129 -});
130 -
131 -const check = (name: string, pass: boolean, detail: string): Check => ({ name, pass, detail });
132 -const describeRect = (rect: Rect | undefined) =>
133 - rect
134 - ? `${Math.round(rect.x)},${Math.round(rect.y)} ${Math.round(rect.width)}×${Math.round(rect.height)}`
135 - : "absent";
136 -const inside = (rect: Rect, snapshot: Snapshot) =>
137 - rect.x >= 0 &&
138 - rect.y >= 0 &&
139 - rect.x + rect.width <= snapshot.viewport.width + 0.5 &&
140 - rect.y + rect.height <= snapshot.viewport.height + 0.5;
141 -
142 -export function homeChecks(snapshot: Snapshot): Check[] {
143 - const { tabBar, newChat } = chrome(snapshot);
144 - return [
145 - check("tab bar visible", tabBar !== undefined, describeRect(tabBar?.rect)),
146 - check(
147 - "tab bar inside the screen",
148 - tabBar?.rect !== undefined && inside(tabBar.rect, snapshot),
149 - describeRect(tabBar?.rect),
150 - ),
151 - check("new chat button visible", newChat !== undefined, describeRect(newChat?.rect)),
152 - ];
153 -}
154 -
155 -export function threadChecks(snapshot: Snapshot, samples: ReadonlyArray<Snapshot>): Check[] {
156 - const { tabBar, backButton, send } = chrome(snapshot);
157 - const overlapping = samples.filter((sample) => {
158 - const state = chrome(sample);
159 - return state.backButton !== undefined && state.tabBar !== undefined;
160 - }).length;
161 - return [
162 - check("tab bar hidden", tabBar === undefined, describeRect(tabBar?.rect)),
163 - check("back button visible", backButton !== undefined, describeRect(backButton?.rect)),
164 - check(
165 - "composer inside the screen",
166 - send?.rect !== undefined && inside(send.rect, snapshot),
167 - describeRect(send?.rect),
168 - ),
169 - check(
170 - "no tabs over the entering thread",
171 - overlapping === 0,
172 - `${overlapping} of ${samples.length} transition samples showed tabs with a Back button`,
173 - ),
174 - ];
175 -}
176 -
177 -export function backChecks(snapshot: Snapshot, samples: ReadonlyArray<Snapshot>): Check[] {
178 - const listWithoutTabs = samples.filter((sample) => {
179 - const state = chrome(sample);
180 - return (
181 - state.backButton === undefined && state.newChat !== undefined && state.tabBar === undefined
182 - );
183 - }).length;
184 - return [
185 - ...homeChecks(snapshot),
186 - check(
187 - "tabs return with the list",
188 - listWithoutTabs === 0,
189 - `${listWithoutTabs} of ${samples.length} transition samples showed the list without tabs`,
190 - ),
191 - ];
192 -}
193 -
194 -/** Downsampled grayscale crops of one region, one per video frame (or one for a PNG). */
195 -async function grayFrames(input: string, region: Rect, scale: number): Promise<Uint8Array[]> {
196 - const even = (value: number) => Math.max(2, Math.floor(value / 2) * 2);
197 - const [x, y, width, height] = [
198 - Math.round(region.x * scale),
199 - Math.round(region.y * scale),
200 - even(region.width * scale),
201 - even(region.height * scale),
202 - ];
203 - const [outWidth, outHeight] = [even(width / 6), even(height / 6)];
204 - const bytes = await new Promise<Buffer>((resolve, reject) => {
205 - const child = NodeChildProcess.spawn(
206 - "ffmpeg",
207 - [
208 - "-v",
209 - "error",
210 - "-i",
211 - input,
212 - // One output frame per recorded frame, so indices match select=eq(n,…) for evidence.
213 - "-fps_mode",
214 - "passthrough",
215 - "-vf",
216 - `crop=${width}:${height}:${x}:${y},scale=${outWidth}:${outHeight},format=gray`,
217 - "-f",
218 - "rawvideo",
219 - "-",
220 - ],
221 - { stdio: ["ignore", "pipe", "pipe"] },
222 - );
223 - const chunks: Buffer[] = [];
224 - let stderr = "";
225 - child.stdout.on("data", (chunk: Buffer) => chunks.push(chunk));
226 - child.stderr.on("data", (chunk) => (stderr += chunk));
227 - child.on("error", reject);
228 - child.on("close", (code) =>
229 - code === 0 ? resolve(Buffer.concat(chunks)) : reject(new Error(`ffmpeg failed: ${stderr}`)),
230 - );
231 - });
232 - const size = outWidth * outHeight;
233 - return Array.from({ length: Math.floor(bytes.length / size) }, (_, index) =>
234 - bytes.subarray(index * size, (index + 1) * size),
235 - );
236 -}
237 -
238 -const distance = (a: Uint8Array, b: Uint8Array) => {
239 - let sum = 0;
240 - for (let index = 0; index < a.length; index++) sum += Math.abs(a[index]! - b[index]!);
241 - return sum / a.length;
242 -};
243 -
244 -/** How far a region has moved from its origin toward its destination look, 0 to 1. */
245 -export interface FrameProgress {
246 - readonly nav: number;
247 - readonly chrome: number;
248 -}
249 -
250 -/**
251 - * A frame lags when the navigation bar is well into the transition while the tab bar region
252 - * still looks like the origin: tabs lingering over an entering thread, or a returned list drawn
253 - * before its tabs.
254 - */
255 -export const chromeLags = (frame: FrameProgress) => frame.nav > 0.3 && frame.chrome < 0.4;
256 -// One lagging frame is capture jitter; a regression lasts several.
257 -const LAG_FRAME_LIMIT = 2;
258 -
259 -const progress = (origin: number, destination: number) =>
260 - origin + destination === 0 ? 0 : origin / (origin + destination);
261 -
262 -async function chromeLagFrames(input: {
263 - readonly video: string;
264 - readonly origin: string;
265 - readonly destination: string;
266 - readonly nav: Rect;
267 - readonly chrome: Rect;
268 - readonly scale: number;
269 -}) {
270 - const [
271 - navFrames,
272 - chromeFrames,
273 - [navOrigin],
274 - [navDestination],
275 - [chromeOrigin],
276 - [chromeDestination],
277 - ] = await Promise.all([
278 - grayFrames(input.video, input.nav, input.scale),
279 - grayFrames(input.video, input.chrome, input.scale),
280 - grayFrames(input.origin, input.nav, input.scale),
281 - grayFrames(input.destination, input.nav, input.scale),
282 - grayFrames(input.origin, input.chrome, input.scale),
283 - grayFrames(input.destination, input.chrome, input.scale),
284 - ]);
285 - if (!navOrigin || !navDestination || !chromeOrigin || !chromeDestination)
286 - throw new Error("Could not read the reference screenshots.");
287 - const verdicts = navFrames.map((nav, index) => {
288 - const chrome = chromeFrames[index]!;
289 - return {
290 - nav: progress(distance(nav, navOrigin), distance(nav, navDestination)),
291 - chrome: progress(distance(chrome, chromeOrigin), distance(chrome, chromeDestination)),
292 - };
293 - });
294 - return verdicts.flatMap((verdict, index) => (chromeLags(verdict) ? [index] : []));
295 -}
296 -
297 -async function saveFrame(video: string, index: number, path: string) {
298 - await run("ffmpeg", [
299 - "-v",
300 - "error",
301 - "-y",
302 - "-i",
303 - video,
304 - "-vf",
305 - `select=eq(n\\,${index})`,
306 - "-frames:v",
307 - "1",
308 - path,
309 - ]);
310 -}
311 -
312 -async function lagCheck(
313 - name: string,
314 - input: Parameters<typeof chromeLagFrames>[0],
315 - evidence: string,
316 -): Promise<{ check: Check; frame?: string }> {
317 - const lagging = await chromeLagFrames(input);
318 - if (lagging.length < LAG_FRAME_LIMIT)
319 - return { check: check(name, true, `${lagging.length} lagging video frames`) };
320 - await saveFrame(input.video, lagging[0]!, evidence);
321 - return {
322 - check: check(name, false, `${lagging.length} video frames (first #${lagging[0]})`),
323 - frame: evidence,
324 - };
325 -}
326 -
327 -function run(
328 - command: string,
329 - args: ReadonlyArray<string>,
330 - options: { readonly env?: NodeJS.ProcessEnv; readonly allowFailure?: boolean } = {},
331 -): Promise<{ readonly code: number; readonly stdout: string; readonly stderr: string }> {
332 - return new Promise((resolve, reject) => {
333 - const child = NodeChildProcess.spawn(command, args, {
334 - cwd: REPO_ROOT,
335 - env: options.env ?? NodeProcess.env,
336 - stdio: ["ignore", "pipe", "pipe"],
337 - });
338 - let stdout = "";
339 - let stderr = "";
340 - child.stdout.on("data", (chunk) => (stdout += chunk));
341 - child.stderr.on("data", (chunk) => (stderr += chunk));
342 - child.on("error", reject);
343 - child.on("close", (code) => {
344 - if (code !== 0 && !options.allowFailure)
345 - reject(new Error(`${command} ${args.join(" ")} failed (${code}): ${stderr || stdout}`));
346 - else resolve({ code: code ?? 1, stdout, stderr });
347 - });
348 - });
349 -}
350 -
351 -const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
352 -
353 -class Device {
354 - readonly spec: E2eDevice;
355 - readonly udid: string;
356 -
357 - constructor(spec: E2eDevice, udid: string) {
358 - this.spec = spec;
359 - this.udid = udid;
360 - }
361 -
362 - async agent(args: ReadonlyArray<string>): Promise<Record<string, unknown>> {
363 - const { stdout } = await run(
364 - NodeProcess.execPath,
365 - [
366 - AGENT_DEVICE,
367 - ...args,
368 - "--platform",
369 - "ios",
370 - "--udid",
371 - this.udid,
372 - "--session",
373 - `t3-e2e-${this.spec.id}`,
374 - "--json",
375 - ],
376 - {
377 - env: { ...NodeProcess.env, AGENT_DEVICE_STATE_DIR: AGENT_DEVICE_STATE },
378 - allowFailure: true,
379 - },
380 - );
381 - const parsed = JSON.parse(stdout) as {
382 - success: boolean;
383 - data?: Record<string, unknown>;
384 - error?: { message: string };
385 - };
386 - if (!parsed.success) throw new Error(`agent-device ${args[0]}: ${parsed.error?.message}`);
387 - return parsed.data ?? {};
388 - }
389 -
390 - async snapshot(): Promise<Snapshot> {
391 - return (await this.agent(["snapshot"])) as unknown as Snapshot;
392 - }
393 -
394 - async waitFor(predicate: (snapshot: Snapshot) => boolean, label: string, timeoutMs = 60_000) {
395 - const deadline = Date.now() + timeoutMs;
396 - let last: Snapshot | null = null;
397 - while (Date.now() < deadline) {
398 - last = await this.snapshot().catch(() => null);
399 - if (last && predicate(last)) return last;
400 - await delay(500);
401 - }
402 - throw new Error(`${this.spec.id}: timed out waiting for ${label}`);
403 - }
404 -
405 - /** Open a URL the way a user would; iOS asks before opening a custom scheme from another app. */
406 - async openUrl(url: string) {
407 - await run("xcrun", ["simctl", "openurl", this.udid, url]);
408 - await delay(1_500);
409 - await this.agent(["alert", "accept"]).catch(() => undefined);
410 - }
411 -
412 - async screenshot(path: string) {
413 - await run("xcrun", ["simctl", "io", this.udid, "screenshot", path]);
414 - }
415 -
416 - /** Run an action while recording and sampling the accessibility tree until the screen settles. */
417 - async transition(directory: string, name: string, action: () => Promise<unknown>) {
418 - const video = NodePath.join(directory, `${name}.mp4`);
419 - await this.agent(["record", "start", video, "--hide-touches"]);
420 - const samples: Snapshot[] = [];
421 - const sampling = { active: true };
422 - const sampler = (async () => {
423 - while (sampling.active) {
424 - const sample = await this.snapshot().catch(() => null);
425 - if (sample) samples.push(sample);
426 - }
427 - })();
428 - try {
429 - await action();
430 - await delay(2_000);
431 - } finally {
432 - sampling.active = false;
433 - await sampler;
434 - await this.agent(["record", "stop"]).catch(() => undefined);
435 - }
436 - const contactSheet = NodePath.join(directory, `${name}-frames.png`);
437 - const sheet = await run(
438 - NodeProcess.execPath,
439 - [AGENT_DEVICE, "record", "contact-sheet", video, "--out", contactSheet],
440 - {
441 - env: { ...NodeProcess.env, AGENT_DEVICE_STATE_DIR: AGENT_DEVICE_STATE },
442 - allowFailure: true,
443 - },
444 - );
445 - return { samples, video, contactSheet: sheet.code === 0 ? contactSheet : undefined };
446 - }
447 -}
448 -
449 -/** Screenshot pixels per snapshot point (3 on most iPhones). */
450 -async function screenshotScale(path: string, snapshot: Snapshot): Promise<number> {
451 - const { stdout } = await run("ffprobe", [
452 - "-v",
453 - "error",
454 - "-select_streams",
455 - "v:0",
456 - "-show_entries",
457 - "stream=width",
458 - "-of",
459 - "csv=p=0",
460 - path,
461 - ]);
462 - return Number(stdout.trim()) / snapshot.viewport.width;
463 -}
464 -
465 -async function ensureSimulator(spec: E2eDevice): Promise<string> {
466 - const name = `T3 E2E ${spec.id}`;
467 - const listing = JSON.parse(
468 - (await run("xcrun", ["simctl", "list", "devices", "available", "-j"])).stdout,
469 - ) as { devices: Record<string, ReadonlyArray<{ udid: string; name: string; state: string }>> };
470 - const existing = (listing.devices[spec.runtime] ?? []).find((device) => device.name === name);
471 - const udid =
472 - existing?.udid ??
473 - (await run("xcrun", ["simctl", "create", name, spec.deviceType, spec.runtime])).stdout.trim();
474 - if (existing?.state !== "Booted")
475 - await run("xcrun", ["simctl", "boot", udid], { allowFailure: true });
476 - await run("xcrun", ["simctl", "bootstatus", udid, "-b"]);
477 - await run("xcrun", ["simctl", "ui", udid, "appearance", "light"]);
478 - await run("xcrun", [
479 - "simctl",
480 - "status_bar",
481 - udid,
482 - "override",
483 - "--time",
484 - "9:41",
485 - "--batteryState",
486 - "charged",
487 - "--batteryLevel",
488 - "100",
489 - ]);
490 - return udid;
491 -}
26 +const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`);
27
493 -function lanAddress(): string {
28 +/** This machine's Tailscale address (100.64.0.0/10), which pool devices reach it on. */
29 +function tailnetAddress(): string {
30 for (const addresses of Object.values(NodeOS.networkInterfaces())) {
31 for (const address of addresses ?? []) {
496 - if (address.family === "IPv4" && !address.internal && !address.address.startsWith("169.254."))
32 + const [first, second] = address.address.split(".").map(Number);
33 + if (address.family === "IPv4" && first === 100 && second! >= 64 && second! < 128)
34 return address.address;
35 }
36 }
500 - throw new Error("No LAN IPv4 address for the simulator to reach Metro.");
37 + throw new Error("No Tailscale IPv4 address: pool devices reach Metro and the fixture over it.");
38 }
39
503 -async function runDevice(
504 - spec: E2eDevice,
505 - outputDirectory: string,
506 - environment: { readonly baseDir: string; readonly port: number },
507 - metroHost: string,
508 -): Promise<DeviceResult> {
509 - const steps: StepResult[] = [];
510 - const directory = NodePath.join(outputDirectory, spec.id);
511 - await NodeFSP.mkdir(directory, { recursive: true });
512 - try {
513 - const udid = await ensureSimulator(spec);
514 - const device = new Device(spec, udid);
515 - // A fresh install drops connections paired by earlier runs; the cached client reinstalls in seconds.
516 - await run("xcrun", ["simctl", "uninstall", udid, BUNDLE_ID], { allowFailure: true });
517 - log(spec.id, "installing the dev client");
518 - await run(NodeProcess.execPath, ["scripts/mobile-native-client.ts", "ensure", "ios", udid], {
519 - env: { ...NodeProcess.env, T3_NATIVE_CLIENT_CACHE: NodeProcess.env.T3_NATIVE_CLIENT_CACHE },
520 - });
521 - const metroUrl = encodeURIComponent(`http://${metroHost}:${METRO_PORT}`);
522 - await device.openUrl(
523 - `t3code-dev://expo-development-client/?url=${metroUrl}&disableOnboarding=1&disableFab=1&disableAutoLaunch=1`,
524 - );
525 - // Snapshots read through an agent-device app session; open attaches one to the running app.
526 - await device.agent(["open", BUNDLE_ID]);
527 - // The app ignores the pairing link until its JS has loaded and shown the workspace.
528 - await device.waitFor(
529 - (snapshot) => chrome(snapshot).newChat !== undefined,
530 - "the app to load",
531 - 120_000,
532 - );
533 - const paired = (snapshot: Snapshot) =>
534 - chrome(snapshot).tabBar !== undefined && chrome(snapshot).threadRows.length > 0;
535 - let home: Snapshot | null = null;
536 - for (let attempt = 1; attempt <= 2 && !home; attempt++) {
537 - log(spec.id, attempt === 1 ? "pairing" : "pairing again");
538 - const pairing = new URLSearchParams({
539 - pairingUrl: await issuePairUrl(environment),
540 - autoConnect: "1",
541 - });
542 - await device.openUrl(`t3code-dev://connections/new?${pairing}`);
543 - home = await device.waitFor(paired, "the paired thread list", 45_000).catch(() => null);
544 - }
545 - if (!home) throw new Error("The app did not show the fixture threads after pairing.");
546 - await delay(1_000);
547 - const homeShot = NodePath.join(directory, "home.png");
548 - await device.screenshot(homeShot);
549 - steps.push({ name: "home", checks: homeChecks(home), screenshot: homeShot, samples: 0 });
550 -
551 - log(spec.id, "opening a thread");
552 - const row = chrome(home).threadRows[0];
553 - if (!row?.label) throw new Error("No thread row to open.");
554 - const opened = await device.transition(directory, "open-thread", () =>
555 - device.agent(["press", `label=${JSON.stringify(row.label)}`]),
556 - );
557 - const thread = await device.waitFor(
558 - (snapshot) => chrome(snapshot).backButton !== undefined,
559 - "the thread screen",
560 - );
561 - const threadShot = NodePath.join(directory, "thread.png");
562 - await device.screenshot(threadShot);
563 - const { tabBar, navigationBar } = chrome(home);
564 - const scale = await screenshotScale(homeShot, home);
565 - const regions =
566 - tabBar?.rect && navigationBar?.rect ? { nav: navigationBar.rect, chrome: tabBar.rect } : null;
567 - const openLag = regions
568 - ? await lagCheck(
569 - "no tabs over the entering thread (video)",
570 - { video: opened.video, origin: homeShot, destination: threadShot, scale, ...regions },
571 - NodePath.join(directory, "open-thread-lag.png"),
572 - )
573 - : null;
574 - steps.push({
575 - name: "open thread",
576 - checks: [...threadChecks(thread, opened.samples), ...(openLag ? [openLag.check] : [])],
577 - lagFrame: openLag?.frame,
578 - screenshot: threadShot,
579 - video: opened.video,
580 - contactSheet: opened.contactSheet,
581 - samples: opened.samples.length,
582 - });
583 -
584 - log(spec.id, "going back");
585 - const back = await device.transition(directory, "back", () =>
586 - device.agent(["press", "id=BackButton"]),
587 - );
588 - const list = await device.waitFor(
589 - (snapshot) => chrome(snapshot).backButton === undefined,
590 - "the thread list",
591 - );
592 - const backShot = NodePath.join(directory, "back.png");
593 - await device.screenshot(backShot);
594 - const backLag = regions
595 - ? await lagCheck(
596 - "tabs return with the list (video)",
597 - { video: back.video, origin: threadShot, destination: backShot, scale, ...regions },
598 - NodePath.join(directory, "back-lag.png"),
599 - )
600 - : null;
601 - steps.push({
602 - name: "back",
603 - checks: [...backChecks(list, back.samples), ...(backLag ? [backLag.check] : [])],
604 - lagFrame: backLag?.frame,
605 - screenshot: backShot,
606 - video: back.video,
607 - contactSheet: back.contactSheet,
608 - samples: back.samples.length,
609 - });
610 - return { device: spec, label: spec.id, steps };
611 - } catch (error) {
612 - return {
613 - device: spec,
614 - label: spec.id,
615 - steps,
616 - error: error instanceof Error ? error.message : String(error),
617 - };
618 - }
40 +async function mapConcurrent<T, R>(
41 + items: ReadonlyArray<T>,
42 + limit: number,
43 + task: (item: T) => Promise<R>,
44 +): Promise<R[]> {
45 + const results: R[] = Array.from({ length: items.length });
46 + let next = 0;
47 + await Promise.all(
48 + Array.from({ length: Math.min(limit, items.length) }, async () => {
49 + while (next < items.length) {
50 + const index = next++;
51 + results[index] = await task(items[index]!);
52 + }
53 + }),
54 + );
55 + return results;
56 }
57
621 -async function issuePairUrl(environment: { readonly baseDir: string; readonly port: number }) {
622 - const { stdout } = await run(NodeProcess.execPath, [
623 - "apps/server/src/bin.ts",
624 - "auth",
625 - "pairing",
626 - "create",
627 - "--base-dir",
628 - environment.baseDir,
629 - "--base-url",
630 - `http://127.0.0.1:${environment.port}`,
631 - "--ttl",
632 - "15m",
633 - "--json",
634 - ]);
58 +async function issuePairUrl(
59 + environment: { readonly baseDir: string; readonly port: number },
60 + host: string,
61 +) {
62 + const { stdout } = await run(
63 + NodeProcess.execPath,
64 + [
65 + "apps/server/src/bin.ts",
66 + "auth",
67 + "pairing",
68 + "create",
69 + "--base-dir",
70 + environment.baseDir,
71 + "--base-url",
72 + `http://${host}:${environment.port}`,
73 + "--ttl",
74 + "15m",
75 + "--json",
76 + ],
77 + { cwd: REPO_ROOT },
78 + );
79 const { pairUrl } = JSON.parse(
80 stdout.slice(stdout.indexOf("{"), stdout.lastIndexOf("}") + 1),
81 ) as {
@@ -641,71 +85,109 @@ async function issuePairUrl(environment: { readonly baseDir: string; readonly po
85 return pairUrl;
86 }
87
644 -const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`);
645 -
646 -const escapeHtml = (value: string) =>
647 - value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
648 -
649 -function renderReport(results: ReadonlyArray<DeviceResult>, outputDirectory: string): string {
650 - const relative = (path: string | undefined) =>
651 - path ? escapeHtml(NodePath.relative(outputDirectory, path)) : "";
652 - const rows = results
653 - .map((result) => {
654 - const steps = result.steps
655 - .map((step) => {
656 - const checks = step.checks
657 - .map(
658 - (item) =>
659 - `<li class="${item.pass ? "pass" : "fail"}">${item.pass ? "✓" : "✗"} ${escapeHtml(item.name)} <small>${escapeHtml(item.detail)}</small></li>`,
660 - )
661 - .join("");
662 - const media = [
663 - step.screenshot
664 - ? `<a href="${relative(step.screenshot)}"><img src="${relative(step.screenshot)}"></a>`
665 - : "",
666 - step.lagFrame
667 - ? `<a href="${relative(step.lagFrame)}"><img class="lag" src="${relative(step.lagFrame)}"></a>`
668 - : "",
669 - step.contactSheet
670 - ? `<a href="${relative(step.contactSheet)}">transition frames</a>`
671 - : "",
672 - step.video ? `<a href="${relative(step.video)}">video</a>` : "",
673 - ].join(" ");
674 - return `<td><h3>${escapeHtml(step.name)}</h3><ul>${checks}</ul>${media}</td>`;
675 - })
676 - .join("");
677 - const error = result.error ? `<td class="fail">${escapeHtml(result.error)}</td>` : "";
678 - return `<tr><th>${escapeHtml(result.label)}</th>${steps}${error}</tr>`;
679 - })
680 - .join("");
681 - const failures = results.filter(
682 - (result) => result.error || result.steps.some((step) => step.checks.some((item) => !item.pass)),
683 - ).length;
684 - return `<!doctype html><meta charset="utf-8"><title>Mobile E2E</title>
685 -<style>body{font:14px system-ui;margin:24px}table{border-collapse:collapse}td,th{border:1px solid #ddd;padding:8px;vertical-align:top}img{width:180px;display:block;margin:6px 0}img.lag{outline:3px solid #b42318}.pass{color:#16794a}.fail{color:#b42318}ul{padding-left:16px;margin:4px 0}small{color:#666}</style>
686 -<h1>Mobile E2E · ${failures === 0 ? "all devices passed" : `${failures} device(s) failed`}</h1>
687 -<table>${rows}</table>`;
88 +async function runPose(
89 + pose: Pose,
90 + pool: Pool,
91 + runId: string,
92 + outputDirectory: string,
93 + metroUrl: string,
94 + pair: () => Promise<string>,
95 +): Promise<PoseResult> {
96 + const directory = NodePath.join(outputDirectory, pose.id);
97 + await NodeFSP.mkdir(directory, { recursive: true });
98 + const steps: StepResult[] = [];
99 + let device: Device | null = null;
100 + try {
101 + log(pose.id, `leasing ${pose.device}`);
102 + device = await Device.lease(pose, pool, runId);
103 + log(pose.id, "installing the dev client");
104 + await device.install();
105 + await device.prepare();
106 + await device.launch(metroUrl);
107 + log(pose.id, "pairing and walking the app");
108 + await runJourney(device, directory, pair, steps);
109 + return { pose, steps };
110 + } catch (error) {
111 + const message = error instanceof Error ? error.message : String(error);
112 + if (!device) return { pose, steps, error: message };
113 + // What the device showed when the journey stopped is usually the whole diagnosis.
114 + const stopped = NodePath.join(directory, "stopped.png");
115 + const shot = await device.screenshot(stopped).then(
116 + () => stopped,
117 + () => undefined,
118 + );
119 + const labels = await device.snapshot().then(
120 + (snapshot) =>
121 + snapshot.nodes
122 + .map((node) => node.label)
123 + .filter(Boolean)
124 + .slice(0, 12)
125 + .join(" | "),
126 + () => "",
127 + );
128 + return {
129 + pose,
130 + steps,
131 + error: labels ? `${message} (screen: ${labels})` : message,
132 + ...(shot ? { stopped: shot } : {}),
133 + };
134 + } finally {
135 + await device?.release();
136 + }
137 }
138
139 async function main() {
140 const args = NodeProcess.argv.slice(2);
141 if (args.includes("--list")) {
693 - for (const device of DEVICES) NodeProcess.stdout.write(`${device.id}\n`);
142 + for (const pose of POSES) NodeProcess.stdout.write(`${pose.id}\t${pose.device}\n`);
143 return;
144 }
696 - const selected = args.flatMap((arg, index) => (args[index - 1] === "--device" ? [arg] : []));
697 - const devices = selected.length
698 - ? DEVICES.filter((device) => selected.includes(device.id))
699 - : DEVICES;
700 - const unknown = selected.filter((id) => !DEVICES.some((device) => device.id === id));
701 - if (unknown.length) throw new Error(`Unknown device ${unknown.join(", ")}. Run with --list.`);
702 - const concurrencyArg = args[args.indexOf("--concurrency") + 1];
703 - const concurrency = args.includes("--concurrency") ? Number(concurrencyArg) : 3;
145 + const values = (flag: string) =>
146 + args.flatMap((arg, index) => (args[index - 1] === flag ? [arg] : []));
147 + const selected = values("--pose");
148 + const unknown = selected.filter((id) => !POSES.some((pose) => pose.id === id));
149 + if (unknown.length) throw new Error(`Unknown pose ${unknown.join(", ")}. Run with --list.`);
150 + const platforms = values("--platform");
151 + const poses = POSES.filter(
152 + (pose) =>
153 + (selected.length === 0 || selected.includes(pose.id)) &&
154 + (platforms.length === 0 || platforms.includes(pose.platform)),
155 + );
156 + const concurrencyFlag = values("--concurrency")[0];
157 + const concurrency = concurrencyFlag ? Number(concurrencyFlag) : undefined;
158 + const review = !args.includes("--no-review");
159 +
160 + const agentDeviceUrl = NodeProcess.env.SIMLOCK_AGENT_DEVICE_URL;
161 + const token = NodeProcess.env.AGENT_DEVICE_DAEMON_AUTH_TOKEN;
162 + if (!agentDeviceUrl || !token)
163 + throw new Error(
164 + "Set SIMLOCK_AGENT_DEVICE_URL (<gateway>/v1/agent-device) and AGENT_DEVICE_DAEMON_AUTH_TOKEN (a Simlock agent token) to run on the device pool.",
165 + );
166 + const host = tailnetAddress();
167
705 - const stamp = new Date().toISOString().replaceAll(":", "-");
706 - const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-e2e", stamp);
168 + const runId = new Date().toISOString().replaceAll(/[:.]/g, "-");
169 + const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-e2e", runId);
170 await NodeFSP.mkdir(outputDirectory, { recursive: true });
171 const rootDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-mobile-e2e-"));
172 + const workspace = await NodeFSP.realpath(outputDirectory);
173 + const config = NodePath.join(rootDir, "agent-device.json");
174 + await NodeFSP.writeFile(
175 + config,
176 + JSON.stringify({
177 + daemonBaseUrl: agentDeviceUrl,
178 + daemonAuthToken: token,
179 + stateDir: NodePath.join(rootDir, "agent-device"),
180 + }),
181 + { mode: 0o600 },
182 + );
183 + const pool: Pool = {
184 + repoRoot: REPO_ROOT,
185 + agentDevice: NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"),
186 + agentDeviceUrl,
187 + workspace,
188 + env: { ...NodeProcess.env, AGENT_DEVICE_CONFIG: config, PWD: workspace },
189 + };
190 +
191 const servers: NodeChildProcess.ChildProcess[] = [];
192 let metro: NodeChildProcess.ChildProcess | null = null;
193 try {
@@ -716,7 +198,7 @@ async function main() {
198 SHOWCASE_ENVIRONMENTS.slice(0, 1),
199 );
200 if (!environment) throw new Error("The fixture server did not start.");
719 - const metroHost = lanAddress();
201 + const metroPort = await reserveAvailablePort();
202 metro = NodeChildProcess.spawn(
203 "vp",
204 [
@@ -728,7 +210,7 @@ async function main() {
210 "t3code-dev",
211 "--lan",
212 "--port",
731 - String(METRO_PORT),
213 + String(metroPort),
214 ],
215 {
216 cwd: NodePath.join(REPO_ROOT, "apps/mobile"),
@@ -738,53 +220,83 @@ async function main() {
220 detached: true,
221 },
222 );
741 - await waitForPort(METRO_PORT, "Metro", 120_000);
742 - await run("curl", [
743 - "-sf",
744 - "-o",
745 - "/dev/null",
746 - `http://127.0.0.1:${METRO_PORT}/apps/mobile/index.bundle?platform=ios&dev=true&minify=false`,
747 - ]);
223 + await waitForPort(metroPort, "Metro", 120_000);
224 + // Warm each platform's bundle once, instead of on every device at once.
225 + for (const platform of new Set(poses.map((pose) => pose.platform))) {
226 + log("e2e", `bundling for ${platform}`);
227 + await run("curl", [
228 + "-sf",
229 + "-o",
230 + "/dev/null",
231 + `http://127.0.0.1:${metroPort}/apps/mobile/index.bundle?platform=${platform}&dev=true&minify=false`,
232 + ]);
233 + }
234
749 - // The first install may build; later ones reuse its artifact.
750 - const results: DeviceResult[] = [];
751 - const [first, ...rest] = devices;
752 - if (first) results.push(await runDevice(first, outputDirectory, environment, metroHost));
753 - for (let index = 0; index < rest.length; index += concurrency) {
754 - results.push(
755 - ...(await Promise.all(
756 - rest
757 - .slice(index, index + concurrency)
758 - .map((device) => runDevice(device, outputDirectory, environment, metroHost)),
759 - )),
235 + const attempt = (pose: Pose, session: string) =>
236 + runPose(pose, pool, session, outputDirectory, `http://${host}:${metroPort}`, () =>
237 + issuePairUrl(environment, host),
238 );
761 - }
239 + const runOne = async (pose: Pose) => {
240 + const first = await attempt(pose, runId);
241 + // A pose that failed before its first screen usually met the pool, not the app.
242 + if (!first.error || first.steps.length > 0) return first;
243 + log(pose.id, `retrying after: ${first.error.split("\n")[0]}`);
244 + return attempt(pose, `${runId}-retry`);
245 + };
246 + // Each platform has its own workers, so each gets its own limit; one Mac runs every
247 + // iOS simulator, and recording several at once on a shared pool times it out.
248 + const byPlatform = await Promise.all(
249 + (["ios", "android"] as const).map((platform) =>
250 + mapConcurrent(
251 + poses.filter((pose) => pose.platform === platform),
252 + concurrency ?? (platform === "ios" ? 2 : 3),
253 + runOne,
254 + ),
255 + ),
256 + );
257 + const results = poses.map((pose) => byPlatform.flat().find((result) => result.pose === pose)!);
258 +
259 + const reviewed = review
260 + ? await mapConcurrent(results, 4, async (result) => {
261 + log(result.pose.id, "agent review");
262 + const findings = await reviewPose(result, outputDirectory).catch((error) => {
263 + log(result.pose.id, `review failed: ${error instanceof Error ? error.message : error}`);
264 + return [];
265 + });
266 + return { ...result, findings };
267 + })
268 + : results;
269 +
270 const report = NodePath.join(outputDirectory, "index.html");
763 - await NodeFSP.writeFile(report, renderReport(results, outputDirectory));
271 + await NodeFSP.writeFile(report, renderReport(reviewed, outputDirectory));
272 await NodeFSP.writeFile(
273 NodePath.join(outputDirectory, "report.json"),
766 - JSON.stringify(results, null, 2),
274 + JSON.stringify(reviewed, null, 2),
275 );
768 - const failed = results.filter(
769 - (result) =>
770 - result.error || result.steps.some((step) => step.checks.some((item) => !item.pass)),
771 - );
772 - for (const result of results) {
773 - const failures = result.steps.flatMap((step) =>
774 - step.checks
775 - .filter((item) => !item.pass)
776 - .map((item) => `${step.name}: ${item.name} (${item.detail})`),
276 + for (const result of reviewed) {
277 + const failures = failedChecks(result).map(
278 + (item) => `${item.step}: ${item.name} (${item.detail})`,
279 + );
280 + log(
281 + result.pose.id,
282 + [result.error, ...failures].filter(Boolean).join("; ") ||
283 + `passed${result.findings?.length ? `, ${result.findings.length} review findings` : ""}`,
284 );
778 - log(result.label, result.error ?? (failures.length ? failures.join("; ") : "passed"));
285 }
286 + const failed = reviewed.filter(poseFailed).length;
287 NodeProcess.stdout.write(
781 - `${JSON.stringify({ report, devices: results.length, failed: failed.length })}\n`,
288 + `${JSON.stringify({
289 + report,
290 + poses: reviewed.length,
291 + failed,
292 + findings: reviewed.reduce((count, result) => count + (result.findings?.length ?? 0), 0),
293 + })}\n`,
294 );
783 - process.exitCode = failed.length ? 1 : 0;
295 + process.exitCode = failed ? 1 : 0;
296 } finally {
297 if (metro?.pid) {
298 try {
787 - process.kill(-metro.pid, "SIGTERM");
299 + NodeProcess.kill(-metro.pid, "SIGTERM");
300 } catch {}
301 await stopProcess(metro);
302 }
scripts/mobile-e2e/checks.test.ts new
+115
@@ -0,0 +1,115 @@
1 +import { describe, expect, it } from "@effect/vitest";
2 +
3 +import { layoutChecks, type Snapshot, type SnapshotNode } from "./checks.ts";
4 +
5 +const APP = "com.t3tools.t3code.dev";
6 +const node = (index: number, fields: Omit<SnapshotNode, "index">): SnapshotNode => ({
7 + index,
8 + hittable: true,
9 + kind: "button",
10 + ...fields,
11 +});
12 +
13 +const failing = (snapshot: Snapshot) =>
14 + Object.fromEntries(
15 + layoutChecks(snapshot, APP)
16 + .filter((item) => !item.pass)
17 + .map((item) => [item.name, item.detail]),
18 + );
19 +
20 +describe("layoutChecks", () => {
21 + it("flags broken chrome but not scrolled content, nested controls or duplicates", () => {
22 + const snapshot: Snapshot = {
23 + appBundleId: APP,
24 + viewport: { width: 400, height: 800 },
25 + nodes: [
26 + node(0, {
27 + kind: "application",
28 + hittable: false,
29 + rect: { x: 0, y: 0, width: 400, height: 800 },
30 + }),
31 + node(1, {
32 + kind: "scroll-area",
33 + parentIndex: 0,
34 + rect: { x: 0, y: 0, width: 400, height: 700 },
35 + }),
36 + node(2, {
37 + label: "Row past the edge",
38 + parentIndex: 1,
39 + rect: { x: 0, y: 680, width: 400, height: 60 },
40 + }),
41 + node(3, { label: "Send", parentIndex: 0, rect: { x: 340, y: 770, width: 80, height: 40 } }),
42 + node(4, {
43 + label: "Card",
44 + parentIndex: 0,
45 + rect: { x: 10, y: 100, width: 200, height: 100 },
46 + }),
47 + node(5, { label: "Inner", parentIndex: 4, rect: { x: 20, y: 110, width: 50, height: 40 } }),
48 + node(6, { label: "Twin", parentIndex: 0, rect: { x: 10, y: 300, width: 80, height: 40 } }),
49 + node(7, { label: "Twin", parentIndex: 0, rect: { x: 10, y: 300, width: 80, height: 40 } }),
50 + node(8, {
51 + label: "Plan",
52 + parentIndex: 0,
53 + rect: { x: 100, y: 400, width: 100, height: 40 },
54 + }),
55 + node(9, {
56 + label: "Stop",
57 + parentIndex: 0,
58 + rect: { x: 120, y: 405, width: 100, height: 40 },
59 + }),
60 + node(10, {
61 + kind: "text-field",
62 + label: "Message",
63 + parentIndex: 0,
64 + rect: { x: 0, y: 600, width: 300, height: 40 },
65 + }),
66 + node(12, {
67 + label: "Row of a covered screen",
68 + parentIndex: 1,
69 + rect: { x: 100, y: 400, width: 100, height: 40 },
70 + }),
71 + node(11, {
72 + kind: "keyboard",
73 + hittable: false,
74 + rect: { x: 0, y: 500, width: 400, height: 300 },
75 + }),
76 + ],
77 + };
78 +
79 + expect(failing(snapshot)).toEqual({
80 + "no control cut off by the screen edge": '"Send" at 340,770 80×40',
81 + "no controls stacked on each other": '"Plan" and "Stop"',
82 + "no input under the keyboard": '"Message" at 0,600 300×40',
83 + });
84 + });
85 +
86 + it("flags an error screen and controls under Android system bars", () => {
87 + const snapshot: Snapshot = {
88 + appBundleId: APP,
89 + viewport: { width: 1080, height: 2400 },
90 + nodes: [
91 + node(0, {
92 + kind: "text",
93 + label: "Render Error",
94 + rect: { x: 0, y: 200, width: 600, height: 60 },
95 + }),
96 + node(1, {
97 + kind: "text",
98 + systemBar: "status",
99 + bundleId: "com.android.systemui",
100 + rect: { x: 0, y: 0, width: 1080, height: 60 },
101 + }),
102 + node(2, {
103 + label: "Search",
104 + bundleId: APP,
105 + rect: { x: 900, y: 10, width: 120, height: 80 },
106 + }),
107 + ],
108 + };
109 +
110 + expect(Object.keys(failing(snapshot))).toEqual([
111 + "no error screen",
112 + "no control under a system bar",
113 + ]);
114 + });
115 +});
scripts/mobile-e2e/checks.ts new
+203
@@ -0,0 +1,203 @@
1 +export interface Rect {
2 + readonly x: number;
3 + readonly y: number;
4 + readonly width: number;
5 + readonly height: number;
6 +}
7 +
8 +/** One node of an agent-device `snapshot --json`, on iOS or Android. */
9 +export interface SnapshotNode {
10 + readonly index: number;
11 + readonly parentIndex?: number;
12 + /** Valid for commands until the next snapshot. */
13 + readonly ref?: string;
14 + readonly kind?: string;
15 + readonly type?: string;
16 + readonly label?: string;
17 + readonly identifier?: string;
18 + readonly rect?: Rect;
19 + readonly hittable?: boolean;
20 + readonly editable?: boolean;
21 + readonly bundleId?: string;
22 + readonly systemBar?: string;
23 +}
24 +
25 +export interface Snapshot {
26 + readonly nodes: ReadonlyArray<SnapshotNode>;
27 + readonly viewport: { readonly width: number; readonly height: number };
28 + readonly appBundleId?: string;
29 + readonly keyboard?: { readonly kind?: string; readonly rect?: Rect };
30 +}
31 +
32 +export interface Check {
33 + readonly name: string;
34 + readonly pass: boolean;
35 + readonly detail: string;
36 +}
37 +
38 +export const check = (name: string, pass: boolean, detail: string): Check => ({
39 + name,
40 + pass,
41 + detail,
42 +});
43 +
44 +// Kinds that are never controls a person taps; iOS also marks plain text hittable.
45 +const NOT_CONTROLS = new Set([
46 + "application",
47 + "text",
48 + "image",
49 + "scroll-area",
50 + "other",
51 + "toolbar",
52 + "tab-bar",
53 + "activity-indicator",
54 + "composeview",
55 + "keyboard",
56 +]);
57 +
58 +const area = (rect: Rect) => Math.max(0, rect.width) * Math.max(0, rect.height);
59 +
60 +const intersection = (a: Rect, b: Rect) =>
61 + Math.max(0, Math.min(a.x + a.width, b.x + b.width) - Math.max(a.x, b.x)) *
62 + Math.max(0, Math.min(a.y + a.height, b.y + b.height) - Math.max(a.y, b.y));
63 +
64 +export const describeRect = (rect: Rect | undefined) =>
65 + rect
66 + ? `${Math.round(rect.x)},${Math.round(rect.y)} ${Math.round(rect.width)}×${Math.round(rect.height)}`
67 + : "absent";
68 +
69 +const name = (node: SnapshotNode) => node.label || node.identifier || node.type || "unlabelled";
70 +
71 +const appOwned = (node: SnapshotNode, snapshot: Snapshot) =>
72 + !node.systemBar && (node.bundleId === undefined || node.bundleId === snapshot.appBundleId);
73 +
74 +const isControl = (node: SnapshotNode, snapshot: Snapshot) =>
75 + node.hittable === true &&
76 + node.rect !== undefined &&
77 + area(node.rect) > 0 &&
78 + !NOT_CONTROLS.has(node.kind ?? "") &&
79 + appOwned(node, snapshot);
80 +
81 +function ancestors(node: SnapshotNode, snapshot: Snapshot): SnapshotNode[] {
82 + const byIndex = new Map(snapshot.nodes.map((candidate) => [candidate.index, candidate]));
83 + const chain: SnapshotNode[] = [];
84 + for (let parent = node.parentIndex; parent !== undefined;) {
85 + const next = byIndex.get(parent);
86 + if (!next || chain.includes(next)) break;
87 + chain.push(next);
88 + parent = next.parentIndex;
89 + }
90 + return chain;
91 +}
92 +
93 +const viewportRect = (snapshot: Snapshot): Rect => ({ x: 0, y: 0, ...snapshot.viewport });
94 +
95 +export function keyboardRect(snapshot: Snapshot): Rect | undefined {
96 + if (snapshot.keyboard?.rect) return snapshot.keyboard.rect;
97 + return snapshot.nodes.find((node) => node.type === "Keyboard" || node.kind === "keyboard")?.rect;
98 +}
99 +
100 +const TAB_LABELS = new Set(["Chats", "Pulls", "Usage", "Settings"]);
101 +
102 +/** The bottom tab bar, as a native tab bar node or the union of its tab buttons. */
103 +export function tabBarRect(snapshot: Snapshot): Rect | undefined {
104 + const native = snapshot.nodes.find((node) => node.kind === "tab-bar" && node.rect)?.rect;
105 + if (native) return native;
106 + const tabs = snapshot.nodes.filter(
107 + (node) =>
108 + node.rect &&
109 + TAB_LABELS.has(node.label ?? "") &&
110 + node.rect.y > snapshot.viewport.height * 0.7 &&
111 + appOwned(node, snapshot),
112 + );
113 + if (tabs.length < 2) return undefined;
114 + const left = Math.min(...tabs.map((tab) => tab.rect!.x));
115 + const top = Math.min(...tabs.map((tab) => tab.rect!.y));
116 + const right = Math.max(...tabs.map((tab) => tab.rect!.x + tab.rect!.width));
117 + const bottom = Math.max(...tabs.map((tab) => tab.rect!.y + tab.rect!.height));
118 + return { x: left, y: top, width: right - left, height: bottom - top };
119 +}
120 +
121 +const ERROR_TEXT =
122 + /\b(Render Error|Uncaught Error|Unhandled JS Exception|Syntax Error|keeps stopping|isn't responding)\b/i;
123 +
124 +/**
125 + * Checks every screen should pass, whatever it shows: the app is in front without an error
126 + * screen, and no control is clipped by the screen edge, stacked on another control, under a
127 + * system bar, or an input under the keyboard.
128 + */
129 +export function layoutChecks(snapshot: Snapshot, bundleId: string): Check[] {
130 + const viewport = viewportRect(snapshot);
131 + const controls = snapshot.nodes.filter((node) => isControl(node, snapshot));
132 + const errors = snapshot.nodes.filter((node) => ERROR_TEXT.test(node.label ?? ""));
133 +
134 + const clipped = controls.filter((node) => {
135 + const visible = intersection(node.rect!, viewport);
136 + if (visible === 0 || visible >= area(node.rect!) * 0.98) return false;
137 + // Content scrolled past the edge is expected; fixed chrome cut off by it is not.
138 + return !ancestors(node, snapshot).some((parent) => parent.kind === "scroll-area");
139 + });
140 +
141 + // Only siblings: a screen covered by another stays in the tree, so cousins overlap
142 + // legitimately. Native tab items report overlapping frames.
143 + const siblings = controls.filter(
144 + (node) => !ancestors(node, snapshot).some((parent) => parent.kind === "tab-bar"),
145 + );
146 + const stacked: string[] = [];
147 + for (const [index, a] of siblings.entries()) {
148 + for (const b of siblings.slice(index + 1)) {
149 + if (a.parentIndex !== b.parentIndex) continue;
150 + const shared = intersection(a.rect!, b.rect!);
151 + if (shared < Math.min(area(a.rect!), area(b.rect!)) * 0.25) continue;
152 + // iOS exposes some elements twice, at the same frame.
153 + const same =
154 + describeRect(a.rect) === describeRect(b.rect) &&
155 + (!a.label || !b.label || a.label === b.label);
156 + if (!same) stacked.push(`"${name(a)}" and "${name(b)}"`);
157 + }
158 + }
159 +
160 + const bars = snapshot.nodes.filter((node) => node.systemBar && node.rect);
161 + const underBars = controls.filter((node) =>
162 + bars.some((bar) => intersection(node.rect!, bar.rect!) > area(node.rect!) * 0.3),
163 + );
164 +
165 + const keyboard = keyboardRect(snapshot);
166 + const coveredInputs = keyboard
167 + ? snapshot.nodes.filter(
168 + (node) =>
169 + (node.kind === "text-field" || node.editable === true) &&
170 + node.rect &&
171 + appOwned(node, snapshot) &&
172 + intersection(node.rect, keyboard) > area(node.rect) * 0.3,
173 + )
174 + : [];
175 +
176 + const list = (nodes: ReadonlyArray<SnapshotNode>) =>
177 + nodes.map((node) => `"${name(node)}" at ${describeRect(node.rect)}`).join(", ");
178 + return [
179 + check(
180 + "app in front",
181 + snapshot.appBundleId === undefined || snapshot.appBundleId === bundleId,
182 + snapshot.appBundleId ?? "unknown",
183 + ),
184 + check("no error screen", errors.length === 0, errors.map(name).join(", ") || "none"),
185 + check("no control cut off by the screen edge", clipped.length === 0, list(clipped) || "none"),
186 + check("no controls stacked on each other", stacked.length === 0, stacked.join("; ") || "none"),
187 + check("no control under a system bar", underBars.length === 0, list(underBars) || "none"),
188 + check(
189 + "no input under the keyboard",
190 + coveredInputs.length === 0,
191 + keyboard ? list(coveredInputs) || "none" : "no keyboard",
192 + ),
193 + ];
194 +}
195 +
196 +export const findLabel = (snapshot: Snapshot, pattern: RegExp) =>
197 + snapshot.nodes.find(
198 + (node) =>
199 + node.rect &&
200 + pattern.test(node.label ?? "") &&
201 + intersection(node.rect, viewportRect(snapshot)) > 0 &&
202 + appOwned(node, snapshot),
203 + );
scripts/mobile-e2e/device.ts new
+229
@@ -0,0 +1,229 @@
1 +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off - Host-side device automation uses Node subprocess and timing APIs directly.
2 +
3 +import * as NodeChildProcess from "node:child_process";
4 +import * as NodePath from "node:path";
5 +import * as NodeProcess from "node:process";
6 +
7 +import type { Snapshot } from "./checks.ts";
8 +import type { Pose } from "./poses.ts";
9 +
10 +export const BUNDLE_ID = "com.t3tools.t3code.dev";
11 +
12 +export interface RunResult {
13 + readonly code: number;
14 + readonly stdout: string;
15 + readonly stderr: string;
16 +}
17 +
18 +export function run(
19 + command: string,
20 + args: ReadonlyArray<string>,
21 + options: {
22 + readonly cwd?: string;
23 + readonly env?: NodeJS.ProcessEnv;
24 + readonly allowFailure?: boolean;
25 + readonly timeoutMs?: number;
26 + } = {},
27 +): Promise<RunResult> {
28 + return new Promise((resolve, reject) => {
29 + const child = NodeChildProcess.spawn(command, args, {
30 + ...(options.cwd ? { cwd: options.cwd } : {}),
31 + env: options.env ?? NodeProcess.env,
32 + stdio: ["ignore", "pipe", "pipe"],
33 + ...(options.timeoutMs ? { timeout: options.timeoutMs } : {}),
34 + });
35 + let stdout = "";
36 + let stderr = "";
37 + child.stdout.on("data", (chunk) => (stdout += chunk));
38 + child.stderr.on("data", (chunk) => (stderr += chunk));
39 + child.on("error", reject);
40 + child.on("close", (code) => {
41 + if (code !== 0 && !options.allowFailure)
42 + reject(new Error(`${command} ${args.join(" ")} failed (${code}): ${stderr || stdout}`));
43 + else resolve({ code: code ?? 1, stdout, stderr });
44 + });
45 + });
46 +}
47 +
48 +export const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
49 +
50 +/** How this run reaches the pool: one agent-device config, from one workspace. */
51 +export interface Pool {
52 + readonly repoRoot: string;
53 + readonly agentDevice: string;
54 + /** The gateway's agent-device front, `<gateway>/v1/agent-device`. */
55 + readonly agentDeviceUrl: string;
56 + /**
57 + * agent-device ties a lease to the workspace it was opened from and reads it from both the
58 + * real cwd and $PWD, so every command runs from this real path with a matching PWD.
59 + */
60 + readonly workspace: string;
61 + readonly env: NodeJS.ProcessEnv;
62 +}
63 +
64 +export class AgentDeviceError extends Error {}
65 +
66 +// A shared pool's workers drop off the gateway and stall under load for seconds at a time.
67 +const TRANSIENT =
68 + /Remote daemon is unavailable|not connected to the gateway|still finishing a previous command|Invalid daemon response|Daemon request timed out/i;
69 +
70 +export class Device {
71 + readonly pose: Pose;
72 + readonly session: string;
73 + readonly pool: Pool;
74 +
75 + private constructor(pose: Pose, session: string, pool: Pool) {
76 + this.pose = pose;
77 + this.session = session;
78 + this.pool = pool;
79 + }
80 +
81 + /** Leases a device for the pose; `open` waits in the pool's queue while every match is busy. */
82 + static async lease(pose: Pose, pool: Pool, runId: string): Promise<Device> {
83 + const device = new Device(pose, `t3-e2e-${runId}-${pose.id}`, pool);
84 + await device.agent(["connect", "proxy", "--daemon-base-url", pool.agentDeviceUrl]);
85 + await device.agent(
86 + [
87 + "open",
88 + "--platform",
89 + pose.platform,
90 + "--device",
91 + pose.device,
92 + // A real keyboard, so the layout under it is what people see.
93 + ...(pose.platform === "android" ? ["--no-test-ime"] : []),
94 + ],
95 + { timeoutMs: 420_000 },
96 + );
97 + return device;
98 + }
99 +
100 + async agent(
101 + args: ReadonlyArray<string>,
102 + options: { readonly timeoutMs?: number } = {},
103 + ): Promise<Record<string, unknown>> {
104 + for (let attempt = 1; ; attempt++) {
105 + try {
106 + return await this.agentOnce(args, options);
107 + } catch (error) {
108 + if (attempt >= 3 || !(error instanceof AgentDeviceError) || !TRANSIENT.test(error.message))
109 + throw error;
110 + await delay(3_000 * attempt);
111 + }
112 + }
113 + }
114 +
115 + private async agentOnce(
116 + args: ReadonlyArray<string>,
117 + options: { readonly timeoutMs?: number },
118 + ): Promise<Record<string, unknown>> {
119 + const { stdout, stderr } = await run(
120 + NodeProcess.execPath,
121 + [this.pool.agentDevice, ...args, "--session", this.session, "--json"],
122 + {
123 + cwd: this.pool.workspace,
124 + env: this.pool.env,
125 + allowFailure: true,
126 + timeoutMs: options.timeoutMs ?? 120_000,
127 + },
128 + );
129 + let parsed: { success: boolean; data?: Record<string, unknown>; error?: { message: string } };
130 + try {
131 + parsed = JSON.parse(stdout);
132 + } catch {
133 + throw new AgentDeviceError(`agent-device ${args[0]}: ${(stderr || stdout).slice(-400)}`);
134 + }
135 + if (!parsed.success)
136 + throw new AgentDeviceError(`agent-device ${args.join(" ")}: ${parsed.error?.message}`);
137 + return parsed.data ?? {};
138 + }
139 +
140 + /** The shared-cache dev client for this checkout; the worker downloads it itself. */
141 + async install() {
142 + await run(
143 + NodeProcess.execPath,
144 + [
145 + NodePath.join(this.pool.repoRoot, "scripts/mobile-native-client.ts"),
146 + "ensure",
147 + this.pose.platform,
148 + "--session",
149 + this.session,
150 + ],
151 + { cwd: this.pool.workspace, env: this.pool.env, timeoutMs: 600_000 },
152 + );
153 + }
154 +
155 + /** Device settings that must be in place before the app starts. */
156 + async prepare() {
157 + await this.agent(["settings", "appearance", this.pose.appearance]);
158 + if (this.pose.textSize) await this.agent(["settings", "text-size", this.pose.textSize]);
159 + if (this.pose.fold) await this.agent(["fold", this.pose.fold]);
160 + }
161 +
162 + async launch(metroUrl: string) {
163 + const url = encodeURIComponent(metroUrl);
164 + await this.agent(
165 + [
166 + "open",
167 + BUNDLE_ID,
168 + `t3code-dev://expo-development-client/?url=${url}&disableOnboarding=1&disableFab=1&disableAutoLaunch=1`,
169 + // A real keyboard, so the layout under it is what people see.
170 + ...(this.pose.platform === "android" ? ["--no-test-ime"] : []),
171 + ],
172 + { timeoutMs: 180_000 },
173 + );
174 + await this.acceptOpenPrompt();
175 + if (this.pose.landscape) await this.agent(["orientation", "landscape-left"]);
176 + }
177 +
178 + async openUrl(url: string) {
179 + await this.agent(["open", BUNDLE_ID, url]);
180 + await this.acceptOpenPrompt();
181 + }
182 +
183 + /** iOS asks before another app opens a custom scheme, and agent-device does not always answer. */
184 + private async acceptOpenPrompt() {
185 + if (this.pose.platform !== "ios") return;
186 + await delay(1_500);
187 + await this.agent(["alert", "accept"]).catch(() => undefined);
188 + }
189 +
190 + async snapshot(): Promise<Snapshot> {
191 + return (await this.agent(["snapshot"])) as unknown as Snapshot;
192 + }
193 +
194 + async waitFor(predicate: (snapshot: Snapshot) => boolean, label: string, timeoutMs = 60_000) {
195 + const deadline = Date.now() + timeoutMs;
196 + while (Date.now() < deadline) {
197 + const snapshot = await this.snapshot().catch(() => null);
198 + if (snapshot && predicate(snapshot)) return snapshot;
199 + await delay(750);
200 + }
201 + throw new Error(`timed out waiting for ${label}`);
202 + }
203 +
204 + async screenshot(path: string) {
205 + // iOS defaults to 1x points, too coarse for anyone reviewing the screenshot.
206 + await this.agent([
207 + "screenshot",
208 + path,
209 + ...(this.pose.platform === "ios" ? ["--pixel-density", "2"] : []),
210 + ]);
211 + }
212 +
213 + /** Runs an action while recording, so transitions can be checked frame by frame. */
214 + async recorded(directory: string, name: string, action: () => Promise<unknown>) {
215 + const video = NodePath.join(directory, `${name}.mp4`);
216 + await this.agent(["record", "start", video, "--hide-touches"]);
217 + try {
218 + await action();
219 + await delay(2_000);
220 + } finally {
221 + await this.agent(["record", "stop"], { timeoutMs: 180_000 }).catch(() => undefined);
222 + }
223 + return video;
224 + }
225 +
226 + async release() {
227 + await this.agent(["close"]).catch(() => undefined);
228 + }
229 +}
scripts/mobile-e2e/journey.ts new
+264
@@ -0,0 +1,264 @@
1 +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off - Host-side device automation.
2 +
3 +import * as NodePath from "node:path";
4 +
5 +import {
6 + check,
7 + type Check,
8 + describeRect,
9 + findLabel,
10 + keyboardRect,
11 + layoutChecks,
12 + type Rect,
13 + type Snapshot,
14 + type SnapshotNode,
15 + tabBarRect,
16 +} from "./checks.ts";
17 +import { BUNDLE_ID, delay, type Device } from "./device.ts";
18 +import { lagCheck } from "./video.ts";
19 +
20 +export interface StepResult {
21 + readonly name: string;
22 + readonly checks: ReadonlyArray<Check>;
23 + readonly screenshot?: string;
24 + readonly video?: string;
25 + readonly lagFrame?: string;
26 +}
27 +
28 +// Threads of the fixture's first environment (scripts/mobile-showcase-environment.ts).
29 +const THREAD_TITLE =
30 + /Make remote coding feel local|Put the command center in your pocket|Tune the handoff haptics/;
31 +const BACK = /^(Back|Navigate up|Go back)$/i;
32 +
33 +const backButton = (snapshot: Snapshot) =>
34 + snapshot.nodes.find((node) => node.identifier === "BackButton") ?? findLabel(snapshot, BACK);
35 +const composer = (snapshot: Snapshot) =>
36 + snapshot.nodes
37 + .filter(
38 + (node) =>
39 + (node.kind === "text-field" || node.kind === "text-view" || node.editable === true) &&
40 + node.rect &&
41 + node.rect.y < snapshot.viewport.height &&
42 + node.bundleId !== "com.android.systemui",
43 + )
44 + .toSorted((a, b) => b.rect!.y - a.rect!.y)[0];
45 +const threadRow = (snapshot: Snapshot) => findLabel(snapshot, THREAD_TITLE);
46 +
47 +const inside = (rect: Rect | undefined, snapshot: Snapshot) =>
48 + rect !== undefined &&
49 + rect.x >= 0 &&
50 + rect.y >= 0 &&
51 + rect.x + rect.width <= snapshot.viewport.width + 1 &&
52 + rect.y + rect.height <= snapshot.viewport.height + 1;
53 +
54 +const topBand = (snapshot: Snapshot): Rect => ({
55 + x: 0,
56 + y: 0,
57 + width: snapshot.viewport.width,
58 + height: snapshot.viewport.height * 0.12,
59 +});
60 +
61 +/**
62 + * Walks one device through the app like a person would: thread list, a thread, scrolling,
63 + * typing, back, settings, then the pose's own change (rotation or fold). Every screen gets
64 + * the generic layout checks plus what that screen must show. Steps land in `steps` as they
65 + * finish, so a journey that stops early still reports what it saw.
66 + */
67 +export async function runJourney(
68 + device: Device,
69 + directory: string,
70 + pair: () => Promise<string>,
71 + steps: StepResult[],
72 +): Promise<void> {
73 + const phone = device.pose.form === "phone";
74 + const tap = (node: SnapshotNode | undefined, label: string) => {
75 + if (!node?.ref) throw new Error(`no ${label} to tap`);
76 + return device.agent(["press", `@${node.ref}`]);
77 + };
78 +
79 + /** Screenshot and check the settled screen, and record it as a step. */
80 + const capture = async (
81 + name: string,
82 + snapshot: Snapshot,
83 + expectations: ReadonlyArray<Check> = [],
84 + extra: Partial<StepResult> = {},
85 + ) => {
86 + const screenshot = NodePath.join(directory, `${steps.length + 1}-${name}.png`);
87 + await device.screenshot(screenshot);
88 + const step = {
89 + name,
90 + checks: [...layoutChecks(snapshot, BUNDLE_ID), ...expectations],
91 + screenshot,
92 + ...extra,
93 + };
94 + steps.push(step);
95 + return step;
96 + };
97 +
98 + // The app ignores the pairing link until its JS has loaded and shown the workspace. An
99 + // error dialog on the way is a finding: record it, dismiss it, and keep going.
100 + const loaded = (snapshot: Snapshot) =>
101 + findLabel(snapshot, /Add environment|New (chat|thread)/i) !== undefined;
102 + const errorDialog = (snapshot: Snapshot) =>
103 + findLabel(snapshot, /^(Dismiss|OK|Close)$/) !== undefined
104 + ? findLabel(snapshot, /^Could not|failed|went wrong/i)
105 + : undefined;
106 + for (let dialogs = 0; ; dialogs++) {
107 + const launch = await device.waitFor(
108 + (snapshot) => loaded(snapshot) || errorDialog(snapshot) !== undefined,
109 + "the app to load",
110 + 180_000,
111 + );
112 + const dialog = errorDialog(launch);
113 + if (!dialog || dialogs >= 3) break;
114 + await capture("launch error", launch, [
115 + check("no error dialog on launch", false, dialog.label ?? "error dialog"),
116 + ]);
117 + await device.agent(["press", `label="${findLabel(launch, /^(Dismiss|OK|Close)$/)!.label}"`]);
118 + await delay(1_000);
119 + }
120 + let threads: Snapshot | null = null;
121 + for (let attempt = 1; attempt <= 2 && !threads; attempt++) {
122 + const pairing = new URLSearchParams({ pairingUrl: await pair(), autoConnect: "1" });
123 + await device.openUrl(`t3code-dev://connections/new?${pairing}`);
124 + threads = await device
125 + .waitFor((snapshot) => threadRow(snapshot) !== undefined, "the paired thread list", 60_000)
126 + .catch(() => null);
127 + }
128 + if (!threads) throw new Error("the app did not show the fixture threads after pairing");
129 + await delay(1_000);
130 + threads = await device.snapshot();
131 + const tabs = tabBarRect(threads);
132 + await capture("threads", threads, [
133 + check("thread list visible", threadRow(threads) !== undefined, "fixture threads"),
134 + ...(phone ? [check("tab bar visible", inside(tabs, threads), describeRect(tabs))] : []),
135 + ]);
136 +
137 + const opening = await device.recorded(directory, "open-thread", () =>
138 + tap(threadRow(threads!), "thread row"),
139 + );
140 + const thread = await device.waitFor(
141 + (snapshot) => composer(snapshot) !== undefined && threadRow(snapshot) !== undefined,
142 + "the thread screen",
143 + );
144 + const threadShot = NodePath.join(directory, `${steps.length + 1}-thread.png`);
145 + await device.screenshot(threadShot);
146 + const openLag =
147 + phone && tabs
148 + ? await lagCheck(
149 + "no tabs over the entering thread (video)",
150 + {
151 + video: opening,
152 + nav: topBand(threads),
153 + chrome: tabs,
154 + viewportWidth: threads.viewport.width,
155 + },
156 + NodePath.join(directory, "open-thread-lag.png"),
157 + )
158 + : null;
159 + steps.push({
160 + name: "thread",
161 + screenshot: threadShot,
162 + video: opening,
163 + ...(openLag?.frame ? { lagFrame: openLag.frame } : {}),
164 + checks: [
165 + ...layoutChecks(thread, BUNDLE_ID),
166 + check(
167 + "composer inside the screen",
168 + inside(composer(thread)?.rect, thread),
169 + describeRect(composer(thread)?.rect),
170 + ),
171 + ...(phone
172 + ? [
173 + check("back button visible", backButton(thread) !== undefined, "back"),
174 + check(
175 + "tab bar hidden",
176 + tabBarRect(thread) === undefined,
177 + describeRect(tabBarRect(thread)),
178 + ),
179 + ]
180 + : []),
181 + ...(openLag ? [openLag.check] : []),
182 + ],
183 + });
184 +
185 + await device.agent(["scroll", "up"]).catch(() => undefined);
186 + await delay(800);
187 + await capture("scrolled up", await device.snapshot());
188 + await device.agent(["scroll", "down"]).catch(() => undefined);
189 + await delay(800);
190 +
191 + await tap(composer(await device.snapshot()), "composer");
192 + await device.agent(["type", "Checking the composer above the keyboard"]);
193 + await delay(1_200);
194 + const typing = await device.snapshot();
195 + await capture("typing", typing, [
196 + check(
197 + "composer visible while typing",
198 + inside(composer(typing)?.rect, typing),
199 + describeRect(composer(typing)?.rect),
200 + ),
201 + check(
202 + "composer above the keyboard",
203 + keyboardRect(typing) === undefined ||
204 + (composer(typing)?.rect?.y ?? Infinity) < keyboardRect(typing)!.y,
205 + `composer ${describeRect(composer(typing)?.rect)}, keyboard ${describeRect(keyboardRect(typing))}`,
206 + ),
207 + ]);
208 + await device.agent(["keyboard", "dismiss"]).catch(() => undefined);
209 + await delay(800);
210 +
211 + if (phone) {
212 + const back = await device.recorded(directory, "back", async () => {
213 + const current = await device.snapshot();
214 + const button = backButton(current);
215 + if (button) await tap(button, "back button");
216 + else await device.agent(["back"]);
217 + });
218 + const list = await device.waitFor(
219 + (snapshot) => backButton(snapshot) === undefined && threadRow(snapshot) !== undefined,
220 + "the thread list",
221 + );
222 + const listShot = NodePath.join(directory, `${steps.length + 1}-back.png`);
223 + await device.screenshot(listShot);
224 + const backLag = tabs
225 + ? await lagCheck(
226 + "tabs return with the list (video)",
227 + {
228 + video: back,
229 + nav: topBand(list),
230 + chrome: tabs,
231 + viewportWidth: list.viewport.width,
232 + },
233 + NodePath.join(directory, "back-lag.png"),
234 + )
235 + : null;
236 + steps.push({
237 + name: "back",
238 + screenshot: listShot,
239 + video: back,
240 + ...(backLag?.frame ? { lagFrame: backLag.frame } : {}),
241 + checks: [
242 + ...layoutChecks(list, BUNDLE_ID),
243 + check("tab bar back", inside(tabBarRect(list), list), describeRect(tabBarRect(list))),
244 + ...(backLag ? [backLag.check] : []),
245 + ],
246 + });
247 + }
248 +
249 + // The labelled tab is not always the hittable node; agent-device resolves the press.
250 + await device.agent(["press", 'label="Settings"']);
251 + await delay(1_500);
252 + await capture("settings", await device.snapshot());
253 +
254 + if (device.pose.form === "tablet" && !device.pose.fold) {
255 + await device.agent(["orientation", device.pose.landscape ? "portrait" : "landscape-left"]);
256 + await delay(2_000);
257 + await capture("rotated", await device.snapshot());
258 + }
259 + if (device.pose.fold) {
260 + await device.agent(["fold", device.pose.fold === "open" ? "closed" : "open"]);
261 + await delay(2_500);
262 + await capture(device.pose.fold === "open" ? "folded" : "unfolded", await device.snapshot());
263 + }
264 +}
scripts/mobile-e2e/poses.ts new
+107
@@ -0,0 +1,107 @@
1 +export interface Pose {
2 + readonly id: string;
3 + readonly platform: "ios" | "android";
4 + /** A model name from the pool's catalog (`GET /v1/catalog`). */
5 + readonly device: string;
6 + readonly form: "phone" | "tablet";
7 + readonly appearance: "light" | "dark";
8 + /** Phones are portrait-only; tablets also run rotated. */
9 + readonly landscape?: boolean;
10 + readonly fold?: "closed" | "open";
11 + readonly textSize?: "accessibility-large";
12 +}
13 +
14 +const pose = (value: Pose) => value;
15 +
16 +export const POSES: ReadonlyArray<Pose> = [
17 + pose({
18 + id: "iphone-se",
19 + platform: "ios",
20 + device: "iPhone SE (3rd generation)",
21 + form: "phone",
22 + appearance: "light",
23 + }),
24 + pose({
25 + id: "iphone-13-mini-dark",
26 + platform: "ios",
27 + device: "iPhone 13 mini",
28 + form: "phone",
29 + appearance: "dark",
30 + }),
31 + pose({
32 + id: "iphone-17-pro",
33 + platform: "ios",
34 + device: "iPhone 17 Pro",
35 + form: "phone",
36 + appearance: "light",
37 + }),
38 + pose({
39 + id: "iphone-17-pro-max-large-text",
40 + platform: "ios",
41 + device: "iPhone 17 Pro Max",
42 + form: "phone",
43 + appearance: "dark",
44 + textSize: "accessibility-large",
45 + }),
46 + pose({
47 + id: "ipad-mini",
48 + platform: "ios",
49 + device: "iPad mini (A17 Pro)",
50 + form: "tablet",
51 + appearance: "light",
52 + }),
53 + pose({
54 + id: "ipad-pro-13-landscape",
55 + platform: "ios",
56 + device: "iPad Pro 13-inch (M5)",
57 + form: "tablet",
58 + appearance: "dark",
59 + landscape: true,
60 + }),
61 + pose({
62 + id: "pixel-9",
63 + platform: "android",
64 + device: "Pixel 9",
65 + form: "phone",
66 + appearance: "light",
67 + }),
68 + pose({
69 + id: "small-phone-dark",
70 + platform: "android",
71 + device: "Small Phone",
72 + form: "phone",
73 + appearance: "dark",
74 + }),
75 + pose({
76 + id: "pixel-9-large-text",
77 + platform: "android",
78 + device: "Pixel 9",
79 + form: "phone",
80 + appearance: "light",
81 + textSize: "accessibility-large",
82 + }),
83 + pose({
84 + id: "pixel-fold-closed",
85 + platform: "android",
86 + device: "Pixel 9 Pro Fold",
87 + form: "phone",
88 + appearance: "light",
89 + fold: "closed",
90 + }),
91 + pose({
92 + id: "pixel-fold-open",
93 + platform: "android",
94 + device: "Pixel 9 Pro Fold",
95 + form: "tablet",
96 + appearance: "dark",
97 + fold: "open",
98 + }),
99 + pose({
100 + id: "pixel-tablet-landscape",
101 + platform: "android",
102 + device: "Pixel Tablet",
103 + form: "tablet",
104 + appearance: "light",
105 + landscape: true,
106 + }),
107 +];
scripts/mobile-e2e/report.ts new
+90
@@ -0,0 +1,90 @@
1 +// @effect-diagnostics nodeBuiltinImport:off - Writes a static HTML report.
2 +
3 +import * as NodePath from "node:path";
4 +
5 +import type { StepResult } from "./journey.ts";
6 +import type { Pose } from "./poses.ts";
7 +
8 +export interface Finding {
9 + readonly step: string;
10 + readonly issue: string;
11 + readonly severity: "high" | "medium" | "low";
12 +}
13 +
14 +export interface PoseResult {
15 + readonly pose: Pose;
16 + readonly steps: ReadonlyArray<StepResult>;
17 + readonly error?: string;
18 + /** Screenshot taken when the journey stopped early. */
19 + readonly stopped?: string;
20 + readonly findings?: ReadonlyArray<Finding>;
21 +}
22 +
23 +export const failedChecks = (result: PoseResult) =>
24 + result.steps.flatMap((step) =>
25 + step.checks.filter((item) => !item.pass).map((item) => ({ step: step.name, ...item })),
26 + );
27 +
28 +export const poseFailed = (result: PoseResult) =>
29 + result.error !== undefined || failedChecks(result).length > 0;
30 +
31 +const escapeHtml = (value: string) =>
32 + value.replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;");
33 +
34 +const describePose = (pose: Pose) =>
35 + [
36 + pose.device,
37 + pose.appearance,
38 + pose.landscape ? "landscape" : "",
39 + pose.fold ? `fold ${pose.fold}` : "",
40 + pose.textSize ? "large text" : "",
41 + ]
42 + .filter(Boolean)
43 + .join(" · ");
44 +
45 +export function renderReport(results: ReadonlyArray<PoseResult>, outputDirectory: string): string {
46 + const relative = (path: string | undefined) =>
47 + path ? escapeHtml(NodePath.relative(outputDirectory, path)) : "";
48 + const rows = results
49 + .map((result) => {
50 + const cells = result.steps
51 + .map((step) => {
52 + const failed = step.checks.filter((item) => !item.pass);
53 + const findings = (result.findings ?? []).filter((finding) => finding.step === step.name);
54 + const notes = [
55 + ...failed.map(
56 + (item) =>
57 + `<li class="fail">✗ ${escapeHtml(item.name)} <small>${escapeHtml(item.detail)}</small></li>`,
58 + ),
59 + ...findings.map(
60 + (finding) =>
61 + `<li class="review ${finding.severity}">◆ ${escapeHtml(finding.issue)}</li>`,
62 + ),
63 + ].join("");
64 + const passed = step.checks.length - failed.length;
65 + const media = [
66 + step.screenshot
67 + ? `<a href="${relative(step.screenshot)}"><img src="${relative(step.screenshot)}"></a>`
68 + : "",
69 + step.lagFrame
70 + ? `<a href="${relative(step.lagFrame)}"><img class="lag" src="${relative(step.lagFrame)}"></a>`
71 + : "",
72 + step.video ? `<a href="${relative(step.video)}">video</a>` : "",
73 + ].join("");
74 + return `<td class="${failed.length ? "bad" : ""}"><h3>${escapeHtml(step.name)}</h3>${media}<small class="pass">${passed}/${step.checks.length} checks</small><ul>${notes}</ul></td>`;
75 + })
76 + .join("");
77 + const error = result.error
78 + ? `<td class="bad"><h3>stopped</h3>${result.stopped ? `<a href="${relative(result.stopped)}"><img src="${relative(result.stopped)}"></a>` : ""}<p class="fail">${escapeHtml(result.error)}</p></td>`
79 + : "";
80 + return `<tr><th>${escapeHtml(result.pose.id)}<br><small>${escapeHtml(describePose(result.pose))}</small></th>${cells}${error}</tr>`;
81 + })
82 + .join("");
83 + const failed = results.filter(poseFailed).length;
84 + const findings = results.reduce((count, result) => count + (result.findings?.length ?? 0), 0);
85 + return `<!doctype html><meta charset="utf-8"><title>Mobile E2E</title>
86 +<style>body{font:14px system-ui;margin:24px}table{border-collapse:collapse}td,th{border:1px solid #ddd;padding:8px;vertical-align:top;text-align:left}td{width:190px}td.bad{background:#fff5f4}h3{margin:0 0 6px;font-size:13px}img{width:180px;display:block;margin:0 0 6px}img.lag{outline:3px solid #b42318}.pass{color:#16794a}.fail{color:#b42318}.review{color:#7a4b00}.review.high{color:#b42318}ul{padding-left:16px;margin:4px 0}small{color:#666}</style>
87 +<h1>Mobile E2E · ${results.length} poses · ${failed === 0 ? "all checks passed" : `${failed} with failing checks`} · ${findings} review findings</h1>
88 +<p><span class="fail">✗</span> failed check <span class="review">◆</span> agent review finding</p>
89 +<table>${rows}</table>`;
90 +}
scripts/mobile-e2e/review.ts new
+68
@@ -0,0 +1,68 @@
1 +// @effect-diagnostics nodeBuiltinImport:off - Runs the Claude Code CLI as a reviewer.
2 +
3 +import * as NodeProcess from "node:process";
4 +
5 +import { run } from "./device.ts";
6 +import type { Finding, PoseResult } from "./report.ts";
7 +
8 +const SCHEMA = {
9 + type: "object",
10 + properties: {
11 + findings: {
12 + type: "array",
13 + items: {
14 + type: "object",
15 + properties: {
16 + step: { type: "string" },
17 + issue: { type: "string" },
18 + severity: { type: "string", enum: ["high", "medium", "low"] },
19 + },
20 + required: ["step", "issue", "severity"],
21 + },
22 + },
23 + },
24 + required: ["findings"],
25 +};
26 +
27 +/**
28 + * Asks an agent to look at one pose's screenshots for what the generic checks cannot see:
29 + * clipped or overlapping text, wrong colours for the appearance, odd gaps, unreadable
30 + * contrast. It reads the images itself and returns structured findings.
31 + */
32 +export async function reviewPose(result: PoseResult, outputDirectory: string): Promise<Finding[]> {
33 + const shots = result.steps.filter((step) => step.screenshot);
34 + if (shots.length === 0) return [];
35 + const { pose } = result;
36 + const prompt = [
37 + `You are reviewing screenshots of the T3 Code mobile app (a client for coding agents) for visual bugs.`,
38 + `Device: ${pose.device} (${pose.platform}, ${pose.form}), ${pose.appearance} appearance${pose.landscape ? ", landscape" : ""}${pose.fold ? `, fold ${pose.fold}` : ""}${pose.textSize ? ", largest accessibility text size" : ""}.`,
39 + `Read each screenshot below with the Read tool. Each is one step of the same session:`,
40 + ...shots.map((step) => `- step "${step.name}": ${step.screenshot}`),
41 + ``,
42 + `Report only real defects a user would notice: text clipped, truncated mid-word where it should wrap, or overlapping other content; controls overlapping, cut off, or under the status bar, notch, home indicator, or keyboard; content hidden behind navigation chrome; colours wrong for the ${pose.appearance} appearance or unreadable contrast; large unexplained empty areas or misaligned layout; error banners or red error screens.`,
43 + `Ignore: the fixture content itself, the status bar clock, a development-build badge, and anything that is plainly a deliberate design choice. Prefer no finding to a speculative one.`,
44 + `Use the exact step names above. Severity: high = broken or unusable, medium = clearly wrong, low = polish.`,
45 + ].join("\n");
46 + const { stdout } = await run(
47 + "claude",
48 + [
49 + "-p",
50 + prompt,
51 + "--output-format",
52 + "json",
53 + "--json-schema",
54 + JSON.stringify(SCHEMA),
55 + "--allowedTools",
56 + "Read",
57 + "--add-dir",
58 + outputDirectory,
59 + "--no-session-persistence",
60 + ...(NodeProcess.env.T3_E2E_REVIEW_MODEL
61 + ? ["--model", NodeProcess.env.T3_E2E_REVIEW_MODEL]
62 + : []),
63 + ],
64 + { cwd: outputDirectory, timeoutMs: 600_000 },
65 + );
66 + const output = JSON.parse(stdout) as { structured_output?: { findings?: Finding[] } };
67 + return output.structured_output?.findings ?? [];
68 +}
scripts/mobile-e2e/video.ts new
+148
@@ -0,0 +1,148 @@
1 +// @effect-diagnostics nodeBuiltinImport:off - Host-side video analysis shells out to ffmpeg.
2 +
3 +import * as NodeChildProcess from "node:child_process";
4 +import * as NodeFSP from "node:fs/promises";
5 +
6 +import { check, type Check, type Rect } from "./checks.ts";
7 +import { run } from "./device.ts";
8 +
9 +/** Pixel width of an image or video, to map snapshot coordinates onto its frames. */
10 +export async function pixelWidth(path: string): Promise<number> {
11 + const { stdout } = await run("ffprobe", [
12 + "-v",
13 + "error",
14 + "-select_streams",
15 + "v:0",
16 + "-show_entries",
17 + "stream=width",
18 + "-of",
19 + "csv=p=0",
20 + path,
21 + ]);
22 + return Number(stdout.trim());
23 +}
24 +
25 +/** Downsampled grayscale crops of one region, one per video frame (or one for a PNG). */
26 +async function grayFrames(input: string, region: Rect, scale: number): Promise<Uint8Array[]> {
27 + const even = (value: number) => Math.max(2, Math.floor(value / 2) * 2);
28 + const [x, y, width, height] = [
29 + Math.round(region.x * scale),
30 + Math.round(region.y * scale),
31 + even(region.width * scale),
32 + even(region.height * scale),
33 + ];
34 + const [outWidth, outHeight] = [even(region.width / 3), even(region.height / 3)];
35 + const bytes = await new Promise<Buffer>((resolve, reject) => {
36 + const child = NodeChildProcess.spawn(
37 + "ffmpeg",
38 + [
39 + "-v",
40 + "error",
41 + "-i",
42 + input,
43 + // One output frame per recorded frame, so indices match select=eq(n,…) for evidence.
44 + "-fps_mode",
45 + "passthrough",
46 + "-vf",
47 + `crop=${width}:${height}:${x}:${y},scale=${outWidth}:${outHeight},format=gray`,
48 + "-f",
49 + "rawvideo",
50 + "-",
51 + ],
52 + { stdio: ["ignore", "pipe", "pipe"] },
53 + );
54 + const chunks: Buffer[] = [];
55 + let stderr = "";
56 + child.stdout.on("data", (chunk: Buffer) => chunks.push(chunk));
57 + child.stderr.on("data", (chunk) => (stderr += chunk));
58 + child.on("error", reject);
59 + child.on("close", (code) =>
60 + code === 0 ? resolve(Buffer.concat(chunks)) : reject(new Error(`ffmpeg failed: ${stderr}`)),
61 + );
62 + });
63 + const size = outWidth * outHeight;
64 + return Array.from({ length: Math.floor(bytes.length / size) }, (_, index) =>
65 + bytes.subarray(index * size, (index + 1) * size),
66 + );
67 +}
68 +
69 +const distance = (a: Uint8Array, b: Uint8Array) => {
70 + let sum = 0;
71 + for (let index = 0; index < a.length; index++) sum += Math.abs(a[index]! - b[index]!);
72 + return sum / a.length;
73 +};
74 +
75 +/** How far a region has moved from its origin toward its destination look, 0 to 1. */
76 +export interface FrameProgress {
77 + readonly nav: number;
78 + readonly chrome: number;
79 +}
80 +
81 +/**
82 + * A frame lags when the top of the screen is well into the transition while the tab bar
83 + * region still looks like the origin: tabs lingering over an entering thread, or a returned
84 + * list drawn before its tabs.
85 + */
86 +export const chromeLags = (frame: FrameProgress) => frame.nav > 0.3 && frame.chrome < 0.4;
87 +// One lagging frame is capture jitter; a regression lasts several.
88 +const LAG_FRAME_LIMIT = 2;
89 +
90 +const progress = (origin: number, destination: number) =>
91 + origin + destination === 0 ? 0 : origin / (origin + destination);
92 +
93 +/**
94 + * Checks that the tab bar moves with the screen during a recorded transition. The first and
95 + * last frames are the before and after looks, so nothing is compared across capture sources.
96 + */
97 +export async function lagCheck(
98 + name: string,
99 + input: {
100 + readonly video: string;
101 + readonly nav: Rect;
102 + readonly chrome: Rect;
103 + /** Snapshot units across the screen, to scale regions onto the video's pixels. */
104 + readonly viewportWidth: number;
105 + },
106 + evidence: string,
107 +): Promise<{ check: Check; frame?: string }> {
108 + // A recording the device failed to export is a gap in evidence, not a lag.
109 + if (!(await NodeFSP.stat(input.video).catch(() => null)))
110 + return { check: check(name, true, "no recording; not checked") };
111 + const scale = (await pixelWidth(input.video)) / input.viewportWidth;
112 + const [navFrames, chromeFrames] = await Promise.all([
113 + grayFrames(input.video, input.nav, scale),
114 + grayFrames(input.video, input.chrome, scale),
115 + ]);
116 + const [navOrigin, navDestination] = [navFrames[0], navFrames.at(-1)];
117 + const [chromeOrigin, chromeDestination] = [chromeFrames[0], chromeFrames.at(-1)];
118 + if (!navOrigin || !navDestination || !chromeOrigin || !chromeDestination)
119 + throw new Error(`Could not read frames of ${input.video}.`);
120 + const lagging = navFrames.flatMap((nav, index) => {
121 + const frame = {
122 + nav: progress(distance(nav, navOrigin), distance(nav, navDestination)),
123 + chrome: progress(
124 + distance(chromeFrames[index]!, chromeOrigin),
125 + distance(chromeFrames[index]!, chromeDestination),
126 + ),
127 + };
128 + return chromeLags(frame) ? [index] : [];
129 + });
130 + if (lagging.length < LAG_FRAME_LIMIT)
131 + return { check: check(name, true, `${lagging.length} lagging video frames`) };
132 + await run("ffmpeg", [
133 + "-v",
134 + "error",
135 + "-y",
136 + "-i",
137 + input.video,
138 + "-vf",
139 + `select=eq(n\\,${lagging[0]})`,
140 + "-frames:v",
141 + "1",
142 + evidence,
143 + ]);
144 + return {
145 + check: check(name, false, `${lagging.length} video frames (first #${lagging[0]})`),
146 + frame: evidence,
147 + };
148 +}
scripts/mobile-native-client.ts
+68 -10
@@ -453,6 +453,32 @@ const remoteCache = Effect.fn("nativeClient.remoteCache")(
453 Effect.orElseSucceed(() => null),
454 );
455
456 +const ReaderLogin = Schema.fromJsonString(
457 + Schema.Struct({ name: Schema.String, token: Schema.String }),
458 +);
459 +
460 +/**
461 + * A URL another machine can download `key` from. The header goes to that machine, so only a
462 + * read-only login (`ci-reader`) is ever handed out, never a developer's writer token.
463 + */
464 +const sharedSource = Effect.fn("nativeClient.sharedSource")(
465 + function* (home: string, key: string) {
466 + const remote = yield* remoteCache(home);
467 + if (!remote) return null;
468 + const fs = yield* FileSystem.FileSystem;
469 + const path = yield* Path.Path;
470 + const reader =
471 + remote.name === "ci-reader"
472 + ? remote
473 + : yield* fs
474 + .readFileString(path.join(home, ".config/t3-task-cache/ci-reader.json"))
475 + .pipe(Effect.flatMap(Schema.decodeUnknownEffect(ReaderLogin)));
476 + const basic = Buffer.from(`${reader.name}:${reader.token}`).toString("base64");
477 + return { url: `${remote.url}/${key}`, header: `authorization:Basic ${basic}` };
478 + },
479 + Effect.orElseSucceed(() => null),
480 +);
481 +
482 const authorized = (remote: RemoteCache, request: HttpClientRequest.HttpClientRequest) =>
483 request.pipe(HttpClientRequest.basicAuth(remote.name, remote.token));
484
@@ -860,16 +886,48 @@ const installIntoSession = Effect.fn("nativeClient.installIntoSession")(function
886 const deviceAbi = platform === "android" ? abi : null;
887 const task = deviceAbi ? `native-client:android-${deviceAbi}` : `native-client:${platform}`;
888 const current = yield* fingerprint(platform);
863 - const artifact = yield* resolveArtifact(home, platform, deviceAbi, current, false).pipe(
864 - Effect.tapError(() => reportTelemetry(home, task, "failed", 0)),
865 - );
866 - yield* Console.error(`Installing into agent-device session ${session}...`);
867 - yield* command(
868 - node,
869 - [agentDevice, "install", bundleId, artifact.file, "--session", session],
870 - true,
871 - workspace,
872 - );
889 + const key = `${platform}/${deviceAbi ? `development-${deviceAbi}` : "development"}/${current}`;
890 + const source = yield* sharedSource(home, key);
891 + // The session's host downloads the client itself; uploading a 166 MB APK through a
892 + // gateway from here takes minutes and can outlast the remote install timeout.
893 + const fetched = source
894 + ? yield* command(
895 + node,
896 + [
897 + agentDevice,
898 + "install-from-source",
899 + source.url,
900 + "--header",
901 + source.header,
902 + "--session",
903 + session,
904 + ],
905 + true,
906 + workspace,
907 + ).pipe(
908 + Effect.as(true),
909 + Effect.catch((error) =>
910 + Console.error(`The device could not download ${key} (${describe(error)}).`).pipe(
911 + Effect.as(false),
912 + ),
913 + ),
914 + )
915 + : false;
916 + const artifact = fetched
917 + ? { source: "remote" as const, buildMs: 0 }
918 + : yield* Effect.gen(function* () {
919 + const local = yield* resolveArtifact(home, platform, deviceAbi, current, false).pipe(
920 + Effect.tapError(() => reportTelemetry(home, task, "failed", 0)),
921 + );
922 + yield* Console.error(`Installing into agent-device session ${session}...`);
923 + yield* command(
924 + node,
925 + [agentDevice, "install", bundleId, local.file, "--session", session],
926 + true,
927 + workspace,
928 + );
929 + return local;
930 + });
931 if (metro) {
932 const url = encodeURIComponent(metro);
933 yield* command(
scripts/mobile-showcase.ts
+1 -1
@@ -553,7 +553,7 @@ async function waitForFileContent(
553 throw new Error(`${label} was not written to ${filePath} within ${timeoutMs}ms.`);
554 }
555
556 -async function reserveAvailablePort(): Promise<number> {
556 +export async function reserveAvailablePort(): Promise<number> {
557 return await new Promise<number>((resolve, reject) => {
558 const server = NodeNet.createServer();
559 server.once("error", reject);