1
#!/usr/bin/env node
2
-// @effect-diagnostics nodeBuiltinImport:off globalTimers:off globalDate:off - Host-side simulator automation uses Node subprocess and timing APIs directly.
2
+// @effect-diagnostics nodeBuiltinImport:off globalDate:off - Host-side device automation uses Node subprocess and filesystem APIs directly.
3
4
import * as NodeChildProcess from "node:child_process";
5
import * as NodeFSP from "node:fs/promises";
8
import * as NodeProcess from "node:process";
9
import * as NodeURL from "node:url";
10
11
-import { startShowcaseEnvironments, stopProcess, waitForPort } from "./mobile-showcase.ts";
11
+import { Device, type Pool, run } from "./mobile-e2e/device.ts";
12
+import { runJourney, type StepResult } from "./mobile-e2e/journey.ts";
13
+import { POSES, type Pose } from "./mobile-e2e/poses.ts";
14
+import { failedChecks, poseFailed, type PoseResult, renderReport } from "./mobile-e2e/report.ts";
15
+import { reviewPose } from "./mobile-e2e/review.ts";
16
+import {
17
+ reserveAvailablePort,
18
+ startShowcaseEnvironments,
19
+ stopProcess,
20
+ waitForPort,
21
+} from "./mobile-showcase.ts";
22
import { SHOWCASE_ENVIRONMENTS } from "./mobile-showcase-environment.ts";
23
24
const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), "..");
15
-const AGENT_DEVICE = NodePath.join(
16
- REPO_ROOT,
17
- "scripts/node_modules/agent-device/bin/agent-device.mjs",
18
-);
19
-const BUNDLE_ID = "com.t3tools.t3code.dev";
20
-const METRO_PORT = 8081;
21
-const AGENT_DEVICE_VERSION = (
22
- JSON.parse(
23
- await NodeFSP.readFile(
24
- NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/package.json"),
25
- "utf8",
26
- ),
27
- ) as { version: string }
28
-).version;
29
-// Persistent so the XCTest runner agent-device builds per simulator survives between runs;
30
-// per version because a daemon refuses clients of another version.
31
-const AGENT_DEVICE_STATE = NodePath.join(
32
- NodeOS.homedir(),
33
- ".cache/t3code/mobile-e2e",
34
- `agent-device-${AGENT_DEVICE_VERSION}`,
35
-);
25
37
-export interface E2eDevice {
38
- readonly id: string;
39
- readonly platform: "ios";
40
- readonly deviceType: string;
41
- readonly runtime: string;
42
-}
43
-
44
-const ios = (id: string, deviceType: string): E2eDevice => ({
45
- id,
46
- platform: "ios",
47
- deviceType: `com.apple.CoreSimulator.SimDeviceType.${deviceType}`,
48
- runtime: "com.apple.CoreSimulator.SimRuntime.iOS-27-0",
49
-});
50
-
51
-export const DEVICES: ReadonlyArray<E2eDevice> = [
52
- ios("iphone-se", "iPhone-SE-3rd-generation"),
53
- ios("iphone-13-mini", "iPhone-13-mini"),
54
- ios("iphone-17", "iPhone-17"),
55
- ios("iphone-17-pro", "iPhone-17-Pro"),
56
- ios("iphone-17-pro-max", "iPhone-17-Pro-Max"),
57
- ios("iphone-air", "iPhone-Air"),
58
-];
59
-
60
-interface Rect {
61
- readonly x: number;
62
- readonly y: number;
63
- readonly width: number;
64
- readonly height: number;
65
-}
66
-
67
-export interface SnapshotNode {
68
- readonly type?: string;
69
- readonly role?: string;
70
- readonly identifier?: string;
71
- readonly label?: string;
72
- readonly rect?: Rect;
73
-}
74
-
75
-export interface Snapshot {
76
- readonly nodes: ReadonlyArray<SnapshotNode>;
77
- readonly viewport: { readonly width: number; readonly height: number };
78
-}
79
-
80
-export interface Check {
81
- readonly name: string;
82
- readonly pass: boolean;
83
- readonly detail: string;
84
-}
85
-
86
-interface StepResult {
87
- readonly name: string;
88
- readonly checks: ReadonlyArray<Check>;
89
- readonly screenshot?: string | undefined;
90
- readonly video?: string | undefined;
91
- readonly contactSheet?: string | undefined;
92
- readonly lagFrame?: string | undefined;
93
- readonly samples: number;
94
-}
95
-
96
-interface DeviceResult {
97
- readonly device: E2eDevice;
98
- readonly label: string;
99
- readonly steps: ReadonlyArray<StepResult>;
100
- readonly error?: string;
101
-}
102
-
103
-const onScreen = (node: SnapshotNode, snapshot: Snapshot) =>
104
- node.rect !== undefined &&
105
- node.rect.width > 0 &&
106
- node.rect.height > 0 &&
107
- node.rect.x < snapshot.viewport.width &&
108
- node.rect.y < snapshot.viewport.height &&
109
- node.rect.x + node.rect.width > 0 &&
110
- node.rect.y + node.rect.height > 0;
111
-
112
-const find = (snapshot: Snapshot, match: (node: SnapshotNode) => boolean) =>
113
- snapshot.nodes.filter((node) => match(node) && onScreen(node, snapshot));
114
-
115
-export const chrome = (snapshot: Snapshot) => ({
116
- tabBar: find(snapshot, (node) => node.type === "TabBar")[0],
117
- navigationBar: find(snapshot, (node) => node.type === "NavigationBar")[0],
118
- backButton: find(snapshot, (node) => node.identifier === "BackButton")[0],
119
- send: find(snapshot, (node) => node.type === "Button" && node.label === "Send")[0],
120
- newChat: find(snapshot, (node) => node.type === "Button" && node.label === "New chat")[0],
121
- // Thread rows are full-width context-menu wrappers; the filter pills are narrow ones.
122
- threadRows: find(
123
- snapshot,
124
- (node) =>
125
- node.role === "MenuView" &&
126
- Boolean(node.label) &&
127
- (node.rect?.width ?? 0) >= snapshot.viewport.width * 0.9,
128
- ),
129
-});
130
-
131
-const check = (name: string, pass: boolean, detail: string): Check => ({ name, pass, detail });
132
-const describeRect = (rect: Rect | undefined) =>
133
- rect
134
- ? `${Math.round(rect.x)},${Math.round(rect.y)} ${Math.round(rect.width)}×${Math.round(rect.height)}`
135
- : "absent";
136
-const inside = (rect: Rect, snapshot: Snapshot) =>
137
- rect.x >= 0 &&
138
- rect.y >= 0 &&
139
- rect.x + rect.width <= snapshot.viewport.width + 0.5 &&
140
- rect.y + rect.height <= snapshot.viewport.height + 0.5;
141
-
142
-export function homeChecks(snapshot: Snapshot): Check[] {
143
- const { tabBar, newChat } = chrome(snapshot);
144
- return [
145
- check("tab bar visible", tabBar !== undefined, describeRect(tabBar?.rect)),
146
- check(
147
- "tab bar inside the screen",
148
- tabBar?.rect !== undefined && inside(tabBar.rect, snapshot),
149
- describeRect(tabBar?.rect),
150
- ),
151
- check("new chat button visible", newChat !== undefined, describeRect(newChat?.rect)),
152
- ];
153
-}
154
-
155
-export function threadChecks(snapshot: Snapshot, samples: ReadonlyArray<Snapshot>): Check[] {
156
- const { tabBar, backButton, send } = chrome(snapshot);
157
- const overlapping = samples.filter((sample) => {
158
- const state = chrome(sample);
159
- return state.backButton !== undefined && state.tabBar !== undefined;
160
- }).length;
161
- return [
162
- check("tab bar hidden", tabBar === undefined, describeRect(tabBar?.rect)),
163
- check("back button visible", backButton !== undefined, describeRect(backButton?.rect)),
164
- check(
165
- "composer inside the screen",
166
- send?.rect !== undefined && inside(send.rect, snapshot),
167
- describeRect(send?.rect),
168
- ),
169
- check(
170
- "no tabs over the entering thread",
171
- overlapping === 0,
172
- `${overlapping} of ${samples.length} transition samples showed tabs with a Back button`,
173
- ),
174
- ];
175
-}
176
-
177
-export function backChecks(snapshot: Snapshot, samples: ReadonlyArray<Snapshot>): Check[] {
178
- const listWithoutTabs = samples.filter((sample) => {
179
- const state = chrome(sample);
180
- return (
181
- state.backButton === undefined && state.newChat !== undefined && state.tabBar === undefined
182
- );
183
- }).length;
184
- return [
185
- ...homeChecks(snapshot),
186
- check(
187
- "tabs return with the list",
188
- listWithoutTabs === 0,
189
- `${listWithoutTabs} of ${samples.length} transition samples showed the list without tabs`,
190
- ),
191
- ];
192
-}
193
-
194
-/** Downsampled grayscale crops of one region, one per video frame (or one for a PNG). */
195
-async function grayFrames(input: string, region: Rect, scale: number): Promise<Uint8Array[]> {
196
- const even = (value: number) => Math.max(2, Math.floor(value / 2) * 2);
197
- const [x, y, width, height] = [
198
- Math.round(region.x * scale),
199
- Math.round(region.y * scale),
200
- even(region.width * scale),
201
- even(region.height * scale),
202
- ];
203
- const [outWidth, outHeight] = [even(width / 6), even(height / 6)];
204
- const bytes = await new Promise<Buffer>((resolve, reject) => {
205
- const child = NodeChildProcess.spawn(
206
- "ffmpeg",
207
- [
208
- "-v",
209
- "error",
210
- "-i",
211
- input,
212
- // One output frame per recorded frame, so indices match select=eq(n,…) for evidence.
213
- "-fps_mode",
214
- "passthrough",
215
- "-vf",
216
- `crop=${width}:${height}:${x}:${y},scale=${outWidth}:${outHeight},format=gray`,
217
- "-f",
218
- "rawvideo",
219
- "-",
220
- ],
221
- { stdio: ["ignore", "pipe", "pipe"] },
222
- );
223
- const chunks: Buffer[] = [];
224
- let stderr = "";
225
- child.stdout.on("data", (chunk: Buffer) => chunks.push(chunk));
226
- child.stderr.on("data", (chunk) => (stderr += chunk));
227
- child.on("error", reject);
228
- child.on("close", (code) =>
229
- code === 0 ? resolve(Buffer.concat(chunks)) : reject(new Error(`ffmpeg failed: ${stderr}`)),
230
- );
231
- });
232
- const size = outWidth * outHeight;
233
- return Array.from({ length: Math.floor(bytes.length / size) }, (_, index) =>
234
- bytes.subarray(index * size, (index + 1) * size),
235
- );
236
-}
237
-
238
-const distance = (a: Uint8Array, b: Uint8Array) => {
239
- let sum = 0;
240
- for (let index = 0; index < a.length; index++) sum += Math.abs(a[index]! - b[index]!);
241
- return sum / a.length;
242
-};
243
-
244
-/** How far a region has moved from its origin toward its destination look, 0 to 1. */
245
-export interface FrameProgress {
246
- readonly nav: number;
247
- readonly chrome: number;
248
-}
249
-
250
-/**
251
- * A frame lags when the navigation bar is well into the transition while the tab bar region
252
- * still looks like the origin: tabs lingering over an entering thread, or a returned list drawn
253
- * before its tabs.
254
- */
255
-export const chromeLags = (frame: FrameProgress) => frame.nav > 0.3 && frame.chrome < 0.4;
256
-// One lagging frame is capture jitter; a regression lasts several.
257
-const LAG_FRAME_LIMIT = 2;
258
-
259
-const progress = (origin: number, destination: number) =>
260
- origin + destination === 0 ? 0 : origin / (origin + destination);
261
-
262
-async function chromeLagFrames(input: {
263
- readonly video: string;
264
- readonly origin: string;
265
- readonly destination: string;
266
- readonly nav: Rect;
267
- readonly chrome: Rect;
268
- readonly scale: number;
269
-}) {
270
- const [
271
- navFrames,
272
- chromeFrames,
273
- [navOrigin],
274
- [navDestination],
275
- [chromeOrigin],
276
- [chromeDestination],
277
- ] = await Promise.all([
278
- grayFrames(input.video, input.nav, input.scale),
279
- grayFrames(input.video, input.chrome, input.scale),
280
- grayFrames(input.origin, input.nav, input.scale),
281
- grayFrames(input.destination, input.nav, input.scale),
282
- grayFrames(input.origin, input.chrome, input.scale),
283
- grayFrames(input.destination, input.chrome, input.scale),
284
- ]);
285
- if (!navOrigin || !navDestination || !chromeOrigin || !chromeDestination)
286
- throw new Error("Could not read the reference screenshots.");
287
- const verdicts = navFrames.map((nav, index) => {
288
- const chrome = chromeFrames[index]!;
289
- return {
290
- nav: progress(distance(nav, navOrigin), distance(nav, navDestination)),
291
- chrome: progress(distance(chrome, chromeOrigin), distance(chrome, chromeDestination)),
292
- };
293
- });
294
- return verdicts.flatMap((verdict, index) => (chromeLags(verdict) ? [index] : []));
295
-}
296
-
297
-async function saveFrame(video: string, index: number, path: string) {
298
- await run("ffmpeg", [
299
- "-v",
300
- "error",
301
- "-y",
302
- "-i",
303
- video,
304
- "-vf",
305
- `select=eq(n\\,${index})`,
306
- "-frames:v",
307
- "1",
308
- path,
309
- ]);
310
-}
311
-
312
-async function lagCheck(
313
- name: string,
314
- input: Parameters<typeof chromeLagFrames>[0],
315
- evidence: string,
316
-): Promise<{ check: Check; frame?: string }> {
317
- const lagging = await chromeLagFrames(input);
318
- if (lagging.length < LAG_FRAME_LIMIT)
319
- return { check: check(name, true, `${lagging.length} lagging video frames`) };
320
- await saveFrame(input.video, lagging[0]!, evidence);
321
- return {
322
- check: check(name, false, `${lagging.length} video frames (first #${lagging[0]})`),
323
- frame: evidence,
324
- };
325
-}
326
-
327
-function run(
328
- command: string,
329
- args: ReadonlyArray<string>,
330
- options: { readonly env?: NodeJS.ProcessEnv; readonly allowFailure?: boolean } = {},
331
-): Promise<{ readonly code: number; readonly stdout: string; readonly stderr: string }> {
332
- return new Promise((resolve, reject) => {
333
- const child = NodeChildProcess.spawn(command, args, {
334
- cwd: REPO_ROOT,
335
- env: options.env ?? NodeProcess.env,
336
- stdio: ["ignore", "pipe", "pipe"],
337
- });
338
- let stdout = "";
339
- let stderr = "";
340
- child.stdout.on("data", (chunk) => (stdout += chunk));
341
- child.stderr.on("data", (chunk) => (stderr += chunk));
342
- child.on("error", reject);
343
- child.on("close", (code) => {
344
- if (code !== 0 && !options.allowFailure)
345
- reject(new Error(`${command} ${args.join(" ")} failed (${code}): ${stderr || stdout}`));
346
- else resolve({ code: code ?? 1, stdout, stderr });
347
- });
348
- });
349
-}
350
-
351
-const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
352
-
353
-class Device {
354
- readonly spec: E2eDevice;
355
- readonly udid: string;
356
-
357
- constructor(spec: E2eDevice, udid: string) {
358
- this.spec = spec;
359
- this.udid = udid;
360
- }
361
-
362
- async agent(args: ReadonlyArray<string>): Promise<Record<string, unknown>> {
363
- const { stdout } = await run(
364
- NodeProcess.execPath,
365
- [
366
- AGENT_DEVICE,
367
- ...args,
368
- "--platform",
369
- "ios",
370
- "--udid",
371
- this.udid,
372
- "--session",
373
- `t3-e2e-${this.spec.id}`,
374
- "--json",
375
- ],
376
- {
377
- env: { ...NodeProcess.env, AGENT_DEVICE_STATE_DIR: AGENT_DEVICE_STATE },
378
- allowFailure: true,
379
- },
380
- );
381
- const parsed = JSON.parse(stdout) as {
382
- success: boolean;
383
- data?: Record<string, unknown>;
384
- error?: { message: string };
385
- };
386
- if (!parsed.success) throw new Error(`agent-device ${args[0]}: ${parsed.error?.message}`);
387
- return parsed.data ?? {};
388
- }
389
-
390
- async snapshot(): Promise<Snapshot> {
391
- return (await this.agent(["snapshot"])) as unknown as Snapshot;
392
- }
393
-
394
- async waitFor(predicate: (snapshot: Snapshot) => boolean, label: string, timeoutMs = 60_000) {
395
- const deadline = Date.now() + timeoutMs;
396
- let last: Snapshot | null = null;
397
- while (Date.now() < deadline) {
398
- last = await this.snapshot().catch(() => null);
399
- if (last && predicate(last)) return last;
400
- await delay(500);
401
- }
402
- throw new Error(`${this.spec.id}: timed out waiting for ${label}`);
403
- }
404
-
405
- /** Open a URL the way a user would; iOS asks before opening a custom scheme from another app. */
406
- async openUrl(url: string) {
407
- await run("xcrun", ["simctl", "openurl", this.udid, url]);
408
- await delay(1_500);
409
- await this.agent(["alert", "accept"]).catch(() => undefined);
410
- }
411
-
412
- async screenshot(path: string) {
413
- await run("xcrun", ["simctl", "io", this.udid, "screenshot", path]);
414
- }
415
-
416
- /** Run an action while recording and sampling the accessibility tree until the screen settles. */
417
- async transition(directory: string, name: string, action: () => Promise<unknown>) {
418
- const video = NodePath.join(directory, `${name}.mp4`);
419
- await this.agent(["record", "start", video, "--hide-touches"]);
420
- const samples: Snapshot[] = [];
421
- const sampling = { active: true };
422
- const sampler = (async () => {
423
- while (sampling.active) {
424
- const sample = await this.snapshot().catch(() => null);
425
- if (sample) samples.push(sample);
426
- }
427
- })();
428
- try {
429
- await action();
430
- await delay(2_000);
431
- } finally {
432
- sampling.active = false;
433
- await sampler;
434
- await this.agent(["record", "stop"]).catch(() => undefined);
435
- }
436
- const contactSheet = NodePath.join(directory, `${name}-frames.png`);
437
- const sheet = await run(
438
- NodeProcess.execPath,
439
- [AGENT_DEVICE, "record", "contact-sheet", video, "--out", contactSheet],
440
- {
441
- env: { ...NodeProcess.env, AGENT_DEVICE_STATE_DIR: AGENT_DEVICE_STATE },
442
- allowFailure: true,
443
- },
444
- );
445
- return { samples, video, contactSheet: sheet.code === 0 ? contactSheet : undefined };
446
- }
447
-}
448
-
449
-/** Screenshot pixels per snapshot point (3 on most iPhones). */
450
-async function screenshotScale(path: string, snapshot: Snapshot): Promise<number> {
451
- const { stdout } = await run("ffprobe", [
452
- "-v",
453
- "error",
454
- "-select_streams",
455
- "v:0",
456
- "-show_entries",
457
- "stream=width",
458
- "-of",
459
- "csv=p=0",
460
- path,
461
- ]);
462
- return Number(stdout.trim()) / snapshot.viewport.width;
463
-}
464
-
465
-async function ensureSimulator(spec: E2eDevice): Promise<string> {
466
- const name = `T3 E2E ${spec.id}`;
467
- const listing = JSON.parse(
468
- (await run("xcrun", ["simctl", "list", "devices", "available", "-j"])).stdout,
469
- ) as { devices: Record<string, ReadonlyArray<{ udid: string; name: string; state: string }>> };
470
- const existing = (listing.devices[spec.runtime] ?? []).find((device) => device.name === name);
471
- const udid =
472
- existing?.udid ??
473
- (await run("xcrun", ["simctl", "create", name, spec.deviceType, spec.runtime])).stdout.trim();
474
- if (existing?.state !== "Booted")
475
- await run("xcrun", ["simctl", "boot", udid], { allowFailure: true });
476
- await run("xcrun", ["simctl", "bootstatus", udid, "-b"]);
477
- await run("xcrun", ["simctl", "ui", udid, "appearance", "light"]);
478
- await run("xcrun", [
479
- "simctl",
480
- "status_bar",
481
- udid,
482
- "override",
483
- "--time",
484
- "9:41",
485
- "--batteryState",
486
- "charged",
487
- "--batteryLevel",
488
- "100",
489
- ]);
490
- return udid;
491
-}
26
+const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`);
27
493
-function lanAddress(): string {
28
+/** This machine's Tailscale address (100.64.0.0/10), which pool devices reach it on. */
29
+function tailnetAddress(): string {
30
for (const addresses of Object.values(NodeOS.networkInterfaces())) {
31
for (const address of addresses ?? []) {
496
- if (address.family === "IPv4" && !address.internal && !address.address.startsWith("169.254."))
32
+ const [first, second] = address.address.split(".").map(Number);
33
+ if (address.family === "IPv4" && first === 100 && second! >= 64 && second! < 128)
34
return address.address;
35
}
36
}
500
- throw new Error("No LAN IPv4 address for the simulator to reach Metro.");
37
+ throw new Error("No Tailscale IPv4 address: pool devices reach Metro and the fixture over it.");
38
}
39
503
-async function runDevice(
504
- spec: E2eDevice,
505
- outputDirectory: string,
506
- environment: { readonly baseDir: string; readonly port: number },
507
- metroHost: string,
508
-): Promise<DeviceResult> {
509
- const steps: StepResult[] = [];
510
- const directory = NodePath.join(outputDirectory, spec.id);
511
- await NodeFSP.mkdir(directory, { recursive: true });
512
- try {
513
- const udid = await ensureSimulator(spec);
514
- const device = new Device(spec, udid);
515
- // A fresh install drops connections paired by earlier runs; the cached client reinstalls in seconds.
516
- await run("xcrun", ["simctl", "uninstall", udid, BUNDLE_ID], { allowFailure: true });
517
- log(spec.id, "installing the dev client");
518
- await run(NodeProcess.execPath, ["scripts/mobile-native-client.ts", "ensure", "ios", udid], {
519
- env: { ...NodeProcess.env, T3_NATIVE_CLIENT_CACHE: NodeProcess.env.T3_NATIVE_CLIENT_CACHE },
520
- });
521
- const metroUrl = encodeURIComponent(`http://${metroHost}:${METRO_PORT}`);
522
- await device.openUrl(
523
- `t3code-dev://expo-development-client/?url=${metroUrl}&disableOnboarding=1&disableFab=1&disableAutoLaunch=1`,
524
- );
525
- // Snapshots read through an agent-device app session; open attaches one to the running app.
526
- await device.agent(["open", BUNDLE_ID]);
527
- // The app ignores the pairing link until its JS has loaded and shown the workspace.
528
- await device.waitFor(
529
- (snapshot) => chrome(snapshot).newChat !== undefined,
530
- "the app to load",
531
- 120_000,
532
- );
533
- const paired = (snapshot: Snapshot) =>
534
- chrome(snapshot).tabBar !== undefined && chrome(snapshot).threadRows.length > 0;
535
- let home: Snapshot | null = null;
536
- for (let attempt = 1; attempt <= 2 && !home; attempt++) {
537
- log(spec.id, attempt === 1 ? "pairing" : "pairing again");
538
- const pairing = new URLSearchParams({
539
- pairingUrl: await issuePairUrl(environment),
540
- autoConnect: "1",
541
- });
542
- await device.openUrl(`t3code-dev://connections/new?${pairing}`);
543
- home = await device.waitFor(paired, "the paired thread list", 45_000).catch(() => null);
544
- }
545
- if (!home) throw new Error("The app did not show the fixture threads after pairing.");
546
- await delay(1_000);
547
- const homeShot = NodePath.join(directory, "home.png");
548
- await device.screenshot(homeShot);
549
- steps.push({ name: "home", checks: homeChecks(home), screenshot: homeShot, samples: 0 });
550
-
551
- log(spec.id, "opening a thread");
552
- const row = chrome(home).threadRows[0];
553
- if (!row?.label) throw new Error("No thread row to open.");
554
- const opened = await device.transition(directory, "open-thread", () =>
555
- device.agent(["press", `label=${JSON.stringify(row.label)}`]),
556
- );
557
- const thread = await device.waitFor(
558
- (snapshot) => chrome(snapshot).backButton !== undefined,
559
- "the thread screen",
560
- );
561
- const threadShot = NodePath.join(directory, "thread.png");
562
- await device.screenshot(threadShot);
563
- const { tabBar, navigationBar } = chrome(home);
564
- const scale = await screenshotScale(homeShot, home);
565
- const regions =
566
- tabBar?.rect && navigationBar?.rect ? { nav: navigationBar.rect, chrome: tabBar.rect } : null;
567
- const openLag = regions
568
- ? await lagCheck(
569
- "no tabs over the entering thread (video)",
570
- { video: opened.video, origin: homeShot, destination: threadShot, scale, ...regions },
571
- NodePath.join(directory, "open-thread-lag.png"),
572
- )
573
- : null;
574
- steps.push({
575
- name: "open thread",
576
- checks: [...threadChecks(thread, opened.samples), ...(openLag ? [openLag.check] : [])],
577
- lagFrame: openLag?.frame,
578
- screenshot: threadShot,
579
- video: opened.video,
580
- contactSheet: opened.contactSheet,
581
- samples: opened.samples.length,
582
- });
583
-
584
- log(spec.id, "going back");
585
- const back = await device.transition(directory, "back", () =>
586
- device.agent(["press", "id=BackButton"]),
587
- );
588
- const list = await device.waitFor(
589
- (snapshot) => chrome(snapshot).backButton === undefined,
590
- "the thread list",
591
- );
592
- const backShot = NodePath.join(directory, "back.png");
593
- await device.screenshot(backShot);
594
- const backLag = regions
595
- ? await lagCheck(
596
- "tabs return with the list (video)",
597
- { video: back.video, origin: threadShot, destination: backShot, scale, ...regions },
598
- NodePath.join(directory, "back-lag.png"),
599
- )
600
- : null;
601
- steps.push({
602
- name: "back",
603
- checks: [...backChecks(list, back.samples), ...(backLag ? [backLag.check] : [])],
604
- lagFrame: backLag?.frame,
605
- screenshot: backShot,
606
- video: back.video,
607
- contactSheet: back.contactSheet,
608
- samples: back.samples.length,
609
- });
610
- return { device: spec, label: spec.id, steps };
611
- } catch (error) {
612
- return {
613
- device: spec,
614
- label: spec.id,
615
- steps,
616
- error: error instanceof Error ? error.message : String(error),
617
- };
618
- }
40
+async function mapConcurrent<T, R>(
41
+ items: ReadonlyArray<T>,
42
+ limit: number,
43
+ task: (item: T) => Promise<R>,
44
+): Promise<R[]> {
45
+ const results: R[] = Array.from({ length: items.length });
46
+ let next = 0;
47
+ await Promise.all(
48
+ Array.from({ length: Math.min(limit, items.length) }, async () => {
49
+ while (next < items.length) {
50
+ const index = next++;
51
+ results[index] = await task(items[index]!);
52
+ }
53
+ }),
54
+ );
55
+ return results;
56
}
57
621
-async function issuePairUrl(environment: { readonly baseDir: string; readonly port: number }) {
622
- const { stdout } = await run(NodeProcess.execPath, [
623
- "apps/server/src/bin.ts",
624
- "auth",
625
- "pairing",
626
- "create",
627
- "--base-dir",
628
- environment.baseDir,
629
- "--base-url",
630
- `http://127.0.0.1:${environment.port}`,
631
- "--ttl",
632
- "15m",
633
- "--json",
634
- ]);
58
+async function issuePairUrl(
59
+ environment: { readonly baseDir: string; readonly port: number },
60
+ host: string,
61
+) {
62
+ const { stdout } = await run(
63
+ NodeProcess.execPath,
64
+ [
65
+ "apps/server/src/bin.ts",
66
+ "auth",
67
+ "pairing",
68
+ "create",
69
+ "--base-dir",
70
+ environment.baseDir,
71
+ "--base-url",
72
+ `http://${host}:${environment.port}`,
73
+ "--ttl",
74
+ "15m",
75
+ "--json",
76
+ ],
77
+ { cwd: REPO_ROOT },
78
+ );
79
const { pairUrl } = JSON.parse(
80
stdout.slice(stdout.indexOf("{"), stdout.lastIndexOf("}") + 1),
81
) as {
85
return pairUrl;
86
}
87
644
-const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`);
645
-
646
-const escapeHtml = (value: string) =>
647
- value.replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">");
648
-
649
-function renderReport(results: ReadonlyArray<DeviceResult>, outputDirectory: string): string {
650
- const relative = (path: string | undefined) =>
651
- path ? escapeHtml(NodePath.relative(outputDirectory, path)) : "";
652
- const rows = results
653
- .map((result) => {
654
- const steps = result.steps
655
- .map((step) => {
656
- const checks = step.checks
657
- .map(
658
- (item) =>
659
- `<li class="${item.pass ? "pass" : "fail"}">${item.pass ? "✓" : "✗"} ${escapeHtml(item.name)} <small>${escapeHtml(item.detail)}</small></li>`,
660
- )
661
- .join("");
662
- const media = [
663
- step.screenshot
664
- ? `<a href="${relative(step.screenshot)}"><img src="${relative(step.screenshot)}"></a>`
665
- : "",
666
- step.lagFrame
667
- ? `<a href="${relative(step.lagFrame)}"><img class="lag" src="${relative(step.lagFrame)}"></a>`
668
- : "",
669
- step.contactSheet
670
- ? `<a href="${relative(step.contactSheet)}">transition frames</a>`
671
- : "",
672
- step.video ? `<a href="${relative(step.video)}">video</a>` : "",
673
- ].join(" ");
674
- return `<td><h3>${escapeHtml(step.name)}</h3><ul>${checks}</ul>${media}</td>`;
675
- })
676
- .join("");
677
- const error = result.error ? `<td class="fail">${escapeHtml(result.error)}</td>` : "";
678
- return `<tr><th>${escapeHtml(result.label)}</th>${steps}${error}</tr>`;
679
- })
680
- .join("");
681
- const failures = results.filter(
682
- (result) => result.error || result.steps.some((step) => step.checks.some((item) => !item.pass)),
683
- ).length;
684
- return `<!doctype html><meta charset="utf-8"><title>Mobile E2E</title>
685
-<style>body{font:14px system-ui;margin:24px}table{border-collapse:collapse}td,th{border:1px solid #ddd;padding:8px;vertical-align:top}img{width:180px;display:block;margin:6px 0}img.lag{outline:3px solid #b42318}.pass{color:#16794a}.fail{color:#b42318}ul{padding-left:16px;margin:4px 0}small{color:#666}</style>
686
-<h1>Mobile E2E · ${failures === 0 ? "all devices passed" : `${failures} device(s) failed`}</h1>
687
-<table>${rows}</table>`;
88
+async function runPose(
89
+ pose: Pose,
90
+ pool: Pool,
91
+ runId: string,
92
+ outputDirectory: string,
93
+ metroUrl: string,
94
+ pair: () => Promise<string>,
95
+): Promise<PoseResult> {
96
+ const directory = NodePath.join(outputDirectory, pose.id);
97
+ await NodeFSP.mkdir(directory, { recursive: true });
98
+ const steps: StepResult[] = [];
99
+ let device: Device | null = null;
100
+ try {
101
+ log(pose.id, `leasing ${pose.device}`);
102
+ device = await Device.lease(pose, pool, runId);
103
+ log(pose.id, "installing the dev client");
104
+ await device.install();
105
+ await device.prepare();
106
+ await device.launch(metroUrl);
107
+ log(pose.id, "pairing and walking the app");
108
+ await runJourney(device, directory, pair, steps);
109
+ return { pose, steps };
110
+ } catch (error) {
111
+ const message = error instanceof Error ? error.message : String(error);
112
+ if (!device) return { pose, steps, error: message };
113
+ // What the device showed when the journey stopped is usually the whole diagnosis.
114
+ const stopped = NodePath.join(directory, "stopped.png");
115
+ const shot = await device.screenshot(stopped).then(
116
+ () => stopped,
117
+ () => undefined,
118
+ );
119
+ const labels = await device.snapshot().then(
120
+ (snapshot) =>
121
+ snapshot.nodes
122
+ .map((node) => node.label)
123
+ .filter(Boolean)
124
+ .slice(0, 12)
125
+ .join(" | "),
126
+ () => "",
127
+ );
128
+ return {
129
+ pose,
130
+ steps,
131
+ error: labels ? `${message} (screen: ${labels})` : message,
132
+ ...(shot ? { stopped: shot } : {}),
133
+ };
134
+ } finally {
135
+ await device?.release();
136
+ }
137
}
138
139
async function main() {
140
const args = NodeProcess.argv.slice(2);
141
if (args.includes("--list")) {
693
- for (const device of DEVICES) NodeProcess.stdout.write(`${device.id}\n`);
142
+ for (const pose of POSES) NodeProcess.stdout.write(`${pose.id}\t${pose.device}\n`);
143
return;
144
}
696
- const selected = args.flatMap((arg, index) => (args[index - 1] === "--device" ? [arg] : []));
697
- const devices = selected.length
698
- ? DEVICES.filter((device) => selected.includes(device.id))
699
- : DEVICES;
700
- const unknown = selected.filter((id) => !DEVICES.some((device) => device.id === id));
701
- if (unknown.length) throw new Error(`Unknown device ${unknown.join(", ")}. Run with --list.`);
702
- const concurrencyArg = args[args.indexOf("--concurrency") + 1];
703
- const concurrency = args.includes("--concurrency") ? Number(concurrencyArg) : 3;
145
+ const values = (flag: string) =>
146
+ args.flatMap((arg, index) => (args[index - 1] === flag ? [arg] : []));
147
+ const selected = values("--pose");
148
+ const unknown = selected.filter((id) => !POSES.some((pose) => pose.id === id));
149
+ if (unknown.length) throw new Error(`Unknown pose ${unknown.join(", ")}. Run with --list.`);
150
+ const platforms = values("--platform");
151
+ const poses = POSES.filter(
152
+ (pose) =>
153
+ (selected.length === 0 || selected.includes(pose.id)) &&
154
+ (platforms.length === 0 || platforms.includes(pose.platform)),
155
+ );
156
+ const concurrencyFlag = values("--concurrency")[0];
157
+ const concurrency = concurrencyFlag ? Number(concurrencyFlag) : undefined;
158
+ const review = !args.includes("--no-review");
159
+
160
+ const agentDeviceUrl = NodeProcess.env.SIMLOCK_AGENT_DEVICE_URL;
161
+ const token = NodeProcess.env.AGENT_DEVICE_DAEMON_AUTH_TOKEN;
162
+ if (!agentDeviceUrl || !token)
163
+ throw new Error(
164
+ "Set SIMLOCK_AGENT_DEVICE_URL (<gateway>/v1/agent-device) and AGENT_DEVICE_DAEMON_AUTH_TOKEN (a Simlock agent token) to run on the device pool.",
165
+ );
166
+ const host = tailnetAddress();
167
705
- const stamp = new Date().toISOString().replaceAll(":", "-");
706
- const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-e2e", stamp);
168
+ const runId = new Date().toISOString().replaceAll(/[:.]/g, "-");
169
+ const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-e2e", runId);
170
await NodeFSP.mkdir(outputDirectory, { recursive: true });
171
const rootDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-mobile-e2e-"));
172
+ const workspace = await NodeFSP.realpath(outputDirectory);
173
+ const config = NodePath.join(rootDir, "agent-device.json");
174
+ await NodeFSP.writeFile(
175
+ config,
176
+ JSON.stringify({
177
+ daemonBaseUrl: agentDeviceUrl,
178
+ daemonAuthToken: token,
179
+ stateDir: NodePath.join(rootDir, "agent-device"),
180
+ }),
181
+ { mode: 0o600 },
182
+ );
183
+ const pool: Pool = {
184
+ repoRoot: REPO_ROOT,
185
+ agentDevice: NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"),
186
+ agentDeviceUrl,
187
+ workspace,
188
+ env: { ...NodeProcess.env, AGENT_DEVICE_CONFIG: config, PWD: workspace },
189
+ };
190
+
191
const servers: NodeChildProcess.ChildProcess[] = [];
192
let metro: NodeChildProcess.ChildProcess | null = null;
193
try {
198
SHOWCASE_ENVIRONMENTS.slice(0, 1),
199
);
200
if (!environment) throw new Error("The fixture server did not start.");
719
- const metroHost = lanAddress();
201
+ const metroPort = await reserveAvailablePort();
202
metro = NodeChildProcess.spawn(
203
"vp",
204
[
210
"t3code-dev",
211
"--lan",
212
"--port",
731
- String(METRO_PORT),
213
+ String(metroPort),
214
],
215
{
216
cwd: NodePath.join(REPO_ROOT, "apps/mobile"),
220
detached: true,
221
},
222
);
741
- await waitForPort(METRO_PORT, "Metro", 120_000);
742
- await run("curl", [
743
- "-sf",
744
- "-o",
745
- "/dev/null",
746
- `http://127.0.0.1:${METRO_PORT}/apps/mobile/index.bundle?platform=ios&dev=true&minify=false`,
747
- ]);
223
+ await waitForPort(metroPort, "Metro", 120_000);
224
+ // Warm each platform's bundle once, instead of on every device at once.
225
+ for (const platform of new Set(poses.map((pose) => pose.platform))) {
226
+ log("e2e", `bundling for ${platform}`);
227
+ await run("curl", [
228
+ "-sf",
229
+ "-o",
230
+ "/dev/null",
231
+ `http://127.0.0.1:${metroPort}/apps/mobile/index.bundle?platform=${platform}&dev=true&minify=false`,
232
+ ]);
233
+ }
234
749
- // The first install may build; later ones reuse its artifact.
750
- const results: DeviceResult[] = [];
751
- const [first, ...rest] = devices;
752
- if (first) results.push(await runDevice(first, outputDirectory, environment, metroHost));
753
- for (let index = 0; index < rest.length; index += concurrency) {
754
- results.push(
755
- ...(await Promise.all(
756
- rest
757
- .slice(index, index + concurrency)
758
- .map((device) => runDevice(device, outputDirectory, environment, metroHost)),
759
- )),
235
+ const attempt = (pose: Pose, session: string) =>
236
+ runPose(pose, pool, session, outputDirectory, `http://${host}:${metroPort}`, () =>
237
+ issuePairUrl(environment, host),
238
);
761
- }
239
+ const runOne = async (pose: Pose) => {
240
+ const first = await attempt(pose, runId);
241
+ // A pose that failed before its first screen usually met the pool, not the app.
242
+ if (!first.error || first.steps.length > 0) return first;
243
+ log(pose.id, `retrying after: ${first.error.split("\n")[0]}`);
244
+ return attempt(pose, `${runId}-retry`);
245
+ };
246
+ // Each platform has its own workers, so each gets its own limit; one Mac runs every
247
+ // iOS simulator, and recording several at once on a shared pool times it out.
248
+ const byPlatform = await Promise.all(
249
+ (["ios", "android"] as const).map((platform) =>
250
+ mapConcurrent(
251
+ poses.filter((pose) => pose.platform === platform),
252
+ concurrency ?? (platform === "ios" ? 2 : 3),
253
+ runOne,
254
+ ),
255
+ ),
256
+ );
257
+ const results = poses.map((pose) => byPlatform.flat().find((result) => result.pose === pose)!);
258
+
259
+ const reviewed = review
260
+ ? await mapConcurrent(results, 4, async (result) => {
261
+ log(result.pose.id, "agent review");
262
+ const findings = await reviewPose(result, outputDirectory).catch((error) => {
263
+ log(result.pose.id, `review failed: ${error instanceof Error ? error.message : error}`);
264
+ return [];
265
+ });
266
+ return { ...result, findings };
267
+ })
268
+ : results;
269
+
270
const report = NodePath.join(outputDirectory, "index.html");
763
- await NodeFSP.writeFile(report, renderReport(results, outputDirectory));
271
+ await NodeFSP.writeFile(report, renderReport(reviewed, outputDirectory));
272
await NodeFSP.writeFile(
273
NodePath.join(outputDirectory, "report.json"),
766
- JSON.stringify(results, null, 2),
274
+ JSON.stringify(reviewed, null, 2),
275
);
768
- const failed = results.filter(
769
- (result) =>
770
- result.error || result.steps.some((step) => step.checks.some((item) => !item.pass)),
771
- );
772
- for (const result of results) {
773
- const failures = result.steps.flatMap((step) =>
774
- step.checks
775
- .filter((item) => !item.pass)
776
- .map((item) => `${step.name}: ${item.name} (${item.detail})`),
276
+ for (const result of reviewed) {
277
+ const failures = failedChecks(result).map(
278
+ (item) => `${item.step}: ${item.name} (${item.detail})`,
279
+ );
280
+ log(
281
+ result.pose.id,
282
+ [result.error, ...failures].filter(Boolean).join("; ") ||
283
+ `passed${result.findings?.length ? `, ${result.findings.length} review findings` : ""}`,
284
);
778
- log(result.label, result.error ?? (failures.length ? failures.join("; ") : "passed"));
285
}
286
+ const failed = reviewed.filter(poseFailed).length;
287
NodeProcess.stdout.write(
781
- `${JSON.stringify({ report, devices: results.length, failed: failed.length })}\n`,
288
+ `${JSON.stringify({
289
+ report,
290
+ poses: reviewed.length,
291
+ failed,
292
+ findings: reviewed.reduce((count, result) => count + (result.findings?.length ?? 0), 0),
293
+ })}\n`,
294
);
783
- process.exitCode = failed.length ? 1 : 0;
295
+ process.exitCode = failed ? 1 : 0;
296
} finally {
297
if (metro?.pid) {
298
try {
787
- process.kill(-metro.pid, "SIGTERM");
299
+ NodeProcess.kill(-metro.pid, "SIGTERM");
300
} catch {}
301
await stopProcess(metro);
302
}