The best way to control Claude Code, Codex and more from any computer or phone

main
ts 297 lines 10.4 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off globalTimers:off - Host-side device automation.
2
3 import * as NodePath from "node:path";
4
5 import {
6 check,
7 type Check,
8 describeRect,
9 findLabel,
10 keyboardRect,
11 layoutChecks,
12 type Rect,
13 type Snapshot,
14 type SnapshotNode,
15 tabBarRect,
16 } from "./checks.ts";
17 import { BUNDLE_ID, delay, type Device } from "./device.ts";
18 import { lagCheck } from "./video.ts";
19
20 export interface StepResult {
21 readonly name: string;
22 readonly checks: ReadonlyArray<Check>;
23 readonly screenshot?: string;
24 readonly video?: string;
25 readonly lagFrame?: string;
26 }
27
28 // Threads of the fixture's first environment (scripts/mobile-showcase-environment.ts).
29 const THREAD_TITLE =
30 /Make remote coding feel local|Put the command center in your pocket|Tune the handoff haptics/;
31 const BACK = /^(Back|Navigate up|Go back)$/i;
32
33 const backButton = (snapshot: Snapshot) =>
34 snapshot.nodes.find((node) => node.identifier === "BackButton") ?? findLabel(snapshot, BACK);
35 const composer = (snapshot: Snapshot) =>
36 snapshot.nodes
37 .filter(
38 (node) =>
39 (node.kind === "text-field" || node.kind === "text-view" || node.editable === true) &&
40 node.rect &&
41 node.rect.y < snapshot.viewport.height &&
42 node.bundleId !== "com.android.systemui",
43 )
44 .toSorted((a, b) => b.rect!.y - a.rect!.y)[0];
45 const threadRow = (snapshot: Snapshot) => findLabel(snapshot, THREAD_TITLE);
46
47 const inside = (rect: Rect | undefined, snapshot: Snapshot) =>
48 rect !== undefined &&
49 rect.x >= 0 &&
50 rect.y >= 0 &&
51 rect.x + rect.width <= snapshot.viewport.width + 1 &&
52 rect.y + rect.height <= snapshot.viewport.height + 1;
53
54 const topBand = (snapshot: Snapshot): Rect => ({
55 x: 0,
56 y: 0,
57 width: snapshot.viewport.width,
58 height: snapshot.viewport.height * 0.12,
59 });
60
61 /**
62 * Walks one device through the app like a person would: thread list, a thread, scrolling,
63 * typing, back, settings, then the pose's own change (rotation or fold). Every screen gets
64 * the generic layout checks plus what that screen must show. Steps land in `steps` as they
65 * finish, so a journey that stops early still reports what it saw.
66 */
67 export async function runJourney(
68 device: Device,
69 directory: string,
70 pair: () => Promise<string>,
71 steps: StepResult[],
72 ): Promise<void> {
73 const phone = device.pose.form === "phone";
74 const tap = async (node: SnapshotNode | undefined, label: string) => {
75 if (!node?.ref || !node.rect) throw new Error(`no ${label} to tap`);
76 // agent-device refuses refs it considers covered, such as on the unfolded Duo's inner
77 // display; the element's centre still takes the tap.
78 await device
79 .agent(["press", `@${node.ref}`])
80 .catch(() =>
81 device.agent([
82 "press",
83 String(Math.round(node.rect!.x + node.rect!.width / 2)),
84 String(Math.round(node.rect!.y + node.rect!.height / 2)),
85 ]),
86 );
87 };
88
89 /** Screenshot and check the settled screen, and record it as a step. */
90 const capture = async (
91 name: string,
92 snapshot: Snapshot,
93 expectations: ReadonlyArray<Check> = [],
94 extra: Partial<StepResult> = {},
95 ) => {
96 const screenshot = NodePath.join(directory, `${steps.length + 1}-${name}.png`);
97 await device.screenshot(screenshot);
98 const step = {
99 name,
100 checks: [...layoutChecks(snapshot, BUNDLE_ID), ...expectations],
101 screenshot,
102 ...extra,
103 };
104 steps.push(step);
105 return step;
106 };
107
108 // The app ignores the pairing link until its JS has loaded and shown the workspace. An
109 // error dialog on the way is a finding: record it, dismiss it, and keep going.
110 const loaded = (snapshot: Snapshot) =>
111 findLabel(snapshot, /Add environment|New (chat|thread)/i) !== undefined;
112 const errorDialog = (snapshot: Snapshot) =>
113 findLabel(snapshot, /^(Dismiss|OK|Close)$/) !== undefined
114 ? findLabel(snapshot, /^Could not|failed|went wrong/i)
115 : undefined;
116 let dialogs = 0;
117 /** Waits for `ready`, recording and dismissing up to three error dialogs on the way. */
118 const settle = async (
119 ready: (snapshot: Snapshot) => boolean,
120 label: string,
121 timeoutMs: number,
122 ) => {
123 for (;;) {
124 const snapshot = await device.waitFor(
125 (current) => ready(current) || errorDialog(current) !== undefined,
126 label,
127 timeoutMs,
128 );
129 const dialog = errorDialog(snapshot);
130 if (!dialog || dialogs >= 3) return snapshot;
131 dialogs++;
132 await capture("launch error", snapshot, [
133 check("no error dialog on launch", false, dialog.label ?? "error dialog"),
134 ]);
135 await device.agent([
136 "press",
137 `label="${findLabel(snapshot, /^(Dismiss|OK|Close)$/)!.label}"`,
138 ]);
139 await delay(1_000);
140 }
141 };
142 await settle(loaded, "the app to load", 180_000);
143 let threads: Snapshot | null = null;
144 for (let attempt = 1; attempt <= 2 && !threads; attempt++) {
145 const pairing = new URLSearchParams({ pairingUrl: await pair(), autoConnect: "1" });
146 await device.openUrl(`t3code-dev://connections/new?${pairing}`);
147 threads = await settle(
148 (snapshot) => threadRow(snapshot) !== undefined,
149 "the paired thread list",
150 60_000,
151 )
152 .then((snapshot) => (threadRow(snapshot) ? snapshot : null))
153 .catch(() => null);
154 }
155 if (!threads) throw new Error("the app did not show the fixture threads after pairing");
156 // Rotated only once the app is up: a rotation during launch is lost.
157 await delay(1_000);
158 threads = await device.orient(device.pose.landscape ? "landscape" : "portrait");
159 const tabs = tabBarRect(threads);
160 // A bottom tab bar gives way to a pushed thread; the folded Duo's side rail stays put.
161 const bottomTabs = phone && tabs && tabs.width > tabs.height ? tabs : undefined;
162 const wide = threads.viewport.width > threads.viewport.height;
163 await capture("threads", threads, [
164 check(
165 device.pose.landscape ? "in landscape" : "in portrait",
166 wide === Boolean(device.pose.landscape),
167 `${Math.round(threads.viewport.width)}×${Math.round(threads.viewport.height)}`,
168 ),
169 check("thread list visible", threadRow(threads) !== undefined, "fixture threads"),
170 ...(phone ? [check("tab bar visible", inside(tabs, threads), describeRect(tabs))] : []),
171 ]);
172
173 const opening = await device.recorded(directory, "open-thread", () =>
174 tap(threadRow(threads!), "thread row"),
175 );
176 const thread = await device.waitFor(
177 (snapshot) => composer(snapshot) !== undefined && threadRow(snapshot) !== undefined,
178 "the thread screen",
179 );
180 const threadShot = NodePath.join(directory, `${steps.length + 1}-thread.png`);
181 await device.screenshot(threadShot);
182 const openLag = bottomTabs
183 ? await lagCheck(
184 "no tabs over the entering thread (video)",
185 {
186 video: opening,
187 nav: topBand(threads),
188 chrome: bottomTabs,
189 viewportWidth: threads.viewport.width,
190 },
191 NodePath.join(directory, "open-thread-lag.png"),
192 )
193 : null;
194 steps.push({
195 name: "thread",
196 screenshot: threadShot,
197 video: opening,
198 ...(openLag?.frame ? { lagFrame: openLag.frame } : {}),
199 checks: [
200 ...layoutChecks(thread, BUNDLE_ID),
201 check(
202 "composer inside the screen",
203 inside(composer(thread)?.rect, thread),
204 describeRect(composer(thread)?.rect),
205 ),
206 ...(phone ? [check("back button visible", backButton(thread) !== undefined, "back")] : []),
207 ...(bottomTabs
208 ? [
209 check(
210 "tab bar hidden",
211 tabBarRect(thread) === undefined,
212 describeRect(tabBarRect(thread)),
213 ),
214 ]
215 : []),
216 ...(openLag ? [openLag.check] : []),
217 ],
218 });
219
220 await device.agent(["scroll", "up"]).catch(() => undefined);
221 await delay(800);
222 await capture("scrolled up", await device.snapshot());
223 await device.agent(["scroll", "down"]).catch(() => undefined);
224 await delay(800);
225
226 await tap(composer(await device.snapshot()), "composer");
227 await device.agent(["type", "Checking the composer above the keyboard"]);
228 await delay(1_200);
229 const typing = await device.snapshot();
230 await capture("typing", typing, [
231 check(
232 "composer visible while typing",
233 inside(composer(typing)?.rect, typing),
234 describeRect(composer(typing)?.rect),
235 ),
236 check(
237 "composer above the keyboard",
238 keyboardRect(typing) === undefined ||
239 (composer(typing)?.rect?.y ?? Infinity) < keyboardRect(typing)!.y,
240 `composer ${describeRect(composer(typing)?.rect)}, keyboard ${describeRect(keyboardRect(typing))}`,
241 ),
242 ]);
243 await device.agent(["keyboard", "dismiss"]).catch(() => undefined);
244 await delay(800);
245
246 if (phone) {
247 const back = await device.recorded(directory, "back", async () => {
248 const current = await device.snapshot();
249 const button = backButton(current);
250 if (button) await tap(button, "back button");
251 else await device.agent(["back"]);
252 });
253 const list = await device.waitFor(
254 (snapshot) => backButton(snapshot) === undefined && threadRow(snapshot) !== undefined,
255 "the thread list",
256 );
257 const listShot = NodePath.join(directory, `${steps.length + 1}-back.png`);
258 await device.screenshot(listShot);
259 const backLag = bottomTabs
260 ? await lagCheck(
261 "tabs return with the list (video)",
262 {
263 video: back,
264 nav: topBand(list),
265 chrome: bottomTabs,
266 viewportWidth: list.viewport.width,
267 },
268 NodePath.join(directory, "back-lag.png"),
269 )
270 : null;
271 steps.push({
272 name: "back",
273 screenshot: listShot,
274 video: back,
275 ...(backLag?.frame ? { lagFrame: backLag.frame } : {}),
276 checks: [
277 ...layoutChecks(list, BUNDLE_ID),
278 check("tab bar back", inside(tabBarRect(list), list), describeRect(tabBarRect(list))),
279 ...(backLag ? [backLag.check] : []),
280 ],
281 });
282 }
283
284 // The labelled tab is not always the hittable node; agent-device resolves the press.
285 await device.agent(["press", 'label="Settings"']);
286 await delay(1_500);
287 await capture("settings", await device.snapshot());
288
289 if (device.pose.form === "tablet" && !device.pose.fold) {
290 await capture("rotated", await device.orient(device.pose.landscape ? "portrait" : "landscape"));
291 }
292 if (device.pose.fold) {
293 await device.agent(["fold", device.pose.fold === "open" ? "closed" : "open"]);
294 await delay(2_500);
295 await capture(device.pose.fold === "open" ? "folded" : "unfolded", await device.snapshot());
296 }
297 }