The best way to control Claude Code, Codex and more from any computer or phone
| 1 | // @effect-diagnostics nodeBuiltinImport:off globalTimers:off - Host-side device automation. |
| 2 | |
| 3 | import * as NodePath from "node:path"; |
| 4 | |
| 5 | import { |
| 6 | check, |
| 7 | type Check, |
| 8 | describeRect, |
| 9 | findLabel, |
| 10 | keyboardRect, |
| 11 | layoutChecks, |
| 12 | type Rect, |
| 13 | type Snapshot, |
| 14 | type SnapshotNode, |
| 15 | tabBarRect, |
| 16 | } from "./checks.ts"; |
| 17 | import { BUNDLE_ID, delay, type Device } from "./device.ts"; |
| 18 | import { lagCheck } from "./video.ts"; |
| 19 | |
| 20 | export interface StepResult { |
| 21 | readonly name: string; |
| 22 | readonly checks: ReadonlyArray<Check>; |
| 23 | readonly screenshot?: string; |
| 24 | readonly video?: string; |
| 25 | readonly lagFrame?: string; |
| 26 | } |
| 27 | |
| 28 | // Threads of the fixture's first environment (scripts/mobile-showcase-environment.ts). |
| 29 | const THREAD_TITLE = |
| 30 | /Make remote coding feel local|Put the command center in your pocket|Tune the handoff haptics/; |
| 31 | const BACK = /^(Back|Navigate up|Go back)$/i; |
| 32 | |
| 33 | const backButton = (snapshot: Snapshot) => |
| 34 | snapshot.nodes.find((node) => node.identifier === "BackButton") ?? findLabel(snapshot, BACK); |
| 35 | const composer = (snapshot: Snapshot) => |
| 36 | snapshot.nodes |
| 37 | .filter( |
| 38 | (node) => |
| 39 | (node.kind === "text-field" || node.kind === "text-view" || node.editable === true) && |
| 40 | node.rect && |
| 41 | node.rect.y < snapshot.viewport.height && |
| 42 | node.bundleId !== "com.android.systemui", |
| 43 | ) |
| 44 | .toSorted((a, b) => b.rect!.y - a.rect!.y)[0]; |
| 45 | const threadRow = (snapshot: Snapshot) => findLabel(snapshot, THREAD_TITLE); |
| 46 | |
| 47 | const inside = (rect: Rect | undefined, snapshot: Snapshot) => |
| 48 | rect !== undefined && |
| 49 | rect.x >= 0 && |
| 50 | rect.y >= 0 && |
| 51 | rect.x + rect.width <= snapshot.viewport.width + 1 && |
| 52 | rect.y + rect.height <= snapshot.viewport.height + 1; |
| 53 | |
| 54 | const topBand = (snapshot: Snapshot): Rect => ({ |
| 55 | x: 0, |
| 56 | y: 0, |
| 57 | width: snapshot.viewport.width, |
| 58 | height: snapshot.viewport.height * 0.12, |
| 59 | }); |
| 60 | |
| 61 | /** |
| 62 | * Walks one device through the app like a person would: thread list, a thread, scrolling, |
| 63 | * typing, back, settings, then the pose's own change (rotation or fold). Every screen gets |
| 64 | * the generic layout checks plus what that screen must show. Steps land in `steps` as they |
| 65 | * finish, so a journey that stops early still reports what it saw. |
| 66 | */ |
| 67 | export async function runJourney( |
| 68 | device: Device, |
| 69 | directory: string, |
| 70 | pair: () => Promise<string>, |
| 71 | steps: StepResult[], |
| 72 | ): Promise<void> { |
| 73 | const phone = device.pose.form === "phone"; |
| 74 | const tap = async (node: SnapshotNode | undefined, label: string) => { |
| 75 | if (!node?.ref || !node.rect) throw new Error(`no ${label} to tap`); |
| 76 | // agent-device refuses refs it considers covered, such as on the unfolded Duo's inner |
| 77 | // display; the element's centre still takes the tap. |
| 78 | await device |
| 79 | .agent(["press", `@${node.ref}`]) |
| 80 | .catch(() => |
| 81 | device.agent([ |
| 82 | "press", |
| 83 | String(Math.round(node.rect!.x + node.rect!.width / 2)), |
| 84 | String(Math.round(node.rect!.y + node.rect!.height / 2)), |
| 85 | ]), |
| 86 | ); |
| 87 | }; |
| 88 | |
| 89 | /** Screenshot and check the settled screen, and record it as a step. */ |
| 90 | const capture = async ( |
| 91 | name: string, |
| 92 | snapshot: Snapshot, |
| 93 | expectations: ReadonlyArray<Check> = [], |
| 94 | extra: Partial<StepResult> = {}, |
| 95 | ) => { |
| 96 | const screenshot = NodePath.join(directory, `${steps.length + 1}-${name}.png`); |
| 97 | await device.screenshot(screenshot); |
| 98 | const step = { |
| 99 | name, |
| 100 | checks: [...layoutChecks(snapshot, BUNDLE_ID), ...expectations], |
| 101 | screenshot, |
| 102 | ...extra, |
| 103 | }; |
| 104 | steps.push(step); |
| 105 | return step; |
| 106 | }; |
| 107 | |
| 108 | // The app ignores the pairing link until its JS has loaded and shown the workspace. An |
| 109 | // error dialog on the way is a finding: record it, dismiss it, and keep going. |
| 110 | const loaded = (snapshot: Snapshot) => |
| 111 | findLabel(snapshot, /Add environment|New (chat|thread)/i) !== undefined; |
| 112 | const errorDialog = (snapshot: Snapshot) => |
| 113 | findLabel(snapshot, /^(Dismiss|OK|Close)$/) !== undefined |
| 114 | ? findLabel(snapshot, /^Could not|failed|went wrong/i) |
| 115 | : undefined; |
| 116 | let dialogs = 0; |
| 117 | /** Waits for `ready`, recording and dismissing up to three error dialogs on the way. */ |
| 118 | const settle = async ( |
| 119 | ready: (snapshot: Snapshot) => boolean, |
| 120 | label: string, |
| 121 | timeoutMs: number, |
| 122 | ) => { |
| 123 | for (;;) { |
| 124 | const snapshot = await device.waitFor( |
| 125 | (current) => ready(current) || errorDialog(current) !== undefined, |
| 126 | label, |
| 127 | timeoutMs, |
| 128 | ); |
| 129 | const dialog = errorDialog(snapshot); |
| 130 | if (!dialog || dialogs >= 3) return snapshot; |
| 131 | dialogs++; |
| 132 | await capture("launch error", snapshot, [ |
| 133 | check("no error dialog on launch", false, dialog.label ?? "error dialog"), |
| 134 | ]); |
| 135 | await device.agent([ |
| 136 | "press", |
| 137 | `label="${findLabel(snapshot, /^(Dismiss|OK|Close)$/)!.label}"`, |
| 138 | ]); |
| 139 | await delay(1_000); |
| 140 | } |
| 141 | }; |
| 142 | await settle(loaded, "the app to load", 180_000); |
| 143 | let threads: Snapshot | null = null; |
| 144 | for (let attempt = 1; attempt <= 2 && !threads; attempt++) { |
| 145 | const pairing = new URLSearchParams({ pairingUrl: await pair(), autoConnect: "1" }); |
| 146 | await device.openUrl(`t3code-dev://connections/new?${pairing}`); |
| 147 | threads = await settle( |
| 148 | (snapshot) => threadRow(snapshot) !== undefined, |
| 149 | "the paired thread list", |
| 150 | 60_000, |
| 151 | ) |
| 152 | .then((snapshot) => (threadRow(snapshot) ? snapshot : null)) |
| 153 | .catch(() => null); |
| 154 | } |
| 155 | if (!threads) throw new Error("the app did not show the fixture threads after pairing"); |
| 156 | // Rotated only once the app is up: a rotation during launch is lost. |
| 157 | await delay(1_000); |
| 158 | threads = await device.orient(device.pose.landscape ? "landscape" : "portrait"); |
| 159 | const tabs = tabBarRect(threads); |
| 160 | // A bottom tab bar gives way to a pushed thread; the folded Duo's side rail stays put. |
| 161 | const bottomTabs = phone && tabs && tabs.width > tabs.height ? tabs : undefined; |
| 162 | const wide = threads.viewport.width > threads.viewport.height; |
| 163 | await capture("threads", threads, [ |
| 164 | check( |
| 165 | device.pose.landscape ? "in landscape" : "in portrait", |
| 166 | wide === Boolean(device.pose.landscape), |
| 167 | `${Math.round(threads.viewport.width)}×${Math.round(threads.viewport.height)}`, |
| 168 | ), |
| 169 | check("thread list visible", threadRow(threads) !== undefined, "fixture threads"), |
| 170 | ...(phone ? [check("tab bar visible", inside(tabs, threads), describeRect(tabs))] : []), |
| 171 | ]); |
| 172 | |
| 173 | const opening = await device.recorded(directory, "open-thread", () => |
| 174 | tap(threadRow(threads!), "thread row"), |
| 175 | ); |
| 176 | const thread = await device.waitFor( |
| 177 | (snapshot) => composer(snapshot) !== undefined && threadRow(snapshot) !== undefined, |
| 178 | "the thread screen", |
| 179 | ); |
| 180 | const threadShot = NodePath.join(directory, `${steps.length + 1}-thread.png`); |
| 181 | await device.screenshot(threadShot); |
| 182 | const openLag = bottomTabs |
| 183 | ? await lagCheck( |
| 184 | "no tabs over the entering thread (video)", |
| 185 | { |
| 186 | video: opening, |
| 187 | nav: topBand(threads), |
| 188 | chrome: bottomTabs, |
| 189 | viewportWidth: threads.viewport.width, |
| 190 | }, |
| 191 | NodePath.join(directory, "open-thread-lag.png"), |
| 192 | ) |
| 193 | : null; |
| 194 | steps.push({ |
| 195 | name: "thread", |
| 196 | screenshot: threadShot, |
| 197 | video: opening, |
| 198 | ...(openLag?.frame ? { lagFrame: openLag.frame } : {}), |
| 199 | checks: [ |
| 200 | ...layoutChecks(thread, BUNDLE_ID), |
| 201 | check( |
| 202 | "composer inside the screen", |
| 203 | inside(composer(thread)?.rect, thread), |
| 204 | describeRect(composer(thread)?.rect), |
| 205 | ), |
| 206 | ...(phone ? [check("back button visible", backButton(thread) !== undefined, "back")] : []), |
| 207 | ...(bottomTabs |
| 208 | ? [ |
| 209 | check( |
| 210 | "tab bar hidden", |
| 211 | tabBarRect(thread) === undefined, |
| 212 | describeRect(tabBarRect(thread)), |
| 213 | ), |
| 214 | ] |
| 215 | : []), |
| 216 | ...(openLag ? [openLag.check] : []), |
| 217 | ], |
| 218 | }); |
| 219 | |
| 220 | await device.agent(["scroll", "up"]).catch(() => undefined); |
| 221 | await delay(800); |
| 222 | await capture("scrolled up", await device.snapshot()); |
| 223 | await device.agent(["scroll", "down"]).catch(() => undefined); |
| 224 | await delay(800); |
| 225 | |
| 226 | await tap(composer(await device.snapshot()), "composer"); |
| 227 | await device.agent(["type", "Checking the composer above the keyboard"]); |
| 228 | await delay(1_200); |
| 229 | const typing = await device.snapshot(); |
| 230 | await capture("typing", typing, [ |
| 231 | check( |
| 232 | "composer visible while typing", |
| 233 | inside(composer(typing)?.rect, typing), |
| 234 | describeRect(composer(typing)?.rect), |
| 235 | ), |
| 236 | check( |
| 237 | "composer above the keyboard", |
| 238 | keyboardRect(typing) === undefined || |
| 239 | (composer(typing)?.rect?.y ?? Infinity) < keyboardRect(typing)!.y, |
| 240 | `composer ${describeRect(composer(typing)?.rect)}, keyboard ${describeRect(keyboardRect(typing))}`, |
| 241 | ), |
| 242 | ]); |
| 243 | await device.agent(["keyboard", "dismiss"]).catch(() => undefined); |
| 244 | await delay(800); |
| 245 | |
| 246 | if (phone) { |
| 247 | const back = await device.recorded(directory, "back", async () => { |
| 248 | const current = await device.snapshot(); |
| 249 | const button = backButton(current); |
| 250 | if (button) await tap(button, "back button"); |
| 251 | else await device.agent(["back"]); |
| 252 | }); |
| 253 | const list = await device.waitFor( |
| 254 | (snapshot) => backButton(snapshot) === undefined && threadRow(snapshot) !== undefined, |
| 255 | "the thread list", |
| 256 | ); |
| 257 | const listShot = NodePath.join(directory, `${steps.length + 1}-back.png`); |
| 258 | await device.screenshot(listShot); |
| 259 | const backLag = bottomTabs |
| 260 | ? await lagCheck( |
| 261 | "tabs return with the list (video)", |
| 262 | { |
| 263 | video: back, |
| 264 | nav: topBand(list), |
| 265 | chrome: bottomTabs, |
| 266 | viewportWidth: list.viewport.width, |
| 267 | }, |
| 268 | NodePath.join(directory, "back-lag.png"), |
| 269 | ) |
| 270 | : null; |
| 271 | steps.push({ |
| 272 | name: "back", |
| 273 | screenshot: listShot, |
| 274 | video: back, |
| 275 | ...(backLag?.frame ? { lagFrame: backLag.frame } : {}), |
| 276 | checks: [ |
| 277 | ...layoutChecks(list, BUNDLE_ID), |
| 278 | check("tab bar back", inside(tabBarRect(list), list), describeRect(tabBarRect(list))), |
| 279 | ...(backLag ? [backLag.check] : []), |
| 280 | ], |
| 281 | }); |
| 282 | } |
| 283 | |
| 284 | // The labelled tab is not always the hittable node; agent-device resolves the press. |
| 285 | await device.agent(["press", 'label="Settings"']); |
| 286 | await delay(1_500); |
| 287 | await capture("settings", await device.snapshot()); |
| 288 | |
| 289 | if (device.pose.form === "tablet" && !device.pose.fold) { |
| 290 | await capture("rotated", await device.orient(device.pose.landscape ? "portrait" : "landscape")); |
| 291 | } |
| 292 | if (device.pose.fold) { |
| 293 | await device.agent(["fold", device.pose.fold === "open" ? "closed" : "open"]); |
| 294 | await delay(2_500); |
| 295 | await capture(device.pose.fold === "open" ? "folded" : "unfolded", await device.snapshot()); |
| 296 | } |
| 297 | } |