The best way to control Claude Code, Codex and more from any computer or phone
| 1 | export interface Rect { |
| 2 | readonly x: number; |
| 3 | readonly y: number; |
| 4 | readonly width: number; |
| 5 | readonly height: number; |
| 6 | } |
| 7 | |
| 8 | /** One node of an agent-device `snapshot --json`, on iOS or Android. */ |
| 9 | export interface SnapshotNode { |
| 10 | readonly index: number; |
| 11 | readonly parentIndex?: number; |
| 12 | /** Valid for commands until the next snapshot. */ |
| 13 | readonly ref?: string; |
| 14 | readonly kind?: string; |
| 15 | readonly type?: string; |
| 16 | readonly label?: string; |
| 17 | readonly identifier?: string; |
| 18 | readonly rect?: Rect; |
| 19 | readonly hittable?: boolean; |
| 20 | readonly editable?: boolean; |
| 21 | readonly bundleId?: string; |
| 22 | readonly systemBar?: string; |
| 23 | } |
| 24 | |
| 25 | export interface Snapshot { |
| 26 | readonly nodes: ReadonlyArray<SnapshotNode>; |
| 27 | readonly viewport: { readonly width: number; readonly height: number }; |
| 28 | readonly appBundleId?: string; |
| 29 | readonly keyboard?: { readonly kind?: string; readonly rect?: Rect }; |
| 30 | } |
| 31 | |
| 32 | export interface Check { |
| 33 | readonly name: string; |
| 34 | readonly pass: boolean; |
| 35 | readonly detail: string; |
| 36 | } |
| 37 | |
| 38 | export const check = (name: string, pass: boolean, detail: string): Check => ({ |
| 39 | name, |
| 40 | pass, |
| 41 | detail, |
| 42 | }); |
| 43 | |
| 44 | // Kinds that are never controls a person taps; iOS also marks plain text hittable. |
| 45 | const NOT_CONTROLS = new Set([ |
| 46 | "application", |
| 47 | "text", |
| 48 | "image", |
| 49 | "scroll-area", |
| 50 | "other", |
| 51 | "toolbar", |
| 52 | "tab-bar", |
| 53 | "activity-indicator", |
| 54 | "composeview", |
| 55 | "keyboard", |
| 56 | ]); |
| 57 | |
| 58 | const area = (rect: Rect) => Math.max(0, rect.width) * Math.max(0, rect.height); |
| 59 | |
| 60 | const intersection = (a: Rect, b: Rect) => |
| 61 | Math.max(0, Math.min(a.x + a.width, b.x + b.width) - Math.max(a.x, b.x)) * |
| 62 | Math.max(0, Math.min(a.y + a.height, b.y + b.height) - Math.max(a.y, b.y)); |
| 63 | |
| 64 | export const describeRect = (rect: Rect | undefined) => |
| 65 | rect |
| 66 | ? `${Math.round(rect.x)},${Math.round(rect.y)} ${Math.round(rect.width)}×${Math.round(rect.height)}` |
| 67 | : "absent"; |
| 68 | |
| 69 | const name = (node: SnapshotNode) => node.label || node.identifier || node.type || "unlabelled"; |
| 70 | |
| 71 | const appOwned = (node: SnapshotNode, snapshot: Snapshot) => |
| 72 | !node.systemBar && (node.bundleId === undefined || node.bundleId === snapshot.appBundleId); |
| 73 | |
| 74 | const isControl = (node: SnapshotNode, snapshot: Snapshot) => |
| 75 | node.hittable === true && |
| 76 | node.rect !== undefined && |
| 77 | area(node.rect) > 0 && |
| 78 | !NOT_CONTROLS.has(node.kind ?? "") && |
| 79 | appOwned(node, snapshot); |
| 80 | |
| 81 | function ancestors(node: SnapshotNode, snapshot: Snapshot): SnapshotNode[] { |
| 82 | const byIndex = new Map(snapshot.nodes.map((candidate) => [candidate.index, candidate])); |
| 83 | const chain: SnapshotNode[] = []; |
| 84 | for (let parent = node.parentIndex; parent !== undefined;) { |
| 85 | const next = byIndex.get(parent); |
| 86 | if (!next || chain.includes(next)) break; |
| 87 | chain.push(next); |
| 88 | parent = next.parentIndex; |
| 89 | } |
| 90 | return chain; |
| 91 | } |
| 92 | |
| 93 | const viewportRect = (snapshot: Snapshot): Rect => ({ x: 0, y: 0, ...snapshot.viewport }); |
| 94 | |
| 95 | export function keyboardRect(snapshot: Snapshot): Rect | undefined { |
| 96 | if (snapshot.keyboard?.rect) return snapshot.keyboard.rect; |
| 97 | return snapshot.nodes.find((node) => node.type === "Keyboard" || node.kind === "keyboard")?.rect; |
| 98 | } |
| 99 | |
| 100 | const TAB_LABELS = new Set(["Chats", "Pulls", "Usage", "Settings"]); |
| 101 | |
| 102 | /** The bottom tab bar, as a native tab bar node or the union of its tab buttons. */ |
| 103 | export function tabBarRect(snapshot: Snapshot): Rect | undefined { |
| 104 | const native = snapshot.nodes.find((node) => node.kind === "tab-bar" && node.rect)?.rect; |
| 105 | if (native) return native; |
| 106 | const tabs = snapshot.nodes.filter( |
| 107 | (node) => |
| 108 | node.rect && |
| 109 | TAB_LABELS.has(node.label ?? "") && |
| 110 | node.rect.y > snapshot.viewport.height * 0.7 && |
| 111 | appOwned(node, snapshot), |
| 112 | ); |
| 113 | if (tabs.length < 2) return undefined; |
| 114 | const left = Math.min(...tabs.map((tab) => tab.rect!.x)); |
| 115 | const top = Math.min(...tabs.map((tab) => tab.rect!.y)); |
| 116 | const right = Math.max(...tabs.map((tab) => tab.rect!.x + tab.rect!.width)); |
| 117 | const bottom = Math.max(...tabs.map((tab) => tab.rect!.y + tab.rect!.height)); |
| 118 | return { x: left, y: top, width: right - left, height: bottom - top }; |
| 119 | } |
| 120 | |
| 121 | const ERROR_TEXT = |
| 122 | /\b(Render Error|Uncaught Error|Unhandled JS Exception|Syntax Error|keeps stopping|isn't responding)\b/i; |
| 123 | |
| 124 | /** |
| 125 | * Checks every screen should pass, whatever it shows: the app is in front without an error |
| 126 | * screen, and no control is clipped by the screen edge, stacked on another control, under a |
| 127 | * system bar, or an input under the keyboard. |
| 128 | */ |
| 129 | export function layoutChecks(snapshot: Snapshot, bundleId: string): Check[] { |
| 130 | const viewport = viewportRect(snapshot); |
| 131 | const controls = snapshot.nodes.filter((node) => isControl(node, snapshot)); |
| 132 | const errors = snapshot.nodes.filter((node) => ERROR_TEXT.test(node.label ?? "")); |
| 133 | |
| 134 | const clipped = controls.filter((node) => { |
| 135 | const visible = intersection(node.rect!, viewport); |
| 136 | if (visible === 0 || visible >= area(node.rect!) * 0.98) return false; |
| 137 | // Content scrolled past the edge is expected; fixed chrome cut off by it is not. |
| 138 | return !ancestors(node, snapshot).some((parent) => parent.kind === "scroll-area"); |
| 139 | }); |
| 140 | |
| 141 | // Only siblings: a screen covered by another stays in the tree, so cousins overlap |
| 142 | // legitimately. Native tab items report overlapping frames. |
| 143 | const siblings = controls.filter( |
| 144 | (node) => !ancestors(node, snapshot).some((parent) => parent.kind === "tab-bar"), |
| 145 | ); |
| 146 | const stacked: string[] = []; |
| 147 | for (const [index, a] of siblings.entries()) { |
| 148 | for (const b of siblings.slice(index + 1)) { |
| 149 | if (a.parentIndex !== b.parentIndex) continue; |
| 150 | const shared = intersection(a.rect!, b.rect!); |
| 151 | if (shared < Math.min(area(a.rect!), area(b.rect!)) * 0.25) continue; |
| 152 | // iOS exposes some elements twice, at the same frame. |
| 153 | const same = |
| 154 | describeRect(a.rect) === describeRect(b.rect) && |
| 155 | (!a.label || !b.label || a.label === b.label); |
| 156 | if (!same) stacked.push(`"${name(a)}" and "${name(b)}"`); |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | const bars = snapshot.nodes.filter((node) => node.systemBar && node.rect); |
| 161 | const underBars = controls.filter((node) => |
| 162 | bars.some((bar) => intersection(node.rect!, bar.rect!) > area(node.rect!) * 0.3), |
| 163 | ); |
| 164 | |
| 165 | const keyboard = keyboardRect(snapshot); |
| 166 | const coveredInputs = keyboard |
| 167 | ? snapshot.nodes.filter( |
| 168 | (node) => |
| 169 | (node.kind === "text-field" || node.editable === true) && |
| 170 | node.rect && |
| 171 | appOwned(node, snapshot) && |
| 172 | intersection(node.rect, keyboard) > area(node.rect) * 0.3, |
| 173 | ) |
| 174 | : []; |
| 175 | |
| 176 | const list = (nodes: ReadonlyArray<SnapshotNode>) => |
| 177 | nodes.map((node) => `"${name(node)}" at ${describeRect(node.rect)}`).join(", "); |
| 178 | return [ |
| 179 | check( |
| 180 | "app in front", |
| 181 | snapshot.appBundleId === undefined || snapshot.appBundleId === bundleId, |
| 182 | snapshot.appBundleId ?? "unknown", |
| 183 | ), |
| 184 | check("no error screen", errors.length === 0, errors.map(name).join(", ") || "none"), |
| 185 | check("no control cut off by the screen edge", clipped.length === 0, list(clipped) || "none"), |
| 186 | check("no controls stacked on each other", stacked.length === 0, stacked.join("; ") || "none"), |
| 187 | check("no control under a system bar", underBars.length === 0, list(underBars) || "none"), |
| 188 | check( |
| 189 | "no input under the keyboard", |
| 190 | coveredInputs.length === 0, |
| 191 | keyboard ? list(coveredInputs) || "none" : "no keyboard", |
| 192 | ), |
| 193 | ]; |
| 194 | } |
| 195 | |
| 196 | export const findLabel = (snapshot: Snapshot, pattern: RegExp) => |
| 197 | snapshot.nodes.find( |
| 198 | (node) => |
| 199 | node.rect && |
| 200 | pattern.test(node.label ?? "") && |
| 201 | intersection(node.rect, viewportRect(snapshot)) > 0 && |
| 202 | appOwned(node, snapshot), |
| 203 | ); |