The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { withDeviceHubQuery } from "@t3tools/client-runtime/state/deviceHubAccess"; |
| 2 | import type { DeviceHubAccess } from "@t3tools/client-runtime/state/deviceHubAccess"; |
| 3 | import type { DevicePlatform } from "@t3tools/contracts"; |
| 4 | |
| 5 | export interface DeviceAxElement { |
| 6 | readonly id: string; |
| 7 | readonly label: string; |
| 8 | readonly role: string; |
| 9 | /** Normalized to the displayed screen: 0..1 on both axes. */ |
| 10 | readonly x: number; |
| 11 | readonly y: number; |
| 12 | readonly width: number; |
| 13 | readonly height: number; |
| 14 | } |
| 15 | |
| 16 | export interface DeviceAxTree { |
| 17 | readonly elements: ReadonlyArray<DeviceAxElement>; |
| 18 | readonly errors: ReadonlyArray<string>; |
| 19 | } |
| 20 | |
| 21 | export interface DeviceEventLogEntry { |
| 22 | readonly id: number; |
| 23 | readonly timestamp: string; |
| 24 | readonly kind: string; |
| 25 | readonly summary: string; |
| 26 | } |
| 27 | |
| 28 | export interface DeviceForegroundInfo { |
| 29 | readonly id: string; |
| 30 | readonly label?: string; |
| 31 | readonly pid?: number; |
| 32 | readonly isReactNative?: boolean; |
| 33 | } |
| 34 | |
| 35 | export interface DeviceHubTarget { |
| 36 | readonly access: DeviceHubAccess; |
| 37 | readonly platform: DevicePlatform; |
| 38 | readonly deviceId: string; |
| 39 | } |
| 40 | |
| 41 | const vendorBase = (target: DeviceHubTarget) => |
| 42 | `${target.access.httpBase}${target.platform === "ios" ? "/vendor/serve-sim" : "/vendor/serve-emu"}`; |
| 43 | |
| 44 | const hubUrl = (target: DeviceHubTarget, path: string, params?: Record<string, string>) => { |
| 45 | const search = params ? `?${new URLSearchParams(params).toString()}` : ""; |
| 46 | return withDeviceHubQuery(`${vendorBase(target)}${path}${search}`, target.access); |
| 47 | }; |
| 48 | |
| 49 | const fetchJson = async ( |
| 50 | target: DeviceHubTarget, |
| 51 | url: string, |
| 52 | signal?: AbortSignal, |
| 53 | ): Promise<unknown> => { |
| 54 | const response = await fetch(url, { |
| 55 | cache: "no-store", |
| 56 | credentials: target.access.credentials ? "include" : "same-origin", |
| 57 | ...(signal ? { signal } : {}), |
| 58 | }); |
| 59 | if (!response.ok) throw new Error(`${response.status} ${response.statusText}`); |
| 60 | return response.json(); |
| 61 | }; |
| 62 | |
| 63 | const isRecord = (value: unknown): value is Record<string, unknown> => |
| 64 | typeof value === "object" && value !== null; |
| 65 | |
| 66 | const numberOr = (value: unknown, fallback: number) => |
| 67 | typeof value === "number" && Number.isFinite(value) ? value : fallback; |
| 68 | |
| 69 | const AX_ELEMENT_LIMIT = 500; |
| 70 | |
| 71 | const flattenIosAxTree = (roots: ReadonlyArray<unknown>): ReadonlyArray<DeviceAxElement> => { |
| 72 | const first = roots[0]; |
| 73 | const rootFrame = isRecord(first) && isRecord(first.frame) ? first.frame : null; |
| 74 | const screenWidth = Math.max(1, numberOr(rootFrame?.width, 1)); |
| 75 | const screenHeight = Math.max(1, numberOr(rootFrame?.height, 1)); |
| 76 | const elements: DeviceAxElement[] = []; |
| 77 | const visit = (node: unknown, path: string) => { |
| 78 | if (elements.length >= AX_ELEMENT_LIMIT || !isRecord(node) || !isRecord(node.frame)) return; |
| 79 | const frame = node.frame; |
| 80 | const width = numberOr(frame.width, 0); |
| 81 | const height = numberOr(frame.height, 0); |
| 82 | const coversScreen = |
| 83 | Math.abs(width - screenWidth) < 0.5 && Math.abs(height - screenHeight) < 0.5; |
| 84 | if (!coversScreen && width > 0 && height > 0) { |
| 85 | elements.push({ |
| 86 | id: typeof node.AXUniqueId === "string" ? node.AXUniqueId : path, |
| 87 | label: typeof node.AXLabel === "string" ? node.AXLabel : "", |
| 88 | role: typeof node.type === "string" ? node.type : "", |
| 89 | x: numberOr(frame.x, 0) / screenWidth, |
| 90 | y: numberOr(frame.y, 0) / screenHeight, |
| 91 | width: width / screenWidth, |
| 92 | height: height / screenHeight, |
| 93 | }); |
| 94 | } |
| 95 | const children = Array.isArray(node.children) ? node.children : []; |
| 96 | children.forEach((child, index) => visit(child, `${path}.${index}`)); |
| 97 | }; |
| 98 | roots.forEach((root, index) => visit(root, String(index))); |
| 99 | return elements; |
| 100 | }; |
| 101 | |
| 102 | export async function fetchDeviceAxTree( |
| 103 | target: DeviceHubTarget, |
| 104 | signal?: AbortSignal, |
| 105 | ): Promise<DeviceAxTree> { |
| 106 | if (target.platform === "ios") { |
| 107 | const payload = await fetchJson( |
| 108 | target, |
| 109 | hubUrl(target, `/helper/${encodeURIComponent(target.deviceId)}/ax`), |
| 110 | signal, |
| 111 | ); |
| 112 | if (!Array.isArray(payload)) { |
| 113 | const error = isRecord(payload) && typeof payload.error === "string" ? payload.error : null; |
| 114 | return { elements: [], errors: [error ?? "Unexpected accessibility payload."] }; |
| 115 | } |
| 116 | return { elements: flattenIosAxTree(payload), errors: [] }; |
| 117 | } |
| 118 | const payload = await fetchJson( |
| 119 | target, |
| 120 | hubUrl(target, "/api/accessibility", { device: target.deviceId }), |
| 121 | signal, |
| 122 | ); |
| 123 | if (!isRecord(payload) || !Array.isArray(payload.nodes)) { |
| 124 | const error = isRecord(payload) && typeof payload.error === "string" ? payload.error : null; |
| 125 | return { elements: [], errors: [error ?? "Unexpected accessibility payload."] }; |
| 126 | } |
| 127 | const nodes = payload.nodes.filter( |
| 128 | (node): node is Record<string, unknown> => isRecord(node) && isRecord(node.bounds), |
| 129 | ); |
| 130 | const root = nodes[0]?.bounds as Record<string, unknown> | undefined; |
| 131 | const screenWidth = Math.max(1, numberOr(root?.right, 1)); |
| 132 | const screenHeight = Math.max(1, numberOr(root?.bottom, 1)); |
| 133 | const elements = nodes.slice(1).flatMap((node): DeviceAxElement[] => { |
| 134 | const bounds = node.bounds as Record<string, unknown>; |
| 135 | const left = numberOr(bounds.left, 0); |
| 136 | const top = numberOr(bounds.top, 0); |
| 137 | const width = (numberOr(bounds.right, left) - left) / screenWidth; |
| 138 | const height = (numberOr(bounds.bottom, top) - top) / screenHeight; |
| 139 | const text = typeof node.text === "string" ? node.text : ""; |
| 140 | const description = typeof node.contentDescription === "string" ? node.contentDescription : ""; |
| 141 | const label = text || description; |
| 142 | if (width >= 0.95 && height >= 0.9) return []; |
| 143 | if (!label && node.clickable !== true) return []; |
| 144 | const className = typeof node.className === "string" ? node.className : ""; |
| 145 | return [ |
| 146 | { |
| 147 | id: String(node.id ?? ""), |
| 148 | label, |
| 149 | role: className.split(".").at(-1) ?? "", |
| 150 | x: left / screenWidth, |
| 151 | y: top / screenHeight, |
| 152 | width, |
| 153 | height, |
| 154 | }, |
| 155 | ]; |
| 156 | }); |
| 157 | return { elements, errors: [] }; |
| 158 | } |
| 159 | |
| 160 | const openEventSource = ( |
| 161 | target: DeviceHubTarget, |
| 162 | url: string, |
| 163 | onMessage: (data: unknown) => void, |
| 164 | ): (() => void) => { |
| 165 | const source = new EventSource(url, { withCredentials: target.access.credentials }); |
| 166 | source.addEventListener("message", (event) => { |
| 167 | try { |
| 168 | onMessage(JSON.parse(String(event.data))); |
| 169 | } catch {} |
| 170 | }); |
| 171 | return () => source.close(); |
| 172 | }; |
| 173 | |
| 174 | export function subscribeDeviceForeground( |
| 175 | target: DeviceHubTarget, |
| 176 | onChange: (app: DeviceForegroundInfo | null) => void, |
| 177 | ): () => void { |
| 178 | if (target.platform !== "ios") return () => {}; |
| 179 | return openEventSource( |
| 180 | target, |
| 181 | hubUrl(target, "/appstate", { device: target.deviceId }), |
| 182 | (data) => { |
| 183 | if (!isRecord(data)) return; |
| 184 | if (data.bundleId === null || data.bundleId === "") { |
| 185 | onChange(null); |
| 186 | return; |
| 187 | } |
| 188 | if (typeof data.bundleId !== "string") return; |
| 189 | onChange({ |
| 190 | id: data.bundleId, |
| 191 | ...(typeof data.pid === "number" ? { pid: data.pid } : {}), |
| 192 | ...(typeof data.isReactNative === "boolean" ? { isReactNative: data.isReactNative } : {}), |
| 193 | }); |
| 194 | }, |
| 195 | ); |
| 196 | } |
| 197 | |
| 198 | const toEventLogEntry = (raw: unknown): DeviceEventLogEntry | null => { |
| 199 | if (!isRecord(raw) || typeof raw.id !== "number") return null; |
| 200 | return { |
| 201 | id: raw.id, |
| 202 | timestamp: typeof raw.timestamp === "string" ? raw.timestamp : "", |
| 203 | kind: typeof raw.kind === "string" ? raw.kind : "", |
| 204 | summary: |
| 205 | typeof raw.summary === "string" ? raw.summary : typeof raw.msg === "string" ? raw.msg : "", |
| 206 | }; |
| 207 | }; |
| 208 | |
| 209 | export const DEVICE_EVENT_LOG_LIMIT = 100; |
| 210 | |
| 211 | /** A reset batch replaces the log; the log keeps the newest entries up to the limit. */ |
| 212 | export function appendDeviceEventLog( |
| 213 | current: ReadonlyArray<DeviceEventLogEntry>, |
| 214 | incoming: ReadonlyArray<DeviceEventLogEntry>, |
| 215 | reset: boolean, |
| 216 | ): ReadonlyArray<DeviceEventLogEntry> { |
| 217 | const merged = reset ? [...incoming] : [...current, ...incoming]; |
| 218 | return merged.length > DEVICE_EVENT_LOG_LIMIT ? merged.slice(-DEVICE_EVENT_LOG_LIMIT) : merged; |
| 219 | } |
| 220 | |
| 221 | export function subscribeDeviceEventLog( |
| 222 | target: DeviceHubTarget, |
| 223 | onEvents: (entries: ReadonlyArray<DeviceEventLogEntry>, reset: boolean) => void, |
| 224 | ): () => void { |
| 225 | if (target.platform !== "ios") return () => {}; |
| 226 | return openEventSource( |
| 227 | target, |
| 228 | hubUrl(target, "/api/event-log/events", { device: target.deviceId, limit: "100" }), |
| 229 | (data) => { |
| 230 | if (!isRecord(data)) return; |
| 231 | if (Array.isArray(data.events)) { |
| 232 | onEvents( |
| 233 | data.events.flatMap((raw) => toEventLogEntry(raw) ?? []), |
| 234 | true, |
| 235 | ); |
| 236 | return; |
| 237 | } |
| 238 | const entry = toEventLogEntry(data.event); |
| 239 | if (entry) onEvents([entry], false); |
| 240 | }, |
| 241 | ); |
| 242 | } |