@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 250 lines 8.26 KB
Raw
1 // @effect-diagnostics globalTimers:off -- Accessibility timeouts run outside Effect fibers.
2
3 import {
4 SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
5 type SnapShotAccessibility,
6 } from "@t3tools/contracts";
7 import type * as Electron from "electron";
8
9 import {
10 accessibleWindowElementTree,
11 accessibleWindowText,
12 compactAccessibilityTree,
13 findAccessibleWindow,
14 isWaylandSession,
15 } from "./snapShot.ts";
16
17 const ACCESSIBILITY_TIMEOUT_MS = 3_000;
18
19 export type AccessibleWindowIdentity = {
20 readonly title: string;
21 readonly bounds: Electron.Rectangle;
22 readonly clientBounds?: Electron.Rectangle;
23 readonly owner: { readonly processId: number; readonly bundleId?: string };
24 readonly accessibilityBoundsReliable?: boolean;
25 };
26
27 export type CapturedWindowAccessibilityContext = {
28 readonly accessibleText?: string;
29 readonly accessibility?: SnapShotAccessibility;
30 };
31
32 export type SnapShotAccessibilityRequest = {
33 readonly active: AccessibleWindowIdentity;
34 readonly platform: NodeJS.Platform;
35 readonly sourceTitle: string;
36 readonly imageSize: Electron.Size;
37 };
38
39 type AccessibilityApp = (typeof import("@crowecawcaw/xa11y"))["App"];
40
41 type AccessibilityReadProgress = {
42 accessibleText: string | undefined;
43 flatComplete: boolean;
44 richComplete: boolean;
45 richLocationsReliable: boolean;
46 richRoot?: Extract<SnapShotAccessibility, { format: "element-tree" }>["root"];
47 richTruncated: boolean;
48 timedOut: boolean;
49 };
50
51 function accessibilityReadSnapshot(
52 progress: AccessibilityReadProgress,
53 imageSize: Electron.Size,
54 ): CapturedWindowAccessibilityContext | undefined {
55 const richTree = progress.richRoot
56 ? compactAccessibilityTree(progress.richRoot, {
57 descendantLocationsReliable: progress.richLocationsReliable,
58 })
59 : undefined;
60 const richText = richTree
61 ? accessibleWindowText(richTree.root, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS)
62 : undefined;
63 const accessibleText = progress.accessibleText ?? richText;
64 const richTruncated = progress.richTruncated || richTree?.truncated === true;
65 const accessibility: SnapShotAccessibility | undefined =
66 progress.richComplete &&
67 richTree &&
68 (!richTruncated || !progress.accessibleText || progress.accessibleText === richText)
69 ? {
70 format: "element-tree",
71 coordinateSpace: "captured-image",
72 imageSize,
73 truncated: richTruncated,
74 root: richTree.root,
75 }
76 : progress.flatComplete && progress.accessibleText
77 ? {
78 format: "flat-text",
79 text: progress.accessibleText,
80 truncated: progress.accessibleText.length >= SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS,
81 }
82 : richTree
83 ? {
84 format: "element-tree",
85 coordinateSpace: "captured-image",
86 imageSize,
87 truncated: true,
88 root: richTree.root,
89 }
90 : undefined;
91 if (!accessibleText && !accessibility) return undefined;
92 return JSON.parse(
93 JSON.stringify({
94 ...(accessibleText ? { accessibleText } : {}),
95 ...(accessibility ? { accessibility } : {}),
96 }),
97 ) as CapturedWindowAccessibilityContext;
98 }
99
100 type AccessibilityElement = Awaited<ReturnType<InstanceType<AccessibilityApp>["children"]>>[number];
101
102 async function windowsForPid(
103 App: AccessibilityApp,
104 processId: number,
105 ): Promise<readonly AccessibilityElement[]> {
106 return await App.byPid(processId, { timeout: 0 })
107 .then((app) => app.children())
108 .catch(() => []);
109 }
110
111 /** Flatpak apps reach AT-SPI through xdg-dbus-proxy, so the compositor PID misses. */
112 async function windowsFromAppList(App: AccessibilityApp): Promise<readonly AccessibilityElement[]> {
113 const apps = await App.list().catch(() => []);
114 return (await Promise.all(apps.map((app) => app.children().catch(() => [])))).flat();
115 }
116
117 async function readCapturedWindowAccessibility(
118 App: AccessibilityApp,
119 request: SnapShotAccessibilityRequest,
120 progress: AccessibilityReadProgress,
121 onStarted: () => void,
122 ): Promise<CapturedWindowAccessibilityContext | undefined> {
123 const { active, platform, sourceTitle, imageSize } = request;
124 const foreground = platform === "win32" ? await App.foreground({ timeout: 0 }) : undefined;
125 const pidWindows =
126 foreground !== undefined
127 ? foreground.pid === active.owner.processId
128 ? await foreground.children().catch(() => [])
129 : []
130 : await windowsForPid(App, active.owner.processId);
131 const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";
132 const captured = {
133 title: active.title,
134 sourceTitle,
135 bounds: active.bounds,
136 clientBounds: active.clientBounds,
137 platform,
138 bundleId: pidWindows.length > 0 ? active.owner.bundleId : undefined,
139 };
140 const window = findAccessibleWindow(
141 pidWindows.length > 0 || foreground !== undefined ? pidWindows : await windowsFromAppList(App),
142 captured,
143 matchMode,
144 { allowUntitledUniqueBounds: foreground === undefined && pidWindows.length > 0 },
145 );
146 if (!window) {
147 onStarted();
148 return undefined;
149 }
150 const accessibleBounds = window.bounds;
151 const matchingBounds =
152 matchMode === "wayland" &&
153 active.clientBounds &&
154 accessibleBounds &&
155 Math.abs(accessibleBounds.width - active.clientBounds.width) <= 2 &&
156 Math.abs(accessibleBounds.height - active.clientBounds.height) <= 2
157 ? active.clientBounds
158 : active.bounds;
159 const locationsReliable =
160 active.accessibilityBoundsReliable !== false &&
161 (matchMode === "screen-bounds" ||
162 (accessibleBounds !== null &&
163 Math.abs(accessibleBounds.x - matchingBounds.x) <= 2 &&
164 Math.abs(accessibleBounds.y - matchingBounds.y) <= 2));
165 const flatRead = window
166 .tree()
167 .then((tree) => {
168 progress.accessibleText =
169 accessibleWindowText(tree, SNAP_SHOT_ACCESSIBLE_TEXT_MAX_CHARS) || undefined;
170 progress.flatComplete = true;
171 })
172 .catch(() => {
173 progress.flatComplete = true;
174 });
175 // Preserve the screenshot frame origin and scale so accessibility coordinates include decorations.
176 const sourceBounds =
177 matchMode === "wayland" && active.clientBounds
178 ? active.bounds
179 : (accessibleBounds ?? active.bounds);
180 const richRead = accessibleWindowElementTree(window, sourceBounds, imageSize, {
181 locationsReliable,
182 onProgress: (root, truncated, descendantLocationsReliable) => {
183 progress.richLocationsReliable = descendantLocationsReliable;
184 progress.richRoot = root;
185 progress.richTruncated = truncated;
186 },
187 shouldContinue: () => !progress.timedOut,
188 verifyDescendantLocations: matchMode === "wayland",
189 })
190 .then((rich) => {
191 progress.richComplete = true;
192 if (rich) {
193 progress.richRoot = rich.root;
194 progress.richTruncated = rich.truncated;
195 }
196 })
197 .catch(() => {
198 progress.richComplete = true;
199 });
200 onStarted();
201 await Promise.all([flatRead, richRead]);
202 return accessibilityReadSnapshot(progress, imageSize);
203 }
204
205 let activeAccessibilityRead: Promise<unknown> | undefined;
206
207 async function raceAccessibleRead<T>(
208 run: () => Promise<T>,
209 timeoutValue: () => T | undefined,
210 ): Promise<T | undefined> {
211 if (activeAccessibilityRead) return undefined;
212 const read = run().catch(() => undefined);
213 activeAccessibilityRead = read;
214 void read.finally(() => {
215 if (activeAccessibilityRead === read) activeAccessibilityRead = undefined;
216 });
217 let timeout: ReturnType<typeof setTimeout> | undefined;
218 try {
219 return await Promise.race([
220 read,
221 new Promise<T | undefined>((resolve) => {
222 timeout = setTimeout(() => resolve(timeoutValue()), ACCESSIBILITY_TIMEOUT_MS);
223 }),
224 ]);
225 } finally {
226 if (timeout) clearTimeout(timeout);
227 }
228 }
229
230 export function readAccessibleWindowContextWithApp(
231 App: AccessibilityApp,
232 request: SnapShotAccessibilityRequest,
233 onStarted: () => void = () => undefined,
234 ): Promise<CapturedWindowAccessibilityContext | undefined> {
235 const progress: AccessibilityReadProgress = {
236 accessibleText: undefined,
237 flatComplete: false,
238 richComplete: false,
239 richLocationsReliable: false,
240 richTruncated: false,
241 timedOut: false,
242 };
243 return raceAccessibleRead(
244 () => readCapturedWindowAccessibility(App, request, progress, onStarted),
245 () => {
246 progress.timedOut = true;
247 return accessibilityReadSnapshot(progress, request.imageSize);
248 },
249 );
250 }