@devries-cameron20 / t3code Mirror · read-only / commits / c4ce7b0fd7
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 02:00 UTC

fix(desktop): update xa11y to 0.15.4 so App Shots read window text faster (#33135)

Matthew Feroz committed Nov 9, 2026 at 00:42 UTC c4ce7b0fd74f6beb37ed6034f5c8416e5aa7e165
9 files changed +75 -64
apps/desktop/package.json
+1 -1
@@ -12,7 +12,7 @@
12 "dependencies": {
13 "@clerk/electron": "catalog:",
14 "@clerk/electron-passkeys": "catalog:",
15 - "@crowecawcaw/xa11y": "0.13.0",
15 + "@crowecawcaw/xa11y": "0.15.4",
16 "@effect/platform-node": "catalog:",
17 "@napi-rs/keyring": "^1.3.0",
18 "@t3tools/client-runtime": "workspace:*",
apps/desktop/src/electron/ElectronWindow.test.ts
+16
@@ -53,6 +53,15 @@ vi.mock("electron", () => ({
53 getAllWindows: getAllWindowsMock,
54 getFocusedWindow: getFocusedWindowMock,
55 }),
56 + screen: {
57 + // A 200% display, so the focus target must carry converted bounds.
58 + dipToScreenRect: (_window: unknown, rect: Electron.Rectangle) => ({
59 + x: rect.x * 2,
60 + y: rect.y * 2,
61 + width: rect.width * 2,
62 + height: rect.height * 2,
63 + }),
64 + },
65 }));
66
67 import * as ElectronWindow from "./ElectronWindow.ts";
@@ -116,6 +125,13 @@ describe("ElectronWindow", () => {
125 });
126 const electronWindow = yield* ElectronWindow.ElectronWindow;
127 yield* electronWindow.prepareReveal(window as unknown as Electron.BrowserWindow);
128 + assert.deepEqual(windowsForegroundPrepareMock.mock.calls[0]?.[0], {
129 + windowId: 41,
130 + processId: process.pid,
131 + title: "T3 Code (Dev)",
132 + bounds: { x: 200, y: 100, width: 2_400, height: 1_600 },
133 + contentBounds: { x: 216, y: 100, width: 2_368, height: 1_584 },
134 + });
135 window.getTitle.mockClear();
136
137 const revealFiber = yield* electronWindow
apps/desktop/src/electron/ElectronWindow.ts
+3 -2
@@ -26,8 +26,9 @@ function windowsForegroundFocusTarget(window: Electron.BrowserWindow) {
26 windowId: window.id,
27 processId: process.pid,
28 title: window.getTitle(),
29 - bounds: window.getBounds(),
30 - contentBounds: window.getContentBounds(),
29 + // xa11y reports physical desktop pixels on Windows; Electron bounds are DIPs.
30 + bounds: Electron.screen.dipToScreenRect(window, window.getBounds()),
31 + contentBounds: Electron.screen.dipToScreenRect(window, window.getContentBounds()),
32 };
33 }
34
apps/desktop/src/electron/WindowsForegroundFocusWorker.ts
+1 -7
@@ -32,13 +32,7 @@ async function findTarget(
32 const children = await App.byPid(target.processId, { timeout: 0 })
33 .then((app) => app.children())
34 .catch(() => []);
35 - return (
36 - children.find((candidate) => matchesTarget(candidate, target)) ??
37 - (await App.list())
38 - .filter((candidate) => candidate.pid === target.processId)
39 - .map((candidate) => candidate.asElement())
40 - .find((candidate) => matchesTarget(candidate, target))
41 - );
35 + return children.find((candidate) => matchesTarget(candidate, target));
36 }
37
38 const cachedElements = new Map<number, Element>();
apps/desktop/src/snapShot/DesktopSnapShot.test.ts
+21 -20
@@ -978,7 +978,7 @@ it.effect.each(["succeeds", "fails"] as const)(
978 },
979 );
980
981 -it.effect("matches Windows accessibility windows on a scaled display", () => {
981 +it.effect("captures and reads Windows windows in physical pixels on a scaled display", () => {
982 const png = Buffer.from([1, 2, 3]);
983 const active = {
984 platform: "windows",
@@ -987,26 +987,26 @@ it.effect("matches Windows accessibility windows on a scaled display", () => {
987 owner: { name: "Editor", processId: 123 },
988 bounds: { x: 10, y: 20, width: 800, height: 600 },
989 } as const;
990 - const dipBounds = { x: 5, y: 10, width: 400, height: 300 };
990 + const window = (name: string, value: string) => ({
991 + role: "window",
992 + name,
993 + bounds: active.bounds,
994 + children: async () => [
995 + {
996 + role: "button",
997 + name: "Save",
998 + bounds: { x: 210, y: 220, width: 200, height: 100 },
999 + children: async () => [],
1000 + },
1001 + ],
1002 + tree: async () => ({ name, value, children: [] }),
1003 + });
1004 activeWindowMock.mockReset().mockResolvedValue(active);
992 - screenToDipRectMock.mockImplementation(() => dipBounds);
1005 + screenToDipRectMock.mockImplementation(() => ({ x: 5, y: 10, width: 400, height: 300 }));
1006 regionCaptureMock.mockReset().mockResolvedValue({ width: 800, height: 600, png });
1007 accessibilityForegroundMock.mockReset().mockResolvedValue({
1008 pid: 123,
996 - asElement: () => ({
997 - role: "window",
998 - name: "Editor",
999 - bounds: dipBounds,
1000 - children: async () => [
1001 - {
1002 - role: "button",
1003 - name: "Save",
1004 - bounds: { x: 105, y: 110, width: 100, height: 50 },
1005 - children: async () => [],
1006 - },
1007 - ],
1008 - tree: async () => ({ name: "Editor", value: "Scaled text", children: [] }),
1009 - }),
1009 + children: async () => [window("Settings", "Wrong window"), window("Editor", "Scaled text")],
1010 });
1011 let metadata = "";
1012 return Effect.scoped(
@@ -1015,6 +1015,7 @@ it.effect("matches Windows accessibility windows on a scaled display", () => {
1015 yield* service.configure(enabledSettings());
1016 yield* service.capture;
1017 const saved = yield* decodePendingMetadata(metadata);
1018 + assert.deepEqual(regionCaptureMock.mock.calls[0]?.[0], active.bounds);
1019 assert.include(saved.source.accessibleText, "Scaled text");
1020 assert.deepEqual(
1021 saved.source.accessibility?.format === "element-tree"
@@ -1402,7 +1403,7 @@ it.each(["darwin", "win32", "linux"] as const)(
1403 accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
1404 accessibilityForegroundMock
1405 .mockReset()
1405 - .mockResolvedValue({ pid: 123, asElement: () => window });
1406 + .mockResolvedValue({ pid: 123, children: async () => [window] });
1407
1408 const result = await readAccessibleWindowContext(
1409 { title: "Editor", bounds, owner: { processId: 123 } },
@@ -1442,7 +1443,7 @@ it.each(["darwin", "win32"] as const)(
1443 accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
1444 accessibilityForegroundMock
1445 .mockReset()
1445 - .mockResolvedValue({ pid: 123, asElement: () => window });
1446 + .mockResolvedValue({ pid: 123, children: async () => [window] });
1447 try {
1448 assert.isUndefined(
1449 await readAccessibleWindowText(
@@ -1484,7 +1485,7 @@ it.each([
1485 accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
1486 accessibilityForegroundMock
1487 .mockReset()
1487 - .mockResolvedValue({ pid: 123, asElement: () => window });
1488 + .mockResolvedValue({ pid: 123, children: async () => [window] });
1489 try {
1490 const result = await readAccessibleWindowContext(
1491 { title: "Editor", bounds, owner: { processId: 123 } },
apps/desktop/src/snapShot/DesktopSnapShot.ts
+1 -2
@@ -508,7 +508,6 @@ async function captureSource({
508 ({ source, png } = await captureRegionWindowSnapshot(
509 regionSnapShotPool,
510 active,
511 - snapShotFlashBounds(active, platform),
511 snapShotThumbnailSize(active),
512 ));
513 } else {
@@ -547,7 +546,7 @@ async function captureSource({
546 }
547 }
548 const accessibleIdentity = active
550 - ? { ...active, bounds: snapShotFlashBounds(active, platform) }
549 + ? active
550 : linuxWindow?.processId
551 ? {
552 title: linuxWindow.title,
apps/desktop/src/snapShot/RegionSnapShot.ts
+1 -2
@@ -175,10 +175,9 @@ export function makeRegionSnapShotPool(
175 export async function captureRegionWindowSnapshot(
176 pool: Pick<RegionSnapShotPool, "capture">,
177 active: ActiveWindow,
178 - region: Electron.Rectangle,
178 maxSize: Electron.Size,
179 ): Promise<{ readonly source: RegionSnapShotSource; readonly png: Buffer }> {
181 - const shot = await pool.capture(region);
180 + const shot = await pool.capture(active.bounds);
181 const scale = Math.min(maxSize.width / shot.width, maxSize.height / shot.height, 1);
182 const png =
183 scale < 1
apps/desktop/src/snapShot/SnapShotAccessibility.ts
+1 -1
@@ -125,7 +125,7 @@ async function readCapturedWindowAccessibility(
125 const pidWindows =
126 foreground !== undefined
127 ? foreground.pid === active.owner.processId
128 - ? [foreground.asElement()]
128 + ? await foreground.children().catch(() => [])
129 : []
130 : await windowsForPid(App, active.owner.processId);
131 const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";
pnpm-lock.yaml
+30 -29
@@ -426,8 +426,8 @@ importers:
426 specifier: 0.0.3
427 version: 0.0.3
428 '@crowecawcaw/xa11y':
429 - specifier: 0.13.0
430 - version: 0.13.0
429 + specifier: 0.15.4
430 + version: 0.15.4
431 '@effect/platform-node':
432 specifier: 4.1.2
433 version: 4.1.2(bufferutil@4.1.0)(effect@4.1.2)(redis@6.2.1)(utf-8-validate@6.0.6)
@@ -3488,47 +3488,48 @@ packages:
3488 peerDependencies:
3489 '@bufbuild/protobuf': ^1.10.0
3490
3491 - '@crowecawcaw/xa11y-darwin-arm64@0.13.0':
3492 - resolution: {integrity: sha512-wT+f9lbE6IfSPs8unb7IHr6/V/EkB4SCMFisTtV3jsTseCKr4/vkgpI1LRdul2r+KrmgQ4JxRpUDVB25dmpqNw==}
3491 + '@crowecawcaw/xa11y-darwin-arm64@0.15.4':
3492 + resolution: {integrity: sha512-/8I+o3JR2tY8thwH/lEcmIuyTxHY8rLW422UFFcP0jI4XRD4+kx5fSjSGV2QIUaQ9tlNsyZkPS0cTYX4oF9GKw==}
3493 engines: {node: '>=18'}
3494 cpu: [arm64]
3495 os: [darwin]
3496
3497 - '@crowecawcaw/xa11y-darwin-x64@0.13.0':
3498 - resolution: {integrity: sha512-8QtLjTzciLIFOTWX1wZRTh6lg+Cw35Rc0VcDtJspS3baHiqkTZD6UyeZKhu5awSirbYZWLeXTuyY2CHCfaQ9rw==}
3497 + '@crowecawcaw/xa11y-darwin-x64@0.15.4':
3498 + resolution: {integrity: sha512-sCEfQrpVXyDwacL2PqsUiLUKUN2lhiBZ8dwW8R1/NuLYJTygGVz0UI1Tuk/td/i6RPupIHAgBgjwNo9Ci1KU0w==}
3499 engines: {node: '>=18'}
3500 cpu: [x64]
3501 os: [darwin]
3502
3503 - '@crowecawcaw/xa11y-linux-arm64-gnu@0.13.0':
3504 - resolution: {integrity: sha512-Yad7Bo25pqyowd92yLxGqr59ASSf7zr6lhzoufeCJU62eOq96jEQhhQw0TlvRU8aRI1ksgSuiUivoxNjYP2/gg==}
3503 + '@crowecawcaw/xa11y-linux-arm64-gnu@0.15.4':
3504 + resolution: {integrity: sha512-v0l2K3vqY4t9a8dK5j2EFZkjOknUv1j89zN++uuYLT5qR4TBbQFJs9ZcqxzLPwO1SqMXlO+LD4W+Ssx5JyQRjg==}
3505 engines: {node: '>=18'}
3506 cpu: [arm64]
3507 os: [linux]
3508 libc: [glibc]
3509
3510 - '@crowecawcaw/xa11y-linux-x64-gnu@0.13.0':
3511 - resolution: {integrity: sha512-h/7UoAr2X8iSHaBsNFt7uW1Ft14zgintTACNYMTX8WFwE+WXdHe3vtJmxa/bG+N8imWSGH1Xkzg52YJaa/C+bw==}
3510 + '@crowecawcaw/xa11y-linux-x64-gnu@0.15.4':
3511 + resolution: {integrity: sha512-ajyJr5J+4YyAtdca7Gdeus18boAaRyNL6JTkk9QG48YXCgLx16y2dOvT0PvpbIPsc31VAH5RcvoHpnRW7E3sJQ==}
3512 engines: {node: '>=18'}
3513 cpu: [x64]
3514 os: [linux]
3515 libc: [glibc]
3516
3517 - '@crowecawcaw/xa11y-win32-arm64-msvc@0.13.0':
3518 - resolution: {integrity: sha512-3vmwZxNg7s6qXdSmYve3EE/zh7vZx75jUxi29odguy9llUvWpWFguvvXJ8EFYh13+TwxchibxnImlZb1XpKG+g==}
3517 + '@crowecawcaw/xa11y-win32-arm64-msvc@0.15.4':
3518 + resolution: {integrity: sha512-RfF0CIHXruhg3mQB3xPpItUgiB+Rh9RdHQ9fMLBd2wtCt8V9jtdcOu37EMrPDKt5ClpAxUr0/kxJrhfrX8ZzSQ==}
3519 engines: {node: '>=18'}
3520 cpu: [arm64]
3521 os: [win32]
3522
3523 - '@crowecawcaw/xa11y-win32-x64-msvc@0.13.0':
3524 - resolution: {integrity: sha512-7kWYSj6AZKIN89EirGJQTkeaCJrElwFXOp9eQKd55vUFGZw2YARGY5HfpXha2BKGgZgT2IRR4CJrmgz2Nb1ETw==}
3523 + '@crowecawcaw/xa11y-win32-x64-msvc@0.15.4':
3524 + resolution: {integrity: sha512-fnx8+QGU76m+F6D3Va+fMg1jq/bOpvTU6tbnfF0pCg93wvsuJ0+xO/MJlFnCZVeOagPm0+wDBVBLUC1cvQOAmw==}
3525 engines: {node: '>=18'}
3526 cpu: [x64]
3527 os: [win32]
3528
3529 - '@crowecawcaw/xa11y@0.13.0':
3530 - resolution: {integrity: sha512-WTYHwlBhImGmmMt81crxaRvNdZXlBNemFyrP5GGHj2WeDTJMBrHgHTaladTypXHXrviFFBBGROl5moijRQ4ODA==}
3529 + '@crowecawcaw/xa11y@0.15.4':
3530 + resolution: {integrity: sha512-aDGW67yj0OPGc8Oo/IJZpZy/252D+qi9rcrlT8T+mXLjsn5+58Ha6cq921O58Yq6aGvNikXBMbap0J6/fM2Mlg==}
3531 engines: {node: '>=18'}
3532 + hasBin: true
3533
3534 '@csstools/color-helpers@6.1.1':
3535 resolution: {integrity: sha512-gLNsunvwf3mCi5u5o46/Z/JcJMnhbHSaZ69rkgPzNM3J4s8hWwpPUQB6/tt0EDFyCiWzxANlx+2LJwpYj4zS1w==}
@@ -13498,32 +13499,32 @@ snapshots:
13499 dependencies:
13500 '@bufbuild/protobuf': 1.10.0
13501
13501 - '@crowecawcaw/xa11y-darwin-arm64@0.13.0':
13502 + '@crowecawcaw/xa11y-darwin-arm64@0.15.4':
13503 optional: true
13504
13504 - '@crowecawcaw/xa11y-darwin-x64@0.13.0':
13505 + '@crowecawcaw/xa11y-darwin-x64@0.15.4':
13506 optional: true
13507
13507 - '@crowecawcaw/xa11y-linux-arm64-gnu@0.13.0':
13508 + '@crowecawcaw/xa11y-linux-arm64-gnu@0.15.4':
13509 optional: true
13510
13510 - '@crowecawcaw/xa11y-linux-x64-gnu@0.13.0':
13511 + '@crowecawcaw/xa11y-linux-x64-gnu@0.15.4':
13512 optional: true
13513
13513 - '@crowecawcaw/xa11y-win32-arm64-msvc@0.13.0':
13514 + '@crowecawcaw/xa11y-win32-arm64-msvc@0.15.4':
13515 optional: true
13516
13516 - '@crowecawcaw/xa11y-win32-x64-msvc@0.13.0':
13517 + '@crowecawcaw/xa11y-win32-x64-msvc@0.15.4':
13518 optional: true
13519
13519 - '@crowecawcaw/xa11y@0.13.0':
13520 + '@crowecawcaw/xa11y@0.15.4':
13521 optionalDependencies:
13521 - '@crowecawcaw/xa11y-darwin-arm64': 0.13.0
13522 - '@crowecawcaw/xa11y-darwin-x64': 0.13.0
13523 - '@crowecawcaw/xa11y-linux-arm64-gnu': 0.13.0
13524 - '@crowecawcaw/xa11y-linux-x64-gnu': 0.13.0
13525 - '@crowecawcaw/xa11y-win32-arm64-msvc': 0.13.0
13526 - '@crowecawcaw/xa11y-win32-x64-msvc': 0.13.0
13522 + '@crowecawcaw/xa11y-darwin-arm64': 0.15.4
13523 + '@crowecawcaw/xa11y-darwin-x64': 0.15.4
13524 + '@crowecawcaw/xa11y-linux-arm64-gnu': 0.15.4
13525 + '@crowecawcaw/xa11y-linux-x64-gnu': 0.15.4
13526 + '@crowecawcaw/xa11y-win32-arm64-msvc': 0.15.4
13527 + '@crowecawcaw/xa11y-win32-x64-msvc': 0.15.4
13528
13529 '@csstools/color-helpers@6.1.1': {}
13530