The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 02:00 UTC
fix(desktop): update xa11y to 0.15.4 so App Shots read window text faster (#33135)
Matthew Feroz committed
Nov 9, 2026 at 00:42 UTC
c4ce7b0fd74f6beb37ed6034f5c8416e5aa7e165
9 files changed
+75
-64
apps/desktop/package.json
+1
-1
@@ -12,7 +12,7 @@
12
"dependencies": {
13
"@clerk/electron": "catalog:",
14
"@clerk/electron-passkeys": "catalog:",
15
- "@crowecawcaw/xa11y": "0.13.0",
15
+ "@crowecawcaw/xa11y": "0.15.4",
16
"@effect/platform-node": "catalog:",
17
"@napi-rs/keyring": "^1.3.0",
18
"@t3tools/client-runtime": "workspace:*",
apps/desktop/src/electron/ElectronWindow.test.ts
+16
@@ -53,6 +53,15 @@ vi.mock("electron", () => ({
53
getAllWindows: getAllWindowsMock,
54
getFocusedWindow: getFocusedWindowMock,
55
}),
56
+ screen: {
57
+ // A 200% display, so the focus target must carry converted bounds.
58
+ dipToScreenRect: (_window: unknown, rect: Electron.Rectangle) => ({
59
+ x: rect.x * 2,
60
+ y: rect.y * 2,
61
+ width: rect.width * 2,
62
+ height: rect.height * 2,
63
+ }),
64
+ },
65
}));
66
67
import * as ElectronWindow from "./ElectronWindow.ts";
@@ -116,6 +125,13 @@ describe("ElectronWindow", () => {
125
});
126
const electronWindow = yield* ElectronWindow.ElectronWindow;
127
yield* electronWindow.prepareReveal(window as unknown as Electron.BrowserWindow);
128
+ assert.deepEqual(windowsForegroundPrepareMock.mock.calls[0]?.[0], {
129
+ windowId: 41,
130
+ processId: process.pid,
131
+ title: "T3 Code (Dev)",
132
+ bounds: { x: 200, y: 100, width: 2_400, height: 1_600 },
133
+ contentBounds: { x: 216, y: 100, width: 2_368, height: 1_584 },
134
+ });
135
window.getTitle.mockClear();
136
137
const revealFiber = yield* electronWindow
apps/desktop/src/electron/ElectronWindow.ts
+3
-2
@@ -26,8 +26,9 @@ function windowsForegroundFocusTarget(window: Electron.BrowserWindow) {
26
windowId: window.id,
27
processId: process.pid,
28
title: window.getTitle(),
29
- bounds: window.getBounds(),
30
- contentBounds: window.getContentBounds(),
29
+ // xa11y reports physical desktop pixels on Windows; Electron bounds are DIPs.
30
+ bounds: Electron.screen.dipToScreenRect(window, window.getBounds()),
31
+ contentBounds: Electron.screen.dipToScreenRect(window, window.getContentBounds()),
32
};
33
}
34
apps/desktop/src/electron/WindowsForegroundFocusWorker.ts
+1
-7
@@ -32,13 +32,7 @@ async function findTarget(
32
const children = await App.byPid(target.processId, { timeout: 0 })
33
.then((app) => app.children())
34
.catch(() => []);
35
- return (
36
- children.find((candidate) => matchesTarget(candidate, target)) ??
37
- (await App.list())
38
- .filter((candidate) => candidate.pid === target.processId)
39
- .map((candidate) => candidate.asElement())
40
- .find((candidate) => matchesTarget(candidate, target))
41
- );
35
+ return children.find((candidate) => matchesTarget(candidate, target));
36
}
37
38
const cachedElements = new Map<number, Element>();
apps/desktop/src/snapShot/DesktopSnapShot.test.ts
+21
-20
@@ -978,7 +978,7 @@ it.effect.each(["succeeds", "fails"] as const)(
978
},
979
);
980
981
-it.effect("matches Windows accessibility windows on a scaled display", () => {
981
+it.effect("captures and reads Windows windows in physical pixels on a scaled display", () => {
982
const png = Buffer.from([1, 2, 3]);
983
const active = {
984
platform: "windows",
@@ -987,26 +987,26 @@ it.effect("matches Windows accessibility windows on a scaled display", () => {
987
owner: { name: "Editor", processId: 123 },
988
bounds: { x: 10, y: 20, width: 800, height: 600 },
989
} as const;
990
- const dipBounds = { x: 5, y: 10, width: 400, height: 300 };
990
+ const window = (name: string, value: string) => ({
991
+ role: "window",
992
+ name,
993
+ bounds: active.bounds,
994
+ children: async () => [
995
+ {
996
+ role: "button",
997
+ name: "Save",
998
+ bounds: { x: 210, y: 220, width: 200, height: 100 },
999
+ children: async () => [],
1000
+ },
1001
+ ],
1002
+ tree: async () => ({ name, value, children: [] }),
1003
+ });
1004
activeWindowMock.mockReset().mockResolvedValue(active);
992
- screenToDipRectMock.mockImplementation(() => dipBounds);
1005
+ screenToDipRectMock.mockImplementation(() => ({ x: 5, y: 10, width: 400, height: 300 }));
1006
regionCaptureMock.mockReset().mockResolvedValue({ width: 800, height: 600, png });
1007
accessibilityForegroundMock.mockReset().mockResolvedValue({
1008
pid: 123,
996
- asElement: () => ({
997
- role: "window",
998
- name: "Editor",
999
- bounds: dipBounds,
1000
- children: async () => [
1001
- {
1002
- role: "button",
1003
- name: "Save",
1004
- bounds: { x: 105, y: 110, width: 100, height: 50 },
1005
- children: async () => [],
1006
- },
1007
- ],
1008
- tree: async () => ({ name: "Editor", value: "Scaled text", children: [] }),
1009
- }),
1009
+ children: async () => [window("Settings", "Wrong window"), window("Editor", "Scaled text")],
1010
});
1011
let metadata = "";
1012
return Effect.scoped(
@@ -1015,6 +1015,7 @@ it.effect("matches Windows accessibility windows on a scaled display", () => {
1015
yield* service.configure(enabledSettings());
1016
yield* service.capture;
1017
const saved = yield* decodePendingMetadata(metadata);
1018
+ assert.deepEqual(regionCaptureMock.mock.calls[0]?.[0], active.bounds);
1019
assert.include(saved.source.accessibleText, "Scaled text");
1020
assert.deepEqual(
1021
saved.source.accessibility?.format === "element-tree"
@@ -1402,7 +1403,7 @@ it.each(["darwin", "win32", "linux"] as const)(
1403
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
1404
accessibilityForegroundMock
1405
.mockReset()
1405
- .mockResolvedValue({ pid: 123, asElement: () => window });
1406
+ .mockResolvedValue({ pid: 123, children: async () => [window] });
1407
1408
const result = await readAccessibleWindowContext(
1409
{ title: "Editor", bounds, owner: { processId: 123 } },
@@ -1442,7 +1443,7 @@ it.each(["darwin", "win32"] as const)(
1443
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
1444
accessibilityForegroundMock
1445
.mockReset()
1445
- .mockResolvedValue({ pid: 123, asElement: () => window });
1446
+ .mockResolvedValue({ pid: 123, children: async () => [window] });
1447
try {
1448
assert.isUndefined(
1449
await readAccessibleWindowText(
@@ -1484,7 +1485,7 @@ it.each([
1485
accessibilityByPidMock.mockReset().mockResolvedValue({ children: async () => [window] });
1486
accessibilityForegroundMock
1487
.mockReset()
1487
- .mockResolvedValue({ pid: 123, asElement: () => window });
1488
+ .mockResolvedValue({ pid: 123, children: async () => [window] });
1489
try {
1490
const result = await readAccessibleWindowContext(
1491
{ title: "Editor", bounds, owner: { processId: 123 } },
apps/desktop/src/snapShot/DesktopSnapShot.ts
+1
-2
@@ -508,7 +508,6 @@ async function captureSource({
508
({ source, png } = await captureRegionWindowSnapshot(
509
regionSnapShotPool,
510
active,
511
- snapShotFlashBounds(active, platform),
511
snapShotThumbnailSize(active),
512
));
513
} else {
@@ -547,7 +546,7 @@ async function captureSource({
546
}
547
}
548
const accessibleIdentity = active
550
- ? { ...active, bounds: snapShotFlashBounds(active, platform) }
549
+ ? active
550
: linuxWindow?.processId
551
? {
552
title: linuxWindow.title,
apps/desktop/src/snapShot/RegionSnapShot.ts
+1
-2
@@ -175,10 +175,9 @@ export function makeRegionSnapShotPool(
175
export async function captureRegionWindowSnapshot(
176
pool: Pick<RegionSnapShotPool, "capture">,
177
active: ActiveWindow,
178
- region: Electron.Rectangle,
178
maxSize: Electron.Size,
179
): Promise<{ readonly source: RegionSnapShotSource; readonly png: Buffer }> {
181
- const shot = await pool.capture(region);
180
+ const shot = await pool.capture(active.bounds);
181
const scale = Math.min(maxSize.width / shot.width, maxSize.height / shot.height, 1);
182
const png =
183
scale < 1
apps/desktop/src/snapShot/SnapShotAccessibility.ts
+1
-1
@@ -125,7 +125,7 @@ async function readCapturedWindowAccessibility(
125
const pidWindows =
126
foreground !== undefined
127
? foreground.pid === active.owner.processId
128
- ? [foreground.asElement()]
128
+ ? await foreground.children().catch(() => [])
129
: []
130
: await windowsForPid(App, active.owner.processId);
131
const matchMode = isWaylandSession(platform, process.env) ? "wayland" : "screen-bounds";
pnpm-lock.yaml
+30
-29
@@ -426,8 +426,8 @@ importers:
426
specifier: 0.0.3
427
version: 0.0.3
428
'@crowecawcaw/xa11y':
429
- specifier: 0.13.0
430
- version: 0.13.0
429
+ specifier: 0.15.4
430
+ version: 0.15.4
431
'@effect/platform-node':
432
specifier: 4.1.2
433
version: 4.1.2(bufferutil@4.1.0)(effect@4.1.2)(redis@6.2.1)(utf-8-validate@6.0.6)
@@ -3488,47 +3488,48 @@ packages:
3488
peerDependencies:
3489
'@bufbuild/protobuf': ^1.10.0
3490
3491
- '@crowecawcaw/xa11y-darwin-arm64@0.13.0':
3492
- resolution: {integrity: sha512-wT+f9lbE6IfSPs8unb7IHr6/V/EkB4SCMFisTtV3jsTseCKr4/vkgpI1LRdul2r+KrmgQ4JxRpUDVB25dmpqNw==}
3491
+ '@crowecawcaw/xa11y-darwin-arm64@0.15.4':
3492
+ resolution: {integrity: sha512-/8I+o3JR2tY8thwH/lEcmIuyTxHY8rLW422UFFcP0jI4XRD4+kx5fSjSGV2QIUaQ9tlNsyZkPS0cTYX4oF9GKw==}
3493
engines: {node: '>=18'}
3494
cpu: [arm64]
3495
os: [darwin]
3496
3497
- '@crowecawcaw/xa11y-darwin-x64@0.13.0':
3498
- resolution: {integrity: sha512-8QtLjTzciLIFOTWX1wZRTh6lg+Cw35Rc0VcDtJspS3baHiqkTZD6UyeZKhu5awSirbYZWLeXTuyY2CHCfaQ9rw==}
3497
+ '@crowecawcaw/xa11y-darwin-x64@0.15.4':
3498
+ resolution: {integrity: sha512-sCEfQrpVXyDwacL2PqsUiLUKUN2lhiBZ8dwW8R1/NuLYJTygGVz0UI1Tuk/td/i6RPupIHAgBgjwNo9Ci1KU0w==}
3499
engines: {node: '>=18'}
3500
cpu: [x64]
3501
os: [darwin]
3502
3503
- '@crowecawcaw/xa11y-linux-arm64-gnu@0.13.0':
3504
- resolution: {integrity: sha512-Yad7Bo25pqyowd92yLxGqr59ASSf7zr6lhzoufeCJU62eOq96jEQhhQw0TlvRU8aRI1ksgSuiUivoxNjYP2/gg==}
3503
+ '@crowecawcaw/xa11y-linux-arm64-gnu@0.15.4':
3504
+ resolution: {integrity: sha512-v0l2K3vqY4t9a8dK5j2EFZkjOknUv1j89zN++uuYLT5qR4TBbQFJs9ZcqxzLPwO1SqMXlO+LD4W+Ssx5JyQRjg==}
3505
engines: {node: '>=18'}
3506
cpu: [arm64]
3507
os: [linux]
3508
libc: [glibc]
3509
3510
- '@crowecawcaw/xa11y-linux-x64-gnu@0.13.0':
3511
- resolution: {integrity: sha512-h/7UoAr2X8iSHaBsNFt7uW1Ft14zgintTACNYMTX8WFwE+WXdHe3vtJmxa/bG+N8imWSGH1Xkzg52YJaa/C+bw==}
3510
+ '@crowecawcaw/xa11y-linux-x64-gnu@0.15.4':
3511
+ resolution: {integrity: sha512-ajyJr5J+4YyAtdca7Gdeus18boAaRyNL6JTkk9QG48YXCgLx16y2dOvT0PvpbIPsc31VAH5RcvoHpnRW7E3sJQ==}
3512
engines: {node: '>=18'}
3513
cpu: [x64]
3514
os: [linux]
3515
libc: [glibc]
3516
3517
- '@crowecawcaw/xa11y-win32-arm64-msvc@0.13.0':
3518
- resolution: {integrity: sha512-3vmwZxNg7s6qXdSmYve3EE/zh7vZx75jUxi29odguy9llUvWpWFguvvXJ8EFYh13+TwxchibxnImlZb1XpKG+g==}
3517
+ '@crowecawcaw/xa11y-win32-arm64-msvc@0.15.4':
3518
+ resolution: {integrity: sha512-RfF0CIHXruhg3mQB3xPpItUgiB+Rh9RdHQ9fMLBd2wtCt8V9jtdcOu37EMrPDKt5ClpAxUr0/kxJrhfrX8ZzSQ==}
3519
engines: {node: '>=18'}
3520
cpu: [arm64]
3521
os: [win32]
3522
3523
- '@crowecawcaw/xa11y-win32-x64-msvc@0.13.0':
3524
- resolution: {integrity: sha512-7kWYSj6AZKIN89EirGJQTkeaCJrElwFXOp9eQKd55vUFGZw2YARGY5HfpXha2BKGgZgT2IRR4CJrmgz2Nb1ETw==}
3523
+ '@crowecawcaw/xa11y-win32-x64-msvc@0.15.4':
3524
+ resolution: {integrity: sha512-fnx8+QGU76m+F6D3Va+fMg1jq/bOpvTU6tbnfF0pCg93wvsuJ0+xO/MJlFnCZVeOagPm0+wDBVBLUC1cvQOAmw==}
3525
engines: {node: '>=18'}
3526
cpu: [x64]
3527
os: [win32]
3528
3529
- '@crowecawcaw/xa11y@0.13.0':
3530
- resolution: {integrity: sha512-WTYHwlBhImGmmMt81crxaRvNdZXlBNemFyrP5GGHj2WeDTJMBrHgHTaladTypXHXrviFFBBGROl5moijRQ4ODA==}
3529
+ '@crowecawcaw/xa11y@0.15.4':
3530
+ resolution: {integrity: sha512-aDGW67yj0OPGc8Oo/IJZpZy/252D+qi9rcrlT8T+mXLjsn5+58Ha6cq921O58Yq6aGvNikXBMbap0J6/fM2Mlg==}
3531
engines: {node: '>=18'}
3532
+ hasBin: true
3533
3534
'@csstools/color-helpers@6.1.1':
3535
resolution: {integrity: sha512-gLNsunvwf3mCi5u5o46/Z/JcJMnhbHSaZ69rkgPzNM3J4s8hWwpPUQB6/tt0EDFyCiWzxANlx+2LJwpYj4zS1w==}
@@ -13498,32 +13499,32 @@ snapshots:
13499
dependencies:
13500
'@bufbuild/protobuf': 1.10.0
13501
13501
- '@crowecawcaw/xa11y-darwin-arm64@0.13.0':
13502
+ '@crowecawcaw/xa11y-darwin-arm64@0.15.4':
13503
optional: true
13504
13504
- '@crowecawcaw/xa11y-darwin-x64@0.13.0':
13505
+ '@crowecawcaw/xa11y-darwin-x64@0.15.4':
13506
optional: true
13507
13507
- '@crowecawcaw/xa11y-linux-arm64-gnu@0.13.0':
13508
+ '@crowecawcaw/xa11y-linux-arm64-gnu@0.15.4':
13509
optional: true
13510
13510
- '@crowecawcaw/xa11y-linux-x64-gnu@0.13.0':
13511
+ '@crowecawcaw/xa11y-linux-x64-gnu@0.15.4':
13512
optional: true
13513
13513
- '@crowecawcaw/xa11y-win32-arm64-msvc@0.13.0':
13514
+ '@crowecawcaw/xa11y-win32-arm64-msvc@0.15.4':
13515
optional: true
13516
13516
- '@crowecawcaw/xa11y-win32-x64-msvc@0.13.0':
13517
+ '@crowecawcaw/xa11y-win32-x64-msvc@0.15.4':
13518
optional: true
13519
13519
- '@crowecawcaw/xa11y@0.13.0':
13520
+ '@crowecawcaw/xa11y@0.15.4':
13521
optionalDependencies:
13521
- '@crowecawcaw/xa11y-darwin-arm64': 0.13.0
13522
- '@crowecawcaw/xa11y-darwin-x64': 0.13.0
13523
- '@crowecawcaw/xa11y-linux-arm64-gnu': 0.13.0
13524
- '@crowecawcaw/xa11y-linux-x64-gnu': 0.13.0
13525
- '@crowecawcaw/xa11y-win32-arm64-msvc': 0.13.0
13526
- '@crowecawcaw/xa11y-win32-x64-msvc': 0.13.0
13522
+ '@crowecawcaw/xa11y-darwin-arm64': 0.15.4
13523
+ '@crowecawcaw/xa11y-darwin-x64': 0.15.4
13524
+ '@crowecawcaw/xa11y-linux-arm64-gnu': 0.15.4
13525
+ '@crowecawcaw/xa11y-linux-x64-gnu': 0.15.4
13526
+ '@crowecawcaw/xa11y-win32-arm64-msvc': 0.15.4
13527
+ '@crowecawcaw/xa11y-win32-x64-msvc': 0.15.4
13528
13529
'@csstools/color-helpers@6.1.1': {}
13530