The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 16:00 UTC
fix(devices): agents can test the mobile app on remote device hosts (#25121)
Co-authored-by: Julius Marminge <julius0216@outlook.com>
Theo Browne committed
Nov 14, 2026 at 00:27 UTC
7bf7527a0a9bf9e7dcd3a6afa3962bfe17a66a78
11 files changed
+156
-28
apps/server/src/device/DeviceService.test.ts
+48
-2
@@ -49,6 +49,25 @@ const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown
49
const decodeBootError = Schema.decodeUnknownSync(Schema.fromJsonString(DeviceBootError));
50
const encodeBootError = Schema.encodeSync(Schema.fromJsonString(DeviceBootError));
51
52
+const pngChunk = (type: string, length: number) => {
53
+ const chunk = new Uint8Array(12 + length);
54
+ new DataView(chunk.buffer).setUint32(0, length);
55
+ chunk.set(new TextEncoder().encode(type), 4);
56
+ return chunk;
57
+};
58
+const minimalPng = Uint8Array.from([
59
+ 137,
60
+ 80,
61
+ 78,
62
+ 71,
63
+ 13,
64
+ 10,
65
+ 26,
66
+ 10,
67
+ ...pngChunk("IHDR", 13),
68
+ ...pngChunk("IEND", 0),
69
+]);
70
+
71
it.effect.each([400, 500])(
72
"retries a hub-rejected (%s) screenshot with simctl only on iOS 26 and older",
73
(status) =>
@@ -112,6 +131,7 @@ const fixture = Effect.fn("fixture")(function* (
131
const androidAvailable = yield* Ref.make(true);
132
const iosAvailable = yield* Ref.make(true);
133
const encoderResult = yield* Ref.make({ code: 0, stdout: " V....D libx264", stderr: "" });
134
+ const screenshotBody = yield* Ref.make(minimalPng);
135
const listing = { hangs: false, requested: yield* Deferred.make<void>() };
136
let booted = false;
137
let shutDown = false;
@@ -150,7 +170,7 @@ const fixture = Effect.fn("fixture")(function* (
170
);
171
}
172
if (request.url.includes("/api/screenshot")) {
153
- return HttpClientResponse.fromWeb(request, new Response(new Uint8Array([137, 80, 78, 71])));
173
+ return HttpClientResponse.fromWeb(request, new Response(yield* Ref.get(screenshotBody)));
174
}
175
if (request.url.endsWith("/shutdown")) {
176
if (connection.failShutdown)
@@ -343,6 +363,7 @@ const fixture = Effect.fn("fixture")(function* (
363
connection,
364
devices,
365
encoderResult,
366
+ screenshotBody,
367
discoveryErrors,
368
androidAvailable,
369
iosAvailable,
@@ -793,12 +814,37 @@ describe("device discovery after server restart", () => {
814
expect((yield* service.state).devices).toEqual([]);
815
const capture = yield* service.screenshot({ deviceId: DeviceId.make("Pixel_API_35") });
816
expect(capture.device.id).toBe("Pixel_API_35");
796
- expect(Array.from(capture.png)).toEqual([137, 80, 78, 71]);
817
+ expect(capture.png).toEqual(minimalPng);
818
expect(requests.some((url) => url.endsWith("/api/devices"))).toBe(true);
819
}),
820
);
821
});
822
823
+describe("device screenshots", () => {
824
+ it.effect("drops text that adb prints around the PNG", () =>
825
+ Effect.gen(function* () {
826
+ const { service, settings, screenshotBody } = yield* fixture();
827
+ yield* Ref.update(settings, (current) => ({ ...current, enableDeviceSupport: true }));
828
+ const warning = new TextEncoder().encode("[Warning] Multiple displays were found\n");
829
+ yield* Ref.set(screenshotBody, Uint8Array.from([...warning, ...minimalPng, 10]));
830
+ const capture = yield* service.screenshot({ deviceId: DeviceId.make("Pixel_API_35") });
831
+ expect(capture.png).toEqual(minimalPng);
832
+ }),
833
+ );
834
+
835
+ it.effect("fails instead of returning bytes that hold no complete PNG", () =>
836
+ Effect.gen(function* () {
837
+ const { service, settings, screenshotBody } = yield* fixture();
838
+ yield* Ref.update(settings, (current) => ({ ...current, enableDeviceSupport: true }));
839
+ yield* Ref.set(screenshotBody, minimalPng.subarray(0, 30));
840
+ const error = yield* Effect.flip(
841
+ service.screenshot({ deviceId: DeviceId.make("Pixel_API_35") }),
842
+ );
843
+ expect(error).toMatchObject({ _tag: "DeviceOperationError", reason: "hub_rejected" });
844
+ }),
845
+ );
846
+});
847
+
848
const missingPlistMessage =
849
"[apple-utils] Failed to read CoreSimulator plist at /Users/test/Library/Developer/CoreSimulator/Devices/deleted-simulator/device.plist:";
850
apps/server/src/device/DeviceService.ts
+11
-1
@@ -32,6 +32,7 @@ import {
32
LOCAL_DEVICE_HOST_LABEL,
33
type ThreadId,
34
} from "@t3tools/contracts";
35
+import { extractPng } from "@t3tools/shared/png";
36
import { SshCommandError } from "@t3tools/ssh/errors";
37
import * as FileSystem from "effect/FileSystem";
38
import { resolveNodeExecutable, nodeRuntimeUnavailableMessage } from "@t3tools/shared/nodeRuntime";
@@ -993,7 +994,16 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function*
994
const screenshot: DeviceService["Service"]["screenshot"] = Effect.fn("DeviceService.screenshot")(
995
function* (input) {
996
const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId);
996
- return { device, png: yield* ready.devices.screenshot(device) };
997
+ const bytes = yield* ready.devices.screenshot(device);
998
+ const png = extractPng(bytes);
999
+ if (png === undefined) {
1000
+ return yield* new DeviceOperationError({
1001
+ operation: "screenshot",
1002
+ reason: "hub_rejected",
1003
+ cause: `The hub returned ${bytes.length} bytes without a complete PNG.`,
1004
+ });
1005
+ }
1006
+ return { device, png };
1007
},
1008
);
1009
evals/mcp-tool-use/README.md
+1
-1
@@ -60,7 +60,7 @@ Check that the case builds with `--dry-run --case <id>`. Then run it on the text
60
61
`device.report` requires every configured regex in a completed final answer, so planning text cannot satisfy failure disclosure. CAP-01 sets `device.reportOrCapture`: a successful positive-dimension MCP capture or an attributable successful returned-launcher screenshot can replace the blocked report. Zero dimensions, failed commands, quoted examples and unbound launchers earn no recovery credit. It is a bounded disclosure check, not a general honesty evaluator. `device.capture.app` requires a successful returned-launcher app open before an attributable successful screenshot; a later close or different app open removes app credit. An unambiguous `open … && screenshot` chain is processed in action order. Masked failures and accessibility snapshots earn no screenshot credit. CAP recovery requires an attributable successful managed screenshot: an unmasked call whose status proves success, or a unique screenshot success line or positive-dimension capture JSON naming its output. Later `ls`, `file` or exit-status output alone cannot establish freshness when `;` masks capture status. `device.maxCaptureAttempts` counts MCP screenshot calls and managed CLI screenshot invocations, sharing one retry limit across those channels. Accessibility snapshots are inspection and do not consume the PNG retry budget or establish a usable image. No new shell grammar is added.
62
63
-OWN-01 uses an authored agent-device ownership claim. Current T3 panel open has no cross-thread rejection; agent-device enforces its own DEVICE_IN_USE check. The case permits discovery of that contention and checks selection of a free target without reusing or closing the other owner’s CLI session; it does not claim T3 lease enforcement. DISC-01 covers an allowed alternate, not a fleet without one. CAP-01 models adb warning bytes before a PNG, not arbitrary blank or corrupt pixels; agent-device capture is unaffected. OWN-01 models same-daemon ownership, not cross-workspace claims. Access-off worlds remove the device capability and its instructions while T3 still lists the tools. Optional remote-install coverage (F2) remains outside this increment.
63
+OWN-01 uses an authored agent-device ownership claim. Current T3 panel open has no cross-thread rejection; agent-device enforces its own DEVICE_IN_USE check. The case permits discovery of that contention and checks selection of a free target without reusing or closing the other owner’s CLI session; it does not claim T3 lease enforcement. DISC-01 covers an allowed alternate, not a fleet without one. CAP-01 models a PNG whose header says 0×0, not arbitrary blank or corrupt pixels; agent-device capture is unaffected. OWN-01 models same-daemon ownership, not cross-workspace claims. Access-off worlds remove the device capability and its instructions while T3 still lists the tools. Optional remote-install coverage (F2) remains outside this increment.
64
65
### Results
66
evals/mcp-tool-use/devices.test.ts
+1
-1
@@ -678,7 +678,7 @@ it.effect("reports an unreadable capture as T3 does: a successful 0×0 screensho
678
);
679
yield* withDevices(sample);
680
const told = JSON.stringify(shellRequest(sample, { tool: "Bash", input: {} }, model).messages);
681
- assert.include(told, "agent-device's own screenshot finds the PNG and is unaffected");
681
+ assert.include(told, "agent-device's own screenshot is unaffected");
682
}).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
683
);
684
evals/mcp-tool-use/devices.ts
+12
-15
@@ -66,7 +66,7 @@ export interface DeviceWorld {
66
readonly deviceDiscoveryFailures?: Readonly<Record<string, DeviceOperationError>> | undefined;
67
/** false: agent device access is off, so the thread holds no `device` capability. */
68
readonly deviceAgentAccess?: boolean | undefined;
69
- /** Every capture reaches T3 with adb screencap's multi-display warning ahead of the PNG. */
69
+ /** Every capture reaches T3 as a PNG whose header says 0×0. */
70
readonly deviceInvalidCapture?: boolean | undefined;
71
/** Devices another thread's agent-device session holds, by device id: that thread's id. */
72
readonly deviceOwners?: Readonly<Record<string, string>> | undefined;
@@ -547,24 +547,19 @@ const failing = <A, R>(effect: Effect.Effect<A, DeviceError, R>) =>
547
const NODE = "/usr/bin/node";
548
const HUB = "http://127.0.0.1:47613";
549
const PLATFORM_NAMES = { ios: "iOS Simulators", android: "Android Emulators" } as const;
550
-/** T3 reads only a capture's PNG header; replay has no screens to send. */
551
-const capture = (platform: DevicePlatform) => {
552
- const png = new Uint8Array(24);
550
+/** A complete PNG holding only the header T3 reads; replay has no screens to send. */
551
+const capture = (width: number, height: number) => {
552
+ const png = new Uint8Array(45);
553
const view = new DataView(png.buffer);
554
view.setUint32(0, 0x89504e47);
555
view.setUint32(4, 0x0d0a1a0a);
556
+ view.setUint32(8, 13);
557
view.setUint32(12, 0x49484452);
557
- view.setUint32(16, platform === "ios" ? 1206 : 1080);
558
- view.setUint32(20, platform === "ios" ? 2622 : 2400);
558
+ view.setUint32(16, width);
559
+ view.setUint32(20, height);
560
+ view.setUint32(37, 0x49454e44);
561
return png;
562
};
561
-/**
562
- * What `adb exec-out screencap -p` prints before the PNG on an emulator with several displays,
563
- * such as a foldable. T3 passes the hub's bytes on, and its PNG check finds no header.
564
- */
565
-const SCREENCAP_WARNING = new TextEncoder().encode(
566
- "[Warning] Multiple displays were found, but no display id was specified! Defaulting to the first display found, however this default is not guaranteed to be consistent across captures. A display id should be specified.\n",
567
-);
563
564
const fakeHost = (
565
world: DeviceWorld,
@@ -604,8 +599,10 @@ const fakeHost = (
599
screenshot: (device) =>
600
Effect.succeed(
601
world.deviceInvalidCapture
607
- ? new Uint8Array([...SCREENCAP_WARNING, ...capture(device.platform)])
608
- : capture(device.platform),
602
+ ? capture(0, 0)
603
+ : device.platform === "ios"
604
+ ? capture(1206, 2622)
605
+ : capture(1080, 2400),
606
),
607
},
608
// T3 checks an Android emulator's host for ffmpeg's H.264 encoder before streaming it.
evals/mcp-tool-use/run.ts
+1
-1
@@ -1519,7 +1519,7 @@ export const shellRequest = (sample: Sample, call: Omit<ShellEntry, "output">, m
1519
...(devices?.describe(shell, typeof command === "string" ? command : undefined) ?? []),
1520
...(world?.deviceInvalidCapture
1521
? [
1522
- "T3's device_screenshot gets each capture with adb screencap's multi-display warning ahead of the PNG, so it reports 0×0. agent-device's own screenshot finds the PNG and is unaffected.",
1522
+ "T3's device_screenshot gets each capture as a PNG whose header says 0×0, so it reports 0×0. agent-device's own screenshot is unaffected.",
1523
]
1524
: []),
1525
...(sample.secretFiles.length > 0
packages/client-runtime/src/device/screenshot.ts
+4
-3
@@ -1,5 +1,6 @@
1
// @effect-diagnostics globalFetch:off - This browser transport runs without an Effect runtime, like the live stream.
2
import * as Schema from "effect/Schema";
3
+import { extractPng } from "@t3tools/shared/png";
4
import { withDeviceHubQuery } from "./hubAccess.ts";
5
import type { DeviceStreamTarget } from "./stream.ts";
6
@@ -24,7 +25,7 @@ export async function captureDeviceScreenshot(target: DeviceStreamTarget, signal
25
signal: AbortSignal.any([signal, AbortSignal.timeout(20_000)]),
26
});
27
if (!response.ok) throw new DeviceScreenshotError({ status: response.status });
27
- const image = await response.blob();
28
- if (!image.size) throw new Error("The device returned an empty screenshot. Try again.");
29
- return image;
28
+ const png = extractPng(new Uint8Array(await response.arrayBuffer()));
29
+ if (!png) throw new Error("The device returned an incomplete screenshot. Try again.");
30
+ return new Blob([png], { type: "image/png" });
31
}
packages/shared/package.json
+4
@@ -3,6 +3,10 @@
3
"private": true,
4
"type": "module",
5
"exports": {
6
+ "./png": {
7
+ "types": "./src/png.ts",
8
+ "import": "./src/png.ts"
9
+ },
10
"./pullRequestAnchor": {
11
"types": "./src/pullRequestAnchor.ts",
12
"import": "./src/pullRequestAnchor.ts"
packages/shared/src/png.ts
new
+24
@@ -0,0 +1,24 @@
1
+const PNG_SIGNATURE = [137, 80, 78, 71, 13, 10, 26, 10];
2
+
3
+/**
4
+ * Returns the first complete PNG in `bytes`, or undefined. `adb exec-out screencap -p` can print
5
+ * warnings, such as a foldable's multiple displays, around the image.
6
+ */
7
+export const extractPng = <Buffer extends ArrayBufferLike>(
8
+ bytes: Uint8Array<Buffer>,
9
+): Uint8Array<Buffer> | undefined => {
10
+ const start = bytes.findIndex((_, index) =>
11
+ PNG_SIGNATURE.every((value, offset) => bytes[index + offset] === value),
12
+ );
13
+ if (start < 0) return undefined;
14
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
15
+ for (let chunk = start + PNG_SIGNATURE.length; chunk + 12 <= bytes.length;) {
16
+ const end = chunk + 12 + view.getUint32(chunk);
17
+ if (end > bytes.length) return undefined;
18
+ if (String.fromCharCode(...bytes.subarray(chunk + 4, chunk + 8)) === "IEND") {
19
+ return bytes.subarray(start, end);
20
+ }
21
+ chunk = end;
22
+ }
23
+ return undefined;
24
+};
scripts/mobile-native-client.test.ts
+24
@@ -1,10 +1,12 @@
1
import * as NodeServices from "@effect/platform-node/NodeServices";
2
import { assert, it } from "@effect/vitest";
3
+import * as HostProcess from "@t3tools/shared/HostProcess";
4
import * as Effect from "effect/Effect";
5
import * as FileSystem from "effect/FileSystem";
6
import * as Path from "effect/Path";
7
8
import {
9
+ androidSdk,
10
buildRunTitle,
11
NativeClientError,
12
prebuildAndroid,
@@ -67,6 +69,28 @@ it.layer(NodeServices.layer)("prebuildAndroid", (it) => {
69
);
70
});
71
72
+it.layer(NodeServices.layer)("androidSdk", (it) => {
73
+ it.effect("skips a configured SDK path that holds no SDK", () =>
74
+ Effect.gen(function* () {
75
+ const fs = yield* FileSystem.FileSystem;
76
+ const path = yield* Path.Path;
77
+ const home = yield* fs.makeTempDirectoryScoped({ prefix: "android-sdk-home-" });
78
+ const sdk = path.join(home, "sdk");
79
+ yield* fs.makeDirectory(path.join(sdk, "platform-tools"), { recursive: true });
80
+
81
+ const resolved = yield* androidSdk.pipe(
82
+ Effect.provideService(HostProcess.Environment, {
83
+ HOME: home,
84
+ ANDROID_HOME: path.join(home, "stale"),
85
+ ANDROID_SDK_ROOT: sdk,
86
+ }),
87
+ );
88
+
89
+ assert.strictEqual(resolved, sdk);
90
+ }),
91
+ );
92
+});
93
+
94
it("finds a Blacksmith build by its run title and reads its state", () => {
95
const fingerprint = "a".repeat(40);
96
const title = buildRunTitle("android", "x86_64", fingerprint);
scripts/mobile-native-client.ts
+26
-4
@@ -149,19 +149,39 @@ const collect = <E>(stream: Stream.Stream<Uint8Array, E>) =>
149
(a, b) => a + b,
150
),
151
);
152
+/** Gradle needs the SDK location; fall back to Android Studio's default install like the device host does. */
153
+export const androidSdk = Effect.gen(function* () {
154
+ const fs = yield* FileSystem.FileSystem;
155
+ const path = yield* Path.Path;
156
+ const environment = yield* HostProcess.Environment;
157
+ const home = environment.HOME ?? environment.USERPROFILE ?? "";
158
+ const candidates = [
159
+ environment.ANDROID_HOME?.trim(),
160
+ environment.ANDROID_SDK_ROOT?.trim(),
161
+ path.join(home, "Library", "Android", "sdk"),
162
+ path.join(home, "Android", "Sdk"),
163
+ path.join(environment.LOCALAPPDATA ?? path.join(home, "AppData", "Local"), "Android", "Sdk"),
164
+ ];
165
+ for (const sdk of candidates) {
166
+ if (!sdk) continue;
167
+ const platformTools = path.join(sdk, "platform-tools");
168
+ if (yield* fs.exists(platformTools).pipe(Effect.orElseSucceed(() => false))) return sdk;
169
+ }
170
+ return undefined;
171
+});
172
+
173
export const resolveAdb = Effect.gen(function* () {
174
if (yield* isCommandAvailable("adb")) return "adb";
154
- const environment = yield* HostProcess.Environment;
175
const path = yield* Path.Path;
176
const executable = (yield* HostProcess.Platform) === "win32" ? "adb.exe" : "adb";
157
- for (const sdk of [environment.ANDROID_SDK_ROOT, environment.ANDROID_HOME]) {
158
- if (!sdk) continue;
177
+ const sdk = yield* androidSdk;
178
+ if (sdk) {
179
const candidate = path.join(sdk, "platform-tools", executable);
180
if (yield* isCommandAvailable(candidate)) return candidate;
181
}
182
return yield* new NativeClientError({
183
message:
164
- "adb was not found on PATH or in ANDROID_SDK_ROOT/ANDROID_HOME. Install Android SDK platform-tools.",
184
+ "adb was not found on PATH, in ANDROID_HOME/ANDROID_SDK_ROOT, or in Android Studio's default SDK location. Install Android SDK platform-tools.",
185
});
186
});
187
@@ -185,12 +205,14 @@ const command = Effect.fn("nativeClient.command")(function* (
205
);
206
const log = inherit ? yield* BuildLog : null;
207
const spawn = yield* resolveSpawnCommand(program === "adb" ? yield* resolveAdb : program, args);
208
+ const sdk = yield* androidSdk;
209
const child = yield* spawner.spawn(
210
ChildProcess.make(spawn.command, spawn.args, {
211
shell: spawn.shell,
212
cwd: cwd ?? (yield* roots).mobile,
213
env: {
214
...environment,
215
+ ...(sdk ? { ANDROID_HOME: sdk } : {}),
216
APP_VARIANT: "development",
217
MOBILE_VERSION_POLICY: "appVersion",
218
T3CODE_IOS_PERSONAL_TEAM: "0",