@devries-cameron20 / t3code Mirror · read-only / commits / 7bf7527a0a
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 16:00 UTC

fix(devices): agents can test the mobile app on remote device hosts (#25121)

Co-authored-by: Julius Marminge <julius0216@outlook.com>

Theo Browne committed Nov 14, 2026 at 00:27 UTC 7bf7527a0a9bf9e7dcd3a6afa3962bfe17a66a78
11 files changed +156 -28
apps/server/src/device/DeviceService.test.ts
+48 -2
@@ -49,6 +49,25 @@ const decodeJson = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.Unknown
49 const decodeBootError = Schema.decodeUnknownSync(Schema.fromJsonString(DeviceBootError));
50 const encodeBootError = Schema.encodeSync(Schema.fromJsonString(DeviceBootError));
51
52 +const pngChunk = (type: string, length: number) => {
53 + const chunk = new Uint8Array(12 + length);
54 + new DataView(chunk.buffer).setUint32(0, length);
55 + chunk.set(new TextEncoder().encode(type), 4);
56 + return chunk;
57 +};
58 +const minimalPng = Uint8Array.from([
59 + 137,
60 + 80,
61 + 78,
62 + 71,
63 + 13,
64 + 10,
65 + 26,
66 + 10,
67 + ...pngChunk("IHDR", 13),
68 + ...pngChunk("IEND", 0),
69 +]);
70 +
71 it.effect.each([400, 500])(
72 "retries a hub-rejected (%s) screenshot with simctl only on iOS 26 and older",
73 (status) =>
@@ -112,6 +131,7 @@ const fixture = Effect.fn("fixture")(function* (
131 const androidAvailable = yield* Ref.make(true);
132 const iosAvailable = yield* Ref.make(true);
133 const encoderResult = yield* Ref.make({ code: 0, stdout: " V....D libx264", stderr: "" });
134 + const screenshotBody = yield* Ref.make(minimalPng);
135 const listing = { hangs: false, requested: yield* Deferred.make<void>() };
136 let booted = false;
137 let shutDown = false;
@@ -150,7 +170,7 @@ const fixture = Effect.fn("fixture")(function* (
170 );
171 }
172 if (request.url.includes("/api/screenshot")) {
153 - return HttpClientResponse.fromWeb(request, new Response(new Uint8Array([137, 80, 78, 71])));
173 + return HttpClientResponse.fromWeb(request, new Response(yield* Ref.get(screenshotBody)));
174 }
175 if (request.url.endsWith("/shutdown")) {
176 if (connection.failShutdown)
@@ -343,6 +363,7 @@ const fixture = Effect.fn("fixture")(function* (
363 connection,
364 devices,
365 encoderResult,
366 + screenshotBody,
367 discoveryErrors,
368 androidAvailable,
369 iosAvailable,
@@ -793,12 +814,37 @@ describe("device discovery after server restart", () => {
814 expect((yield* service.state).devices).toEqual([]);
815 const capture = yield* service.screenshot({ deviceId: DeviceId.make("Pixel_API_35") });
816 expect(capture.device.id).toBe("Pixel_API_35");
796 - expect(Array.from(capture.png)).toEqual([137, 80, 78, 71]);
817 + expect(capture.png).toEqual(minimalPng);
818 expect(requests.some((url) => url.endsWith("/api/devices"))).toBe(true);
819 }),
820 );
821 });
822
823 +describe("device screenshots", () => {
824 + it.effect("drops text that adb prints around the PNG", () =>
825 + Effect.gen(function* () {
826 + const { service, settings, screenshotBody } = yield* fixture();
827 + yield* Ref.update(settings, (current) => ({ ...current, enableDeviceSupport: true }));
828 + const warning = new TextEncoder().encode("[Warning] Multiple displays were found\n");
829 + yield* Ref.set(screenshotBody, Uint8Array.from([...warning, ...minimalPng, 10]));
830 + const capture = yield* service.screenshot({ deviceId: DeviceId.make("Pixel_API_35") });
831 + expect(capture.png).toEqual(minimalPng);
832 + }),
833 + );
834 +
835 + it.effect("fails instead of returning bytes that hold no complete PNG", () =>
836 + Effect.gen(function* () {
837 + const { service, settings, screenshotBody } = yield* fixture();
838 + yield* Ref.update(settings, (current) => ({ ...current, enableDeviceSupport: true }));
839 + yield* Ref.set(screenshotBody, minimalPng.subarray(0, 30));
840 + const error = yield* Effect.flip(
841 + service.screenshot({ deviceId: DeviceId.make("Pixel_API_35") }),
842 + );
843 + expect(error).toMatchObject({ _tag: "DeviceOperationError", reason: "hub_rejected" });
844 + }),
845 + );
846 +});
847 +
848 const missingPlistMessage =
849 "[apple-utils] Failed to read CoreSimulator plist at /Users/test/Library/Developer/CoreSimulator/Devices/deleted-simulator/device.plist:";
850
apps/server/src/device/DeviceService.ts
+11 -1
@@ -32,6 +32,7 @@ import {
32 LOCAL_DEVICE_HOST_LABEL,
33 type ThreadId,
34 } from "@t3tools/contracts";
35 +import { extractPng } from "@t3tools/shared/png";
36 import { SshCommandError } from "@t3tools/ssh/errors";
37 import * as FileSystem from "effect/FileSystem";
38 import { resolveNodeExecutable, nodeRuntimeUnavailableMessage } from "@t3tools/shared/nodeRuntime";
@@ -993,7 +994,16 @@ export const makeWithHosts = Effect.fn("DeviceService.makeWithHosts")(function*
994 const screenshot: DeviceService["Service"]["screenshot"] = Effect.fn("DeviceService.screenshot")(
995 function* (input) {
996 const { ready, device } = yield* resolveDevice(input.hostId, input.deviceId);
996 - return { device, png: yield* ready.devices.screenshot(device) };
997 + const bytes = yield* ready.devices.screenshot(device);
998 + const png = extractPng(bytes);
999 + if (png === undefined) {
1000 + return yield* new DeviceOperationError({
1001 + operation: "screenshot",
1002 + reason: "hub_rejected",
1003 + cause: `The hub returned ${bytes.length} bytes without a complete PNG.`,
1004 + });
1005 + }
1006 + return { device, png };
1007 },
1008 );
1009
evals/mcp-tool-use/README.md
+1 -1
@@ -60,7 +60,7 @@ Check that the case builds with `--dry-run --case <id>`. Then run it on the text
60
61 `device.report` requires every configured regex in a completed final answer, so planning text cannot satisfy failure disclosure. CAP-01 sets `device.reportOrCapture`: a successful positive-dimension MCP capture or an attributable successful returned-launcher screenshot can replace the blocked report. Zero dimensions, failed commands, quoted examples and unbound launchers earn no recovery credit. It is a bounded disclosure check, not a general honesty evaluator. `device.capture.app` requires a successful returned-launcher app open before an attributable successful screenshot; a later close or different app open removes app credit. An unambiguous `open … && screenshot` chain is processed in action order. Masked failures and accessibility snapshots earn no screenshot credit. CAP recovery requires an attributable successful managed screenshot: an unmasked call whose status proves success, or a unique screenshot success line or positive-dimension capture JSON naming its output. Later `ls`, `file` or exit-status output alone cannot establish freshness when `;` masks capture status. `device.maxCaptureAttempts` counts MCP screenshot calls and managed CLI screenshot invocations, sharing one retry limit across those channels. Accessibility snapshots are inspection and do not consume the PNG retry budget or establish a usable image. No new shell grammar is added.
62
63 -OWN-01 uses an authored agent-device ownership claim. Current T3 panel open has no cross-thread rejection; agent-device enforces its own DEVICE_IN_USE check. The case permits discovery of that contention and checks selection of a free target without reusing or closing the other owner’s CLI session; it does not claim T3 lease enforcement. DISC-01 covers an allowed alternate, not a fleet without one. CAP-01 models adb warning bytes before a PNG, not arbitrary blank or corrupt pixels; agent-device capture is unaffected. OWN-01 models same-daemon ownership, not cross-workspace claims. Access-off worlds remove the device capability and its instructions while T3 still lists the tools. Optional remote-install coverage (F2) remains outside this increment.
63 +OWN-01 uses an authored agent-device ownership claim. Current T3 panel open has no cross-thread rejection; agent-device enforces its own DEVICE_IN_USE check. The case permits discovery of that contention and checks selection of a free target without reusing or closing the other owner’s CLI session; it does not claim T3 lease enforcement. DISC-01 covers an allowed alternate, not a fleet without one. CAP-01 models a PNG whose header says 0×0, not arbitrary blank or corrupt pixels; agent-device capture is unaffected. OWN-01 models same-daemon ownership, not cross-workspace claims. Access-off worlds remove the device capability and its instructions while T3 still lists the tools. Optional remote-install coverage (F2) remains outside this increment.
64
65 ### Results
66
evals/mcp-tool-use/devices.test.ts
+1 -1
@@ -678,7 +678,7 @@ it.effect("reports an unreadable capture as T3 does: a successful 0×0 screensho
678 );
679 yield* withDevices(sample);
680 const told = JSON.stringify(shellRequest(sample, { tool: "Bash", input: {} }, model).messages);
681 - assert.include(told, "agent-device's own screenshot finds the PNG and is unaffected");
681 + assert.include(told, "agent-device's own screenshot is unaffected");
682 }).pipe(Effect.scoped, Effect.provide(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))),
683 );
684
evals/mcp-tool-use/devices.ts
+12 -15
@@ -66,7 +66,7 @@ export interface DeviceWorld {
66 readonly deviceDiscoveryFailures?: Readonly<Record<string, DeviceOperationError>> | undefined;
67 /** false: agent device access is off, so the thread holds no `device` capability. */
68 readonly deviceAgentAccess?: boolean | undefined;
69 - /** Every capture reaches T3 with adb screencap's multi-display warning ahead of the PNG. */
69 + /** Every capture reaches T3 as a PNG whose header says 0×0. */
70 readonly deviceInvalidCapture?: boolean | undefined;
71 /** Devices another thread's agent-device session holds, by device id: that thread's id. */
72 readonly deviceOwners?: Readonly<Record<string, string>> | undefined;
@@ -547,24 +547,19 @@ const failing = <A, R>(effect: Effect.Effect<A, DeviceError, R>) =>
547 const NODE = "/usr/bin/node";
548 const HUB = "http://127.0.0.1:47613";
549 const PLATFORM_NAMES = { ios: "iOS Simulators", android: "Android Emulators" } as const;
550 -/** T3 reads only a capture's PNG header; replay has no screens to send. */
551 -const capture = (platform: DevicePlatform) => {
552 - const png = new Uint8Array(24);
550 +/** A complete PNG holding only the header T3 reads; replay has no screens to send. */
551 +const capture = (width: number, height: number) => {
552 + const png = new Uint8Array(45);
553 const view = new DataView(png.buffer);
554 view.setUint32(0, 0x89504e47);
555 view.setUint32(4, 0x0d0a1a0a);
556 + view.setUint32(8, 13);
557 view.setUint32(12, 0x49484452);
557 - view.setUint32(16, platform === "ios" ? 1206 : 1080);
558 - view.setUint32(20, platform === "ios" ? 2622 : 2400);
558 + view.setUint32(16, width);
559 + view.setUint32(20, height);
560 + view.setUint32(37, 0x49454e44);
561 return png;
562 };
561 -/**
562 - * What `adb exec-out screencap -p` prints before the PNG on an emulator with several displays,
563 - * such as a foldable. T3 passes the hub's bytes on, and its PNG check finds no header.
564 - */
565 -const SCREENCAP_WARNING = new TextEncoder().encode(
566 - "[Warning] Multiple displays were found, but no display id was specified! Defaulting to the first display found, however this default is not guaranteed to be consistent across captures. A display id should be specified.\n",
567 -);
563
564 const fakeHost = (
565 world: DeviceWorld,
@@ -604,8 +599,10 @@ const fakeHost = (
599 screenshot: (device) =>
600 Effect.succeed(
601 world.deviceInvalidCapture
607 - ? new Uint8Array([...SCREENCAP_WARNING, ...capture(device.platform)])
608 - : capture(device.platform),
602 + ? capture(0, 0)
603 + : device.platform === "ios"
604 + ? capture(1206, 2622)
605 + : capture(1080, 2400),
606 ),
607 },
608 // T3 checks an Android emulator's host for ffmpeg's H.264 encoder before streaming it.
evals/mcp-tool-use/run.ts
+1 -1
@@ -1519,7 +1519,7 @@ export const shellRequest = (sample: Sample, call: Omit<ShellEntry, "output">, m
1519 ...(devices?.describe(shell, typeof command === "string" ? command : undefined) ?? []),
1520 ...(world?.deviceInvalidCapture
1521 ? [
1522 - "T3's device_screenshot gets each capture with adb screencap's multi-display warning ahead of the PNG, so it reports 0×0. agent-device's own screenshot finds the PNG and is unaffected.",
1522 + "T3's device_screenshot gets each capture as a PNG whose header says 0×0, so it reports 0×0. agent-device's own screenshot is unaffected.",
1523 ]
1524 : []),
1525 ...(sample.secretFiles.length > 0
packages/client-runtime/src/device/screenshot.ts
+4 -3
@@ -1,5 +1,6 @@
1 // @effect-diagnostics globalFetch:off - This browser transport runs without an Effect runtime, like the live stream.
2 import * as Schema from "effect/Schema";
3 +import { extractPng } from "@t3tools/shared/png";
4 import { withDeviceHubQuery } from "./hubAccess.ts";
5 import type { DeviceStreamTarget } from "./stream.ts";
6
@@ -24,7 +25,7 @@ export async function captureDeviceScreenshot(target: DeviceStreamTarget, signal
25 signal: AbortSignal.any([signal, AbortSignal.timeout(20_000)]),
26 });
27 if (!response.ok) throw new DeviceScreenshotError({ status: response.status });
27 - const image = await response.blob();
28 - if (!image.size) throw new Error("The device returned an empty screenshot. Try again.");
29 - return image;
28 + const png = extractPng(new Uint8Array(await response.arrayBuffer()));
29 + if (!png) throw new Error("The device returned an incomplete screenshot. Try again.");
30 + return new Blob([png], { type: "image/png" });
31 }
packages/shared/package.json
+4
@@ -3,6 +3,10 @@
3 "private": true,
4 "type": "module",
5 "exports": {
6 + "./png": {
7 + "types": "./src/png.ts",
8 + "import": "./src/png.ts"
9 + },
10 "./pullRequestAnchor": {
11 "types": "./src/pullRequestAnchor.ts",
12 "import": "./src/pullRequestAnchor.ts"
packages/shared/src/png.ts new
+24
@@ -0,0 +1,24 @@
1 +const PNG_SIGNATURE = [137, 80, 78, 71, 13, 10, 26, 10];
2 +
3 +/**
4 + * Returns the first complete PNG in `bytes`, or undefined. `adb exec-out screencap -p` can print
5 + * warnings, such as a foldable's multiple displays, around the image.
6 + */
7 +export const extractPng = <Buffer extends ArrayBufferLike>(
8 + bytes: Uint8Array<Buffer>,
9 +): Uint8Array<Buffer> | undefined => {
10 + const start = bytes.findIndex((_, index) =>
11 + PNG_SIGNATURE.every((value, offset) => bytes[index + offset] === value),
12 + );
13 + if (start < 0) return undefined;
14 + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
15 + for (let chunk = start + PNG_SIGNATURE.length; chunk + 12 <= bytes.length;) {
16 + const end = chunk + 12 + view.getUint32(chunk);
17 + if (end > bytes.length) return undefined;
18 + if (String.fromCharCode(...bytes.subarray(chunk + 4, chunk + 8)) === "IEND") {
19 + return bytes.subarray(start, end);
20 + }
21 + chunk = end;
22 + }
23 + return undefined;
24 +};
scripts/mobile-native-client.test.ts
+24
@@ -1,10 +1,12 @@
1 import * as NodeServices from "@effect/platform-node/NodeServices";
2 import { assert, it } from "@effect/vitest";
3 +import * as HostProcess from "@t3tools/shared/HostProcess";
4 import * as Effect from "effect/Effect";
5 import * as FileSystem from "effect/FileSystem";
6 import * as Path from "effect/Path";
7
8 import {
9 + androidSdk,
10 buildRunTitle,
11 NativeClientError,
12 prebuildAndroid,
@@ -67,6 +69,28 @@ it.layer(NodeServices.layer)("prebuildAndroid", (it) => {
69 );
70 });
71
72 +it.layer(NodeServices.layer)("androidSdk", (it) => {
73 + it.effect("skips a configured SDK path that holds no SDK", () =>
74 + Effect.gen(function* () {
75 + const fs = yield* FileSystem.FileSystem;
76 + const path = yield* Path.Path;
77 + const home = yield* fs.makeTempDirectoryScoped({ prefix: "android-sdk-home-" });
78 + const sdk = path.join(home, "sdk");
79 + yield* fs.makeDirectory(path.join(sdk, "platform-tools"), { recursive: true });
80 +
81 + const resolved = yield* androidSdk.pipe(
82 + Effect.provideService(HostProcess.Environment, {
83 + HOME: home,
84 + ANDROID_HOME: path.join(home, "stale"),
85 + ANDROID_SDK_ROOT: sdk,
86 + }),
87 + );
88 +
89 + assert.strictEqual(resolved, sdk);
90 + }),
91 + );
92 +});
93 +
94 it("finds a Blacksmith build by its run title and reads its state", () => {
95 const fingerprint = "a".repeat(40);
96 const title = buildRunTitle("android", "x86_64", fingerprint);
scripts/mobile-native-client.ts
+26 -4
@@ -149,19 +149,39 @@ const collect = <E>(stream: Stream.Stream<Uint8Array, E>) =>
149 (a, b) => a + b,
150 ),
151 );
152 +/** Gradle needs the SDK location; fall back to Android Studio's default install like the device host does. */
153 +export const androidSdk = Effect.gen(function* () {
154 + const fs = yield* FileSystem.FileSystem;
155 + const path = yield* Path.Path;
156 + const environment = yield* HostProcess.Environment;
157 + const home = environment.HOME ?? environment.USERPROFILE ?? "";
158 + const candidates = [
159 + environment.ANDROID_HOME?.trim(),
160 + environment.ANDROID_SDK_ROOT?.trim(),
161 + path.join(home, "Library", "Android", "sdk"),
162 + path.join(home, "Android", "Sdk"),
163 + path.join(environment.LOCALAPPDATA ?? path.join(home, "AppData", "Local"), "Android", "Sdk"),
164 + ];
165 + for (const sdk of candidates) {
166 + if (!sdk) continue;
167 + const platformTools = path.join(sdk, "platform-tools");
168 + if (yield* fs.exists(platformTools).pipe(Effect.orElseSucceed(() => false))) return sdk;
169 + }
170 + return undefined;
171 +});
172 +
173 export const resolveAdb = Effect.gen(function* () {
174 if (yield* isCommandAvailable("adb")) return "adb";
154 - const environment = yield* HostProcess.Environment;
175 const path = yield* Path.Path;
176 const executable = (yield* HostProcess.Platform) === "win32" ? "adb.exe" : "adb";
157 - for (const sdk of [environment.ANDROID_SDK_ROOT, environment.ANDROID_HOME]) {
158 - if (!sdk) continue;
177 + const sdk = yield* androidSdk;
178 + if (sdk) {
179 const candidate = path.join(sdk, "platform-tools", executable);
180 if (yield* isCommandAvailable(candidate)) return candidate;
181 }
182 return yield* new NativeClientError({
183 message:
164 - "adb was not found on PATH or in ANDROID_SDK_ROOT/ANDROID_HOME. Install Android SDK platform-tools.",
184 + "adb was not found on PATH, in ANDROID_HOME/ANDROID_SDK_ROOT, or in Android Studio's default SDK location. Install Android SDK platform-tools.",
185 });
186 });
187
@@ -185,12 +205,14 @@ const command = Effect.fn("nativeClient.command")(function* (
205 );
206 const log = inherit ? yield* BuildLog : null;
207 const spawn = yield* resolveSpawnCommand(program === "adb" ? yield* resolveAdb : program, args);
208 + const sdk = yield* androidSdk;
209 const child = yield* spawner.spawn(
210 ChildProcess.make(spawn.command, spawn.args, {
211 shell: spawn.shell,
212 cwd: cwd ?? (yield* roots).mobile,
213 env: {
214 ...environment,
215 + ...(sdk ? { ANDROID_HOME: sdk } : {}),
216 APP_VARIANT: "development",
217 MOBILE_VERSION_POLICY: "appVersion",
218 T3CODE_IOS_PERSONAL_TEAM: "0",