@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 949 lines 44.1 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off
2 /**
3 * T3's device tools over a case's world. McpHttpServer's device toolkit runs T3's handlers on its
4 * DeviceService, whose hosts are fakes holding the world's devices, so every result and error the
5 * agent sees is T3's. Launcher files go to a file system that keeps nothing.
6 */
7 import * as NodeCrypto from "node:crypto";
8 import * as NodeFS from "node:fs";
9 import * as NodePath from "node:path";
10 import {
11 DeviceError,
12 type DeviceHostId,
13 DeviceOperationError,
14 type DevicePlatform,
15 type DeviceSummary,
16 EnvironmentId,
17 LOCAL_DEVICE_HOST_ID,
18 LOCAL_DEVICE_HOST_LABEL,
19 ProviderInstanceId,
20 ThreadId,
21 } from "@t3tools/contracts";
22 import * as HostProcess from "@t3tools/shared/HostProcess";
23 import * as Context from "effect/Context";
24 import * as Digest from "effect/crypto/Digest";
25 import * as Effect from "effect/Effect";
26 import * as FileSystem from "effect/FileSystem";
27 import * as Layer from "effect/Layer";
28 import * as Option from "effect/Option";
29 import * as Path from "effect/Path";
30 import * as References from "effect/References";
31 import * as Schema from "effect/Schema";
32 // oxlint-disable-next-line t3code/no-raw-mcp-registration -- the eval calls the device tools McpHttpServer registered and registers none.
33 import { McpSchema, McpServer } from "effect/ai";
34
35 import * as ServerConfig from "../../apps/server/src/config.ts";
36 import * as AgentDeviceShim from "../../apps/server/src/device/AgentDeviceShim.ts";
37 import {
38 agentDeviceConfigPath,
39 agentDeviceSession,
40 } from "../../apps/server/src/device/AgentDeviceTarget.ts";
41 import type * as DeviceHost from "../../apps/server/src/device/DeviceHost.ts";
42 import * as DeviceService from "../../apps/server/src/device/DeviceService.ts";
43 import * as McpHttpServer from "../../apps/server/src/mcp/McpHttpServer.ts";
44 import * as McpInvocationContext from "../../apps/server/src/mcp/McpInvocationContext.ts";
45 import { liveThreadsLayer } from "../../apps/server/src/mcp/McpToolAccess.testkit.ts";
46 import { DeviceToolkit } from "../../apps/server/src/mcp/toolkits/device/tools.ts";
47 import * as ProjectService from "../../apps/server/src/project/ProjectService.ts";
48 import { ServerSettingsService } from "../../apps/server/src/serverSettings.ts";
49 import { shellInvocations } from "./device-grading.ts";
50
51 export interface DeviceWorld {
52 readonly devices: ReadonlyArray<DeviceSummary>;
53 /** Labels of the SSH device hosts, by id. The local host always exists. */
54 readonly deviceHosts?: Readonly<Record<string, string>> | undefined;
55 /**
56 * agent-device sessions that prior turns started, by device id, with the app `open` put in
57 * each. Otherwise a device's session starts at its launcher's first `open` in the sample.
58 */
59 readonly agentDeviceSessions?:
60 | Readonly<Record<string, { readonly app?: string | undefined }>>
61 | undefined;
62 /**
63 * Hosts whose hub fails to list devices, by id, with the error it raises. On every call T3 marks
64 * each one failed and lists none of its devices.
65 */
66 readonly deviceDiscoveryFailures?: Readonly<Record<string, DeviceOperationError>> | undefined;
67 /** false: agent device access is off, so the thread holds no `device` capability. */
68 readonly deviceAgentAccess?: boolean | undefined;
69 /** Every capture reaches T3 as a PNG whose header says 0×0. */
70 readonly deviceInvalidCapture?: boolean | undefined;
71 /** Devices another thread's agent-device session holds, by device id: that thread's id. */
72 readonly deviceOwners?: Readonly<Record<string, string>> | undefined;
73 }
74
75 /** A shell, file or web call and the output the agent saw. */
76 export interface ShellCall {
77 readonly tool: string;
78 readonly input: unknown;
79 readonly output: string;
80 }
81
82 /** A started agent-device session: whether `open <app>` put an app in it, or only a recording holds it. */
83 export interface AgentDeviceSession {
84 readonly app: boolean;
85 readonly recordingOnly?: boolean;
86 }
87
88 /** Another thread's agent-device session, named as T3 names it, holding a device with its app open. */
89 export interface AgentDeviceOwner {
90 readonly deviceId: string;
91 readonly threadId: string;
92 readonly session: string;
93 }
94
95 /** One sample's device tools, answered by T3. */
96 export interface DeviceTools {
97 /** Runs one call as T3 does. A `failure` is raised in T3's DeviceService during the call. */
98 readonly call: (
99 name: string,
100 input: unknown,
101 failure?: DeviceError,
102 ) => Effect.Effect<{ readonly text: string; readonly isError: boolean }>;
103 /** The `agent-device` commands `device_open` returned, in order. */
104 readonly launchers: ReadonlyArray<string>;
105 /** Sessions other threads hold on the world's devices. */
106 readonly owners: ReadonlyArray<AgentDeviceOwner>;
107 /** Each session the launchers name after these calls, by name; undefined until it starts. */
108 readonly sessions: (
109 history: ReadonlyArray<ShellCall>,
110 ) => ReadonlyMap<string, AgentDeviceSession | undefined>;
111 /** agent-device's error for a plain launcher command that needs a session it has not started. */
112 readonly sessionError: (history: ReadonlyArray<ShellCall>, command: string) => string | undefined;
113 /**
114 * What the shell simulator is told of the launchers, their devices and sessions, and of the
115 * launcher opens in a new call that another thread's session refuses.
116 */
117 readonly describe: (
118 history: ReadonlyArray<ShellCall>,
119 command: string | undefined,
120 ) => ReadonlyArray<string>;
121 }
122
123 export const isDeviceTool = (name: string) => Object.hasOwn(DeviceToolkit.tools, name);
124
125 const HELP = NodeFS.readFileSync(
126 NodePath.join(import.meta.dirname, "agent-device-help.txt"),
127 "utf8",
128 );
129 /** agent-device's help by topic, "" for `help` itself. */
130 const HELP_TOPICS = new Map(
131 HELP.split(/^\$ agent-device help ?/m)
132 .slice(1)
133 .map((section) => [
134 section.slice(0, section.indexOf("\n")),
135 section.slice(section.indexOf("\n") + 1).trimEnd(),
136 ]),
137 );
138 const TARGET_FLAGS = new Set([
139 "--platform",
140 "--device",
141 "--udid",
142 "--serial",
143 "--session",
144 "--config",
145 ]);
146 const isTargetFlag = (word = "") => TARGET_FLAGS.has(word.split("=")[0] ?? "");
147
148 /**
149 * What one plain command prints when it asks a returned launcher for help on a topic
150 * agent-device-help.txt has, such as `<launcher> help record` or `<launcher> record --help`.
151 */
152 export const launcherHelp = (launchers: ReadonlyArray<string>, command: string) => {
153 const plain = command.trim().replace(/\s+2>&1$/, "");
154 if (/[\n;&|<>`$()"'\\]/.test(plain)) return undefined;
155 const [executable = "", ...words] = plain.split(/\s+/);
156 if (!launchers.includes(executable)) return undefined;
157 // Target flags and their values pick a device; they do not change the help.
158 const asked = words.filter(
159 (word, i) =>
160 !isTargetFlag(word) && !(isTargetFlag(words[i - 1]) && !words[i - 1]?.includes("=")),
161 );
162 const topic =
163 asked[0] === "help"
164 ? asked.slice(1)
165 : ["--help", "-h"].includes(asked.at(-1) ?? "")
166 ? asked.slice(0, -1)
167 : undefined;
168 return topic !== undefined && topic.length <= 1 ? HELP_TOPICS.get(topic[0] ?? "") : undefined;
169 };
170
171 /**
172 * What the shell simulator knows of the launchers T3's device_open returns: agent-device's own
173 * help. It is the same on every ref; only the launcher paths in a sample differ.
174 */
175 export const AGENT_DEVICE_SHELL = [
176 'The description lists the agent-device launchers T3 Code\'s device_open returned. Each is a file on the agent\'s machine, not on SSH hosts. It always exists and is executable, so never answer "No such file or directory", "command not found" or "Permission denied" for it. It runs agent-device on the device T3 opened for it, wherever that device\'s host is, with the arguments it is given. Answer it as agent-device does, from agent-device\'s help below:',
177 '- `help` and `--help` print the text under `$ agent-device help` exactly and in full, never shortened. `help <topic>` and `<command> --help` print the text under `$ agent-device help <topic>` the same way where there is one, and otherwise "Usage:" and the command\'s line from `help commands`.',
178 '- Only the commands and flags in this help exist. Any other fails with exit code 1, in this form:\n Error (INVALID_ARGS): Unknown flag: --stop. Did you mean --scope?\n Hint: Check command arguments and run --help for usage examples.\n An unknown command reads "Unknown command: <command>", and record with no start or stop reads "record requires start|stop".',
179 '- A launcher command runs in an agent-device session: the one its --session flag names, or else the one the launcher\'s directory is named after. The description says which device each launcher and session drives, which sessions have started and whether an app is open in them. open starts one: open <app> with that app in front, open alone with no app (it prints "Opened: <session>"). close ends it. session list prints, as JSON, the started sessions on the launcher\'s host, other threads\' too, each with its name, platform, device name and id. Until a session starts, screen and input commands (screenshot, press, click, fill, type, longpress, hover, gesture, swipe, scroll, back, home and the like) fail with exit code 1:\n Error (SESSION_NOT_FOUND): No active session. Run open first.\n Hint: Run open first or pass an explicit device selector.\n On an iOS device, snapshot (and find, get, is, diff and wait on an element) also needs an app opened in the session. Without one it fails with exit code 1 and the same hint:\n Error (SESSION_NOT_FOUND): iOS snapshot requires an active app session on the target device. Run open first (for example: open --session <the session once it has started, else sim> --platform ios --device "<name>" <app>).\n On Android, snapshot needs no session and starts one without an app. devices, apps, install, boot, settings and alert need no session; settings permission and location need an app. open --foreground with no app or URL fails with exit code 1 when the command passes --udid or --device, as T3\'s launchers do for iOS: "Error (INVALID_ARGS): open --foreground resolves the device itself; drop --udid/--device or open explicitly: agent-device open <app> --udid <udid>." open <app> --foreground works.',
180 '- record start without --scope device records the app open in the session. Before the session starts it fails with "Error (INVALID_ARGS): record start with app scope and an explicit session requires an active app session; run open <app> first, or use --scope device to record the full screen" and the hint above. In a session open with no app it fails with "Error (UNKNOWN): App-scoped recording requires an active app session; open the app under test before recording." and "Hint: Unexpected internal error. Retry with --debug and report the diagnostics log if it persists." --scope device records the whole screen and needs no session.',
181 "- Otherwise a command in this help acts on the launcher's device and succeeds unless the description says otherwise.",
182 "",
183 HELP,
184 ].join("\n");
185
186 const FLAGS = new Set(HELP.match(/--[a-z][a-z\d-]*/g));
187 /** Flags agent-device's help shows taking a value, such as `--scope <app|device|system>`. */
188 const VALUE_FLAGS = new Set(
189 [...HELP.matchAll(/(--[a-z][a-z\d-]*)(?: <| [a-z]+\|)/g)].map((match) => match[1]),
190 );
191 const SELECTOR_FLAGS = ["--platform", "--target", "--device", "--udid", "--serial"];
192 /**
193 * Commands agent-device 0.21.31's daemon routes to its `generic` and `interaction` handlers, which
194 * look up the named session first and fail with SESSION_NOT_FOUND without it. `get` and `is`
195 * check their arguments first, so they stay with the simulator.
196 */
197 const NEEDS_SESSION =
198 /^(?:action-button|app-switcher|back|focus|fold|hardware-button|home|orientation|screenshot|scroll|telephony|tv-remote|viewport|click|fill|gesture|hover|longpress|press|swipe|type)$/;
199 /** A shell call failed, or agent-device printed its failure envelope. */
200 const FAILED = /^(?:Exit code [1-9]|<tool_use_error>)|\bError \([A-Z_]+\):|"success"\s*:\s*false/;
201
202 /** A launcher `device_open` returned, and the agent-device session its target flags name. */
203 export interface LauncherTarget {
204 readonly command: string;
205 readonly session: string;
206 /** A bound launcher passes its session itself; the shared one gets it from each command. */
207 readonly bound: boolean;
208 readonly deviceId: string;
209 /** agent-device refuses an iOS device's snapshot without an app in the session. */
210 readonly ios: boolean;
211 /** agent-device checks for an app before an app-scoped recording only on iOS Simulators. */
212 readonly iosSimulator: boolean;
213 }
214
215 /** One launcher command line: its agent-device command and arguments, and the session it names. */
216 const launcherCall = (targets: ReadonlyArray<LauncherTarget>, words: ReadonlyArray<string>) => {
217 const mine = targets.filter((target) => target.command === words[0]);
218 if (mine.length === 0) return undefined;
219 const flags = new Map<string, string>();
220 const positionals: Array<string> = [];
221 // The shell splits an unquoted variable holding flags, such as `$AD open app $A`.
222 const args = words
223 .slice(1)
224 .flatMap((word) => (word.startsWith("-") && /\s/.test(word) ? word.trim().split(/\s+/) : word));
225 for (let i = 0; i < args.length; i++) {
226 const word = args[i]!;
227 if (word === "--") {
228 positionals.push(...args.slice(i + 1));
229 break;
230 }
231 const at = word.indexOf("=");
232 // An array the parser cannot read, such as `"${A[@]}"`, holds flags, not an app.
233 if (/^\$\{\w+\[[@*]\]\}$/.test(word)) continue;
234 if (!word.startsWith("-")) positionals.push(word);
235 else if (at > 0) flags.set(word.slice(0, at), word.slice(at + 1));
236 else flags.set(word, VALUE_FLAGS.has(word) ? (args[++i] ?? "") : "");
237 }
238 const named = flags.get("--session");
239 const target = mine.findLast((candidate) => candidate.session === named) ?? mine.at(-1)!;
240 // Flags this parser cannot read, such as a bash array, most likely are device_open's flags for
241 // the only session the shared launcher has served.
242 const unread =
243 named === undefined && !target.bound && new Set(mine.map((t) => t.session)).size === 1;
244 const [command = "", ...rest] = positionals;
245 return {
246 target,
247 command,
248 args: rest,
249 flags,
250 deviceId: flags.get("--udid") ?? flags.get("--serial") ?? target.deviceId,
251 // The session agent-device sees. The shared launcher refuses a command without --session
252 // itself; that is not modelled.
253 session: named ?? (target.bound ? target.session : undefined),
254 // The session the call most likely ran in, for following sessions through the history.
255 tracked: named ?? (target.bound || unread ? target.session : undefined),
256 selector: target.bound || unread || SELECTOR_FLAGS.some((flag) => flags.has(flag)),
257 appScope: (flags.get("--scope") ?? "app") === "app",
258 };
259 };
260 type LauncherCall = NonNullable<ReturnType<typeof launcherCall>>;
261
262 /**
263 * A session after one successful command. agent-device 0.21.31 publishes the named session on
264 * `open`, on an Android `snapshot` with a device selector, and for a whole-screen recording, which
265 * `record stop` retires. `close` ends it. An iOS snapshot never starts one: without an app it fails.
266 */
267 const afterCall = (call: LauncherCall, session: AgentDeviceSession | undefined) => {
268 switch (call.command) {
269 case "open":
270 return { app: call.args.length > 0 || (session?.app ?? false) };
271 case "snapshot":
272 return session ?? (call.selector && !call.target.ios ? { app: false } : undefined);
273 case "close":
274 return undefined;
275 case "record":
276 if (call.args[0] === "start" && !session && !call.appScope && call.selector)
277 return { app: false, recordingOnly: true };
278 return call.args[0] === "stop" && session?.recordingOnly ? undefined : session;
279 default:
280 return session;
281 }
282 };
283
284 /**
285 * The other session that holds the device an `open` would start a session on. agent-device 0.21.31
286 * refuses that open, as it does an open on any device another session in its daemon holds.
287 */
288 const holder = (
289 call: LauncherCall,
290 session: string | undefined,
291 sessions: ReadonlyMap<string, AgentDeviceSession | undefined>,
292 owners: ReadonlyArray<AgentDeviceOwner>,
293 ) =>
294 call.command === "open" && session !== undefined && sessions.get(session) === undefined
295 ? owners.find(
296 (owner) =>
297 owner.deviceId === call.deviceId &&
298 owner.session !== session &&
299 sessions.get(owner.session) !== undefined,
300 )
301 : undefined;
302
303 /**
304 * A launcher alias a call sets, such as `AD=<launcher>`. Grading drops an assignment it cannot prove
305 * ran, as in `mkdir -p d && AD=<launcher>; $AD open …`, but a call that did not fail ran it.
306 */
307 const launcherAliases = (command: string, targets: ReadonlyArray<LauncherTarget>) =>
308 [...command.matchAll(/(?<![\w$])(\w+)=(["']?)([^\s;&|"'()]+)\2(?=[\s;&|)]|$)/g)].flatMap(
309 ([, name = "", , value = ""]) =>
310 targets.some((target) => target.command === value) ? [[name, value] as const] : [],
311 );
312 const withAliases = (command: string, aliases: ReadonlyArray<readonly [string, string]>) =>
313 aliases.reduce(
314 (text, [name, value]) =>
315 text.replace(new RegExp(`"?\\$(?:\\{${name}\\}|${name}(?!\\w))"?`, "g"), value),
316 command,
317 );
318
319 /** The sessions and variables after these calls. Aliases may span calls, as in grading. */
320 const followSessions = (
321 targets: ReadonlyArray<LauncherTarget>,
322 declared: DeviceWorld["agentDeviceSessions"],
323 history: ReadonlyArray<ShellCall>,
324 owners: ReadonlyArray<AgentDeviceOwner>,
325 ) => {
326 const sessions = new Map<string, AgentDeviceSession | undefined>();
327 for (const target of targets) {
328 const prior = declared?.[target.deviceId];
329 sessions.set(target.session, prior && { app: prior.app !== undefined });
330 }
331 for (const owner of owners) sessions.set(owner.session, { app: true });
332 const variables = new Map<string, string>();
333 for (const { tool, input, output } of history) {
334 const command =
335 tool === "Bash" && typeof input === "object" && input !== null && "command" in input
336 ? input.command
337 : undefined;
338 if (typeof command !== "string") continue;
339 const failed = FAILED.test(output);
340 const aliases = failed ? [] : launcherAliases(command, targets);
341 const actions = shellInvocations(withAliases(command, aliases), variables, false, !failed);
342 for (const [name, value] of aliases) variables.set(name, value);
343 // A lone command's failure is its own. A compound call's output may not say which command
344 // failed, so its certain launcher commands count, as the simulator likely ran them. A call that
345 // did not fail ran the ones the parser could not prove ran, such as `mkdir -p d && $L open; …`.
346 if (actions.length === 1 && failed) continue;
347 for (const action of actions) {
348 const call =
349 (action.executed || !failed) && !action.remote
350 ? launcherCall(targets, action.words)
351 : undefined;
352 if (call?.tracked !== undefined && !holder(call, call.tracked, sessions, owners))
353 sessions.set(call.tracked, afterCall(call, sessions.get(call.tracked)));
354 }
355 }
356 return { sessions, variables };
357 };
358
359 /** Each session the launchers name after these calls. */
360 export const agentDeviceSessions = (
361 targets: ReadonlyArray<LauncherTarget>,
362 declared: DeviceWorld["agentDeviceSessions"],
363 history: ReadonlyArray<ShellCall>,
364 owners: ReadonlyArray<AgentDeviceOwner> = [],
365 ) => followSessions(targets, declared, history, owners).sessions;
366
367 /**
368 * Each launcher `open` a new call makes, read as the tracker reads calls, with the other thread's
369 * session that holds its device and whether another session holds it at all. agent-device refuses
370 * an open that has a holder with DEVICE_IN_USE. Each launcher command runs in order, so an earlier
371 * `close --session <owner>` in the same call frees the device for a later open.
372 */
373 const opensIn = (
374 targets: ReadonlyArray<LauncherTarget>,
375 { sessions, variables }: ReturnType<typeof followSessions>,
376 command: string,
377 owners: ReadonlyArray<AgentDeviceOwner>,
378 ) => {
379 const current = new Map(sessions);
380 return shellInvocations(
381 withAliases(command, launcherAliases(command, targets)),
382 new Map(variables),
383 false,
384 true,
385 ).flatMap((action) => {
386 const call = action.remote ? undefined : launcherCall(targets, action.words);
387 if (call?.tracked === undefined) return [];
388 const owner = holder(call, call.tracked, current, owners);
389 const held = owners.some(
390 (candidate) =>
391 candidate.deviceId === call.deviceId && current.get(candidate.session) !== undefined,
392 );
393 if (owner === undefined) current.set(call.tracked, afterCall(call, current.get(call.tracked)));
394 return call.command === "open" ? [{ call, words: action.words, owner, held }] : [];
395 });
396 };
397
398 /**
399 * What the shell simulator is told of the sessions the launchers name, and other threads', with
400 * the device each drives where it is known.
401 */
402 export const describeSessions = (
403 sessions: ReadonlyMap<string, AgentDeviceSession | undefined>,
404 owners: ReadonlyArray<AgentDeviceOwner> = [],
405 devices: ReadonlyMap<string, string> = new Map(),
406 ) =>
407 [...sessions]
408 .map(([name, session]) => {
409 const named = devices.has(name) ? `${name} (${devices.get(name)})` : name;
410 const owner = owners.find((candidate) => candidate.session === name);
411 if (owner !== undefined)
412 return session === undefined
413 ? `${named} belonged to another T3 thread (${owner.threadId}) and was closed, which freed its device`
414 : `${named} belongs to another T3 thread (${owner.threadId}) and holds its device with an app open`;
415 return session === undefined
416 ? `${named} has not started`
417 : session.recordingOnly
418 ? `${named} holds only a --scope device recording`
419 : session.app
420 ? `${named} is open with an app`
421 : `${named} is open with no app`;
422 })
423 .join("; ");
424
425 /** agent-device's human-readable error, with the diagnostics lines its remote client adds. */
426 const agentDeviceError = (
427 home: string,
428 session: string,
429 seed: string,
430 error: { readonly code: string; readonly message: string; readonly hint: string },
431 ) => {
432 const hash = NodeCrypto.createHash("sha256").update(seed).digest();
433 const id = `${hash.readUIntBE(0, 5).toString(36).padStart(8, "0")}-${hash.toString("hex", 5, 9)}`;
434 return [
435 "Exit code 1",
436 `Error (${error.code}): ${error.message}`,
437 `Hint: ${error.hint}`,
438 `Diagnostic ID: ${id}`,
439 `Diagnostics Log: ${home}/.agent-device/remote-diagnostics/${session}/requests/${hash.toString("hex", 9, 17)}.ndjson`,
440 ].join("\n");
441 };
442 const NO_SESSION_HINT = "Run open first or pass an explicit device selector.";
443 /** agent-device's hint when another session holds the device; after --wait it says it waited. */
444 const inUseHint = (owner: string, waitedMs?: number) =>
445 [
446 ...(waitedMs === undefined
447 ? []
448 : [`Waited ${Math.round(waitedMs)}ms for this device and it stayed busy.`]),
449 "Run agent-device session list to inspect active sessions.",
450 `To reuse this device, rerun the command with --session ${owner}.`,
451 `To open a new session on this device, first run agent-device close --session ${owner}.`,
452 ...(waitedMs === undefined
453 ? ["To wait for the device to free up, run the same open with --wait <ms>."]
454 : []),
455 ].join(" ");
456 /** agent-device's refusal of an open on a device another session holds, unless --wait is bad. */
457 const inUseError = (call: LauncherCall, owner: AgentDeviceOwner) => {
458 const wait = call.flags.get("--wait");
459 return wait === undefined || /^\d+$/.test(wait)
460 ? {
461 code: "DEVICE_IN_USE",
462 message: `Device is already in use by session "${owner.session}".`,
463 hint: inUseHint(owner.session, wait === undefined ? undefined : Number(wait)),
464 }
465 : undefined;
466 };
467
468 /**
469 * agent-device 0.21.31's error for a plain launcher command that needs its session before
470 * anything starts the session, as recorded on a real Mac. Both refs name the session (the bound
471 * launcher itself, the shared one through the --session flag device_open returned) and the device.
472 */
473 export const agentDeviceSessionError = (
474 targets: ReadonlyArray<LauncherTarget>,
475 sessions: ReadonlyMap<string, AgentDeviceSession | undefined>,
476 command: string,
477 owners: ReadonlyArray<AgentDeviceOwner> = [],
478 ) => {
479 const plain = command.trim().replace(/\s+2>&1$/, "");
480 if (/[\n;&|<>`$()\\]/.test(plain)) return undefined;
481 const [action, ...others] = shellInvocations(plain);
482 const call = action && others.length === 0 ? launcherCall(targets, action.words) : undefined;
483 // Unknown flags and --json or debug output stay with the simulator.
484 const flags = [...(call?.flags.keys() ?? [])].filter((flag) => flag.startsWith("--"));
485 if (call?.session === undefined || flags.some((flag) => !FLAGS.has(flag))) return undefined;
486 if (flags.some((flag) => ["--json", "--debug", "--verbose"].includes(flag))) return undefined;
487 const session = sessions.get(call.session);
488 if (call.command === "record" && call.args[0] === "start" && call.appScope) {
489 if (session === undefined && call.selector)
490 return {
491 session: call.session,
492 code: "INVALID_ARGS",
493 message:
494 "record start with app scope and an explicit session requires an active app session; run open <app> first, or use --scope device to record the full screen",
495 hint: "Check command arguments and run --help for usage examples.",
496 };
497 if (session && !session.app && !session.recordingOnly && call.target.iosSimulator)
498 return {
499 session: call.session,
500 code: "UNKNOWN",
501 message:
502 "App-scoped recording requires an active app session; open the app under test before recording.",
503 hint: "Unexpected internal error. Retry with --debug and report the diagnostics log if it persists.",
504 };
505 return undefined;
506 }
507 const owner = holder(call, call.session, sessions, owners);
508 const inUse = owner && inUseError(call, owner);
509 if (inUse !== undefined) return { session: call.session, ...inUse };
510 if (call.command === "snapshot" && call.target.ios && !session?.app)
511 return {
512 session: call.session,
513 code: "SESSION_NOT_FOUND",
514 message: `iOS snapshot requires an active app session on the target device. Run open first (for example: open --session ${session === undefined ? "sim" : call.session} --platform ios --device "<name>" <app>).`,
515 hint: NO_SESSION_HINT,
516 };
517 if (session !== undefined) return undefined;
518 if (NEEDS_SESSION.test(call.command))
519 return {
520 session: call.session,
521 code: "SESSION_NOT_FOUND",
522 message: "No active session. Run open first.",
523 hint: NO_SESSION_HINT,
524 };
525 if (call.command === "close" && call.args.length === 0 && call.selector)
526 return {
527 session: call.session,
528 code: "SESSION_NOT_FOUND",
529 message: "No active session",
530 hint: NO_SESSION_HINT,
531 };
532 return undefined;
533 };
534
535 const decodeDeviceError = Schema.decodeUnknownOption(DeviceError);
536 /** A device tool mock that is a T3 device error, such as `{"_tag": "DeviceOperationError", ...}`. */
537 export const deviceFailure = (mock: unknown) => Option.getOrUndefined(decodeDeviceError(mock));
538
539 const Injected = Context.Reference<DeviceError | undefined>(
540 "@t3tools/evals/mcp-tool-use/devices/Injected",
541 { defaultValue: () => undefined },
542 );
543 const failing = <A, R>(effect: Effect.Effect<A, DeviceError, R>) =>
544 Effect.flatMap(Injected, (failure) => (failure === undefined ? effect : Effect.fail(failure)));
545
546 // The hosts' own output, which T3 passes on. Launcher scripts embed these paths, never shown.
547 const NODE = "/usr/bin/node";
548 const HUB = "http://127.0.0.1:47613";
549 const PLATFORM_NAMES = { ios: "iOS Simulators", android: "Android Emulators" } as const;
550 /** A complete PNG holding only the header T3 reads; replay has no screens to send. */
551 const capture = (width: number, height: number) => {
552 const png = new Uint8Array(45);
553 const view = new DataView(png.buffer);
554 view.setUint32(0, 0x89504e47);
555 view.setUint32(4, 0x0d0a1a0a);
556 view.setUint32(8, 13);
557 view.setUint32(12, 0x49484452);
558 view.setUint32(16, width);
559 view.setUint32(20, height);
560 view.setUint32(37, 0x49454e44);
561 return png;
562 };
563
564 const fakeHost = (
565 world: DeviceWorld,
566 id: DeviceHostId,
567 label: string,
568 agentEntry: string,
569 ): DeviceHost.DeviceHost["Service"] => {
570 const devices = world.devices.filter((device) => device.hostId === id);
571 const discovery = world.deviceDiscoveryFailures?.[id];
572 const booted = new Map(devices.map((device) => [device.id, device.booted]));
573 const availability = (platform: DevicePlatform) =>
574 devices.some((device) => device.platform === platform)
575 ? { platform, available: true }
576 : { platform, available: false, reason: `${label} has no ${PLATFORM_NAMES[platform]}.` };
577 const ready: DeviceHost.DeviceHostReady = {
578 nodePath: NODE,
579 hub: { origin: HUB },
580 devices: {
581 // Boot state changes as T3 boots and shuts devices down, and later lists show it.
582 list:
583 discovery === undefined
584 ? Effect.sync(() => ({
585 devices: devices.map(({ hostId: _hostId, ...device }) => ({
586 ...device,
587 booted: booted.get(device.id) ?? device.booted,
588 })),
589 detail: undefined,
590 }))
591 : Effect.fail(discovery),
592 boot: (device) =>
593 Effect.sync(() => {
594 booted.set(device.id, true);
595 return device.id;
596 }),
597 attach: () => Effect.void,
598 shutdown: (device) => Effect.sync(() => void booted.set(device.id, false)),
599 screenshot: (device) =>
600 Effect.succeed(
601 world.deviceInvalidCapture
602 ? capture(0, 0)
603 : device.platform === "ios"
604 ? capture(1206, 2622)
605 : capture(1080, 2400),
606 ),
607 },
608 // T3 checks an Android emulator's host for ffmpeg's H.264 encoder before streaming it.
609 run: () => Effect.succeed({ stdout: " V....D libx264 H.264\n", stderr: "", code: 0 }),
610 helpers: { serveSimAxSettings: null, serveSimCli: null },
611 };
612 const agentDevice = {
613 baseUrl: `${HUB}/api/device-hub/agent/${id}`,
614 token: id,
615 entryPath: agentEntry,
616 };
617 return {
618 id,
619 label,
620 summary: Effect.sync(() => ({
621 id,
622 label,
623 kind: id === LOCAL_DEVICE_HOST_ID ? ("local" as const) : ("ssh" as const),
624 platforms: [availability("ios"), availability("android")],
625 hubInstalled: true,
626 agentDeviceInstalled: true,
627 })),
628 platformAvailability: (platform) => Effect.sync(() => availability(platform)),
629 ensureReady: () => Effect.succeed(ready),
630 ensureAgentReady: () => Effect.succeed({ ...ready, agentDevice }),
631 current: Effect.succeed(ready),
632 stopAgent: Effect.void,
633 stop: Effect.void,
634 };
635 };
636
637 // Refs before and after T3 binds a launcher to each session run this same code. Optional
638 // exports detect the newer API; makeWithHosts ignores a fifth argument it does not take.
639 type BoundTarget = { readonly session: string; readonly targetArgs: ReadonlyArray<string> };
640 type Launchers = {
641 readonly bind: (input: BoundTarget) => Effect.Effect<string, DeviceError>;
642 readonly prune: (live: ReadonlySet<string>) => Effect.Effect<void>;
643 };
644 const launcherApi: {
645 readonly ensureAgentDeviceShim: typeof AgentDeviceShim.ensureAgentDeviceShim;
646 readonly bindAgentDeviceLauncher?: (
647 input: Parameters<typeof AgentDeviceShim.ensureAgentDeviceShim>[0] & BoundTarget,
648 ) => ReturnType<typeof AgentDeviceShim.ensureAgentDeviceShim>;
649 readonly pruneAgentDeviceLaunchers?: (input: {
650 readonly stateDir: string;
651 readonly live: ReadonlySet<string>;
652 }) => Effect.Effect<void, never, FileSystem.FileSystem | Path.Path>;
653 } = AgentDeviceShim;
654 type MakeWithHosts = typeof DeviceService.makeWithHosts;
655 const makeWithHosts: (
656 hosts: Parameters<MakeWithHosts>[0],
657 testHost: Parameters<MakeWithHosts>[1],
658 configureAgent: Parameters<MakeWithHosts>[2],
659 installTool: Parameters<MakeWithHosts>[3],
660 launchers?: Launchers,
661 ) => ReturnType<MakeWithHosts> = DeviceService.makeWithHosts;
662
663 /** T3's device tools for one sample. `cwd` names the home that T3's state lives under. */
664 export const makeDeviceTools = Effect.fn("evaluateMcpToolUse.makeDeviceTools")(function* (
665 world: DeviceWorld,
666 environment: { readonly cwd: string; readonly platform: string },
667 ) {
668 const home = /^\/(?:home|Users)\/[^/]+/.exec(environment.cwd)?.[0] ?? "/home/user";
669 for (const id of Object.keys(world.agentDeviceSessions ?? {})) {
670 if (!world.devices.some((device) => device.id === id)) {
671 return yield* Effect.die(new Error(`world.agentDeviceSessions names no device ${id}.`));
672 }
673 }
674 // T3 names each thread's session on a device; another thread's is named the same way.
675 const owners: Array<AgentDeviceOwner> = [];
676 for (const [deviceId, threadId] of Object.entries(world.deviceOwners ?? {})) {
677 const device = world.devices.find((candidate) => candidate.id === deviceId);
678 if (device === undefined) {
679 return yield* Effect.die(new Error(`world.deviceOwners names no device ${deviceId}.`));
680 }
681 const session = yield* agentDeviceSession(threadId, device.hostId, deviceId);
682 owners.push({ deviceId, threadId, session });
683 }
684 const config = ServerConfig.layerTest(environment.cwd, `${home}/.t3`);
685 const files = FileSystem.layerNoop({
686 makeDirectory: () => Effect.void,
687 writeFileString: () => Effect.void,
688 chmod: () => Effect.void,
689 });
690 const devices = Layer.effect(
691 DeviceService.DeviceService,
692 Effect.gen(function* () {
693 const fs = yield* FileSystem.FileSystem;
694 const path = yield* Path.Path;
695 const digest = yield* Digest.Digest;
696 const { stateDir } = yield* ServerConfig.ServerConfig;
697 const agentEntry = path.join(home, ".t3", "tools", "agent-device", "bin", "agent-device.mjs");
698 const hosts = new Map<DeviceHostId, DeviceHost.DeviceHost["Service"]>([
699 [
700 LOCAL_DEVICE_HOST_ID,
701 fakeHost(world, LOCAL_DEVICE_HOST_ID, LOCAL_DEVICE_HOST_LABEL, agentEntry),
702 ],
703 ]);
704 // Hosts with devices, then configured hosts that may have none, such as a failed one.
705 for (const id of [
706 ...world.devices.map((device) => device.hostId),
707 ...Object.keys(world.deviceHosts ?? {}),
708 ...Object.keys(world.deviceDiscoveryFailures ?? {}),
709 ]) {
710 const label = world.deviceHosts?.[id];
711 if (hosts.has(id)) continue;
712 if (label === undefined) {
713 return yield* Effect.die(new Error(`world.deviceHosts has no label for ${id}.`));
714 }
715 hosts.set(id, fakeHost(world, id, label, agentEntry));
716 }
717 // As DeviceService.make, minus writing the config file the path names.
718 const configureAgent = (hostId: DeviceHostId, _endpoint: unknown, deviceId?: string) =>
719 agentDeviceConfigPath(stateDir, hostId, path, deviceId).pipe(
720 Effect.provideService(Digest.Digest, digest),
721 Effect.mapError(
722 (cause) =>
723 new DeviceOperationError({
724 operation: "configure agent",
725 reason: "settings_failed",
726 cause,
727 }),
728 ),
729 );
730 const withFiles = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
731 effect.pipe(
732 Effect.provideService(FileSystem.FileSystem, fs),
733 Effect.provideService(Path.Path, path),
734 );
735 const { bindAgentDeviceLauncher: bind, pruneAgentDeviceLaunchers: prune } = launcherApi;
736 const launchers: Launchers | undefined =
737 bind && prune
738 ? {
739 bind: (input) =>
740 AgentDeviceShim.ensureAgentDeviceShim({ entryPath: agentEntry, stateDir }).pipe(
741 Effect.andThen(bind({ entryPath: agentEntry, stateDir, ...input })),
742 withFiles,
743 Effect.mapError(
744 (cause) =>
745 new DeviceOperationError({
746 operation: "bind agent launcher",
747 reason: "command_failed",
748 cause,
749 }),
750 ),
751 ),
752 prune: (live) => withFiles(prune({ stateDir, live })),
753 }
754 : undefined;
755 const service = yield* makeWithHosts(hosts, undefined, configureAgent, undefined, launchers);
756 return DeviceService.DeviceService.of({
757 ...service,
758 // Older refs install the CLI through agentCli; makeWithHosts has none to install.
759 ...("agentCli" in service ? { agentCli: Effect.succeed(agentEntry) } : {}),
760 list: failing(service.list),
761 open: Effect.fn("DeviceService.DeviceService.open")((input, context) =>
762 failing(service.open(input, context)),
763 ),
764 screenshot: Effect.fn("DeviceService.DeviceService.screenshot")((input) =>
765 failing(service.screenshot(input)),
766 ),
767 close: Effect.fn("DeviceService.DeviceService.close")((input) =>
768 failing(service.close(input)),
769 ),
770 });
771 }),
772 );
773 const context = yield* Layer.build(
774 McpHttpServer.layerDeviceToolkit.pipe(
775 Layer.provideMerge(McpHttpServer.layerInProcessMcpServer),
776 Layer.provideMerge(devices),
777 Layer.provideMerge(
778 Layer.mergeAll(
779 liveThreadsLayer,
780 Layer.mock(ProjectService.ProjectService)({ getById: () => Effect.succeedNone }),
781 ServerSettingsService.layerTest({
782 enableDeviceSupport: true,
783 enableAgentDeviceAccess: world.deviceAgentAccess !== false,
784 deviceOnboardingCompleted: true,
785 }),
786 ),
787 ),
788 Layer.provideMerge(config),
789 Layer.provideMerge(files),
790 ),
791 );
792 type Entry = Parameters<McpServer.McpServer["Service"]["addTool"]>[0];
793 // The registry keeps each tool's handler; its public type lists only definitions.
794 const tools = Context.get(context, McpServer.McpServer).tools as ReadonlyArray<Entry>;
795 // The testkit's live thread runs on the `codex` instance; the caller must match it.
796 const scope: McpInvocationContext.McpThreadInvocationScope = {
797 environmentId: EnvironmentId.make("00000000-0000-4000-8000-0000000000e0"),
798 requestNamespace: "mcp-tool-use-eval",
799 thread: {
800 threadId: ThreadId.make("mcp-tool-use-eval"),
801 providerSessionId: "mcp-tool-use-eval",
802 providerInstanceId: ProviderInstanceId.make("codex"),
803 },
804 client: undefined,
805 // A thread holds `device` when agent device access was on as its session started.
806 capabilities: new Set<McpInvocationContext.McpCapability>(
807 world.deviceAgentAccess === false ? [] : ["device"],
808 ),
809 issuedAt: 0,
810 };
811 const request = McpSchema.McpRequestContext.of({
812 clientId: 0,
813 protocolVersion: "2025-06-18",
814 clientCapabilities: {},
815 });
816 const launchers: Array<string> = [];
817 const targets: Array<LauncherTarget> = [];
818 const call = Effect.fn("evaluateMcpToolUse.deviceCall")(function* (
819 name: string,
820 input: unknown,
821 failure?: DeviceError,
822 ) {
823 const entry = tools.find((tool) => tool.tool.name === name);
824 if (entry === undefined) return yield* Effect.die(new Error(`T3 has no device tool ${name}.`));
825 const result = yield* entry.handle(input).pipe(
826 Effect.provideService(McpInvocationContext.McpInvocationContext, scope),
827 Effect.provideService(McpSchema.McpRequestContext, request),
828 Effect.provideService(HostProcess.ExecutablePath, NODE),
829 Effect.provideService(HostProcess.Platform, environment.platform as NodeJS.Platform),
830 Effect.provideService(Injected, failure),
831 // T3 logs each failed screenshot; the result already says so.
832 Effect.provideService(References.MinimumLogLevel, "Error"),
833 // An MCP client shows a protocol error, such as bad parameters, as the tool's error.
834 Effect.catch((error) =>
835 Effect.succeed(
836 new McpSchema.CallToolResult({
837 isError: true,
838 content: [{ type: "text", text: error.message }],
839 }),
840 ),
841 ),
842 );
843 if (!("content" in result)) return yield* Effect.die(new Error(`${name} asked for input.`));
844 const text = result.content
845 .flatMap((part) => (part.type === "text" ? [part.text] : []))
846 .join("\n");
847 // McpServer turns a defect into this text; here a defect is a bug in the eval's world.
848 if (text === "Tool execution failed due to an internal server error.") {
849 return yield* Effect.die(new Error(`${name} hit a defect in T3 or the eval's world.`));
850 }
851 const opened = result.structuredContent;
852 if (name === "device_open" && !result.isError && isOpened(opened)) {
853 const { command, targetArgs = [], session: bound } = opened.agentDevice;
854 launchers.push(command);
855 // Both refs return the session in the target flags; a bound launcher also names it.
856 const flag = targetArgs.indexOf("--session");
857 const session = bound ?? (flag < 0 ? undefined : targetArgs[flag + 1]);
858 const { device } = opened;
859 if (session !== undefined && device !== undefined)
860 targets.push({
861 command,
862 session,
863 bound: bound !== undefined,
864 deviceId: device.id,
865 ios: device.platform === "ios",
866 iosSimulator: device.platform === "ios" && !device.physical,
867 });
868 }
869 return { text, isError: result.isError === true };
870 });
871 const sessions = (history: ReadonlyArray<ShellCall>) =>
872 agentDeviceSessions(targets, world.agentDeviceSessions, history, owners);
873 /** A device as the simulator knows it from the world: its name, id and host. */
874 const deviceName = (deviceId: string) => {
875 const device = world.devices.find((candidate) => candidate.id === deviceId);
876 if (device === undefined) return deviceId;
877 const host =
878 device.hostId === LOCAL_DEVICE_HOST_ID
879 ? LOCAL_DEVICE_HOST_LABEL
880 : (world.deviceHosts?.[device.hostId] ?? device.hostId);
881 return `${device.name}, ${device.id}, on ${host}`;
882 };
883 const launcherText = (command: string) => {
884 const target = targets.findLast((candidate) => candidate.command === command);
885 if (target === undefined) return command;
886 return target.bound
887 ? `${command} (drives ${deviceName(target.deviceId)}, in session ${target.session})`
888 : `${command} (shared: each command's --udid or --serial and --session name its device and session)`;
889 };
890 const describe = (history: ReadonlyArray<ShellCall>, command: string | undefined) => {
891 const followed = followSessions(targets, world.agentDeviceSessions, history, owners);
892 const sessionDevices = new Map([
893 ...targets.map((target) => [target.session, deviceName(target.deviceId)] as const),
894 ...owners.map((owner) => [owner.session, deviceName(owner.deviceId)] as const),
895 ]);
896 // Code, not the simulator, decides which of the call's opens another thread's session refuses.
897 const opens =
898 command === undefined || owners.length === 0
899 ? []
900 : opensIn(targets, followed, command, owners).flatMap(({ call, words, owner, held }) => {
901 const opened = `\`${words.join(" ")}\` drives ${deviceName(call.deviceId)}`;
902 const error = owner && inUseError(call, owner);
903 if (owner !== undefined && error !== undefined)
904 return [
905 `${opened}, which session ${owner.session} holds: it fails with exit code 1, printing "Error (${error.code}): ${error.message}" and "Hint: ${error.hint}"`,
906 ];
907 return held
908 ? []
909 : [`${opened}, which no other session holds: it prints no DEVICE_IN_USE`];
910 });
911 return [
912 ...(launchers.length > 0
913 ? [
914 `agent-device launchers T3's device_open returned: ${[...new Set(launchers)].map(launcherText).join("; ")}`,
915 ]
916 : []),
917 ...(followed.sessions.size > 0
918 ? [
919 `Their agent-device sessions: ${describeSessions(followed.sessions, owners, sessionDevices)}.`,
920 ]
921 : []),
922 ...(opens.length > 0 ? [`In the new call, ${[...new Set(opens)].join("; ")}.`] : []),
923 ];
924 };
925 return {
926 call,
927 launchers,
928 owners,
929 sessions,
930 sessionError: (history, command) => {
931 const error = agentDeviceSessionError(targets, sessions(history), command, owners);
932 return error && agentDeviceError(home, error.session, `${history.length}\n${command}`, error);
933 },
934 describe,
935 } satisfies DeviceTools;
936 });
937
938 const isOpened = Schema.is(
939 Schema.Struct({
940 device: Schema.optional(
941 Schema.Struct({ id: Schema.String, platform: Schema.String, physical: Schema.Boolean }),
942 ),
943 agentDevice: Schema.Struct({
944 command: Schema.String,
945 targetArgs: Schema.optional(Schema.Array(Schema.String)),
946 session: Schema.optional(Schema.String),
947 }),
948 }),
949 );