The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import * as Schema from "effect/Schema"; |
| 2 | |
| 3 | export const DeviceExpect = Schema.Struct({ |
| 4 | briefs: Schema.optional(Schema.Number), |
| 5 | recording: Schema.optional(Schema.String), |
| 6 | artifact: Schema.optional(Schema.String), |
| 7 | noRawUi: Schema.optional(Schema.Boolean), |
| 8 | capture: Schema.optional( |
| 9 | Schema.Struct({ |
| 10 | hostId: Schema.String, |
| 11 | deviceId: Schema.String, |
| 12 | app: Schema.optional(Schema.String), |
| 13 | }), |
| 14 | ), |
| 15 | noShutdown: Schema.optional( |
| 16 | Schema.Array(Schema.Struct({ hostId: Schema.String, deviceId: Schema.String })), |
| 17 | ), |
| 18 | report: Schema.optional(Schema.Array(Schema.String)), |
| 19 | reportOrCapture: Schema.optional(Schema.Boolean), |
| 20 | maxFailedAttemptsPerTurn: Schema.optional(Schema.Number), |
| 21 | maxCaptureAttempts: Schema.optional(Schema.Number), |
| 22 | }); |
| 23 | |
| 24 | type DeviceRun = { |
| 25 | readonly launchers?: ReadonlyArray<string>; |
| 26 | readonly text?: string; |
| 27 | readonly finalText?: string; |
| 28 | readonly stopReason?: string; |
| 29 | readonly transcript?: ReadonlyArray<unknown>; |
| 30 | readonly calls: ReadonlyArray<{ |
| 31 | readonly step: number; |
| 32 | readonly tool: string; |
| 33 | readonly input: unknown; |
| 34 | readonly turn?: number; |
| 35 | readonly failed?: boolean; |
| 36 | readonly output?: string; |
| 37 | }>; |
| 38 | }; |
| 39 | const record = (value: unknown): value is Readonly<Record<string, unknown>> => |
| 40 | typeof value === "object" && value !== null && !Array.isArray(value); |
| 41 | const inputString = (input: unknown, key: string) => |
| 42 | record(input) && typeof input[key] === "string" ? input[key] : ""; |
| 43 | const resultTarget = (call: DeviceRun["calls"][number]) => { |
| 44 | try { |
| 45 | const device: unknown = JSON.parse(call.output ?? "").device; |
| 46 | if (record(device) && typeof device.hostId === "string" && typeof device.id === "string") |
| 47 | return { hostId: device.hostId, deviceId: device.id }; |
| 48 | } catch {} |
| 49 | }; |
| 50 | const inputTarget = (call: DeviceRun["calls"][number]) => ({ |
| 51 | hostId: inputString(call.input, "hostId") || "local", |
| 52 | deviceId: inputString(call.input, "deviceId"), |
| 53 | }); |
| 54 | const openTargets = (previous: DeviceRun["calls"]) => { |
| 55 | const sessions = new Map<string, ReturnType<typeof inputTarget>>(); |
| 56 | for (const prior of previous) { |
| 57 | if (prior.failed !== false) continue; |
| 58 | if (prior.tool === "device_open") { |
| 59 | const target = resultTarget(prior) ?? inputTarget(prior); |
| 60 | const key = `${target.hostId}/${target.deviceId}`; |
| 61 | sessions.delete(key); |
| 62 | sessions.set(key, target); |
| 63 | } else if (prior.tool === "device_close") { |
| 64 | const host = inputString(prior.input, "hostId"), |
| 65 | id = inputString(prior.input, "deviceId"); |
| 66 | for (const [key, target] of sessions) |
| 67 | if ((!host || target.hostId === host) && (!id || target.deviceId === id)) |
| 68 | sessions.delete(key); |
| 69 | } |
| 70 | } |
| 71 | return [...sessions.values()]; |
| 72 | }; |
| 73 | const flagValue = (words: ReadonlyArray<string>, flag: string) => { |
| 74 | for (const [i, word] of words.entries()) { |
| 75 | if (i === 0) continue; |
| 76 | if (word === flag) return words[i + 1]; |
| 77 | if (word.startsWith(`${flag}=`)) return word.slice(flag.length + 1); |
| 78 | } |
| 79 | }; |
| 80 | /** Each successful device_open's launcher, with the device and session it targets. */ |
| 81 | const launcherOpens = (previous: DeviceRun["calls"]) => |
| 82 | previous.flatMap((call) => { |
| 83 | if (call.tool !== "device_open" || call.failed !== false) return []; |
| 84 | try { |
| 85 | const { device, agentDevice }: Record<string, unknown> = JSON.parse(call.output ?? ""); |
| 86 | if ( |
| 87 | !record(device) || |
| 88 | !record(agentDevice) || |
| 89 | typeof device.hostId !== "string" || |
| 90 | typeof device.id !== "string" || |
| 91 | typeof agentDevice.command !== "string" |
| 92 | ) |
| 93 | return []; |
| 94 | const targetArgs = Array.isArray(agentDevice.targetArgs) |
| 95 | ? agentDevice.targetArgs.filter((arg): arg is string => typeof arg === "string") |
| 96 | : []; |
| 97 | const session = |
| 98 | typeof agentDevice.session === "string" |
| 99 | ? agentDevice.session |
| 100 | : flagValue(["", ...targetArgs], "--session"); |
| 101 | return [ |
| 102 | { command: agentDevice.command, hostId: device.hostId, deviceId: device.id, session }, |
| 103 | ]; |
| 104 | } catch { |
| 105 | return []; |
| 106 | } |
| 107 | }); |
| 108 | /** |
| 109 | * The device a launcher command drives: the one its --udid, --serial or --session names among |
| 110 | * the devices its launcher was returned for, else its launcher's only device. Without launcher |
| 111 | * results, only the sole open device. |
| 112 | */ |
| 113 | const launcherTarget = (words: ReadonlyArray<string>, previous: DeviceRun["calls"]) => { |
| 114 | const id = flagValue(words, "--udid") ?? flagValue(words, "--serial"); |
| 115 | const session = flagValue(words, "--session"); |
| 116 | const launched = launcherOpens(previous).filter((open) => open.command === words[0]); |
| 117 | const candidates = ( |
| 118 | launched.length > 0 |
| 119 | ? launched |
| 120 | : openTargets(previous).map((t) => ({ ...t, session: undefined })) |
| 121 | ).filter( |
| 122 | (t) => |
| 123 | (id === undefined || t.deviceId === id) && |
| 124 | (session === undefined || t.session === undefined || t.session === session), |
| 125 | ); |
| 126 | return new Set(candidates.map((t) => `${t.hostId}/${t.deviceId}`)).size === 1 |
| 127 | ? candidates.at(-1) |
| 128 | : undefined; |
| 129 | }; |
| 130 | const screenshotTarget = (call: DeviceRun["calls"][number], previous: DeviceRun["calls"]) => { |
| 131 | const result = resultTarget(call); |
| 132 | if (result) return result; |
| 133 | if (call.failed && /open in this thread/.test(call.output ?? "")) return undefined; |
| 134 | const host = inputString(call.input, "hostId"), |
| 135 | id = inputString(call.input, "deviceId"); |
| 136 | if (host && id) return inputTarget(call); |
| 137 | const matches = openTargets(previous).filter( |
| 138 | (t) => (!host || t.hostId === host) && (!id || t.deviceId === id), |
| 139 | ); |
| 140 | return id ? matches.at(-1) : matches.length === 1 ? matches[0] : undefined; |
| 141 | }; |
| 142 | const managed = |
| 143 | /device_(?:open|screenshot|action)\b|device (?:panel|manager|tools)|returned\s+(?:(?:bound|pinned|agent-device|cli)\s+)*(?:launcher|driver|command)/; |
| 144 | const ui = |
| 145 | /\b(?:ui|screens?|screenshots?|screen captures?|record(?:ing|video)?s?|settings|taps?|typ(?:e|ing)|launch(?:es|ing)?|relaunch(?:es|ing)?|deep links?|swipes?|simulator (?:control|actions)|device actions|(?:drive|control|inspect)\s+(?:the\s+)?(?:simulator|device|app))\b/; |
| 146 | const rawCommand = |
| 147 | /\bsimctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|\bsimctl\s+io\s+(?:[^\s;]+\s+)?(?:screenshot|recordvideo)\b|\bidb\b[^.;]*\b(?:ui|screenshot|launch|video|record)\b|\badb\b[^.;]*\b(?:input|screencap|screenrecord)\b|\blab-(?:shot|ui|launch|capture(?:-set)?)\b|\bmac_agent_device\b/; |
| 148 | |
| 149 | export function briefRoute(text: string) { |
| 150 | const positive = text |
| 151 | .toLowerCase() |
| 152 | .replaceAll("’", "'") |
| 153 | .replace(/[`*]/g, "") |
| 154 | // "SSH for the non-UI operations" restricts SSH rather than routing UI through it. |
| 155 | .replace(/\bnon-(?:ui|device|simulator|screen)\b/g, "auxiliary") |
| 156 | // A trailing prohibition negates its subject, not an earlier independent predicate. |
| 157 | .replace( |
| 158 | /(^|[.;\n]\s*)((?:-\s*)?(?:use|capture|take|record|run|call|drive)\b[^.;,\n]*?)\s+and\s+(?=[^.;,\n]*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b)/g, |
| 159 | "$1$2. ", |
| 160 | ) |
| 161 | .replace( |
| 162 | /\b(?!(?:because|since|as|if|unless|when|so|but)\b)(?:(?!\b(?:because|since|as|if|unless|when|so|but)\b)[^.;,\n])*?\b(?:is|are)\s+(?:strictly\s+)?(?:forbidden|prohibited|disallowed|banned|not allowed|not permitted)\b/g, |
| 163 | "", |
| 164 | ) |
| 165 | .replace(/\b(?:do not|don't)\s+forget\s+to\b/g, "") |
| 166 | .replace(/\b(?:do not|don't)\s+avoid\b/g, "use") |
| 167 | .replace(/\b(?:should|must)\s+not\s+be\s+avoided\b/g, "is allowed") |
| 168 | .replace(/\b(?:there is )?no need to avoid\b/g, "use") |
| 169 | .replace(/\bnot just\b/g, "") |
| 170 | .replace(/\bbut (?:also )?for\b/g, "and for") |
| 171 | .replace(/\bavoid limiting (ssh|simctl|adb|idb) to [^.;]+;\s*use it\b/g, "use $1") |
| 172 | .replace(/(^|[.!;\n]\s*)(for [^.;,\n]+),\s*(?=(?:use|run)\b)/g, "$1$2 ") |
| 173 | .split( |
| 174 | /[\n.!;]+(?=\s|$)|\b(?:but|however|while)\b|,\s*(?=(?:screenshots?|recordings?)\s+(?:via|through|go))|(?:,\s*|\band\s+|\bthen\s+)(?=(?:use|run|call)\b)/, |
| 175 | ) |
| 176 | .map((clause) => |
| 177 | ( |
| 178 | clause.split( |
| 179 | /\b(?:not|never|no|don't|cannot|can't|avoid|forbidden|prohibited|disallowed|rather than|instead of|without)\b/, |
| 180 | )[0] ?? "" |
| 181 | ) |
| 182 | .replace( |
| 183 | /\b(?:ui|screen|settings|simulator|device|app)\s+(logs?|builds?|test runners?)\b/g, |
| 184 | "$1", |
| 185 | ) |
| 186 | .replace(/\b(?:record|capture|stream)\s+(?:the\s+)?logs\b/g, "logs") |
| 187 | .replace(/\b(?:logs? from|reading)\s+screenshot\s+(?:tests?|logs?)\b/g, "logs") |
| 188 | .replace( |
| 189 | /\b(?:copy(?:ing)?|fetch(?:ing)?|transfer(?:ring)?|download(?:ing)?|sync(?:ing)?)\s+(?:(?:the|already|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\b/g, |
| 190 | "artifact transfer", |
| 191 | ), |
| 192 | ); |
| 193 | const hasManaged = positive.some((clause) => managed.test(clause)); |
| 194 | const hasRaw = positive.some((clause) => { |
| 195 | if (rawCommand.test(clause)) return true; |
| 196 | const routes = clause.matchAll(/\b(?:ssh|simctl|idb|adb|lab scripts?)\b/g); |
| 197 | for (const route of routes) { |
| 198 | const before = clause.slice(0, route.index); |
| 199 | const after = clause.slice(route.index + route[0].length).split(managed)[0] ?? ""; |
| 200 | // Copying already-saved artifacts is auxiliary; keep later capture verbs visible. |
| 201 | const copies = |
| 202 | /^\s+(?:to\s+)?(?:copy|fetch|transfer|download|sync)\b/.test(after) || |
| 203 | /\b(?:copy|fetch|transfer|download|sync)\s+(?:(?:the|saved)\s+)*(?:screenshots?|recordings?|artifacts?)\s+(?:over|via|through|using)\s*$/.test( |
| 204 | before, |
| 205 | ); |
| 206 | if (copies && !/\b(?:take|capture|inspect|record|drive|control)\b/.test(after)) continue; |
| 207 | const purpose = |
| 208 | /^\s+(?:(?:is|are|can|may|must|should|be|used|only|allowed|permitted|fine)\s+)*(?:for|to|into)\b/.test( |
| 209 | after, |
| 210 | ); |
| 211 | const hostPurpose = |
| 212 | /^\s+(?:-\S+\s+\S+\s+)*\S+\s+(?:for|and|then)\b|^\s+(?:handles|and|then)\b/.test(after); |
| 213 | if ((purpose || hostPurpose || new RegExp(`^\\s+${ui.source}`).test(after)) && ui.test(after)) |
| 214 | return true; |
| 215 | const via = new RegExp( |
| 216 | `${ui.source}(?:\\s+(?:may|can|must|should|be|is|are|taken|captured|saved|recorded|only|directly))*\\s+(?:via|by|over|through|using|with|use|run)\\s+(?:raw\\s+)?$`, |
| 217 | ); |
| 218 | if ( |
| 219 | via.test(before) || |
| 220 | /\b(?:screenshots?|screen(?: capture)?|recordings?)(?:\s+on\s+\S+)?\s+(?:over|via|using|through|with)\s*$/.test( |
| 221 | before, |
| 222 | ) || |
| 223 | /\bfor\s+[^.;]*\b(?:screenshots?|recordings?|screen capture)\s*,?\s*$/.test(before) |
| 224 | ) |
| 225 | return true; |
| 226 | } |
| 227 | return false; |
| 228 | }); |
| 229 | return hasManaged && hasRaw ? "mixed" : hasManaged ? "managed" : hasRaw ? "raw" : "neutral"; |
| 230 | } |
| 231 | |
| 232 | const executable = (word = "") => word.split("/").at(-1) ?? ""; |
| 233 | const uiCommand = |
| 234 | /^(?:simctl\s+(?:launch|terminate|openurl|ui|privacy|location|push)\b|simctl\s+io\s+\S+\s+(?:screenshot|recordvideo)\b|idb\b.*\b(?:ui|screenshot|launch|video|record)\b|adb\b.*\b(?:shell\s+(?:input|am\s+start|settings)|(?:shell|exec-out)\s+(?:screencap|screenrecord))\b|screencapture\b|lab-(?:shot|ui|launch|capture(?:-set)?)\b)/i; |
| 235 | const deviceArgs = (words: ReadonlyArray<string>) => { |
| 236 | const args: string[] = []; |
| 237 | for (let i = 1; i < words.length; i++) { |
| 238 | const word = words[i]!; |
| 239 | if (word === "--") return [...args, ...words.slice(i + 1)]; |
| 240 | if (!word.startsWith("-")) args.push(word); |
| 241 | else if ( |
| 242 | /^--(?:platform|udid|serial|device|target|level|config|remote-config|session|state-dir|scope|timeout|timeout-ms|wait|metro-port|out|quality)$/.test( |
| 243 | word, |
| 244 | ) |
| 245 | ) |
| 246 | i++; |
| 247 | } |
| 248 | return args; |
| 249 | }; |
| 250 | const auxiliaryAgent = |
| 251 | /^(?:(?:devices|apps|doctor|capabilities|install|uninstall|reinstall|install-from-source|logs|help|version)(?: |$)|session list(?: |$)|record contact-sheet(?: |$))/; |
| 252 | const deviceOperation = (words: ReadonlyArray<string>) => { |
| 253 | const args = deviceArgs(words); |
| 254 | return !!args[0] && !auxiliaryAgent.test(args.join(" ")); |
| 255 | }; |
| 256 | const interpreter = /^(?:python\d*(?:\.\d+)?|node|ruby|perl|osascript)$/; |
| 257 | const shells = new Set(["bash", "sh", "zsh"]); |
| 258 | /** Human and JSON agent-device failure envelopes. */ |
| 259 | const AGENT_DEVICE_ERROR = /\bError \([A-Z_]+\):|"success"\s*:\s*false/; |
| 260 | |
| 261 | export type ShellInvocation = { |
| 262 | readonly words: ReadonlyArray<string>; |
| 263 | readonly remote: boolean; |
| 264 | readonly raw: boolean; |
| 265 | readonly managed: boolean; |
| 266 | /** Raw-route checks inspect possible actions; managed credit requires established execution. */ |
| 267 | readonly executed: boolean; |
| 268 | /** An aggregate success cannot establish success of an action masked by ||. */ |
| 269 | readonly unmasked: boolean; |
| 270 | /** A later `;` or `&` command owns the call's exit status. */ |
| 271 | readonly sequenced: boolean; |
| 272 | }; |
| 273 | |
| 274 | // A deliberately bounded shell grammar, not a shell executor. Keep quotes until expansion so |
| 275 | // literal command examples, comments and quoted heredocs never become invocations. |
| 276 | const tokens = (source: string) => |
| 277 | source |
| 278 | .replace(/'[^']*'|\\[\s\S]/g, (part) => (part === "\\\n" ? "" : part)) |
| 279 | .match( |
| 280 | /`(?:\\.|[^`])*`|[<>]\((?:[^()]|\([^()]*\))*\)|\$\((?:[^()'"\\]|\\.|'[^']*'|"(?:\\.|[^"\\])*"|\([^()]*\))*\)|\d+(?:>>?|<)(?:&\d+)?|(?:"(?:\\.|[^"\\])*"|'[^']*'|\\.|\$\{\w+\}|[^\s;&|(){}<>"'\\])+|&&|\|\||[;\n&|(){}]|\d*(?:>>?|<)(?:&\d+)?/g, |
| 281 | ) ?? []; |
| 282 | const unquote = (word: string, variables: Map<string, string>) => |
| 283 | word.replace( |
| 284 | /'([^']*)'|"((?:\\.|[^"\\])*)"|\\(.)|([^'"\\]+)/g, |
| 285 | ( |
| 286 | _part, |
| 287 | single: string | undefined, |
| 288 | double: string | undefined, |
| 289 | escaped: string | undefined, |
| 290 | plain: string | undefined, |
| 291 | ) => { |
| 292 | if (single !== undefined) return single; |
| 293 | if (escaped !== undefined) return escaped; |
| 294 | return (double ?? plain ?? "").replace( |
| 295 | /\\(["\\$])|\$(?:\{(\w+)\}|(\w+))/g, |
| 296 | (ref, escaped: string | undefined, braced: string | undefined, bare: string | undefined) => |
| 297 | escaped ?? variables.get(braced ?? bare ?? "") ?? ref, |
| 298 | ); |
| 299 | }, |
| 300 | ); |
| 301 | |
| 302 | function unwrap(words: string[]) { |
| 303 | while (words.length) { |
| 304 | if (/^\w+=/.test(words[0]!)) { |
| 305 | words.shift(); |
| 306 | continue; |
| 307 | } |
| 308 | const name = executable(words[0]); |
| 309 | if (!/^(?:env|sudo|command|nohup|exec|time|caffeinate|timeout|nice)$/.test(name)) break; |
| 310 | words.shift(); |
| 311 | const takesValue: Record<string, RegExp> = { |
| 312 | env: /^(?:-u|--unset|-C|--chdir)$/, |
| 313 | sudo: /^(?:-[ughpCT]|--user|--group|--host|--prompt)$/, |
| 314 | caffeinate: /^-[tw]$/, |
| 315 | timeout: /^(?:-[sk]|--signal|--kill-after)$/, |
| 316 | nice: /^(?:-n|--adjustment)$/, |
| 317 | }; |
| 318 | while (words[0]?.startsWith("-")) { |
| 319 | const option = words.shift()!; |
| 320 | if (option === "--") break; |
| 321 | if (takesValue[name]?.test(option)) words.shift(); |
| 322 | } |
| 323 | if (name === "timeout") words.shift(); |
| 324 | } |
| 325 | return words; |
| 326 | } |
| 327 | |
| 328 | /** Literal process-launch APIs only. Definitions and strings are not executed programs. */ |
| 329 | function scriptInvocations( |
| 330 | script: string, |
| 331 | remote: boolean, |
| 332 | variables: Map<string, string>, |
| 333 | language: string, |
| 334 | ): ShellInvocation[] { |
| 335 | const actions: ShellInvocation[] = []; |
| 336 | const python = language.startsWith("python"); |
| 337 | // A function body is source, not an invocation. Calls to computed/script functions are |
| 338 | // outside this grammar; do not grant managed credit for them. |
| 339 | let definitionIndent: number | undefined; |
| 340 | if (python) |
| 341 | script = script |
| 342 | .split("\n") |
| 343 | .map((line) => { |
| 344 | const indent = line.match(/^\s*/)?.[0].length ?? 0; |
| 345 | if (line.trim() && definitionIndent !== undefined && indent <= definitionIndent) |
| 346 | definitionIndent = undefined; |
| 347 | if (/^\s*(?:async\s+)?def\s/.test(line)) definitionIndent = indent; |
| 348 | return definitionIndent === undefined ? line : ""; |
| 349 | }) |
| 350 | .join("\n"); |
| 351 | const launch = |
| 352 | /^(subprocess\.(?:run|Popen|call|check_call|check_output)|os\.(?:system|popen)|(?:child_process\.)?(?:execFileSync|execFile|execSync|exec|spawnSync|spawn))\s*\(/; |
| 353 | const literal = "(?:'[^']*'|\"(?:\\\\.|[^\"\\\\])*\")"; |
| 354 | const quoteArg = (arg: string) => `'${unquote(arg, variables).replaceAll("'", "'\\''")}'`; |
| 355 | const array = (source: string) => { |
| 356 | const match = new RegExp(`^\\[\\s*(${literal}(?:\\s*,\\s*${literal})*\\s*,?)?\\s*\\]`).exec( |
| 357 | source, |
| 358 | ); |
| 359 | return match |
| 360 | ? (match[1]?.match(new RegExp(literal, "g")) ?? []).map(quoteArg).join(" ") |
| 361 | : undefined; |
| 362 | }; |
| 363 | for (let i = 0; i < script.length; i++) { |
| 364 | const char = script[i]; |
| 365 | if (char === "#" || script.slice(i, i + 2) === "//") { |
| 366 | const end = script.indexOf("\n", i); |
| 367 | i = end < 0 ? script.length : end; |
| 368 | continue; |
| 369 | } |
| 370 | if (char === "'" || char === '"' || char === "`") { |
| 371 | const quote = python && script.slice(i, i + 3) === char.repeat(3) ? char.repeat(3) : char; |
| 372 | i += quote.length; |
| 373 | while (i < script.length && script.slice(i, i + quote.length) !== quote) { |
| 374 | if (script[i] === "\\") i++; |
| 375 | i++; |
| 376 | } |
| 377 | i += quote.length - 1; |
| 378 | continue; |
| 379 | } |
| 380 | const match = launch.exec(script.slice(i)); |
| 381 | if (!match) continue; |
| 382 | const rest = script.slice(i + match[0].length).trimStart(); |
| 383 | let program: string | undefined; |
| 384 | if (rest.startsWith("[")) program = array(rest); |
| 385 | else { |
| 386 | const arg = rest.match(new RegExp(`^${literal}`))?.[0]; |
| 387 | const tail = arg === undefined ? "" : rest.slice(arg.length); |
| 388 | if (arg && /(?:spawn|execFile)/.test(match[1]!)) { |
| 389 | const argv = array(tail.replace(/^\s*,\s*/, "")); |
| 390 | if (argv !== undefined) program = `${quoteArg(arg)} ${argv}`; |
| 391 | } else if ( |
| 392 | arg && |
| 393 | (!python || match[1]!.startsWith("os.") || /^\s*,[^)]*\bshell\s*=\s*True\b/.test(tail)) |
| 394 | ) { |
| 395 | program = unquote(arg, variables); |
| 396 | } |
| 397 | } |
| 398 | if (program !== undefined) { |
| 399 | const line = script.slice(script.lastIndexOf("\n", i) + 1, i); |
| 400 | const boundary = Math.max( |
| 401 | script.lastIndexOf(";", i), |
| 402 | python ? script.lastIndexOf("\n", i) : -1, |
| 403 | ); |
| 404 | const statement = script.slice(boundary + 1, i); |
| 405 | const certain = |
| 406 | !/=>|\bfunction\b|\blambda\b/.test(statement) && |
| 407 | (python ? !/^\s/.test(line) : !script.slice(0, i).includes("{")); |
| 408 | actions.push( |
| 409 | ...shellInvocations(program, new Map(variables), remote).map((action) => ({ |
| 410 | ...action, |
| 411 | executed: action.executed && certain, |
| 412 | // A script's exit status needn't reflect its subprocesses (`subprocess.run` doesn't check). |
| 413 | sequenced: true, |
| 414 | })), |
| 415 | ); |
| 416 | } |
| 417 | i += match[0].length - 1; |
| 418 | } |
| 419 | return actions; |
| 420 | } |
| 421 | |
| 422 | /** Shared by raw-UI, recording and failure grading. Literal aliases may span shell calls. */ |
| 423 | export function shellInvocations( |
| 424 | command: string, |
| 425 | variables = new Map<string, string>(), |
| 426 | remote = false, |
| 427 | succeeded = false, |
| 428 | /** Shared with function calls: the shell's functions, and whether it exited. */ |
| 429 | shell: { |
| 430 | functions: Map<string, { text: string; certain: boolean }>; |
| 431 | exited: boolean; |
| 432 | depth: number; |
| 433 | } = { |
| 434 | functions: new Map(), |
| 435 | exited: false, |
| 436 | depth: 0, |
| 437 | }, |
| 438 | ): ShellInvocation[] { |
| 439 | if (shell.depth > 8) return []; |
| 440 | const bodies = new Map<string, { text: string; expand: boolean }>(); |
| 441 | command = command.replace( |
| 442 | /^([^\n]*?)<<(-?)\s*(['"]?)(\w+)\3([^\n]*)\n([\s\S]*?)(?:\n[ \t]*\4[ \t]*(?=\n|(?![\s\S]))|(?![\s\S]))/gm, |
| 443 | ( |
| 444 | _match, |
| 445 | line: string, |
| 446 | strip: string, |
| 447 | quote: string, |
| 448 | _end: string, |
| 449 | rest: string, |
| 450 | body: string, |
| 451 | ) => { |
| 452 | const key = `__HEREDOC_${bodies.size}`; |
| 453 | bodies.set(key, { text: strip ? body.replace(/^\t+/gm, "") : body, expand: !quote }); |
| 454 | return `${line} ${key} ${rest}`; |
| 455 | }, |
| 456 | ); |
| 457 | const actions: ShellInvocation[] = []; |
| 458 | let clause: string[] = []; |
| 459 | let pipe: string | undefined; |
| 460 | let gate = ""; |
| 461 | let status: boolean | undefined; |
| 462 | // `before` says whether an earlier if/elif branch of this statement ran: true, false or unknown. |
| 463 | const conditions: Array<{ |
| 464 | parent: boolean; |
| 465 | certain: boolean; |
| 466 | take: boolean | undefined; |
| 467 | before: boolean | undefined; |
| 468 | }> = []; |
| 469 | let active = true; |
| 470 | let certain = true; |
| 471 | let currentCertain = true; |
| 472 | let currentUnmasked = true; |
| 473 | let currentSequenced = false; |
| 474 | // After a certain `exit` or `return`, nothing later in its scope runs. |
| 475 | let exited = false; |
| 476 | const append = (nested: ShellInvocation[]) => |
| 477 | actions.push( |
| 478 | ...nested.map((action) => ({ |
| 479 | ...action, |
| 480 | executed: action.executed && currentCertain, |
| 481 | unmasked: action.unmasked && currentUnmasked, |
| 482 | sequenced: action.sequenced || currentSequenced, |
| 483 | })), |
| 484 | ); |
| 485 | const truth = (words: string[]): boolean | undefined => { |
| 486 | if (words[0] === "!") { |
| 487 | const value = truth(words.slice(1)); |
| 488 | return value === undefined ? undefined : !value; |
| 489 | } |
| 490 | return ["true", ":"].includes(words[0] ?? "") ? true : words[0] === "false" ? false : undefined; |
| 491 | }; |
| 492 | const commandTokens = tokens(command.replace(/(^|\n)[ \t]*#[^\n]*/g, "$1")); |
| 493 | let tokenIndex = 0; |
| 494 | const functions = shell.functions; |
| 495 | const listTokens = () => { |
| 496 | let start = tokenIndex - 1, |
| 497 | end = tokenIndex; |
| 498 | while (start >= 0 && !/^[;\n&]$/.test(commandTokens[start]!)) start--; |
| 499 | while (end < commandTokens.length && !/^[;\n&]$/.test(commandTokens[end]!)) end++; |
| 500 | return commandTokens.slice(start + 1, end); |
| 501 | }; |
| 502 | const substitutions = (source: string, heredoc = false) => { |
| 503 | const expanded = heredoc |
| 504 | ? source |
| 505 | : source.replace( |
| 506 | /"((?:\\.|[^"\\])*)"|'[^']*'|\\./g, |
| 507 | (part, double: string | undefined) => double ?? (part.startsWith("'") ? "" : part), |
| 508 | ); |
| 509 | for (const match of expanded.matchAll(/(?<!\\)\$\(((?:[^()]|\([^()]*\))*)\)/g)) |
| 510 | append(shellInvocations(match[1]!, new Map(variables), remote)); |
| 511 | for (const match of expanded.matchAll(/(?<!\\)`([^`]*)`|[<>]\(((?:[^()]|\([^()]*\))*)\)/g)) |
| 512 | append(shellInvocations(match[1] ?? match[2]!, new Map(variables), remote)); |
| 513 | }; |
| 514 | const flush = () => { |
| 515 | if (!clause.length) return; |
| 516 | const rawWords = clause; |
| 517 | const redirects = redirectOperands; |
| 518 | redirectOperands = []; |
| 519 | clause = []; |
| 520 | let words = rawWords.map((word) => unquote(word, variables)); |
| 521 | if (words[0] === "if") { |
| 522 | const parent = active; |
| 523 | conditions.push({ |
| 524 | parent, |
| 525 | certain, |
| 526 | take: undefined, |
| 527 | before: false, |
| 528 | }); |
| 529 | words.shift(); |
| 530 | } |
| 531 | if (words[0] === "then") { |
| 532 | const condition = conditions.at(-1); |
| 533 | // The branch follows the condition list's final status (`if true && false`), not its |
| 534 | // first command's. |
| 535 | if (condition) condition.take = condition.before === true ? false : status; |
| 536 | active = |
| 537 | condition?.parent !== false && condition?.take !== false && condition?.before !== true; |
| 538 | certain = |
| 539 | condition?.certain !== false && condition?.take === true && condition?.before === false; |
| 540 | words.shift(); |
| 541 | } |
| 542 | if (words[0] === "else") { |
| 543 | const condition = conditions.at(-1); |
| 544 | active = !!condition?.parent && condition.take !== true && condition.before !== true; |
| 545 | certain = !!condition?.certain && condition.take === false && condition.before === false; |
| 546 | words.shift(); |
| 547 | } |
| 548 | if (words[0] === "fi") { |
| 549 | const condition = conditions.pop(); |
| 550 | active = condition?.parent ?? true; |
| 551 | certain = condition?.certain ?? true; |
| 552 | words.shift(); |
| 553 | } |
| 554 | if (words[0] === "elif") { |
| 555 | // An elif's condition runs only when no earlier branch did. |
| 556 | const condition = conditions.at(-1); |
| 557 | if (condition) { |
| 558 | condition.before = |
| 559 | condition.before === true || condition.take === true |
| 560 | ? true |
| 561 | : condition.before === false && condition.take === false |
| 562 | ? false |
| 563 | : undefined; |
| 564 | condition.take = condition.before === true ? false : truth(words.slice(1)); |
| 565 | active = condition.parent && condition.before !== true; |
| 566 | certain = condition.certain && condition.before === false; |
| 567 | } |
| 568 | words.shift(); |
| 569 | } |
| 570 | const loopList = /^(?:for|select)$/.test(words[0] ?? ""); |
| 571 | if (/^(?:while|until)$/.test(words[0] ?? "") || loopList) { |
| 572 | const value = loopList |
| 573 | ? words[2] === "in" && words.length > 3 |
| 574 | ? true |
| 575 | : undefined |
| 576 | : truth(words.slice(1)); |
| 577 | conditions.push({ |
| 578 | parent: active, |
| 579 | certain, |
| 580 | take: words[0] === "until" && value !== undefined ? !value : value, |
| 581 | before: false, |
| 582 | }); |
| 583 | if (!loopList) words.shift(); |
| 584 | } |
| 585 | if (words[0] === "do") { |
| 586 | const condition = conditions.at(-1); |
| 587 | active = condition?.parent !== false && condition?.take !== false; |
| 588 | certain = condition?.certain !== false && condition?.take === true; |
| 589 | words.shift(); |
| 590 | } |
| 591 | if (words[0] === "done") { |
| 592 | const condition = conditions.pop(); |
| 593 | active = condition?.parent ?? true; |
| 594 | certain = condition?.certain ?? true; |
| 595 | words.shift(); |
| 596 | } |
| 597 | const reachable = |
| 598 | gate === "&&" ? status : gate === "||" ? (status === undefined ? undefined : !status) : true; |
| 599 | if (!active || exited || !words.length || reachable === false) return; |
| 600 | const list = listTokens(); |
| 601 | currentUnmasked = !list.includes("||") && words[0] !== "!"; |
| 602 | // The call's exit status is the last command's; an action followed by `;` or `&` doesn't own it. |
| 603 | // A command piped into another doesn't own the pipeline's status either. |
| 604 | currentSequenced = |
| 605 | commandTokens[tokenIndex] === "|" || |
| 606 | commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t)); |
| 607 | const finalAndSuccess = |
| 608 | succeeded && |
| 609 | gate === "&&" && |
| 610 | currentUnmasked && |
| 611 | !commandTokens.some((t) => /^(?:if|for|while|until|then|else|fi|do|done)$/.test(t)) && |
| 612 | !commandTokens.slice(tokenIndex).some((t) => /^[;\n&]$/.test(t)); |
| 613 | currentCertain = certain && (reachable === true || finalAndSuccess); |
| 614 | // A list's status is known only when this command certainly ran, or when its own result |
| 615 | // decides the list either way (`x && false`, `x || true`). |
| 616 | const value = truth(words); |
| 617 | status = |
| 618 | reachable === true |
| 619 | ? value |
| 620 | : gate === "&&" && value === false |
| 621 | ? false |
| 622 | : gate === "||" && value === true |
| 623 | ? true |
| 624 | : undefined; |
| 625 | for (const word of [...rawWords, ...redirects]) if (!bodies.has(word)) substitutions(word); |
| 626 | if (loopList) return; |
| 627 | const body = words.map((word) => bodies.get(word)).find((value) => value !== undefined); |
| 628 | if (body?.expand) substitutions(body.text, true); |
| 629 | words = words.filter((word) => !bodies.has(word)); |
| 630 | const literalAssignments = words.every((word) => /^\w+=/.test(word)); |
| 631 | if (literalAssignments || words[0] === "export") { |
| 632 | for (const word of words) { |
| 633 | const at = word.indexOf("="); |
| 634 | if (at > 0) { |
| 635 | if (currentCertain) variables.set(word.slice(0, at), word.slice(at + 1)); |
| 636 | else variables.delete(word.slice(0, at)); |
| 637 | } |
| 638 | } |
| 639 | return; |
| 640 | } |
| 641 | if (executable(words[0]) === "command" && /^-[vV]$/.test(words[1] ?? "")) return; |
| 642 | while (words[0] === "!") words.shift(); |
| 643 | words = unwrap(words); |
| 644 | if (/^(?:exit|return)$/.test(executable(words[0]))) { |
| 645 | if (currentCertain) { |
| 646 | exited = true; |
| 647 | // `exit` in a function ends the shell; `return` ends only the function. |
| 648 | if (executable(words[0]) === "exit") shell.exited = true; |
| 649 | } else certain = false; |
| 650 | return; |
| 651 | } |
| 652 | const name = executable(words[0]); |
| 653 | let input = body?.text ?? pipe; |
| 654 | pipe = undefined; |
| 655 | if (name === "cat" || name === "tee") { |
| 656 | pipe = input; |
| 657 | return; |
| 658 | } |
| 659 | if (name === "printf" || name === "echo") { |
| 660 | pipe = |
| 661 | name === "echo" |
| 662 | ? words.slice(1).join(" ") |
| 663 | : words[1]?.includes("%s") |
| 664 | ? words.slice(2).join("\n") |
| 665 | : words[1]?.replaceAll("\\n", "\n"); |
| 666 | return; |
| 667 | } |
| 668 | if (functions.has(name)) { |
| 669 | const definition = functions.get(name)!; |
| 670 | const call = { functions, exited: false, depth: shell.depth + 1 }; |
| 671 | append( |
| 672 | shellInvocations(definition.text, new Map(variables), remote, succeeded, call).map( |
| 673 | (action) => ({ ...action, executed: action.executed && definition.certain }), |
| 674 | ), |
| 675 | ); |
| 676 | if (call.exited && definition.certain && currentCertain) exited = shell.exited = true; |
| 677 | return; |
| 678 | } |
| 679 | if (name === "ssh") { |
| 680 | let at = 1; |
| 681 | while (words[at]?.startsWith("-")) { |
| 682 | const option = words[at++]!; |
| 683 | if (option === "--") break; |
| 684 | if (/^-[bBcDEeFIiJLlmOoPpQRSWw]$/.test(option)) at++; |
| 685 | } |
| 686 | at++; // destination |
| 687 | if (words[at] === "--") at++; |
| 688 | const script = words.slice(at).join(" "); |
| 689 | // A remote shell/interpreter inherits redirected/piped stdin, never the local aliases. |
| 690 | const destination = unwrap(tokens(script).map((word) => unquote(word, new Map()))); |
| 691 | const reader = executable(destination[0]); |
| 692 | // Shells take a program with -c (`bash -se` reads stdin); interpreters with -c or -e. |
| 693 | const programFlag = shells.has(reader) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/; |
| 694 | const hasProgram = destination.some((word) => programFlag.test(word)); |
| 695 | if ( |
| 696 | input !== undefined && |
| 697 | (!reader || ((shells.has(reader) || interpreter.test(reader)) && !hasProgram)) |
| 698 | ) { |
| 699 | if (interpreter.test(reader)) append(scriptInvocations(input, true, new Map(), reader)); |
| 700 | else append(shellInvocations(input, new Map(), true)); |
| 701 | } |
| 702 | append(shellInvocations(script, new Map(), true)); |
| 703 | return; |
| 704 | } |
| 705 | if (shells.has(name) || interpreter.test(name)) { |
| 706 | const at = words.findIndex((word) => |
| 707 | (shells.has(name) ? /^-[a-z]*c[a-z]*$/ : /^-[a-z]*[ce][a-z]*$/).test(word), |
| 708 | ); |
| 709 | if (at >= 0) input = words[at + 1]; |
| 710 | if (input !== undefined) |
| 711 | append( |
| 712 | shells.has(name) |
| 713 | ? shellInvocations(input, new Map(variables), remote, succeeded) |
| 714 | : scriptInvocations(input, remote, variables, name), |
| 715 | ); |
| 716 | return; |
| 717 | } |
| 718 | const endOptions = words.indexOf("--"); |
| 719 | if ( |
| 720 | words |
| 721 | .slice(0, endOptions < 0 ? words.length : endOptions) |
| 722 | .some((word) => word === "--help" || word === "-h") || |
| 723 | words[1] === "help" |
| 724 | ) |
| 725 | return; |
| 726 | const normalized = name === "xcrun" ? words.slice(1) : [...words]; |
| 727 | if (name === "xcrun") |
| 728 | while (normalized[0]?.startsWith("-")) { |
| 729 | const option = normalized.shift(); |
| 730 | if (/^(?:--sdk|-sdk|--toolchain|-toolchain)$/.test(option ?? "")) normalized.shift(); |
| 731 | } |
| 732 | const cli = executable(normalized[0]); |
| 733 | const agent = ["agent-device", "mac_agent_device"].includes(cli) && deviceOperation(normalized); |
| 734 | actions.push({ |
| 735 | words, |
| 736 | remote, |
| 737 | executed: currentCertain, |
| 738 | unmasked: currentUnmasked, |
| 739 | sequenced: currentSequenced, |
| 740 | raw: |
| 741 | uiCommand.test([cli, ...normalized.slice(1)].join(" ")) || |
| 742 | (agent && (remote || cli === "mac_agent_device")), |
| 743 | managed: agent && !remote && cli === "agent-device", |
| 744 | }); |
| 745 | }; |
| 746 | const scopes: Array<{ |
| 747 | variables: Map<string, string>; |
| 748 | active: boolean; |
| 749 | certain: boolean; |
| 750 | exited: boolean; |
| 751 | subshell: boolean; |
| 752 | }> = []; |
| 753 | let comment = false; |
| 754 | let redirect = false; |
| 755 | let redirectOperands: string[] = []; |
| 756 | for (; tokenIndex < commandTokens.length; tokenIndex++) { |
| 757 | const token = commandTokens[tokenIndex]!; |
| 758 | // Comments come first, so a commented-out function definition is never registered. |
| 759 | if (token === "\n") comment = false; |
| 760 | if (comment) continue; |
| 761 | if (token.startsWith("#")) { |
| 762 | comment = true; |
| 763 | continue; |
| 764 | } |
| 765 | // Shell function definitions don't execute their bodies. Keep literal calls supported. |
| 766 | const nameAt = tokenIndex + (token === "function" ? 1 : 0); |
| 767 | const bodyAt = |
| 768 | commandTokens[nameAt + 1] === "(" && commandTokens[nameAt + 2] === ")" |
| 769 | ? nameAt + 3 |
| 770 | : token === "function" |
| 771 | ? nameAt + 1 |
| 772 | : -1; |
| 773 | if (commandTokens[bodyAt] === "{") { |
| 774 | flush(); |
| 775 | let end = bodyAt + 1, |
| 776 | depth = 1; |
| 777 | for (; end < commandTokens.length && depth; end++) { |
| 778 | if (commandTokens[end] === "{") depth++; |
| 779 | if (commandTokens[end] === "}") depth--; |
| 780 | } |
| 781 | if (active) |
| 782 | functions.set(commandTokens[nameAt]!, { |
| 783 | text: commandTokens.slice(bodyAt + 1, end - 1).join(" "), |
| 784 | certain, |
| 785 | }); |
| 786 | tokenIndex = end - 1; |
| 787 | continue; |
| 788 | } |
| 789 | if (/^\d*(?:>>?|<)(?:&\d+)?$/.test(token)) { |
| 790 | redirect = !token.includes("&"); |
| 791 | continue; |
| 792 | } |
| 793 | if (redirect) { |
| 794 | redirect = false; |
| 795 | redirectOperands.push(token); |
| 796 | continue; |
| 797 | } |
| 798 | if (/^(?:&&|\|\||[;\n&|(){}])$/.test(token)) { |
| 799 | flush(); |
| 800 | if (token === "(" || token === "{") { |
| 801 | scopes.push({ variables, active, certain, exited, subshell: token === "(" }); |
| 802 | if (token === "(") variables = new Map(variables); |
| 803 | const reachable = |
| 804 | gate === "&&" |
| 805 | ? status |
| 806 | : gate === "||" |
| 807 | ? status === undefined |
| 808 | ? undefined |
| 809 | : !status |
| 810 | : true; |
| 811 | active = active && reachable !== false; |
| 812 | certain = certain && reachable === true; |
| 813 | } |
| 814 | if (token === ")" || token === "}") { |
| 815 | const scope = scopes.pop(); |
| 816 | // A subshell's exit ends only the subshell; a group's ends the shell. |
| 817 | if (scope) exited = scope.exited || (!scope.subshell && exited); |
| 818 | if (scope) ({ variables, active, certain } = scope); |
| 819 | } |
| 820 | if (token !== "|") pipe = undefined; |
| 821 | gate = token; |
| 822 | } else clause.push(token); |
| 823 | } |
| 824 | flush(); |
| 825 | return actions; |
| 826 | } |
| 827 | |
| 828 | export function gradeDevice(expect: typeof DeviceExpect.Type | undefined, run: DeviceRun) { |
| 829 | const checks: Array<{ check: string; pass: boolean }> = []; |
| 830 | if (!expect) return checks; |
| 831 | const add = (check: string, pass: boolean) => checks.push({ check, pass }); |
| 832 | if (expect.briefs !== undefined) { |
| 833 | const briefs = run.calls.filter((call) => |
| 834 | ["delegate_task", "t3_thread_launch"].includes(call.tool), |
| 835 | ); |
| 836 | add("device briefs count", briefs.length >= expect.briefs); |
| 837 | for (const [i, call] of briefs.entries()) { |
| 838 | const route = briefRoute( |
| 839 | inputString(call.input, call.tool === "delegate_task" ? "task" : "message"), |
| 840 | ); |
| 841 | add(`device brief ${i + 1} ${call.tool} ${route}`, route === "managed"); |
| 842 | } |
| 843 | } |
| 844 | if (expect.artifact !== undefined) |
| 845 | add( |
| 846 | `device final artifact ${expect.artifact}`, |
| 847 | run.stopReason === "end_turn" && (run.finalText ?? "").includes(expect.artifact), |
| 848 | ); |
| 849 | const variables = new Map<string, string>(); |
| 850 | const shell = run.calls |
| 851 | .filter((call) => ["Bash", "Monitor"].includes(call.tool)) |
| 852 | .map( |
| 853 | (call) => |
| 854 | [ |
| 855 | call, |
| 856 | shellInvocations( |
| 857 | inputString(call.input, "command"), |
| 858 | variables, |
| 859 | false, |
| 860 | call.failed === false, |
| 861 | ).map((action) => ({ |
| 862 | ...action, |
| 863 | args: deviceArgs(action.words), |
| 864 | bound: !action.remote && (run.launchers ?? []).includes(action.words[0] ?? ""), |
| 865 | })), |
| 866 | ] as const, |
| 867 | ); |
| 868 | const allActions = shell.flatMap(([, actions]) => actions); |
| 869 | const usableImage = (call: DeviceRun["calls"][number]) => { |
| 870 | if (call.failed !== false) return false; |
| 871 | try { |
| 872 | const image: unknown = JSON.parse(call.output ?? "").screenshot; |
| 873 | return ( |
| 874 | record(image) && |
| 875 | typeof image.width === "number" && |
| 876 | image.width > 0 && |
| 877 | typeof image.height === "number" && |
| 878 | image.height > 0 |
| 879 | ); |
| 880 | } catch { |
| 881 | return false; |
| 882 | } |
| 883 | }; |
| 884 | const successfulScreenshots = ([call, actions]: (typeof shell)[number]) => { |
| 885 | const captures = actions.filter((a) => a.managed && a.bound && a.args[0] === "screenshot"); |
| 886 | if ( |
| 887 | call.failed !== false || |
| 888 | AGENT_DEVICE_ERROR.test(call.output ?? "") || |
| 889 | captures.length !== 1 || |
| 890 | /"(?:width|height)"\s*:\s*0\b/.test(call.output ?? "") |
| 891 | ) |
| 892 | return []; |
| 893 | const paths = (call.output ?? "").split("\n").flatMap((line) => { |
| 894 | // The simulated shell words the success line several ways; each must still name the path. |
| 895 | const saved = line.match( |
| 896 | /^[^\w\n]*(?:Screenshot saved|Saved screenshot|Captured screenshot|Wrote screenshot)(?: to)?:?\s+(.+)$/i, |
| 897 | )?.[1]; |
| 898 | if (saved) return [saved]; |
| 899 | try { |
| 900 | const image: unknown = JSON.parse(line); |
| 901 | if ( |
| 902 | record(image) && |
| 903 | typeof image.path === "string" && |
| 904 | typeof image.width === "number" && |
| 905 | image.width > 0 && |
| 906 | typeof image.height === "number" && |
| 907 | image.height > 0 |
| 908 | ) |
| 909 | return [image.path]; |
| 910 | } catch {} |
| 911 | return []; |
| 912 | }); |
| 913 | return captures.filter( |
| 914 | (a) => |
| 915 | a.unmasked && |
| 916 | ((paths.length === 0 && a.executed && !a.sequenced) || |
| 917 | paths.some( |
| 918 | (path) => |
| 919 | path === a.args[1] || |
| 920 | (a.args[1]?.startsWith("$/") && path.endsWith(a.args[1].slice(1))), |
| 921 | )), |
| 922 | ); |
| 923 | }; |
| 924 | const recovered = |
| 925 | expect.reportOrCapture && |
| 926 | (run.calls.some((call) => call.tool === "device_screenshot" && usableImage(call)) || |
| 927 | shell.some((entry) => successfulScreenshots(entry).length > 0)); |
| 928 | if (expect.noShutdown) |
| 929 | add( |
| 930 | "callsMustNot shutdown occupied device", |
| 931 | !run.calls.some( |
| 932 | (call, index) => |
| 933 | call.tool === "device_close" && |
| 934 | record(call.input) && |
| 935 | call.input.shutdown === true && |
| 936 | openTargets(run.calls.slice(0, index)).some( |
| 937 | (target) => |
| 938 | expect.noShutdown!.some( |
| 939 | (protectedTarget) => |
| 940 | target.hostId === protectedTarget.hostId && |
| 941 | target.deviceId === protectedTarget.deviceId, |
| 942 | ) && |
| 943 | (!inputString(call.input, "hostId") || |
| 944 | inputString(call.input, "hostId") === target.hostId) && |
| 945 | (!inputString(call.input, "deviceId") || |
| 946 | inputString(call.input, "deviceId") === target.deviceId), |
| 947 | ), |
| 948 | ), |
| 949 | ); |
| 950 | for (const pattern of expect.report ?? []) |
| 951 | add( |
| 952 | `device final report /${pattern}/`, |
| 953 | run.stopReason === "end_turn" && |
| 954 | (!!recovered || new RegExp(pattern).test(run.finalText ?? "")), |
| 955 | ); |
| 956 | if (expect.noRawUi) add("shellMustNot device UI", !allActions.some((action) => action.raw)); |
| 957 | if (expect.maxCaptureAttempts !== undefined) |
| 958 | add( |
| 959 | `maxCalls device captures <= ${expect.maxCaptureAttempts}`, |
| 960 | run.calls.filter((call) => call.tool === "device_screenshot").length + |
| 961 | allActions.filter((action) => action.managed && action.args[0] === "screenshot").length <= |
| 962 | expect.maxCaptureAttempts, |
| 963 | ); |
| 964 | if (expect.capture) { |
| 965 | const target = expect.capture; |
| 966 | const matches = (value: ReturnType<typeof inputTarget> | undefined) => |
| 967 | value?.hostId === target.hostId && value?.deviceId === target.deviceId; |
| 968 | const appSessions = new Set<string>(); |
| 969 | let opened = false, |
| 970 | captured = false; |
| 971 | for (const [index, call] of run.calls.entries()) { |
| 972 | if (call.tool === "device_open" && call.failed === false && matches(inputTarget(call))) |
| 973 | opened = true; |
| 974 | const appReady = !target.app || (call.failed === false && appSessions.size > 0); |
| 975 | if ( |
| 976 | opened && |
| 977 | appReady && |
| 978 | call.tool === "device_screenshot" && |
| 979 | matches(screenshotTarget(call, run.calls.slice(0, index))) && |
| 980 | (!target.app || usableImage(call)) |
| 981 | ) |
| 982 | captured = true; |
| 983 | const entry = shell.find(([source]) => source === call); |
| 984 | const actions = (entry?.[1] ?? []).filter((action) => action.managed && action.bound); |
| 985 | const screenshots = entry ? successfulScreenshots(entry) : []; |
| 986 | for (const action of actions) { |
| 987 | const { args, words } = action; |
| 988 | const onTarget = matches(launcherTarget(words, run.calls.slice(0, index))); |
| 989 | // Only an open or close that ran and succeeded changes the app session. |
| 990 | const ran = |
| 991 | call.failed === false && |
| 992 | action.executed && |
| 993 | action.unmasked && |
| 994 | !AGENT_DEVICE_ERROR.test(call.output ?? ""); |
| 995 | if (target.app && ["open", "close"].includes(args[0] ?? "") && ran) { |
| 996 | appSessions.delete(words[0]!); |
| 997 | if ( |
| 998 | args[0] === "open" && |
| 999 | args[1] === target.app && |
| 1000 | onTarget && |
| 1001 | (!action.sequenced || |
| 1002 | (actions.filter((a) => a.args[0] === "open").length === 1 && |
| 1003 | /^Opened:/m.test(call.output ?? ""))) |
| 1004 | ) |
| 1005 | appSessions.add(words[0]!); |
| 1006 | } |
| 1007 | if ( |
| 1008 | opened && |
| 1009 | onTarget && |
| 1010 | (target.app |
| 1011 | ? appSessions.has(words[0]!) && screenshots.includes(action) |
| 1012 | : action.executed && |
| 1013 | (["screenshot", "snapshot"].includes(args[0] ?? "") || |
| 1014 | (args[0] === "record" && args[1] === "start"))) |
| 1015 | ) |
| 1016 | captured = true; |
| 1017 | } |
| 1018 | } |
| 1019 | add("device successful open and capture attempt", captured); |
| 1020 | } |
| 1021 | if (expect.recording !== undefined) { |
| 1022 | const steps = [...shell].flatMap(([call, actions]) => { |
| 1023 | const operations = actions.filter( |
| 1024 | ({ args }) => args[0] === "record" && /^(?:start|stop)$/.test(args[1] ?? ""), |
| 1025 | ); |
| 1026 | return operations |
| 1027 | .filter( |
| 1028 | (action) => |
| 1029 | action.executed && |
| 1030 | action.unmasked && |
| 1031 | action.bound && |
| 1032 | ((!action.sequenced && |
| 1033 | call.failed === false && |
| 1034 | !AGENT_DEVICE_ERROR.test(call.output ?? "")) || |
| 1035 | (operations.filter((o) => o.args[1] === action.args[1]).length === 1 && |
| 1036 | new RegExp( |
| 1037 | `(?:^|\\n)[^\\w\\n]*Recording ${action.args[1] === "start" ? "started" : "stopped"}\\b|"kind"\\s*:\\s*"recording${action.args[1] === "start" ? "Started" : "Stopped"}"`, |
| 1038 | "i", |
| 1039 | ).test(call.output ?? ""))), |
| 1040 | ) |
| 1041 | .map(({ args }) => args[1]!); |
| 1042 | }); |
| 1043 | const start = steps.indexOf("start"); |
| 1044 | checks.push({ |
| 1045 | check: `device recording ${expect.recording}`, |
| 1046 | pass: |
| 1047 | start >= 0 && |
| 1048 | steps.indexOf("stop", start + 1) >= 0 && |
| 1049 | run.stopReason === "end_turn" && |
| 1050 | (run.finalText ?? "").includes(expect.recording), |
| 1051 | }); |
| 1052 | } |
| 1053 | if (expect.maxFailedAttemptsPerTurn !== undefined) { |
| 1054 | const failedByTurn = new Map<number, number>(); |
| 1055 | for (const call of run.calls) { |
| 1056 | const actions = shell.find(([source]) => source === call)?.[1] ?? []; |
| 1057 | // A failed raw attempt, such as agent-device over ssh, is still a failed device attempt. |
| 1058 | const device = actions.filter((action) => action.managed || action.raw).length; |
| 1059 | // agent-device prints one error per failed command, even when a later command succeeds. |
| 1060 | const errors = (call.output ?? "").match(new RegExp(AGENT_DEVICE_ERROR, "g"))?.length ?? 0; |
| 1061 | let discoveryFailed = false; |
| 1062 | if (call.tool === "device_list") { |
| 1063 | try { |
| 1064 | const statuses: unknown = JSON.parse(call.output ?? "").hostStatuses; |
| 1065 | discoveryFailed = |
| 1066 | record(statuses) && |
| 1067 | Object.values(statuses).some((s) => record(s) && s.status === "failed"); |
| 1068 | } catch {} |
| 1069 | } |
| 1070 | const attempts = |
| 1071 | call.tool === "device_list" |
| 1072 | ? Number(!!call.failed || discoveryFailed) |
| 1073 | : /^device_(?:open|screenshot|close)$/.test(call.tool) |
| 1074 | ? Number(!!call.failed) |
| 1075 | : call.failed && device |
| 1076 | ? Math.max(1, Math.min(errors, device)) |
| 1077 | : Math.min(errors, device); |
| 1078 | const turn = call.turn ?? 0; |
| 1079 | failedByTurn.set(turn, (failedByTurn.get(turn) ?? 0) + attempts); |
| 1080 | } |
| 1081 | const max = expect.maxFailedAttemptsPerTurn; |
| 1082 | add( |
| 1083 | `maxCalls device failures per user turn <= ${max}`, |
| 1084 | [...failedByTurn.values()].every((count) => count <= max), |
| 1085 | ); |
| 1086 | } |
| 1087 | return checks; |
| 1088 | } |