The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | #!/usr/bin/env node |
| 2 | // @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off globalFetch:off - Host-side device automation uses Node subprocess and filesystem APIs directly. |
| 3 | |
| 4 | import * as NodeChildProcess from "node:child_process"; |
| 5 | import * as NodeFS from "node:fs"; |
| 6 | import * as NodeFSP from "node:fs/promises"; |
| 7 | import * as NodeOS from "node:os"; |
| 8 | import * as NodePath from "node:path"; |
| 9 | import * as NodeProcess from "node:process"; |
| 10 | import * as NodeReadline from "node:readline"; |
| 11 | import * as NodeURL from "node:url"; |
| 12 | |
| 13 | import { CASES } from "./mobile-behavior/cases.ts"; |
| 14 | import { blockingResults, resultsFor, summarize, type SampleRow } from "./mobile-behavior/grade.ts"; |
| 15 | import { Device, type Pool, poolError, run } from "./mobile-e2e/device.ts"; |
| 16 | import type { Pose } from "./mobile-e2e/poses.ts"; |
| 17 | |
| 18 | const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), ".."); |
| 19 | |
| 20 | /** The devices the behavior cases run on; each runs in parallel with its own server and Metro. */ |
| 21 | const TARGETS = [ |
| 22 | { id: "iphone-18-pro", device: "iPhone 18 Pro", form: "phone", behaviorPose: "phone" }, |
| 23 | { id: "ipad-pro-13", device: "iPad Pro 13-inch (M5)", form: "tablet", behaviorPose: "ipad" }, |
| 24 | ] as const; |
| 25 | type Target = (typeof TARGETS)[number]; |
| 26 | |
| 27 | const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`); |
| 28 | |
| 29 | interface TargetRun { |
| 30 | readonly target: Target; |
| 31 | readonly directory: string; |
| 32 | readonly error?: string; |
| 33 | } |
| 34 | |
| 35 | /** Keeps a lease alive while the cases run: they go minutes without an agent-device call. */ |
| 36 | function renewWhileRunning(pool: Pool, stateDir: string, session: string, token: string) { |
| 37 | const gateway = pool.agentDeviceUrl.replace(/\/v1\/agent-device\/?$/, ""); |
| 38 | const timer = setInterval(() => { |
| 39 | const record = NodePath.join(stateDir, "remote-connections", `${session}.json`); |
| 40 | void NodeFSP.readFile(record, "utf8") |
| 41 | .then((text) => (JSON.parse(text) as { leaseId: string }).leaseId) |
| 42 | .then((leaseId) => |
| 43 | fetch(`${gateway}/v1/leases/${encodeURIComponent(leaseId)}/renew`, { |
| 44 | method: "POST", |
| 45 | headers: { authorization: `Bearer ${token}` }, |
| 46 | }), |
| 47 | ) |
| 48 | .catch(() => undefined); |
| 49 | }, 90_000); |
| 50 | return () => clearInterval(timer); |
| 51 | } |
| 52 | |
| 53 | async function runTarget( |
| 54 | target: Target, |
| 55 | pool: Pool, |
| 56 | runId: string, |
| 57 | outputDirectory: string, |
| 58 | config: string, |
| 59 | stateDir: string, |
| 60 | token: string, |
| 61 | passThrough: ReadonlyArray<string>, |
| 62 | ): Promise<TargetRun> { |
| 63 | const directory = NodePath.join(outputDirectory, target.id); |
| 64 | await NodeFSP.mkdir(directory, { recursive: true }); |
| 65 | const pose: Pose = { |
| 66 | id: target.id, |
| 67 | platform: "ios", |
| 68 | device: target.device, |
| 69 | form: target.form, |
| 70 | appearance: "light", |
| 71 | }; |
| 72 | let device: Device | null = null; |
| 73 | for (let attempt = 1; !device; attempt++) { |
| 74 | try { |
| 75 | log(target.id, `leasing ${target.device}`); |
| 76 | device = await Device.lease(pose, pool, attempt === 1 ? runId : `${runId}-retry`); |
| 77 | log(target.id, "installing the dev client"); |
| 78 | await device.install(); |
| 79 | } catch (error) { |
| 80 | const message = error instanceof Error ? error.message : String(error); |
| 81 | await device?.release(); |
| 82 | device = null; |
| 83 | if (attempt >= 2 || !poolError(message)) return { target, directory, error: message }; |
| 84 | log(target.id, `retrying after: ${message.split("\n")[0]}`); |
| 85 | } |
| 86 | } |
| 87 | const stopRenewing = renewWhileRunning(pool, stateDir, device.session, token); |
| 88 | try { |
| 89 | log(target.id, "running the behavior cases"); |
| 90 | const child = NodeChildProcess.spawn( |
| 91 | NodeProcess.execPath, |
| 92 | [ |
| 93 | "scripts/mobile-behavior/run.ts", |
| 94 | "--out", |
| 95 | directory, |
| 96 | "--label", |
| 97 | target.id, |
| 98 | "--pose", |
| 99 | target.behaviorPose, |
| 100 | "--device-config", |
| 101 | config, |
| 102 | "--device-session", |
| 103 | device.session, |
| 104 | "--agent-device", |
| 105 | pool.agentDevice, |
| 106 | ...passThrough, |
| 107 | ], |
| 108 | { cwd: REPO_ROOT, env: NodeProcess.env, stdio: ["ignore", "pipe", "pipe"] }, |
| 109 | ); |
| 110 | const transcript = NodeFS.createWriteStream(NodePath.join(directory, "run.log")); |
| 111 | // The scoreboard and any fatal error go to stdout, progress to stderr; the transcript keeps both. |
| 112 | NodeReadline.createInterface({ input: child.stdout }).on("line", (line) => |
| 113 | transcript.write(`${line}\n`), |
| 114 | ); |
| 115 | NodeReadline.createInterface({ input: child.stderr }).on("line", (line) => { |
| 116 | transcript.write(`${line}\n`); |
| 117 | if (/#\d+: /.test(line)) log(target.id, line); |
| 118 | }); |
| 119 | const code = await new Promise<number>((resolve) => |
| 120 | child.on("close", (exit) => resolve(exit ?? 1)), |
| 121 | ); |
| 122 | transcript.end(); |
| 123 | const hasResults = NodeFS.existsSync(NodePath.join(directory, "results.jsonl")); |
| 124 | return hasResults || code === 0 |
| 125 | ? { target, directory } |
| 126 | : { target, directory, error: `the cases stopped (exit ${code}); see ${directory}/run.log` }; |
| 127 | } finally { |
| 128 | stopRenewing(); |
| 129 | await device.release(); |
| 130 | } |
| 131 | } |
| 132 | |
| 133 | async function main() { |
| 134 | const args = NodeProcess.argv.slice(2); |
| 135 | const values = (flag: string) => |
| 136 | args.flatMap((arg, index) => (args[index - 1] === flag ? [arg] : [])); |
| 137 | if (args.includes("--list")) { |
| 138 | for (const target of TARGETS) NodeProcess.stdout.write(`${target.id}\t${target.device}\n`); |
| 139 | for (const testCase of CASES) |
| 140 | NodeProcess.stdout.write(`${testCase.id}\t${testCase.kind}\t${testCase.source}\n`); |
| 141 | return; |
| 142 | } |
| 143 | const selected = values("--device"); |
| 144 | const unknown = selected.filter((id) => !TARGETS.some((target) => target.id === id)); |
| 145 | if (unknown.length) throw new Error(`Unknown device ${unknown.join(", ")}. Run with --list.`); |
| 146 | const targets = TARGETS.filter((target) => selected.length === 0 || selected.includes(target.id)); |
| 147 | const passThrough = [ |
| 148 | ...values("--samples").flatMap((value) => ["--samples", value]), |
| 149 | ...values("--case").flatMap((value) => ["--case", value]), |
| 150 | ]; |
| 151 | |
| 152 | const agentDeviceUrl = NodeProcess.env.SIMLOCK_AGENT_DEVICE_URL; |
| 153 | const token = NodeProcess.env.AGENT_DEVICE_DAEMON_AUTH_TOKEN; |
| 154 | if (!agentDeviceUrl || !token) |
| 155 | throw new Error( |
| 156 | "Set SIMLOCK_AGENT_DEVICE_URL (<gateway>/v1/agent-device) and AGENT_DEVICE_DAEMON_AUTH_TOKEN (a Simlock agent token) to run on the device pool.", |
| 157 | ); |
| 158 | |
| 159 | const runId = new Date().toISOString().replaceAll(/[:.]/g, "-"); |
| 160 | const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-behavior", runId); |
| 161 | await NodeFSP.mkdir(outputDirectory, { recursive: true }); |
| 162 | const rootDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-mobile-behavior-")); |
| 163 | const workspace = await NodeFSP.realpath(outputDirectory); |
| 164 | const config = NodePath.join(rootDir, "agent-device.json"); |
| 165 | const stateDir = NodePath.join(rootDir, "agent-device"); |
| 166 | await NodeFSP.writeFile( |
| 167 | config, |
| 168 | JSON.stringify({ daemonBaseUrl: agentDeviceUrl, daemonAuthToken: token, stateDir }), |
| 169 | { mode: 0o600 }, |
| 170 | ); |
| 171 | const pool: Pool = { |
| 172 | repoRoot: REPO_ROOT, |
| 173 | agentDevice: NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"), |
| 174 | agentDeviceUrl, |
| 175 | workspace, |
| 176 | env: { ...NodeProcess.env, AGENT_DEVICE_CONFIG: config, PWD: workspace }, |
| 177 | }; |
| 178 | try { |
| 179 | // A cache miss builds the dev client, which can outlast a lease, so it is ready first. |
| 180 | log("behavior", "preparing the ios dev client"); |
| 181 | await run( |
| 182 | NodeProcess.execPath, |
| 183 | ["scripts/mobile-native-client.ts", "ensure", "ios", "--prepare"], |
| 184 | { cwd: REPO_ROOT, timeoutMs: 1_800_000 }, |
| 185 | ); |
| 186 | const runs = await Promise.all( |
| 187 | targets.map((target) => |
| 188 | runTarget(target, pool, runId, outputDirectory, config, stateDir, token, passThrough), |
| 189 | ), |
| 190 | ); |
| 191 | |
| 192 | const rows = runs.flatMap((entry) => { |
| 193 | const file = NodePath.join(entry.directory, "results.jsonl"); |
| 194 | if (!NodeFS.existsSync(file)) return []; |
| 195 | return NodeFS.readFileSync(file, "utf8") |
| 196 | .split("\n") |
| 197 | .filter(Boolean) |
| 198 | .map((line) => JSON.parse(line) as SampleRow); |
| 199 | }); |
| 200 | const results = resultsFor(rows, CASES); |
| 201 | const stopped = runs.filter((entry) => entry.error); |
| 202 | const summary = summarize({ |
| 203 | cases: CASES, |
| 204 | results, |
| 205 | header: [ |
| 206 | `# Mobile behavior · ${runId}`, |
| 207 | "", |
| 208 | "Cells show pass or FAIL, passed/samples, and the key metric's median. `bad` cases reproduce known bugs and are expected to fail until they are fixed.", |
| 209 | ...stopped.map((entry) => `\n**${entry.target.id} did not finish:** ${entry.error}`), |
| 210 | ].join("\n"), |
| 211 | }); |
| 212 | const summaryFile = NodePath.join(outputDirectory, "summary.md"); |
| 213 | await NodeFSP.writeFile(summaryFile, summary); |
| 214 | const blocking = blockingResults(results, CASES); |
| 215 | for (const result of blocking) |
| 216 | log(result.device, `${result.id}: ${result.pass === null ? "incomplete" : "FAIL"}`); |
| 217 | for (const entry of stopped) log(entry.target.id, entry.error!); |
| 218 | NodeProcess.stdout.write( |
| 219 | `${JSON.stringify({ summary: summaryFile, cases: results.length, blocking: blocking.length, stopped: stopped.length })}\n`, |
| 220 | ); |
| 221 | process.exitCode = blocking.length || stopped.length ? 1 : 0; |
| 222 | } finally { |
| 223 | await NodeFSP.rm(rootDir, { recursive: true, force: true }); |
| 224 | } |
| 225 | } |
| 226 | |
| 227 | if (import.meta.main) { |
| 228 | await main(); |
| 229 | } |