The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | #!/usr/bin/env node |
| 2 | // @effect-diagnostics nodeBuiltinImport:off globalJsonInEffect:off - owns Metro and writes result files. |
| 3 | /** |
| 4 | * Drives the T3 Code iOS dev client on one simulator against an isolated server with scripted |
| 5 | * replies, and grades app behavior from the dev-only probe and server state. See README.md. |
| 6 | */ |
| 7 | import * as NodeChildProcess from "node:child_process"; |
| 8 | import * as NodeFS from "node:fs"; |
| 9 | import * as NodeOS from "node:os"; |
| 10 | import * as NodePath from "node:path"; |
| 11 | import * as NodeRuntime from "@effect/platform-node/NodeRuntime"; |
| 12 | import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; |
| 13 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 14 | import * as Clock from "effect/Clock"; |
| 15 | import * as Console from "effect/Console"; |
| 16 | import * as Effect from "effect/Effect"; |
| 17 | import * as Option from "effect/Option"; |
| 18 | import { Command, Flag } from "effect/cli"; |
| 19 | import { FetchHttpClient, HttpClient } from "effect/http"; |
| 20 | |
| 21 | import { CASES, type MobileBehaviorCase } from "./cases.ts"; |
| 22 | import { APP_ID, DeviceError, makeDevice, SCHEME } from "./device.ts"; |
| 23 | import { freePort, startEnvironment, uuid } from "./environment.ts"; |
| 24 | import { blockingResults, checkFails, resultsFor, summarize, type SampleRow } from "./grade.ts"; |
| 25 | import { SCENARIOS, type Pose, type ScenarioContext } from "./scenarios.ts"; |
| 26 | |
| 27 | const REPO_ROOT = NodePath.resolve(import.meta.dirname, "../.."); |
| 28 | const POSES = ["phone", "duo-closed", "duo-open", "ipad"] as const satisfies ReadonlyArray<Pose>; |
| 29 | |
| 30 | /** This machine's address as the simulator sees it: the tailnet address, else the first LAN IPv4. */ |
| 31 | function reachableHost(): string { |
| 32 | const tailnet = NodeChildProcess.spawnSync("tailscale", ["ip", "-4"], { encoding: "utf8" }); |
| 33 | const ip = tailnet.status === 0 ? tailnet.stdout.trim().split("\n")[0] : undefined; |
| 34 | if (ip) return ip; |
| 35 | const lan = Object.values(NodeOS.networkInterfaces()) |
| 36 | .flatMap((entries) => entries ?? []) |
| 37 | .find((entry) => entry.family === "IPv4" && !entry.internal); |
| 38 | return lan?.address ?? "127.0.0.1"; |
| 39 | } |
| 40 | |
| 41 | /** Metro with the behavior probe compiled in, stopped with its process group when the scope ends. */ |
| 42 | const startMetro = Effect.fn("run.startMetro")(function* (host: string, out: string) { |
| 43 | const port = yield* freePort(); |
| 44 | const log = NodeFS.openSync(NodePath.join(out, "metro.log"), "w"); |
| 45 | yield* Effect.acquireRelease( |
| 46 | Effect.sync(() => |
| 47 | NodeChildProcess.spawn( |
| 48 | "npx", |
| 49 | ["expo", "start", "--dev-client", "--scheme", SCHEME, "--lan", "--port", String(port)], |
| 50 | { |
| 51 | cwd: NodePath.join(REPO_ROOT, "apps/mobile"), |
| 52 | detached: true, |
| 53 | stdio: ["ignore", log, log], |
| 54 | env: { |
| 55 | ...process.env, |
| 56 | APP_VARIANT: "development", |
| 57 | CI: "1", |
| 58 | // Lazily fetched chunks sometimes arrive broken on pool emulators and throw SyntaxErrors. |
| 59 | EXPO_NO_METRO_LAZY: "1", |
| 60 | EXPO_PUBLIC_T3_BEHAVIOR_PROBE: "1", |
| 61 | REACT_NATIVE_PACKAGER_HOSTNAME: host, |
| 62 | }, |
| 63 | }, |
| 64 | ), |
| 65 | ), |
| 66 | (child) => Effect.sync(() => child.pid !== undefined && process.kill(-child.pid, "SIGTERM")), |
| 67 | ); |
| 68 | const metro = `http://${host}:${port}`; |
| 69 | for (let attempt = 0; attempt < 120; attempt++) { |
| 70 | const ready = yield* HttpClient.get(`${metro}/status`).pipe( |
| 71 | Effect.flatMap((response) => response.text), |
| 72 | Effect.provide(FetchHttpClient.layer), |
| 73 | Effect.orElseSucceed(() => ""), |
| 74 | ); |
| 75 | if (ready.includes("running")) return metro; |
| 76 | yield* Effect.sleep("1 second"); |
| 77 | } |
| 78 | return yield* new DeviceError({ message: `Metro never started; see ${out}/metro.log.` }); |
| 79 | }); |
| 80 | |
| 81 | const decodeRows = (file: string): ReadonlyArray<SampleRow> => |
| 82 | NodeFS.readFileSync(file, "utf8") |
| 83 | .split("\n") |
| 84 | .filter(Boolean) |
| 85 | .map((line) => JSON.parse(line) as SampleRow); |
| 86 | |
| 87 | const command = Command.make( |
| 88 | "mobile-behavior", |
| 89 | { |
| 90 | out: Flag.String("out").pipe(Flag.withDescription("Directory for results and run state.")), |
| 91 | device: Flag.String("device").pipe( |
| 92 | Flag.withDescription( |
| 93 | "Simulator UDID; leave out for a Simlock lease, whose gateway picks it.", |
| 94 | ), |
| 95 | Flag.withDefault(""), |
| 96 | ), |
| 97 | deviceConfig: Flag.String("device-config").pipe( |
| 98 | Flag.withDescription("The --config path device_open returned for this device."), |
| 99 | ), |
| 100 | deviceSession: Flag.String("device-session").pipe( |
| 101 | Flag.withDescription("The --session name device_open returned for this device."), |
| 102 | ), |
| 103 | deviceName: Flag.String("device-name").pipe( |
| 104 | Flag.withDescription( |
| 105 | 'Simulator name as Metro lists it, e.g. "iPhone 17 Pro". Leave out when this run\'s Metro serves one device.', |
| 106 | ), |
| 107 | Flag.withDefault(""), |
| 108 | ), |
| 109 | pose: Flag.Literals("pose", POSES).pipe(Flag.withDefault<Pose>("phone")), |
| 110 | label: Flag.String("label").pipe( |
| 111 | Flag.withDescription("Column name in the scoreboard; defaults to the pose."), |
| 112 | Flag.optional, |
| 113 | ), |
| 114 | cases: Flag.String("case").pipe( |
| 115 | Flag.withDescription("Comma-separated case ids."), |
| 116 | Flag.optional, |
| 117 | ), |
| 118 | samples: Flag.Int("samples").pipe(Flag.withDefault(3)), |
| 119 | baseline: Flag.String("baseline").pipe( |
| 120 | Flag.withDescription("results.jsonl from an earlier run to compare against."), |
| 121 | Flag.optional, |
| 122 | ), |
| 123 | agentDevice: Flag.String("agent-device").pipe( |
| 124 | Flag.withDefault( |
| 125 | NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"), |
| 126 | ), |
| 127 | ), |
| 128 | forgetOtherEnvironments: Flag.Boolean("forget-other-environments").pipe( |
| 129 | Flag.withDescription( |
| 130 | "Remove the app's other saved environments first. Only for a simulator kept for this eval.", |
| 131 | ), |
| 132 | Flag.withDefault(false), |
| 133 | ), |
| 134 | host: Flag.String("host").pipe( |
| 135 | Flag.withDescription( |
| 136 | "Address the simulator uses to reach this machine; defaults to the tailnet IP.", |
| 137 | ), |
| 138 | Flag.optional, |
| 139 | ), |
| 140 | }, |
| 141 | Effect.fn("mobileBehavior.main")(function* (flags) { |
| 142 | const out = NodePath.resolve(flags.out); |
| 143 | // Read first: the baseline may be this --out's own results.jsonl, which the run rewrites. |
| 144 | const baseline = Option.map(flags.baseline, (file) => resultsFor(decodeRows(file), CASES)); |
| 145 | NodeFS.mkdirSync(out, { recursive: true }); |
| 146 | const label = Option.getOrElse(flags.label, () => flags.pose); |
| 147 | const wanted = Option.map(flags.cases, (ids) => new Set(ids.split(",").map((id) => id.trim()))); |
| 148 | const selected = CASES.filter( |
| 149 | (testCase) => |
| 150 | Option.match(wanted, { onNone: () => true, onSome: (ids) => ids.has(testCase.id) }) && |
| 151 | (testCase.poses?.includes(flags.pose) ?? true), |
| 152 | ); |
| 153 | const unknown = Option.match(wanted, { |
| 154 | onNone: () => [], |
| 155 | onSome: (ids) => [...ids].filter((id) => !CASES.some((testCase) => testCase.id === id)), |
| 156 | }); |
| 157 | if (unknown.length > 0) |
| 158 | return yield* new DeviceError({ message: `Unknown case id(s): ${unknown.join(", ")}.` }); |
| 159 | if (selected.length === 0) return yield* new DeviceError({ message: "No cases match." }); |
| 160 | const host = Option.getOrElse(flags.host, reachableHost); |
| 161 | const startedAt = yield* Clock.currentTimeMillis; |
| 162 | |
| 163 | const env = yield* startEnvironment({ baseDir: NodePath.join(out, "env"), deviceHost: host }); |
| 164 | const tag = (yield* uuid).slice(0, 4); |
| 165 | yield* Console.error(`Seeding a 60-turn thread on ${env.origin}...`); |
| 166 | const longThread = yield* env.longThread(`Long settled thread ${tag}`, 60); |
| 167 | |
| 168 | const metro = yield* startMetro(host, out); |
| 169 | const device = makeDevice({ |
| 170 | agentDevice: flags.agentDevice, |
| 171 | targetArgs: [ |
| 172 | "--platform", |
| 173 | "ios", |
| 174 | ...(flags.device ? ["--udid", flags.device] : []), |
| 175 | "--config", |
| 176 | flags.deviceConfig, |
| 177 | "--session", |
| 178 | flags.deviceSession, |
| 179 | ], |
| 180 | metro, |
| 181 | deviceName: flags.deviceName, |
| 182 | }); |
| 183 | yield* Console.error(`Opening the dev client against ${metro}...`); |
| 184 | // agent-device answers iOS's "Open in" prompt; the flags keep dev-menu chrome off the app. |
| 185 | const metroUrl = new URL(metro); |
| 186 | const launchUrl = `${SCHEME}://expo-development-client/?url=${encodeURIComponent(metro)}&disableOnboarding=1&__expo_disable_fab=1&__expo_disable_auto_launch=1`; |
| 187 | yield* device |
| 188 | .run( |
| 189 | [ |
| 190 | "open", |
| 191 | APP_ID, |
| 192 | "--relaunch", |
| 193 | "--metro-host", |
| 194 | metroUrl.hostname, |
| 195 | "--metro-port", |
| 196 | metroUrl.port, |
| 197 | "--launch-url", |
| 198 | launchUrl, |
| 199 | ], |
| 200 | 240_000, |
| 201 | ) |
| 202 | .pipe(Effect.retry({ times: 2 })); |
| 203 | for (let attempt = 0; ; attempt++) { |
| 204 | const loaded = yield* device |
| 205 | .evaluate("typeof __t3Behavior") |
| 206 | .pipe(Effect.orElseSucceed(() => "")); |
| 207 | if (loaded === "object") break; |
| 208 | // iOS asks before a URL opens another app; busy hosts can miss the first accept. |
| 209 | if (attempt % 5 === 0) yield* device.acceptOpenPrompt.pipe(Effect.ignore); |
| 210 | if (attempt > 120) |
| 211 | return yield* new DeviceError({ |
| 212 | message: |
| 213 | "The probe never loaded. Is this a dev client built from this checkout's native fingerprint?", |
| 214 | }); |
| 215 | yield* Effect.sleep("2 seconds"); |
| 216 | } |
| 217 | if (flags.pose === "duo-closed" || flags.pose === "duo-open") |
| 218 | yield* device.fold(flags.pose === "duo-closed" ? "closed" : "open"); |
| 219 | const forget = (ids: ReadonlyArray<string>) => |
| 220 | device |
| 221 | .evaluate(`__t3Behavior.forgetEnvironments(${JSON.stringify(ids)})`) |
| 222 | .pipe(Effect.andThen(Effect.sleep("3 seconds"))); |
| 223 | const pairing = yield* env.pairingUrl(); |
| 224 | yield* device.openUrl( |
| 225 | `${SCHEME}://connections/new?pairingUrl=${encodeURIComponent(pairing)}&autoConnect=1`, |
| 226 | ); |
| 227 | // Registered once pairing starts, so a stopped run never leaves its environment in the app. |
| 228 | yield* Effect.addFinalizer(() => forget([env.environmentId]).pipe(Effect.ignore)); |
| 229 | const seeded = (state: { homeList: unknown; threads: Record<string, string> }) => |
| 230 | state.homeList !== null && longThread in state.threads; |
| 231 | yield* device.waitFor("the paired environment's threads on Home", seeded, 60_000); |
| 232 | // Other saved environments keep cached threads (some stuck working) that skew status cases. |
| 233 | const others = (yield* device.state).environments.filter((id) => id !== env.environmentId); |
| 234 | if (others.length > 0 && !flags.forgetOtherEnvironments) |
| 235 | return yield* new DeviceError({ |
| 236 | message: `The app has ${others.length} other saved environment(s). Use a simulator with no other connections, or pass --forget-other-environments to remove them.`, |
| 237 | }); |
| 238 | if (others.length > 0) yield* forget(others); |
| 239 | const ctx: ScenarioContext = { |
| 240 | env, |
| 241 | device, |
| 242 | pose: flags.pose, |
| 243 | fixtures: { longThread }, |
| 244 | }; |
| 245 | const scenarios = [...new Set(selected.map((testCase) => testCase.scenario))]; |
| 246 | const rows: SampleRow[] = []; |
| 247 | const resultsFile = NodeFS.openSync(NodePath.join(out, "results.jsonl"), "w"); |
| 248 | for (const scenario of scenarios) { |
| 249 | const cases = selected.filter((testCase) => testCase.scenario === scenario); |
| 250 | for (let sample = 1; sample <= flags.samples; sample++) { |
| 251 | const outcome = yield* env.interruptAll().pipe( |
| 252 | Effect.andThen(env.archiveAllExcept(longThread)), |
| 253 | Effect.andThen(device.reload(seeded)), |
| 254 | Effect.andThen(SCENARIOS[scenario](ctx)), |
| 255 | Effect.map((metrics) => ({ metrics, error: undefined })), |
| 256 | Effect.catch((error) => Effect.succeed({ metrics: {}, error: error.message })), |
| 257 | // A bad probe reply or snapshot must cost one sample, not the run. |
| 258 | Effect.catchDefect((defect) => Effect.succeed({ metrics: {}, error: String(defect) })), |
| 259 | ); |
| 260 | for (const testCase of cases) { |
| 261 | const failures = outcome.error |
| 262 | ? [] |
| 263 | : testCase.checks.flatMap((check) => checkFails(check, outcome.metrics) ?? []); |
| 264 | const row: SampleRow = { |
| 265 | id: testCase.id, |
| 266 | device: label, |
| 267 | sample, |
| 268 | pass: outcome.error ? null : failures.length === 0, |
| 269 | failures, |
| 270 | metrics: outcome.metrics, |
| 271 | ...(outcome.error ? { error: outcome.error } : {}), |
| 272 | }; |
| 273 | rows.push(row); |
| 274 | NodeFS.writeSync(resultsFile, `${JSON.stringify(row)}\n`); |
| 275 | yield* Console.error( |
| 276 | `${testCase.id} #${sample}: ${row.pass === null ? row.error : row.pass ? "pass" : `FAIL ${failures.join("; ")}`}`, |
| 277 | ); |
| 278 | } |
| 279 | } |
| 280 | } |
| 281 | NodeFS.closeSync(resultsFile); |
| 282 | const minutes = Math.round(((yield* Clock.currentTimeMillis) - startedAt) / 60_000); |
| 283 | const results = resultsFor(rows, CASES); |
| 284 | const summary = summarize({ |
| 285 | cases: selected as ReadonlyArray<MobileBehaviorCase>, |
| 286 | results, |
| 287 | ...Option.match(baseline, { onNone: () => ({}), onSome: (rows) => ({ baseline: rows }) }), |
| 288 | header: `# Mobile behavior: ${label}\n\n${flags.deviceName || label}, ${flags.samples} samples per case, ${minutes} min. Cells show pass or FAIL and the key metric's median.`, |
| 289 | }); |
| 290 | NodeFS.writeFileSync(NodePath.join(out, "summary.md"), summary); |
| 291 | yield* Console.log(summary); |
| 292 | if (blockingResults(results, CASES).length > 0) process.exitCode = 1; |
| 293 | }, Effect.scoped), |
| 294 | ); |
| 295 | |
| 296 | Command.run(command, { version: "0.0.0" }).pipe( |
| 297 | Effect.provide([NodeServices.layer, NodeCrypto.layer]), |
| 298 | NodeRuntime.runMain, |
| 299 | ); |