@devries-cameron20 / t3code Mirror · read-only / commits / ff71186669
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 14:30 UTC

test(mobile): vp run mobile:behavior grades app behavior on pool devices

Brings Theo's mobile behavior checks (#25876) into the local test suite beside mobile:e2e. A dev-only probe reads the app's own state through Metro's Hermes inspector, and a scripted Fx provider drives replies, so 25 cases grade scroll, frame drops, drafts, focus, status, and links without a model. The command leases an iPhone 18 Pro and an iPad Pro 13 from the Simlock pool, runs each with its own server and Metro, keeps the leases renewed, and writes one scoreboard to artifacts/mobile-behavior/. Pool errors retry, Metro serves no lazy chunks, and a run fails only on guard or control cases; bad cases reproduce known bugs. Co-Authored-By: Theo Browne <me@theo.li> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 27, 2026 at 21:31 UTC ff71186669c863aa976cee0fdb00c20c64ea92b4
18 files changed +3159 -3
.gitignore
+1
@@ -37,6 +37,7 @@ apps/mobile/.android-cache/
37 /.generated/
38 artifacts/app-store/screenshots/
39 artifacts/mobile-e2e/
40 +artifacts/mobile-behavior/
41 .github/pr-assets/
42 native/**/target/
43 node_modules/
apps/mobile/index.ts
+2 -1
@@ -1,5 +1,6 @@
1 -// Must stay first: the render counter wraps the DevTools hook before any React commit.
1 +// Must stay first: the render counter and behavior probe wrap the DevTools hook before any React commit.
2 import "./src/features/development/render-counter-entry";
3 +import "./src/features/development/behavior-probe-entry";
4 // Next, so errors thrown while the rest of the app loads are recorded too.
5 import "./src/features/observability/observe";
6 import { registerRootComponent } from "expo";
apps/mobile/src/features/development/behavior-probe-entry.ts new
+4
@@ -0,0 +1,4 @@
1 +// Metro folds this condition before collecting dependencies, excluding the probe from release bundles.
2 +if (__DEV__ && process.env.EXPO_PUBLIC_T3_BEHAVIOR_PROBE === "1") {
3 + require("./behavior-probe");
4 +}
apps/mobile/src/features/development/behavior-probe.ts new
+333
@@ -0,0 +1,333 @@
1 +// Read by evals/mobile-behavior through the Hermes inspector; product modules load lazily so import order is unchanged.
2 +
3 +interface Fiber {
4 + readonly tag: number;
5 + readonly type: unknown;
6 + readonly child: Fiber | null;
7 + readonly sibling: Fiber | null;
8 + readonly memoizedProps: Record<string, unknown> | null;
9 +}
10 +interface DevToolsHook {
11 + onCommitFiberRoot?: (id: unknown, root: { current: Fiber }, ...rest: unknown[]) => unknown;
12 + __t3BehaviorProbe?: true;
13 +}
14 +interface ListState {
15 + readonly scroll: number;
16 + readonly scrollLength: number;
17 + readonly contentLength: number;
18 + readonly isAtEnd: boolean;
19 + readonly start: number;
20 + readonly end: number;
21 + readonly data: ReadonlyArray<{ readonly id?: string; readonly key?: string }>;
22 + readonly positionAtIndex: (index: number) => number;
23 +}
24 +interface Feed {
25 + readonly threadId: string;
26 + readonly environmentId: string;
27 + readonly list: { readonly current: { getState?: () => ListState } | null } | undefined;
28 + following: boolean | null;
29 +}
30 +interface Atom {
31 + readonly _atom: true;
32 +}
33 +
34 +// Every root React has committed; LogBox and other overlays commit their own.
35 +const roots = new Set<{ current: Fiber }>();
36 +
37 +function installCommitHook() {
38 + const hook = (globalThis as { __REACT_DEVTOOLS_GLOBAL_HOOK__?: DevToolsHook })
39 + .__REACT_DEVTOOLS_GLOBAL_HOOK__;
40 + if (!hook || typeof hook.onCommitFiberRoot !== "function") {
41 + setTimeout(installCommitHook, 50);
42 + return;
43 + }
44 + if (hook.__t3BehaviorProbe) return;
45 + hook.__t3BehaviorProbe = true;
46 + const original = hook.onCommitFiberRoot.bind(hook);
47 + hook.onCommitFiberRoot = (id, next, ...rest) => {
48 + roots.add(next);
49 + return original(id, next, ...rest);
50 + };
51 +}
52 +
53 +/** Visits every mounted fiber; `enter` returns the context passed to the fiber's children. */
54 +function walk<C>(fiber: Fiber | null, context: C, enter: (fiber: Fiber, context: C) => C) {
55 + let node = fiber;
56 + while (node) {
57 + const next = enter(node, context);
58 + if (node.child) walk(node.child, next, enter);
59 + node = node.sibling;
60 + }
61 +}
62 +
63 +const readAtom = <T>(atom: Atom): T => {
64 + const { appAtomRegistry } = require("../../state/atom-registry") as {
65 + appAtomRegistry: { get: (atom: Atom) => unknown };
66 + };
67 + return appAtomRegistry.get(atom) as T;
68 +};
69 +
70 +/** Mounted thread feeds, the composer text the native editor was given, and the status pill. */
71 +function scanTree() {
72 + const feeds = new Map<string, Feed>();
73 + const editors: Array<{ value: string; expanded: boolean }> = [];
74 + let floatingStatus: string | null = null;
75 + let homeList: Array<{
76 + type: string;
77 + threadId?: string;
78 + status?: string;
79 + pinned?: boolean;
80 + count?: number;
81 + expanded?: boolean;
82 + }> | null = null;
83 + const enter = (fiber: Fiber, feed: Feed | null): Feed | null => {
84 + const props = fiber.memoizedProps;
85 + if (!props || typeof props !== "object") return feed;
86 + if ("listRef" in props && typeof props.threadId === "string" && "detailRef" in props) {
87 + const existing = feeds.get(props.threadId);
88 + if (existing) return existing;
89 + const created: Feed = {
90 + threadId: props.threadId,
91 + environmentId: String(props.environmentId),
92 + list: props.listRef as Feed["list"],
93 + following: null,
94 + };
95 + feeds.set(props.threadId, created);
96 + return created;
97 + }
98 + if (feed && "maintainScrollAtEnd" in props && "maintainVisibleContentPosition" in props)
99 + feed.following = props.maintainScrollAtEnd !== false;
100 + if (typeof props.value === "string" && "chipsInert" in props)
101 + editors.push({ value: props.value, expanded: props.chipsInert === false });
102 + if ("status" in props && "showScrollToEnd" in props) {
103 + const status = props.status as { kind?: string } | null;
104 + floatingStatus = status?.kind ?? null;
105 + }
106 + if (homeList === null && Array.isArray(props.data) && "renderItem" in props) {
107 + const data = props.data as ReadonlyArray<{
108 + type?: string;
109 + item?: { thread?: { id?: string }; pinned?: boolean };
110 + count?: number;
111 + expanded?: boolean;
112 + }>;
113 + if (data.some((entry) => entry?.type?.startsWith("v2-"))) {
114 + const { resolveThreadListV2Status } = require("../threads/threadListV2") as {
115 + resolveThreadListV2Status: (thread: unknown) => string;
116 + };
117 + homeList = data.map((entry) => ({
118 + type: String(entry.type),
119 + ...(entry.item?.thread?.id
120 + ? {
121 + threadId: entry.item.thread.id,
122 + status: resolveThreadListV2Status(entry.item.thread),
123 + pinned: entry.item.pinned === true,
124 + }
125 + : {}),
126 + ...(typeof entry.count === "number" ? { count: entry.count } : {}),
127 + ...(typeof entry.expanded === "boolean" ? { expanded: entry.expanded } : {}),
128 + }));
129 + }
130 + }
131 + return feed;
132 + };
133 + for (const root of roots) walk(root.current, null, enter);
134 + return { feeds: [...feeds.values()], editors, floatingStatus, homeList };
135 +}
136 +
137 +function feedState(feed: Feed) {
138 + const state = feed.list?.current?.getState?.();
139 + if (!state) return { threadId: feed.threadId, ready: false as const };
140 + const start = Math.max(0, state.start);
141 + const end = Math.min(state.data.length - 1, state.end);
142 + const rows: Record<string, number> = {};
143 + for (let index = start; index <= end; index++) {
144 + const row = state.data[index];
145 + const key = row?.id ?? row?.key;
146 + if (key) rows[key] = Math.round(state.positionAtIndex(index) - state.scroll);
147 + }
148 + return {
149 + threadId: feed.threadId,
150 + ready: true as const,
151 + scroll: Math.round(state.scroll),
152 + viewport: Math.round(state.scrollLength),
153 + content: Math.round(state.contentLength),
154 + fromEnd: Math.round(state.contentLength - state.scroll - state.scrollLength),
155 + atEnd: state.isAtEnd,
156 + following: feed.following,
157 + count: state.data.length,
158 + /** Screen y of each visible row, keyed by row id. */
159 + rows,
160 + };
161 +}
162 +
163 +function route() {
164 + const { rootNavigationRef } =
165 + require("../../lib/rootNavigation") as typeof import("../../lib/rootNavigation");
166 + if (!rootNavigationRef.isReady()) return null;
167 + const current = rootNavigationRef.getCurrentRoute();
168 + const { workspaceHasColumns } = require("../workspace/workspace-selection-store") as {
169 + workspaceHasColumns: () => boolean;
170 + };
171 + return {
172 + name: current?.name ?? null,
173 + params: (current?.params ?? {}) as Record<string, unknown>,
174 + columns: workspaceHasColumns(),
175 + };
176 +}
177 +
178 +function snapshot() {
179 + const scan = scanTree();
180 + const at = route();
181 + const threadId = typeof at?.params.threadId === "string" ? at.params.threadId : null;
182 + const environmentId =
183 + typeof at?.params.environmentId === "string" ? at.params.environmentId : null;
184 + const threadKey = threadId && environmentId ? `${environmentId}:${threadId}` : null;
185 + const { composerDraftsAtom } = require("../../state/use-composer-drafts") as {
186 + composerDraftsAtom: Atom;
187 + };
188 + const { threadComposerErrorsAtom } = require("../../state/thread-composer-error") as {
189 + threadComposerErrorsAtom: Atom;
190 + };
191 + const drafts = readAtom<Record<string, { text?: string }>>(composerDraftsAtom);
192 + const errors = readAtom<Record<string, { message: string }>>(threadComposerErrorsAtom);
193 + const { KeyboardController } = require("react-native-keyboard-controller") as {
194 + KeyboardController: { isVisible: () => boolean };
195 + };
196 + const feed = visibleFeed(scan.feeds);
197 + const { environmentThreadShells } = require("../../state/threads") as {
198 + environmentThreadShells: { threadShellsAtom: Atom };
199 + };
200 + const { resolveThreadListV2Status } = require("../threads/threadListV2") as {
201 + resolveThreadListV2Status: (thread: unknown) => string;
202 + };
203 + const shells = readAtom<ReadonlyArray<{ id: string; environmentId: string }>>(
204 + environmentThreadShells.threadShellsAtom,
205 + );
206 + return {
207 + /** Saved environments the app has threads from. */
208 + environments: [...new Set(shells.map((shell) => shell.environmentId))],
209 + /** The app's own status for every thread it knows, including rows folded into a shelf. */
210 + threads: Object.fromEntries(
211 + shells.map((shell) => [shell.id, resolveThreadListV2Status(shell)]),
212 + ),
213 + route: at,
214 + feed: feed ? feedState(feed) : null,
215 + composer: {
216 + key: threadKey,
217 + draft: threadKey ? (drafts[threadKey]?.text ?? "") : null,
218 + drafts: Object.fromEntries(
219 + Object.entries(drafts).map(([key, draft]) => [key, draft.text ?? ""]),
220 + ),
221 + editors: scan.editors,
222 + error: threadKey ? (errors[threadKey]?.message ?? null) : null,
223 + keyboard: KeyboardController.isVisible(),
224 + },
225 + floatingStatus: scan.floatingStatus,
226 + homeList: scan.homeList,
227 + };
228 +}
229 +
230 +const frames = { js: [] as number[], on: false };
231 +
232 +function jsFrame(time: number) {
233 + if (!frames.on) return;
234 + frames.js.push(time);
235 + requestAnimationFrame(jsFrame);
236 +}
237 +
238 +function uiFramesStart() {
239 + "worklet";
240 + const ui = globalThis as { __t3UiFrames?: number[]; __t3UiFramesOn?: boolean };
241 + ui.__t3UiFrames = [];
242 + ui.__t3UiFramesOn = true;
243 + const tick = (time: number) => {
244 + if (!ui.__t3UiFramesOn) return;
245 + ui.__t3UiFrames!.push(time);
246 + requestAnimationFrame(tick);
247 + };
248 + requestAnimationFrame(tick);
249 +}
250 +
251 +function uiFramesStop() {
252 + "worklet";
253 + const ui = globalThis as { __t3UiFrames?: number[]; __t3UiFramesOn?: boolean };
254 + ui.__t3UiFramesOn = false;
255 + return ui.__t3UiFrames ?? [];
256 +}
257 +
258 +let samples: Array<Record<string, unknown>> = [];
259 +let sampled: Feed | null = null;
260 +
261 +// The feed is found once per recording; a tree walk on every frame would itself cause jank.
262 +function sampleFeed(time: number) {
263 + if (!sampled) return;
264 + const { following: _following, ...state } = feedState(sampled);
265 + samples.push({ at: time, ...state });
266 + requestAnimationFrame(sampleFeed);
267 +}
268 +
269 +function visibleFeed(feeds: ReadonlyArray<Feed>) {
270 + const threadId = route()?.params.threadId;
271 + return feeds.find((feed) => feed.threadId === threadId) ?? feeds.at(-1) ?? null;
272 +}
273 +
274 +installCommitHook();
275 +
276 +Object.assign(globalThis, {
277 + __t3Behavior: {
278 + open(url: string) {
279 + const { Linking } = require("react-native") as typeof import("react-native");
280 + void Linking.openURL(url);
281 + return true;
282 + },
283 + /** Restarts the JS app; deferred so the inspector call can reply first. */
284 + reload() {
285 + const { DevSettings } = require("react-native") as typeof import("react-native");
286 + setTimeout(() => DevSettings.reload(), 100);
287 + return true;
288 + },
289 + state: () => JSON.stringify(snapshot()),
290 + /** Removes these saved environments (and their cached threads) through the app's own command. */
291 + async forgetEnvironments(ids: ReadonlyArray<string>) {
292 + const { runAtomCommand } = require("@t3tools/client-runtime/state/runtime") as {
293 + runAtomCommand: (
294 + registry: unknown,
295 + command: unknown,
296 + value: unknown,
297 + options: object,
298 + ) => Promise<unknown>;
299 + };
300 + const { appAtomRegistry } = require("../../state/atom-registry") as {
301 + appAtomRegistry: unknown;
302 + };
303 + const { environmentCatalog } = require("../../connection/catalog") as {
304 + environmentCatalog: { remove: unknown };
305 + };
306 + for (const id of ids)
307 + await runAtomCommand(appAtomRegistry, environmentCatalog.remove, id, {
308 + label: "behavior probe",
309 + });
310 + return true;
311 + },
312 + /** Records JS and UI frame times, and the visible feed on every JS frame. */
313 + recordStart(withFeed: boolean) {
314 + const { scheduleOnUI } =
315 + require("react-native-worklets") as typeof import("react-native-worklets");
316 + frames.js = [];
317 + frames.on = true;
318 + requestAnimationFrame(jsFrame);
319 + scheduleOnUI(uiFramesStart);
320 + samples = [];
321 + sampled = withFeed ? visibleFeed(scanTree().feeds) : null;
322 + if (sampled) requestAnimationFrame(sampleFeed);
323 + return true;
324 + },
325 + recordStop() {
326 + const { runOnUISync } =
327 + require("react-native-worklets") as typeof import("react-native-worklets");
328 + frames.on = false;
329 + sampled = null;
330 + return JSON.stringify({ js: frames.js, ui: runOnUISync(uiFramesStop), feed: samples });
331 + },
332 + },
333 +});
knip.jsonc
+4 -2
@@ -21,12 +21,14 @@
21 // worktree setup (from t3.json) run directly. release-desktop.yml bundles the
22 // Windows verify script with esbuild through a path knip does not trace,
23 // electron-builder loads the macOS signer by path, and mobile-e2e.ts runs
24 - // agent-device's CLI by path. release-desktop.yml runs the desktop launch
25 - // check through xvfb-run, which knip does not trace.
24 + // agent-device's CLI by path, and mobile-behavior.ts spawns its runner by path.
25 + // release-desktop.yml runs the desktop launch check through xvfb-run, which knip
26 + // does not trace.
27 "entry": [
28 // These fixture scripts are injected into a browser page by path.
29 "fixtures/*/*.js",
30 "knip-schemas.ts",
31 + "mobile-behavior/run.ts",
32 "mobile-native-client.ts",
33 "setup-worktree.ts",
34 "sign-macos.ts",
package.json
+1
@@ -34,6 +34,7 @@
34 "knip:production": "knip --production --preprocessor ./scripts/knip-schemas.ts",
35 "lint:mobile": "node scripts/mobile-native-static-check.ts",
36 "mobile:install": "node scripts/mobile-native-client.ts ensure",
37 + "mobile:behavior": "node scripts/mobile-behavior.ts",
38 "mobile:e2e": "node scripts/mobile-e2e.ts",
39 "test": "vp run -r test",
40 "test:cached": "node scripts/test-cached.ts",
pnpm-lock.yaml
+3
@@ -2281,6 +2281,9 @@ importers:
2281 '@electron/osx-sign':
2282 specifier: 2.7.0
2283 version: 2.7.0
2284 + '@t3tools/contracts':
2285 + specifier: workspace:*
2286 + version: link:../packages/contracts
2287 '@t3tools/shared':
2288 specifier: workspace:*
2289 version: link:../packages/shared
scripts/mobile-behavior.ts new
+229
@@ -0,0 +1,229 @@
1 +#!/usr/bin/env node
2 +// @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off globalFetch:off - Host-side device automation uses Node subprocess and filesystem APIs directly.
3 +
4 +import * as NodeChildProcess from "node:child_process";
5 +import * as NodeFS from "node:fs";
6 +import * as NodeFSP from "node:fs/promises";
7 +import * as NodeOS from "node:os";
8 +import * as NodePath from "node:path";
9 +import * as NodeProcess from "node:process";
10 +import * as NodeReadline from "node:readline";
11 +import * as NodeURL from "node:url";
12 +
13 +import { CASES } from "./mobile-behavior/cases.ts";
14 +import { blockingResults, resultsFor, summarize, type SampleRow } from "./mobile-behavior/grade.ts";
15 +import { Device, type Pool, poolError, run } from "./mobile-e2e/device.ts";
16 +import type { Pose } from "./mobile-e2e/poses.ts";
17 +
18 +const REPO_ROOT = NodePath.resolve(NodePath.dirname(NodeURL.fileURLToPath(import.meta.url)), "..");
19 +
20 +/** The devices the behavior cases run on; each runs in parallel with its own server and Metro. */
21 +const TARGETS = [
22 + { id: "iphone-18-pro", device: "iPhone 18 Pro", form: "phone", behaviorPose: "phone" },
23 + { id: "ipad-pro-13", device: "iPad Pro 13-inch (M5)", form: "tablet", behaviorPose: "ipad" },
24 +] as const;
25 +type Target = (typeof TARGETS)[number];
26 +
27 +const log = (scope: string, message: string) => NodeProcess.stderr.write(`[${scope}] ${message}\n`);
28 +
29 +interface TargetRun {
30 + readonly target: Target;
31 + readonly directory: string;
32 + readonly error?: string;
33 +}
34 +
35 +/** Keeps a lease alive while the cases run: they go minutes without an agent-device call. */
36 +function renewWhileRunning(pool: Pool, stateDir: string, session: string, token: string) {
37 + const gateway = pool.agentDeviceUrl.replace(/\/v1\/agent-device\/?$/, "");
38 + const timer = setInterval(() => {
39 + const record = NodePath.join(stateDir, "remote-connections", `${session}.json`);
40 + void NodeFSP.readFile(record, "utf8")
41 + .then((text) => (JSON.parse(text) as { leaseId: string }).leaseId)
42 + .then((leaseId) =>
43 + fetch(`${gateway}/v1/leases/${encodeURIComponent(leaseId)}/renew`, {
44 + method: "POST",
45 + headers: { authorization: `Bearer ${token}` },
46 + }),
47 + )
48 + .catch(() => undefined);
49 + }, 90_000);
50 + return () => clearInterval(timer);
51 +}
52 +
53 +async function runTarget(
54 + target: Target,
55 + pool: Pool,
56 + runId: string,
57 + outputDirectory: string,
58 + config: string,
59 + stateDir: string,
60 + token: string,
61 + passThrough: ReadonlyArray<string>,
62 +): Promise<TargetRun> {
63 + const directory = NodePath.join(outputDirectory, target.id);
64 + await NodeFSP.mkdir(directory, { recursive: true });
65 + const pose: Pose = {
66 + id: target.id,
67 + platform: "ios",
68 + device: target.device,
69 + form: target.form,
70 + appearance: "light",
71 + };
72 + let device: Device | null = null;
73 + for (let attempt = 1; !device; attempt++) {
74 + try {
75 + log(target.id, `leasing ${target.device}`);
76 + device = await Device.lease(pose, pool, attempt === 1 ? runId : `${runId}-retry`);
77 + log(target.id, "installing the dev client");
78 + await device.install();
79 + } catch (error) {
80 + const message = error instanceof Error ? error.message : String(error);
81 + await device?.release();
82 + device = null;
83 + if (attempt >= 2 || !poolError(message)) return { target, directory, error: message };
84 + log(target.id, `retrying after: ${message.split("\n")[0]}`);
85 + }
86 + }
87 + const stopRenewing = renewWhileRunning(pool, stateDir, device.session, token);
88 + try {
89 + log(target.id, "running the behavior cases");
90 + const child = NodeChildProcess.spawn(
91 + NodeProcess.execPath,
92 + [
93 + "scripts/mobile-behavior/run.ts",
94 + "--out",
95 + directory,
96 + "--label",
97 + target.id,
98 + "--pose",
99 + target.behaviorPose,
100 + "--device-config",
101 + config,
102 + "--device-session",
103 + device.session,
104 + "--agent-device",
105 + pool.agentDevice,
106 + ...passThrough,
107 + ],
108 + { cwd: REPO_ROOT, env: NodeProcess.env, stdio: ["ignore", "pipe", "pipe"] },
109 + );
110 + const transcript = NodeFS.createWriteStream(NodePath.join(directory, "run.log"));
111 + // The scoreboard and any fatal error go to stdout, progress to stderr; the transcript keeps both.
112 + NodeReadline.createInterface({ input: child.stdout }).on("line", (line) =>
113 + transcript.write(`${line}\n`),
114 + );
115 + NodeReadline.createInterface({ input: child.stderr }).on("line", (line) => {
116 + transcript.write(`${line}\n`);
117 + if (/#\d+: /.test(line)) log(target.id, line);
118 + });
119 + const code = await new Promise<number>((resolve) =>
120 + child.on("close", (exit) => resolve(exit ?? 1)),
121 + );
122 + transcript.end();
123 + const hasResults = NodeFS.existsSync(NodePath.join(directory, "results.jsonl"));
124 + return hasResults || code === 0
125 + ? { target, directory }
126 + : { target, directory, error: `the cases stopped (exit ${code}); see ${directory}/run.log` };
127 + } finally {
128 + stopRenewing();
129 + await device.release();
130 + }
131 +}
132 +
133 +async function main() {
134 + const args = NodeProcess.argv.slice(2);
135 + const values = (flag: string) =>
136 + args.flatMap((arg, index) => (args[index - 1] === flag ? [arg] : []));
137 + if (args.includes("--list")) {
138 + for (const target of TARGETS) NodeProcess.stdout.write(`${target.id}\t${target.device}\n`);
139 + for (const testCase of CASES)
140 + NodeProcess.stdout.write(`${testCase.id}\t${testCase.kind}\t${testCase.source}\n`);
141 + return;
142 + }
143 + const selected = values("--device");
144 + const unknown = selected.filter((id) => !TARGETS.some((target) => target.id === id));
145 + if (unknown.length) throw new Error(`Unknown device ${unknown.join(", ")}. Run with --list.`);
146 + const targets = TARGETS.filter((target) => selected.length === 0 || selected.includes(target.id));
147 + const passThrough = [
148 + ...values("--samples").flatMap((value) => ["--samples", value]),
149 + ...values("--case").flatMap((value) => ["--case", value]),
150 + ];
151 +
152 + const agentDeviceUrl = NodeProcess.env.SIMLOCK_AGENT_DEVICE_URL;
153 + const token = NodeProcess.env.AGENT_DEVICE_DAEMON_AUTH_TOKEN;
154 + if (!agentDeviceUrl || !token)
155 + throw new Error(
156 + "Set SIMLOCK_AGENT_DEVICE_URL (<gateway>/v1/agent-device) and AGENT_DEVICE_DAEMON_AUTH_TOKEN (a Simlock agent token) to run on the device pool.",
157 + );
158 +
159 + const runId = new Date().toISOString().replaceAll(/[:.]/g, "-");
160 + const outputDirectory = NodePath.join(REPO_ROOT, "artifacts/mobile-behavior", runId);
161 + await NodeFSP.mkdir(outputDirectory, { recursive: true });
162 + const rootDir = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-mobile-behavior-"));
163 + const workspace = await NodeFSP.realpath(outputDirectory);
164 + const config = NodePath.join(rootDir, "agent-device.json");
165 + const stateDir = NodePath.join(rootDir, "agent-device");
166 + await NodeFSP.writeFile(
167 + config,
168 + JSON.stringify({ daemonBaseUrl: agentDeviceUrl, daemonAuthToken: token, stateDir }),
169 + { mode: 0o600 },
170 + );
171 + const pool: Pool = {
172 + repoRoot: REPO_ROOT,
173 + agentDevice: NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"),
174 + agentDeviceUrl,
175 + workspace,
176 + env: { ...NodeProcess.env, AGENT_DEVICE_CONFIG: config, PWD: workspace },
177 + };
178 + try {
179 + // A cache miss builds the dev client, which can outlast a lease, so it is ready first.
180 + log("behavior", "preparing the ios dev client");
181 + await run(
182 + NodeProcess.execPath,
183 + ["scripts/mobile-native-client.ts", "ensure", "ios", "--prepare"],
184 + { cwd: REPO_ROOT, timeoutMs: 1_800_000 },
185 + );
186 + const runs = await Promise.all(
187 + targets.map((target) =>
188 + runTarget(target, pool, runId, outputDirectory, config, stateDir, token, passThrough),
189 + ),
190 + );
191 +
192 + const rows = runs.flatMap((entry) => {
193 + const file = NodePath.join(entry.directory, "results.jsonl");
194 + if (!NodeFS.existsSync(file)) return [];
195 + return NodeFS.readFileSync(file, "utf8")
196 + .split("\n")
197 + .filter(Boolean)
198 + .map((line) => JSON.parse(line) as SampleRow);
199 + });
200 + const results = resultsFor(rows, CASES);
201 + const stopped = runs.filter((entry) => entry.error);
202 + const summary = summarize({
203 + cases: CASES,
204 + results,
205 + header: [
206 + `# Mobile behavior · ${runId}`,
207 + "",
208 + "Cells show pass or FAIL, passed/samples, and the key metric's median. `bad` cases reproduce known bugs and are expected to fail until they are fixed.",
209 + ...stopped.map((entry) => `\n**${entry.target.id} did not finish:** ${entry.error}`),
210 + ].join("\n"),
211 + });
212 + const summaryFile = NodePath.join(outputDirectory, "summary.md");
213 + await NodeFSP.writeFile(summaryFile, summary);
214 + const blocking = blockingResults(results, CASES);
215 + for (const result of blocking)
216 + log(result.device, `${result.id}: ${result.pass === null ? "incomplete" : "FAIL"}`);
217 + for (const entry of stopped) log(entry.target.id, entry.error!);
218 + NodeProcess.stdout.write(
219 + `${JSON.stringify({ summary: summaryFile, cases: results.length, blocking: blocking.length, stopped: stopped.length })}\n`,
220 + );
221 + process.exitCode = blocking.length || stopped.length ? 1 : 0;
222 + } finally {
223 + await NodeFSP.rm(rootDir, { recursive: true, force: true });
224 + }
225 +}
226 +
227 +if (import.meta.main) {
228 + await main();
229 +}
scripts/mobile-behavior/README.md new
+32
@@ -0,0 +1,32 @@
1 +# Mobile behavior
2 +
3 +`vp run mobile:behavior` checks what the iOS app does, from its own state rather than pixels: scroll position, dropped frames, composer drafts and focus, status labels, and which thread opens. It leases an iPhone 18 Pro and an iPad Pro 13 from the Simlock pool and runs the cases on both in parallel. It is the companion to `vp run mobile:e2e`, which checks layout.
4 +
5 +No model runs. Each device gets its own server whose only provider, `evalbot`, is an Fx instance backed by [`fake-fx.ts`](fake-fx.ts). A `[[eval:…]]` marker in a message picks its scripted reply: a paced stream, a held turn, a question, two subagents, or a failure. Each device also gets its own Metro with the dev-only probe ([`behavior-probe.ts`](../../apps/mobile/src/features/development/behavior-probe.ts)) compiled in. The probe reads app state through Metro's Hermes inspector, and agent-device drives the touches.
6 +
7 +## Run
8 +
9 +Needs `SIMLOCK_AGENT_DEVICE_URL` and `AGENT_DEVICE_DAEMON_AUTH_TOKEN`, like `mobile:e2e`.
10 +
11 +```sh
12 +vp run mobile:behavior # both devices, 3 samples per case
13 +vp run mobile:behavior --device iphone-18-pro --samples 1 --case scroll-up-never-teleports
14 +vp run mobile:behavior --list # devices and cases
15 +```
16 +
17 +Output lands in `artifacts/mobile-behavior/<run>/`: `summary.md` is the scoreboard. Each device's folder holds `results.jsonl` (one row per case sample), `run.log`, `metro.log`, and the server under `env/`. A full run takes about 15 minutes.
18 +
19 +A case fails when half or more of its graded samples fail, and is `incomplete` when errors outnumber graded samples. The run fails on a `guard` or `control` case that fails or is incomplete. `bad` cases reproduce known bugs and fail until those are fixed.
20 +
21 +## Metrics
22 +
23 +- **Scroll.** The probe reads the thread list's state on every JS frame and records each visible row's y on screen. `backwardPx` is the largest move against the drag. `teleports` counts one-frame moves over 400 pt, faster than a real fling. `maxFromEndPx` is the distance from the latest message.
24 +- **Jank.** JS-thread frames from `requestAnimationFrame` in the probe, UI-thread frames from a worklet loop. Simulators render at 60 Hz, and injected swipes load the simulator's UI thread, so swipe cases grade JS frames only.
25 +- **Composer.** The draft store, the editor's text from the accessibility tree, whether it expanded, and whether the keyboard is up.
26 +- **Status.** Each Home row's status from the app's own `resolveThreadListV2Status`, and the Working shelf count, against server state.
27 +
28 +## Add a case
29 +
30 +Cases are data in [`cases.ts`](cases.ts): an `id`, a `kind` (`bad` fails on `main` today, `guard` must keep passing, `control` is ordinary use), its `source` issue, the `scenario` that produces its metrics, a `key` metric, and `checks`. A new flow goes in [`scenarios.ts`](scenarios.ts). Add a case before its fix, so it fails on `main` while the bug exists; flip it from `bad` to `guard` when the fix lands.
31 +
32 +Not covered: Live Activities and notification taps (they need a T3 Connect relay and delivered pushes), native editor scroll, the background-work pill and async questions (Fx has no background tasks), and 120 Hz pacing.
scripts/mobile-behavior/cases.ts new
+338
@@ -0,0 +1,338 @@
1 +/**
2 + * Cases for the mobile behavior eval. `bad` fails on main today and must pass once fixed;
3 + * `guard` passes on main and must keep passing (its issue is fixed, or does not show on iOS);
4 + * `control` is ordinary use that must keep passing. Each names the scenario that produces its metrics and the checks on them.
5 + */
6 +import type { Check } from "./grade.ts";
7 +import type { Pose, ScenarioName } from "./scenarios.ts";
8 +
9 +export interface MobileBehaviorCase {
10 + readonly id: string;
11 + readonly area: "scroll" | "jank" | "composer" | "status" | "wrong-thread";
12 + readonly kind: "bad" | "guard" | "control";
13 + /** The issue or PR it came from. */
14 + readonly source: string;
15 + /** The real bad experience, or the good one a guard protects. */
16 + readonly experience: string;
17 + readonly scenario: ScenarioName;
18 + /** The metric the scoreboard shows. */
19 + readonly key: string;
20 + readonly checks: ReadonlyArray<Check>;
21 + /** Omit to run on every pose. */
22 + readonly poses?: ReadonlyArray<Pose>;
23 +}
24 +
25 +/**
26 + * Frame budget checks for jank cases: at most 5% of UI frames and 10% of JS frames dropped.
27 + * Swipe-driven cases check JS frames only: the injected swipes cost the simulator's UI thread.
28 + */
29 +const smooth: ReadonlyArray<Check> = [
30 + { metric: "uiDroppedPct", max: 5 },
31 + { metric: "jsDroppedPct", max: 10 },
32 +];
33 +
34 +export const CASES: ReadonlyArray<MobileBehaviorCase> = [
35 + {
36 + id: "scroll-up-never-moves-backward",
37 + area: "scroll",
38 + kind: "guard",
39 + source: "#22350",
40 + experience: "Flicking up a long thread, content jerks back down toward the latest message.",
41 + scenario: "scrollUpLongThread",
42 + key: "backwardPx",
43 + checks: [
44 + { metric: "movedPx", min: 300 },
45 + { metric: "backwardPx", max: 24 },
46 + ],
47 + },
48 + {
49 + id: "scroll-up-never-teleports",
50 + area: "scroll",
51 + kind: "guard",
52 + source: "#12072",
53 + experience: "Dragging up a little after scroll-to-end, the viewport jumps far away.",
54 + scenario: "scrollUpLongThread",
55 + key: "largestStepPx",
56 + checks: [
57 + { metric: "movedPx", min: 300 },
58 + { metric: "teleports", equals: 0 },
59 + ],
60 + },
61 + {
62 + id: "streaming-stays-at-latest",
63 + area: "scroll",
64 + kind: "bad",
65 + source: "#15469",
66 + experience:
67 + "While a reply streams at the bottom, the thread jumps far above the latest message.",
68 + scenario: "streamInView",
69 + key: "settledFromEndPx",
70 + checks: [
71 + { metric: "settledFromEndPx", max: 64 },
72 + { metric: "followingAtEnd", equals: true },
73 + ],
74 + },
75 + {
76 + id: "send-in-long-thread-stays-at-end",
77 + area: "scroll",
78 + kind: "guard",
79 + source: "#13706",
80 + experience:
81 + "Sending in a long thread, the timeline jumps far up instead of showing the new message.",
82 + scenario: "sendInLongThread",
83 + key: "maxFromEndPx",
84 + checks: [
85 + { metric: "maxFromEndPx", max: 1500 },
86 + { metric: "finalFromEndPx", max: 24 },
87 + ],
88 + },
89 + {
90 + id: "background-finish-keeps-scroll",
91 + area: "scroll",
92 + kind: "guard",
93 + source: "#15165",
94 + experience: "Another thread finishing resets the open thread's scroll position.",
95 + scenario: "backgroundFinish",
96 + key: "largestStepPx",
97 + checks: [{ metric: "largestStepPx", max: 4 }],
98 + },
99 + {
100 + id: "jank-streaming-long-reply",
101 + area: "jank",
102 + kind: "bad",
103 + source: "#15641, #14010, #19120",
104 + experience: "A long reply streaming into a long thread freezes or stutters the app.",
105 + scenario: "streamInView",
106 + key: "jsDroppedPct",
107 + checks: smooth,
108 + },
109 + {
110 + id: "jank-scrolling-settled-thread",
111 + area: "jank",
112 + kind: "control",
113 + source: "#13925 (Android)",
114 + experience: "Scrolling a long settled thread drops frames.",
115 + scenario: "scrollUpLongThread",
116 + key: "jsP99Ms",
117 + checks: [
118 + { metric: "movedPx", min: 300 },
119 + { metric: "jsDroppedPct", max: 10 },
120 + ],
121 + },
122 + {
123 + id: "jank-scrolling-subagent-thread",
124 + area: "jank",
125 + kind: "control",
126 + source: "#18376 (Android)",
127 + experience: "Scrolling a subagent's thread is far less smooth than its parent.",
128 + scenario: "openSubagentThread",
129 + key: "jsP99Ms",
130 + checks: [
131 + { metric: "movedPx", min: 300 },
132 + { metric: "jsDroppedPct", max: 10 },
133 + ],
134 + },
135 + {
136 + id: "draft-survives-thread-switch",
137 + area: "composer",
138 + kind: "guard",
139 + source: "#17422",
140 + experience: "A typed draft is gone after switching threads and coming back.",
141 + scenario: "draftThreadSwitch",
142 + key: "draftKept",
143 + checks: [
144 + { metric: "draftKept", equals: true },
145 + { metric: "editorShowsDraft", equals: true },
146 + ],
147 + },
148 + {
149 + id: "draft-never-leaks-to-other-thread",
150 + area: "composer",
151 + kind: "guard",
152 + source: "#7390",
153 + experience: "Text typed in a streaming thread shows up in another thread's composer.",
154 + scenario: "draftThreadSwitch",
155 + key: "leakedIntoOtherEditor",
156 + checks: [
157 + { metric: "leakedIntoOtherDraft", equals: false },
158 + { metric: "leakedIntoOtherEditor", equals: false },
159 + ],
160 + },
161 + {
162 + id: "draft-survives-background",
163 + area: "composer",
164 + kind: "guard",
165 + source: "#17422, #7390",
166 + experience: "A typed draft is gone after the app goes to the background and back.",
167 + scenario: "draftBackground",
168 + key: "draftKept",
169 + checks: [
170 + { metric: "draftKept", equals: true },
171 + { metric: "editorShowsDraft", equals: true },
172 + ],
173 + },
174 + {
175 + id: "composer-clears-after-send",
176 + area: "composer",
177 + kind: "guard",
178 + source: "#20982",
179 + experience: "After sending, the message text stays in the composer.",
180 + scenario: "sendInLongThread",
181 + key: "editorCleared",
182 + checks: [
183 + { metric: "draftCleared", equals: true },
184 + { metric: "editorCleared", equals: true },
185 + ],
186 + },
187 + {
188 + id: "composer-tap-focuses-text",
189 + area: "composer",
190 + kind: "guard",
191 + source: "#24303",
192 + experience: "Tapping the composer's text area opens the model picker instead of the keyboard.",
193 + scenario: "tapComposer",
194 + key: "editorFocused",
195 + checks: [
196 + { metric: "stayedOnThread", equals: true },
197 + { metric: "editorFocused", equals: true },
198 + ],
199 + },
200 + {
201 + id: "composer-keeps-focus-after-command-pick",
202 + area: "composer",
203 + kind: "guard",
204 + source: "#10956",
205 + experience: "Picking a skill or command from the / menu closes the keyboard.",
206 + scenario: "skillPick",
207 + key: "editorFocused",
208 + checks: [{ metric: "editorFocused", equals: true }],
209 + },
210 + {
211 + id: "failed-send-is-never-silent",
212 + area: "composer",
213 + kind: "guard",
214 + source: "#15801",
215 + experience: "A send the server will reject looks sent, with no error.",
216 + scenario: "failedSend",
217 + // The cell shows which path ran: Send blocked, or a rejected send with an error.
218 + key: "sendBlocked",
219 + checks: [{ metric: "silentFailure", equals: false }],
220 + },
221 + {
222 + id: "working-shelf-clears-when-done",
223 + area: "status",
224 + kind: "bad",
225 + source: "#24901",
226 + experience: "A finished thread stays in the Working shelf.",
227 + scenario: "workingShelfFinish",
228 + key: "shelfCountAfter",
229 + checks: [
230 + { metric: "shelfShownWhileWorking", equals: true },
231 + { metric: "shelfCountAfter", equals: 0 },
232 + { metric: "statusAfter", equals: "ready" },
233 + ],
234 + },
235 + {
236 + id: "pending-question-shows-input",
237 + area: "status",
238 + kind: "guard",
239 + source: "#24901",
240 + experience: "A thread waiting on a question shows as Working instead of needing input.",
241 + scenario: "pendingQuestion",
242 + key: "status",
243 + checks: [
244 + { metric: "status", equals: "input" },
245 + { metric: "shelfCount", equals: 0 },
246 + ],
247 + },
248 + {
249 + id: "running-subagents-not-idle",
250 + area: "status",
251 + kind: "guard",
252 + source: "#16464, #4962",
253 + experience: "A thread with subagents still running looks idle in the list and the thread.",
254 + scenario: "subagentsRunning",
255 + key: "status",
256 + checks: [
257 + { metric: "status", equals: "working" },
258 + { metric: "threadShowsStatus", equals: true },
259 + ],
260 + },
261 + {
262 + id: "long-press-pin-pins-that-row",
263 + area: "wrong-thread",
264 + kind: "guard",
265 + source: "#20428",
266 + experience: "Pin from a row's long-press menu pins a different thread.",
267 + scenario: "pinFromLongPress",
268 + key: "targetPinned",
269 + checks: [
270 + { metric: "targetPinned", equals: true },
271 + { metric: "otherPinned", equals: 0 },
272 + ],
273 + },
274 + {
275 + id: "link-opens-new-thread",
276 + area: "wrong-thread",
277 + kind: "guard",
278 + source: "#14689 (web)",
279 + experience: "A link to a just-created thread redirects to another thread.",
280 + scenario: "linkToNewThread",
281 + key: "opensTarget",
282 + checks: [{ metric: "opensTarget", equals: true }],
283 + },
284 + {
285 + id: "link-opens-archived-thread",
286 + area: "wrong-thread",
287 + kind: "guard",
288 + source: "#16443",
289 + experience: "A link to an archived thread sends you Home.",
290 + scenario: "linkToArchivedThread",
291 + key: "opensTarget",
292 + checks: [{ metric: "opensTarget", equals: true }],
293 + },
294 + {
295 + id: "folded-duo-back-opens-right-thread",
296 + area: "wrong-thread",
297 + kind: "guard",
298 + source: "#20633",
299 + experience: "On a folded iPhone Duo, after going back, the next thread opens off screen.",
300 + scenario: "duoBackThenOpen",
301 + key: "titleOnScreen",
302 + poses: ["duo-closed"],
303 + checks: [
304 + { metric: "opensTarget", equals: true },
305 + { metric: "titleOnScreen", equals: true },
306 + ],
307 + },
308 + {
309 + id: "control-open-thread-from-list",
310 + area: "wrong-thread",
311 + kind: "control",
312 + source: "control",
313 + experience: "Tapping a row opens that thread.",
314 + scenario: "openFromList",
315 + key: "opensTarget",
316 + checks: [{ metric: "opensTarget", equals: true }],
317 + },
318 + {
319 + id: "control-send-reaches-server",
320 + area: "composer",
321 + kind: "control",
322 + source: "control",
323 + experience: "A message sent from the composer reaches the server once.",
324 + scenario: "sendInLongThread",
325 + key: "serverMessages",
326 + checks: [{ metric: "serverMessages", equals: 1 }],
327 + },
328 + {
329 + id: "control-streaming-reply-renders",
330 + area: "scroll",
331 + kind: "control",
332 + source: "control",
333 + experience: "A streaming reply grows in the open thread.",
334 + scenario: "streamInView",
335 + key: "contentGrewPx",
336 + checks: [{ metric: "contentGrewPx", min: 500 }],
337 + },
338 +];
scripts/mobile-behavior/device.ts new
+307
@@ -0,0 +1,307 @@
1 +// @effect-diagnostics nodeBuiltinImport:off preferSchemaOverJson:off - drives agent-device and the Hermes inspector; the probe types its own JSON.
2 +/**
3 + * One simulator, driven two ways: T3's `agent-device` CLI for real touches and system actions,
4 + * and the dev-only behavior probe (`apps/mobile/src/features/development/behavior-probe.ts`)
5 + * through Metro's Hermes inspector for exact app state.
6 + */
7 +import * as NodeChildProcess from "node:child_process";
8 +import * as NodeOS from "node:os";
9 +import * as NodePath from "node:path";
10 +import * as NodeSocket from "@effect/platform-node/NodeSocket";
11 +import * as Clock from "effect/Clock";
12 +import * as Effect from "effect/Effect";
13 +import * as Schedule from "effect/Schedule";
14 +import * as Schema from "effect/Schema";
15 +import { FetchHttpClient, HttpClient, HttpClientResponse } from "effect/http";
16 +import { Socket } from "effect/socket";
17 +
18 +import { poolError } from "../mobile-e2e/device.ts";
19 +
20 +const InspectorTargets = Schema.Array(
21 + Schema.Struct({
22 + appId: Schema.String,
23 + deviceName: Schema.optional(Schema.String),
24 + webSocketDebuggerUrl: Schema.String,
25 + }),
26 +);
27 +
28 +export class DeviceError extends Schema.TaggedError<DeviceError>()("DeviceError", {
29 + message: Schema.String,
30 +}) {}
31 +
32 +export const APP_ID = "com.t3tools.t3code.dev";
33 +export const SCHEME = "t3code-dev";
34 +
35 +export interface ProbeState {
36 + readonly route: {
37 + readonly name: string | null;
38 + readonly params: Record<string, unknown>;
39 + readonly columns: boolean;
40 + } | null;
41 + readonly feed: FeedSample | null;
42 + readonly composer: {
43 + readonly key: string | null;
44 + readonly draft: string | null;
45 + readonly drafts: Record<string, string>;
46 + readonly editors: ReadonlyArray<{ readonly value: string; readonly expanded: boolean }>;
47 + readonly error: string | null;
48 + readonly keyboard: boolean;
49 + };
50 + readonly floatingStatus: string | null;
51 + readonly environments: ReadonlyArray<string>;
52 + /** The app's status for every thread it knows, by thread id. */
53 + readonly threads: Record<string, string>;
54 + readonly homeList: ReadonlyArray<{
55 + readonly type: string;
56 + readonly threadId?: string;
57 + readonly status?: string;
58 + readonly pinned?: boolean;
59 + readonly count?: number;
60 + readonly expanded?: boolean;
61 + }> | null;
62 +}
63 +
64 +export interface FeedSample {
65 + readonly at?: number;
66 + readonly threadId: string;
67 + readonly ready: boolean;
68 + readonly scroll?: number;
69 + readonly viewport?: number;
70 + readonly content?: number;
71 + readonly fromEnd?: number;
72 + readonly atEnd?: boolean;
73 + readonly following?: boolean | null;
74 + readonly count?: number;
75 + readonly rows?: Record<string, number>;
76 +}
77 +
78 +export interface Recording {
79 + readonly js: ReadonlyArray<number>;
80 + readonly ui: ReadonlyArray<number>;
81 + readonly feed: ReadonlyArray<FeedSample>;
82 +}
83 +
84 +export interface AxNode {
85 + readonly type?: string;
86 + readonly label?: string;
87 + readonly value?: string;
88 + readonly kind?: string;
89 + readonly editable?: boolean;
90 + readonly enabled?: boolean;
91 + readonly rect?: {
92 + readonly x: number;
93 + readonly y: number;
94 + readonly width: number;
95 + readonly height: number;
96 + };
97 +}
98 +
99 +export interface DeviceOptions {
100 + /** `agent-device` executable and the target args `device_open` returned. */
101 + readonly agentDevice: string;
102 + readonly targetArgs: ReadonlyArray<string>;
103 + readonly metro: string;
104 + /** Simulator name as Metro's inspector lists it; empty for the only app on this run's Metro. */
105 + readonly deviceName: string;
106 +}
107 +
108 +export function makeDevice(options: DeviceOptions) {
109 + const run = (args: ReadonlyArray<string>, timeoutMs = 90_000) =>
110 + Effect.try({
111 + try: () =>
112 + NodeChildProcess.execFileSync(options.agentDevice, [...args, ...options.targetArgs], {
113 + encoding: "utf8",
114 + timeout: timeoutMs,
115 + stdio: ["ignore", "pipe", "pipe"],
116 + }),
117 + catch: (cause) => {
118 + // The thrown message starts with the full command line; the reason is in its output.
119 + const output = cause as { readonly stdout?: string; readonly stderr?: string };
120 + const reason = `${output.stderr ?? ""}${output.stdout ?? ""}`.trim() || String(cause);
121 + return new DeviceError({
122 + message: `agent-device ${args.join(" ")} failed: ${reason.slice(-400)}`,
123 + });
124 + },
125 + }).pipe(
126 + // A shared pool's workers drop off the gateway for seconds at a time.
127 + Effect.retry({
128 + times: 4,
129 + while: (error) => poolError(error.message),
130 + schedule: Schedule.spaced("5 seconds"),
131 + }),
132 + Effect.withSpan("device.agentDevice", { attributes: { command: args[0] } }),
133 + );
134 +
135 + const evaluate = Effect.fn("device.evaluate")(
136 + function* (expression: string) {
137 + const targets = yield* HttpClient.get(`${options.metro}/json/list`).pipe(
138 + Effect.flatMap(HttpClientResponse.schemaBodyJson(InspectorTargets)),
139 + Effect.provide(FetchHttpClient.layer),
140 + Effect.mapError(
141 + () => new DeviceError({ message: `Metro at ${options.metro} did not answer.` }),
142 + ),
143 + );
144 + // Pool simulators carry generated names; a run's own Metro then serves only its device.
145 + const target = targets.findLast(
146 + (entry) =>
147 + entry.appId === APP_ID &&
148 + (!options.deviceName || entry.deviceName === options.deviceName),
149 + );
150 + if (!target)
151 + return yield* new DeviceError({
152 + message: `No app on ${options.deviceName} is attached to Metro.`,
153 + });
154 + // Expo drops inspector sockets whose Origin is not the URL Metro advertises.
155 + const socket = yield* Socket.makeWebSocket(target.webSocketDebuggerUrl, {
156 + headers: { origin: new URL(options.metro).origin },
157 + openTimeout: "10 seconds",
158 + });
159 + const writer = yield* socket.writer;
160 + const reader = yield* socket.reader;
161 + yield* writer.write(
162 + JSON.stringify({
163 + id: 1,
164 + method: "Runtime.evaluate",
165 + params: { expression, returnByValue: true, awaitPromise: true },
166 + }),
167 + );
168 + while (true) {
169 + for (const frame of yield* reader.pull) {
170 + const reply = JSON.parse(
171 + typeof frame === "string" ? frame : new TextDecoder().decode(frame),
172 + ) as {
173 + id?: number;
174 + result?: { result?: { value?: unknown }; exceptionDetails?: unknown };
175 + };
176 + if (reply.id !== 1) continue;
177 + if (reply.result?.exceptionDetails)
178 + return yield* new DeviceError({
179 + message: `${expression} threw ${JSON.stringify(reply.result.exceptionDetails).slice(0, 400)}`,
180 + });
181 + return reply.result?.result?.value;
182 + }
183 + }
184 + },
185 + Effect.scoped,
186 + Effect.provide(NodeSocket.layerWebSocketConstructorWS),
187 + Effect.timeoutOrElse({
188 + duration: "20 seconds",
189 + orElse: () => Effect.fail(new DeviceError({ message: "The inspector did not answer." })),
190 + }),
191 + Effect.mapError((error) =>
192 + error._tag === "DeviceError"
193 + ? error
194 + : new DeviceError({ message: `Inspector: ${error.message}` }),
195 + ),
196 + );
197 +
198 + const state = evaluate("__t3Behavior.state()").pipe(
199 + Effect.map((value) => JSON.parse(String(value)) as ProbeState),
200 + );
201 +
202 + /** Polls the probe until `predicate` holds; fails with the last state after `timeoutMs`. */
203 + const waitFor = Effect.fn("device.waitFor")(function* (
204 + description: string,
205 + predicate: (state: ProbeState) => boolean,
206 + timeoutMs = 15_000,
207 + ) {
208 + const deadline = (yield* Clock.currentTimeMillis) + timeoutMs;
209 + let last: ProbeState | null = null;
210 + while ((yield* Clock.currentTimeMillis) < deadline) {
211 + last = yield* state.pipe(Effect.orElseSucceed(() => null));
212 + if (last && predicate(last)) return last;
213 + yield* Effect.sleep("250 millis");
214 + }
215 + return yield* new DeviceError({
216 + message: `Timed out waiting for ${description}. Last route: ${JSON.stringify(last?.route ?? null)}`,
217 + });
218 + });
219 +
220 + const nodes = run(["snapshot", "--json"]).pipe(
221 + Effect.map((out) => (JSON.parse(out) as { data: { nodes: ReadonlyArray<AxNode> } }).data.nodes),
222 + );
223 +
224 + const center = (node: AxNode) => ({
225 + x: Math.round(node.rect!.x + node.rect!.width / 2),
226 + y: Math.round(node.rect!.y + node.rect!.height / 2),
227 + });
228 +
229 + const screenSize = run([
230 + "screenshot",
231 + NodePath.join(NodeOS.tmpdir(), "t3-mobile-behavior-screen.png"),
232 + "--json",
233 + ]).pipe(
234 + Effect.map((out) => {
235 + const { data } = JSON.parse(out) as { data: { logicalWidth: number; logicalHeight: number } };
236 + return { width: data.logicalWidth, height: data.logicalHeight };
237 + }),
238 + );
239 +
240 + return {
241 + options,
242 + run,
243 + evaluate,
244 + state,
245 + waitFor,
246 + nodes,
247 + center,
248 + open: (path: string) =>
249 + evaluate(`__t3Behavior.open(${JSON.stringify(`${SCHEME}://${path.replace(/^\//, "")}`)})`),
250 + openUrl: (url: string) => evaluate(`__t3Behavior.open(${JSON.stringify(url)})`),
251 + /** Reloads JS, then opens Home; the app restores the last thread, and split layouts keep one open. */
252 + reload: Effect.fn("device.reload")(function* (ready: (state: ProbeState) => boolean) {
253 + yield* evaluate("__t3Behavior.reload()").pipe(Effect.ignore);
254 + yield* Effect.sleep("2 seconds");
255 + yield* waitFor("the thread list after reload", ready, 60_000);
256 + yield* evaluate(`__t3Behavior.open(${JSON.stringify(`${SCHEME}://`)})`);
257 + // Split layouts may keep a thread beside Home, so they get a short wait for the Home route.
258 + const home = (value: ProbeState) => ready(value) && value.route?.name === "Home";
259 + const state = yield* waitFor("Home", home, 5_000).pipe(Effect.option);
260 + if (state._tag === "Some") return state.value;
261 + return yield* waitFor(
262 + "the split layout",
263 + (value) => ready(value) && value.route?.columns === true,
264 + );
265 + }),
266 + recordStart: (withFeed: boolean) => evaluate(`__t3Behavior.recordStart(${withFeed})`),
267 + recordStop: evaluate("__t3Behavior.recordStop()").pipe(
268 + Effect.map((value) => JSON.parse(String(value)) as Recording),
269 + ),
270 + press: (selector: string) => run(["press", selector]),
271 + pressAt: (x: number, y: number) => run(["press", String(x), String(y)]),
272 + longPress: (selector: string) => run(["longpress", selector, "800"]),
273 + type: (text: string) => run(["type", text]),
274 + /** Drags from y1 to y2 at x; a downward drag scrolls the content up. */
275 + swipe: (x: number, y1: number, y2: number) =>
276 + run(["swipe", String(x), String(y1), String(x), String(y2)]),
277 + /**
278 + * Answers iOS's "Open in" prompt. XCTest cannot see the iOS 27.1 prompt, so when `alert` fails
279 + * this taps where its Open button sits, just right of and below the screen's center.
280 + */
281 + acceptOpenPrompt: run(["alert", "accept"], 60_000).pipe(
282 + Effect.catch(() =>
283 + screenSize.pipe(
284 + Effect.flatMap(({ width, height }) =>
285 + run(["press", String(Math.round(width / 2 + 76)), String(Math.round(height / 2 + 8))]),
286 + ),
287 + ),
288 + ),
289 + ),
290 + home: run(["home"]),
291 + /** The Duo simulator often follows a fold without a hinge reading, so the screen's shape confirms it. */
292 + fold: (pose: "closed" | "open") =>
293 + run(["fold", pose]).pipe(
294 + Effect.catch((error) =>
295 + screenSize.pipe(
296 + Effect.filterOrFail(
297 + ({ width, height }) => (pose === "closed") === width < height,
298 + () => error,
299 + ),
300 + ),
301 + ),
302 + ),
303 + back: run(["back"]),
304 + };
305 +}
306 +
307 +export type Device = ReturnType<typeof makeDevice>;
scripts/mobile-behavior/environment.ts new
+420
@@ -0,0 +1,420 @@
1 +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off - the eval owns a server subprocess and a scratch workspace.
2 +/**
3 + * An isolated T3 server for one device run. Its only provider is `evalbot`, an Fx instance
4 + * backed by `fake-fx.ts`, so every reply is scripted. State lives under the run's `--out`.
5 + */
6 +import * as NodeChildProcess from "node:child_process";
7 +import * as NodeFS from "node:fs";
8 +import * as NodeNet from "node:net";
9 +import * as NodePath from "node:path";
10 +import * as NodeSocket from "@effect/platform-node/NodeSocket";
11 +import {
12 + CommandId,
13 + EnvironmentId,
14 + ExecutionEnvironmentDescriptor,
15 + MessageId,
16 + ORCHESTRATION_PROTOCOL_HEADER,
17 + ORCHESTRATION_PROTOCOL_VERSION,
18 + OrchestrationV2ShellSnapshot,
19 + ProjectId,
20 + ProviderInstanceId,
21 + ThreadId,
22 + WsRpcGroup,
23 + type OrchestrationV2ThreadProjection,
24 +} from "@t3tools/contracts";
25 +import * as Crypto from "effect/Crypto";
26 +import * as Effect from "effect/Effect";
27 +import * as Layer from "effect/Layer";
28 +import * as Schema from "effect/Schema";
29 +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
30 +import { RpcClient, RpcSerialization } from "effect/rpc";
31 +import { Socket } from "effect/socket";
32 +
33 +const REPO_ROOT = NodePath.resolve(import.meta.dirname, "../..");
34 +export const EVAL_INSTANCE = ProviderInstanceId.make("evalbot");
35 +export const EVAL_MODEL = "eval/scripted";
36 +
37 +export const uuid = Effect.flatMap(Crypto.Crypto, (crypto) => crypto.randomUUIDv4());
38 +
39 +export class EnvironmentError extends Schema.TaggedError<EnvironmentError>()("EnvironmentError", {
40 + message: Schema.String,
41 +}) {}
42 +
43 +export const freePort = () =>
44 + Effect.callback<number>((resume) => {
45 + const server = NodeNet.createServer();
46 + server.listen(0, () => {
47 + const address = server.address();
48 + server.close(() =>
49 + resume(Effect.succeed(typeof address === "object" && address ? address.port : 0)),
50 + );
51 + });
52 + });
53 +
54 +const waitForPort = Effect.fn("environment.waitForPort")(function* (port: number) {
55 + for (let attempt = 0; attempt < 120; attempt++) {
56 + const open = yield* Effect.callback<boolean>((resume) => {
57 + const socket = NodeNet.connect(port, "127.0.0.1");
58 + socket.once("connect", () => (socket.destroy(), resume(Effect.succeed(true))));
59 + socket.once("error", () => resume(Effect.succeed(false)));
60 + });
61 + if (open) return;
62 + yield* Effect.sleep("500 millis");
63 + }
64 + return yield* new EnvironmentError({ message: `The server never listened on ${port}.` });
65 +});
66 +
67 +/** Runs a server CLI command and returns both output streams. */
68 +const runCli = (args: ReadonlyArray<string>) =>
69 + Effect.try({
70 + try: () => {
71 + const result = NodeChildProcess.spawnSync("node", ["apps/server/src/bin.ts", ...args], {
72 + cwd: REPO_ROOT,
73 + encoding: "utf8",
74 + });
75 + if (result.status !== 0) throw new Error(result.stderr);
76 + return { stdout: result.stdout, stderr: result.stderr };
77 + },
78 + catch: (cause) =>
79 + new EnvironmentError({ message: `bin.ts ${args.join(" ")} failed: ${cause}` }),
80 + });
81 +
82 +function writeScratch(baseDir: string) {
83 + const workspace = NodePath.join(baseDir, "workspace/acme-app");
84 + const control = NodePath.join(baseDir, "control");
85 + NodeFS.mkdirSync(NodePath.join(baseDir, "userdata"), { recursive: true });
86 + NodeFS.mkdirSync(control, { recursive: true });
87 + NodeFS.mkdirSync(workspace, { recursive: true });
88 + NodeFS.writeFileSync(NodePath.join(workspace, "README.md"), "# acme app\n");
89 + // The / menu's pickable row; without it, fx finds only skills in folders above `--out`.
90 + const skill = NodePath.join(workspace, ".fx/skills/eval-notes");
91 + NodeFS.mkdirSync(skill, { recursive: true });
92 + NodeFS.writeFileSync(
93 + NodePath.join(skill, "SKILL.md"),
94 + "---\nname: eval-notes\ndescription: Takes notes for the eval.\n---\nTake notes.\n",
95 + );
96 + const git = (...args: string[]) =>
97 + NodeChildProcess.execFileSync("git", ["-C", workspace, ...args], { stdio: "ignore" });
98 + git("init", "-q");
99 + git("add", "-A");
100 + git("-c", "user.name=eval", "-c", "user.email=eval@example.com", "commit", "-qm", "init");
101 + const launcher = NodePath.join(baseDir, "fx");
102 + NodeFS.writeFileSync(
103 + launcher,
104 + `#!/bin/sh\nexec node ${JSON.stringify(NodePath.join(import.meta.dirname, "fake-fx.ts"))} "$@"\n`,
105 + { mode: 0o755 },
106 + );
107 + NodeFS.writeFileSync(
108 + NodePath.join(baseDir, "userdata/settings.json"),
109 + JSON.stringify({
110 + providerInstances: {
111 + [EVAL_INSTANCE]: {
112 + driver: "fx",
113 + displayName: "Eval Bot",
114 + environment: [{ name: "T3_EVAL_FX_CONTROL_DIR", value: control }],
115 + config: { enabled: true, binaryPath: launcher },
116 + },
117 + },
118 + }),
119 + );
120 + return { workspace, control };
121 +}
122 +
123 +/** Starts the server in the caller's scope; the scope's end stops it. */
124 +export const startEnvironment = Effect.fn("environment.start")(function* (input: {
125 + readonly baseDir: string;
126 + /** Origin the device uses to reach this machine, without a port. */
127 + readonly deviceHost: string;
128 +}) {
129 + NodeFS.rmSync(input.baseDir, { recursive: true, force: true });
130 + const { workspace, control } = writeScratch(input.baseDir);
131 + const home = NodePath.join(input.baseDir, "home");
132 + NodeFS.mkdirSync(home, { recursive: true });
133 + const port = yield* freePort();
134 + const log = NodeFS.openSync(NodePath.join(input.baseDir, "server.log"), "w");
135 + yield* Effect.acquireRelease(
136 + Effect.sync(() =>
137 + NodeChildProcess.spawn(
138 + "node",
139 + [
140 + "apps/server/src/bin.ts",
141 + "serve",
142 + "--host",
143 + "0.0.0.0",
144 + "--port",
145 + String(port),
146 + "--base-dir",
147 + input.baseDir,
148 + "--no-browser",
149 + "--log-level",
150 + "warn",
151 + workspace,
152 + ],
153 + // A scratch HOME keeps the user's own skills and config out of the app under test.
154 + { cwd: REPO_ROOT, stdio: ["ignore", log, log], env: { ...process.env, HOME: home } },
155 + ),
156 + ),
157 + (child) => Effect.sync(() => child.kill("SIGTERM")),
158 + );
159 + yield* waitForPort(port);
160 + const origin = `http://127.0.0.1:${port}`;
161 + const issued = yield* runCli([
162 + "auth",
163 + "session",
164 + "issue",
165 + "--base-dir",
166 + input.baseDir,
167 + "--token-only",
168 + ]);
169 + // The token is the last line of stdout; stderr is only a fallback.
170 + const lastLine = (text: string) =>
171 + text
172 + .split("\n")
173 + .filter((line) => line.trim())
174 + .at(-1)
175 + ?.trim();
176 + const token = lastLine(issued.stdout) ?? lastLine(issued.stderr) ?? "";
177 + const http = (yield* HttpClient.HttpClient).pipe(
178 + HttpClient.mapRequest((request) =>
179 + request.pipe(
180 + HttpClientRequest.bearerToken(token),
181 + HttpClientRequest.setHeader(
182 + ORCHESTRATION_PROTOCOL_HEADER,
183 + String(ORCHESTRATION_PROTOCOL_VERSION),
184 + ),
185 + ),
186 + ),
187 + HttpClient.filterStatusOk,
188 + );
189 + const descriptor = yield* http
190 + .get(`${origin}/.well-known/t3/environment`)
191 + .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(ExecutionEnvironmentDescriptor)));
192 + const shell = () =>
193 + http
194 + .get(`${origin}/api/orchestration/shell`)
195 + .pipe(Effect.flatMap(HttpClientResponse.schemaBodyJson(OrchestrationV2ShellSnapshot)));
196 + const createProject = Effect.fn("environment.createProject")(function* (
197 + title: string,
198 + workspaceRoot: string,
199 + ) {
200 + const projectId = ProjectId.make(yield* uuid);
201 + yield* http.execute(
202 + HttpClientRequest.post(`${origin}/api/projects/mutate`).pipe(
203 + HttpClientRequest.bodyJsonUnsafe({
204 + type: "project.create",
205 + commandId: yield* uuid,
206 + projectId,
207 + title,
208 + workspaceRoot,
209 + }),
210 + ),
211 + );
212 + return projectId;
213 + });
214 + const projectId = yield* createProject("acme-app", workspace);
215 + const ticket = yield* http
216 + .execute(HttpClientRequest.post(`${origin}/api/auth/websocket-ticket`))
217 + .pipe(
218 + Effect.flatMap(HttpClientResponse.schemaBodyJson(Schema.Struct({ ticket: Schema.String }))),
219 + );
220 + // Built into the caller's scope: `Effect.provide` would close the socket once `make` returns.
221 + const protocol = yield* Layer.build(
222 + RpcClient.layerProtocolSocket().pipe(
223 + Layer.provide(
224 + Socket.layerWebSocket(
225 + `${origin.replace(/^http/, "ws")}/ws?orchestrationProtocol=${ORCHESTRATION_PROTOCOL_VERSION}&wsTicket=${encodeURIComponent(ticket.ticket)}`,
226 + ),
227 + ),
228 + Layer.provide(RpcSerialization.layerJson),
229 + Layer.provide(NodeSocket.layerWebSocketConstructorWS),
230 + ),
231 + );
232 + const client = yield* RpcClient.make(WsRpcGroup).pipe(Effect.provideContext(protocol));
233 + const environmentId = EnvironmentId.make(descriptor.environmentId);
234 +
235 + const command = Effect.map(uuid, CommandId.make);
236 + const projection = (threadId: ThreadId) =>
237 + client["orchestration.getThreadProjection"]({ threadId });
238 +
239 + const create = Effect.fn("environment.create")(function* (
240 + title: string,
241 + prompt?: string,
242 + project: ProjectId = projectId,
243 + ) {
244 + const threadId = ThreadId.make(yield* uuid);
245 + yield* client["orchestration.launchThread"]({
246 + commandId: yield* command,
247 + threadId,
248 + projectId: project,
249 + title,
250 + modelSelection: { instanceId: EVAL_INSTANCE, model: EVAL_MODEL },
251 + runtimeMode: "full-access",
252 + interactionMode: "default",
253 + workspaceStrategy: { type: "root" },
254 + ...(prompt === undefined ? {} : { initialMessage: { text: prompt, attachments: [] } }),
255 + });
256 + return threadId;
257 + });
258 +
259 + /** Starts a thread whose first reply follows `prompt`'s `[[eval:…]]` script. */
260 + const launch = (title: string, prompt: string) => create(title, prompt);
261 +
262 + const until = Effect.fn("environment.until")(function* <A>(
263 + description: string,
264 + threadId: ThreadId,
265 + pick: (value: OrchestrationV2ThreadProjection) => A | undefined,
266 + ) {
267 + for (let attempt = 0; attempt < 150; attempt++) {
268 + const found = pick(yield* projection(threadId));
269 + if (found !== undefined) return found;
270 + yield* Effect.sleep("200 millis");
271 + }
272 + return yield* new EnvironmentError({ message: `Thread ${threadId} never had ${description}.` });
273 + });
274 +
275 + const send = Effect.fn("environment.send")(function* (threadId: ThreadId, text: string) {
276 + yield* client["orchestration.dispatchCommand"]({
277 + type: "message.dispatch",
278 + commandId: yield* command,
279 + threadId,
280 + messageId: MessageId.make(yield* uuid),
281 + text,
282 + attachments: [],
283 + dispatchMode: { type: "queue_after_active" },
284 + createdBy: "user",
285 + creationSource: "server",
286 + });
287 + });
288 +
289 + const isSettled = (value: OrchestrationV2ThreadProjection) =>
290 + value.runs.length > 0 &&
291 + value.runs.every((run) =>
292 + ["completed", "failed", "interrupted", "cancelled"].includes(run.status),
293 + );
294 +
295 + /** Waits until every run of the thread has ended. */
296 + const settled = Effect.fn("environment.settled")(function* (threadId: ThreadId) {
297 + for (let attempt = 0; attempt < 600; attempt++) {
298 + const value = yield* projection(threadId);
299 + if (isSettled(value)) return value;
300 + yield* Effect.sleep("200 millis");
301 + }
302 + return yield* new EnvironmentError({ message: `Thread ${threadId} never settled.` });
303 + });
304 +
305 + /** A settled thread of `turns` request and reply pairs, for scroll and jank cases. */
306 + const longThread = Effect.fn("environment.longThread")(function* (title: string, turns: number) {
307 + const threadId = yield* launch(title, "Summarize the module. [[eval:stream:400:0]]");
308 + yield* settled(threadId);
309 + for (let turn = 1; turn < turns; turn++) {
310 + const text = `Next part ${turn}. [[eval:stream:${turn % 3 === 0 ? 900 : 300}:0]]`;
311 + yield* send(threadId, text);
312 + // The message and its run land in one command, so once it shows, `settled` sees the run.
313 + yield* until("its next message", threadId, (value) =>
314 + value.messages.some((message) => message.text === text) ? true : undefined,
315 + );
316 + yield* settled(threadId);
317 + }
318 + return threadId;
319 + });
320 +
321 + return {
322 + /** An empty thread in its own project; `lock` then disables `evalbot` there, so sends are rejected. */
323 + launchLockable: Effect.fn("environment.launchLockable")(function* (title: string) {
324 + const root = NodePath.join(input.baseDir, `workspace/locked-${(yield* uuid).slice(0, 6)}`);
325 + NodeFS.mkdirSync(root, { recursive: true });
326 + const project = yield* createProject("locked", root);
327 + const threadId = yield* create(title, undefined, project);
328 + const lock = client["server.updateSettings"]({
329 + patch: {
330 + projectSettingsOverrides: {
331 + [project]: { providerInstances: { [EVAL_INSTANCE]: { enabled: false } } },
332 + },
333 + },
334 + });
335 + return { threadId, lock };
336 + }),
337 + /** Archives every thread but `keep`, so earlier samples' rows never crowd Home. */
338 + archiveAllExcept: Effect.fn("environment.archiveAllExcept")(function* (keep: ThreadId) {
339 + for (const thread of (yield* shell()).threads) {
340 + if (thread.id === keep) continue;
341 + yield* client["orchestration.dispatchCommand"]({
342 + type: "thread.archive",
343 + commandId: yield* command,
344 + threadId: thread.id,
345 + });
346 + }
347 + }),
348 + /** Stops every unfinished run, so one sample's held turns never leak into the next. */
349 + interruptAll: Effect.fn("environment.interruptAll")(function* () {
350 + const threads = (yield* shell()).threads;
351 + for (const thread of threads)
352 + for (const run of (yield* projection(thread.id)).runs) {
353 + if (["completed", "failed", "interrupted", "cancelled"].includes(run.status)) continue;
354 + yield* client["orchestration.dispatchCommand"]({
355 + type: "run.interrupt",
356 + commandId: yield* command,
357 + threadId: thread.id,
358 + runId: run.id,
359 + });
360 + }
361 + for (const thread of threads)
362 + if ((yield* projection(thread.id)).runs.length) yield* settled(thread.id);
363 + }),
364 + waitForPendingInput: (threadId: ThreadId) =>
365 + until("a pending question", threadId, (value) =>
366 + value.runtimeRequests.find((request) => request.status === "pending"),
367 + ),
368 + waitForSubagents: (threadId: ThreadId) =>
369 + until("running subagents", threadId, (value) =>
370 + value.subagents.length >= 2 ? value.subagents : undefined,
371 + ),
372 + origin,
373 + environmentId,
374 + archive: (threadId: ThreadId) =>
375 + command.pipe(
376 + Effect.flatMap((commandId) =>
377 + client["orchestration.dispatchCommand"]({ type: "thread.archive", commandId, threadId }),
378 + ),
379 + Effect.asVoid,
380 + ),
381 + unpin: (threadId: ThreadId) =>
382 + command.pipe(
383 + Effect.flatMap((commandId) =>
384 + client["orchestration.dispatchCommand"]({ type: "thread.unpin", commandId, threadId }),
385 + ),
386 + Effect.asVoid,
387 + ),
388 + projection,
389 + launch,
390 + send,
391 + settled,
392 + longThread,
393 + shell,
394 + release: (key: string) =>
395 + Effect.sync(() => NodeFS.writeFileSync(NodePath.join(control, `release-${key}`), "")),
396 + pairingUrl: () =>
397 + runCli([
398 + "auth",
399 + "pairing",
400 + "create",
401 + "--base-dir",
402 + input.baseDir,
403 + "--base-url",
404 + `http://${input.deviceHost}:${port}`,
405 + "--ttl",
406 + "15m",
407 + "--label",
408 + "mobile-behavior-eval",
409 + ]).pipe(
410 + Effect.flatMap((out) => {
411 + const url = /Pair URL:\s*(\S+)/.exec(`${out.stdout}\n${out.stderr}`)?.[1];
412 + return url
413 + ? Effect.succeed(url)
414 + : Effect.fail(new EnvironmentError({ message: "No pairing URL in CLI output." }));
415 + }),
416 + ),
417 + };
418 +}, Effect.provide(FetchHttpClient.layer));
419 +
420 +export type EvalEnvironment = Effect.Success<ReturnType<typeof startEnvironment>>;
scripts/mobile-behavior/fake-fx.ts new
+262
@@ -0,0 +1,262 @@
1 +// @effect-diagnostics nodeBuiltinImport:off globalTimers:off preferSchemaOverJson:off globalConsole:off - a tiny stdio ACP agent spawned by the server; plain Node keeps it dependency-free.
2 +/**
3 + * A scripted stand-in for the `fx` CLI, registered as an Fx provider instance so the
4 + * server streams deterministic replies with no model. A `[[eval:<script>]]` marker in
5 + * the prompt picks the reply; see `scriptOf`. `hold` scripts wait for a release file
6 + * named after the marker's key in `T3_EVAL_FX_CONTROL_DIR`.
7 + */
8 +import * as NodeFS from "node:fs";
9 +import * as NodePath from "node:path";
10 +import * as NodeReadline from "node:readline";
11 +
12 +const MODEL = "eval/scripted";
13 +
14 +type Json = null | boolean | number | string | Json[] | { [key: string]: Json };
15 +interface Message {
16 + readonly id?: number | string;
17 + readonly method?: string;
18 + readonly params?: { [key: string]: Json };
19 + readonly result?: Json;
20 + readonly error?: Json;
21 +}
22 +
23 +const controlDir = process.env.T3_EVAL_FX_CONTROL_DIR ?? "";
24 +const send = (message: object) => process.stdout.write(`${JSON.stringify(message)}\n`);
25 +const sleep = (ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms));
26 +const update = (sessionId: string, value: object) =>
27 + send({ jsonrpc: "2.0", method: "session/update", params: { sessionId, update: value } });
28 +
29 +const select = (id: string, category: string, currentValue: string, values: string[]) => ({
30 + id,
31 + name: id,
32 + category,
33 + type: "select",
34 + currentValue,
35 + options: values.map((value) => ({ value, name: value })),
36 +});
37 +// The Fx adapter needs a mode option and the modes list; it sets mode and effort per session.
38 +const configOptions = [
39 + select("model", "model", MODEL, [MODEL]),
40 + select("mode", "mode", "full-access", ["auto", "ask", "full-access"]),
41 + select("effort", "thought_level", "auto", ["auto", "low", "medium", "high"]),
42 +];
43 +const modes = {
44 + currentModeId: "full-access",
45 + availableModes: ["auto", "ask", "full-access"].map((id) => ({ id, name: id })),
46 +};
47 +
48 +let nextOutgoingId = 1_000_000;
49 +const pendingReplies = new Map<number, (message: Message) => void>();
50 +const cancelled = new Set<string>();
51 +let sessionCount = 0;
52 +
53 +function request(method: string, params: object): Promise<Message> {
54 + const id = nextOutgoingId++;
55 + send({ jsonrpc: "2.0", id, method, params });
56 + return new Promise((resolve) => pendingReplies.set(id, resolve));
57 +}
58 +
59 +/** Paragraphs of plain filler with a heading every fifth paragraph, so the feed lays out like a real reply. */
60 +export function fillerText(words: number, seed = 0): string {
61 + const vocabulary =
62 + "the agent reads the file then edits the module and runs the focused test before it writes a short summary of what changed and why the change is safe".split(
63 + " ",
64 + );
65 + const out: string[] = [];
66 + let paragraph: string[] = [];
67 + for (let index = 0; index < words; index++) {
68 + paragraph.push(vocabulary[(index * 7 + seed) % vocabulary.length]!);
69 + if (paragraph.length === 60 || index === words - 1) {
70 + const count = out.length;
71 + out.push(`${count % 5 === 0 ? `## Part ${count / 5 + 1}\n\n` : ""}${paragraph.join(" ")}.`);
72 + paragraph = [];
73 + }
74 + }
75 + return out.join("\n\n");
76 +}
77 +
78 +interface Script {
79 + readonly kind: "reply" | "stream" | "hold" | "question" | "subagents" | "fail";
80 + readonly words: number;
81 + readonly chunkMs: number;
82 + readonly key: string;
83 +}
84 +
85 +/** `[[eval:stream:3000:40]]` streams 3000 words in 40 ms chunks; `[[eval:hold:k]]` replies when `k` is released. */
86 +export function scriptOf(prompt: string): Script {
87 + const marker = /\[\[eval:([a-z]+)(?::([^\]:]+))?(?::([^\]:]+))?\]\]/.exec(prompt);
88 + const kind = marker?.[1];
89 + if (kind === "stream")
90 + return {
91 + kind,
92 + words: Number(marker?.[2] ?? 3000),
93 + chunkMs: Number(marker?.[3] ?? 40),
94 + key: "",
95 + };
96 + if (kind === "hold" || kind === "subagents" || kind === "question")
97 + return { kind, words: 40, chunkMs: 0, key: marker?.[2] ?? kind };
98 + if (kind === "fail") return { kind, words: 0, chunkMs: 0, key: "" };
99 + return { kind: "reply", words: 40, chunkMs: 0, key: "" };
100 +}
101 +
102 +async function waitForRelease(sessionId: string, key: string) {
103 + const file = NodePath.join(controlDir, `release-${key}`);
104 + while (!cancelled.has(sessionId) && !NodeFS.existsSync(file)) await sleep(100);
105 +}
106 +
107 +async function streamText(sessionId: string, text: string, chunkMs: number, messageId: string) {
108 + const tokens = text.split(/(?<= )/);
109 + for (let index = 0; index < tokens.length; index += 4) {
110 + if (cancelled.has(sessionId)) return;
111 + update(sessionId, {
112 + sessionUpdate: "agent_message_chunk",
113 + messageId,
114 + content: { type: "text", text: tokens.slice(index, index + 4).join("") },
115 + });
116 + if (chunkMs > 0) await sleep(chunkMs);
117 + }
118 +}
119 +
120 +async function runPrompt(sessionId: string, prompt: string, turn: number): Promise<object> {
121 + const script = scriptOf(prompt);
122 + const messageId = `${sessionId}-m${turn}`;
123 + if (script.kind === "fail") throw new Error("Scripted provider failure.");
124 + if (script.kind === "hold") await waitForRelease(sessionId, script.key);
125 + if (script.kind === "question") {
126 + const reply = await request("elicitation/create", {
127 + sessionId,
128 + mode: "form",
129 + message: "Which option should I use?",
130 + requestedSchema: {
131 + type: "object",
132 + properties: {
133 + choice: {
134 + type: "string",
135 + title: "Option",
136 + description: "Which option should I use?",
137 + enum: ["First", "Second"],
138 + },
139 + },
140 + required: ["choice"],
141 + },
142 + });
143 + if (reply.error) return { stopReason: "cancelled" };
144 + }
145 + if (script.kind === "subagents") {
146 + for (const name of ["one", "two"]) {
147 + const toolCallId = `${sessionId}-sub-${name}-${turn}`;
148 + update(sessionId, {
149 + sessionUpdate: "tool_call",
150 + toolCallId,
151 + name: "subagent",
152 + title: `Subagent · read ${name}`,
153 + kind: "other",
154 + status: "pending",
155 + rawInput: { request: { action: "run", task: `Read file ${name}` } },
156 + });
157 + update(sessionId, { sessionUpdate: "tool_call_update", toolCallId, status: "in_progress" });
158 + }
159 + await waitForRelease(sessionId, script.key);
160 + for (const name of ["one", "two"])
161 + update(sessionId, {
162 + sessionUpdate: "tool_call_update",
163 + toolCallId: `${sessionId}-sub-${name}-${turn}`,
164 + status: "completed",
165 + content: [
166 + {
167 + type: "content",
168 + // A long reply gives the child thread something to scroll.
169 + content: {
170 + type: "text",
171 + text: JSON.stringify({ ok: true, result: fillerText(1500, turn) }),
172 + },
173 + },
174 + ],
175 + });
176 + }
177 + if (cancelled.has(sessionId)) return { stopReason: "cancelled" };
178 + await streamText(sessionId, fillerText(script.words, turn), script.chunkMs, messageId);
179 + return { stopReason: cancelled.has(sessionId) ? "cancelled" : "end_turn" };
180 +}
181 +
182 +const turns = new Map<string, number>();
183 +
184 +async function handle(message: Message) {
185 + if (message.method === undefined) {
186 + if (typeof message.id === "number") pendingReplies.get(message.id)?.(message);
187 + return;
188 + }
189 + const reply = (result: object) => send({ jsonrpc: "2.0", id: message.id, result });
190 + const params = message.params ?? {};
191 + const sessionId = typeof params.sessionId === "string" ? params.sessionId : "";
192 + switch (message.method) {
193 + case "initialize":
194 + return reply({
195 + protocolVersion: 1,
196 + agentCapabilities: {
197 + loadSession: true,
198 + promptCapabilities: { image: true, audio: false, embeddedContext: true },
199 + mcpCapabilities: { http: true, sse: true, acp: true },
200 + sessionCapabilities: { list: {}, resume: {}, close: {}, systemPrompt: {} },
201 + },
202 + agentInfo: { name: "fx", title: "fx", version: "0.0.14" },
203 + authMethods: [],
204 + });
205 + case "session/new": {
206 + sessionCount += 1;
207 + const sessionId = `eval-session-${process.pid}-${sessionCount}`;
208 + return reply({ sessionId, configOptions, modes });
209 + }
210 + case "session/load":
211 + case "session/resume":
212 + return reply({ configOptions, modes });
213 + case "session/set_config_option":
214 + return reply({ configOptions });
215 + case "session/cancel":
216 + cancelled.add(sessionId);
217 + return;
218 + case "session/prompt": {
219 + cancelled.delete(sessionId);
220 + const turn = (turns.get(sessionId) ?? 0) + 1;
221 + turns.set(sessionId, turn);
222 + const blocks = Array.isArray(params.prompt) ? params.prompt : [];
223 + const text = blocks
224 + .map((block) =>
225 + block && typeof block === "object" && !Array.isArray(block) ? block.text : "",
226 + )
227 + .filter((value) => typeof value === "string")
228 + .join("\n");
229 + try {
230 + return reply(await runPrompt(sessionId, text, turn));
231 + } catch (error) {
232 + return send({
233 + jsonrpc: "2.0",
234 + id: message.id,
235 + error: { code: -32603, message: error instanceof Error ? error.message : String(error) },
236 + });
237 + }
238 + }
239 + default:
240 + if (message.id !== undefined) return reply({});
241 + }
242 +}
243 +
244 +function main(args: ReadonlyArray<string>) {
245 + const probes: Record<string, string> = {
246 + "--version": "fx 0.0.14",
247 + status: JSON.stringify({ auth: "eval", model: MODEL }),
248 + models: JSON.stringify({ ids: [MODEL] }),
249 + };
250 + const probe = probes[args[0] ?? ""];
251 + if (probe !== undefined) return console.log(probe);
252 + if (args[0] !== "acp") {
253 + console.error(`fake-fx: unsupported command ${args.join(" ")}`);
254 + process.exitCode = 2;
255 + return;
256 + }
257 + NodeReadline.createInterface({ input: process.stdin }).on("line", (line) => {
258 + if (line.trim()) void handle(JSON.parse(line) as Message);
259 + });
260 +}
261 +
262 +if (import.meta.main) main(process.argv.slice(2));
scripts/mobile-behavior/grade.ts new
+233
@@ -0,0 +1,233 @@
1 +/** Pure grading for the mobile behavior eval: frame and scroll math, checks, and the summary. */
2 +import type { FeedSample } from "./device.ts";
3 +
4 +export type Metrics = Readonly<Record<string, number | boolean | string | null>>;
5 +
6 +export interface Check {
7 + readonly metric: string;
8 + readonly max?: number;
9 + readonly min?: number;
10 + readonly equals?: number | boolean | string | null;
11 +}
12 +
13 +const percentile = (sorted: ReadonlyArray<number>, p: number) =>
14 + sorted.length === 0 ? 0 : sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))]!;
15 +
16 +export function median(values: ReadonlyArray<number>): number {
17 + const sorted = [...values].sort((a, b) => a - b);
18 + const middle = Math.floor(sorted.length / 2);
19 + return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2;
20 +}
21 +
22 +const round = (value: number, digits = 1) => Number(value.toFixed(digits));
23 +
24 +/**
25 + * Frame times from rAF timestamps, graded against the display's budget: `refreshMs` when given,
26 + * else 120 Hz when the fastest frames come under 12 ms and 60 Hz otherwise. A steady low frame
27 + * rate still counts as dropped frames. `dropped` counts the frames that should have run inside
28 + * each long gap.
29 + */
30 +export function frameStats(
31 + timestamps: ReadonlyArray<number>,
32 + prefix: string,
33 + refreshMs?: number,
34 +): Metrics {
35 + const gaps = timestamps.slice(1).map((time, index) => time - timestamps[index]!);
36 + if (gaps.length < 10) return { [`${prefix}Frames`]: gaps.length };
37 + const sorted = [...gaps].sort((a, b) => a - b);
38 + const refresh = refreshMs ?? (percentile(sorted, 0.05) < 12 ? 1000 / 120 : 1000 / 60);
39 + const dropped = gaps.reduce(
40 + (total, gap) => total + Math.max(0, Math.round(gap / refresh) - 1),
41 + 0,
42 + );
43 + return {
44 + [`${prefix}Frames`]: gaps.length,
45 + [`${prefix}RefreshHz`]: Math.round(1000 / refresh),
46 + [`${prefix}P50Ms`]: round(percentile(sorted, 0.5)),
47 + [`${prefix}P95Ms`]: round(percentile(sorted, 0.95)),
48 + [`${prefix}P99Ms`]: round(percentile(sorted, 0.99)),
49 + [`${prefix}MaxMs`]: round(sorted.at(-1)!),
50 + [`${prefix}DroppedPct`]: round((100 * dropped) / (gaps.length + dropped)),
51 + };
52 +}
53 +
54 +/** Fastest real fling step: about 8000 pt/s at 60 Hz. A bigger one-frame move is a jump. */
55 +export const TELEPORT_PX = 400;
56 +
57 +/**
58 + * How visible rows moved between JS frames. A row's screen y is its list position minus the
59 + * scroll offset, so a list that shifts its offset to hold content still reads as no motion.
60 + * `direction` is the way rows should move: +1 while dragging content down (scrolling up).
61 + */
62 +export function feedMotion(samples: ReadonlyArray<FeedSample>, direction: 1 | -1 | 0): Metrics {
63 + let backward = 0;
64 + let largest = 0;
65 + let teleports = 0;
66 + let moved = 0;
67 + for (let index = 1; index < samples.length; index++) {
68 + const before = samples[index - 1]!;
69 + const after = samples[index]!;
70 + const shared = Object.keys(before.rows ?? {}).filter((key) => after.rows?.[key] !== undefined);
71 + let step = 0;
72 + for (const key of shared) {
73 + const rowStep = after.rows![key]! - before.rows![key]!;
74 + if (Math.abs(rowStep) > Math.abs(step)) step = rowStep;
75 + }
76 + // A jump can replace every visible row; with the same content, the offset change is the move.
77 + if (shared.length === 0 && before.content === after.content)
78 + step = (before.scroll ?? 0) - (after.scroll ?? 0);
79 + moved += Math.abs(step);
80 + largest = Math.max(largest, Math.abs(step));
81 + if (Math.abs(step) > TELEPORT_PX) teleports++;
82 + if (direction !== 0) backward = Math.max(backward, -step * direction);
83 + }
84 + const fromEnd = samples.flatMap((sample) =>
85 + sample.fromEnd === undefined ? [] : [sample.fromEnd],
86 + );
87 + return {
88 + samples: samples.length,
89 + movedPx: Math.round(moved),
90 + backwardPx: Math.round(backward),
91 + largestStepPx: Math.round(largest),
92 + teleports,
93 + maxFromEndPx: fromEnd.length ? Math.max(...fromEnd) : null,
94 + finalFromEndPx: fromEnd.at(-1) ?? null,
95 + };
96 +}
97 +
98 +export function checkFails(check: Check, metrics: Metrics): string | null {
99 + const value = metrics[check.metric];
100 + if (value === undefined) return `${check.metric} missing`;
101 + if (check.equals !== undefined)
102 + return value === check.equals ? null : `${check.metric}=${value}, want ${check.equals}`;
103 + if (typeof value !== "number") return `${check.metric}=${value}, want a number`;
104 + if (check.max !== undefined && value > check.max)
105 + return `${check.metric}=${value} > ${check.max}`;
106 + if (check.min !== undefined && value < check.min)
107 + return `${check.metric}=${value} < ${check.min}`;
108 + return null;
109 +}
110 +
111 +export interface SampleRow {
112 + readonly id: string;
113 + readonly device: string;
114 + readonly sample: number;
115 + readonly pass: boolean | null;
116 + readonly failures: ReadonlyArray<string>;
117 + readonly metrics: Metrics;
118 + /** Why the sample has no grade: the scenario failed before it could measure. */
119 + readonly error?: string;
120 +}
121 +
122 +export interface CaseResult {
123 + readonly id: string;
124 + readonly device: string;
125 + readonly samples: number;
126 + readonly passed: number;
127 + readonly errors: number;
128 + /**
129 + * `null` when too few samples were graded: errors outnumber graded samples, or none ran.
130 + * Otherwise a case fails when half or more of its graded samples fail.
131 + */
132 + readonly pass: boolean | null;
133 + readonly key: number | boolean | string | null;
134 +}
135 +
136 +export function caseResult(
137 + id: string,
138 + device: string,
139 + key: string,
140 + rows: ReadonlyArray<SampleRow>,
141 +): CaseResult {
142 + const graded = rows.filter((row) => row.pass !== null);
143 + const errors = rows.filter((row) => row.error !== undefined).length;
144 + const passed = graded.filter((row) => row.pass).length;
145 + const values = graded.map((row) => row.metrics[key]);
146 + const numbers = values.filter((value): value is number => typeof value === "number");
147 + return {
148 + id,
149 + device,
150 + samples: graded.length,
151 + passed,
152 + errors,
153 + pass: graded.length === 0 || errors > graded.length ? null : passed > graded.length / 2,
154 + key:
155 + numbers.length === values.length && numbers.length > 0
156 + ? round(median(numbers))
157 + : (values[Math.floor(values.length / 2)] ?? null),
158 + };
159 +}
160 +
161 +const mark = (result: CaseResult) =>
162 + `${result.pass === null ? "incomplete" : result.pass ? "pass" : "FAIL"} ${result.passed}/${result.samples + result.errors}`;
163 +
164 +/** Markdown scoreboard. With a baseline, each row shows the earlier result. */
165 +export function summarize(input: {
166 + readonly cases: ReadonlyArray<{
167 + readonly id: string;
168 + readonly kind: string;
169 + readonly source: string;
170 + readonly key: string;
171 + }>;
172 + readonly results: ReadonlyArray<CaseResult>;
173 + readonly baseline?: ReadonlyArray<CaseResult>;
174 + readonly header: string;
175 +}): string {
176 + const devices = [...new Set(input.results.map((result) => result.device))];
177 + const cell = (result: CaseResult | undefined) =>
178 + result ? `${mark(result)} (${result.key ?? "-"})` : "-";
179 + const lines = [
180 + input.header,
181 + "",
182 + `| Case | Kind | Source | Key metric | ${devices.join(" | ")} |`,
183 + `| --- | --- | --- | --- | ${devices.map(() => "---").join(" | ")} |`,
184 + ];
185 + for (const testCase of input.cases) {
186 + const cells = devices.map((device) => {
187 + const now = input.results.find((r) => r.id === testCase.id && r.device === device);
188 + const before = input.baseline?.find((r) => r.id === testCase.id && r.device === device);
189 + return before ? `${cell(now)} was ${cell(before)}` : cell(now);
190 + });
191 + if (cells.every((value) => value === "-")) continue;
192 + lines.push(
193 + `| \`${testCase.id}\` | ${testCase.kind} | ${testCase.source} | ${testCase.key} | ${cells.join(" | ")} |`,
194 + );
195 + }
196 + const ran = input.results.filter((result) => result.pass !== null);
197 + const incomplete = input.results.length - ran.length;
198 + lines.push(
199 + "",
200 + `${ran.filter((r) => r.pass).length} of ${ran.length} case runs pass${incomplete ? `, ${incomplete} incomplete` : ""}.`,
201 + );
202 + return `${lines.join("\n")}\n`;
203 +}
204 +
205 +export function resultsFor(
206 + rows: ReadonlyArray<SampleRow>,
207 + cases: ReadonlyArray<{ readonly id: string; readonly key: string }>,
208 +): ReadonlyArray<CaseResult> {
209 + const keys = [...new Set(rows.map((row) => `${row.id}\u0000${row.device}`))];
210 + return keys.map((key) => {
211 + const [id, device] = key.split("\u0000") as [string, string];
212 + return caseResult(
213 + id,
214 + device,
215 + cases.find((entry) => entry.id === id)?.key ?? "",
216 + rows.filter((row) => row.id === id && row.device === device),
217 + );
218 + });
219 +}
220 +
221 +/**
222 + * Results that should stop a release: a guard or control that failed or could not be graded.
223 + * `bad` cases reproduce known bugs and fail until those are fixed.
224 + */
225 +export function blockingResults(
226 + results: ReadonlyArray<CaseResult>,
227 + cases: ReadonlyArray<{ readonly id: string; readonly kind: string }>,
228 +): ReadonlyArray<CaseResult> {
229 + return results.filter(
230 + (result) =>
231 + result.pass !== true && cases.find((entry) => entry.id === result.id)?.kind !== "bad",
232 + );
233 +}
scripts/mobile-behavior/run.test.ts new
+94
@@ -0,0 +1,94 @@
1 +import { expect, it } from "@effect/vitest";
2 +
3 +import { scriptOf } from "./fake-fx.ts";
4 +import { caseResult, checkFails, feedMotion, frameStats, type SampleRow } from "./grade.ts";
5 +
6 +const steady = (count: number, gap: number) =>
7 + Array.from({ length: count }, (_, index) => index * gap);
8 +
9 +it("counts the frames a long gap swallowed, against the device's own refresh rate", () => {
10 + const at120 = steady(120, 1000 / 120);
11 + expect(frameStats(at120, "ui")).toMatchObject({ uiRefreshHz: 120, uiDroppedPct: 0 });
12 +
13 + // One 50 ms stall at 60 Hz hides two frames.
14 + const stalled = [...steady(60, 1000 / 60), 59 * (1000 / 60) + 50];
15 + expect(frameStats(stalled, "js")).toMatchObject({ jsRefreshHz: 60, jsDroppedPct: 3.2 });
16 +
17 + // JS rAF fires on a timer, so a late frame makes the next gap short; the UI rate sets the budget.
18 + const jittered = steady(300, 1000 / 60).map((time, index) =>
19 + index % 15 === 7 ? time + 6 : time,
20 + );
21 + expect(frameStats(jittered, "js")).toMatchObject({ jsRefreshHz: 120 });
22 + expect(frameStats(jittered, "js", 1000 / 60)).toMatchObject({ jsRefreshHz: 60, jsDroppedPct: 0 });
23 +
24 + // A steady 20 FPS is two dropped frames in every three, not a slow display.
25 + expect(frameStats(steady(60, 50), "ui")).toMatchObject({ uiRefreshHz: 60, uiDroppedPct: 66.7 });
26 +});
27 +
28 +it("reads row motion on screen, so an offset change that keeps content still is not a jump", () => {
29 + const rows = (entries: Record<string, number>) => ({ threadId: "t", ready: true, rows: entries });
30 + // Scrolling up: rows move down the screen; the list then grows above and shifts its offset.
31 + const smooth = [rows({ a: 100, b: 400 }), rows({ a: 180, b: 480 }), rows({ a: 180, b: 480 })];
32 + expect(feedMotion(smooth, 1)).toMatchObject({ backwardPx: 0, teleports: 0, largestStepPx: 80 });
33 +
34 + const jerk = [rows({ a: 100 }), rows({ a: 180 }), rows({ a: 60 })];
35 + expect(feedMotion(jerk, 1)).toMatchObject({ backwardPx: 120 });
36 +
37 + const teleport = [rows({ a: 100 }), rows({ a: 1300 })];
38 + expect(feedMotion(teleport, 1)).toMatchObject({ teleports: 1, largestStepPx: 1200 });
39 +
40 + // A jump past every visible row leaves no shared row; the offset change is the move.
41 + const offset = (scroll: number, entries: Record<string, number>) => ({
42 + ...rows(entries),
43 + scroll,
44 + content: 9000,
45 + });
46 + expect(feedMotion([offset(6000, { a: 100 }), offset(1000, { z: 100 })], 1)).toMatchObject({
47 + teleports: 1,
48 + largestStepPx: 5000,
49 + });
50 +});
51 +
52 +it("fails a case when half or more of its graded samples fail", () => {
53 + const row = (pass: boolean | null, value: number): SampleRow => ({
54 + id: "c",
55 + device: "phone",
56 + sample: 1,
57 + pass,
58 + failures: [],
59 + metrics: { backwardPx: value },
60 + });
61 + expect(
62 + caseResult("c", "phone", "backwardPx", [row(true, 1), row(true, 3), row(false, 90)]),
63 + ).toMatchObject({
64 + pass: true,
65 + key: 3,
66 + });
67 + expect(caseResult("c", "phone", "backwardPx", [row(true, 1), row(false, 90)]).pass).toBe(false);
68 + expect(caseResult("c", "phone", "backwardPx", [row(null, 0)]).pass).toBe(null);
69 +
70 + // Errors that outnumber graded samples leave the case incomplete, not passing.
71 + const errored: SampleRow = { ...row(null, 0), error: "Inspector timed out" };
72 + expect(caseResult("c", "phone", "backwardPx", [row(true, 1), errored, errored])).toMatchObject({
73 + pass: null,
74 + errors: 2,
75 + });
76 +});
77 +
78 +it("checks exact values and bounds", () => {
79 + expect(checkFails({ metric: "rowStatus", equals: "input" }, { rowStatus: "working" })).toBe(
80 + "rowStatus=working, want input",
81 + );
82 + expect(checkFails({ metric: "backwardPx", max: 24 }, { backwardPx: 24 })).toBe(null);
83 + expect(checkFails({ metric: "contentGrewPx", min: 500 }, {})).toBe("contentGrewPx missing");
84 +});
85 +
86 +it("reads the fake provider's script from the prompt marker", () => {
87 + expect(scriptOf("Write it. [[eval:stream:4000:25]]")).toMatchObject({
88 + kind: "stream",
89 + words: 4000,
90 + chunkMs: 25,
91 + });
92 + expect(scriptOf("Run. [[eval:hold:bg-1]]")).toMatchObject({ kind: "hold", key: "bg-1" });
93 + expect(scriptOf("plain text")).toMatchObject({ kind: "reply" });
94 +});
scripts/mobile-behavior/run.ts new
+298
@@ -0,0 +1,298 @@
1 +#!/usr/bin/env node
2 +// @effect-diagnostics nodeBuiltinImport:off preferSchemaOverJson:off - owns Metro and writes result files.
3 +/**
4 + * Drives the T3 Code iOS dev client on one simulator against an isolated server with scripted
5 + * replies, and grades app behavior from the dev-only probe and server state. See README.md.
6 + */
7 +import * as NodeChildProcess from "node:child_process";
8 +import * as NodeFS from "node:fs";
9 +import * as NodeOS from "node:os";
10 +import * as NodePath from "node:path";
11 +import * as NodeRuntime from "@effect/platform-node/NodeRuntime";
12 +import * as NodeServices from "@effect/platform-node/NodeServices";
13 +import * as Clock from "effect/Clock";
14 +import * as Console from "effect/Console";
15 +import * as Effect from "effect/Effect";
16 +import * as Option from "effect/Option";
17 +import { Command, Flag } from "effect/cli";
18 +import { FetchHttpClient, HttpClient } from "effect/http";
19 +
20 +import { CASES, type MobileBehaviorCase } from "./cases.ts";
21 +import { APP_ID, DeviceError, makeDevice, SCHEME } from "./device.ts";
22 +import { freePort, startEnvironment, uuid } from "./environment.ts";
23 +import { blockingResults, checkFails, resultsFor, summarize, type SampleRow } from "./grade.ts";
24 +import { SCENARIOS, type Pose, type ScenarioContext } from "./scenarios.ts";
25 +
26 +const REPO_ROOT = NodePath.resolve(import.meta.dirname, "../..");
27 +const POSES = ["phone", "duo-closed", "duo-open", "ipad"] as const satisfies ReadonlyArray<Pose>;
28 +
29 +/** This machine's address as the simulator sees it: the tailnet address, else the first LAN IPv4. */
30 +function reachableHost(): string {
31 + const tailnet = NodeChildProcess.spawnSync("tailscale", ["ip", "-4"], { encoding: "utf8" });
32 + const ip = tailnet.status === 0 ? tailnet.stdout.trim().split("\n")[0] : undefined;
33 + if (ip) return ip;
34 + const lan = Object.values(NodeOS.networkInterfaces())
35 + .flatMap((entries) => entries ?? [])
36 + .find((entry) => entry.family === "IPv4" && !entry.internal);
37 + return lan?.address ?? "127.0.0.1";
38 +}
39 +
40 +/** Metro with the behavior probe compiled in, stopped with its process group when the scope ends. */
41 +const startMetro = Effect.fn("run.startMetro")(function* (host: string, out: string) {
42 + const port = yield* freePort();
43 + const log = NodeFS.openSync(NodePath.join(out, "metro.log"), "w");
44 + yield* Effect.acquireRelease(
45 + Effect.sync(() =>
46 + NodeChildProcess.spawn(
47 + "npx",
48 + ["expo", "start", "--dev-client", "--scheme", SCHEME, "--lan", "--port", String(port)],
49 + {
50 + cwd: NodePath.join(REPO_ROOT, "apps/mobile"),
51 + detached: true,
52 + stdio: ["ignore", log, log],
53 + env: {
54 + ...process.env,
55 + APP_VARIANT: "development",
56 + CI: "1",
57 + // Lazily fetched chunks sometimes arrive broken on pool emulators and throw SyntaxErrors.
58 + EXPO_NO_METRO_LAZY: "1",
59 + EXPO_PUBLIC_T3_BEHAVIOR_PROBE: "1",
60 + REACT_NATIVE_PACKAGER_HOSTNAME: host,
61 + },
62 + },
63 + ),
64 + ),
65 + (child) => Effect.sync(() => child.pid !== undefined && process.kill(-child.pid, "SIGTERM")),
66 + );
67 + const metro = `http://${host}:${port}`;
68 + for (let attempt = 0; attempt < 120; attempt++) {
69 + const ready = yield* HttpClient.get(`${metro}/status`).pipe(
70 + Effect.flatMap((response) => response.text),
71 + Effect.provide(FetchHttpClient.layer),
72 + Effect.orElseSucceed(() => ""),
73 + );
74 + if (ready.includes("running")) return metro;
75 + yield* Effect.sleep("1 second");
76 + }
77 + return yield* new DeviceError({ message: `Metro never started; see ${out}/metro.log.` });
78 +});
79 +
80 +const decodeRows = (file: string): ReadonlyArray<SampleRow> =>
81 + NodeFS.readFileSync(file, "utf8")
82 + .split("\n")
83 + .filter(Boolean)
84 + .map((line) => JSON.parse(line) as SampleRow);
85 +
86 +const command = Command.make(
87 + "mobile-behavior",
88 + {
89 + out: Flag.String("out").pipe(Flag.withDescription("Directory for results and run state.")),
90 + device: Flag.String("device").pipe(
91 + Flag.withDescription(
92 + "Simulator UDID; leave out for a Simlock lease, whose gateway picks it.",
93 + ),
94 + Flag.withDefault(""),
95 + ),
96 + deviceConfig: Flag.String("device-config").pipe(
97 + Flag.withDescription("The --config path device_open returned for this device."),
98 + ),
99 + deviceSession: Flag.String("device-session").pipe(
100 + Flag.withDescription("The --session name device_open returned for this device."),
101 + ),
102 + deviceName: Flag.String("device-name").pipe(
103 + Flag.withDescription(
104 + 'Simulator name as Metro lists it, e.g. "iPhone 17 Pro". Leave out when this run\'s Metro serves one device.',
105 + ),
106 + Flag.withDefault(""),
107 + ),
108 + pose: Flag.Literals("pose", POSES).pipe(Flag.withDefault<Pose>("phone")),
109 + label: Flag.String("label").pipe(
110 + Flag.withDescription("Column name in the scoreboard; defaults to the pose."),
111 + Flag.optional,
112 + ),
113 + cases: Flag.String("case").pipe(
114 + Flag.withDescription("Comma-separated case ids."),
115 + Flag.optional,
116 + ),
117 + samples: Flag.Int("samples").pipe(Flag.withDefault(3)),
118 + baseline: Flag.String("baseline").pipe(
119 + Flag.withDescription("results.jsonl from an earlier run to compare against."),
120 + Flag.optional,
121 + ),
122 + agentDevice: Flag.String("agent-device").pipe(
123 + Flag.withDefault(
124 + NodePath.join(REPO_ROOT, "scripts/node_modules/agent-device/bin/agent-device.mjs"),
125 + ),
126 + ),
127 + forgetOtherEnvironments: Flag.Boolean("forget-other-environments").pipe(
128 + Flag.withDescription(
129 + "Remove the app's other saved environments first. Only for a simulator kept for this eval.",
130 + ),
131 + Flag.withDefault(false),
132 + ),
133 + host: Flag.String("host").pipe(
134 + Flag.withDescription(
135 + "Address the simulator uses to reach this machine; defaults to the tailnet IP.",
136 + ),
137 + Flag.optional,
138 + ),
139 + },
140 + Effect.fn("mobileBehavior.main")(function* (flags) {
141 + const out = NodePath.resolve(flags.out);
142 + // Read first: the baseline may be this --out's own results.jsonl, which the run rewrites.
143 + const baseline = Option.map(flags.baseline, (file) => resultsFor(decodeRows(file), CASES));
144 + NodeFS.mkdirSync(out, { recursive: true });
145 + const label = Option.getOrElse(flags.label, () => flags.pose);
146 + const wanted = Option.map(flags.cases, (ids) => new Set(ids.split(",").map((id) => id.trim())));
147 + const selected = CASES.filter(
148 + (testCase) =>
149 + Option.match(wanted, { onNone: () => true, onSome: (ids) => ids.has(testCase.id) }) &&
150 + (testCase.poses?.includes(flags.pose) ?? true),
151 + );
152 + const unknown = Option.match(wanted, {
153 + onNone: () => [],
154 + onSome: (ids) => [...ids].filter((id) => !CASES.some((testCase) => testCase.id === id)),
155 + });
156 + if (unknown.length > 0)
157 + return yield* new DeviceError({ message: `Unknown case id(s): ${unknown.join(", ")}.` });
158 + if (selected.length === 0) return yield* new DeviceError({ message: "No cases match." });
159 + const host = Option.getOrElse(flags.host, reachableHost);
160 + const startedAt = yield* Clock.currentTimeMillis;
161 +
162 + const env = yield* startEnvironment({ baseDir: NodePath.join(out, "env"), deviceHost: host });
163 + const tag = (yield* uuid).slice(0, 4);
164 + yield* Console.error(`Seeding a 60-turn thread on ${env.origin}...`);
165 + const longThread = yield* env.longThread(`Long settled thread ${tag}`, 60);
166 +
167 + const metro = yield* startMetro(host, out);
168 + const device = makeDevice({
169 + agentDevice: flags.agentDevice,
170 + targetArgs: [
171 + "--platform",
172 + "ios",
173 + ...(flags.device ? ["--udid", flags.device] : []),
174 + "--config",
175 + flags.deviceConfig,
176 + "--session",
177 + flags.deviceSession,
178 + ],
179 + metro,
180 + deviceName: flags.deviceName,
181 + });
182 + yield* Console.error(`Opening the dev client against ${metro}...`);
183 + // agent-device answers iOS's "Open in" prompt; the flags keep dev-menu chrome off the app.
184 + const metroUrl = new URL(metro);
185 + const launchUrl = `${SCHEME}://expo-development-client/?url=${encodeURIComponent(metro)}&disableOnboarding=1&__expo_disable_fab=1&__expo_disable_auto_launch=1`;
186 + yield* device
187 + .run(
188 + [
189 + "open",
190 + APP_ID,
191 + "--relaunch",
192 + "--metro-host",
193 + metroUrl.hostname,
194 + "--metro-port",
195 + metroUrl.port,
196 + "--launch-url",
197 + launchUrl,
198 + ],
199 + 240_000,
200 + )
201 + .pipe(Effect.retry({ times: 2 }));
202 + for (let attempt = 0; ; attempt++) {
203 + const loaded = yield* device
204 + .evaluate("typeof __t3Behavior")
205 + .pipe(Effect.orElseSucceed(() => ""));
206 + if (loaded === "object") break;
207 + // iOS asks before a URL opens another app; busy hosts can miss the first accept.
208 + if (attempt % 5 === 0) yield* device.acceptOpenPrompt.pipe(Effect.ignore);
209 + if (attempt > 120)
210 + return yield* new DeviceError({
211 + message:
212 + "The probe never loaded. Is this a dev client built from this checkout's native fingerprint?",
213 + });
214 + yield* Effect.sleep("2 seconds");
215 + }
216 + if (flags.pose === "duo-closed" || flags.pose === "duo-open")
217 + yield* device.fold(flags.pose === "duo-closed" ? "closed" : "open");
218 + const forget = (ids: ReadonlyArray<string>) =>
219 + device
220 + .evaluate(`__t3Behavior.forgetEnvironments(${JSON.stringify(ids)})`)
221 + .pipe(Effect.andThen(Effect.sleep("3 seconds")));
222 + const pairing = yield* env.pairingUrl();
223 + yield* device.openUrl(
224 + `${SCHEME}://connections/new?pairingUrl=${encodeURIComponent(pairing)}&autoConnect=1`,
225 + );
226 + // Registered once pairing starts, so a stopped run never leaves its environment in the app.
227 + yield* Effect.addFinalizer(() => forget([env.environmentId]).pipe(Effect.ignore));
228 + const seeded = (state: { homeList: unknown; threads: Record<string, string> }) =>
229 + state.homeList !== null && longThread in state.threads;
230 + yield* device.waitFor("the paired environment's threads on Home", seeded, 60_000);
231 + // Other saved environments keep cached threads (some stuck working) that skew status cases.
232 + const others = (yield* device.state).environments.filter((id) => id !== env.environmentId);
233 + if (others.length > 0 && !flags.forgetOtherEnvironments)
234 + return yield* new DeviceError({
235 + message: `The app has ${others.length} other saved environment(s). Use a simulator with no other connections, or pass --forget-other-environments to remove them.`,
236 + });
237 + if (others.length > 0) yield* forget(others);
238 + const ctx: ScenarioContext = {
239 + env,
240 + device,
241 + pose: flags.pose,
242 + fixtures: { longThread },
243 + };
244 + const scenarios = [...new Set(selected.map((testCase) => testCase.scenario))];
245 + const rows: SampleRow[] = [];
246 + const resultsFile = NodeFS.openSync(NodePath.join(out, "results.jsonl"), "w");
247 + for (const scenario of scenarios) {
248 + const cases = selected.filter((testCase) => testCase.scenario === scenario);
249 + for (let sample = 1; sample <= flags.samples; sample++) {
250 + const outcome = yield* env.interruptAll().pipe(
251 + Effect.andThen(env.archiveAllExcept(longThread)),
252 + Effect.andThen(device.reload(seeded)),
253 + Effect.andThen(SCENARIOS[scenario](ctx)),
254 + Effect.map((metrics) => ({ metrics, error: undefined })),
255 + Effect.catch((error) => Effect.succeed({ metrics: {}, error: error.message })),
256 + // A bad probe reply or snapshot must cost one sample, not the run.
257 + Effect.catchDefect((defect) => Effect.succeed({ metrics: {}, error: String(defect) })),
258 + );
259 + for (const testCase of cases) {
260 + const failures = outcome.error
261 + ? []
262 + : testCase.checks.flatMap((check) => checkFails(check, outcome.metrics) ?? []);
263 + const row: SampleRow = {
264 + id: testCase.id,
265 + device: label,
266 + sample,
267 + pass: outcome.error ? null : failures.length === 0,
268 + failures,
269 + metrics: outcome.metrics,
270 + ...(outcome.error ? { error: outcome.error } : {}),
271 + };
272 + rows.push(row);
273 + NodeFS.writeSync(resultsFile, `${JSON.stringify(row)}\n`);
274 + yield* Console.error(
275 + `${testCase.id} #${sample}: ${row.pass === null ? row.error : row.pass ? "pass" : `FAIL ${failures.join("; ")}`}`,
276 + );
277 + }
278 + }
279 + }
280 + NodeFS.closeSync(resultsFile);
281 + const minutes = Math.round(((yield* Clock.currentTimeMillis) - startedAt) / 60_000);
282 + const results = resultsFor(rows, CASES);
283 + const summary = summarize({
284 + cases: selected as ReadonlyArray<MobileBehaviorCase>,
285 + results,
286 + ...Option.match(baseline, { onNone: () => ({}), onSome: (rows) => ({ baseline: rows }) }),
287 + header: `# Mobile behavior: ${label}\n\n${flags.deviceName || label}, ${flags.samples} samples per case, ${minutes} min. Cells show pass or FAIL and the key metric's median.`,
288 + });
289 + NodeFS.writeFileSync(NodePath.join(out, "summary.md"), summary);
290 + yield* Console.log(summary);
291 + if (blockingResults(results, CASES).length > 0) process.exitCode = 1;
292 + }, Effect.scoped),
293 +);
294 +
295 +Command.run(command, { version: "0.0.0" }).pipe(
296 + Effect.provide(NodeServices.layer),
297 + NodeRuntime.runMain,
298 +);
scripts/mobile-behavior/scenarios.ts new
+597
@@ -0,0 +1,597 @@
1 +// @effect-diagnostics globalDate:off - scenario windows are wall-clock by nature.
2 +/**
3 + * Scenarios drive one flow on the device and return metrics. Cases in `cases.ts` name a
4 + * scenario and check its metrics, so one flow can serve several issues.
5 + */
6 +import type { ThreadId } from "@t3tools/contracts";
7 +import type * as Crypto from "effect/Crypto";
8 +import * as Effect from "effect/Effect";
9 +import * as Schema from "effect/Schema";
10 +
11 +import {
12 + APP_ID,
13 + SCHEME,
14 + type AxNode,
15 + type Device,
16 + type FeedSample,
17 + type ProbeState,
18 +} from "./device.ts";
19 +import { uuid, type EvalEnvironment } from "./environment.ts";
20 +import { feedMotion, frameStats, type Metrics } from "./grade.ts";
21 +
22 +export type Pose = "phone" | "duo-closed" | "duo-open" | "ipad";
23 +
24 +export interface ScenarioContext {
25 + readonly env: EvalEnvironment;
26 + readonly device: Device;
27 + readonly pose: Pose;
28 + readonly fixtures: { readonly longThread: ThreadId };
29 +}
30 +
31 +// Child thread ids carry `:`, so each segment is encoded.
32 +const threadPath = (ctx: ScenarioContext, threadId: string) =>
33 + `threads/${encodeURIComponent(ctx.env.environmentId)}/${encodeURIComponent(threadId)}`;
34 +
35 +const isThread = (state: ProbeState, threadId: string) =>
36 + state.route?.name === "Thread" && state.route.params.threadId === threadId;
37 +
38 +const openThread = Effect.fn("scenario.openThread")(function* (
39 + ctx: ScenarioContext,
40 + threadId: string,
41 +) {
42 + yield* ctx.device.open(threadPath(ctx, threadId));
43 + return yield* ctx.device.waitFor(
44 + `thread ${threadId} with a ready feed`,
45 + (state) => isThread(state, threadId) && state.feed?.threadId === threadId && state.feed.ready,
46 + 20_000,
47 + );
48 +});
49 +
50 +/** Screen size in points, and the x of the thread column (the right column in split view). */
51 +const geometry = Effect.fn("scenario.geometry")(function* (ctx: ScenarioContext) {
52 + const nodes = yield* ctx.device.nodes;
53 + const rect = nodes[0]?.rect ?? { x: 0, y: 0, width: 400, height: 870 };
54 + const state = yield* ctx.device.state;
55 + return {
56 + width: rect.width,
57 + height: rect.height,
58 + threadX: Math.round(rect.width * (state.route?.columns ? 0.72 : 0.5)),
59 + };
60 +});
61 +
62 +const findNode = Effect.fn("scenario.findNode")(function* (
63 + ctx: ScenarioContext,
64 + description: string,
65 + match: (node: { label?: string; value?: string; kind?: string; editable?: boolean }) => boolean,
66 +) {
67 + const node = (yield* ctx.device.nodes).find(match);
68 + if (!node?.rect) return yield* new ScenarioFailure({ message: `No ${description} on screen.` });
69 + return node;
70 +});
71 +
72 +const composerNode = (ctx: ScenarioContext) =>
73 + findNode(
74 + ctx,
75 + "composer text view",
76 + (node) => node.kind === "text-view" && node.editable === true,
77 + );
78 +
79 +/** The text the native editor shows, read from the accessibility tree rather than React props. */
80 +const composerText = Effect.fn("scenario.composerText")(function* (ctx: ScenarioContext) {
81 + const node = yield* composerNode(ctx);
82 + return node.value ?? node.label ?? "";
83 +});
84 +
85 +/** A fresh settled thread, so drafts left by earlier samples never mix in. */
86 +const freshThread = Effect.fn("scenario.freshThread")(function* (
87 + ctx: ScenarioContext,
88 + prompt = "Hi. [[eval:reply]]",
89 +) {
90 + const threadId = yield* ctx.env.launch(`Fresh ${(yield* uuid).slice(0, 6)}`, prompt);
91 + if (!prompt.includes("[[eval:stream")) yield* ctx.env.settled(threadId);
92 + return threadId;
93 +});
94 +
95 +/** Replaces the composer's text, the way a person would after clearing it. */
96 +const typeInComposer = Effect.fn("scenario.typeInComposer")(function* (
97 + ctx: ScenarioContext,
98 + text: string,
99 +) {
100 + const { x, y } = ctx.device.center(yield* composerNode(ctx));
101 + yield* ctx.device.run(["fill", String(x), String(y), text]);
102 + // The composer slides up with the keyboard; a tap before it settles lands on the keyboard.
103 + yield* ctx.device.waitFor("the keyboard", (state) => state.composer.keyboard, 10_000);
104 + yield* Effect.sleep("1 second");
105 +});
106 +
107 +const countUserText = (
108 + projection: { readonly messages: ReadonlyArray<{ readonly text?: string }> },
109 + text: string,
110 +) => projection.messages.filter((message) => message.text === text).length;
111 +
112 +/** Starts a scripted reply in `threadId` and waits until the app shows it streaming. */
113 +const startStream = Effect.fn("scenario.startStream")(function* (
114 + ctx: ScenarioContext,
115 + threadId: ThreadId,
116 + script: string,
117 +) {
118 + const before = (yield* ctx.device.state).feed?.content ?? 0;
119 + yield* ctx.env.send(threadId, `Write the long report. ${script}`);
120 + yield* ctx.device.waitFor(
121 + "the reply to start streaming in the feed",
122 + (state) => (state.feed?.content ?? 0) > before + 200,
123 + 20_000,
124 + );
125 +});
126 +
127 +/** Records frames (and the visible feed) while `action` runs; the action's failure is not the metric. */
128 +/** Records frames (and the visible feed) while `action` runs; stops recording even if it fails. */
129 +const recordFor = <E, R>(
130 + ctx: ScenarioContext,
131 + withFeed: boolean,
132 + action: Effect.Effect<unknown, E, R>,
133 +) =>
134 + ctx.device.recordStart(withFeed).pipe(
135 + Effect.andThen(Effect.exit(action)),
136 + Effect.flatMap((exit) =>
137 + ctx.device.recordStop.pipe(Effect.flatMap((recording) => Effect.as(exit, recording))),
138 + ),
139 + );
140 +
141 +/** JS frame times jitter, so the display's refresh rate comes from the UI thread's vsync times. */
142 +const frameMetrics = (recording: { js: ReadonlyArray<number>; ui: ReadonlyArray<number> }) => {
143 + const ui = frameStats(recording.ui, "ui");
144 + const refreshMs = typeof ui.uiRefreshHz === "number" ? 1000 / ui.uiRefreshHz : undefined;
145 + return { ...frameStats(recording.js, "js", refreshMs), ...ui };
146 +};
147 +
148 +/** Every scenario fails with this one error, so the table has a single function type. */
149 +export class ScenarioFailure extends Schema.TaggedError<ScenarioFailure>()("ScenarioFailure", {
150 + message: Schema.String,
151 +}) {}
152 +
153 +const failure = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
154 + Effect.mapError(
155 + effect,
156 + (error) =>
157 + new ScenarioFailure({ message: error instanceof Error ? error.message : String(error) }),
158 + );
159 +
160 +type Scenario = (ctx: ScenarioContext) => Effect.Effect<Metrics, ScenarioFailure, Crypto.Crypto>;
161 +
162 +export const SCENARIOS = {
163 + /** Six upward flings in the long settled thread. */
164 + scrollUpLongThread: Effect.fn("scenario.scrollUpLongThread")(function* (ctx: ScenarioContext) {
165 + yield* openThread(ctx, ctx.fixtures.longThread);
166 + const { height, threadX } = yield* geometry(ctx);
167 + const recording = yield* recordFor(
168 + ctx,
169 + true,
170 + Effect.gen(function* () {
171 + for (let fling = 0; fling < 6; fling++)
172 + yield* ctx.device.swipe(threadX, Math.round(height * 0.35), Math.round(height * 0.8));
173 + yield* Effect.sleep("1 second");
174 + }),
175 + );
176 + return { ...feedMotion(recording.feed, 1), ...frameMetrics(recording) } satisfies Metrics;
177 + }, failure),
178 +
179 + /** A long reply streams into the open thread for eight seconds. */
180 + streamInView: Effect.fn("scenario.streamInView")(function* (ctx: ScenarioContext) {
181 + yield* openThread(ctx, ctx.fixtures.longThread);
182 + yield* startStream(ctx, ctx.fixtures.longThread, "[[eval:stream:4000:25]]");
183 + const recording = yield* recordFor(ctx, true, Effect.sleep("8 seconds"));
184 + const after = yield* ctx.device.state;
185 + yield* ctx.env.interruptAll();
186 + const motion = feedMotion(recording.feed, 0);
187 + return {
188 + ...motion,
189 + ...frameMetrics(recording),
190 + settledFromEndPx: settledFromEnd(recording.feed),
191 + contentGrewPx: (recording.feed.at(-1)?.content ?? 0) - (recording.feed[0]?.content ?? 0),
192 + followingAtEnd: after.feed?.following === true,
193 + } satisfies Metrics;
194 + }, failure),
195 +
196 + /** Types and sends a message from the composer at the end of the long thread. */
197 + sendInLongThread: Effect.fn("scenario.sendInLongThread")(function* (ctx: ScenarioContext) {
198 + yield* openThread(ctx, ctx.fixtures.longThread);
199 + const text = `Follow-up ${(yield* uuid).slice(0, 8)} [[eval:reply]]`;
200 + yield* typeInComposer(ctx, text);
201 + const recording = yield* recordFor(
202 + ctx,
203 + true,
204 + Effect.gen(function* () {
205 + yield* ctx.device.press('label="Send"');
206 + yield* Effect.sleep("4 seconds");
207 + }),
208 + );
209 + const after = yield* ctx.device.state;
210 + const nativeText = yield* composerText(ctx);
211 + const projection = yield* ctx.env.settled(ctx.fixtures.longThread);
212 + const motion = feedMotion(recording.feed, 0);
213 + return {
214 + maxFromEndPx: motion.maxFromEndPx ?? null,
215 + finalFromEndPx: motion.finalFromEndPx ?? null,
216 + serverMessages: countUserText(projection, text),
217 + draftCleared: after.composer.draft === "",
218 + editorCleared: nativeText === "",
219 + } satisfies Metrics;
220 + }, failure),
221 +
222 + /** A thread in the background finishes while the long thread sits mid-scroll. */
223 + backgroundFinish: Effect.fn("scenario.backgroundFinish")(function* (ctx: ScenarioContext) {
224 + const key = `bg-${(yield* uuid).slice(0, 8)}`;
225 + const other = yield* ctx.env.launch("Background work", `Run the job. [[eval:hold:${key}]]`);
226 + yield* openThread(ctx, ctx.fixtures.longThread);
227 + const { height, threadX } = yield* geometry(ctx);
228 + for (let fling = 0; fling < 2; fling++)
229 + yield* ctx.device.swipe(threadX, Math.round(height * 0.4), Math.round(height * 0.7));
230 + yield* waitForStill(ctx);
231 + const recording = yield* recordFor(
232 + ctx,
233 + true,
234 + Effect.gen(function* () {
235 + yield* ctx.env.release(key);
236 + yield* ctx.env.settled(other);
237 + yield* Effect.sleep("2 seconds");
238 + }),
239 + );
240 + return { ...feedMotion(recording.feed, 0) } satisfies Metrics;
241 + }, failure),
242 +
243 + /** Types in a streaming thread, switches to another thread, and comes back. */
244 + draftThreadSwitch: Effect.fn("scenario.draftThreadSwitch")(function* (ctx: ScenarioContext) {
245 + const text = `Draft ${(yield* uuid).slice(0, 8)}`;
246 + const other = yield* freshThread(ctx);
247 + const streaming = yield* freshThread(ctx, "Write the long report. [[eval:stream:4000:25]]");
248 + yield* openThread(ctx, streaming);
249 + yield* typeInComposer(ctx, text);
250 + yield* openThread(ctx, other);
251 + yield* Effect.sleep("1 second");
252 + const onOther = yield* ctx.device.state;
253 + const otherText = yield* composerText(ctx);
254 + yield* openThread(ctx, streaming);
255 + yield* Effect.sleep("1 second");
256 + const back = yield* ctx.device.state;
257 + const backText = yield* composerText(ctx);
258 + yield* ctx.env.interruptAll();
259 + const otherKey = `${ctx.env.environmentId}:${other}`;
260 + return {
261 + draftKept: back.composer.draft === text,
262 + editorShowsDraft: backText === text,
263 + leakedIntoOtherDraft: (onOther.composer.drafts[otherKey] ?? "") !== "",
264 + leakedIntoOtherEditor: otherText.includes(text),
265 + } satisfies Metrics;
266 + }, failure),
267 +
268 + /** Types a draft, sends the app to the background, and brings it back. */
269 + draftBackground: Effect.fn("scenario.draftBackground")(function* (ctx: ScenarioContext) {
270 + const text = `Draft ${(yield* uuid).slice(0, 8)}`;
271 + const threadId = yield* freshThread(ctx);
272 + yield* openThread(ctx, threadId);
273 + yield* typeInComposer(ctx, text);
274 + yield* ctx.device.home;
275 + yield* Effect.sleep("3 seconds");
276 + // A link back to the thread foregrounds the running app; a plain open would reload its JS.
277 + yield* ctx.device.run([
278 + "open",
279 + APP_ID,
280 + "--launch-url",
281 + `${SCHEME}://${threadPath(ctx, threadId)}`,
282 + ]);
283 + const back = yield* ctx.device.waitFor("the thread after foreground", (state) =>
284 + isThread(state, threadId),
285 + );
286 + return {
287 + draftKept: back.composer.draft === text,
288 + editorShowsDraft: (yield* composerText(ctx)) === text,
289 + } satisfies Metrics;
290 + }, failure),
291 +
292 + /** Taps the middle of the composer's text area. */
293 + tapComposer: Effect.fn("scenario.tapComposer")(function* (ctx: ScenarioContext) {
294 + const threadId = yield* freshThread(ctx);
295 + yield* openThread(ctx, threadId);
296 + const node = yield* composerNode(ctx);
297 + const { x, y } = ctx.device.center(node);
298 + yield* ctx.device.pressAt(x, y);
299 + yield* Effect.sleep("1500 millis");
300 + const after = yield* ctx.device.state;
301 + return {
302 + stayedOnThread: isThread(after, threadId),
303 + editorFocused:
304 + after.composer.editors.some((editor) => editor.expanded) && after.composer.keyboard,
305 + } satisfies Metrics;
306 + }, failure),
307 +
308 + /** Types `/eval`, picks the workspace's `eval-notes` skill, and checks the editor keeps focus. */
309 + skillPick: Effect.fn("scenario.skillPick")(function* (ctx: ScenarioContext) {
310 + yield* openThread(ctx, yield* freshThread(ctx));
311 + // Short screens fit one menu row above the keyboard, so filter the menu to one row.
312 + yield* typeInComposer(ctx, "/eval");
313 + const composer = yield* composerNode(ctx);
314 + // Menu rows read "<icon>, skill:<name>, <description>". Skills arrive with a workspace
315 + // snapshot the draft change requests.
316 + let item: AxNode | undefined;
317 + for (let attempt = 0; attempt < 10 && !item?.rect; attempt++) {
318 + yield* Effect.sleep("1 second");
319 + item = (yield* ctx.device.nodes).find(
320 + (node) =>
321 + node.rect !== undefined &&
322 + node.rect.y + node.rect.height <= composer.rect!.y &&
323 + node.label?.includes(", skill:eval-notes") === true,
324 + );
325 + }
326 + if (!item?.rect)
327 + return yield* new ScenarioFailure({ message: "The / menu did not list skill:eval-notes." });
328 + const { x, y } = ctx.device.center(item);
329 + yield* ctx.device.pressAt(x, y);
330 + yield* Effect.sleep("1500 millis");
331 + const after = yield* ctx.device.state;
332 + return {
333 + editorFocused:
334 + after.composer.editors.some((editor) => editor.expanded) && after.composer.keyboard,
335 + } satisfies Metrics;
336 + }, failure),
337 +
338 + /** Sends from a thread whose provider instance the server no longer has. */
339 + failedSend: Effect.fn("scenario.failedSend")(function* (ctx: ScenarioContext) {
340 + const { threadId, lock } = yield* ctx.env.launchLockable("Locked thread");
341 + yield* openThread(ctx, threadId);
342 + const text = `Will fail ${(yield* uuid).slice(0, 8)}`;
343 + yield* typeInComposer(ctx, text);
344 + yield* lock;
345 + // The app learns of the setting a moment after the server commits it.
346 + yield* Effect.sleep("2 seconds");
347 + // Either outcome is honest: Send turns off, or the rejected send shows an error.
348 + const sendBlocked =
349 + (yield* ctx.device.nodes).find((node) => node.label === "Send")?.enabled === false;
350 + if (!sendBlocked) yield* ctx.device.press('label="Send"');
351 + const shown = sendBlocked
352 + ? false
353 + : (yield* ctx.device
354 + .waitFor("a composer error", (state) => state.composer.error !== null, 8_000)
355 + .pipe(Effect.option))._tag === "Some";
356 + return {
357 + sendBlocked,
358 + errorShown: shown,
359 + silentFailure: !sendBlocked && !shown,
360 + } satisfies Metrics;
361 + }, failure),
362 +
363 + /** A working thread finishes while Home is open. */
364 + workingShelfFinish: Effect.fn("scenario.workingShelfFinish")(function* (ctx: ScenarioContext) {
365 + const key = `shelf-${(yield* uuid).slice(0, 8)}`;
366 + const threadId = yield* ctx.env.launch("Shelf work", `Run the job. [[eval:hold:${key}]]`);
367 + const working = yield* ctx.device.waitFor(
368 + "the thread to show as working",
369 + (state) => state.threads[threadId] === "working",
370 + );
371 + yield* ctx.env.release(key);
372 + yield* ctx.env.settled(threadId);
373 + // The shelf keeps a finished row for WORKING_EXIT_GRACE_MS (2 s) on purpose.
374 + yield* Effect.sleep("5 seconds");
375 + const after = yield* ctx.device.state;
376 + return {
377 + shelfShownWhileWorking: shelfCount(working) > 0,
378 + shelfCountAfter: shelfCount(after),
379 + // For diagnosis: threads the app itself calls working, beside the shelf's count.
380 + appWorkingThreads: Object.values(after.threads).filter((status) => status === "working")
381 + .length,
382 + statusAfter: after.threads[threadId] ?? "missing",
383 + } satisfies Metrics;
384 + }, failure),
385 +
386 + /** A thread asks a question while Home is open. */
387 + pendingQuestion: Effect.fn("scenario.pendingQuestion")(function* (ctx: ScenarioContext) {
388 + const threadId = yield* ctx.env.launch("Question", `Ask me first. [[eval:question]]`);
389 + yield* ctx.env.waitForPendingInput(threadId);
390 + yield* Effect.sleep("2 seconds");
391 + const after = yield* ctx.device.state;
392 + yield* ctx.env.interruptAll();
393 + return {
394 + status: after.threads[threadId] ?? "missing",
395 + shelfCount: shelfCount(after),
396 + } satisfies Metrics;
397 + }, failure),
398 +
399 + /** A thread runs two subagents; Home and the thread must not look idle. */
400 + subagentsRunning: Effect.fn("scenario.subagentsRunning")(function* (ctx: ScenarioContext) {
401 + const key = `sub-${(yield* uuid).slice(0, 8)}`;
402 + const threadId = yield* ctx.env.launch("Subagents", `Fan out. [[eval:subagents:${key}]]`);
403 + yield* ctx.env.waitForSubagents(threadId);
404 + yield* Effect.sleep("2 seconds");
405 + const home = yield* ctx.device.state;
406 + yield* openThread(ctx, threadId);
407 + yield* Effect.sleep("1500 millis");
408 + const thread = yield* ctx.device.state;
409 + yield* ctx.env.release(key);
410 + yield* ctx.env.settled(threadId);
411 + return {
412 + status: home.threads[threadId] ?? "missing",
413 + threadShowsStatus: thread.floatingStatus !== null,
414 + } satisfies Metrics;
415 + }, failure),
416 +
417 + /** Scrolls a finished subagent's child thread. */
418 + openSubagentThread: Effect.fn("scenario.openSubagentThread")(function* (ctx: ScenarioContext) {
419 + const key = `child-${(yield* uuid).slice(0, 8)}`;
420 + const threadId = yield* ctx.env.launch("Parent", `Fan out. [[eval:subagents:${key}]]`);
421 + yield* ctx.env.waitForSubagents(threadId);
422 + yield* ctx.env.release(key);
423 + const projection = yield* ctx.env.settled(threadId);
424 + const child = projection.subagents.find(
425 + (subagent) => subagent.childThreadId !== null,
426 + )?.childThreadId;
427 + if (!child)
428 + return yield* new ScenarioFailure({ message: "The subagent made no child thread." });
429 + yield* openThread(ctx, child);
430 + const { height, threadX } = yield* geometry(ctx);
431 + const recording = yield* recordFor(
432 + ctx,
433 + true,
434 + Effect.gen(function* () {
435 + yield* ctx.device.swipe(threadX, Math.round(height * 0.7), Math.round(height * 0.3));
436 + yield* ctx.device.swipe(threadX, Math.round(height * 0.3), Math.round(height * 0.7));
437 + }),
438 + );
439 + return {
440 + movedPx: feedMotion(recording.feed, 0).movedPx ?? 0,
441 + ...frameMetrics(recording),
442 + } satisfies Metrics;
443 + }, failure),
444 +
445 + /** Long-presses a row on Home and picks Pin. */
446 + pinFromLongPress: Effect.fn("scenario.pinFromLongPress")(function* (ctx: ScenarioContext) {
447 + const title = `Pin target ${(yield* uuid).slice(0, 6)}`;
448 + const target = yield* ctx.env.launch(title, "Hi. [[eval:reply]]");
449 + yield* ctx.env.settled(target);
450 + yield* waitForRow(ctx, title);
451 + // A just-inserted row can swallow the first long-press while it animates in.
452 + for (let attempt = 0; attempt < 2; attempt++) {
453 + yield* ctx.device.longPress(rowSelector(title));
454 + yield* Effect.sleep("1 second");
455 + if ((yield* ctx.device.nodes).some((node) => node.label === "Pin")) break;
456 + }
457 + yield* ctx.device.press('label="Pin"');
458 + yield* Effect.sleep("2 seconds");
459 + const shell = yield* ctx.env.shell();
460 + const pinned = shell.threads
461 + .filter((thread) => thread.pinnedAt != null)
462 + .map((thread) => thread.id);
463 + for (const id of pinned) yield* ctx.env.unpin(id);
464 + return {
465 + targetPinned: pinned.includes(target),
466 + otherPinned: pinned.filter((id) => id !== target).length,
467 + } satisfies Metrics;
468 + }, failure),
469 +
470 + /** Opens a link to a thread created after the app loaded its list. */
471 + linkToNewThread: Effect.fn("scenario.linkToNewThread")(function* (ctx: ScenarioContext) {
472 + const threadId = yield* ctx.env.launch("Linked thread", "Hi. [[eval:reply]]");
473 + yield* ctx.device.open(threadPath(ctx, threadId));
474 + yield* Effect.sleep("4 seconds");
475 + const after = yield* ctx.device.state;
476 + return { opensTarget: isThread(after, threadId) } satisfies Metrics;
477 + }, failure),
478 +
479 + /** Opens a link to an archived thread. */
480 + linkToArchivedThread: Effect.fn("scenario.linkToArchivedThread")(function* (
481 + ctx: ScenarioContext,
482 + ) {
483 + const threadId = yield* ctx.env.launch("Archived thread", "Hi. [[eval:reply]]");
484 + yield* ctx.env.settled(threadId);
485 + yield* ctx.env.archive(threadId);
486 + yield* Effect.sleep("1 second");
487 + yield* ctx.device.open(threadPath(ctx, threadId));
488 + yield* Effect.sleep("4 seconds");
489 + const after = yield* ctx.device.state;
490 + return { opensTarget: isThread(after, threadId) } satisfies Metrics;
491 + }, failure),
492 +
493 + /** Taps a row on Home. */
494 + openFromList: Effect.fn("scenario.openFromList")(function* (ctx: ScenarioContext) {
495 + const title = `Open me ${(yield* uuid).slice(0, 6)}`;
496 + const target = yield* ctx.env.launch(title, "Hi. [[eval:reply]]");
497 + yield* ctx.env.settled(target);
498 + yield* waitForRow(ctx, title);
499 + // A just-inserted row animates in, and a tap during that misses it.
500 + yield* Effect.sleep("1 second");
501 + yield* ctx.device.press(rowSelector(title));
502 + const after = yield* ctx.device
503 + .waitFor("the tapped thread", (state) => isThread(state, target))
504 + .pipe(Effect.option);
505 + return { opensTarget: after._tag === "Some" } satisfies Metrics;
506 + }, failure),
507 +
508 + /** On a folded Duo: open a thread from the list, go back, and open another. */
509 + duoBackThenOpen: Effect.fn("scenario.duoBackThenOpen")(function* (ctx: ScenarioContext) {
510 + const tag = (yield* uuid).slice(0, 6);
511 + const first = yield* ctx.env.launch(`Duo first ${tag}`, "Hi. [[eval:reply]]");
512 + const next = yield* ctx.env.launch(`Duo next ${tag}`, "Hi. [[eval:reply]]");
513 + yield* ctx.env.settled(first);
514 + yield* ctx.env.settled(next);
515 + yield* waitForRow(ctx, `Duo first ${tag}`);
516 + yield* ctx.device.press(rowSelector(`Duo first ${tag}`));
517 + yield* ctx.device.waitFor("the first thread", (state) => isThread(state, first));
518 + yield* ctx.device.back;
519 + yield* ctx.device.waitFor("Home", (state) => state.route?.name === "Home");
520 + yield* ctx.device.press(rowSelector(`Duo next ${tag}`));
521 + yield* Effect.sleep("2 seconds");
522 + const after = yield* ctx.device.state;
523 + // A thread can mount in a hidden column, so its title must sit inside the visible screen.
524 + const nodes = yield* ctx.device.nodes;
525 + const screen = nodes[0]?.rect;
526 + // The header title is a text node inside a navigation bar; a Home row with the title is not.
527 + const bars = nodes.flatMap((node) =>
528 + node.kind === "navigation-bar" && node.rect ? [node.rect] : [],
529 + );
530 + const title = nodes.find(
531 + (node) =>
532 + node.label === `Duo next ${tag}` &&
533 + bars.some((bar) => node.rect && node.rect.y >= bar.y && node.rect.y < bar.y + bar.height),
534 + )?.rect;
535 + return {
536 + opensTarget: isThread(after, next),
537 + titleOnScreen:
538 + screen !== undefined &&
539 + title !== undefined &&
540 + title.x >= screen.x &&
541 + title.x + title.width <= screen.x + screen.width,
542 + } satisfies Metrics;
543 + }, failure),
544 +} satisfies Record<string, Scenario>;
545 +
546 +export type ScenarioName = keyof typeof SCENARIOS;
547 +
548 +/** Row rects are relative to their list, so rows are tapped by label, not by point. */
549 +function rowSelector(title: string) {
550 + return `label="${title}"`;
551 +}
552 +
553 +/** Waits until fling momentum ends: the feed's offset holds still across half a second. */
554 +const waitForStill = Effect.fn("scenario.waitForStill")(function* (ctx: ScenarioContext) {
555 + let previous = -1;
556 + for (let attempt = 0; attempt < 20; attempt++) {
557 + const scroll = (yield* ctx.device.state).feed?.scroll ?? -1;
558 + if (scroll === previous) return;
559 + previous = scroll;
560 + yield* Effect.sleep("500 millis");
561 + }
562 +});
563 +
564 +/** Largest distance from the end on frames where the content did not just grow. */
565 +function settledFromEnd(samples: ReadonlyArray<FeedSample>) {
566 + let worst = 0;
567 + for (let index = 1; index < samples.length; index++)
568 + if (samples[index]!.content === samples[index - 1]!.content)
569 + worst = Math.max(worst, samples[index]!.fromEnd ?? 0);
570 + return worst;
571 +}
572 +
573 +/** Waits until a row with this title is on screen. */
574 +const waitForRow = Effect.fn("scenario.waitForRow")(function* (
575 + ctx: ScenarioContext,
576 + title: string,
577 +) {
578 + for (let attempt = 0; attempt < 20; attempt++) {
579 + const nodes = yield* ctx.device.nodes;
580 + if (nodes.some((node) => node.label?.includes(title))) return;
581 + // A crowded list can hold the new row below the fold; scroll to the top once.
582 + if (attempt === 4) yield* ctx.device.run(["scroll", "top"]).pipe(Effect.ignore);
583 + // A finished row can stay folded in the Working shelf; open it once if it is collapsed.
584 + const shelf = nodes.find((node) => /^\d+ working threads?$/.test(node.label ?? ""));
585 + const collapsed = (yield* ctx.device.state).homeList?.some(
586 + (entry) => entry.type === "v2-working-shelf" && entry.expanded === false,
587 + );
588 + if (attempt === 10 && collapsed && shelf?.label)
589 + yield* ctx.device.press(`label="${shelf.label}"`);
590 + yield* Effect.sleep("500 millis");
591 + }
592 + return yield* new ScenarioFailure({ message: `Row "${title}" never appeared on Home.` });
593 +});
594 +
595 +function shelfCount(state: ProbeState) {
596 + return state.homeList?.find((entry) => entry.type === "v2-working-shelf")?.count ?? 0;
597 +}
scripts/package.json
+1
@@ -10,6 +10,7 @@
10 "@effect/platform-node": "catalog:",
11 "@electron/asar": "^4.3.0",
12 "@electron/osx-sign": "2.7.0",
13 + "@t3tools/contracts": "workspace:*",
14 "@t3tools/shared": "workspace:*",
15 "@t3tools/tailscale": "workspace:*",
16 "agent-device": "0.21.32",