The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | // @effect-diagnostics globalDate:off - scenario windows are wall-clock by nature. |
| 2 | /** |
| 3 | * Scenarios drive one flow on the device and return metrics. Cases in `cases.ts` name a |
| 4 | * scenario and check its metrics, so one flow can serve several issues. |
| 5 | */ |
| 6 | import type { ThreadId } from "@t3tools/contracts"; |
| 7 | import type * as SecureRandom from "effect/crypto/SecureRandom"; |
| 8 | import * as Effect from "effect/Effect"; |
| 9 | import * as Schema from "effect/Schema"; |
| 10 | |
| 11 | import { |
| 12 | APP_ID, |
| 13 | SCHEME, |
| 14 | type AxNode, |
| 15 | type Device, |
| 16 | type FeedSample, |
| 17 | type ProbeState, |
| 18 | } from "./device.ts"; |
| 19 | import { uuid, type EvalEnvironment } from "./environment.ts"; |
| 20 | import { feedMotion, frameStats, type Metrics } from "./grade.ts"; |
| 21 | |
| 22 | export type Pose = "phone" | "duo-closed" | "duo-open" | "ipad"; |
| 23 | |
| 24 | export interface ScenarioContext { |
| 25 | readonly env: EvalEnvironment; |
| 26 | readonly device: Device; |
| 27 | readonly pose: Pose; |
| 28 | readonly fixtures: { readonly longThread: ThreadId }; |
| 29 | } |
| 30 | |
| 31 | // Child thread ids carry `:`, so each segment is encoded. |
| 32 | const threadPath = (ctx: ScenarioContext, threadId: string) => |
| 33 | `threads/${encodeURIComponent(ctx.env.environmentId)}/${encodeURIComponent(threadId)}`; |
| 34 | |
| 35 | const isThread = (state: ProbeState, threadId: string) => |
| 36 | state.route?.name === "Thread" && state.route.params.threadId === threadId; |
| 37 | |
| 38 | const openThread = Effect.fn("scenario.openThread")(function* ( |
| 39 | ctx: ScenarioContext, |
| 40 | threadId: string, |
| 41 | ) { |
| 42 | yield* ctx.device.open(threadPath(ctx, threadId)); |
| 43 | return yield* ctx.device.waitFor( |
| 44 | `thread ${threadId} with a ready feed`, |
| 45 | (state) => isThread(state, threadId) && state.feed?.threadId === threadId && state.feed.ready, |
| 46 | 20_000, |
| 47 | ); |
| 48 | }); |
| 49 | |
| 50 | /** Screen size in points, and the x of the thread column (the right column in split view). */ |
| 51 | const geometry = Effect.fn("scenario.geometry")(function* (ctx: ScenarioContext) { |
| 52 | const nodes = yield* ctx.device.nodes; |
| 53 | const rect = nodes[0]?.rect ?? { x: 0, y: 0, width: 400, height: 870 }; |
| 54 | const state = yield* ctx.device.state; |
| 55 | return { |
| 56 | width: rect.width, |
| 57 | height: rect.height, |
| 58 | threadX: Math.round(rect.width * (state.route?.columns ? 0.72 : 0.5)), |
| 59 | }; |
| 60 | }); |
| 61 | |
| 62 | const findNode = Effect.fn("scenario.findNode")(function* ( |
| 63 | ctx: ScenarioContext, |
| 64 | description: string, |
| 65 | match: (node: { label?: string; value?: string; kind?: string; editable?: boolean }) => boolean, |
| 66 | ) { |
| 67 | const node = (yield* ctx.device.nodes).find(match); |
| 68 | if (!node?.rect) return yield* new ScenarioFailure({ message: `No ${description} on screen.` }); |
| 69 | return node; |
| 70 | }); |
| 71 | |
| 72 | const composerNode = (ctx: ScenarioContext) => |
| 73 | findNode( |
| 74 | ctx, |
| 75 | "composer text view", |
| 76 | (node) => node.kind === "text-view" && node.editable === true, |
| 77 | ); |
| 78 | |
| 79 | /** The text the native editor shows, read from the accessibility tree rather than React props. */ |
| 80 | const composerText = Effect.fn("scenario.composerText")(function* (ctx: ScenarioContext) { |
| 81 | const node = yield* composerNode(ctx); |
| 82 | return node.value ?? node.label ?? ""; |
| 83 | }); |
| 84 | |
| 85 | /** A fresh settled thread, so drafts left by earlier samples never mix in. */ |
| 86 | const freshThread = Effect.fn("scenario.freshThread")(function* ( |
| 87 | ctx: ScenarioContext, |
| 88 | prompt = "Hi. [[eval:reply]]", |
| 89 | ) { |
| 90 | const threadId = yield* ctx.env.launch(`Fresh ${(yield* uuid).slice(0, 6)}`, prompt); |
| 91 | if (!prompt.includes("[[eval:stream")) yield* ctx.env.settled(threadId); |
| 92 | return threadId; |
| 93 | }); |
| 94 | |
| 95 | /** Replaces the composer's text, the way a person would after clearing it. */ |
| 96 | const typeInComposer = Effect.fn("scenario.typeInComposer")(function* ( |
| 97 | ctx: ScenarioContext, |
| 98 | text: string, |
| 99 | ) { |
| 100 | const { x, y } = ctx.device.center(yield* composerNode(ctx)); |
| 101 | yield* ctx.device.run(["fill", String(x), String(y), text]); |
| 102 | // The composer slides up with the keyboard; a tap before it settles lands on the keyboard. |
| 103 | yield* ctx.device.waitFor("the keyboard", (state) => state.composer.keyboard, 10_000); |
| 104 | yield* Effect.sleep("1 second"); |
| 105 | }); |
| 106 | |
| 107 | const countUserText = ( |
| 108 | projection: { readonly messages: ReadonlyArray<{ readonly text?: string }> }, |
| 109 | text: string, |
| 110 | ) => projection.messages.filter((message) => message.text === text).length; |
| 111 | |
| 112 | /** Starts a scripted reply in `threadId` and waits until the app shows it streaming. */ |
| 113 | const startStream = Effect.fn("scenario.startStream")(function* ( |
| 114 | ctx: ScenarioContext, |
| 115 | threadId: ThreadId, |
| 116 | script: string, |
| 117 | ) { |
| 118 | const before = (yield* ctx.device.state).feed?.content ?? 0; |
| 119 | yield* ctx.env.send(threadId, `Write the long report. ${script}`); |
| 120 | yield* ctx.device.waitFor( |
| 121 | "the reply to start streaming in the feed", |
| 122 | (state) => (state.feed?.content ?? 0) > before + 200, |
| 123 | 20_000, |
| 124 | ); |
| 125 | }); |
| 126 | |
| 127 | /** Records frames (and the visible feed) while `action` runs; the action's failure is not the metric. */ |
| 128 | /** Records frames (and the visible feed) while `action` runs; stops recording even if it fails. */ |
| 129 | const recordFor = <E, R>( |
| 130 | ctx: ScenarioContext, |
| 131 | withFeed: boolean, |
| 132 | action: Effect.Effect<unknown, E, R>, |
| 133 | ) => |
| 134 | ctx.device.recordStart(withFeed).pipe( |
| 135 | Effect.andThen(Effect.exit(action)), |
| 136 | Effect.flatMap((exit) => |
| 137 | ctx.device.recordStop.pipe(Effect.flatMap((recording) => Effect.as(exit, recording))), |
| 138 | ), |
| 139 | ); |
| 140 | |
| 141 | /** JS frame times jitter, so the display's refresh rate comes from the UI thread's vsync times. */ |
| 142 | const frameMetrics = (recording: { js: ReadonlyArray<number>; ui: ReadonlyArray<number> }) => { |
| 143 | const ui = frameStats(recording.ui, "ui"); |
| 144 | const refreshMs = typeof ui.uiRefreshHz === "number" ? 1000 / ui.uiRefreshHz : undefined; |
| 145 | return { ...frameStats(recording.js, "js", refreshMs), ...ui }; |
| 146 | }; |
| 147 | |
| 148 | /** Every scenario fails with this one error, so the table has a single function type. */ |
| 149 | export class ScenarioFailure extends Schema.TaggedError<ScenarioFailure>()("ScenarioFailure", { |
| 150 | message: Schema.String, |
| 151 | }) {} |
| 152 | |
| 153 | const failure = <A, E, R>(effect: Effect.Effect<A, E, R>) => |
| 154 | Effect.mapError( |
| 155 | effect, |
| 156 | (error) => |
| 157 | new ScenarioFailure({ message: error instanceof Error ? error.message : String(error) }), |
| 158 | ); |
| 159 | |
| 160 | type Scenario = ( |
| 161 | ctx: ScenarioContext, |
| 162 | ) => Effect.Effect<Metrics, ScenarioFailure, SecureRandom.SecureRandom>; |
| 163 | |
| 164 | export const SCENARIOS = { |
| 165 | /** Six upward flings in the long settled thread. */ |
| 166 | scrollUpLongThread: Effect.fn("scenario.scrollUpLongThread")(function* (ctx: ScenarioContext) { |
| 167 | yield* openThread(ctx, ctx.fixtures.longThread); |
| 168 | const { height, threadX } = yield* geometry(ctx); |
| 169 | const recording = yield* recordFor( |
| 170 | ctx, |
| 171 | true, |
| 172 | Effect.gen(function* () { |
| 173 | for (let fling = 0; fling < 6; fling++) |
| 174 | yield* ctx.device.swipe(threadX, Math.round(height * 0.35), Math.round(height * 0.8)); |
| 175 | yield* Effect.sleep("1 second"); |
| 176 | }), |
| 177 | ); |
| 178 | return { ...feedMotion(recording.feed, 1), ...frameMetrics(recording) } satisfies Metrics; |
| 179 | }, failure), |
| 180 | |
| 181 | /** A long reply streams into the open thread for eight seconds. */ |
| 182 | streamInView: Effect.fn("scenario.streamInView")(function* (ctx: ScenarioContext) { |
| 183 | yield* openThread(ctx, ctx.fixtures.longThread); |
| 184 | yield* startStream(ctx, ctx.fixtures.longThread, "[[eval:stream:4000:25]]"); |
| 185 | const recording = yield* recordFor(ctx, true, Effect.sleep("8 seconds")); |
| 186 | const after = yield* ctx.device.state; |
| 187 | yield* ctx.env.interruptAll(); |
| 188 | const motion = feedMotion(recording.feed, 0); |
| 189 | return { |
| 190 | ...motion, |
| 191 | ...frameMetrics(recording), |
| 192 | settledFromEndPx: settledFromEnd(recording.feed), |
| 193 | contentGrewPx: (recording.feed.at(-1)?.content ?? 0) - (recording.feed[0]?.content ?? 0), |
| 194 | followingAtEnd: after.feed?.following === true, |
| 195 | } satisfies Metrics; |
| 196 | }, failure), |
| 197 | |
| 198 | /** Types and sends a message from the composer at the end of the long thread. */ |
| 199 | sendInLongThread: Effect.fn("scenario.sendInLongThread")(function* (ctx: ScenarioContext) { |
| 200 | yield* openThread(ctx, ctx.fixtures.longThread); |
| 201 | const text = `Follow-up ${(yield* uuid).slice(0, 8)} [[eval:reply]]`; |
| 202 | yield* typeInComposer(ctx, text); |
| 203 | const recording = yield* recordFor( |
| 204 | ctx, |
| 205 | true, |
| 206 | Effect.gen(function* () { |
| 207 | yield* ctx.device.press('label="Send"'); |
| 208 | yield* Effect.sleep("4 seconds"); |
| 209 | }), |
| 210 | ); |
| 211 | const after = yield* ctx.device.state; |
| 212 | const nativeText = yield* composerText(ctx); |
| 213 | const projection = yield* ctx.env.settled(ctx.fixtures.longThread); |
| 214 | const motion = feedMotion(recording.feed, 0); |
| 215 | return { |
| 216 | maxFromEndPx: motion.maxFromEndPx ?? null, |
| 217 | finalFromEndPx: motion.finalFromEndPx ?? null, |
| 218 | serverMessages: countUserText(projection, text), |
| 219 | draftCleared: after.composer.draft === "", |
| 220 | editorCleared: nativeText === "", |
| 221 | } satisfies Metrics; |
| 222 | }, failure), |
| 223 | |
| 224 | /** A thread in the background finishes while the long thread sits mid-scroll. */ |
| 225 | backgroundFinish: Effect.fn("scenario.backgroundFinish")(function* (ctx: ScenarioContext) { |
| 226 | const key = `bg-${(yield* uuid).slice(0, 8)}`; |
| 227 | const other = yield* ctx.env.launch("Background work", `Run the job. [[eval:hold:${key}]]`); |
| 228 | yield* openThread(ctx, ctx.fixtures.longThread); |
| 229 | const { height, threadX } = yield* geometry(ctx); |
| 230 | for (let fling = 0; fling < 2; fling++) |
| 231 | yield* ctx.device.swipe(threadX, Math.round(height * 0.4), Math.round(height * 0.7)); |
| 232 | yield* waitForStill(ctx); |
| 233 | const recording = yield* recordFor( |
| 234 | ctx, |
| 235 | true, |
| 236 | Effect.gen(function* () { |
| 237 | yield* ctx.env.release(key); |
| 238 | yield* ctx.env.settled(other); |
| 239 | yield* Effect.sleep("2 seconds"); |
| 240 | }), |
| 241 | ); |
| 242 | return { ...feedMotion(recording.feed, 0) } satisfies Metrics; |
| 243 | }, failure), |
| 244 | |
| 245 | /** Types in a streaming thread, switches to another thread, and comes back. */ |
| 246 | draftThreadSwitch: Effect.fn("scenario.draftThreadSwitch")(function* (ctx: ScenarioContext) { |
| 247 | const text = `Draft ${(yield* uuid).slice(0, 8)}`; |
| 248 | const other = yield* freshThread(ctx); |
| 249 | const streaming = yield* freshThread(ctx, "Write the long report. [[eval:stream:4000:25]]"); |
| 250 | yield* openThread(ctx, streaming); |
| 251 | yield* typeInComposer(ctx, text); |
| 252 | yield* openThread(ctx, other); |
| 253 | yield* Effect.sleep("1 second"); |
| 254 | const onOther = yield* ctx.device.state; |
| 255 | const otherText = yield* composerText(ctx); |
| 256 | yield* openThread(ctx, streaming); |
| 257 | yield* Effect.sleep("1 second"); |
| 258 | const back = yield* ctx.device.state; |
| 259 | const backText = yield* composerText(ctx); |
| 260 | yield* ctx.env.interruptAll(); |
| 261 | const otherKey = `${ctx.env.environmentId}:${other}`; |
| 262 | return { |
| 263 | draftKept: back.composer.draft === text, |
| 264 | editorShowsDraft: backText === text, |
| 265 | leakedIntoOtherDraft: (onOther.composer.drafts[otherKey] ?? "") !== "", |
| 266 | leakedIntoOtherEditor: otherText.includes(text), |
| 267 | } satisfies Metrics; |
| 268 | }, failure), |
| 269 | |
| 270 | /** Types a draft, sends the app to the background, and brings it back. */ |
| 271 | draftBackground: Effect.fn("scenario.draftBackground")(function* (ctx: ScenarioContext) { |
| 272 | const text = `Draft ${(yield* uuid).slice(0, 8)}`; |
| 273 | const threadId = yield* freshThread(ctx); |
| 274 | yield* openThread(ctx, threadId); |
| 275 | yield* typeInComposer(ctx, text); |
| 276 | yield* ctx.device.home; |
| 277 | yield* Effect.sleep("3 seconds"); |
| 278 | // A link back to the thread foregrounds the running app; a plain open would reload its JS. |
| 279 | yield* ctx.device.run([ |
| 280 | "open", |
| 281 | APP_ID, |
| 282 | "--launch-url", |
| 283 | `${SCHEME}://${threadPath(ctx, threadId)}`, |
| 284 | ]); |
| 285 | const back = yield* ctx.device.waitFor("the thread after foreground", (state) => |
| 286 | isThread(state, threadId), |
| 287 | ); |
| 288 | return { |
| 289 | draftKept: back.composer.draft === text, |
| 290 | editorShowsDraft: (yield* composerText(ctx)) === text, |
| 291 | } satisfies Metrics; |
| 292 | }, failure), |
| 293 | |
| 294 | /** Taps the middle of the composer's text area. */ |
| 295 | tapComposer: Effect.fn("scenario.tapComposer")(function* (ctx: ScenarioContext) { |
| 296 | const threadId = yield* freshThread(ctx); |
| 297 | yield* openThread(ctx, threadId); |
| 298 | const node = yield* composerNode(ctx); |
| 299 | const { x, y } = ctx.device.center(node); |
| 300 | yield* ctx.device.pressAt(x, y); |
| 301 | yield* Effect.sleep("1500 millis"); |
| 302 | const after = yield* ctx.device.state; |
| 303 | return { |
| 304 | stayedOnThread: isThread(after, threadId), |
| 305 | editorFocused: |
| 306 | after.composer.editors.some((editor) => editor.expanded) && after.composer.keyboard, |
| 307 | } satisfies Metrics; |
| 308 | }, failure), |
| 309 | |
| 310 | /** Types `/eval`, picks the workspace's `eval-notes` skill, and checks the editor keeps focus. */ |
| 311 | skillPick: Effect.fn("scenario.skillPick")(function* (ctx: ScenarioContext) { |
| 312 | yield* openThread(ctx, yield* freshThread(ctx)); |
| 313 | // Short screens fit one menu row above the keyboard, so filter the menu to one row. |
| 314 | yield* typeInComposer(ctx, "/eval"); |
| 315 | const composer = yield* composerNode(ctx); |
| 316 | // Menu rows read "<icon>, skill:<name>, <description>". Skills arrive with a workspace |
| 317 | // snapshot the draft change requests. |
| 318 | let item: AxNode | undefined; |
| 319 | for (let attempt = 0; attempt < 10 && !item?.rect; attempt++) { |
| 320 | yield* Effect.sleep("1 second"); |
| 321 | item = (yield* ctx.device.nodes).find( |
| 322 | (node) => |
| 323 | node.rect !== undefined && |
| 324 | node.rect.y + node.rect.height <= composer.rect!.y && |
| 325 | node.label?.includes(", skill:eval-notes") === true, |
| 326 | ); |
| 327 | } |
| 328 | if (!item?.rect) |
| 329 | return yield* new ScenarioFailure({ message: "The / menu did not list skill:eval-notes." }); |
| 330 | const { x, y } = ctx.device.center(item); |
| 331 | yield* ctx.device.pressAt(x, y); |
| 332 | yield* Effect.sleep("1500 millis"); |
| 333 | const after = yield* ctx.device.state; |
| 334 | return { |
| 335 | editorFocused: |
| 336 | after.composer.editors.some((editor) => editor.expanded) && after.composer.keyboard, |
| 337 | } satisfies Metrics; |
| 338 | }, failure), |
| 339 | |
| 340 | /** Sends from a thread whose provider instance the server no longer has. */ |
| 341 | failedSend: Effect.fn("scenario.failedSend")(function* (ctx: ScenarioContext) { |
| 342 | const { threadId, lock } = yield* ctx.env.launchLockable("Locked thread"); |
| 343 | yield* openThread(ctx, threadId); |
| 344 | const text = `Will fail ${(yield* uuid).slice(0, 8)}`; |
| 345 | yield* typeInComposer(ctx, text); |
| 346 | yield* lock; |
| 347 | // The app learns of the setting a moment after the server commits it. |
| 348 | yield* Effect.sleep("2 seconds"); |
| 349 | // Either outcome is honest: Send turns off, or the rejected send shows an error. |
| 350 | const sendBlocked = |
| 351 | (yield* ctx.device.nodes).find((node) => node.label === "Send")?.enabled === false; |
| 352 | if (!sendBlocked) yield* ctx.device.press('label="Send"'); |
| 353 | const shown = sendBlocked |
| 354 | ? false |
| 355 | : (yield* ctx.device |
| 356 | .waitFor("a composer error", (state) => state.composer.error !== null, 8_000) |
| 357 | .pipe(Effect.option))._tag === "Some"; |
| 358 | return { |
| 359 | sendBlocked, |
| 360 | errorShown: shown, |
| 361 | silentFailure: !sendBlocked && !shown, |
| 362 | } satisfies Metrics; |
| 363 | }, failure), |
| 364 | |
| 365 | /** A working thread finishes while Home is open. */ |
| 366 | workingShelfFinish: Effect.fn("scenario.workingShelfFinish")(function* (ctx: ScenarioContext) { |
| 367 | const key = `shelf-${(yield* uuid).slice(0, 8)}`; |
| 368 | const threadId = yield* ctx.env.launch("Shelf work", `Run the job. [[eval:hold:${key}]]`); |
| 369 | const working = yield* ctx.device.waitFor( |
| 370 | "the thread to show as working", |
| 371 | (state) => state.threads[threadId] === "working", |
| 372 | ); |
| 373 | yield* ctx.env.release(key); |
| 374 | yield* ctx.env.settled(threadId); |
| 375 | // The shelf keeps a finished row for WORKING_EXIT_GRACE_MS (2 s) on purpose. |
| 376 | yield* Effect.sleep("5 seconds"); |
| 377 | const after = yield* ctx.device.state; |
| 378 | return { |
| 379 | shelfShownWhileWorking: shelfCount(working) > 0, |
| 380 | shelfCountAfter: shelfCount(after), |
| 381 | // For diagnosis: threads the app itself calls working, beside the shelf's count. |
| 382 | appWorkingThreads: Object.values(after.threads).filter((status) => status === "working") |
| 383 | .length, |
| 384 | statusAfter: after.threads[threadId] ?? "missing", |
| 385 | } satisfies Metrics; |
| 386 | }, failure), |
| 387 | |
| 388 | /** A thread asks a question while Home is open. */ |
| 389 | pendingQuestion: Effect.fn("scenario.pendingQuestion")(function* (ctx: ScenarioContext) { |
| 390 | const threadId = yield* ctx.env.launch("Question", `Ask me first. [[eval:question]]`); |
| 391 | yield* ctx.env.waitForPendingInput(threadId); |
| 392 | yield* Effect.sleep("2 seconds"); |
| 393 | const after = yield* ctx.device.state; |
| 394 | yield* ctx.env.interruptAll(); |
| 395 | return { |
| 396 | status: after.threads[threadId] ?? "missing", |
| 397 | shelfCount: shelfCount(after), |
| 398 | } satisfies Metrics; |
| 399 | }, failure), |
| 400 | |
| 401 | /** A thread runs two subagents; Home and the thread must not look idle. */ |
| 402 | subagentsRunning: Effect.fn("scenario.subagentsRunning")(function* (ctx: ScenarioContext) { |
| 403 | const key = `sub-${(yield* uuid).slice(0, 8)}`; |
| 404 | const threadId = yield* ctx.env.launch("Subagents", `Fan out. [[eval:subagents:${key}]]`); |
| 405 | yield* ctx.env.waitForSubagents(threadId); |
| 406 | yield* Effect.sleep("2 seconds"); |
| 407 | const home = yield* ctx.device.state; |
| 408 | yield* openThread(ctx, threadId); |
| 409 | yield* Effect.sleep("1500 millis"); |
| 410 | const thread = yield* ctx.device.state; |
| 411 | yield* ctx.env.release(key); |
| 412 | yield* ctx.env.settled(threadId); |
| 413 | return { |
| 414 | status: home.threads[threadId] ?? "missing", |
| 415 | threadShowsStatus: thread.floatingStatus !== null, |
| 416 | } satisfies Metrics; |
| 417 | }, failure), |
| 418 | |
| 419 | /** Scrolls a finished subagent's child thread. */ |
| 420 | openSubagentThread: Effect.fn("scenario.openSubagentThread")(function* (ctx: ScenarioContext) { |
| 421 | const key = `child-${(yield* uuid).slice(0, 8)}`; |
| 422 | const threadId = yield* ctx.env.launch("Parent", `Fan out. [[eval:subagents:${key}]]`); |
| 423 | yield* ctx.env.waitForSubagents(threadId); |
| 424 | yield* ctx.env.release(key); |
| 425 | const projection = yield* ctx.env.settled(threadId); |
| 426 | const child = projection.subagents.find( |
| 427 | (subagent) => subagent.childThreadId !== null, |
| 428 | )?.childThreadId; |
| 429 | if (!child) |
| 430 | return yield* new ScenarioFailure({ message: "The subagent made no child thread." }); |
| 431 | yield* openThread(ctx, child); |
| 432 | const { height, threadX } = yield* geometry(ctx); |
| 433 | const recording = yield* recordFor( |
| 434 | ctx, |
| 435 | true, |
| 436 | Effect.gen(function* () { |
| 437 | yield* ctx.device.swipe(threadX, Math.round(height * 0.7), Math.round(height * 0.3)); |
| 438 | yield* ctx.device.swipe(threadX, Math.round(height * 0.3), Math.round(height * 0.7)); |
| 439 | }), |
| 440 | ); |
| 441 | return { |
| 442 | movedPx: feedMotion(recording.feed, 0).movedPx ?? 0, |
| 443 | ...frameMetrics(recording), |
| 444 | } satisfies Metrics; |
| 445 | }, failure), |
| 446 | |
| 447 | /** Long-presses a row on Home and picks Pin. */ |
| 448 | pinFromLongPress: Effect.fn("scenario.pinFromLongPress")(function* (ctx: ScenarioContext) { |
| 449 | const title = `Pin target ${(yield* uuid).slice(0, 6)}`; |
| 450 | const target = yield* ctx.env.launch(title, "Hi. [[eval:reply]]"); |
| 451 | yield* ctx.env.settled(target); |
| 452 | yield* waitForRow(ctx, title); |
| 453 | // A just-inserted row can swallow the first long-press while it animates in. |
| 454 | for (let attempt = 0; attempt < 2; attempt++) { |
| 455 | yield* ctx.device.longPress(rowSelector(title)); |
| 456 | yield* Effect.sleep("1 second"); |
| 457 | if ((yield* ctx.device.nodes).some((node) => node.label === "Pin")) break; |
| 458 | } |
| 459 | yield* ctx.device.press('label="Pin"'); |
| 460 | yield* Effect.sleep("2 seconds"); |
| 461 | const shell = yield* ctx.env.shell(); |
| 462 | const pinned = shell.threads |
| 463 | .filter((thread) => thread.pinnedAt != null) |
| 464 | .map((thread) => thread.id); |
| 465 | for (const id of pinned) yield* ctx.env.unpin(id); |
| 466 | return { |
| 467 | targetPinned: pinned.includes(target), |
| 468 | otherPinned: pinned.filter((id) => id !== target).length, |
| 469 | } satisfies Metrics; |
| 470 | }, failure), |
| 471 | |
| 472 | /** Opens a link to a thread created after the app loaded its list. */ |
| 473 | linkToNewThread: Effect.fn("scenario.linkToNewThread")(function* (ctx: ScenarioContext) { |
| 474 | const threadId = yield* ctx.env.launch("Linked thread", "Hi. [[eval:reply]]"); |
| 475 | yield* ctx.device.open(threadPath(ctx, threadId)); |
| 476 | yield* Effect.sleep("4 seconds"); |
| 477 | const after = yield* ctx.device.state; |
| 478 | return { opensTarget: isThread(after, threadId) } satisfies Metrics; |
| 479 | }, failure), |
| 480 | |
| 481 | /** Opens a link to an archived thread. */ |
| 482 | linkToArchivedThread: Effect.fn("scenario.linkToArchivedThread")(function* ( |
| 483 | ctx: ScenarioContext, |
| 484 | ) { |
| 485 | const threadId = yield* ctx.env.launch("Archived thread", "Hi. [[eval:reply]]"); |
| 486 | yield* ctx.env.settled(threadId); |
| 487 | yield* ctx.env.archive(threadId); |
| 488 | yield* Effect.sleep("1 second"); |
| 489 | yield* ctx.device.open(threadPath(ctx, threadId)); |
| 490 | yield* Effect.sleep("4 seconds"); |
| 491 | const after = yield* ctx.device.state; |
| 492 | return { opensTarget: isThread(after, threadId) } satisfies Metrics; |
| 493 | }, failure), |
| 494 | |
| 495 | /** Taps a row on Home. */ |
| 496 | openFromList: Effect.fn("scenario.openFromList")(function* (ctx: ScenarioContext) { |
| 497 | const title = `Open me ${(yield* uuid).slice(0, 6)}`; |
| 498 | const target = yield* ctx.env.launch(title, "Hi. [[eval:reply]]"); |
| 499 | yield* ctx.env.settled(target); |
| 500 | yield* waitForRow(ctx, title); |
| 501 | // A just-inserted row animates in, and a tap during that misses it. |
| 502 | yield* Effect.sleep("1 second"); |
| 503 | yield* ctx.device.press(rowSelector(title)); |
| 504 | const after = yield* ctx.device |
| 505 | .waitFor("the tapped thread", (state) => isThread(state, target)) |
| 506 | .pipe(Effect.option); |
| 507 | return { opensTarget: after._tag === "Some" } satisfies Metrics; |
| 508 | }, failure), |
| 509 | |
| 510 | /** On a folded Duo: open a thread from the list, go back, and open another. */ |
| 511 | duoBackThenOpen: Effect.fn("scenario.duoBackThenOpen")(function* (ctx: ScenarioContext) { |
| 512 | const tag = (yield* uuid).slice(0, 6); |
| 513 | const first = yield* ctx.env.launch(`Duo first ${tag}`, "Hi. [[eval:reply]]"); |
| 514 | const next = yield* ctx.env.launch(`Duo next ${tag}`, "Hi. [[eval:reply]]"); |
| 515 | yield* ctx.env.settled(first); |
| 516 | yield* ctx.env.settled(next); |
| 517 | yield* waitForRow(ctx, `Duo first ${tag}`); |
| 518 | yield* ctx.device.press(rowSelector(`Duo first ${tag}`)); |
| 519 | yield* ctx.device.waitFor("the first thread", (state) => isThread(state, first)); |
| 520 | yield* ctx.device.back; |
| 521 | yield* ctx.device.waitFor("Home", (state) => state.route?.name === "Home"); |
| 522 | yield* ctx.device.press(rowSelector(`Duo next ${tag}`)); |
| 523 | yield* Effect.sleep("2 seconds"); |
| 524 | const after = yield* ctx.device.state; |
| 525 | // A thread can mount in a hidden column, so its title must sit inside the visible screen. |
| 526 | const nodes = yield* ctx.device.nodes; |
| 527 | const screen = nodes[0]?.rect; |
| 528 | // The header title is a text node inside a navigation bar; a Home row with the title is not. |
| 529 | const bars = nodes.flatMap((node) => |
| 530 | node.kind === "navigation-bar" && node.rect ? [node.rect] : [], |
| 531 | ); |
| 532 | const title = nodes.find( |
| 533 | (node) => |
| 534 | node.label === `Duo next ${tag}` && |
| 535 | bars.some((bar) => node.rect && node.rect.y >= bar.y && node.rect.y < bar.y + bar.height), |
| 536 | )?.rect; |
| 537 | return { |
| 538 | opensTarget: isThread(after, next), |
| 539 | titleOnScreen: |
| 540 | screen !== undefined && |
| 541 | title !== undefined && |
| 542 | title.x >= screen.x && |
| 543 | title.x + title.width <= screen.x + screen.width, |
| 544 | } satisfies Metrics; |
| 545 | }, failure), |
| 546 | } satisfies Record<string, Scenario>; |
| 547 | |
| 548 | export type ScenarioName = keyof typeof SCENARIOS; |
| 549 | |
| 550 | /** Row rects are relative to their list, so rows are tapped by label, not by point. */ |
| 551 | function rowSelector(title: string) { |
| 552 | return `label="${title}"`; |
| 553 | } |
| 554 | |
| 555 | /** Waits until fling momentum ends: the feed's offset holds still across half a second. */ |
| 556 | const waitForStill = Effect.fn("scenario.waitForStill")(function* (ctx: ScenarioContext) { |
| 557 | let previous = -1; |
| 558 | for (let attempt = 0; attempt < 20; attempt++) { |
| 559 | const scroll = (yield* ctx.device.state).feed?.scroll ?? -1; |
| 560 | if (scroll === previous) return; |
| 561 | previous = scroll; |
| 562 | yield* Effect.sleep("500 millis"); |
| 563 | } |
| 564 | }); |
| 565 | |
| 566 | /** Largest distance from the end on frames where the content did not just grow. */ |
| 567 | function settledFromEnd(samples: ReadonlyArray<FeedSample>) { |
| 568 | let worst = 0; |
| 569 | for (let index = 1; index < samples.length; index++) |
| 570 | if (samples[index]!.content === samples[index - 1]!.content) |
| 571 | worst = Math.max(worst, samples[index]!.fromEnd ?? 0); |
| 572 | return worst; |
| 573 | } |
| 574 | |
| 575 | /** Waits until a row with this title is on screen. */ |
| 576 | const waitForRow = Effect.fn("scenario.waitForRow")(function* ( |
| 577 | ctx: ScenarioContext, |
| 578 | title: string, |
| 579 | ) { |
| 580 | for (let attempt = 0; attempt < 20; attempt++) { |
| 581 | const nodes = yield* ctx.device.nodes; |
| 582 | if (nodes.some((node) => node.label?.includes(title))) return; |
| 583 | // A crowded list can hold the new row below the fold; scroll to the top once. |
| 584 | if (attempt === 4) yield* ctx.device.run(["scroll", "top"]).pipe(Effect.ignore); |
| 585 | // A finished row can stay folded in the Working shelf; open it once if it is collapsed. |
| 586 | const shelf = nodes.find((node) => /^\d+ working threads?$/.test(node.label ?? "")); |
| 587 | const collapsed = (yield* ctx.device.state).homeList?.some( |
| 588 | (entry) => entry.type === "v2-working-shelf" && entry.expanded === false, |
| 589 | ); |
| 590 | if (attempt === 10 && collapsed && shelf?.label) |
| 591 | yield* ctx.device.press(`label="${shelf.label}"`); |
| 592 | yield* Effect.sleep("500 millis"); |
| 593 | } |
| 594 | return yield* new ScenarioFailure({ message: `Row "${title}" never appeared on Home.` }); |
| 595 | }); |
| 596 | |
| 597 | function shelfCount(state: ProbeState) { |
| 598 | return state.homeList?.find((entry) => entry.type === "v2-working-shelf")?.count ?? 0; |
| 599 | } |