The best way to control Claude Code, Codex and more from any computer or phone
| 1 | /** |
| 2 | * Cases for the mobile behavior eval. `bad` fails on main today and must pass once fixed; |
| 3 | * `guard` passes on main and must keep passing (its issue is fixed, or does not show on iOS); |
| 4 | * `control` is ordinary use that must keep passing. Each names the scenario that produces its metrics and the checks on them. |
| 5 | */ |
| 6 | import type { Check } from "./grade.ts"; |
| 7 | import type { Pose, ScenarioName } from "./scenarios.ts"; |
| 8 | |
| 9 | export interface MobileBehaviorCase { |
| 10 | readonly id: string; |
| 11 | readonly area: "scroll" | "jank" | "composer" | "status" | "wrong-thread"; |
| 12 | readonly kind: "bad" | "guard" | "control"; |
| 13 | /** The issue or PR it came from. */ |
| 14 | readonly source: string; |
| 15 | /** The real bad experience, or the good one a guard protects. */ |
| 16 | readonly experience: string; |
| 17 | readonly scenario: ScenarioName; |
| 18 | /** The metric the scoreboard shows. */ |
| 19 | readonly key: string; |
| 20 | readonly checks: ReadonlyArray<Check>; |
| 21 | /** Omit to run on every pose. */ |
| 22 | readonly poses?: ReadonlyArray<Pose>; |
| 23 | } |
| 24 | |
| 25 | /** |
| 26 | * Frame budget checks for jank cases: at most 5% of UI frames and 10% of JS frames dropped. |
| 27 | * Swipe-driven cases check JS frames only: the injected swipes cost the simulator's UI thread. |
| 28 | */ |
| 29 | const smooth: ReadonlyArray<Check> = [ |
| 30 | { metric: "uiDroppedPct", max: 5 }, |
| 31 | { metric: "jsDroppedPct", max: 10 }, |
| 32 | ]; |
| 33 | |
| 34 | export const CASES: ReadonlyArray<MobileBehaviorCase> = [ |
| 35 | { |
| 36 | id: "scroll-up-never-moves-backward", |
| 37 | area: "scroll", |
| 38 | kind: "guard", |
| 39 | source: "#22350", |
| 40 | experience: "Flicking up a long thread, content jerks back down toward the latest message.", |
| 41 | scenario: "scrollUpLongThread", |
| 42 | key: "backwardPx", |
| 43 | checks: [ |
| 44 | { metric: "movedPx", min: 300 }, |
| 45 | { metric: "backwardPx", max: 24 }, |
| 46 | ], |
| 47 | }, |
| 48 | { |
| 49 | id: "scroll-up-never-teleports", |
| 50 | area: "scroll", |
| 51 | kind: "guard", |
| 52 | source: "#12072", |
| 53 | experience: "Dragging up a little after scroll-to-end, the viewport jumps far away.", |
| 54 | scenario: "scrollUpLongThread", |
| 55 | key: "largestStepPx", |
| 56 | checks: [ |
| 57 | { metric: "movedPx", min: 300 }, |
| 58 | { metric: "teleports", equals: 0 }, |
| 59 | ], |
| 60 | }, |
| 61 | { |
| 62 | id: "streaming-stays-at-latest", |
| 63 | area: "scroll", |
| 64 | kind: "bad", |
| 65 | source: "#15469", |
| 66 | experience: |
| 67 | "While a reply streams at the bottom, the thread jumps far above the latest message.", |
| 68 | scenario: "streamInView", |
| 69 | key: "settledFromEndPx", |
| 70 | checks: [ |
| 71 | { metric: "settledFromEndPx", max: 64 }, |
| 72 | { metric: "followingAtEnd", equals: true }, |
| 73 | ], |
| 74 | }, |
| 75 | { |
| 76 | id: "send-in-long-thread-stays-at-end", |
| 77 | area: "scroll", |
| 78 | kind: "guard", |
| 79 | source: "#13706", |
| 80 | experience: |
| 81 | "Sending in a long thread, the timeline jumps far up instead of showing the new message.", |
| 82 | scenario: "sendInLongThread", |
| 83 | key: "maxFromEndPx", |
| 84 | checks: [ |
| 85 | { metric: "maxFromEndPx", max: 1500 }, |
| 86 | { metric: "finalFromEndPx", max: 24 }, |
| 87 | ], |
| 88 | }, |
| 89 | { |
| 90 | id: "background-finish-keeps-scroll", |
| 91 | area: "scroll", |
| 92 | kind: "guard", |
| 93 | source: "#15165", |
| 94 | experience: "Another thread finishing resets the open thread's scroll position.", |
| 95 | scenario: "backgroundFinish", |
| 96 | key: "largestStepPx", |
| 97 | checks: [{ metric: "largestStepPx", max: 4 }], |
| 98 | }, |
| 99 | { |
| 100 | id: "jank-streaming-long-reply", |
| 101 | area: "jank", |
| 102 | kind: "bad", |
| 103 | source: "#15641, #14010, #19120", |
| 104 | experience: "A long reply streaming into a long thread freezes or stutters the app.", |
| 105 | scenario: "streamInView", |
| 106 | key: "jsDroppedPct", |
| 107 | checks: smooth, |
| 108 | }, |
| 109 | { |
| 110 | id: "jank-scrolling-settled-thread", |
| 111 | area: "jank", |
| 112 | kind: "control", |
| 113 | source: "#13925 (Android)", |
| 114 | experience: "Scrolling a long settled thread drops frames.", |
| 115 | scenario: "scrollUpLongThread", |
| 116 | key: "jsP99Ms", |
| 117 | checks: [ |
| 118 | { metric: "movedPx", min: 300 }, |
| 119 | { metric: "jsDroppedPct", max: 10 }, |
| 120 | ], |
| 121 | }, |
| 122 | { |
| 123 | id: "jank-scrolling-subagent-thread", |
| 124 | area: "jank", |
| 125 | kind: "control", |
| 126 | source: "#18376 (Android)", |
| 127 | experience: "Scrolling a subagent's thread is far less smooth than its parent.", |
| 128 | scenario: "openSubagentThread", |
| 129 | key: "jsP99Ms", |
| 130 | checks: [ |
| 131 | { metric: "movedPx", min: 300 }, |
| 132 | { metric: "jsDroppedPct", max: 10 }, |
| 133 | ], |
| 134 | }, |
| 135 | { |
| 136 | id: "draft-survives-thread-switch", |
| 137 | area: "composer", |
| 138 | kind: "guard", |
| 139 | source: "#17422", |
| 140 | experience: "A typed draft is gone after switching threads and coming back.", |
| 141 | scenario: "draftThreadSwitch", |
| 142 | key: "draftKept", |
| 143 | checks: [ |
| 144 | { metric: "draftKept", equals: true }, |
| 145 | { metric: "editorShowsDraft", equals: true }, |
| 146 | ], |
| 147 | }, |
| 148 | { |
| 149 | id: "draft-never-leaks-to-other-thread", |
| 150 | area: "composer", |
| 151 | kind: "guard", |
| 152 | source: "#7390", |
| 153 | experience: "Text typed in a streaming thread shows up in another thread's composer.", |
| 154 | scenario: "draftThreadSwitch", |
| 155 | key: "leakedIntoOtherEditor", |
| 156 | checks: [ |
| 157 | { metric: "leakedIntoOtherDraft", equals: false }, |
| 158 | { metric: "leakedIntoOtherEditor", equals: false }, |
| 159 | ], |
| 160 | }, |
| 161 | { |
| 162 | id: "draft-survives-background", |
| 163 | area: "composer", |
| 164 | kind: "guard", |
| 165 | source: "#17422, #7390", |
| 166 | experience: "A typed draft is gone after the app goes to the background and back.", |
| 167 | scenario: "draftBackground", |
| 168 | key: "draftKept", |
| 169 | checks: [ |
| 170 | { metric: "draftKept", equals: true }, |
| 171 | { metric: "editorShowsDraft", equals: true }, |
| 172 | ], |
| 173 | }, |
| 174 | { |
| 175 | id: "composer-clears-after-send", |
| 176 | area: "composer", |
| 177 | kind: "guard", |
| 178 | source: "#20982", |
| 179 | experience: "After sending, the message text stays in the composer.", |
| 180 | scenario: "sendInLongThread", |
| 181 | key: "editorCleared", |
| 182 | checks: [ |
| 183 | { metric: "draftCleared", equals: true }, |
| 184 | { metric: "editorCleared", equals: true }, |
| 185 | ], |
| 186 | }, |
| 187 | { |
| 188 | id: "composer-tap-focuses-text", |
| 189 | area: "composer", |
| 190 | kind: "guard", |
| 191 | source: "#24303", |
| 192 | experience: "Tapping the composer's text area opens the model picker instead of the keyboard.", |
| 193 | scenario: "tapComposer", |
| 194 | key: "editorFocused", |
| 195 | checks: [ |
| 196 | { metric: "stayedOnThread", equals: true }, |
| 197 | { metric: "editorFocused", equals: true }, |
| 198 | ], |
| 199 | }, |
| 200 | { |
| 201 | id: "composer-keeps-focus-after-command-pick", |
| 202 | area: "composer", |
| 203 | kind: "guard", |
| 204 | source: "#10956", |
| 205 | experience: "Picking a skill or command from the / menu closes the keyboard.", |
| 206 | scenario: "skillPick", |
| 207 | key: "editorFocused", |
| 208 | checks: [{ metric: "editorFocused", equals: true }], |
| 209 | }, |
| 210 | { |
| 211 | id: "failed-send-is-never-silent", |
| 212 | area: "composer", |
| 213 | kind: "guard", |
| 214 | source: "#15801", |
| 215 | experience: "A send the server will reject looks sent, with no error.", |
| 216 | scenario: "failedSend", |
| 217 | // The cell shows which path ran: Send blocked, or a rejected send with an error. |
| 218 | key: "sendBlocked", |
| 219 | checks: [{ metric: "silentFailure", equals: false }], |
| 220 | }, |
| 221 | { |
| 222 | id: "working-shelf-clears-when-done", |
| 223 | area: "status", |
| 224 | kind: "bad", |
| 225 | source: "#24901", |
| 226 | experience: "A finished thread stays in the Working shelf.", |
| 227 | scenario: "workingShelfFinish", |
| 228 | key: "shelfCountAfter", |
| 229 | checks: [ |
| 230 | { metric: "shelfShownWhileWorking", equals: true }, |
| 231 | { metric: "shelfCountAfter", equals: 0 }, |
| 232 | { metric: "statusAfter", equals: "ready" }, |
| 233 | ], |
| 234 | }, |
| 235 | { |
| 236 | id: "pending-question-shows-input", |
| 237 | area: "status", |
| 238 | kind: "guard", |
| 239 | source: "#24901", |
| 240 | experience: "A thread waiting on a question shows as Working instead of needing input.", |
| 241 | scenario: "pendingQuestion", |
| 242 | key: "status", |
| 243 | checks: [ |
| 244 | { metric: "status", equals: "input" }, |
| 245 | { metric: "shelfCount", equals: 0 }, |
| 246 | ], |
| 247 | }, |
| 248 | { |
| 249 | id: "running-subagents-not-idle", |
| 250 | area: "status", |
| 251 | kind: "guard", |
| 252 | source: "#16464, #4962", |
| 253 | experience: "A thread with subagents still running looks idle in the list and the thread.", |
| 254 | scenario: "subagentsRunning", |
| 255 | key: "status", |
| 256 | checks: [ |
| 257 | { metric: "status", equals: "working" }, |
| 258 | { metric: "threadShowsStatus", equals: true }, |
| 259 | ], |
| 260 | }, |
| 261 | { |
| 262 | id: "long-press-pin-pins-that-row", |
| 263 | area: "wrong-thread", |
| 264 | kind: "guard", |
| 265 | source: "#20428", |
| 266 | experience: "Pin from a row's long-press menu pins a different thread.", |
| 267 | scenario: "pinFromLongPress", |
| 268 | key: "targetPinned", |
| 269 | checks: [ |
| 270 | { metric: "targetPinned", equals: true }, |
| 271 | { metric: "otherPinned", equals: 0 }, |
| 272 | ], |
| 273 | }, |
| 274 | { |
| 275 | id: "link-opens-new-thread", |
| 276 | area: "wrong-thread", |
| 277 | kind: "guard", |
| 278 | source: "#14689 (web)", |
| 279 | experience: "A link to a just-created thread redirects to another thread.", |
| 280 | scenario: "linkToNewThread", |
| 281 | key: "opensTarget", |
| 282 | checks: [{ metric: "opensTarget", equals: true }], |
| 283 | }, |
| 284 | { |
| 285 | id: "link-opens-archived-thread", |
| 286 | area: "wrong-thread", |
| 287 | kind: "guard", |
| 288 | source: "#16443", |
| 289 | experience: "A link to an archived thread sends you Home.", |
| 290 | scenario: "linkToArchivedThread", |
| 291 | key: "opensTarget", |
| 292 | checks: [{ metric: "opensTarget", equals: true }], |
| 293 | }, |
| 294 | { |
| 295 | id: "folded-duo-back-opens-right-thread", |
| 296 | area: "wrong-thread", |
| 297 | kind: "guard", |
| 298 | source: "#20633", |
| 299 | experience: "On a folded iPhone Duo, after going back, the next thread opens off screen.", |
| 300 | scenario: "duoBackThenOpen", |
| 301 | key: "titleOnScreen", |
| 302 | poses: ["duo-closed"], |
| 303 | checks: [ |
| 304 | { metric: "opensTarget", equals: true }, |
| 305 | { metric: "titleOnScreen", equals: true }, |
| 306 | ], |
| 307 | }, |
| 308 | { |
| 309 | id: "control-open-thread-from-list", |
| 310 | area: "wrong-thread", |
| 311 | kind: "control", |
| 312 | source: "control", |
| 313 | experience: "Tapping a row opens that thread.", |
| 314 | scenario: "openFromList", |
| 315 | key: "opensTarget", |
| 316 | checks: [{ metric: "opensTarget", equals: true }], |
| 317 | }, |
| 318 | { |
| 319 | id: "control-send-reaches-server", |
| 320 | area: "composer", |
| 321 | kind: "control", |
| 322 | source: "control", |
| 323 | experience: "A message sent from the composer reaches the server once.", |
| 324 | scenario: "sendInLongThread", |
| 325 | key: "serverMessages", |
| 326 | checks: [{ metric: "serverMessages", equals: 1 }], |
| 327 | }, |
| 328 | { |
| 329 | id: "control-streaming-reply-renders", |
| 330 | area: "scroll", |
| 331 | kind: "control", |
| 332 | source: "control", |
| 333 | experience: "A streaming reply grows in the open thread.", |
| 334 | scenario: "streamInView", |
| 335 | key: "contentGrewPx", |
| 336 | checks: [{ metric: "contentGrewPx", min: 500 }], |
| 337 | }, |
| 338 | ]; |