The best way to control Claude Code, Codex and more from any computer or phone
| 1 | /** Pure grading for the mobile behavior eval: frame and scroll math, checks, and the summary. */ |
| 2 | import type { FeedSample } from "./device.ts"; |
| 3 | |
| 4 | export type Metrics = Readonly<Record<string, number | boolean | string | null>>; |
| 5 | |
| 6 | export interface Check { |
| 7 | readonly metric: string; |
| 8 | readonly max?: number; |
| 9 | readonly min?: number; |
| 10 | readonly equals?: number | boolean | string | null; |
| 11 | } |
| 12 | |
| 13 | const percentile = (sorted: ReadonlyArray<number>, p: number) => |
| 14 | sorted.length === 0 ? 0 : sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))]!; |
| 15 | |
| 16 | export function median(values: ReadonlyArray<number>): number { |
| 17 | const sorted = [...values].sort((a, b) => a - b); |
| 18 | const middle = Math.floor(sorted.length / 2); |
| 19 | return sorted.length % 2 ? sorted[middle]! : (sorted[middle - 1]! + sorted[middle]!) / 2; |
| 20 | } |
| 21 | |
| 22 | const round = (value: number, digits = 1) => Number(value.toFixed(digits)); |
| 23 | |
| 24 | /** |
| 25 | * Frame times from rAF timestamps, graded against the display's budget: `refreshMs` when given, |
| 26 | * else 120 Hz when the fastest frames come under 12 ms and 60 Hz otherwise. A steady low frame |
| 27 | * rate still counts as dropped frames. `dropped` counts the frames that should have run inside |
| 28 | * each long gap. |
| 29 | */ |
| 30 | export function frameStats( |
| 31 | timestamps: ReadonlyArray<number>, |
| 32 | prefix: string, |
| 33 | refreshMs?: number, |
| 34 | ): Metrics { |
| 35 | const gaps = timestamps.slice(1).map((time, index) => time - timestamps[index]!); |
| 36 | if (gaps.length < 10) return { [`${prefix}Frames`]: gaps.length }; |
| 37 | const sorted = [...gaps].sort((a, b) => a - b); |
| 38 | const refresh = refreshMs ?? (percentile(sorted, 0.05) < 12 ? 1000 / 120 : 1000 / 60); |
| 39 | const dropped = gaps.reduce( |
| 40 | (total, gap) => total + Math.max(0, Math.round(gap / refresh) - 1), |
| 41 | 0, |
| 42 | ); |
| 43 | return { |
| 44 | [`${prefix}Frames`]: gaps.length, |
| 45 | [`${prefix}RefreshHz`]: Math.round(1000 / refresh), |
| 46 | [`${prefix}P50Ms`]: round(percentile(sorted, 0.5)), |
| 47 | [`${prefix}P95Ms`]: round(percentile(sorted, 0.95)), |
| 48 | [`${prefix}P99Ms`]: round(percentile(sorted, 0.99)), |
| 49 | [`${prefix}MaxMs`]: round(sorted.at(-1)!), |
| 50 | [`${prefix}DroppedPct`]: round((100 * dropped) / (gaps.length + dropped)), |
| 51 | }; |
| 52 | } |
| 53 | |
| 54 | /** Fastest real fling step: about 8000 pt/s at 60 Hz. A bigger one-frame move is a jump. */ |
| 55 | export const TELEPORT_PX = 400; |
| 56 | |
| 57 | /** |
| 58 | * How visible rows moved between JS frames. A row's screen y is its list position minus the |
| 59 | * scroll offset, so a list that shifts its offset to hold content still reads as no motion. |
| 60 | * `direction` is the way rows should move: +1 while dragging content down (scrolling up). |
| 61 | */ |
| 62 | export function feedMotion(samples: ReadonlyArray<FeedSample>, direction: 1 | -1 | 0): Metrics { |
| 63 | let backward = 0; |
| 64 | let largest = 0; |
| 65 | let teleports = 0; |
| 66 | let moved = 0; |
| 67 | for (let index = 1; index < samples.length; index++) { |
| 68 | const before = samples[index - 1]!; |
| 69 | const after = samples[index]!; |
| 70 | const shared = Object.keys(before.rows ?? {}).filter((key) => after.rows?.[key] !== undefined); |
| 71 | let step = 0; |
| 72 | for (const key of shared) { |
| 73 | const rowStep = after.rows![key]! - before.rows![key]!; |
| 74 | if (Math.abs(rowStep) > Math.abs(step)) step = rowStep; |
| 75 | } |
| 76 | // A jump can replace every visible row; with the same content, the offset change is the move. |
| 77 | if (shared.length === 0 && before.content === after.content) |
| 78 | step = (before.scroll ?? 0) - (after.scroll ?? 0); |
| 79 | moved += Math.abs(step); |
| 80 | largest = Math.max(largest, Math.abs(step)); |
| 81 | if (Math.abs(step) > TELEPORT_PX) teleports++; |
| 82 | if (direction !== 0) backward = Math.max(backward, -step * direction); |
| 83 | } |
| 84 | const fromEnd = samples.flatMap((sample) => |
| 85 | sample.fromEnd === undefined ? [] : [sample.fromEnd], |
| 86 | ); |
| 87 | return { |
| 88 | samples: samples.length, |
| 89 | movedPx: Math.round(moved), |
| 90 | backwardPx: Math.round(backward), |
| 91 | largestStepPx: Math.round(largest), |
| 92 | teleports, |
| 93 | maxFromEndPx: fromEnd.length ? Math.max(...fromEnd) : null, |
| 94 | finalFromEndPx: fromEnd.at(-1) ?? null, |
| 95 | }; |
| 96 | } |
| 97 | |
| 98 | export function checkFails(check: Check, metrics: Metrics): string | null { |
| 99 | const value = metrics[check.metric]; |
| 100 | if (value === undefined) return `${check.metric} missing`; |
| 101 | if (check.equals !== undefined) |
| 102 | return value === check.equals ? null : `${check.metric}=${value}, want ${check.equals}`; |
| 103 | if (typeof value !== "number") return `${check.metric}=${value}, want a number`; |
| 104 | if (check.max !== undefined && value > check.max) |
| 105 | return `${check.metric}=${value} > ${check.max}`; |
| 106 | if (check.min !== undefined && value < check.min) |
| 107 | return `${check.metric}=${value} < ${check.min}`; |
| 108 | return null; |
| 109 | } |
| 110 | |
| 111 | export interface SampleRow { |
| 112 | readonly id: string; |
| 113 | readonly device: string; |
| 114 | readonly sample: number; |
| 115 | readonly pass: boolean | null; |
| 116 | readonly failures: ReadonlyArray<string>; |
| 117 | readonly metrics: Metrics; |
| 118 | /** Why the sample has no grade: the scenario failed before it could measure. */ |
| 119 | readonly error?: string; |
| 120 | } |
| 121 | |
| 122 | export interface CaseResult { |
| 123 | readonly id: string; |
| 124 | readonly device: string; |
| 125 | readonly samples: number; |
| 126 | readonly passed: number; |
| 127 | readonly errors: number; |
| 128 | /** |
| 129 | * `null` when too few samples were graded: errors outnumber graded samples, or none ran. |
| 130 | * Otherwise a case fails when half or more of its graded samples fail. |
| 131 | */ |
| 132 | readonly pass: boolean | null; |
| 133 | readonly key: number | boolean | string | null; |
| 134 | } |
| 135 | |
| 136 | export function caseResult( |
| 137 | id: string, |
| 138 | device: string, |
| 139 | key: string, |
| 140 | rows: ReadonlyArray<SampleRow>, |
| 141 | ): CaseResult { |
| 142 | const graded = rows.filter((row) => row.pass !== null); |
| 143 | const errors = rows.filter((row) => row.error !== undefined).length; |
| 144 | const passed = graded.filter((row) => row.pass).length; |
| 145 | const values = graded.map((row) => row.metrics[key]); |
| 146 | const numbers = values.filter((value): value is number => typeof value === "number"); |
| 147 | return { |
| 148 | id, |
| 149 | device, |
| 150 | samples: graded.length, |
| 151 | passed, |
| 152 | errors, |
| 153 | pass: graded.length === 0 || errors > graded.length ? null : passed > graded.length / 2, |
| 154 | key: |
| 155 | numbers.length === values.length && numbers.length > 0 |
| 156 | ? round(median(numbers)) |
| 157 | : (values[Math.floor(values.length / 2)] ?? null), |
| 158 | }; |
| 159 | } |
| 160 | |
| 161 | const mark = (result: CaseResult) => |
| 162 | `${result.pass === null ? "incomplete" : result.pass ? "pass" : "FAIL"} ${result.passed}/${result.samples + result.errors}`; |
| 163 | |
| 164 | /** Markdown scoreboard. With a baseline, each row shows the earlier result. */ |
| 165 | export function summarize(input: { |
| 166 | readonly cases: ReadonlyArray<{ |
| 167 | readonly id: string; |
| 168 | readonly kind: string; |
| 169 | readonly source: string; |
| 170 | readonly key: string; |
| 171 | }>; |
| 172 | readonly results: ReadonlyArray<CaseResult>; |
| 173 | readonly baseline?: ReadonlyArray<CaseResult>; |
| 174 | readonly header: string; |
| 175 | }): string { |
| 176 | const devices = [...new Set(input.results.map((result) => result.device))]; |
| 177 | const cell = (result: CaseResult | undefined) => |
| 178 | result ? `${mark(result)} (${result.key ?? "-"})` : "-"; |
| 179 | const lines = [ |
| 180 | input.header, |
| 181 | "", |
| 182 | `| Case | Kind | Source | Key metric | ${devices.join(" | ")} |`, |
| 183 | `| --- | --- | --- | --- | ${devices.map(() => "---").join(" | ")} |`, |
| 184 | ]; |
| 185 | for (const testCase of input.cases) { |
| 186 | const cells = devices.map((device) => { |
| 187 | const now = input.results.find((r) => r.id === testCase.id && r.device === device); |
| 188 | const before = input.baseline?.find((r) => r.id === testCase.id && r.device === device); |
| 189 | return before ? `${cell(now)} was ${cell(before)}` : cell(now); |
| 190 | }); |
| 191 | if (cells.every((value) => value === "-")) continue; |
| 192 | lines.push( |
| 193 | `| \`${testCase.id}\` | ${testCase.kind} | ${testCase.source} | ${testCase.key} | ${cells.join(" | ")} |`, |
| 194 | ); |
| 195 | } |
| 196 | const ran = input.results.filter((result) => result.pass !== null); |
| 197 | const incomplete = input.results.length - ran.length; |
| 198 | lines.push( |
| 199 | "", |
| 200 | `${ran.filter((r) => r.pass).length} of ${ran.length} case runs pass${incomplete ? `, ${incomplete} incomplete` : ""}.`, |
| 201 | ); |
| 202 | return `${lines.join("\n")}\n`; |
| 203 | } |
| 204 | |
| 205 | export function resultsFor( |
| 206 | rows: ReadonlyArray<SampleRow>, |
| 207 | cases: ReadonlyArray<{ readonly id: string; readonly key: string }>, |
| 208 | ): ReadonlyArray<CaseResult> { |
| 209 | const keys = [...new Set(rows.map((row) => `${row.id}\u0000${row.device}`))]; |
| 210 | return keys.map((key) => { |
| 211 | const [id, device] = key.split("\u0000") as [string, string]; |
| 212 | return caseResult( |
| 213 | id, |
| 214 | device, |
| 215 | cases.find((entry) => entry.id === id)?.key ?? "", |
| 216 | rows.filter((row) => row.id === id && row.device === device), |
| 217 | ); |
| 218 | }); |
| 219 | } |
| 220 | |
| 221 | /** |
| 222 | * Results that should stop a release: a guard or control that failed or could not be graded. |
| 223 | * `bad` cases reproduce known bugs and fail until those are fixed. |
| 224 | */ |
| 225 | export function blockingResults( |
| 226 | results: ReadonlyArray<CaseResult>, |
| 227 | cases: ReadonlyArray<{ readonly id: string; readonly kind: string }>, |
| 228 | ): ReadonlyArray<CaseResult> { |
| 229 | return results.filter( |
| 230 | (result) => |
| 231 | result.pass !== true && cases.find((entry) => entry.id === result.id)?.kind !== "bad", |
| 232 | ); |
| 233 | } |