The best way to control Claude Code, Codex and more from any computer or phone
| 1 | // @effect-diagnostics nodeBuiltinImport:off - Host-side video analysis shells out to ffmpeg. |
| 2 | |
| 3 | import * as NodeChildProcess from "node:child_process"; |
| 4 | import * as NodeFSP from "node:fs/promises"; |
| 5 | |
| 6 | import { check, type Check, type Rect } from "./checks.ts"; |
| 7 | import { run } from "./device.ts"; |
| 8 | |
| 9 | /** Pixel width of an image or video, to map snapshot coordinates onto its frames. */ |
| 10 | async function pixelWidth(path: string): Promise<number> { |
| 11 | const { stdout } = await run("ffprobe", [ |
| 12 | "-v", |
| 13 | "error", |
| 14 | "-select_streams", |
| 15 | "v:0", |
| 16 | "-show_entries", |
| 17 | "stream=width", |
| 18 | "-of", |
| 19 | "csv=p=0", |
| 20 | path, |
| 21 | ]); |
| 22 | return Number(stdout.trim()); |
| 23 | } |
| 24 | |
| 25 | /** Downsampled grayscale crops of one region, one per video frame (or one for a PNG). */ |
| 26 | async function grayFrames(input: string, region: Rect, scale: number): Promise<Uint8Array[]> { |
| 27 | const even = (value: number) => Math.max(2, Math.floor(value / 2) * 2); |
| 28 | const [x, y, width, height] = [ |
| 29 | Math.round(region.x * scale), |
| 30 | Math.round(region.y * scale), |
| 31 | even(region.width * scale), |
| 32 | even(region.height * scale), |
| 33 | ]; |
| 34 | const [outWidth, outHeight] = [even(region.width / 3), even(region.height / 3)]; |
| 35 | const bytes = await new Promise<Buffer>((resolve, reject) => { |
| 36 | const child = NodeChildProcess.spawn( |
| 37 | "ffmpeg", |
| 38 | [ |
| 39 | "-v", |
| 40 | "error", |
| 41 | "-i", |
| 42 | input, |
| 43 | // One output frame per recorded frame, so indices match select=eq(n,…) for evidence. |
| 44 | "-fps_mode", |
| 45 | "passthrough", |
| 46 | "-vf", |
| 47 | `crop=${width}:${height}:${x}:${y},scale=${outWidth}:${outHeight},format=gray`, |
| 48 | "-f", |
| 49 | "rawvideo", |
| 50 | "-", |
| 51 | ], |
| 52 | { stdio: ["ignore", "pipe", "pipe"] }, |
| 53 | ); |
| 54 | const chunks: Buffer[] = []; |
| 55 | let stderr = ""; |
| 56 | child.stdout.on("data", (chunk: Buffer) => chunks.push(chunk)); |
| 57 | child.stderr.on("data", (chunk) => (stderr += chunk)); |
| 58 | child.on("error", reject); |
| 59 | child.on("close", (code) => |
| 60 | code === 0 ? resolve(Buffer.concat(chunks)) : reject(new Error(`ffmpeg failed: ${stderr}`)), |
| 61 | ); |
| 62 | }); |
| 63 | const size = outWidth * outHeight; |
| 64 | return Array.from({ length: Math.floor(bytes.length / size) }, (_, index) => |
| 65 | bytes.subarray(index * size, (index + 1) * size), |
| 66 | ); |
| 67 | } |
| 68 | |
| 69 | const distance = (a: Uint8Array, b: Uint8Array) => { |
| 70 | let sum = 0; |
| 71 | for (let index = 0; index < a.length; index++) sum += Math.abs(a[index]! - b[index]!); |
| 72 | return sum / a.length; |
| 73 | }; |
| 74 | |
| 75 | /** How far a region has moved from its origin toward its destination look, 0 to 1. */ |
| 76 | export interface FrameProgress { |
| 77 | readonly nav: number; |
| 78 | readonly chrome: number; |
| 79 | } |
| 80 | |
| 81 | /** |
| 82 | * A frame lags when the top of the screen is well into the transition while the tab bar |
| 83 | * region still looks like the origin: tabs lingering over an entering thread, or a returned |
| 84 | * list drawn before its tabs. |
| 85 | */ |
| 86 | const chromeLags = (frame: FrameProgress) => frame.nav > 0.3 && frame.chrome < 0.4; |
| 87 | // One lagging frame is capture jitter; a regression lasts several. |
| 88 | const LAG_FRAME_LIMIT = 2; |
| 89 | |
| 90 | const progress = (origin: number, destination: number) => |
| 91 | origin + destination === 0 ? 0 : origin / (origin + destination); |
| 92 | |
| 93 | /** |
| 94 | * Checks that the tab bar moves with the screen during a recorded transition. The first and |
| 95 | * last frames are the before and after looks, so nothing is compared across capture sources. |
| 96 | */ |
| 97 | export async function lagCheck( |
| 98 | name: string, |
| 99 | input: { |
| 100 | readonly video: string; |
| 101 | readonly nav: Rect; |
| 102 | readonly chrome: Rect; |
| 103 | /** Snapshot units across the screen, to scale regions onto the video's pixels. */ |
| 104 | readonly viewportWidth: number; |
| 105 | }, |
| 106 | evidence: string, |
| 107 | ): Promise<{ check: Check; frame?: string }> { |
| 108 | // A recording the device failed to export is a gap in evidence, not a lag. |
| 109 | if (!(await NodeFSP.stat(input.video).catch(() => null))) |
| 110 | return { check: check(name, true, "no recording; not checked") }; |
| 111 | const scale = (await pixelWidth(input.video)) / input.viewportWidth; |
| 112 | const [navFrames, chromeFrames] = await Promise.all([ |
| 113 | grayFrames(input.video, input.nav, scale), |
| 114 | grayFrames(input.video, input.chrome, scale), |
| 115 | ]); |
| 116 | const [navOrigin, navDestination] = [navFrames[0], navFrames.at(-1)]; |
| 117 | const [chromeOrigin, chromeDestination] = [chromeFrames[0], chromeFrames.at(-1)]; |
| 118 | if (!navOrigin || !navDestination || !chromeOrigin || !chromeDestination) |
| 119 | throw new Error(`Could not read frames of ${input.video}.`); |
| 120 | const lagging = navFrames.flatMap((nav, index) => { |
| 121 | const frame = { |
| 122 | nav: progress(distance(nav, navOrigin), distance(nav, navDestination)), |
| 123 | chrome: progress( |
| 124 | distance(chromeFrames[index]!, chromeOrigin), |
| 125 | distance(chromeFrames[index]!, chromeDestination), |
| 126 | ), |
| 127 | }; |
| 128 | return chromeLags(frame) ? [index] : []; |
| 129 | }); |
| 130 | if (lagging.length < LAG_FRAME_LIMIT) |
| 131 | return { check: check(name, true, `${lagging.length} lagging video frames`) }; |
| 132 | await run("ffmpeg", [ |
| 133 | "-v", |
| 134 | "error", |
| 135 | "-y", |
| 136 | "-i", |
| 137 | input.video, |
| 138 | "-vf", |
| 139 | `select=eq(n\\,${lagging[0]})`, |
| 140 | "-frames:v", |
| 141 | "1", |
| 142 | evidence, |
| 143 | ]); |
| 144 | return { |
| 145 | check: check(name, false, `${lagging.length} video frames (first #${lagging[0]})`), |
| 146 | frame: evidence, |
| 147 | }; |
| 148 | } |