The best way to control Claude Code, Codex and more from any computer or phone
| 1 | // @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off - Playwright callbacks run outside the Effect runtime. |
| 2 | |
| 3 | import { |
| 4 | type BrowserRecordingFrameRate, |
| 5 | PROVIDER_SEND_TURN_MAX_FILE_BYTES, |
| 6 | type PreviewAutomationClickInput, |
| 7 | type PreviewAutomationConsoleEntry, |
| 8 | type PreviewAutomationDragInput, |
| 9 | type PreviewAutomationEvaluateInput, |
| 10 | type PreviewAutomationHoverInput, |
| 11 | type PreviewAutomationNetworkEntry, |
| 12 | type PreviewAutomationPressInput, |
| 13 | type PreviewAutomationScrollInput, |
| 14 | type PreviewAutomationSelectInput, |
| 15 | type PreviewAutomationSelectResult, |
| 16 | type PreviewAutomationSnapshot, |
| 17 | type PreviewAutomationSnapshotMetadata, |
| 18 | type PreviewAutomationSnapshotStage, |
| 19 | type PreviewAutomationTypeInput, |
| 20 | type PreviewAutomationUploadInput, |
| 21 | type PreviewAutomationWaitForInput, |
| 22 | } from "@t3tools/contracts"; |
| 23 | import { constVoid } from "effect/Function"; |
| 24 | import * as Schema from "effect/Schema"; |
| 25 | import type { CDPSession, ConsoleMessage, Frame, Locator, Page } from "playwright-core"; |
| 26 | import * as NodeCrypto from "node:crypto"; |
| 27 | import * as NodeUtil from "node:util"; |
| 28 | import { BrowserControlInterrupted } from "./SessionControl.ts"; |
| 29 | |
| 30 | const isBrowserControlInterrupted = Schema.is(BrowserControlInterrupted); |
| 31 | |
| 32 | const MAX_EVALUATION_BYTES = 64_000; |
| 33 | const MAX_VISIBLE_TEXT_LENGTH = 20_000; |
| 34 | const MAX_SCREENSHOT_WIDTH = 1280; |
| 35 | // Two attempts must fit in a snapshot's 10-second deadline. |
| 36 | const SCREENSHOT_TIMEOUT_MS = 3_000; |
| 37 | const WAIT_POLL_MS = 100; |
| 38 | export const DIAGNOSTIC_BUFFER_LIMIT = 200; |
| 39 | |
| 40 | export class ServerBrowserOperationError extends Schema.TaggedError<ServerBrowserOperationError>()( |
| 41 | "ServerBrowserOperationError", |
| 42 | { |
| 43 | tag: Schema.String, |
| 44 | message: Schema.String, |
| 45 | detail: Schema.optional(Schema.Unknown), |
| 46 | cause: Schema.optional(Schema.Defect()), |
| 47 | }, |
| 48 | ) {} |
| 49 | |
| 50 | const isServerBrowserOperationError = Schema.is(ServerBrowserOperationError); |
| 51 | |
| 52 | export const toOperationError = ( |
| 53 | cause: unknown, |
| 54 | timeoutMs?: number, |
| 55 | ): ServerBrowserOperationError => { |
| 56 | if (isBrowserControlInterrupted(cause)) |
| 57 | return new ServerBrowserOperationError({ |
| 58 | tag: "PreviewAutomationControlInterruptedError", |
| 59 | message: cause.message, |
| 60 | detail: cause.reason, |
| 61 | }); |
| 62 | if (isServerBrowserOperationError(cause)) return cause; |
| 63 | const message = cause instanceof Error ? cause.message : String(cause); |
| 64 | const firstLine = message.split("\n")[0] ?? message; |
| 65 | if (cause instanceof Error && cause.name === "TimeoutError") { |
| 66 | return new ServerBrowserOperationError({ |
| 67 | tag: "PreviewAutomationTimeoutError", |
| 68 | message: withActionabilityReason(firstLine, cause), |
| 69 | detail: timeoutMs === undefined ? undefined : { timeoutMs }, |
| 70 | }); |
| 71 | } |
| 72 | if (/while parsing selector|Unknown engine|Unexpected token/i.test(message)) { |
| 73 | return new ServerBrowserOperationError({ |
| 74 | tag: "PreviewAutomationInvalidSelectorError", |
| 75 | message: firstLine, |
| 76 | }); |
| 77 | } |
| 78 | if (/not an <input>|not editable|not an editable/i.test(message)) { |
| 79 | return new ServerBrowserOperationError({ |
| 80 | tag: "PreviewAutomationTargetNotEditableError", |
| 81 | message: firstLine, |
| 82 | }); |
| 83 | } |
| 84 | return new ServerBrowserOperationError({ |
| 85 | tag: "PreviewAutomationExecutionError", |
| 86 | message: firstLine, |
| 87 | }); |
| 88 | }; |
| 89 | |
| 90 | /** Playwright puts why an element was not actionable in its call log, below the first line. */ |
| 91 | const withActionabilityReason = (message: string, cause: Error) => { |
| 92 | // Playwright dims call-log lines with ANSI escapes when colour is on. |
| 93 | const reason = NodeUtil.stripVTControlCharacters(cause.message) |
| 94 | .split("\n") |
| 95 | .map((line) => line.trim()) |
| 96 | .findLast((line) => line.startsWith("- element is ")) |
| 97 | ?.slice(2); |
| 98 | return reason === undefined ? message : `${message} Playwright last saw: ${reason}.`; |
| 99 | }; |
| 100 | |
| 101 | const DEFAULT_TIMEOUT_MS = 15_000; |
| 102 | const TARGET_COUNT_MS = 250; |
| 103 | const MAX_TARGET_CHARS = 200; |
| 104 | |
| 105 | const pageRefs = new WeakMap<Page, { generation: string; refs: Map<string, string> }>(); |
| 106 | // A compact runtime namespace prevents old refs from aliasing after a server restart. |
| 107 | const refNamespace = NodeCrypto.randomUUID().slice(0, 8); |
| 108 | let snapshotSequence = 0; |
| 109 | const nextRefGeneration = () => `${refNamespace}-${(++snapshotSequence).toString(36)}`; |
| 110 | |
| 111 | /** A takeover revokes previously issued refs even when the document stays unchanged. */ |
| 112 | export const invalidateRefs = (page: Page) => { |
| 113 | const state = pageRefs.get(page); |
| 114 | if (state) { |
| 115 | state.generation = nextRefGeneration(); |
| 116 | state.refs.clear(); |
| 117 | } |
| 118 | }; |
| 119 | |
| 120 | const refsFor = (page: Page) => { |
| 121 | let state = pageRefs.get(page); |
| 122 | if (!state) { |
| 123 | state = { generation: nextRefGeneration(), refs: new Map<string, string>() }; |
| 124 | pageRefs.set(page, state); |
| 125 | page.on("framenavigated", () => invalidateRefs(page)); |
| 126 | page.on("framedetached", () => invalidateRefs(page)); |
| 127 | } |
| 128 | return state; |
| 129 | }; |
| 130 | |
| 131 | const targetLocator = ( |
| 132 | page: Page, |
| 133 | input: { readonly locator?: string | undefined; readonly selector?: string | undefined }, |
| 134 | ): Locator | null => { |
| 135 | const selector = input.locator ?? input.selector; |
| 136 | if (selector === undefined) return null; |
| 137 | const engines = selector.replace( |
| 138 | /\\.|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`/g, |
| 139 | " ", |
| 140 | ); |
| 141 | if (/(?:^|>>)\s*\*?aria-ref\s*=/.test(engines)) { |
| 142 | const ref = /^\s*aria-ref\s*=\s*(\S+)\s*$/.exec(selector)?.[1]; |
| 143 | const nativeRef = ref === undefined ? undefined : pageRefs.get(page)?.refs.get(ref); |
| 144 | if (nativeRef === undefined) { |
| 145 | throw new ServerBrowserOperationError({ |
| 146 | tag: "PreviewAutomationInvalidSelectorError", |
| 147 | message: |
| 148 | "This element ref is stale or belongs to another tab. Take a fresh snapshot and use its locator.", |
| 149 | detail: { staleRef: true }, |
| 150 | }); |
| 151 | } |
| 152 | return page.locator(`aria-ref=${nativeRef}`); |
| 153 | } |
| 154 | // Playwright's strict locators reject ambiguous controls rather than acting on row one. |
| 155 | return page.locator(selector); |
| 156 | }; |
| 157 | |
| 158 | export type PointerReporter = ( |
| 159 | point: { readonly x: number; readonly y: number }, |
| 160 | phase: "move" | "click", |
| 161 | ) => Promise<void>; |
| 162 | const noPointer: PointerReporter = async () => {}; |
| 163 | |
| 164 | // Locator.scrollIntoViewIfNeeded resolves its node once and fails when the page replaces it; the |
| 165 | // input actions after it re-resolve, so the scroll does too, within the same timeout. |
| 166 | const DETACHED = /^locator\.scrollIntoViewIfNeeded: Element is not attached to the DOM(?:\n|$)/; |
| 167 | |
| 168 | const scrollIntoView = async (locator: Locator, timeout: number) => { |
| 169 | const deadline = Date.now() + timeout; |
| 170 | for (;;) { |
| 171 | try { |
| 172 | return await locator.scrollIntoViewIfNeeded({ timeout: Math.max(1, deadline - Date.now()) }); |
| 173 | } catch (cause) { |
| 174 | if (!(cause instanceof Error && DETACHED.test(cause.message)) || Date.now() >= deadline) { |
| 175 | throw cause; |
| 176 | } |
| 177 | } |
| 178 | } |
| 179 | }; |
| 180 | |
| 181 | const targetPoint = async ( |
| 182 | page: Page, |
| 183 | locator: Locator | null, |
| 184 | input: { readonly x?: number | undefined; readonly y?: number | undefined }, |
| 185 | timeout: () => number, |
| 186 | ) => { |
| 187 | if (locator === null) return { x: input.x ?? 0, y: input.y ?? 0 }; |
| 188 | await scrollIntoView(locator, timeout()); |
| 189 | const box = await locator.boundingBox({ timeout: timeout() }); |
| 190 | if (box) return { x: box.x + box.width / 2, y: box.y + box.height / 2 }; |
| 191 | const viewport = page.viewportSize(); |
| 192 | return { x: (viewport?.width ?? 0) / 2, y: (viewport?.height ?? 0) / 2 }; |
| 193 | }; |
| 194 | |
| 195 | const SNAPSHOT_SCRIPT = `(() => { |
| 196 | return { |
| 197 | url: location.href, |
| 198 | title: document.title, |
| 199 | loading: document.readyState !== "complete", |
| 200 | visibleText: document.body?.innerText || "", |
| 201 | interactiveElements: [], |
| 202 | }; |
| 203 | })()`; |
| 204 | |
| 205 | const MASKED_VALUE = "••••••••"; |
| 206 | |
| 207 | /** |
| 208 | * A `data:` URL carries its content, which can be a whole file, and URLs and diagnostics |
| 209 | * reach tool results and provider logs. Keeps only the media type: the payload can hold any |
| 210 | * character and parameters are page-chosen too. |
| 211 | */ |
| 212 | export const withoutDataPayload = (url: string) => { |
| 213 | // Read the scheme as a browser would: it ignores leading controls and spaces, and tabs and |
| 214 | // newlines anywhere. A snapshot's YAML value may be quoted and escaped; an escape that isn't a |
| 215 | // tab or newline still decodes to something that can't start a scheme. |
| 216 | const value = url |
| 217 | .replace(/^["']/, "") |
| 218 | .replace(/\\(x[\da-f]{2}|u[\da-f]{4}|.)/gi, (_escape, code: string) => |
| 219 | code.length > 1 |
| 220 | ? String.fromCharCode(Number.parseInt(code.slice(1), 16)) |
| 221 | : ({ t: "\t", n: "\n", r: "\r", '"': '"', "\\": "\\" }[code] ?? "\u0001"), |
| 222 | ) |
| 223 | .replace(/[\t\n\r]/g, "") |
| 224 | .replace(/^[\u0000-\u0020]+/, ""); |
| 225 | const parsed = /^data:([^,;]*)/i.exec(value); |
| 226 | if (!parsed) return url; |
| 227 | const type = parsed[1]!.trim(); |
| 228 | const mediaType = /^[\w.+-]{1,64}\/[\w.+-]{1,64}$/.test(type) ? type : ""; |
| 229 | return `data:${mediaType},…`; |
| 230 | }; |
| 231 | |
| 232 | /** |
| 233 | * Chromium's own console messages, which carry no arguments, quote `data:` URLs whole, as in a |
| 234 | * blocked navigation or a CSP violation. Everything after the first one goes, since Chromium |
| 235 | * accepts any type and a payload can hold any character. What a page logs itself is its own |
| 236 | * text, like anything else it shows, so it stays. |
| 237 | */ |
| 238 | export const consoleText = (message: ConsoleMessage) => |
| 239 | message.args().length > 0 |
| 240 | ? message.text() |
| 241 | : message.text().replace(/(?<![a-z\d])data:[^,;]*[,;][\s\S]*/i, withoutDataPayload); |
| 242 | |
| 243 | // Split on LF alone: a value keeps any U+2028 that a multiline regex would treat as a line end. |
| 244 | const withoutTreeDataPayloads = (tree: string) => |
| 245 | tree |
| 246 | .split("\n") |
| 247 | .map((line) => { |
| 248 | const url = /^(\s*- \/url: )(.*)$/s.exec(line); |
| 249 | return url ? url[1] + withoutDataPayload(url[2]!) : line; |
| 250 | }) |
| 251 | .join("\n"); |
| 252 | |
| 253 | // The broker waits 15s for the host; a snapshot, retry included, answers well before. |
| 254 | const SNAPSHOT_TIMEOUT_MS = 10_000; |
| 255 | const FRAME_ANSWER_MS = 1_000; |
| 256 | |
| 257 | // Include password and CSS-masked text in open shadow roots, normalized like accessible text. |
| 258 | const DOTTED_SCRIPT = `(() => { |
| 259 | const values = []; |
| 260 | const visit = (root) => { |
| 261 | for (const element of root.querySelectorAll("*")) { |
| 262 | if (element.shadowRoot) visit(element.shadowRoot); |
| 263 | const field = element.localName === "input" || element.localName === "textarea"; |
| 264 | if (!field && !element.isContentEditable) continue; |
| 265 | const dots = getComputedStyle(element).getPropertyValue("-webkit-text-security"); |
| 266 | if (element.type !== "password" && (dots === "none" || dots === "")) continue; |
| 267 | // Raw as page text shows it, and normalized as the tree does. |
| 268 | for (const value of field ? [element.value, element.defaultValue] : [element.textContent ?? ""]) |
| 269 | values.push(value, value.replace(/[\\u200b\\u00ad]/g, "").trim().replace(/\\s+/g, " ")); |
| 270 | } |
| 271 | }; |
| 272 | visit(document); |
| 273 | return [...new Set(values.filter((value) => value.trim()))]; |
| 274 | })()`; |
| 275 | |
| 276 | const pageChanged = () => |
| 277 | new ServerBrowserOperationError({ |
| 278 | tag: "PreviewAutomationExecutionError", |
| 279 | message: "The page changed while capturing its snapshot. Take another snapshot.", |
| 280 | }); |
| 281 | |
| 282 | type SnapshotStage = typeof PreviewAutomationSnapshotStage.Type; |
| 283 | |
| 284 | /** |
| 285 | * Running stages are counted, since a retry can overlap the previous attempt's unanswered |
| 286 | * screenshot. A stage whose own timeout fired has stopped running but is still the cause. |
| 287 | */ |
| 288 | interface SnapshotStages { |
| 289 | readonly running: Map<SnapshotStage, number>; |
| 290 | readonly timedOut: Set<SnapshotStage>; |
| 291 | } |
| 292 | |
| 293 | const snapshotTimedOut = (stages?: SnapshotStages) => |
| 294 | new ServerBrowserOperationError({ |
| 295 | tag: "PreviewAutomationTimeoutError", |
| 296 | message: `Snapshot capture timed out after ${SNAPSHOT_TIMEOUT_MS}ms. Try the snapshot again.`, |
| 297 | detail: { |
| 298 | timeoutMs: SNAPSHOT_TIMEOUT_MS, |
| 299 | pendingStages: stages ? [...new Set([...stages.timedOut, ...stages.running.keys()])] : [], |
| 300 | }, |
| 301 | }); |
| 302 | |
| 303 | const frameUnanswered = () => |
| 304 | new ServerBrowserOperationError({ |
| 305 | tag: "PreviewAutomationTimeoutError", |
| 306 | message: `A frame on the page didn't answer within ${FRAME_ANSWER_MS}ms, so its content couldn't be checked for hidden values. Try the snapshot again.`, |
| 307 | detail: { timeoutMs: FRAME_ANSWER_MS, pendingStages: ["frames"] }, |
| 308 | }); |
| 309 | |
| 310 | const isTimeout = (cause: unknown) => cause instanceof Error && cause.name === "TimeoutError"; |
| 311 | |
| 312 | const VALUE_TOO_LONG = "A password field holds a value too long to hide, so no snapshot was taken."; |
| 313 | |
| 314 | /** Reuse one CDP session per out-of-process frame across snapshots. */ |
| 315 | const frameSessions = new WeakMap<Frame, Promise<CDPSession>>(); |
| 316 | |
| 317 | /** Returns undefined on failure or frame timeout. */ |
| 318 | const answered = <A>(promise: Promise<A>) => { |
| 319 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 320 | const late = new Promise<undefined>((resolve) => { |
| 321 | timer = setTimeout(() => resolve(undefined), FRAME_ANSWER_MS); |
| 322 | }); |
| 323 | return Promise.race([promise.catch(() => undefined), late]).finally(() => clearTimeout(timer)); |
| 324 | }; |
| 325 | |
| 326 | /** Isolated worlds prevent spoofing; desktop out-of-process frames use page worlds. Missing own-frame reads or changed frames fail; other missing reads return undefined. */ |
| 327 | const openReaders = async (page: Page, cdp: CDPSession) => { |
| 328 | const frames = page.frames(); |
| 329 | const owners = new Map<Frame, CDPSession | null | undefined>( |
| 330 | await Promise.all( |
| 331 | frames.map(async (frame) => { |
| 332 | if (frame === page.mainFrame()) return [frame, cdp] as const; |
| 333 | const session = frameSessions.get(frame) ?? page.context().newCDPSession(frame); |
| 334 | frameSessions.set(frame, session); |
| 335 | try { |
| 336 | return [frame, await session] as const; |
| 337 | } catch (cause) { |
| 338 | frameSessions.delete(frame); |
| 339 | // CdpRelay supports only the page; other refusals fall back to the parent session, failing if the frame is absent. |
| 340 | const desktop = String(cause).includes("Only this preview tab's page can be attached"); |
| 341 | return [frame, desktop ? null : undefined] as const; |
| 342 | } |
| 343 | }), |
| 344 | ), |
| 345 | ); |
| 346 | const ownerOf = (frame: Frame): CDPSession | null => { |
| 347 | const owner = owners.get(frame); |
| 348 | if (owner !== undefined) return owner; |
| 349 | const parent = frame.parentFrame(); |
| 350 | return parent ? ownerOf(parent) : null; |
| 351 | }; |
| 352 | const sessionReader = (frame: Frame, session: CDPSession) => { |
| 353 | const owned = frames.filter((each) => ownerOf(each) === session); |
| 354 | const open = async () => { |
| 355 | const { frameTree } = await session.send("Page.getFrameTree"); |
| 356 | const ids = (node: typeof frameTree): Array<string> => [ |
| 357 | node.frame.id, |
| 358 | ...(node.childFrames ?? []).flatMap(ids), |
| 359 | ]; |
| 360 | // A frame Playwright knows but this session doesn't would go unread. |
| 361 | if (ids(frameTree).length !== owned.length) throw pageChanged(); |
| 362 | return Promise.all( |
| 363 | ids(frameTree).map(async (frameId) => { |
| 364 | const world = await session.send("Page.createIsolatedWorld", { |
| 365 | frameId, |
| 366 | worldName: "t3-preview", |
| 367 | }); |
| 368 | return world.executionContextId; |
| 369 | }), |
| 370 | ); |
| 371 | }; |
| 372 | let contexts: ReturnType<typeof open> | undefined; |
| 373 | const read = async () => { |
| 374 | const values = (contexts ??= open()).then((ids) => |
| 375 | Promise.all( |
| 376 | ids.map(async (contextId) => { |
| 377 | const { result, exceptionDetails } = await session.send("Runtime.evaluate", { |
| 378 | contextId, |
| 379 | expression: DOTTED_SCRIPT, |
| 380 | returnByValue: true, |
| 381 | }); |
| 382 | if (exceptionDetails) throw pageChanged(); |
| 383 | return result.value as Array<string>; |
| 384 | }), |
| 385 | ), |
| 386 | ); |
| 387 | try { |
| 388 | return (await values).flat(); |
| 389 | } catch (cause) { |
| 390 | // The frame may have moved to another process; the next snapshot opens a new session. |
| 391 | if (session !== cdp) { |
| 392 | frameSessions.delete(frame); |
| 393 | void session.detach().catch(constVoid); |
| 394 | } |
| 395 | throw cause; |
| 396 | } |
| 397 | }; |
| 398 | return { frames: owned, read }; |
| 399 | }; |
| 400 | const readers = [ |
| 401 | sessionReader(page.mainFrame(), cdp), |
| 402 | ...frames.flatMap((frame) => { |
| 403 | const owner = owners.get(frame); |
| 404 | if (owner) return owner === cdp ? [] : [sessionReader(frame, owner)]; |
| 405 | const own = () => frame.evaluate(DOTTED_SCRIPT) as Promise<Array<string>>; |
| 406 | return ownerOf(frame) === null ? [{ frames: [frame], read: own }] : []; |
| 407 | }), |
| 408 | ]; |
| 409 | return { |
| 410 | readers, |
| 411 | read: async (before?: ReadonlyArray<unknown>) => { |
| 412 | const now = page.frames(); |
| 413 | if (now.length !== frames.length || now.some((frame, i) => frame !== frames[i])) { |
| 414 | throw pageChanged(); |
| 415 | } |
| 416 | return Promise.all( |
| 417 | readers.map((reader, i) => { |
| 418 | if (i === 0) return reader.read().catch(() => Promise.reject(pageChanged())); |
| 419 | return before && !before[i] ? undefined : answered(reader.read()); |
| 420 | }), |
| 421 | ); |
| 422 | }, |
| 423 | }; |
| 424 | }; |
| 425 | |
| 426 | /** Short values need word boundaries to avoid masking unrelated text. */ |
| 427 | const dottedPattern = (values: ReadonlyArray<string>) => { |
| 428 | const sources = values |
| 429 | .flatMap((value) => { |
| 430 | const json = JSON.stringify(value).slice(1, -1); |
| 431 | // Playwright writes control characters like JSON does, with \xNN for the unnamed ones. |
| 432 | const quoted = value.replace(/[\\"\p{Cc}]/gu, (char) => |
| 433 | '\\"\b\f\n\r\t'.includes(char) |
| 434 | ? JSON.stringify(char).slice(1, -1) |
| 435 | : `\\x${char.charCodeAt(0).toString(16).padStart(2, "0")}`, |
| 436 | ); |
| 437 | const forms = [value, json, quoted].flatMap((form) => [form, form.replaceAll("'", "''")]); |
| 438 | return [...new Set(forms)].map((form) => ({ form, word: value.length < 4 })); |
| 439 | }) |
| 440 | // Longest first, so a form inside another never leaves the rest showing. |
| 441 | .toSorted((a, b) => b.form.length - a.form.length) |
| 442 | .map(({ form, word }) => { |
| 443 | const literal = form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); |
| 444 | return word ? `(?<![\\p{L}\\p{N}_])${literal}(?![\\p{L}\\p{N}_])` : literal; |
| 445 | }); |
| 446 | if (sources.length === 0) return undefined; |
| 447 | const pattern = new RegExp(sources.join("|"), "gu"); |
| 448 | try { |
| 449 | // Force regex compilation for one-byte and two-byte strings now; size errors name every value. |
| 450 | pattern.test(""); |
| 451 | pattern.test("\u2022"); |
| 452 | } catch { |
| 453 | throw new ServerBrowserOperationError({ |
| 454 | tag: "PreviewAutomationExecutionError", |
| 455 | message: VALUE_TOO_LONG, |
| 456 | }); |
| 457 | } |
| 458 | return pattern; |
| 459 | }; |
| 460 | |
| 461 | // Playwright single-quotes keys containing ': ' and leaves /.../ names unquoted. Recognize only its attributes so name text cannot spoof them. |
| 462 | const TREE_LINE = |
| 463 | /^(\s*- '?[^\s"':]+)( "(?:[^"\\]|\\.)*"| \/(?:.*?\/)?)?((?: \[(?:active|checked(?:=mixed)?|disabled|expanded|level=\d+|pressed(?:=mixed)?|selected|ref=(?:f\d+)?e\d+|cursor=pointer|box=-?\d+,-?\d+,\d+,\d+)\])*'?)(:.*)?$/; |
| 464 | |
| 465 | /** Preserve roles and attributes so refs work; unquoted /.../ names can make matches cross boundaries. */ |
| 466 | const maskTree = (tree: string, dotted: RegExp) => |
| 467 | tree |
| 468 | .split("\n") |
| 469 | .map((line) => { |
| 470 | const node = TREE_LINE.exec(line); |
| 471 | const name = node?.[1]?.length ?? 0; |
| 472 | const attributes = name + (node?.[2]?.length ?? 0); |
| 473 | const end = attributes + (node?.[3]?.length ?? 0); |
| 474 | const kept = (from: number, to: number) => to <= name || (from >= attributes && to <= end); |
| 475 | return line.replace(dotted, (match: string, offset: number) => |
| 476 | kept(offset, offset + match.length) ? match : MASKED_VALUE, |
| 477 | ); |
| 478 | }) |
| 479 | .join("\n"); |
| 480 | |
| 481 | /** Chromium sizes the capture, so the reported size is read from the PNG header. */ |
| 482 | const pngSize = (base64: string) => { |
| 483 | const header = Buffer.from(base64.slice(0, 32), "base64"); |
| 484 | return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) }; |
| 485 | }; |
| 486 | |
| 487 | export const jpegSize = (base64: string) => { |
| 488 | const bytes = Buffer.from(base64, "base64"); |
| 489 | if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8) |
| 490 | throw new Error("Invalid JPEG screenshot."); |
| 491 | let offset = 2; |
| 492 | while (offset < bytes.length) { |
| 493 | if (bytes[offset++] !== 0xff) throw new Error("Invalid JPEG screenshot."); |
| 494 | while (bytes[offset] === 0xff) offset++; |
| 495 | const marker = bytes[offset++]; |
| 496 | if (marker === undefined || marker === 0xd9 || marker === 0xda) break; |
| 497 | if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue; |
| 498 | if (offset + 2 > bytes.length) break; |
| 499 | const length = bytes.readUInt16BE(offset); |
| 500 | if (length < 2 || offset + length > bytes.length) break; |
| 501 | if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) { |
| 502 | if (length < 7) break; |
| 503 | const height = bytes.readUInt16BE(offset + 3); |
| 504 | const width = bytes.readUInt16BE(offset + 5); |
| 505 | if (width && height) return { width, height }; |
| 506 | break; |
| 507 | } |
| 508 | offset += length; |
| 509 | } |
| 510 | throw new Error("Invalid JPEG screenshot."); |
| 511 | }; |
| 512 | |
| 513 | const scaledSize = (size: { readonly width: number; readonly height: number }, scale: number) => |
| 514 | scale < 1 |
| 515 | ? { width: Math.round(size.width * scale), height: Math.round(size.height * scale) } |
| 516 | : size; |
| 517 | |
| 518 | /** |
| 519 | * A page's display ratio and CSS viewport, read in an isolated world since page script can |
| 520 | * reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom. |
| 521 | */ |
| 522 | const readPageMetrics = async (cdp: CDPSession) => { |
| 523 | const { frameTree } = await cdp.send("Page.getFrameTree"); |
| 524 | const { executionContextId } = await cdp.send("Page.createIsolatedWorld", { |
| 525 | frameId: frameTree.frame.id, |
| 526 | worldName: "t3-preview-metrics", |
| 527 | }); |
| 528 | let { cssVisualViewport } = await cdp.send("Page.getLayoutMetrics"); |
| 529 | // Desktop zoom applies asynchronously; bracket the CSS metrics to reject a torn read. |
| 530 | for (let attempt = 0; attempt < 3; attempt += 1) { |
| 531 | const zoom = cssVisualViewport.zoom ?? 1; |
| 532 | const { result } = await cdp.send("Runtime.evaluate", { |
| 533 | expression: "({ ratio: devicePixelRatio, width: innerWidth, height: innerHeight })", |
| 534 | contextId: executionContextId, |
| 535 | returnByValue: true, |
| 536 | }); |
| 537 | ({ cssVisualViewport } = await cdp.send("Page.getLayoutMetrics")); |
| 538 | if ((cssVisualViewport.zoom ?? 1) === zoom) { |
| 539 | const { ratio, width, height } = result.value as { |
| 540 | readonly ratio: number; |
| 541 | readonly width: number; |
| 542 | readonly height: number; |
| 543 | }; |
| 544 | return { ratio, width, height, zoom }; |
| 545 | } |
| 546 | } |
| 547 | throw new Error( |
| 548 | "The page's zoom kept changing while the snapshot read its size. Try the snapshot again.", |
| 549 | ); |
| 550 | }; |
| 551 | |
| 552 | // Never pass a clip scale: Chromium mirrors the top-left of the page into a `backdrop-filter` |
| 553 | // in any capture it shrinks. |
| 554 | export const captureViewport = async ( |
| 555 | cdp: CDPSession, |
| 556 | options: { readonly format: "png" | "jpeg"; readonly quality?: number }, |
| 557 | ) => { |
| 558 | const capture = cdp.send("Page.captureScreenshot", { |
| 559 | format: options.format, |
| 560 | ...(options.quality === undefined ? {} : { quality: options.quality }), |
| 561 | }); |
| 562 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 563 | const expired = new Promise<never>((_, reject) => { |
| 564 | timer = setTimeout( |
| 565 | () => |
| 566 | reject( |
| 567 | new ServerBrowserOperationError({ |
| 568 | tag: "PreviewAutomationExecutionError", |
| 569 | message: |
| 570 | "The browser is not producing screenshot frames. For a desktop tab, show or restore the T3 Code window and try again.", |
| 571 | }), |
| 572 | ), |
| 573 | SCREENSHOT_TIMEOUT_MS, |
| 574 | ); |
| 575 | }); |
| 576 | // Hidden native guests can leave CDP captures pending forever. This deadline also |
| 577 | // releases a viewer's still-capture lock, which an agent snapshot may be waiting for. |
| 578 | const { data } = await Promise.race([capture, expired]).finally(() => clearTimeout(timer)); |
| 579 | return data; |
| 580 | }; |
| 581 | |
| 582 | export const captureFinalRecordingFrame = (cdp: CDPSession, quality: number) => |
| 583 | answered( |
| 584 | cdp |
| 585 | .send("Runtime.evaluate", { |
| 586 | // Capture the destination of finite transitions, not their initially transparent frame. |
| 587 | expression: `(async () => { |
| 588 | const paint = () => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve))); |
| 589 | await paint(); |
| 590 | await Promise.allSettled(document.getAnimations() |
| 591 | .filter(animation => animation.playState === "running" && animation.effect?.getComputedTiming().iterations !== Infinity) |
| 592 | .map(animation => animation.finished)); |
| 593 | await paint(); |
| 594 | })()`, |
| 595 | awaitPromise: true, |
| 596 | }) |
| 597 | .then(() => captureViewport(cdp, { format: "jpeg", quality })), |
| 598 | ); |
| 599 | |
| 600 | /** Runs in the page's own isolated world, as the server has no image library. */ |
| 601 | const SHRINK_PNG = `async function (png, width, height) { |
| 602 | const bytes = Uint8Array.from(atob(png), (char) => char.charCodeAt(0)); |
| 603 | const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/png" })); |
| 604 | // Namespaced, as an XML document's createElement makes a plain element with no getContext. |
| 605 | const canvas = document.createElementNS("http://www.w3.org/1999/xhtml", "canvas"); |
| 606 | canvas.width = width; |
| 607 | canvas.height = height; |
| 608 | const context = canvas.getContext("2d"); |
| 609 | context.imageSmoothingQuality = "high"; |
| 610 | context.drawImage(bitmap, 0, 0, width, height); |
| 611 | bitmap.close(); |
| 612 | return canvas.toDataURL("image/png").slice("data:image/png;base64,".length); |
| 613 | }`; |
| 614 | |
| 615 | const shrinkPng = async (cdp: CDPSession, png: string, scale: number) => { |
| 616 | const size = pngSize(png); |
| 617 | const { frameTree } = await cdp.send("Page.getFrameTree"); |
| 618 | const { executionContextId } = await cdp.send("Page.createIsolatedWorld", { |
| 619 | frameId: frameTree.frame.id, |
| 620 | worldName: "t3-preview-screenshot", |
| 621 | }); |
| 622 | const { result, exceptionDetails } = await cdp.send("Runtime.callFunctionOn", { |
| 623 | functionDeclaration: SHRINK_PNG, |
| 624 | executionContextId, |
| 625 | arguments: [ |
| 626 | { value: png }, |
| 627 | { value: Math.round(size.width * scale) }, |
| 628 | { value: Math.round(size.height * scale) }, |
| 629 | ], |
| 630 | awaitPromise: true, |
| 631 | returnByValue: true, |
| 632 | }); |
| 633 | if (exceptionDetails) throw new Error("The snapshot could not shrink its screenshot."); |
| 634 | return result.value as string; |
| 635 | }; |
| 636 | |
| 637 | type SnapshotInput = { |
| 638 | readonly page: Page; |
| 639 | readonly cdp: CDPSession; |
| 640 | /** Headless pages render at the scale the server set; a desktop page at its display's. */ |
| 641 | readonly renderScale: number | "display"; |
| 642 | readonly fullScreenshot?: boolean | undefined; |
| 643 | readonly consoleEntries: ReadonlyArray<PreviewAutomationConsoleEntry>; |
| 644 | readonly networkEntries: ReadonlyArray<PreviewAutomationNetworkEntry>; |
| 645 | readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"]; |
| 646 | }; |
| 647 | |
| 648 | /** The scale a page renders at and its viewport in device-independent pixels. */ |
| 649 | const renderingOf = async (input: SnapshotInput) => { |
| 650 | if (input.renderScale !== "display") { |
| 651 | const viewport = input.page.viewportSize() ?? { width: 1280, height: 800 }; |
| 652 | return { renderScale: input.renderScale, viewport }; |
| 653 | } |
| 654 | // The display ratio a desktop page reports includes its page zoom, and its sizes are CSS pixels. |
| 655 | const page = await readPageMetrics(input.cdp); |
| 656 | return { |
| 657 | renderScale: page.ratio / page.zoom, |
| 658 | viewport: { |
| 659 | width: Math.round(page.width * page.zoom), |
| 660 | height: Math.round(page.height * page.zoom), |
| 661 | }, |
| 662 | }; |
| 663 | }; |
| 664 | |
| 665 | const capture = async ( |
| 666 | input: SnapshotInput, |
| 667 | deadline: number, |
| 668 | stages: SnapshotStages, |
| 669 | metadataOnly: boolean, |
| 670 | ) => { |
| 671 | const { running } = stages; |
| 672 | const track = async <A>(stage: SnapshotStage, operation: () => Promise<A>) => { |
| 673 | running.set(stage, (running.get(stage) ?? 0) + 1); |
| 674 | try { |
| 675 | const result = await operation(); |
| 676 | stages.timedOut.delete(stage); |
| 677 | return result; |
| 678 | } catch (cause) { |
| 679 | if (isTimeout(cause)) stages.timedOut.add(stage); |
| 680 | throw cause; |
| 681 | } finally { |
| 682 | const count = (running.get(stage) ?? 1) - 1; |
| 683 | if (count === 0) running.delete(stage); |
| 684 | else running.set(stage, count); |
| 685 | } |
| 686 | }; |
| 687 | const { renderScale, viewport } = await track("metrics", () => renderingOf(input)); |
| 688 | // A late metrics reply must not begin a capture after viewers have resumed. |
| 689 | if (Date.now() >= deadline) throw snapshotTimedOut(); |
| 690 | const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * renderScale)); |
| 691 | const state = refsFor(input.page); |
| 692 | invalidateRefs(input.page); |
| 693 | const generation = state.generation; |
| 694 | const { readers, read } = await track("frames", () => openReaders(input.page, input.cdp)); |
| 695 | const before = await track("frames", () => read()); |
| 696 | // Bracket the tree with value reads to detect changes; await tree completion before retrying failed captures. |
| 697 | const pageAndScreenshot = Promise.all([ |
| 698 | track( |
| 699 | "page state", |
| 700 | () => |
| 701 | input.page.evaluate(SNAPSHOT_SCRIPT) as Promise< |
| 702 | Pick< |
| 703 | PreviewAutomationSnapshot, |
| 704 | "url" | "title" | "loading" | "visibleText" | "interactiveElements" |
| 705 | > |
| 706 | >, |
| 707 | ), |
| 708 | // A cheap real frame still sizes the screenshot; computing it from metrics drifted from the PNG. |
| 709 | track("screenshot", () => |
| 710 | metadataOnly |
| 711 | ? captureViewport(input.cdp, { format: "jpeg", quality: 10 }).then((data) => ({ |
| 712 | data, |
| 713 | full: undefined, |
| 714 | })) |
| 715 | : captureViewport(input.cdp, { format: "png" }).then(async (full) => |
| 716 | scale < 1 |
| 717 | ? { |
| 718 | data: await shrinkPng(input.cdp, full, scale), |
| 719 | full: input.fullScreenshot ? full : undefined, |
| 720 | } |
| 721 | : { data: full, full: undefined }, |
| 722 | ), |
| 723 | ), |
| 724 | ]); |
| 725 | void pageAndScreenshot.catch(constVoid); |
| 726 | // Leaves time for the second read; a frame that never loads holds the tree until then. |
| 727 | const tree = await track("accessibility tree", () => |
| 728 | input.page.ariaSnapshot({ |
| 729 | mode: "ai", |
| 730 | boxes: true, |
| 731 | timeout: Math.max(1, deadline - Date.now() - 2_000), |
| 732 | }), |
| 733 | ); |
| 734 | const after = await track("frames", () => read(before)); |
| 735 | // A value that changed meanwhile may be in the tree under neither reading. |
| 736 | if (after.some((values, i) => values && JSON.stringify(values) !== JSON.stringify(before[i]))) |
| 737 | throw pageChanged(); |
| 738 | const dotted = dottedPattern(after.flatMap((values) => values ?? [])); |
| 739 | // Unread frames can expose values only if their content appears in the tree. |
| 740 | const unread = readers.flatMap((reader, i) => (after[i] ? [] : reader.frames)); |
| 741 | const filled = /^\s*- iframe (?:\[active\] )?\[ref=(\S+?)\].*:$/gm; |
| 742 | for (const [, ref] of unread.length ? tree.matchAll(filled) : []) { |
| 743 | const iframe = await track("frames", () => |
| 744 | input.page.locator(`aria-ref=${ref}`).elementHandle({ timeout: 2_000 }), |
| 745 | ); |
| 746 | if (!iframe) throw pageChanged(); |
| 747 | const frame = await track("frames", () => iframe.contentFrame()); |
| 748 | void iframe.dispose().catch(constVoid); |
| 749 | if (frame && unread.includes(frame)) throw frameUnanswered(); |
| 750 | } |
| 751 | const [page, { full, data }] = await pageAndScreenshot; |
| 752 | if (state.generation !== generation) throw pageChanged(); |
| 753 | const mask = (text: string) => (dotted ? text.replace(dotted, MASKED_VALUE) : text); |
| 754 | // Redacted before the cut, so a payload crossing it can't leave its start showing, and before |
| 755 | // masking, which could hide the `data:` scheme behind a password value's dots. |
| 756 | const redacted = withoutTreeDataPayloads(tree); |
| 757 | const accessibilityTree = (dotted ? maskTree(redacted, dotted) : redacted) |
| 758 | .slice(0, MAX_VISIBLE_TEXT_LENGTH) |
| 759 | .replace(/\[ref=((?:f\d+)?e\d+)\]/g, (_match, nativeRef: string) => { |
| 760 | const ref = `t3-${generation}-${nativeRef}`; |
| 761 | state.refs.set(ref, nativeRef); |
| 762 | return `[ref=${ref}]`; |
| 763 | }); |
| 764 | return { |
| 765 | ...page, |
| 766 | url: withoutDataPayload(page.url), |
| 767 | // Mask text before truncating so a value crossing the cut cannot leak its prefix. |
| 768 | visibleText: mask(page.visibleText).slice(0, MAX_VISIBLE_TEXT_LENGTH), |
| 769 | accessibilityTree, |
| 770 | consoleEntries: [...input.consoleEntries], |
| 771 | networkEntries: [...input.networkEntries], |
| 772 | actionTimeline: [...input.actionTimeline], |
| 773 | // Reports the size the PNG path would have shrunk to. |
| 774 | screenshot: metadataOnly |
| 775 | ? scaledSize(jpegSize(data), scale) |
| 776 | : { mimeType: "image/png" as const, data, ...pngSize(data) }, |
| 777 | ...(full !== undefined ? { fullScreenshot: { data: full, ...pngSize(full) } } : {}), |
| 778 | }; |
| 779 | }; |
| 780 | |
| 781 | /** Snapshots begun per page; observers' snapshots aren't queued, so two can overlap. */ |
| 782 | const snapshotTurns = new WeakMap<Page, number>(); |
| 783 | |
| 784 | const runSnapshot = async (input: SnapshotInput, metadataOnly: boolean) => { |
| 785 | const deadline = Date.now() + SNAPSHOT_TIMEOUT_MS; |
| 786 | const stages: SnapshotStages = { running: new Map(), timedOut: new Set() }; |
| 787 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 788 | const expired = new Promise<never>((_, reject) => { |
| 789 | timer = setTimeout(() => reject(snapshotTimedOut(stages)), SNAPSHOT_TIMEOUT_MS); |
| 790 | }); |
| 791 | const turn = (snapshotTurns.get(input.page) ?? 0) + 1; |
| 792 | snapshotTurns.set(input.page, turn); |
| 793 | // Retry only before the deadline and the next snapshot, whose refs a later retry would revoke. |
| 794 | const captured = capture(input, deadline, stages, metadataOnly).catch((cause: unknown) => |
| 795 | Date.now() < deadline && |
| 796 | snapshotTurns.get(input.page) === turn && |
| 797 | !(cause instanceof Error && cause.message === VALUE_TOO_LONG) |
| 798 | ? capture(input, deadline, stages, metadataOnly) |
| 799 | : Promise.reject(cause), |
| 800 | ); |
| 801 | try { |
| 802 | return await Promise.race([captured, expired]); |
| 803 | } catch (cause) { |
| 804 | throw isTimeout(cause) ? snapshotTimedOut(stages) : cause; |
| 805 | } finally { |
| 806 | clearTimeout(timer); |
| 807 | } |
| 808 | }; |
| 809 | |
| 810 | export const snapshot = (input: SnapshotInput) => |
| 811 | runSnapshot(input, false) as Promise<PreviewAutomationSnapshot>; |
| 812 | |
| 813 | export const snapshotMetadata = (input: SnapshotInput) => |
| 814 | runSnapshot(input, true) as Promise<PreviewAutomationSnapshotMetadata>; |
| 815 | |
| 816 | /** Return when a dialog opens; waiting for the action to finish would block until dialog resolution. */ |
| 817 | export const click = async ( |
| 818 | page: Page, |
| 819 | input: PreviewAutomationClickInput, |
| 820 | pointer: PointerReporter = noPointer, |
| 821 | ): Promise<{ readonly x: number; readonly y: number }> => { |
| 822 | // One deadline across steps keeps the reply inside the broker's grace; Playwright reads 0 as none. |
| 823 | const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); |
| 824 | const remaining = () => Math.max(1, deadline - Date.now()); |
| 825 | const locator = targetLocator(page, input); |
| 826 | try { |
| 827 | return await clickTarget(page, locator, input, remaining, pointer); |
| 828 | } catch (cause) { |
| 829 | throw locator === null ? cause : await namedTargetTimeout(locator, input, cause); |
| 830 | } |
| 831 | }; |
| 832 | |
| 833 | /** |
| 834 | * Playwright's timeout names neither the target nor why it waited. Counting only after the |
| 835 | * wait failed keeps auto-wait intact, but the count describes the page after the timeout, not |
| 836 | * what the click did: Playwright can dispatch the click and then time out waiting for the |
| 837 | * navigation it started. |
| 838 | */ |
| 839 | const namedTargetTimeout = async ( |
| 840 | locator: Locator, |
| 841 | input: { readonly locator?: string | undefined; readonly selector?: string | undefined }, |
| 842 | cause: unknown, |
| 843 | ) => { |
| 844 | if (!(cause instanceof Error && cause.name === "TimeoutError")) return cause; |
| 845 | const selector = JSON.stringify(input.locator ?? input.selector); |
| 846 | // The warning leads and the target is capped, so a reply cut short still carries the warning. |
| 847 | const target = |
| 848 | selector.length > MAX_TARGET_CHARS ? `${selector.slice(0, MAX_TARGET_CHARS)}...` : selector; |
| 849 | // The host's reply must still beat the broker's short grace after the operation timeout. |
| 850 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 851 | const matches = await Promise.race([ |
| 852 | locator.count().catch(() => undefined), |
| 853 | new Promise<undefined>((resolve) => { |
| 854 | timer = setTimeout(() => resolve(undefined), TARGET_COUNT_MS); |
| 855 | }), |
| 856 | ]).finally(() => clearTimeout(timer)); |
| 857 | const reason = |
| 858 | matches === undefined |
| 859 | ? `The click may have happened; check the page before retrying. Whether ${target} still matches an element is unknown.` |
| 860 | : matches === 0 |
| 861 | ? `The click may have happened and removed its target or navigated; check the page before retrying. No element matches ${target} after the timeout; take a fresh snapshot and use its locator.` |
| 862 | : `The click may have happened; check the page before retrying. ${target} still matches an element after the timeout.`; |
| 863 | const message = withActionabilityReason(reason, cause); |
| 864 | return new ServerBrowserOperationError({ |
| 865 | tag: "PreviewAutomationTimeoutError", |
| 866 | message, |
| 867 | detail: { reason: message }, |
| 868 | cause, |
| 869 | }); |
| 870 | }; |
| 871 | |
| 872 | const clickTarget = async ( |
| 873 | page: Page, |
| 874 | locator: Locator | null, |
| 875 | input: PreviewAutomationClickInput, |
| 876 | timeout: () => number, |
| 877 | pointer: PointerReporter, |
| 878 | ) => { |
| 879 | const point = await targetPoint(page, locator, input, timeout); |
| 880 | await pointer(point, "click"); |
| 881 | const options = { button: input.button ?? "left", clickCount: input.clickCount ?? 1 } as const; |
| 882 | const clicked = |
| 883 | locator === null |
| 884 | ? page.mouse.click(point.x, point.y, options) |
| 885 | : locator.click({ ...options, timeout: timeout() }); |
| 886 | let onDialog = constVoid; |
| 887 | const dialogOpened = new Promise<"dialog">((resolve) => { |
| 888 | onDialog = () => resolve("dialog"); |
| 889 | page.once("dialog", onDialog); |
| 890 | }); |
| 891 | try { |
| 892 | if ((await Promise.race([clicked, dialogOpened])) === "dialog") void clicked.catch(constVoid); |
| 893 | } finally { |
| 894 | page.off("dialog", onDialog); |
| 895 | } |
| 896 | return point; |
| 897 | }; |
| 898 | |
| 899 | type FocusCandidate = { |
| 900 | readonly tagName: string; |
| 901 | readonly type?: string; |
| 902 | readonly disabled?: boolean; |
| 903 | readonly readOnly?: boolean; |
| 904 | readonly isContentEditable?: boolean; |
| 905 | readonly shadowRoot?: { readonly activeElement: FocusCandidate | null } | null; |
| 906 | readonly focus?: () => void; |
| 907 | getRootNode(): { readonly activeElement?: FocusCandidate | null }; |
| 908 | contains(other: FocusCandidate): boolean; |
| 909 | }; |
| 910 | |
| 911 | // Serialized into the page's main world, so it must not reference module scope. |
| 912 | const focusTextTarget = (target: FocusCandidate | null) => { |
| 913 | let element = |
| 914 | target ?? |
| 915 | (globalThis as unknown as { document: { activeElement: FocusCandidate | null } }).document |
| 916 | .activeElement; |
| 917 | if (target === null) { |
| 918 | while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement; |
| 919 | // Focus inside a frame cannot be checked from here, and an editable frame would pass as contentEditable. |
| 920 | if (element && ["IFRAME", "FRAME", "FENCEDFRAME", "OBJECT", "EMBED"].includes(element.tagName)) |
| 921 | return false; |
| 922 | } |
| 923 | if (element == null) return false; |
| 924 | // Require an enabled, focused text target to avoid losing input or sending it to the previous field. |
| 925 | const nonText = [ |
| 926 | "button", |
| 927 | "checkbox", |
| 928 | "color", |
| 929 | "file", |
| 930 | "hidden", |
| 931 | "image", |
| 932 | "radio", |
| 933 | "range", |
| 934 | "reset", |
| 935 | "submit", |
| 936 | ]; |
| 937 | const textControl = |
| 938 | element.tagName === "TEXTAREA" || |
| 939 | (element.tagName === "INPUT" && !nonText.includes(element.type ?? "text")); |
| 940 | if (!(textControl || element.isContentEditable) || element.disabled || element.readOnly) { |
| 941 | return false; |
| 942 | } |
| 943 | element.focus?.(); |
| 944 | const active = element.getRootNode().activeElement; |
| 945 | return active != null && (active === element || element.contains(active)); |
| 946 | }; |
| 947 | |
| 948 | // A `*:focus` locator needs Playwright's utility world, which some desktop tabs never get. |
| 949 | const focusedTextField = async (page: Page, timeout: number) => { |
| 950 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 951 | const expired = new Promise<never>((_, reject) => { |
| 952 | timer = setTimeout( |
| 953 | () => |
| 954 | reject( |
| 955 | new ServerBrowserOperationError({ |
| 956 | tag: "PreviewAutomationTimeoutError", |
| 957 | message: `The page did not report its focused element within ${timeout}ms, so no text was typed.`, |
| 958 | detail: { timeoutMs: timeout }, |
| 959 | }), |
| 960 | ), |
| 961 | timeout, |
| 962 | ); |
| 963 | }); |
| 964 | return await Promise.race([page.evaluate(focusTextTarget, null), expired]).finally(() => |
| 965 | clearTimeout(timer), |
| 966 | ); |
| 967 | }; |
| 968 | |
| 969 | export const type = async (page: Page, input: PreviewAutomationTypeInput) => { |
| 970 | const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; |
| 971 | const locator = targetLocator(page, input); |
| 972 | if (locator !== null && input.clear) { |
| 973 | await locator.fill(input.text, { timeout }); |
| 974 | return; |
| 975 | } |
| 976 | const focused = |
| 977 | locator === null |
| 978 | ? await focusedTextField(page, timeout) |
| 979 | : await locator.evaluate(focusTextTarget, undefined, { timeout }); |
| 980 | if (focused !== true) { |
| 981 | throw new ServerBrowserOperationError({ |
| 982 | tag: "PreviewAutomationTargetNotEditableError", |
| 983 | message: "The target is not an enabled text field, so no text was typed.", |
| 984 | }); |
| 985 | } |
| 986 | if (input.clear) { |
| 987 | await page.keyboard.press("ControlOrMeta+A"); |
| 988 | await page.keyboard.press("Delete"); |
| 989 | } |
| 990 | await page.keyboard.insertText(input.text); |
| 991 | }; |
| 992 | |
| 993 | export const hover = async ( |
| 994 | page: Page, |
| 995 | input: PreviewAutomationHoverInput, |
| 996 | pointer: PointerReporter = noPointer, |
| 997 | ) => { |
| 998 | const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); |
| 999 | const remaining = () => Math.max(1, deadline - Date.now()); |
| 1000 | const locator = targetLocator(page, input); |
| 1001 | await pointer(await targetPoint(page, locator, input, remaining), "move"); |
| 1002 | if (locator === null) { |
| 1003 | await page.mouse.move(input.x ?? 0, input.y ?? 0); |
| 1004 | return; |
| 1005 | } |
| 1006 | await locator.hover({ timeout: remaining() }); |
| 1007 | }; |
| 1008 | |
| 1009 | export const select = async ( |
| 1010 | page: Page, |
| 1011 | input: PreviewAutomationSelectInput, |
| 1012 | ): Promise<PreviewAutomationSelectResult> => { |
| 1013 | const locator = targetLocator(page, input)!; |
| 1014 | const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; |
| 1015 | const options = await locator.evaluate( |
| 1016 | (element) => |
| 1017 | element.tagName === "SELECT" |
| 1018 | ? Array.from( |
| 1019 | (element as unknown as { options: ArrayLike<{ value: string; label: string }> }) |
| 1020 | .options, |
| 1021 | ({ value, label }) => ({ value, label: label.trim() }), |
| 1022 | ) |
| 1023 | : null, |
| 1024 | undefined, |
| 1025 | { timeout }, |
| 1026 | ); |
| 1027 | const values = |
| 1028 | options && |
| 1029 | input.values.map( |
| 1030 | (entry) => |
| 1031 | ( |
| 1032 | options.find((option) => option.value === entry) ?? |
| 1033 | options.find((option) => option.label === entry.trim()) |
| 1034 | )?.value ?? entry, |
| 1035 | ); |
| 1036 | if (values === null) |
| 1037 | throw new ServerBrowserOperationError({ |
| 1038 | tag: "PreviewAutomationTargetNotEditableError", |
| 1039 | message: "This element is not a <select>. Click a custom dropdown, then click its option.", |
| 1040 | }); |
| 1041 | return { selected: await locator.selectOption(values, { timeout }) }; |
| 1042 | }; |
| 1043 | |
| 1044 | export const drag = async ( |
| 1045 | page: Page, |
| 1046 | input: PreviewAutomationDragInput, |
| 1047 | pointer: PointerReporter = noPointer, |
| 1048 | ) => { |
| 1049 | const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS); |
| 1050 | const remaining = () => Math.max(1, deadline - Date.now()); |
| 1051 | const source = targetLocator(page, { locator: input.source })!; |
| 1052 | const target = targetLocator(page, { locator: input.target })!; |
| 1053 | await pointer(await targetPoint(page, source, {}, remaining), "move"); |
| 1054 | // Observe drag rejection immediately; an unhandled rejection during cursor movement exits the server. |
| 1055 | // Chromium delivers a single move before dragstart; steps give the page mousemoves during the drag. |
| 1056 | const dragFailure = source.dragTo(target, { timeout: remaining(), steps: 5 }).then( |
| 1057 | () => null, |
| 1058 | (error: unknown) => ({ error }), |
| 1059 | ); |
| 1060 | const end = await target.boundingBox({ timeout: remaining() }).catch(() => null); |
| 1061 | if (end) await pointer({ x: end.x + end.width / 2, y: end.y + end.height / 2 }, "move"); |
| 1062 | const failure = await dragFailure; |
| 1063 | if (failure) throw failure.error; |
| 1064 | }; |
| 1065 | |
| 1066 | export const setInputFiles = async (page: Page, input: PreviewAutomationUploadInput) => { |
| 1067 | const locator = targetLocator(page, input); |
| 1068 | if (locator === null) return false; |
| 1069 | await locator.setInputFiles([...input.paths], { timeout: input.timeoutMs ?? DEFAULT_TIMEOUT_MS }); |
| 1070 | return true; |
| 1071 | }; |
| 1072 | |
| 1073 | export const press = async (page: Page, input: PreviewAutomationPressInput) => { |
| 1074 | await page.keyboard.press([...(input.modifiers ?? []), input.key].join("+")); |
| 1075 | }; |
| 1076 | |
| 1077 | export const scroll = async (page: Page, input: PreviewAutomationScrollInput) => { |
| 1078 | const delta = [input.deltaX ?? 0, input.deltaY ?? 0] as const; |
| 1079 | const locator = targetLocator(page, input); |
| 1080 | if (locator === null) { |
| 1081 | // Page-side code is passed as source: the server compiles without DOM types. |
| 1082 | await page.evaluate(`scrollBy(${delta[0]}, ${delta[1]})`); |
| 1083 | return; |
| 1084 | } |
| 1085 | await locator.evaluate((element, [x, y]) => element.scrollBy(x, y), delta); |
| 1086 | }; |
| 1087 | |
| 1088 | const UNRESPONSIVE_SCRIPT_PROBE_MS = 1_000; |
| 1089 | |
| 1090 | /** |
| 1091 | * A same-site navigation reuses the renderer, so it never commits while a page script holds it. |
| 1092 | * Call before starting the navigation: Chrome does not deliver a terminate once one is pending. |
| 1093 | */ |
| 1094 | export const stopUnresponsiveScript = async (cdp: CDPSession, budgetMs = Infinity) => { |
| 1095 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 1096 | const answered = cdp.send("Runtime.evaluate", { expression: "0" }).then( |
| 1097 | () => true, |
| 1098 | () => true, |
| 1099 | ); |
| 1100 | const expired = new Promise<boolean>((resolve) => { |
| 1101 | timer = setTimeout(() => resolve(false), Math.min(budgetMs, UNRESPONSIVE_SCRIPT_PROBE_MS)); |
| 1102 | }); |
| 1103 | const responsive = await Promise.race([answered, expired]).finally(() => clearTimeout(timer)); |
| 1104 | // Not awaited: a renderer blocked on something other than script may never acknowledge it. |
| 1105 | if (!responsive) void cdp.send("Runtime.terminateExecution").catch(constVoid); |
| 1106 | }; |
| 1107 | |
| 1108 | /** Terminate expired page scripts so a hung expression cannot block the tab's control queue. */ |
| 1109 | export const evaluate = async ( |
| 1110 | cdp: CDPSession, |
| 1111 | input: PreviewAutomationEvaluateInput, |
| 1112 | timeoutMs: number, |
| 1113 | ) => { |
| 1114 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 1115 | let timedOut = false; |
| 1116 | const expired = new Promise<never>((_, reject) => { |
| 1117 | timer = setTimeout(() => { |
| 1118 | timedOut = true; |
| 1119 | // Stops a busy script; an awaited promise is abandoned and settles unread. |
| 1120 | void cdp.send("Runtime.terminateExecution").catch(constVoid); |
| 1121 | reject( |
| 1122 | new ServerBrowserOperationError({ |
| 1123 | tag: "PreviewAutomationTimeoutError", |
| 1124 | message: `Evaluation did not finish within ${timeoutMs}ms and was stopped.`, |
| 1125 | }), |
| 1126 | ); |
| 1127 | }, timeoutMs); |
| 1128 | }); |
| 1129 | const params = { |
| 1130 | expression: input.expression, |
| 1131 | awaitPromise: input.awaitPromise ?? true, |
| 1132 | returnByValue: true, |
| 1133 | }; |
| 1134 | // The devtools console's replMode allows top-level await but returns a trailing promise unawaited, |
| 1135 | // so it only retries what the page's parser rejected. A parse error has a scriptId but no stack and |
| 1136 | // ran nothing; a browser-internal rejection has neither. |
| 1137 | const evaluation = cdp.send("Runtime.evaluate", params).then((classic) => { |
| 1138 | const failure = classic.exceptionDetails; |
| 1139 | const unparsed = |
| 1140 | failure?.exception?.className === "SyntaxError" && |
| 1141 | failure.scriptId !== undefined && |
| 1142 | failure.stackTrace === undefined; |
| 1143 | return unparsed && !timedOut && input.expression.includes("await") |
| 1144 | ? cdp.send("Runtime.evaluate", { ...params, replMode: true }) |
| 1145 | : classic; |
| 1146 | }); |
| 1147 | void evaluation.catch(constVoid); |
| 1148 | const result = await Promise.race([evaluation, expired]).finally(() => clearTimeout(timer)); |
| 1149 | if (result.exceptionDetails) { |
| 1150 | throw new ServerBrowserOperationError({ |
| 1151 | tag: "PreviewAutomationExecutionError", |
| 1152 | message: result.exceptionDetails.exception?.description ?? result.exceptionDetails.text, |
| 1153 | }); |
| 1154 | } |
| 1155 | const value = |
| 1156 | "value" in result.result ? result.result.value : (result.result.description ?? null); |
| 1157 | const actualBytes = Buffer.byteLength(JSON.stringify(value ?? null), "utf8"); |
| 1158 | if (actualBytes > MAX_EVALUATION_BYTES) { |
| 1159 | throw new ServerBrowserOperationError({ |
| 1160 | tag: "PreviewAutomationResultTooLargeError", |
| 1161 | message: `Evaluation result is ${actualBytes} bytes; the limit is ${MAX_EVALUATION_BYTES}.`, |
| 1162 | detail: { maximumBytes: MAX_EVALUATION_BYTES }, |
| 1163 | }); |
| 1164 | } |
| 1165 | return value; |
| 1166 | }; |
| 1167 | |
| 1168 | export const waitFor = async (page: Page, input: PreviewAutomationWaitForInput) => { |
| 1169 | const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS; |
| 1170 | const locator = targetLocator(page, input); |
| 1171 | const { text, urlIncludes } = input; |
| 1172 | const checks: Array<() => Promise<boolean>> = []; |
| 1173 | if (locator !== null) checks.push(async () => (await locator.count()) > 0); |
| 1174 | if (text !== undefined) { |
| 1175 | // A navigation can destroy the context mid-check; that counts as not yet. |
| 1176 | checks.push(() => |
| 1177 | page |
| 1178 | .evaluate(`(document.body?.innerText ?? "").includes(${JSON.stringify(text)})`) |
| 1179 | .then((found) => found === true) |
| 1180 | .catch(() => false), |
| 1181 | ); |
| 1182 | } |
| 1183 | if (urlIncludes !== undefined) checks.push(async () => page.url().includes(urlIncludes)); |
| 1184 | // `count()` and `evaluate()` take no timeout; a stalled check would hold the tab's control queue. |
| 1185 | let timer: ReturnType<typeof setTimeout> | undefined; |
| 1186 | const expired = new Promise<never>((_, reject) => { |
| 1187 | timer = setTimeout( |
| 1188 | () => |
| 1189 | reject( |
| 1190 | new ServerBrowserOperationError({ |
| 1191 | tag: "PreviewAutomationTimeoutError", |
| 1192 | message: `Waited ${timeoutMs}ms without every condition holding.`, |
| 1193 | }), |
| 1194 | ), |
| 1195 | timeoutMs, |
| 1196 | ); |
| 1197 | }); |
| 1198 | try { |
| 1199 | for (;;) { |
| 1200 | const results = await Promise.race([Promise.all(checks.map((check) => check())), expired]); |
| 1201 | if (results.every(Boolean)) return; |
| 1202 | await Promise.race([new Promise((resolve) => setTimeout(resolve, WAIT_POLL_MS)), expired]); |
| 1203 | } |
| 1204 | } finally { |
| 1205 | clearTimeout(timer); |
| 1206 | } |
| 1207 | }; |
| 1208 | |
| 1209 | const NET_ERROR_CODES: Readonly<Record<string, number>> = { |
| 1210 | ERR_FAILED: -2, |
| 1211 | ERR_TIMED_OUT: -7, |
| 1212 | ERR_CONNECTION_CLOSED: -100, |
| 1213 | ERR_CONNECTION_RESET: -101, |
| 1214 | ERR_CONNECTION_REFUSED: -102, |
| 1215 | ERR_NAME_NOT_RESOLVED: -105, |
| 1216 | ERR_INTERNET_DISCONNECTED: -106, |
| 1217 | ERR_ADDRESS_UNREACHABLE: -109, |
| 1218 | ERR_CERT_AUTHORITY_INVALID: -202, |
| 1219 | ERR_EMPTY_RESPONSE: -324, |
| 1220 | }; |
| 1221 | |
| 1222 | export const parseNetError = (errorText: string) => { |
| 1223 | // Names can hold digits, as in ERR_HTTP2_PROTOCOL_ERROR. |
| 1224 | const description = /ERR_[A-Z\d_]+/.exec(errorText)?.[0] ?? errorText; |
| 1225 | return { description, code: NET_ERROR_CODES[description] ?? -2 }; |
| 1226 | }; |
| 1227 | |
| 1228 | export const recordingEncoderScript = (frameRate: BrowserRecordingFrameRate) => `(() => { |
| 1229 | const canvas = document.createElement("canvas"); |
| 1230 | const context = canvas.getContext("2d"); |
| 1231 | const chunks = []; |
| 1232 | let recorder = null; |
| 1233 | let started = null; |
| 1234 | let stopped = null; |
| 1235 | let sizeBytes = 0; |
| 1236 | let tooLarge = false; |
| 1237 | let frame = null; |
| 1238 | let cursor = null; |
| 1239 | let ring = null; |
| 1240 | const AGENT_CURSOR_PATH = new Path2D("M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"); |
| 1241 | let ringTimer = null; |
| 1242 | let scale = 1; |
| 1243 | let offsetX = 0; |
| 1244 | let offsetY = 0; |
| 1245 | const paint = () => { |
| 1246 | if (!frame) return; |
| 1247 | const fit = Math.min(canvas.width / frame.width, canvas.height / frame.height); |
| 1248 | const width = frame.width * fit; |
| 1249 | const height = frame.height * fit; |
| 1250 | offsetX = (canvas.width - width) / 2; |
| 1251 | offsetY = (canvas.height - height) / 2; |
| 1252 | if (width < canvas.width || height < canvas.height) { |
| 1253 | context.fillStyle = "#000"; |
| 1254 | context.fillRect(0, 0, canvas.width, canvas.height); |
| 1255 | } |
| 1256 | context.drawImage(frame, offsetX, offsetY, width, height); |
| 1257 | scale = (width / frame.width) * frame.cssScale; |
| 1258 | if (ring) { |
| 1259 | const age = (performance.now() - ring.at) / 400; |
| 1260 | if (age < 1) { |
| 1261 | context.beginPath(); |
| 1262 | context.arc(offsetX + ring.x * scale, offsetY + ring.y * scale, (8 + 18 * age) * scale, 0, Math.PI * 2); |
| 1263 | context.strokeStyle = "rgba(59,130,246," + (0.9 * (1 - age)) + ")"; |
| 1264 | context.lineWidth = 3 * scale; |
| 1265 | context.stroke(); |
| 1266 | } |
| 1267 | } |
| 1268 | if (cursor) { |
| 1269 | // The agent cursor (lucide MousePointer2, as in the live panel), never the human arrow. |
| 1270 | const s = 20 / 24 * scale; |
| 1271 | context.save(); |
| 1272 | context.translate(offsetX + cursor.x * scale - 2 * scale, offsetY + cursor.y * scale - 2 * scale); |
| 1273 | context.scale(s, s); |
| 1274 | context.lineWidth = 2; |
| 1275 | context.lineJoin = "round"; |
| 1276 | context.lineCap = "round"; |
| 1277 | context.fillStyle = "#fff"; |
| 1278 | context.strokeStyle = "#2563eb"; |
| 1279 | context.shadowColor = "rgba(0,0,0,0.25)"; |
| 1280 | context.shadowBlur = 2; |
| 1281 | context.fill(AGENT_CURSOR_PATH); |
| 1282 | context.shadowColor = "transparent"; |
| 1283 | context.stroke(AGENT_CURSOR_PATH); |
| 1284 | context.restore(); |
| 1285 | } |
| 1286 | }; |
| 1287 | window.__t3Recorder = { |
| 1288 | async frame(base64, cssWidth) { |
| 1289 | if (tooLarge) return false; |
| 1290 | // A per-character atob copy costs ~30 ms on a 3200x1800 frame and throttles recordings. |
| 1291 | const bytes = Uint8Array.fromBase64(base64); |
| 1292 | const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/jpeg" })); |
| 1293 | if (tooLarge) { bitmap.close(); return false; } |
| 1294 | if (!recorder) { |
| 1295 | canvas.width = bitmap.width; |
| 1296 | canvas.height = bitmap.height; |
| 1297 | const mimeType = ["video/mp4;codecs=avc1.640033", "video/webm;codecs=vp9", "video/webm"].find((type) => MediaRecorder.isTypeSupported(type)); |
| 1298 | const bitsPerSecond = Math.min(50e6, Math.max(2.5e6, bitmap.width * bitmap.height * ${frameRate} * 0.05)); |
| 1299 | recorder = new MediaRecorder(canvas.captureStream(${frameRate}), { mimeType, videoBitsPerSecond: bitsPerSecond }); |
| 1300 | started = new Promise((resolve, reject) => { |
| 1301 | recorder.onstart = resolve; |
| 1302 | recorder.onerror = (event) => reject(event.error); |
| 1303 | }); |
| 1304 | stopped = new Promise((resolve) => { recorder.onstop = resolve; }); |
| 1305 | recorder.ondataavailable = (event) => { |
| 1306 | if (tooLarge || event.data.size === 0) return; |
| 1307 | sizeBytes += event.data.size; |
| 1308 | if (sizeBytes > ${PROVIDER_SEND_TURN_MAX_FILE_BYTES}) { |
| 1309 | tooLarge = true; |
| 1310 | chunks.length = 0; |
| 1311 | clearInterval(ringTimer); |
| 1312 | frame?.close(); |
| 1313 | frame = null; |
| 1314 | if (recorder.state !== "inactive") recorder.stop(); |
| 1315 | for (const track of recorder.stream.getTracks()) track.stop(); |
| 1316 | return; |
| 1317 | } |
| 1318 | chunks.push(event.data); |
| 1319 | }; |
| 1320 | recorder.start(1000); |
| 1321 | } |
| 1322 | // Frame px per page CSS px, before fitting into the canvas. |
| 1323 | bitmap.cssScale = bitmap.width / cssWidth; |
| 1324 | frame?.close(); |
| 1325 | frame = bitmap; |
| 1326 | paint(); |
| 1327 | await started; |
| 1328 | return true; |
| 1329 | }, |
| 1330 | cursor(x, y, click) { |
| 1331 | if (tooLarge) return; |
| 1332 | cursor = { x, y }; |
| 1333 | if (click) { |
| 1334 | ring = { x, y, at: performance.now() }; |
| 1335 | clearInterval(ringTimer); |
| 1336 | ringTimer = setInterval(() => { |
| 1337 | paint(); |
| 1338 | if (performance.now() - ring.at > 400) { ring = null; clearInterval(ringTimer); paint(); } |
| 1339 | }, 33); |
| 1340 | } |
| 1341 | paint(); |
| 1342 | }, |
| 1343 | async stop() { |
| 1344 | if (!recorder) return { mimeType: null, count: 0, bytes: 0 }; |
| 1345 | if (recorder.state !== "inactive") { |
| 1346 | // Consume the cached frame, then wait for the final paint to reach the |
| 1347 | // capture stream. Animation ticks alone can stop before it is delivered. |
| 1348 | const video = document.createElement("video"); |
| 1349 | video.muted = true; |
| 1350 | video.srcObject = recorder.stream; |
| 1351 | const nextFrame = () => Promise.race([ |
| 1352 | new Promise(resolve => video.requestVideoFrameCallback(resolve)), |
| 1353 | stopped, |
| 1354 | ]); |
| 1355 | const initial = nextFrame(); |
| 1356 | void video.play().catch(() => {}); |
| 1357 | await initial; |
| 1358 | if (recorder.state !== "inactive") { |
| 1359 | const delivered = nextFrame(); |
| 1360 | recorder.stream.getVideoTracks()[0].requestFrame(); |
| 1361 | paint(); |
| 1362 | await delivered; |
| 1363 | } |
| 1364 | video.pause(); |
| 1365 | video.srcObject = null; |
| 1366 | if (recorder.state !== "inactive") recorder.stop(); |
| 1367 | } |
| 1368 | await stopped; |
| 1369 | return { mimeType: recorder.mimeType, count: chunks.length, bytes: sizeBytes }; |
| 1370 | }, |
| 1371 | async chunk(index) { |
| 1372 | const bytes = new Uint8Array(await chunks[index].arrayBuffer()); |
| 1373 | if (typeof bytes.toBase64 === "function") return bytes.toBase64(); |
| 1374 | let binary = ""; |
| 1375 | for (let offset = 0; offset < bytes.length; offset += 0x8000) { |
| 1376 | binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000)); |
| 1377 | } |
| 1378 | return btoa(binary); |
| 1379 | }, |
| 1380 | download() { |
| 1381 | const link = document.createElement("a"); |
| 1382 | link.href = URL.createObjectURL(new Blob(chunks, { type: recorder.mimeType })); |
| 1383 | link.download = "recording"; |
| 1384 | link.click(); |
| 1385 | }, |
| 1386 | }; |
| 1387 | })()`; |