The best way to control Claude Code, Codex and more from any computer or phone

main
ts 1,387 lines 54.7 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off globalDate:off globalTimers:off - Playwright callbacks run outside the Effect runtime.
2
3 import {
4 type BrowserRecordingFrameRate,
5 PROVIDER_SEND_TURN_MAX_FILE_BYTES,
6 type PreviewAutomationClickInput,
7 type PreviewAutomationConsoleEntry,
8 type PreviewAutomationDragInput,
9 type PreviewAutomationEvaluateInput,
10 type PreviewAutomationHoverInput,
11 type PreviewAutomationNetworkEntry,
12 type PreviewAutomationPressInput,
13 type PreviewAutomationScrollInput,
14 type PreviewAutomationSelectInput,
15 type PreviewAutomationSelectResult,
16 type PreviewAutomationSnapshot,
17 type PreviewAutomationSnapshotMetadata,
18 type PreviewAutomationSnapshotStage,
19 type PreviewAutomationTypeInput,
20 type PreviewAutomationUploadInput,
21 type PreviewAutomationWaitForInput,
22 } from "@t3tools/contracts";
23 import { constVoid } from "effect/Function";
24 import * as Schema from "effect/Schema";
25 import type { CDPSession, ConsoleMessage, Frame, Locator, Page } from "playwright-core";
26 import * as NodeCrypto from "node:crypto";
27 import * as NodeUtil from "node:util";
28 import { BrowserControlInterrupted } from "./SessionControl.ts";
29
30 const isBrowserControlInterrupted = Schema.is(BrowserControlInterrupted);
31
32 const MAX_EVALUATION_BYTES = 64_000;
33 const MAX_VISIBLE_TEXT_LENGTH = 20_000;
34 const MAX_SCREENSHOT_WIDTH = 1280;
35 // Two attempts must fit in a snapshot's 10-second deadline.
36 const SCREENSHOT_TIMEOUT_MS = 3_000;
37 const WAIT_POLL_MS = 100;
38 export const DIAGNOSTIC_BUFFER_LIMIT = 200;
39
40 export class ServerBrowserOperationError extends Schema.TaggedError<ServerBrowserOperationError>()(
41 "ServerBrowserOperationError",
42 {
43 tag: Schema.String,
44 message: Schema.String,
45 detail: Schema.optional(Schema.Unknown),
46 cause: Schema.optional(Schema.Defect()),
47 },
48 ) {}
49
50 const isServerBrowserOperationError = Schema.is(ServerBrowserOperationError);
51
52 export const toOperationError = (
53 cause: unknown,
54 timeoutMs?: number,
55 ): ServerBrowserOperationError => {
56 if (isBrowserControlInterrupted(cause))
57 return new ServerBrowserOperationError({
58 tag: "PreviewAutomationControlInterruptedError",
59 message: cause.message,
60 detail: cause.reason,
61 });
62 if (isServerBrowserOperationError(cause)) return cause;
63 const message = cause instanceof Error ? cause.message : String(cause);
64 const firstLine = message.split("\n")[0] ?? message;
65 if (cause instanceof Error && cause.name === "TimeoutError") {
66 return new ServerBrowserOperationError({
67 tag: "PreviewAutomationTimeoutError",
68 message: withActionabilityReason(firstLine, cause),
69 detail: timeoutMs === undefined ? undefined : { timeoutMs },
70 });
71 }
72 if (/while parsing selector|Unknown engine|Unexpected token/i.test(message)) {
73 return new ServerBrowserOperationError({
74 tag: "PreviewAutomationInvalidSelectorError",
75 message: firstLine,
76 });
77 }
78 if (/not an <input>|not editable|not an editable/i.test(message)) {
79 return new ServerBrowserOperationError({
80 tag: "PreviewAutomationTargetNotEditableError",
81 message: firstLine,
82 });
83 }
84 return new ServerBrowserOperationError({
85 tag: "PreviewAutomationExecutionError",
86 message: firstLine,
87 });
88 };
89
90 /** Playwright puts why an element was not actionable in its call log, below the first line. */
91 const withActionabilityReason = (message: string, cause: Error) => {
92 // Playwright dims call-log lines with ANSI escapes when colour is on.
93 const reason = NodeUtil.stripVTControlCharacters(cause.message)
94 .split("\n")
95 .map((line) => line.trim())
96 .findLast((line) => line.startsWith("- element is "))
97 ?.slice(2);
98 return reason === undefined ? message : `${message} Playwright last saw: ${reason}.`;
99 };
100
101 const DEFAULT_TIMEOUT_MS = 15_000;
102 const TARGET_COUNT_MS = 250;
103 const MAX_TARGET_CHARS = 200;
104
105 const pageRefs = new WeakMap<Page, { generation: string; refs: Map<string, string> }>();
106 // A compact runtime namespace prevents old refs from aliasing after a server restart.
107 const refNamespace = NodeCrypto.randomUUID().slice(0, 8);
108 let snapshotSequence = 0;
109 const nextRefGeneration = () => `${refNamespace}-${(++snapshotSequence).toString(36)}`;
110
111 /** A takeover revokes previously issued refs even when the document stays unchanged. */
112 export const invalidateRefs = (page: Page) => {
113 const state = pageRefs.get(page);
114 if (state) {
115 state.generation = nextRefGeneration();
116 state.refs.clear();
117 }
118 };
119
120 const refsFor = (page: Page) => {
121 let state = pageRefs.get(page);
122 if (!state) {
123 state = { generation: nextRefGeneration(), refs: new Map<string, string>() };
124 pageRefs.set(page, state);
125 page.on("framenavigated", () => invalidateRefs(page));
126 page.on("framedetached", () => invalidateRefs(page));
127 }
128 return state;
129 };
130
131 const targetLocator = (
132 page: Page,
133 input: { readonly locator?: string | undefined; readonly selector?: string | undefined },
134 ): Locator | null => {
135 const selector = input.locator ?? input.selector;
136 if (selector === undefined) return null;
137 const engines = selector.replace(
138 /\\.|"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|`(?:\\.|[^`\\])*`/g,
139 " ",
140 );
141 if (/(?:^|>>)\s*\*?aria-ref\s*=/.test(engines)) {
142 const ref = /^\s*aria-ref\s*=\s*(\S+)\s*$/.exec(selector)?.[1];
143 const nativeRef = ref === undefined ? undefined : pageRefs.get(page)?.refs.get(ref);
144 if (nativeRef === undefined) {
145 throw new ServerBrowserOperationError({
146 tag: "PreviewAutomationInvalidSelectorError",
147 message:
148 "This element ref is stale or belongs to another tab. Take a fresh snapshot and use its locator.",
149 detail: { staleRef: true },
150 });
151 }
152 return page.locator(`aria-ref=${nativeRef}`);
153 }
154 // Playwright's strict locators reject ambiguous controls rather than acting on row one.
155 return page.locator(selector);
156 };
157
158 export type PointerReporter = (
159 point: { readonly x: number; readonly y: number },
160 phase: "move" | "click",
161 ) => Promise<void>;
162 const noPointer: PointerReporter = async () => {};
163
164 // Locator.scrollIntoViewIfNeeded resolves its node once and fails when the page replaces it; the
165 // input actions after it re-resolve, so the scroll does too, within the same timeout.
166 const DETACHED = /^locator\.scrollIntoViewIfNeeded: Element is not attached to the DOM(?:\n|$)/;
167
168 const scrollIntoView = async (locator: Locator, timeout: number) => {
169 const deadline = Date.now() + timeout;
170 for (;;) {
171 try {
172 return await locator.scrollIntoViewIfNeeded({ timeout: Math.max(1, deadline - Date.now()) });
173 } catch (cause) {
174 if (!(cause instanceof Error && DETACHED.test(cause.message)) || Date.now() >= deadline) {
175 throw cause;
176 }
177 }
178 }
179 };
180
181 const targetPoint = async (
182 page: Page,
183 locator: Locator | null,
184 input: { readonly x?: number | undefined; readonly y?: number | undefined },
185 timeout: () => number,
186 ) => {
187 if (locator === null) return { x: input.x ?? 0, y: input.y ?? 0 };
188 await scrollIntoView(locator, timeout());
189 const box = await locator.boundingBox({ timeout: timeout() });
190 if (box) return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
191 const viewport = page.viewportSize();
192 return { x: (viewport?.width ?? 0) / 2, y: (viewport?.height ?? 0) / 2 };
193 };
194
195 const SNAPSHOT_SCRIPT = `(() => {
196 return {
197 url: location.href,
198 title: document.title,
199 loading: document.readyState !== "complete",
200 visibleText: document.body?.innerText || "",
201 interactiveElements: [],
202 };
203 })()`;
204
205 const MASKED_VALUE = "••••••••";
206
207 /**
208 * A `data:` URL carries its content, which can be a whole file, and URLs and diagnostics
209 * reach tool results and provider logs. Keeps only the media type: the payload can hold any
210 * character and parameters are page-chosen too.
211 */
212 export const withoutDataPayload = (url: string) => {
213 // Read the scheme as a browser would: it ignores leading controls and spaces, and tabs and
214 // newlines anywhere. A snapshot's YAML value may be quoted and escaped; an escape that isn't a
215 // tab or newline still decodes to something that can't start a scheme.
216 const value = url
217 .replace(/^["']/, "")
218 .replace(/\\(x[\da-f]{2}|u[\da-f]{4}|.)/gi, (_escape, code: string) =>
219 code.length > 1
220 ? String.fromCharCode(Number.parseInt(code.slice(1), 16))
221 : ({ t: "\t", n: "\n", r: "\r", '"': '"', "\\": "\\" }[code] ?? "\u0001"),
222 )
223 .replace(/[\t\n\r]/g, "")
224 .replace(/^[\u0000-\u0020]+/, "");
225 const parsed = /^data:([^,;]*)/i.exec(value);
226 if (!parsed) return url;
227 const type = parsed[1]!.trim();
228 const mediaType = /^[\w.+-]{1,64}\/[\w.+-]{1,64}$/.test(type) ? type : "";
229 return `data:${mediaType},…`;
230 };
231
232 /**
233 * Chromium's own console messages, which carry no arguments, quote `data:` URLs whole, as in a
234 * blocked navigation or a CSP violation. Everything after the first one goes, since Chromium
235 * accepts any type and a payload can hold any character. What a page logs itself is its own
236 * text, like anything else it shows, so it stays.
237 */
238 export const consoleText = (message: ConsoleMessage) =>
239 message.args().length > 0
240 ? message.text()
241 : message.text().replace(/(?<![a-z\d])data:[^,;]*[,;][\s\S]*/i, withoutDataPayload);
242
243 // Split on LF alone: a value keeps any U+2028 that a multiline regex would treat as a line end.
244 const withoutTreeDataPayloads = (tree: string) =>
245 tree
246 .split("\n")
247 .map((line) => {
248 const url = /^(\s*- \/url: )(.*)$/s.exec(line);
249 return url ? url[1] + withoutDataPayload(url[2]!) : line;
250 })
251 .join("\n");
252
253 // The broker waits 15s for the host; a snapshot, retry included, answers well before.
254 const SNAPSHOT_TIMEOUT_MS = 10_000;
255 const FRAME_ANSWER_MS = 1_000;
256
257 // Include password and CSS-masked text in open shadow roots, normalized like accessible text.
258 const DOTTED_SCRIPT = `(() => {
259 const values = [];
260 const visit = (root) => {
261 for (const element of root.querySelectorAll("*")) {
262 if (element.shadowRoot) visit(element.shadowRoot);
263 const field = element.localName === "input" || element.localName === "textarea";
264 if (!field && !element.isContentEditable) continue;
265 const dots = getComputedStyle(element).getPropertyValue("-webkit-text-security");
266 if (element.type !== "password" && (dots === "none" || dots === "")) continue;
267 // Raw as page text shows it, and normalized as the tree does.
268 for (const value of field ? [element.value, element.defaultValue] : [element.textContent ?? ""])
269 values.push(value, value.replace(/[\\u200b\\u00ad]/g, "").trim().replace(/\\s+/g, " "));
270 }
271 };
272 visit(document);
273 return [...new Set(values.filter((value) => value.trim()))];
274 })()`;
275
276 const pageChanged = () =>
277 new ServerBrowserOperationError({
278 tag: "PreviewAutomationExecutionError",
279 message: "The page changed while capturing its snapshot. Take another snapshot.",
280 });
281
282 type SnapshotStage = typeof PreviewAutomationSnapshotStage.Type;
283
284 /**
285 * Running stages are counted, since a retry can overlap the previous attempt's unanswered
286 * screenshot. A stage whose own timeout fired has stopped running but is still the cause.
287 */
288 interface SnapshotStages {
289 readonly running: Map<SnapshotStage, number>;
290 readonly timedOut: Set<SnapshotStage>;
291 }
292
293 const snapshotTimedOut = (stages?: SnapshotStages) =>
294 new ServerBrowserOperationError({
295 tag: "PreviewAutomationTimeoutError",
296 message: `Snapshot capture timed out after ${SNAPSHOT_TIMEOUT_MS}ms. Try the snapshot again.`,
297 detail: {
298 timeoutMs: SNAPSHOT_TIMEOUT_MS,
299 pendingStages: stages ? [...new Set([...stages.timedOut, ...stages.running.keys()])] : [],
300 },
301 });
302
303 const frameUnanswered = () =>
304 new ServerBrowserOperationError({
305 tag: "PreviewAutomationTimeoutError",
306 message: `A frame on the page didn't answer within ${FRAME_ANSWER_MS}ms, so its content couldn't be checked for hidden values. Try the snapshot again.`,
307 detail: { timeoutMs: FRAME_ANSWER_MS, pendingStages: ["frames"] },
308 });
309
310 const isTimeout = (cause: unknown) => cause instanceof Error && cause.name === "TimeoutError";
311
312 const VALUE_TOO_LONG = "A password field holds a value too long to hide, so no snapshot was taken.";
313
314 /** Reuse one CDP session per out-of-process frame across snapshots. */
315 const frameSessions = new WeakMap<Frame, Promise<CDPSession>>();
316
317 /** Returns undefined on failure or frame timeout. */
318 const answered = <A>(promise: Promise<A>) => {
319 let timer: ReturnType<typeof setTimeout> | undefined;
320 const late = new Promise<undefined>((resolve) => {
321 timer = setTimeout(() => resolve(undefined), FRAME_ANSWER_MS);
322 });
323 return Promise.race([promise.catch(() => undefined), late]).finally(() => clearTimeout(timer));
324 };
325
326 /** Isolated worlds prevent spoofing; desktop out-of-process frames use page worlds. Missing own-frame reads or changed frames fail; other missing reads return undefined. */
327 const openReaders = async (page: Page, cdp: CDPSession) => {
328 const frames = page.frames();
329 const owners = new Map<Frame, CDPSession | null | undefined>(
330 await Promise.all(
331 frames.map(async (frame) => {
332 if (frame === page.mainFrame()) return [frame, cdp] as const;
333 const session = frameSessions.get(frame) ?? page.context().newCDPSession(frame);
334 frameSessions.set(frame, session);
335 try {
336 return [frame, await session] as const;
337 } catch (cause) {
338 frameSessions.delete(frame);
339 // CdpRelay supports only the page; other refusals fall back to the parent session, failing if the frame is absent.
340 const desktop = String(cause).includes("Only this preview tab's page can be attached");
341 return [frame, desktop ? null : undefined] as const;
342 }
343 }),
344 ),
345 );
346 const ownerOf = (frame: Frame): CDPSession | null => {
347 const owner = owners.get(frame);
348 if (owner !== undefined) return owner;
349 const parent = frame.parentFrame();
350 return parent ? ownerOf(parent) : null;
351 };
352 const sessionReader = (frame: Frame, session: CDPSession) => {
353 const owned = frames.filter((each) => ownerOf(each) === session);
354 const open = async () => {
355 const { frameTree } = await session.send("Page.getFrameTree");
356 const ids = (node: typeof frameTree): Array<string> => [
357 node.frame.id,
358 ...(node.childFrames ?? []).flatMap(ids),
359 ];
360 // A frame Playwright knows but this session doesn't would go unread.
361 if (ids(frameTree).length !== owned.length) throw pageChanged();
362 return Promise.all(
363 ids(frameTree).map(async (frameId) => {
364 const world = await session.send("Page.createIsolatedWorld", {
365 frameId,
366 worldName: "t3-preview",
367 });
368 return world.executionContextId;
369 }),
370 );
371 };
372 let contexts: ReturnType<typeof open> | undefined;
373 const read = async () => {
374 const values = (contexts ??= open()).then((ids) =>
375 Promise.all(
376 ids.map(async (contextId) => {
377 const { result, exceptionDetails } = await session.send("Runtime.evaluate", {
378 contextId,
379 expression: DOTTED_SCRIPT,
380 returnByValue: true,
381 });
382 if (exceptionDetails) throw pageChanged();
383 return result.value as Array<string>;
384 }),
385 ),
386 );
387 try {
388 return (await values).flat();
389 } catch (cause) {
390 // The frame may have moved to another process; the next snapshot opens a new session.
391 if (session !== cdp) {
392 frameSessions.delete(frame);
393 void session.detach().catch(constVoid);
394 }
395 throw cause;
396 }
397 };
398 return { frames: owned, read };
399 };
400 const readers = [
401 sessionReader(page.mainFrame(), cdp),
402 ...frames.flatMap((frame) => {
403 const owner = owners.get(frame);
404 if (owner) return owner === cdp ? [] : [sessionReader(frame, owner)];
405 const own = () => frame.evaluate(DOTTED_SCRIPT) as Promise<Array<string>>;
406 return ownerOf(frame) === null ? [{ frames: [frame], read: own }] : [];
407 }),
408 ];
409 return {
410 readers,
411 read: async (before?: ReadonlyArray<unknown>) => {
412 const now = page.frames();
413 if (now.length !== frames.length || now.some((frame, i) => frame !== frames[i])) {
414 throw pageChanged();
415 }
416 return Promise.all(
417 readers.map((reader, i) => {
418 if (i === 0) return reader.read().catch(() => Promise.reject(pageChanged()));
419 return before && !before[i] ? undefined : answered(reader.read());
420 }),
421 );
422 },
423 };
424 };
425
426 /** Short values need word boundaries to avoid masking unrelated text. */
427 const dottedPattern = (values: ReadonlyArray<string>) => {
428 const sources = values
429 .flatMap((value) => {
430 const json = JSON.stringify(value).slice(1, -1);
431 // Playwright writes control characters like JSON does, with \xNN for the unnamed ones.
432 const quoted = value.replace(/[\\"\p{Cc}]/gu, (char) =>
433 '\\"\b\f\n\r\t'.includes(char)
434 ? JSON.stringify(char).slice(1, -1)
435 : `\\x${char.charCodeAt(0).toString(16).padStart(2, "0")}`,
436 );
437 const forms = [value, json, quoted].flatMap((form) => [form, form.replaceAll("'", "''")]);
438 return [...new Set(forms)].map((form) => ({ form, word: value.length < 4 }));
439 })
440 // Longest first, so a form inside another never leaves the rest showing.
441 .toSorted((a, b) => b.form.length - a.form.length)
442 .map(({ form, word }) => {
443 const literal = form.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
444 return word ? `(?<![\\p{L}\\p{N}_])${literal}(?![\\p{L}\\p{N}_])` : literal;
445 });
446 if (sources.length === 0) return undefined;
447 const pattern = new RegExp(sources.join("|"), "gu");
448 try {
449 // Force regex compilation for one-byte and two-byte strings now; size errors name every value.
450 pattern.test("");
451 pattern.test("\u2022");
452 } catch {
453 throw new ServerBrowserOperationError({
454 tag: "PreviewAutomationExecutionError",
455 message: VALUE_TOO_LONG,
456 });
457 }
458 return pattern;
459 };
460
461 // Playwright single-quotes keys containing ': ' and leaves /.../ names unquoted. Recognize only its attributes so name text cannot spoof them.
462 const TREE_LINE =
463 /^(\s*- '?[^\s"':]+)( "(?:[^"\\]|\\.)*"| \/(?:.*?\/)?)?((?: \[(?:active|checked(?:=mixed)?|disabled|expanded|level=\d+|pressed(?:=mixed)?|selected|ref=(?:f\d+)?e\d+|cursor=pointer|box=-?\d+,-?\d+,\d+,\d+)\])*'?)(:.*)?$/;
464
465 /** Preserve roles and attributes so refs work; unquoted /.../ names can make matches cross boundaries. */
466 const maskTree = (tree: string, dotted: RegExp) =>
467 tree
468 .split("\n")
469 .map((line) => {
470 const node = TREE_LINE.exec(line);
471 const name = node?.[1]?.length ?? 0;
472 const attributes = name + (node?.[2]?.length ?? 0);
473 const end = attributes + (node?.[3]?.length ?? 0);
474 const kept = (from: number, to: number) => to <= name || (from >= attributes && to <= end);
475 return line.replace(dotted, (match: string, offset: number) =>
476 kept(offset, offset + match.length) ? match : MASKED_VALUE,
477 );
478 })
479 .join("\n");
480
481 /** Chromium sizes the capture, so the reported size is read from the PNG header. */
482 const pngSize = (base64: string) => {
483 const header = Buffer.from(base64.slice(0, 32), "base64");
484 return { width: header.readUInt32BE(16), height: header.readUInt32BE(20) };
485 };
486
487 export const jpegSize = (base64: string) => {
488 const bytes = Buffer.from(base64, "base64");
489 if (bytes.length < 4 || bytes.readUInt16BE(0) !== 0xffd8)
490 throw new Error("Invalid JPEG screenshot.");
491 let offset = 2;
492 while (offset < bytes.length) {
493 if (bytes[offset++] !== 0xff) throw new Error("Invalid JPEG screenshot.");
494 while (bytes[offset] === 0xff) offset++;
495 const marker = bytes[offset++];
496 if (marker === undefined || marker === 0xd9 || marker === 0xda) break;
497 if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) continue;
498 if (offset + 2 > bytes.length) break;
499 const length = bytes.readUInt16BE(offset);
500 if (length < 2 || offset + length > bytes.length) break;
501 if (marker >= 0xc0 && marker <= 0xcf && ![0xc4, 0xc8, 0xcc].includes(marker)) {
502 if (length < 7) break;
503 const height = bytes.readUInt16BE(offset + 3);
504 const width = bytes.readUInt16BE(offset + 5);
505 if (width && height) return { width, height };
506 break;
507 }
508 offset += length;
509 }
510 throw new Error("Invalid JPEG screenshot.");
511 };
512
513 const scaledSize = (size: { readonly width: number; readonly height: number }, scale: number) =>
514 scale < 1
515 ? { width: Math.round(size.width * scale), height: Math.round(size.height * scale) }
516 : size;
517
518 /**
519 * A page's display ratio and CSS viewport, read in an isolated world since page script can
520 * reassign `devicePixelRatio`, `innerWidth`, and `innerHeight` in its own, and its page zoom.
521 */
522 const readPageMetrics = async (cdp: CDPSession) => {
523 const { frameTree } = await cdp.send("Page.getFrameTree");
524 const { executionContextId } = await cdp.send("Page.createIsolatedWorld", {
525 frameId: frameTree.frame.id,
526 worldName: "t3-preview-metrics",
527 });
528 let { cssVisualViewport } = await cdp.send("Page.getLayoutMetrics");
529 // Desktop zoom applies asynchronously; bracket the CSS metrics to reject a torn read.
530 for (let attempt = 0; attempt < 3; attempt += 1) {
531 const zoom = cssVisualViewport.zoom ?? 1;
532 const { result } = await cdp.send("Runtime.evaluate", {
533 expression: "({ ratio: devicePixelRatio, width: innerWidth, height: innerHeight })",
534 contextId: executionContextId,
535 returnByValue: true,
536 });
537 ({ cssVisualViewport } = await cdp.send("Page.getLayoutMetrics"));
538 if ((cssVisualViewport.zoom ?? 1) === zoom) {
539 const { ratio, width, height } = result.value as {
540 readonly ratio: number;
541 readonly width: number;
542 readonly height: number;
543 };
544 return { ratio, width, height, zoom };
545 }
546 }
547 throw new Error(
548 "The page's zoom kept changing while the snapshot read its size. Try the snapshot again.",
549 );
550 };
551
552 // Never pass a clip scale: Chromium mirrors the top-left of the page into a `backdrop-filter`
553 // in any capture it shrinks.
554 export const captureViewport = async (
555 cdp: CDPSession,
556 options: { readonly format: "png" | "jpeg"; readonly quality?: number },
557 ) => {
558 const capture = cdp.send("Page.captureScreenshot", {
559 format: options.format,
560 ...(options.quality === undefined ? {} : { quality: options.quality }),
561 });
562 let timer: ReturnType<typeof setTimeout> | undefined;
563 const expired = new Promise<never>((_, reject) => {
564 timer = setTimeout(
565 () =>
566 reject(
567 new ServerBrowserOperationError({
568 tag: "PreviewAutomationExecutionError",
569 message:
570 "The browser is not producing screenshot frames. For a desktop tab, show or restore the T3 Code window and try again.",
571 }),
572 ),
573 SCREENSHOT_TIMEOUT_MS,
574 );
575 });
576 // Hidden native guests can leave CDP captures pending forever. This deadline also
577 // releases a viewer's still-capture lock, which an agent snapshot may be waiting for.
578 const { data } = await Promise.race([capture, expired]).finally(() => clearTimeout(timer));
579 return data;
580 };
581
582 export const captureFinalRecordingFrame = (cdp: CDPSession, quality: number) =>
583 answered(
584 cdp
585 .send("Runtime.evaluate", {
586 // Capture the destination of finite transitions, not their initially transparent frame.
587 expression: `(async () => {
588 const paint = () => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)));
589 await paint();
590 await Promise.allSettled(document.getAnimations()
591 .filter(animation => animation.playState === "running" && animation.effect?.getComputedTiming().iterations !== Infinity)
592 .map(animation => animation.finished));
593 await paint();
594 })()`,
595 awaitPromise: true,
596 })
597 .then(() => captureViewport(cdp, { format: "jpeg", quality })),
598 );
599
600 /** Runs in the page's own isolated world, as the server has no image library. */
601 const SHRINK_PNG = `async function (png, width, height) {
602 const bytes = Uint8Array.from(atob(png), (char) => char.charCodeAt(0));
603 const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/png" }));
604 // Namespaced, as an XML document's createElement makes a plain element with no getContext.
605 const canvas = document.createElementNS("http://www.w3.org/1999/xhtml", "canvas");
606 canvas.width = width;
607 canvas.height = height;
608 const context = canvas.getContext("2d");
609 context.imageSmoothingQuality = "high";
610 context.drawImage(bitmap, 0, 0, width, height);
611 bitmap.close();
612 return canvas.toDataURL("image/png").slice("data:image/png;base64,".length);
613 }`;
614
615 const shrinkPng = async (cdp: CDPSession, png: string, scale: number) => {
616 const size = pngSize(png);
617 const { frameTree } = await cdp.send("Page.getFrameTree");
618 const { executionContextId } = await cdp.send("Page.createIsolatedWorld", {
619 frameId: frameTree.frame.id,
620 worldName: "t3-preview-screenshot",
621 });
622 const { result, exceptionDetails } = await cdp.send("Runtime.callFunctionOn", {
623 functionDeclaration: SHRINK_PNG,
624 executionContextId,
625 arguments: [
626 { value: png },
627 { value: Math.round(size.width * scale) },
628 { value: Math.round(size.height * scale) },
629 ],
630 awaitPromise: true,
631 returnByValue: true,
632 });
633 if (exceptionDetails) throw new Error("The snapshot could not shrink its screenshot.");
634 return result.value as string;
635 };
636
637 type SnapshotInput = {
638 readonly page: Page;
639 readonly cdp: CDPSession;
640 /** Headless pages render at the scale the server set; a desktop page at its display's. */
641 readonly renderScale: number | "display";
642 readonly fullScreenshot?: boolean | undefined;
643 readonly consoleEntries: ReadonlyArray<PreviewAutomationConsoleEntry>;
644 readonly networkEntries: ReadonlyArray<PreviewAutomationNetworkEntry>;
645 readonly actionTimeline: PreviewAutomationSnapshot["actionTimeline"];
646 };
647
648 /** The scale a page renders at and its viewport in device-independent pixels. */
649 const renderingOf = async (input: SnapshotInput) => {
650 if (input.renderScale !== "display") {
651 const viewport = input.page.viewportSize() ?? { width: 1280, height: 800 };
652 return { renderScale: input.renderScale, viewport };
653 }
654 // The display ratio a desktop page reports includes its page zoom, and its sizes are CSS pixels.
655 const page = await readPageMetrics(input.cdp);
656 return {
657 renderScale: page.ratio / page.zoom,
658 viewport: {
659 width: Math.round(page.width * page.zoom),
660 height: Math.round(page.height * page.zoom),
661 },
662 };
663 };
664
665 const capture = async (
666 input: SnapshotInput,
667 deadline: number,
668 stages: SnapshotStages,
669 metadataOnly: boolean,
670 ) => {
671 const { running } = stages;
672 const track = async <A>(stage: SnapshotStage, operation: () => Promise<A>) => {
673 running.set(stage, (running.get(stage) ?? 0) + 1);
674 try {
675 const result = await operation();
676 stages.timedOut.delete(stage);
677 return result;
678 } catch (cause) {
679 if (isTimeout(cause)) stages.timedOut.add(stage);
680 throw cause;
681 } finally {
682 const count = (running.get(stage) ?? 1) - 1;
683 if (count === 0) running.delete(stage);
684 else running.set(stage, count);
685 }
686 };
687 const { renderScale, viewport } = await track("metrics", () => renderingOf(input));
688 // A late metrics reply must not begin a capture after viewers have resumed.
689 if (Date.now() >= deadline) throw snapshotTimedOut();
690 const scale = Math.min(1, MAX_SCREENSHOT_WIDTH / (viewport.width * renderScale));
691 const state = refsFor(input.page);
692 invalidateRefs(input.page);
693 const generation = state.generation;
694 const { readers, read } = await track("frames", () => openReaders(input.page, input.cdp));
695 const before = await track("frames", () => read());
696 // Bracket the tree with value reads to detect changes; await tree completion before retrying failed captures.
697 const pageAndScreenshot = Promise.all([
698 track(
699 "page state",
700 () =>
701 input.page.evaluate(SNAPSHOT_SCRIPT) as Promise<
702 Pick<
703 PreviewAutomationSnapshot,
704 "url" | "title" | "loading" | "visibleText" | "interactiveElements"
705 >
706 >,
707 ),
708 // A cheap real frame still sizes the screenshot; computing it from metrics drifted from the PNG.
709 track("screenshot", () =>
710 metadataOnly
711 ? captureViewport(input.cdp, { format: "jpeg", quality: 10 }).then((data) => ({
712 data,
713 full: undefined,
714 }))
715 : captureViewport(input.cdp, { format: "png" }).then(async (full) =>
716 scale < 1
717 ? {
718 data: await shrinkPng(input.cdp, full, scale),
719 full: input.fullScreenshot ? full : undefined,
720 }
721 : { data: full, full: undefined },
722 ),
723 ),
724 ]);
725 void pageAndScreenshot.catch(constVoid);
726 // Leaves time for the second read; a frame that never loads holds the tree until then.
727 const tree = await track("accessibility tree", () =>
728 input.page.ariaSnapshot({
729 mode: "ai",
730 boxes: true,
731 timeout: Math.max(1, deadline - Date.now() - 2_000),
732 }),
733 );
734 const after = await track("frames", () => read(before));
735 // A value that changed meanwhile may be in the tree under neither reading.
736 if (after.some((values, i) => values && JSON.stringify(values) !== JSON.stringify(before[i])))
737 throw pageChanged();
738 const dotted = dottedPattern(after.flatMap((values) => values ?? []));
739 // Unread frames can expose values only if their content appears in the tree.
740 const unread = readers.flatMap((reader, i) => (after[i] ? [] : reader.frames));
741 const filled = /^\s*- iframe (?:\[active\] )?\[ref=(\S+?)\].*:$/gm;
742 for (const [, ref] of unread.length ? tree.matchAll(filled) : []) {
743 const iframe = await track("frames", () =>
744 input.page.locator(`aria-ref=${ref}`).elementHandle({ timeout: 2_000 }),
745 );
746 if (!iframe) throw pageChanged();
747 const frame = await track("frames", () => iframe.contentFrame());
748 void iframe.dispose().catch(constVoid);
749 if (frame && unread.includes(frame)) throw frameUnanswered();
750 }
751 const [page, { full, data }] = await pageAndScreenshot;
752 if (state.generation !== generation) throw pageChanged();
753 const mask = (text: string) => (dotted ? text.replace(dotted, MASKED_VALUE) : text);
754 // Redacted before the cut, so a payload crossing it can't leave its start showing, and before
755 // masking, which could hide the `data:` scheme behind a password value's dots.
756 const redacted = withoutTreeDataPayloads(tree);
757 const accessibilityTree = (dotted ? maskTree(redacted, dotted) : redacted)
758 .slice(0, MAX_VISIBLE_TEXT_LENGTH)
759 .replace(/\[ref=((?:f\d+)?e\d+)\]/g, (_match, nativeRef: string) => {
760 const ref = `t3-${generation}-${nativeRef}`;
761 state.refs.set(ref, nativeRef);
762 return `[ref=${ref}]`;
763 });
764 return {
765 ...page,
766 url: withoutDataPayload(page.url),
767 // Mask text before truncating so a value crossing the cut cannot leak its prefix.
768 visibleText: mask(page.visibleText).slice(0, MAX_VISIBLE_TEXT_LENGTH),
769 accessibilityTree,
770 consoleEntries: [...input.consoleEntries],
771 networkEntries: [...input.networkEntries],
772 actionTimeline: [...input.actionTimeline],
773 // Reports the size the PNG path would have shrunk to.
774 screenshot: metadataOnly
775 ? scaledSize(jpegSize(data), scale)
776 : { mimeType: "image/png" as const, data, ...pngSize(data) },
777 ...(full !== undefined ? { fullScreenshot: { data: full, ...pngSize(full) } } : {}),
778 };
779 };
780
781 /** Snapshots begun per page; observers' snapshots aren't queued, so two can overlap. */
782 const snapshotTurns = new WeakMap<Page, number>();
783
784 const runSnapshot = async (input: SnapshotInput, metadataOnly: boolean) => {
785 const deadline = Date.now() + SNAPSHOT_TIMEOUT_MS;
786 const stages: SnapshotStages = { running: new Map(), timedOut: new Set() };
787 let timer: ReturnType<typeof setTimeout> | undefined;
788 const expired = new Promise<never>((_, reject) => {
789 timer = setTimeout(() => reject(snapshotTimedOut(stages)), SNAPSHOT_TIMEOUT_MS);
790 });
791 const turn = (snapshotTurns.get(input.page) ?? 0) + 1;
792 snapshotTurns.set(input.page, turn);
793 // Retry only before the deadline and the next snapshot, whose refs a later retry would revoke.
794 const captured = capture(input, deadline, stages, metadataOnly).catch((cause: unknown) =>
795 Date.now() < deadline &&
796 snapshotTurns.get(input.page) === turn &&
797 !(cause instanceof Error && cause.message === VALUE_TOO_LONG)
798 ? capture(input, deadline, stages, metadataOnly)
799 : Promise.reject(cause),
800 );
801 try {
802 return await Promise.race([captured, expired]);
803 } catch (cause) {
804 throw isTimeout(cause) ? snapshotTimedOut(stages) : cause;
805 } finally {
806 clearTimeout(timer);
807 }
808 };
809
810 export const snapshot = (input: SnapshotInput) =>
811 runSnapshot(input, false) as Promise<PreviewAutomationSnapshot>;
812
813 export const snapshotMetadata = (input: SnapshotInput) =>
814 runSnapshot(input, true) as Promise<PreviewAutomationSnapshotMetadata>;
815
816 /** Return when a dialog opens; waiting for the action to finish would block until dialog resolution. */
817 export const click = async (
818 page: Page,
819 input: PreviewAutomationClickInput,
820 pointer: PointerReporter = noPointer,
821 ): Promise<{ readonly x: number; readonly y: number }> => {
822 // One deadline across steps keeps the reply inside the broker's grace; Playwright reads 0 as none.
823 const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS);
824 const remaining = () => Math.max(1, deadline - Date.now());
825 const locator = targetLocator(page, input);
826 try {
827 return await clickTarget(page, locator, input, remaining, pointer);
828 } catch (cause) {
829 throw locator === null ? cause : await namedTargetTimeout(locator, input, cause);
830 }
831 };
832
833 /**
834 * Playwright's timeout names neither the target nor why it waited. Counting only after the
835 * wait failed keeps auto-wait intact, but the count describes the page after the timeout, not
836 * what the click did: Playwright can dispatch the click and then time out waiting for the
837 * navigation it started.
838 */
839 const namedTargetTimeout = async (
840 locator: Locator,
841 input: { readonly locator?: string | undefined; readonly selector?: string | undefined },
842 cause: unknown,
843 ) => {
844 if (!(cause instanceof Error && cause.name === "TimeoutError")) return cause;
845 const selector = JSON.stringify(input.locator ?? input.selector);
846 // The warning leads and the target is capped, so a reply cut short still carries the warning.
847 const target =
848 selector.length > MAX_TARGET_CHARS ? `${selector.slice(0, MAX_TARGET_CHARS)}...` : selector;
849 // The host's reply must still beat the broker's short grace after the operation timeout.
850 let timer: ReturnType<typeof setTimeout> | undefined;
851 const matches = await Promise.race([
852 locator.count().catch(() => undefined),
853 new Promise<undefined>((resolve) => {
854 timer = setTimeout(() => resolve(undefined), TARGET_COUNT_MS);
855 }),
856 ]).finally(() => clearTimeout(timer));
857 const reason =
858 matches === undefined
859 ? `The click may have happened; check the page before retrying. Whether ${target} still matches an element is unknown.`
860 : matches === 0
861 ? `The click may have happened and removed its target or navigated; check the page before retrying. No element matches ${target} after the timeout; take a fresh snapshot and use its locator.`
862 : `The click may have happened; check the page before retrying. ${target} still matches an element after the timeout.`;
863 const message = withActionabilityReason(reason, cause);
864 return new ServerBrowserOperationError({
865 tag: "PreviewAutomationTimeoutError",
866 message,
867 detail: { reason: message },
868 cause,
869 });
870 };
871
872 const clickTarget = async (
873 page: Page,
874 locator: Locator | null,
875 input: PreviewAutomationClickInput,
876 timeout: () => number,
877 pointer: PointerReporter,
878 ) => {
879 const point = await targetPoint(page, locator, input, timeout);
880 await pointer(point, "click");
881 const options = { button: input.button ?? "left", clickCount: input.clickCount ?? 1 } as const;
882 const clicked =
883 locator === null
884 ? page.mouse.click(point.x, point.y, options)
885 : locator.click({ ...options, timeout: timeout() });
886 let onDialog = constVoid;
887 const dialogOpened = new Promise<"dialog">((resolve) => {
888 onDialog = () => resolve("dialog");
889 page.once("dialog", onDialog);
890 });
891 try {
892 if ((await Promise.race([clicked, dialogOpened])) === "dialog") void clicked.catch(constVoid);
893 } finally {
894 page.off("dialog", onDialog);
895 }
896 return point;
897 };
898
899 type FocusCandidate = {
900 readonly tagName: string;
901 readonly type?: string;
902 readonly disabled?: boolean;
903 readonly readOnly?: boolean;
904 readonly isContentEditable?: boolean;
905 readonly shadowRoot?: { readonly activeElement: FocusCandidate | null } | null;
906 readonly focus?: () => void;
907 getRootNode(): { readonly activeElement?: FocusCandidate | null };
908 contains(other: FocusCandidate): boolean;
909 };
910
911 // Serialized into the page's main world, so it must not reference module scope.
912 const focusTextTarget = (target: FocusCandidate | null) => {
913 let element =
914 target ??
915 (globalThis as unknown as { document: { activeElement: FocusCandidate | null } }).document
916 .activeElement;
917 if (target === null) {
918 while (element?.shadowRoot?.activeElement) element = element.shadowRoot.activeElement;
919 // Focus inside a frame cannot be checked from here, and an editable frame would pass as contentEditable.
920 if (element && ["IFRAME", "FRAME", "FENCEDFRAME", "OBJECT", "EMBED"].includes(element.tagName))
921 return false;
922 }
923 if (element == null) return false;
924 // Require an enabled, focused text target to avoid losing input or sending it to the previous field.
925 const nonText = [
926 "button",
927 "checkbox",
928 "color",
929 "file",
930 "hidden",
931 "image",
932 "radio",
933 "range",
934 "reset",
935 "submit",
936 ];
937 const textControl =
938 element.tagName === "TEXTAREA" ||
939 (element.tagName === "INPUT" && !nonText.includes(element.type ?? "text"));
940 if (!(textControl || element.isContentEditable) || element.disabled || element.readOnly) {
941 return false;
942 }
943 element.focus?.();
944 const active = element.getRootNode().activeElement;
945 return active != null && (active === element || element.contains(active));
946 };
947
948 // A `*:focus` locator needs Playwright's utility world, which some desktop tabs never get.
949 const focusedTextField = async (page: Page, timeout: number) => {
950 let timer: ReturnType<typeof setTimeout> | undefined;
951 const expired = new Promise<never>((_, reject) => {
952 timer = setTimeout(
953 () =>
954 reject(
955 new ServerBrowserOperationError({
956 tag: "PreviewAutomationTimeoutError",
957 message: `The page did not report its focused element within ${timeout}ms, so no text was typed.`,
958 detail: { timeoutMs: timeout },
959 }),
960 ),
961 timeout,
962 );
963 });
964 return await Promise.race([page.evaluate(focusTextTarget, null), expired]).finally(() =>
965 clearTimeout(timer),
966 );
967 };
968
969 export const type = async (page: Page, input: PreviewAutomationTypeInput) => {
970 const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
971 const locator = targetLocator(page, input);
972 if (locator !== null && input.clear) {
973 await locator.fill(input.text, { timeout });
974 return;
975 }
976 const focused =
977 locator === null
978 ? await focusedTextField(page, timeout)
979 : await locator.evaluate(focusTextTarget, undefined, { timeout });
980 if (focused !== true) {
981 throw new ServerBrowserOperationError({
982 tag: "PreviewAutomationTargetNotEditableError",
983 message: "The target is not an enabled text field, so no text was typed.",
984 });
985 }
986 if (input.clear) {
987 await page.keyboard.press("ControlOrMeta+A");
988 await page.keyboard.press("Delete");
989 }
990 await page.keyboard.insertText(input.text);
991 };
992
993 export const hover = async (
994 page: Page,
995 input: PreviewAutomationHoverInput,
996 pointer: PointerReporter = noPointer,
997 ) => {
998 const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS);
999 const remaining = () => Math.max(1, deadline - Date.now());
1000 const locator = targetLocator(page, input);
1001 await pointer(await targetPoint(page, locator, input, remaining), "move");
1002 if (locator === null) {
1003 await page.mouse.move(input.x ?? 0, input.y ?? 0);
1004 return;
1005 }
1006 await locator.hover({ timeout: remaining() });
1007 };
1008
1009 export const select = async (
1010 page: Page,
1011 input: PreviewAutomationSelectInput,
1012 ): Promise<PreviewAutomationSelectResult> => {
1013 const locator = targetLocator(page, input)!;
1014 const timeout = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
1015 const options = await locator.evaluate(
1016 (element) =>
1017 element.tagName === "SELECT"
1018 ? Array.from(
1019 (element as unknown as { options: ArrayLike<{ value: string; label: string }> })
1020 .options,
1021 ({ value, label }) => ({ value, label: label.trim() }),
1022 )
1023 : null,
1024 undefined,
1025 { timeout },
1026 );
1027 const values =
1028 options &&
1029 input.values.map(
1030 (entry) =>
1031 (
1032 options.find((option) => option.value === entry) ??
1033 options.find((option) => option.label === entry.trim())
1034 )?.value ?? entry,
1035 );
1036 if (values === null)
1037 throw new ServerBrowserOperationError({
1038 tag: "PreviewAutomationTargetNotEditableError",
1039 message: "This element is not a <select>. Click a custom dropdown, then click its option.",
1040 });
1041 return { selected: await locator.selectOption(values, { timeout }) };
1042 };
1043
1044 export const drag = async (
1045 page: Page,
1046 input: PreviewAutomationDragInput,
1047 pointer: PointerReporter = noPointer,
1048 ) => {
1049 const deadline = Date.now() + (input.timeoutMs ?? DEFAULT_TIMEOUT_MS);
1050 const remaining = () => Math.max(1, deadline - Date.now());
1051 const source = targetLocator(page, { locator: input.source })!;
1052 const target = targetLocator(page, { locator: input.target })!;
1053 await pointer(await targetPoint(page, source, {}, remaining), "move");
1054 // Observe drag rejection immediately; an unhandled rejection during cursor movement exits the server.
1055 // Chromium delivers a single move before dragstart; steps give the page mousemoves during the drag.
1056 const dragFailure = source.dragTo(target, { timeout: remaining(), steps: 5 }).then(
1057 () => null,
1058 (error: unknown) => ({ error }),
1059 );
1060 const end = await target.boundingBox({ timeout: remaining() }).catch(() => null);
1061 if (end) await pointer({ x: end.x + end.width / 2, y: end.y + end.height / 2 }, "move");
1062 const failure = await dragFailure;
1063 if (failure) throw failure.error;
1064 };
1065
1066 export const setInputFiles = async (page: Page, input: PreviewAutomationUploadInput) => {
1067 const locator = targetLocator(page, input);
1068 if (locator === null) return false;
1069 await locator.setInputFiles([...input.paths], { timeout: input.timeoutMs ?? DEFAULT_TIMEOUT_MS });
1070 return true;
1071 };
1072
1073 export const press = async (page: Page, input: PreviewAutomationPressInput) => {
1074 await page.keyboard.press([...(input.modifiers ?? []), input.key].join("+"));
1075 };
1076
1077 export const scroll = async (page: Page, input: PreviewAutomationScrollInput) => {
1078 const delta = [input.deltaX ?? 0, input.deltaY ?? 0] as const;
1079 const locator = targetLocator(page, input);
1080 if (locator === null) {
1081 // Page-side code is passed as source: the server compiles without DOM types.
1082 await page.evaluate(`scrollBy(${delta[0]}, ${delta[1]})`);
1083 return;
1084 }
1085 await locator.evaluate((element, [x, y]) => element.scrollBy(x, y), delta);
1086 };
1087
1088 const UNRESPONSIVE_SCRIPT_PROBE_MS = 1_000;
1089
1090 /**
1091 * A same-site navigation reuses the renderer, so it never commits while a page script holds it.
1092 * Call before starting the navigation: Chrome does not deliver a terminate once one is pending.
1093 */
1094 export const stopUnresponsiveScript = async (cdp: CDPSession, budgetMs = Infinity) => {
1095 let timer: ReturnType<typeof setTimeout> | undefined;
1096 const answered = cdp.send("Runtime.evaluate", { expression: "0" }).then(
1097 () => true,
1098 () => true,
1099 );
1100 const expired = new Promise<boolean>((resolve) => {
1101 timer = setTimeout(() => resolve(false), Math.min(budgetMs, UNRESPONSIVE_SCRIPT_PROBE_MS));
1102 });
1103 const responsive = await Promise.race([answered, expired]).finally(() => clearTimeout(timer));
1104 // Not awaited: a renderer blocked on something other than script may never acknowledge it.
1105 if (!responsive) void cdp.send("Runtime.terminateExecution").catch(constVoid);
1106 };
1107
1108 /** Terminate expired page scripts so a hung expression cannot block the tab's control queue. */
1109 export const evaluate = async (
1110 cdp: CDPSession,
1111 input: PreviewAutomationEvaluateInput,
1112 timeoutMs: number,
1113 ) => {
1114 let timer: ReturnType<typeof setTimeout> | undefined;
1115 let timedOut = false;
1116 const expired = new Promise<never>((_, reject) => {
1117 timer = setTimeout(() => {
1118 timedOut = true;
1119 // Stops a busy script; an awaited promise is abandoned and settles unread.
1120 void cdp.send("Runtime.terminateExecution").catch(constVoid);
1121 reject(
1122 new ServerBrowserOperationError({
1123 tag: "PreviewAutomationTimeoutError",
1124 message: `Evaluation did not finish within ${timeoutMs}ms and was stopped.`,
1125 }),
1126 );
1127 }, timeoutMs);
1128 });
1129 const params = {
1130 expression: input.expression,
1131 awaitPromise: input.awaitPromise ?? true,
1132 returnByValue: true,
1133 };
1134 // The devtools console's replMode allows top-level await but returns a trailing promise unawaited,
1135 // so it only retries what the page's parser rejected. A parse error has a scriptId but no stack and
1136 // ran nothing; a browser-internal rejection has neither.
1137 const evaluation = cdp.send("Runtime.evaluate", params).then((classic) => {
1138 const failure = classic.exceptionDetails;
1139 const unparsed =
1140 failure?.exception?.className === "SyntaxError" &&
1141 failure.scriptId !== undefined &&
1142 failure.stackTrace === undefined;
1143 return unparsed && !timedOut && input.expression.includes("await")
1144 ? cdp.send("Runtime.evaluate", { ...params, replMode: true })
1145 : classic;
1146 });
1147 void evaluation.catch(constVoid);
1148 const result = await Promise.race([evaluation, expired]).finally(() => clearTimeout(timer));
1149 if (result.exceptionDetails) {
1150 throw new ServerBrowserOperationError({
1151 tag: "PreviewAutomationExecutionError",
1152 message: result.exceptionDetails.exception?.description ?? result.exceptionDetails.text,
1153 });
1154 }
1155 const value =
1156 "value" in result.result ? result.result.value : (result.result.description ?? null);
1157 const actualBytes = Buffer.byteLength(JSON.stringify(value ?? null), "utf8");
1158 if (actualBytes > MAX_EVALUATION_BYTES) {
1159 throw new ServerBrowserOperationError({
1160 tag: "PreviewAutomationResultTooLargeError",
1161 message: `Evaluation result is ${actualBytes} bytes; the limit is ${MAX_EVALUATION_BYTES}.`,
1162 detail: { maximumBytes: MAX_EVALUATION_BYTES },
1163 });
1164 }
1165 return value;
1166 };
1167
1168 export const waitFor = async (page: Page, input: PreviewAutomationWaitForInput) => {
1169 const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
1170 const locator = targetLocator(page, input);
1171 const { text, urlIncludes } = input;
1172 const checks: Array<() => Promise<boolean>> = [];
1173 if (locator !== null) checks.push(async () => (await locator.count()) > 0);
1174 if (text !== undefined) {
1175 // A navigation can destroy the context mid-check; that counts as not yet.
1176 checks.push(() =>
1177 page
1178 .evaluate(`(document.body?.innerText ?? "").includes(${JSON.stringify(text)})`)
1179 .then((found) => found === true)
1180 .catch(() => false),
1181 );
1182 }
1183 if (urlIncludes !== undefined) checks.push(async () => page.url().includes(urlIncludes));
1184 // `count()` and `evaluate()` take no timeout; a stalled check would hold the tab's control queue.
1185 let timer: ReturnType<typeof setTimeout> | undefined;
1186 const expired = new Promise<never>((_, reject) => {
1187 timer = setTimeout(
1188 () =>
1189 reject(
1190 new ServerBrowserOperationError({
1191 tag: "PreviewAutomationTimeoutError",
1192 message: `Waited ${timeoutMs}ms without every condition holding.`,
1193 }),
1194 ),
1195 timeoutMs,
1196 );
1197 });
1198 try {
1199 for (;;) {
1200 const results = await Promise.race([Promise.all(checks.map((check) => check())), expired]);
1201 if (results.every(Boolean)) return;
1202 await Promise.race([new Promise((resolve) => setTimeout(resolve, WAIT_POLL_MS)), expired]);
1203 }
1204 } finally {
1205 clearTimeout(timer);
1206 }
1207 };
1208
1209 const NET_ERROR_CODES: Readonly<Record<string, number>> = {
1210 ERR_FAILED: -2,
1211 ERR_TIMED_OUT: -7,
1212 ERR_CONNECTION_CLOSED: -100,
1213 ERR_CONNECTION_RESET: -101,
1214 ERR_CONNECTION_REFUSED: -102,
1215 ERR_NAME_NOT_RESOLVED: -105,
1216 ERR_INTERNET_DISCONNECTED: -106,
1217 ERR_ADDRESS_UNREACHABLE: -109,
1218 ERR_CERT_AUTHORITY_INVALID: -202,
1219 ERR_EMPTY_RESPONSE: -324,
1220 };
1221
1222 export const parseNetError = (errorText: string) => {
1223 // Names can hold digits, as in ERR_HTTP2_PROTOCOL_ERROR.
1224 const description = /ERR_[A-Z\d_]+/.exec(errorText)?.[0] ?? errorText;
1225 return { description, code: NET_ERROR_CODES[description] ?? -2 };
1226 };
1227
1228 export const recordingEncoderScript = (frameRate: BrowserRecordingFrameRate) => `(() => {
1229 const canvas = document.createElement("canvas");
1230 const context = canvas.getContext("2d");
1231 const chunks = [];
1232 let recorder = null;
1233 let started = null;
1234 let stopped = null;
1235 let sizeBytes = 0;
1236 let tooLarge = false;
1237 let frame = null;
1238 let cursor = null;
1239 let ring = null;
1240 const AGENT_CURSOR_PATH = new Path2D("M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z");
1241 let ringTimer = null;
1242 let scale = 1;
1243 let offsetX = 0;
1244 let offsetY = 0;
1245 const paint = () => {
1246 if (!frame) return;
1247 const fit = Math.min(canvas.width / frame.width, canvas.height / frame.height);
1248 const width = frame.width * fit;
1249 const height = frame.height * fit;
1250 offsetX = (canvas.width - width) / 2;
1251 offsetY = (canvas.height - height) / 2;
1252 if (width < canvas.width || height < canvas.height) {
1253 context.fillStyle = "#000";
1254 context.fillRect(0, 0, canvas.width, canvas.height);
1255 }
1256 context.drawImage(frame, offsetX, offsetY, width, height);
1257 scale = (width / frame.width) * frame.cssScale;
1258 if (ring) {
1259 const age = (performance.now() - ring.at) / 400;
1260 if (age < 1) {
1261 context.beginPath();
1262 context.arc(offsetX + ring.x * scale, offsetY + ring.y * scale, (8 + 18 * age) * scale, 0, Math.PI * 2);
1263 context.strokeStyle = "rgba(59,130,246," + (0.9 * (1 - age)) + ")";
1264 context.lineWidth = 3 * scale;
1265 context.stroke();
1266 }
1267 }
1268 if (cursor) {
1269 // The agent cursor (lucide MousePointer2, as in the live panel), never the human arrow.
1270 const s = 20 / 24 * scale;
1271 context.save();
1272 context.translate(offsetX + cursor.x * scale - 2 * scale, offsetY + cursor.y * scale - 2 * scale);
1273 context.scale(s, s);
1274 context.lineWidth = 2;
1275 context.lineJoin = "round";
1276 context.lineCap = "round";
1277 context.fillStyle = "#fff";
1278 context.strokeStyle = "#2563eb";
1279 context.shadowColor = "rgba(0,0,0,0.25)";
1280 context.shadowBlur = 2;
1281 context.fill(AGENT_CURSOR_PATH);
1282 context.shadowColor = "transparent";
1283 context.stroke(AGENT_CURSOR_PATH);
1284 context.restore();
1285 }
1286 };
1287 window.__t3Recorder = {
1288 async frame(base64, cssWidth) {
1289 if (tooLarge) return false;
1290 // A per-character atob copy costs ~30 ms on a 3200x1800 frame and throttles recordings.
1291 const bytes = Uint8Array.fromBase64(base64);
1292 const bitmap = await createImageBitmap(new Blob([bytes], { type: "image/jpeg" }));
1293 if (tooLarge) { bitmap.close(); return false; }
1294 if (!recorder) {
1295 canvas.width = bitmap.width;
1296 canvas.height = bitmap.height;
1297 const mimeType = ["video/mp4;codecs=avc1.640033", "video/webm;codecs=vp9", "video/webm"].find((type) => MediaRecorder.isTypeSupported(type));
1298 const bitsPerSecond = Math.min(50e6, Math.max(2.5e6, bitmap.width * bitmap.height * ${frameRate} * 0.05));
1299 recorder = new MediaRecorder(canvas.captureStream(${frameRate}), { mimeType, videoBitsPerSecond: bitsPerSecond });
1300 started = new Promise((resolve, reject) => {
1301 recorder.onstart = resolve;
1302 recorder.onerror = (event) => reject(event.error);
1303 });
1304 stopped = new Promise((resolve) => { recorder.onstop = resolve; });
1305 recorder.ondataavailable = (event) => {
1306 if (tooLarge || event.data.size === 0) return;
1307 sizeBytes += event.data.size;
1308 if (sizeBytes > ${PROVIDER_SEND_TURN_MAX_FILE_BYTES}) {
1309 tooLarge = true;
1310 chunks.length = 0;
1311 clearInterval(ringTimer);
1312 frame?.close();
1313 frame = null;
1314 if (recorder.state !== "inactive") recorder.stop();
1315 for (const track of recorder.stream.getTracks()) track.stop();
1316 return;
1317 }
1318 chunks.push(event.data);
1319 };
1320 recorder.start(1000);
1321 }
1322 // Frame px per page CSS px, before fitting into the canvas.
1323 bitmap.cssScale = bitmap.width / cssWidth;
1324 frame?.close();
1325 frame = bitmap;
1326 paint();
1327 await started;
1328 return true;
1329 },
1330 cursor(x, y, click) {
1331 if (tooLarge) return;
1332 cursor = { x, y };
1333 if (click) {
1334 ring = { x, y, at: performance.now() };
1335 clearInterval(ringTimer);
1336 ringTimer = setInterval(() => {
1337 paint();
1338 if (performance.now() - ring.at > 400) { ring = null; clearInterval(ringTimer); paint(); }
1339 }, 33);
1340 }
1341 paint();
1342 },
1343 async stop() {
1344 if (!recorder) return { mimeType: null, count: 0, bytes: 0 };
1345 if (recorder.state !== "inactive") {
1346 // Consume the cached frame, then wait for the final paint to reach the
1347 // capture stream. Animation ticks alone can stop before it is delivered.
1348 const video = document.createElement("video");
1349 video.muted = true;
1350 video.srcObject = recorder.stream;
1351 const nextFrame = () => Promise.race([
1352 new Promise(resolve => video.requestVideoFrameCallback(resolve)),
1353 stopped,
1354 ]);
1355 const initial = nextFrame();
1356 void video.play().catch(() => {});
1357 await initial;
1358 if (recorder.state !== "inactive") {
1359 const delivered = nextFrame();
1360 recorder.stream.getVideoTracks()[0].requestFrame();
1361 paint();
1362 await delivered;
1363 }
1364 video.pause();
1365 video.srcObject = null;
1366 if (recorder.state !== "inactive") recorder.stop();
1367 }
1368 await stopped;
1369 return { mimeType: recorder.mimeType, count: chunks.length, bytes: sizeBytes };
1370 },
1371 async chunk(index) {
1372 const bytes = new Uint8Array(await chunks[index].arrayBuffer());
1373 if (typeof bytes.toBase64 === "function") return bytes.toBase64();
1374 let binary = "";
1375 for (let offset = 0; offset < bytes.length; offset += 0x8000) {
1376 binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000));
1377 }
1378 return btoa(binary);
1379 },
1380 download() {
1381 const link = document.createElement("a");
1382 link.href = URL.createObjectURL(new Blob(chunks, { type: recorder.mimeType }));
1383 link.download = "recording";
1384 link.click();
1385 },
1386 };
1387 })()`;