The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { |
| 2 | createPreviewFramePainter, |
| 3 | createPreviewStreamClient, |
| 4 | previewStreamControlLabel, |
| 5 | previewStreamModifiers, |
| 6 | type PreviewStreamClient, |
| 7 | type PreviewStreamControl, |
| 8 | type PreviewStreamDownload, |
| 9 | type PreviewStreamFileChooser, |
| 10 | uploadPreviewStreamFiles, |
| 11 | type PreviewStreamInput, |
| 12 | type PreviewStreamMouseButton, |
| 13 | type PreviewStreamPointer, |
| 14 | type PreviewStreamViewport, |
| 15 | previewStreamHostSetupMessage, |
| 16 | } from "@t3tools/client-runtime/preview/server-browser-stream"; |
| 17 | import type { |
| 18 | EnvironmentId, |
| 19 | PreviewStreamHostSetup, |
| 20 | PreviewViewportSetting, |
| 21 | } from "@t3tools/contracts"; |
| 22 | import { |
| 23 | type ClipboardEvent, |
| 24 | type FormEvent, |
| 25 | type KeyboardEvent, |
| 26 | type MouseEvent, |
| 27 | type PointerEvent, |
| 28 | type Ref, |
| 29 | useEffect, |
| 30 | useEffectEvent, |
| 31 | useImperativeHandle, |
| 32 | useRef, |
| 33 | useState, |
| 34 | } from "react"; |
| 35 | import { msg, T, useGT, useMessages } from "gt-react"; |
| 36 | |
| 37 | import { AgentCursorMark } from "~/components/preview/AgentBrowserCursor"; |
| 38 | import { CommandBlock } from "~/components/CommandBlock"; |
| 39 | import { Button } from "~/components/ui/button"; |
| 40 | import { Input } from "~/components/ui/input"; |
| 41 | import { toastManager } from "~/components/ui/toast"; |
| 42 | import { thrownErrorMessage } from "@t3tools/client-runtime/errors"; |
| 43 | import { m } from "~/i18n/messages"; |
| 44 | import { cn } from "~/lib/utils"; |
| 45 | import { refreshPreviewStreamAccess, usePreviewStreamAccess } from "~/state/previewStream"; |
| 46 | import { observeResize } from "~/lib/observeResize"; |
| 47 | |
| 48 | export interface ServerBrowserHandle { |
| 49 | readonly navigate: (url: string) => void; |
| 50 | readonly history: (delta: -1 | 1) => void; |
| 51 | readonly reload: (options?: { readonly ignoreCache?: boolean }) => void; |
| 52 | readonly stopLoading: () => void; |
| 53 | readonly viewport: (setting: PreviewViewportSetting) => void; |
| 54 | readonly canvas: () => HTMLCanvasElement | null; |
| 55 | } |
| 56 | |
| 57 | const RESIZE_THROTTLE_MS = 150; |
| 58 | // Reuse nearby probes when a slow link would return the tap's answer after it ends. |
| 59 | const PROBE_REUSE_PX = 24; |
| 60 | const PROBE_REUSE_MS = 10_000; |
| 61 | // Android keyboards report Backspace as text deletion, so keep a deletable sentinel. |
| 62 | const INPUT_SENTINEL = "\u200b"; |
| 63 | const BACKSPACE = { key: "Backspace", code: "Backspace", keyCode: 8 } as const; |
| 64 | const DELETE = { key: "Delete", code: "Delete", keyCode: 46 } as const; |
| 65 | const COPY = { key: "c", code: "KeyC", keyCode: 67 } as const; |
| 66 | const CUT = { key: "x", code: "KeyX", keyCode: 88 } as const; |
| 67 | const CTRL = 2; |
| 68 | const MAX_UNAUTHORIZED_REFUSALS = 3; |
| 69 | const TAP_SLOP_PX = 8; |
| 70 | const MULTI_CLICK_MS = 500; |
| 71 | const MULTI_CLICK_SLOP_PX = 4; |
| 72 | const WHEEL_LINE_PX = 16; |
| 73 | // The server answers a new viewer with a still within seconds, so silence means Chromium is not painting. |
| 74 | const NO_FRAME_TIMEOUT_MS = 10_000; |
| 75 | |
| 76 | type MouseInput = Extract<PreviewStreamInput, { type: "mouse" }>; |
| 77 | type WheelInput = Extract<PreviewStreamInput, { type: "wheel" }>; |
| 78 | |
| 79 | interface PagePoint { |
| 80 | readonly x: number; |
| 81 | readonly y: number; |
| 82 | readonly scale: number; |
| 83 | } |
| 84 | |
| 85 | interface TouchGesture { |
| 86 | readonly pointerId: number; |
| 87 | readonly startX: number; |
| 88 | readonly startY: number; |
| 89 | lastX: number; |
| 90 | lastY: number; |
| 91 | panning: boolean; |
| 92 | } |
| 93 | |
| 94 | interface TouchProbe { |
| 95 | readonly x: number; |
| 96 | readonly y: number; |
| 97 | editable: boolean | null; |
| 98 | answered: boolean; |
| 99 | tapped: boolean; |
| 100 | } |
| 101 | |
| 102 | const buttonOf = (button: number): PreviewStreamMouseButton => |
| 103 | button === 0 ? "left" : button === 1 ? "middle" : button === 2 ? "right" : "none"; |
| 104 | |
| 105 | const pressedButtonOf = (buttons: number): PreviewStreamMouseButton => |
| 106 | buttons & 1 ? "left" : buttons & 2 ? "right" : buttons & 4 ? "middle" : "none"; |
| 107 | |
| 108 | // Mouse thumb buttons: `button === 3` is Back, `button === 4` is Forward. |
| 109 | const historyDeltaOf = (button: number): -1 | 1 | null => |
| 110 | button === 3 ? -1 : button === 4 ? 1 : null; |
| 111 | |
| 112 | /** Browsers may refuse a write that arrives after the key press; a toast button is a fresh gesture. */ |
| 113 | async function copyPageText(text: string) { |
| 114 | try { |
| 115 | await navigator.clipboard.writeText(text); |
| 116 | } catch { |
| 117 | const id = toastManager.add({ |
| 118 | type: "info", |
| 119 | title: m(msg("The page copied text")), |
| 120 | actionProps: { |
| 121 | children: m(msg("Copy", { $context: "verb, copies text to the clipboard" })), |
| 122 | onClick: () => { |
| 123 | toastManager.close(id); |
| 124 | void navigator.clipboard.writeText(text).catch(() => undefined); |
| 125 | }, |
| 126 | }, |
| 127 | }); |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | function offerDownload(download: PreviewStreamDownload) { |
| 132 | const id = toastManager.add({ |
| 133 | type: "info", |
| 134 | title: m(msg("Downloaded {fileName}", { fileName: download.fileName })), |
| 135 | actionProps: { |
| 136 | children: m(msg("Save", { $context: "verb, saves a downloaded file" })), |
| 137 | onClick: () => { |
| 138 | toastManager.close(id); |
| 139 | const anchor = document.createElement("a"); |
| 140 | anchor.href = download.url; |
| 141 | anchor.download = download.fileName; |
| 142 | anchor.click(); |
| 143 | }, |
| 144 | }, |
| 145 | }); |
| 146 | } |
| 147 | |
| 148 | export function ServerBrowserSurface(props: { |
| 149 | readonly environmentId: EnvironmentId; |
| 150 | readonly threadId: string; |
| 151 | readonly tabId: string; |
| 152 | readonly visible: boolean; |
| 153 | readonly followSize?: boolean; |
| 154 | readonly controlPosition?: "top" | "bottom"; |
| 155 | readonly onFirstFrame?: () => void; |
| 156 | readonly onNoFrames?: () => void; |
| 157 | readonly onViewport?: (viewport: PreviewStreamViewport) => void; |
| 158 | readonly onControl?: (control: PreviewStreamControl | null) => void; |
| 159 | readonly onPopup?: (tabId: string) => void; |
| 160 | readonly className?: string; |
| 161 | readonly pdfUrl?: string | null; |
| 162 | readonly ref?: Ref<ServerBrowserHandle>; |
| 163 | }) { |
| 164 | const { |
| 165 | environmentId, |
| 166 | threadId, |
| 167 | tabId, |
| 168 | visible, |
| 169 | followSize = true, |
| 170 | controlPosition = "top", |
| 171 | onFirstFrame, |
| 172 | onNoFrames, |
| 173 | onViewport, |
| 174 | onControl, |
| 175 | onPopup, |
| 176 | className, |
| 177 | pdfUrl = null, |
| 178 | ref, |
| 179 | } = props; |
| 180 | const gt = useGT(); |
| 181 | const resolve = useMessages(); |
| 182 | const canvasRef = useRef<HTMLCanvasElement | null>(null); |
| 183 | const inputRef = useRef<HTMLTextAreaElement | null>(null); |
| 184 | const keySentRef = useRef(false); |
| 185 | const clientRef = useRef<PreviewStreamClient | null>(null); |
| 186 | const viewportRef = useRef<PreviewStreamViewport | null>(null); |
| 187 | const sizeRef = useRef<{ width: number; height: number } | null>(null); |
| 188 | const hasFrameRef = useRef(false); |
| 189 | const controlRef = useRef<PreviewStreamControl | null>(null); |
| 190 | const [control, setControl] = useState<PreviewStreamControl | null>(null); |
| 191 | const [promptText, setPromptText] = useState(""); |
| 192 | const [fileChooser, setFileChooser] = useState<PreviewStreamFileChooser | null>(null); |
| 193 | const [agentCursor, setAgentCursor] = useState<AgentCursorPlacement | null>(null); |
| 194 | const fileInputRef = useRef<HTMLInputElement | null>(null); |
| 195 | const answerFileChooser = (files: ReadonlyArray<File>) => { |
| 196 | const chooser = fileChooser; |
| 197 | if (!chooser) return; |
| 198 | setFileChooser(null); |
| 199 | void uploadPreviewStreamFiles(chooser, files).catch((cause: unknown) => |
| 200 | toastManager.add({ |
| 201 | type: "error", |
| 202 | title: gt("Could not send the files to the page"), |
| 203 | description: thrownErrorMessage(cause, m) ?? undefined, |
| 204 | }), |
| 205 | ); |
| 206 | }; |
| 207 | const unauthorizedRef = useRef(0); |
| 208 | const [accessDenied, setAccessDenied] = useState(false); |
| 209 | const access = usePreviewStreamAccess(environmentId, { retry: visible && !accessDenied }); |
| 210 | const [hostSetup, setHostSetup] = useState<PreviewStreamHostSetup | null>(null); |
| 211 | const pendingMoveRef = useRef<MouseInput | null>(null); |
| 212 | const pendingWheelRef = useRef<WheelInput | null>(null); |
| 213 | const inputFrameRef = useRef<number | null>(null); |
| 214 | const mouseButtonsRef = useRef(0); |
| 215 | const mouseClicksRef = useRef({ left: 1, middle: 1, right: 1, none: 1 }); |
| 216 | const lastClickRef = useRef<{ |
| 217 | button: PreviewStreamMouseButton; |
| 218 | time: number; |
| 219 | x: number; |
| 220 | y: number; |
| 221 | count: number; |
| 222 | } | null>(null); |
| 223 | const touchRef = useRef<TouchGesture | null>(null); |
| 224 | const probeRef = useRef<TouchProbe | null>(null); |
| 225 | const lastProbeRef = useRef<{ x: number; y: number; editable: boolean; at: number } | null>(null); |
| 226 | const firstFrame = useEffectEvent(() => onFirstFrame?.()); |
| 227 | const noFrames = useEffectEvent(() => onNoFrames?.()); |
| 228 | const [framesStalled, setFramesStalled] = useState(false); |
| 229 | const viewportChanged = useEffectEvent((viewport: PreviewStreamViewport) => |
| 230 | onViewport?.(viewport), |
| 231 | ); |
| 232 | const controlChanged = useEffectEvent((next: PreviewStreamControl | null) => onControl?.(next)); |
| 233 | const popupOpened = useEffectEvent((popupTabId: string) => onPopup?.(popupTabId)); |
| 234 | const [cap, setCap] = useState<{ width: number; height: number } | null>(null); |
| 235 | |
| 236 | // Keep clearInput stable: a new identity reconnects the stream. |
| 237 | const send = (input: PreviewStreamInput) => { |
| 238 | clientRef.current?.send(input); |
| 239 | }; |
| 240 | |
| 241 | const clearInput = () => { |
| 242 | if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current); |
| 243 | inputFrameRef.current = null; |
| 244 | pendingMoveRef.current = pendingWheelRef.current = null; |
| 245 | mouseButtonsRef.current = 0; |
| 246 | mouseClicksRef.current = { left: 1, middle: 1, right: 1, none: 1 }; |
| 247 | lastClickRef.current = null; |
| 248 | touchRef.current = probeRef.current = lastProbeRef.current = null; |
| 249 | keySentRef.current = false; |
| 250 | if (inputRef.current) inputRef.current.value = INPUT_SENTINEL; |
| 251 | inputRef.current?.blur(); |
| 252 | }; |
| 253 | |
| 254 | const flushInput = () => { |
| 255 | if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current); |
| 256 | inputFrameRef.current = null; |
| 257 | const move = pendingMoveRef.current; |
| 258 | const wheel = pendingWheelRef.current; |
| 259 | pendingMoveRef.current = null; |
| 260 | pendingWheelRef.current = null; |
| 261 | if (move) send(move); |
| 262 | if (wheel) send(wheel); |
| 263 | }; |
| 264 | |
| 265 | const scheduleFlush = () => { |
| 266 | if (inputFrameRef.current === null) inputFrameRef.current = requestAnimationFrame(flushInput); |
| 267 | }; |
| 268 | |
| 269 | const queueWheel = (point: PagePoint, deltaX: number, deltaY: number, modifiers: number) => { |
| 270 | const pending = pendingWheelRef.current; |
| 271 | pendingWheelRef.current = { |
| 272 | type: "wheel", |
| 273 | x: point.x, |
| 274 | y: point.y, |
| 275 | deltaX: (pending?.deltaX ?? 0) + deltaX, |
| 276 | deltaY: (pending?.deltaY ?? 0) + deltaY, |
| 277 | modifiers, |
| 278 | }; |
| 279 | scheduleFlush(); |
| 280 | }; |
| 281 | |
| 282 | const pagePoint = (clientX: number, clientY: number, clamp: boolean): PagePoint | null => { |
| 283 | const canvas = canvasRef.current; |
| 284 | const viewport = viewportRef.current; |
| 285 | if (!canvas || !viewport || !hasFrameRef.current || controlRef.current?.controller !== "you") |
| 286 | return null; |
| 287 | const rect = canvas.getBoundingClientRect(); |
| 288 | const fit = Math.min(rect.width / canvas.width, rect.height / canvas.height); |
| 289 | const width = canvas.width * fit; |
| 290 | const height = canvas.height * fit; |
| 291 | if (!(width > 0 && height > 0)) return null; |
| 292 | const scale = viewport.width / width; |
| 293 | const x = (clientX - rect.left - (rect.width - width) / 2) * scale; |
| 294 | const y = (clientY - rect.top - (rect.height - height) / 2) * (viewport.height / height); |
| 295 | const inside = x >= 0 && y >= 0 && x <= viewport.width && y <= viewport.height; |
| 296 | if (!inside && !clamp) return null; |
| 297 | return { |
| 298 | x: Math.min(Math.max(x, 0), viewport.width), |
| 299 | y: Math.min(Math.max(y, 0), viewport.height), |
| 300 | scale, |
| 301 | }; |
| 302 | }; |
| 303 | |
| 304 | const countClick = (button: PreviewStreamMouseButton, x: number, y: number, time: number) => { |
| 305 | const last = lastClickRef.current; |
| 306 | const count = |
| 307 | last && |
| 308 | last.button === button && |
| 309 | time - last.time < MULTI_CLICK_MS && |
| 310 | Math.hypot(x - last.x, y - last.y) < MULTI_CLICK_SLOP_PX |
| 311 | ? last.count + 1 |
| 312 | : 1; |
| 313 | lastClickRef.current = { button, time, x, y, count }; |
| 314 | return count; |
| 315 | }; |
| 316 | |
| 317 | const focusInput = () => inputRef.current?.focus({ preventScroll: true }); |
| 318 | |
| 319 | useImperativeHandle( |
| 320 | ref, |
| 321 | () => ({ |
| 322 | navigate: (url) => send({ type: "navigate", url }), |
| 323 | history: (delta) => send({ type: "history", delta }), |
| 324 | reload: (options) => |
| 325 | send({ type: "reload", ...(options?.ignoreCache ? { ignoreCache: true } : {}) }), |
| 326 | stopLoading: () => send({ type: "stopLoading" }), |
| 327 | viewport: (setting) => send({ type: "viewport", setting }), |
| 328 | canvas: () => (!pdfUrl && hasFrameRef.current ? canvasRef.current : null), |
| 329 | }), |
| 330 | [pdfUrl, send], |
| 331 | ); |
| 332 | |
| 333 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- observes the canvas container's size, sends resizes to the stream and grows the frame cap |
| 334 | useEffect(() => { |
| 335 | const element = canvasRef.current?.parentElement; |
| 336 | if (!element) return; |
| 337 | let timer: ReturnType<typeof setTimeout> | null = null; |
| 338 | let capTimer: ReturnType<typeof setTimeout> | null = null; |
| 339 | const growCap = () => { |
| 340 | capTimer = null; |
| 341 | const size = sizeRef.current; |
| 342 | if (!size) return; |
| 343 | const ratio = window.devicePixelRatio || 1; |
| 344 | const width = Math.round(size.width * ratio); |
| 345 | const height = Math.round(size.height * ratio); |
| 346 | setCap((current) => |
| 347 | current !== null && current.width >= width && current.height >= height |
| 348 | ? current |
| 349 | : { |
| 350 | width: Math.max(width, current?.width ?? 0), |
| 351 | height: Math.max(height, current?.height ?? 0), |
| 352 | }, |
| 353 | ); |
| 354 | }; |
| 355 | const measure = () => { |
| 356 | timer = null; |
| 357 | const rect = element.getBoundingClientRect(); |
| 358 | if (rect.width < 1 || rect.height < 1) return; |
| 359 | const size = { width: Math.round(rect.width), height: Math.round(rect.height) }; |
| 360 | const previous = sizeRef.current; |
| 361 | if (previous?.width === size.width && previous.height === size.height) return; |
| 362 | sizeRef.current = size; |
| 363 | if (followSize) clientRef.current?.send({ type: "resize", ...size }); |
| 364 | // Growing the cap reconnects the stream; wait until resizing settles. |
| 365 | if (capTimer !== null) clearTimeout(capTimer); |
| 366 | capTimer = setTimeout(growCap, previous === null ? 0 : RESIZE_THROTTLE_MS * 2); |
| 367 | }; |
| 368 | const stopObserving = observeResize(element, () => { |
| 369 | if (timer !== null) return; |
| 370 | timer = setTimeout(measure, sizeRef.current === null ? 0 : RESIZE_THROTTLE_MS); |
| 371 | }); |
| 372 | return () => { |
| 373 | stopObserving(); |
| 374 | if (timer !== null) clearTimeout(timer); |
| 375 | if (capTimer !== null) clearTimeout(capTimer); |
| 376 | }; |
| 377 | }, [followSize]); |
| 378 | |
| 379 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- opens the preview stream client and paints its frames onto the canvas while visible |
| 380 | useEffect(() => { |
| 381 | const canvas = canvasRef.current; |
| 382 | if (!visible || accessDenied || hostSetup || !access || !cap || !canvas) return; |
| 383 | let refreshTimer: ReturnType<typeof setTimeout> | null = null; |
| 384 | let noFrameTimer: ReturnType<typeof setTimeout> | null = null; |
| 385 | const clearNoFrameTimer = () => { |
| 386 | if (noFrameTimer !== null) clearTimeout(noFrameTimer); |
| 387 | noFrameTimer = null; |
| 388 | }; |
| 389 | const painter = createPreviewFramePainter(canvas, () => { |
| 390 | if (hasFrameRef.current) return; |
| 391 | hasFrameRef.current = true; |
| 392 | clearNoFrameTimer(); |
| 393 | setFramesStalled(false); |
| 394 | firstFrame(); |
| 395 | }); |
| 396 | const client = createPreviewStreamClient( |
| 397 | { access, threadId, tabId, maxWidth: cap.width, maxHeight: cap.height }, |
| 398 | { |
| 399 | onFrame: (jpeg) => { |
| 400 | unauthorizedRef.current = 0; |
| 401 | painter.paint(jpeg); |
| 402 | }, |
| 403 | onProbe: (result) => { |
| 404 | if (controlRef.current?.controller !== "you") return; |
| 405 | lastProbeRef.current = { ...result, at: performance.now() }; |
| 406 | const probe = probeRef.current; |
| 407 | if (!probe || probe.x !== result.x || probe.y !== result.y) return; |
| 408 | probe.editable = result.editable; |
| 409 | probe.answered = true; |
| 410 | if (!probe.tapped) return; |
| 411 | probeRef.current = null; |
| 412 | if (result.editable) inputRef.current?.focus({ preventScroll: true }); |
| 413 | else inputRef.current?.blur(); |
| 414 | }, |
| 415 | onPointer: (pointer) => { |
| 416 | const placed = placeAgentCursor(pointer, canvasRef.current, viewportRef.current); |
| 417 | if (placed) setAgentCursor(placed); |
| 418 | }, |
| 419 | onClipboard: (text) => void copyPageText(text), |
| 420 | onDownload: offerDownload, |
| 421 | onPopup: popupOpened, |
| 422 | onFileChooser: setFileChooser, |
| 423 | onViewport: (viewport) => { |
| 424 | viewportRef.current = viewport; |
| 425 | viewportChanged(viewport); |
| 426 | }, |
| 427 | onControl: (next) => { |
| 428 | const previous = controlRef.current; |
| 429 | controlRef.current = next; |
| 430 | setControl(next); |
| 431 | controlChanged(next); |
| 432 | if ( |
| 433 | next.dialog?.message !== previous?.dialog?.message || |
| 434 | next.dialog?.defaultValue !== previous?.dialog?.defaultValue |
| 435 | ) |
| 436 | setPromptText(next.dialog?.defaultValue ?? ""); |
| 437 | if (next.controller !== "you") { |
| 438 | clearInput(); |
| 439 | } else if (previous?.controller !== "you") { |
| 440 | const size = sizeRef.current; |
| 441 | if (followSize && size) client.send({ type: "resize", ...size }); |
| 442 | } |
| 443 | }, |
| 444 | onConnectedChange: (connected) => { |
| 445 | clearNoFrameTimer(); |
| 446 | if (connected) { |
| 447 | if (!hasFrameRef.current) |
| 448 | noFrameTimer = setTimeout(() => { |
| 449 | setFramesStalled(true); |
| 450 | noFrames(); |
| 451 | }, NO_FRAME_TIMEOUT_MS); |
| 452 | return; |
| 453 | } |
| 454 | setFileChooser(null); |
| 455 | controlRef.current = null; |
| 456 | setControl(null); |
| 457 | controlChanged(null); |
| 458 | clearInput(); |
| 459 | }, |
| 460 | onHostSetup: setHostSetup, |
| 461 | onUnauthorized: () => { |
| 462 | const refusals = ++unauthorizedRef.current; |
| 463 | if (refusals >= MAX_UNAUTHORIZED_REFUSALS) { |
| 464 | inputRef.current?.blur(); |
| 465 | setAccessDenied(true); |
| 466 | return; |
| 467 | } |
| 468 | refreshTimer = setTimeout( |
| 469 | () => refreshPreviewStreamAccess(environmentId), |
| 470 | refusals === 1 ? 0 : 1_000 * 2 ** (refusals - 1), |
| 471 | ); |
| 472 | }, |
| 473 | }, |
| 474 | ); |
| 475 | clientRef.current = client; |
| 476 | return () => { |
| 477 | painter.stop(); |
| 478 | clearNoFrameTimer(); |
| 479 | if (refreshTimer !== null) clearTimeout(refreshTimer); |
| 480 | client.stop(); |
| 481 | if (clientRef.current === client) clientRef.current = null; |
| 482 | controlRef.current = null; |
| 483 | setControl(null); |
| 484 | controlChanged(null); |
| 485 | clearInput(); |
| 486 | }; |
| 487 | }, [ |
| 488 | access, |
| 489 | accessDenied, |
| 490 | cap, |
| 491 | clearInput, |
| 492 | environmentId, |
| 493 | followSize, |
| 494 | hostSetup, |
| 495 | tabId, |
| 496 | threadId, |
| 497 | visible, |
| 498 | ]); |
| 499 | |
| 500 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- registers a non-passive wheel listener on the canvas so it can stop the panel scrolling |
| 501 | useEffect(() => { |
| 502 | const canvas = canvasRef.current; |
| 503 | if (!canvas) return; |
| 504 | // Registered natively: React's wheel listener is passive and cannot stop the panel scrolling. |
| 505 | const onWheel = (event: WheelEvent) => { |
| 506 | const point = pagePoint(event.clientX, event.clientY, false); |
| 507 | if (!point) return; |
| 508 | event.preventDefault(); |
| 509 | const unit = |
| 510 | event.deltaMode === WheelEvent.DOM_DELTA_LINE |
| 511 | ? WHEEL_LINE_PX |
| 512 | : event.deltaMode === WheelEvent.DOM_DELTA_PAGE |
| 513 | ? (viewportRef.current?.height ?? 0) |
| 514 | : 1; |
| 515 | queueWheel(point, event.deltaX * unit, event.deltaY * unit, previewStreamModifiers(event)); |
| 516 | }; |
| 517 | canvas.addEventListener("wheel", onWheel, { passive: false }); |
| 518 | return () => canvas.removeEventListener("wheel", onWheel); |
| 519 | }, [pagePoint, queueWheel]); |
| 520 | |
| 521 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- cancels the pending input animation frame when the component unmounts |
| 522 | useEffect( |
| 523 | () => () => { |
| 524 | if (inputFrameRef.current !== null) cancelAnimationFrame(inputFrameRef.current); |
| 525 | }, |
| 526 | [], |
| 527 | ); |
| 528 | |
| 529 | const handlePointer = (event: PointerEvent<HTMLCanvasElement>) => { |
| 530 | if (controlRef.current?.controller !== "you") return; |
| 531 | // Thumb buttons drive this tab's history in handleMouseUp, never the page. |
| 532 | if (historyDeltaOf(event.button) !== null) return; |
| 533 | const { type, pointerId, clientX, clientY } = event; |
| 534 | const cancelled = type === "pointercancel"; |
| 535 | const touch = touchRef.current; |
| 536 | if (cancelled && touch?.pointerId === pointerId) { |
| 537 | touchRef.current = probeRef.current = null; |
| 538 | return; |
| 539 | } |
| 540 | if (event.pointerType === "touch" && !cancelled) { |
| 541 | const point = pagePoint(clientX, clientY, type === "pointermove"); |
| 542 | if (type === "pointerdown") { |
| 543 | if (!event.isPrimary) return; |
| 544 | touchRef.current = { |
| 545 | pointerId, |
| 546 | startX: clientX, |
| 547 | startY: clientY, |
| 548 | lastX: clientX, |
| 549 | lastY: clientY, |
| 550 | panning: false, |
| 551 | }; |
| 552 | const last = lastProbeRef.current; |
| 553 | const reusable = |
| 554 | point && |
| 555 | last && |
| 556 | performance.now() - last.at < PROBE_REUSE_MS && |
| 557 | Math.hypot(point.x - last.x, point.y - last.y) < PROBE_REUSE_PX; |
| 558 | probeRef.current = point |
| 559 | ? { |
| 560 | x: point.x, |
| 561 | y: point.y, |
| 562 | editable: reusable ? last.editable : null, |
| 563 | answered: false, |
| 564 | tapped: false, |
| 565 | } |
| 566 | : null; |
| 567 | if (point) send({ type: "probe", x: point.x, y: point.y }); |
| 568 | return; |
| 569 | } |
| 570 | if (!touch || touch.pointerId !== pointerId) return; |
| 571 | if (type === "pointermove") { |
| 572 | if ( |
| 573 | !touch.panning && |
| 574 | Math.hypot(clientX - touch.startX, clientY - touch.startY) < TAP_SLOP_PX |
| 575 | ) |
| 576 | return; |
| 577 | touch.panning = true; |
| 578 | if (point) |
| 579 | queueWheel( |
| 580 | point, |
| 581 | (touch.lastX - clientX) * point.scale, |
| 582 | (touch.lastY - clientY) * point.scale, |
| 583 | 0, |
| 584 | ); |
| 585 | touch.lastX = clientX; |
| 586 | touch.lastY = clientY; |
| 587 | return; |
| 588 | } |
| 589 | touchRef.current = null; |
| 590 | const probe = probeRef.current; |
| 591 | if (touch.panning) { |
| 592 | probeRef.current = null; |
| 593 | return; |
| 594 | } |
| 595 | if (!point) return; |
| 596 | // iOS requires focus during the tap; late probe replies only raise Android's keyboard. |
| 597 | if (probe?.editable === true) focusInput(); |
| 598 | else if (probe?.editable === false) inputRef.current?.blur(); |
| 599 | if (probe && !probe.answered) probe.tapped = true; |
| 600 | else probeRef.current = null; |
| 601 | flushInput(); |
| 602 | const clickCount = countClick("left", clientX, clientY, event.timeStamp); |
| 603 | const at = { x: point.x, y: point.y, modifiers: 0 }; |
| 604 | send({ type: "mouse", action: "move", ...at, button: "none", buttons: 0, clickCount: 0 }); |
| 605 | send({ type: "mouse", action: "down", ...at, button: "left", buttons: 1, clickCount }); |
| 606 | send({ type: "mouse", action: "up", ...at, button: "left", buttons: 0, clickCount }); |
| 607 | return; |
| 608 | } |
| 609 | const previous = mouseButtonsRef.current; |
| 610 | if ((cancelled || type === "pointerup") && previous === 0) return; |
| 611 | if (cancelled || type === "pointerup") |
| 612 | mouseButtonsRef.current = cancelled ? 0 : event.buttons & 7; |
| 613 | const point = pagePoint( |
| 614 | clientX, |
| 615 | clientY, |
| 616 | type !== "pointerdown" && |
| 617 | (cancelled || type === "pointerup" || previous !== 0 || event.buttons !== 0), |
| 618 | ); |
| 619 | if (!point) return; |
| 620 | const at = { x: point.x, y: point.y, modifiers: previewStreamModifiers(event) }; |
| 621 | const mouse = (action: "down" | "up", button: PreviewStreamMouseButton, buttons: number) => { |
| 622 | if (action === "down") |
| 623 | mouseClicksRef.current[button] = countClick(button, clientX, clientY, event.timeStamp); |
| 624 | send({ |
| 625 | type: "mouse", |
| 626 | action, |
| 627 | ...at, |
| 628 | button, |
| 629 | buttons, |
| 630 | clickCount: mouseClicksRef.current[button], |
| 631 | }); |
| 632 | }; |
| 633 | if (type === "pointerdown") { |
| 634 | focusInput(); |
| 635 | event.currentTarget.setPointerCapture(pointerId); |
| 636 | flushInput(); |
| 637 | mouseButtonsRef.current = event.buttons & 7; |
| 638 | mouse("down", buttonOf(event.button), event.buttons); |
| 639 | } else if (type === "pointerup") { |
| 640 | flushInput(); |
| 641 | mouse("up", buttonOf(event.button), event.buttons); |
| 642 | } else { |
| 643 | // Chords arrive as pointermove. Cancellation releases every held button. |
| 644 | const next = cancelled ? 0 : event.buttons & 7; |
| 645 | const changed = previous ^ next; |
| 646 | if (previous !== 0 && changed !== 0) { |
| 647 | flushInput(); |
| 648 | let buttons = previous; |
| 649 | for (const bit of [1, 2, 4]) { |
| 650 | if (!(changed & bit)) continue; |
| 651 | buttons ^= bit; |
| 652 | mouse(next & bit ? "down" : "up", pressedButtonOf(bit), buttons); |
| 653 | } |
| 654 | mouseButtonsRef.current = next; |
| 655 | } |
| 656 | if (cancelled) return; |
| 657 | pendingMoveRef.current = { |
| 658 | type: "mouse", |
| 659 | action: "move", |
| 660 | ...at, |
| 661 | button: pressedButtonOf(event.buttons), |
| 662 | buttons: event.buttons, |
| 663 | clickCount: 0, |
| 664 | }; |
| 665 | scheduleFlush(); |
| 666 | } |
| 667 | }; |
| 668 | |
| 669 | // Chromium runs a thumb button's back/forward on mouseup in this document, which |
| 670 | // would move the app's router. Cancel it and send the press to the preview, as |
| 671 | // the toolbar does. |
| 672 | const handleMouseUp = (event: MouseEvent<HTMLDivElement>) => { |
| 673 | const delta = historyDeltaOf(event.button); |
| 674 | if (delta === null) return; |
| 675 | event.preventDefault(); |
| 676 | send({ type: "history", delta }); |
| 677 | }; |
| 678 | |
| 679 | const resetInput = (textarea: HTMLTextAreaElement) => { |
| 680 | keySentRef.current = false; |
| 681 | textarea.value = INPUT_SENTINEL; |
| 682 | textarea.setSelectionRange(INPUT_SENTINEL.length, INPUT_SENTINEL.length); |
| 683 | }; |
| 684 | |
| 685 | const sendKeyPress = ( |
| 686 | key: typeof BACKSPACE | typeof DELETE | typeof COPY | typeof CUT, |
| 687 | modifiers = 0, |
| 688 | ) => { |
| 689 | send({ type: "key", action: "down", ...key, modifiers }); |
| 690 | send({ type: "key", action: "up", ...key, modifiers }); |
| 691 | }; |
| 692 | |
| 693 | // Edit menu Copy and Cut fire no key, so send the shortcut the server copies on. |
| 694 | const handleClipboard = (key: typeof COPY | typeof CUT, event: ClipboardEvent) => { |
| 695 | // Keep the sentinel off the clipboard. |
| 696 | event.preventDefault(); |
| 697 | if (controlRef.current?.controller !== "you") return; |
| 698 | sendKeyPress(key, CTRL); |
| 699 | }; |
| 700 | |
| 701 | const handleKey = (action: "down" | "up", event: KeyboardEvent<HTMLTextAreaElement>) => { |
| 702 | if (controlRef.current?.controller !== "you") return; |
| 703 | keySentRef.current = false; |
| 704 | if ( |
| 705 | event.nativeEvent.isComposing || |
| 706 | event.keyCode === 229 || |
| 707 | event.key === "Process" || |
| 708 | event.key === "Unidentified" |
| 709 | ) { |
| 710 | return; |
| 711 | } |
| 712 | if (event.key === "Escape" && event.shiftKey) { |
| 713 | event.preventDefault(); |
| 714 | event.stopPropagation(); |
| 715 | if (action === "down") event.currentTarget.blur(); |
| 716 | return; |
| 717 | } |
| 718 | const shortcut = event.ctrlKey || event.metaKey; |
| 719 | if (shortcut && event.key.toLowerCase() === "v") return; |
| 720 | if (event.shiftKey && event.key === "Insert") return; |
| 721 | // Enter carries "\r" like Puppeteer's key table, so forms submit and textareas break lines. |
| 722 | const text = shortcut |
| 723 | ? undefined |
| 724 | : [...event.key].length === 1 |
| 725 | ? event.key |
| 726 | : event.key === "Enter" |
| 727 | ? "\r" |
| 728 | : undefined; |
| 729 | send({ |
| 730 | type: "key", |
| 731 | action, |
| 732 | key: event.key, |
| 733 | code: event.code, |
| 734 | keyCode: event.keyCode, |
| 735 | ...(action === "down" && text !== undefined ? { text } : {}), |
| 736 | modifiers: previewStreamModifiers(event), |
| 737 | }); |
| 738 | // Some Android keyboards edit the textarea even when keydown is prevented. |
| 739 | keySentRef.current = |
| 740 | action === "down" && |
| 741 | !shortcut && |
| 742 | (text !== undefined || event.key === "Backspace" || event.key === "Delete"); |
| 743 | // The key copies or cuts in the page, so stop the browser's own copy or cut repeating it. |
| 744 | if ( |
| 745 | event.ctrlKey !== event.metaKey && |
| 746 | !event.altKey && |
| 747 | !event.shiftKey && |
| 748 | ["c", "x"].includes(event.key.toLowerCase()) |
| 749 | ) { |
| 750 | event.preventDefault(); |
| 751 | } |
| 752 | if (shortcut) return; |
| 753 | event.preventDefault(); |
| 754 | event.stopPropagation(); |
| 755 | }; |
| 756 | |
| 757 | const handleInput = (event: FormEvent<HTMLTextAreaElement>) => { |
| 758 | const native = event.nativeEvent; |
| 759 | if (native instanceof InputEvent && native.isComposing) return; |
| 760 | const textarea = event.currentTarget; |
| 761 | const inputType = native instanceof InputEvent ? native.inputType : ""; |
| 762 | if (keySentRef.current) keySentRef.current = false; |
| 763 | else if (inputType === "deleteContentBackward") sendKeyPress(BACKSPACE); |
| 764 | else if (inputType === "deleteContentForward") sendKeyPress(DELETE); |
| 765 | else { |
| 766 | const text = textarea.value.replace(/^\u200b/, ""); |
| 767 | if (text) send({ type: "text", text }); |
| 768 | } |
| 769 | resetInput(textarea); |
| 770 | }; |
| 771 | |
| 772 | return ( |
| 773 | <div |
| 774 | className={cn("relative flex flex-col overflow-hidden", className)} |
| 775 | data-server-browser-surface={tabId} |
| 776 | onMouseUp={handleMouseUp} |
| 777 | > |
| 778 | <div |
| 779 | className={cn( |
| 780 | "visible relative z-20 flex shrink-0 items-center justify-between gap-2 border-border bg-background px-2 py-1", |
| 781 | controlPosition === "bottom" ? "order-last border-t" : "border-b", |
| 782 | )} |
| 783 | > |
| 784 | <span role="status" className="text-xs text-muted-foreground"> |
| 785 | {previewStreamControlLabel(control, resolve)} |
| 786 | </span> |
| 787 | {control?.canOperate ? ( |
| 788 | <Button |
| 789 | variant="outline" |
| 790 | size="xs" |
| 791 | disabled={control.controller === "another-viewer"} |
| 792 | onClick={() => |
| 793 | send({ type: control.controller === "you" ? "releaseControl" : "takeControl" }) |
| 794 | } |
| 795 | > |
| 796 | {control.controller === "you" ? <T>Release control</T> : <T>Take control</T>} |
| 797 | </Button> |
| 798 | ) : null} |
| 799 | </div> |
| 800 | <div className="relative min-h-0 flex-1"> |
| 801 | {pdfUrl ? ( |
| 802 | <embed |
| 803 | src={pdfUrl} |
| 804 | type="application/pdf" |
| 805 | title={gt("PDF document")} |
| 806 | className="absolute inset-0 z-10 size-full border-0" |
| 807 | /> |
| 808 | ) : null} |
| 809 | <canvas |
| 810 | ref={canvasRef} |
| 811 | className="block size-full touch-none object-contain" |
| 812 | onPointerDown={handlePointer} |
| 813 | onPointerMove={handlePointer} |
| 814 | onPointerUp={handlePointer} |
| 815 | onPointerCancel={handlePointer} |
| 816 | onMouseDown={(event) => event.preventDefault()} |
| 817 | onContextMenu={(event) => event.preventDefault()} |
| 818 | /> |
| 819 | {!pdfUrl && agentCursor && control?.controller !== "you" ? ( |
| 820 | <AgentCursorMark |
| 821 | {...agentCursor} |
| 822 | controller={control?.controller === "agent" ? "agent" : "none"} |
| 823 | /> |
| 824 | ) : null} |
| 825 | {/* Pin top-left to avoid focus scrolling; 16px prevents iOS focus zoom. */} |
| 826 | <textarea |
| 827 | ref={inputRef} |
| 828 | aria-label={gt("Browser page")} |
| 829 | aria-description={gt("Press Shift+Escape to leave the browser page.")} |
| 830 | autoCapitalize="off" |
| 831 | autoComplete="off" |
| 832 | autoCorrect="off" |
| 833 | spellCheck={false} |
| 834 | disabled={pdfUrl !== null || control?.controller !== "you"} |
| 835 | defaultValue={INPUT_SENTINEL} |
| 836 | onFocus={(event) => resetInput(event.currentTarget)} |
| 837 | className="sr-only top-0 left-0 text-base" |
| 838 | onKeyDown={(event) => handleKey("down", event)} |
| 839 | onKeyUp={(event) => handleKey("up", event)} |
| 840 | onInput={handleInput} |
| 841 | onCompositionEnd={(event) => { |
| 842 | const text = event.data.replace(/^\u200b/, ""); |
| 843 | if (text) send({ type: "text", text }); |
| 844 | resetInput(event.currentTarget); |
| 845 | }} |
| 846 | onCopy={(event) => handleClipboard(COPY, event)} |
| 847 | onCut={(event) => handleClipboard(CUT, event)} |
| 848 | onPaste={(event) => { |
| 849 | event.preventDefault(); |
| 850 | const text = event.clipboardData.getData("text/plain"); |
| 851 | if (text) send({ type: "text", text }); |
| 852 | }} |
| 853 | /> |
| 854 | {visible && framesStalled && !pdfUrl ? ( |
| 855 | <div className="absolute inset-0 flex items-center justify-center bg-background p-4 text-center"> |
| 856 | <p role="alert" className="max-w-sm text-sm text-muted-foreground"> |
| 857 | <T> |
| 858 | The server's browser loaded this page but is not drawing it. Try reloading, or open |
| 859 | the page in a browser on this computer. |
| 860 | </T> |
| 861 | </p> |
| 862 | </div> |
| 863 | ) : null} |
| 864 | {fileChooser && control?.controller === "you" ? ( |
| 865 | <div |
| 866 | className="absolute inset-x-2 top-2 z-10 flex flex-col gap-2 rounded-lg border border-border bg-background p-3 shadow-lg" |
| 867 | role="dialog" |
| 868 | aria-label={gt("Choose files for the page")} |
| 869 | > |
| 870 | <p className="text-sm"> |
| 871 | {fileChooser.multiple ? ( |
| 872 | <T>The page asks for files.</T> |
| 873 | ) : ( |
| 874 | <T>The page asks for a file.</T> |
| 875 | )} |
| 876 | </p> |
| 877 | <input |
| 878 | ref={fileInputRef} |
| 879 | type="file" |
| 880 | className="hidden" |
| 881 | multiple={fileChooser.multiple} |
| 882 | accept={fileChooser.accept} |
| 883 | onChange={(event) => { |
| 884 | const files = [...(event.currentTarget.files ?? [])]; |
| 885 | event.currentTarget.value = ""; |
| 886 | if (files.length > 0) answerFileChooser(files); |
| 887 | }} |
| 888 | /> |
| 889 | <div className="flex justify-end gap-2"> |
| 890 | <Button variant="outline" size="sm" onClick={() => answerFileChooser([])}> |
| 891 | <T>Cancel</T> |
| 892 | </Button> |
| 893 | <Button size="sm" onClick={() => fileInputRef.current?.click()}> |
| 894 | {fileChooser.multiple ? <T>Choose files</T> : <T>Choose file</T>} |
| 895 | </Button> |
| 896 | </div> |
| 897 | </div> |
| 898 | ) : null} |
| 899 | {control?.dialog ? ( |
| 900 | <div |
| 901 | className="absolute inset-x-2 top-2 z-10 flex flex-col gap-2 rounded-lg border border-border bg-background p-3 shadow-lg" |
| 902 | role="dialog" |
| 903 | aria-label={gt("Browser dialog")} |
| 904 | > |
| 905 | <p className="break-words text-sm">{control.dialog.message}</p> |
| 906 | {control.controller === "you" ? ( |
| 907 | <> |
| 908 | {control.dialog.type === "prompt" ? ( |
| 909 | <Input |
| 910 | aria-label={gt("Dialog response")} |
| 911 | value={promptText} |
| 912 | onChange={(event) => setPromptText(event.target.value)} |
| 913 | /> |
| 914 | ) : null} |
| 915 | <div className="flex justify-end gap-2"> |
| 916 | <Button |
| 917 | variant="outline" |
| 918 | size="sm" |
| 919 | onClick={() => send({ type: "dialog", accept: false })} |
| 920 | > |
| 921 | <T>Dismiss</T> |
| 922 | </Button> |
| 923 | <Button |
| 924 | size="sm" |
| 925 | onClick={() => |
| 926 | send({ |
| 927 | type: "dialog", |
| 928 | accept: true, |
| 929 | ...(control.dialog?.type === "prompt" ? { promptText } : {}), |
| 930 | }) |
| 931 | } |
| 932 | > |
| 933 | <T>Accept</T> |
| 934 | </Button> |
| 935 | </div> |
| 936 | </> |
| 937 | ) : ( |
| 938 | <p className="text-xs text-muted-foreground"> |
| 939 | <T>Take control to respond.</T> |
| 940 | </p> |
| 941 | )} |
| 942 | </div> |
| 943 | ) : null} |
| 944 | </div> |
| 945 | {visible && hostSetup ? ( |
| 946 | <div className="visible absolute inset-0 z-20 flex flex-col items-center justify-center gap-3 bg-background p-4 text-center"> |
| 947 | <p role="alert" className="max-w-sm text-sm text-muted-foreground"> |
| 948 | {previewStreamHostSetupMessage(hostSetup, resolve)} |
| 949 | </p> |
| 950 | <CommandBlock command={hostSetup.command} className="w-full max-w-md text-left" /> |
| 951 | <Button variant="outline" size="sm" onClick={() => setHostSetup(null)}> |
| 952 | <T>Try again</T> |
| 953 | </Button> |
| 954 | </div> |
| 955 | ) : null} |
| 956 | {visible && accessDenied ? ( |
| 957 | // The page can be invisible beneath an empty or unreachable state; reconnect must remain reachable. |
| 958 | <div className="visible absolute inset-0 z-20 flex flex-col items-center justify-center gap-2 bg-background p-3 text-center"> |
| 959 | <p role="alert" className="text-xs text-muted-foreground"> |
| 960 | <T>Browser connection was refused.</T> |
| 961 | </p> |
| 962 | <Button |
| 963 | variant="outline" |
| 964 | size="sm" |
| 965 | onClick={() => { |
| 966 | unauthorizedRef.current = 0; |
| 967 | setAccessDenied(false); |
| 968 | refreshPreviewStreamAccess(environmentId); |
| 969 | }} |
| 970 | > |
| 971 | <T>Reconnect</T> |
| 972 | </Button> |
| 973 | </div> |
| 974 | ) : null} |
| 975 | </div> |
| 976 | ); |
| 977 | } |
| 978 | |
| 979 | interface AgentCursorPlacement { |
| 980 | readonly phase: PreviewStreamPointer["phase"]; |
| 981 | readonly sequence: number; |
| 982 | readonly left: number; |
| 983 | readonly top: number; |
| 984 | } |
| 985 | |
| 986 | function placeAgentCursor( |
| 987 | pointer: PreviewStreamPointer, |
| 988 | canvas: HTMLCanvasElement | null, |
| 989 | viewport: PreviewStreamViewport | null, |
| 990 | ): AgentCursorPlacement | null { |
| 991 | if (!canvas || !viewport || canvas.width < 1 || canvas.height < 1) return null; |
| 992 | const box = { width: canvas.clientWidth, height: canvas.clientHeight }; |
| 993 | const fit = Math.min(box.width / canvas.width, box.height / canvas.height); |
| 994 | const width = canvas.width * fit; |
| 995 | const height = canvas.height * fit; |
| 996 | return { |
| 997 | phase: pointer.phase, |
| 998 | sequence: pointer.sequence, |
| 999 | left: (box.width - width) / 2 + (pointer.x * width) / viewport.width, |
| 1000 | top: (box.height - height) / 2 + (pointer.y * height) / viewport.height, |
| 1001 | }; |
| 1002 | } |