@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 707 lines 26.4 KB
Raw
1 import {
2 createPreviewFramePainter,
3 createPreviewStreamClient,
4 previewStreamModifiers,
5 type PreviewStreamClient,
6 type PreviewStreamControl,
7 type PreviewStreamInput,
8 type PreviewStreamPointer,
9 type PreviewStreamViewport,
10 } from "@t3tools/client-runtime/preview/server-browser-stream";
11
12 import type { PreviewStreamConfiguration, PreviewStreamMessage } from "./preview-stream-document";
13
14 declare global {
15 interface Window {
16 ReactNativeWebView: { postMessage: (message: string) => void };
17 }
18 }
19
20 interface PresentationVideo extends HTMLVideoElement {
21 webkitSetPresentationMode?: (mode: "inline" | "picture-in-picture") => void;
22 webkitPresentationMode?: string;
23 }
24
25 type WheelInput = Extract<PreviewStreamInput, { type: "wheel" }>;
26 type MouseInput = Extract<PreviewStreamInput, { type: "mouse" }>;
27 const mouseButton = (button: number): MouseInput["button"] =>
28 button === 0 ? "left" : button === 1 ? "middle" : button === 2 ? "right" : "none";
29
30 const RESIZE_DEBOUNCE_MS = 150;
31 const TAP_SLOP_PX = 8;
32 const MULTI_CLICK_MS = 500;
33 const MULTI_CLICK_SLOP_PX = 4;
34 const WHEEL_LINE_PX = 16;
35 const SCROLL_REPORT_PX = 24;
36 const MAX_PIXEL_RATIO = 2;
37 // Reuse nearby probe answers: slow replies arrive too late after touch-end to raise the iOS keyboard.
38 const PROBE_REUSE_PX = 24;
39 const PROBE_REUSE_MS = 10_000;
40 // A sentinel makes backspace fire input; Gboard keydown supplies only keyCode 229.
41 const SENTINEL = "\u200b";
42 const AGENT_CURSOR_ACTIVE_MS = 700;
43 const AGENT_CURSOR_SVG =
44 '<svg width="20" height="20" viewBox="0 0 24 24" fill="#fff" stroke="#3b82f6" stroke-width="2" stroke-linecap="round" stroke-linejoin="round"><path d="M4.037 4.688a.495.495 0 0 1 .651-.651l16 6.5a.5.5 0 0 1-.063.947l-6.124 1.58a2 2 0 0 0-1.438 1.435l-1.579 6.126a.5.5 0 0 1-.947.063z"/></svg>';
45
46 interface Viewer {
47 readonly stop: () => void;
48 readonly command: (input: PreviewStreamInput) => void;
49 readonly togglePictureInPicture: () => void;
50 }
51
52 let activeViewer: Viewer | null = null;
53
54 export function stop() {
55 activeViewer?.stop();
56 activeViewer = null;
57 }
58
59 export function command(input: PreviewStreamInput) {
60 activeViewer?.command(input);
61 }
62
63 export function pictureInPicture() {
64 activeViewer?.togglePictureInPicture();
65 }
66
67 export function start(configuration: PreviewStreamConfiguration) {
68 stop();
69 const post = (message: PreviewStreamMessage) => {
70 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- The native WebView bridge takes one string.
71 window.ReactNativeWebView.postMessage(JSON.stringify(message));
72 };
73 const { interactive } = configuration;
74 document.body.style.background = configuration.background;
75 const container = document.createElement("div");
76 const canvas = document.createElement("canvas");
77 canvas.setAttribute("role", "img");
78 canvas.setAttribute("aria-label", configuration.labels.page);
79 const agentCursor = document.createElement("div");
80 agentCursor.className = "agent-cursor";
81 agentCursor.setAttribute("aria-hidden", "true");
82 agentCursor.innerHTML = AGENT_CURSOR_SVG;
83 container.append(canvas, agentCursor);
84 const input = document.createElement("textarea");
85 input.setAttribute("aria-label", configuration.labels.pageInput);
86 input.autocapitalize = "off";
87 input.autocomplete = "off";
88 input.spellcheck = false;
89 input.setAttribute("autocorrect", "off");
90 input.disabled = true;
91 document.body.replaceChildren(container, ...(interactive ? [input] : []));
92
93 let stopped = false;
94 let streaming = false;
95 let viewport: PreviewStreamViewport | null = null;
96 let size: { width: number; height: number } | null = null;
97 let cap: { width: number; height: number } | null = null;
98 let client: PreviewStreamClient | null = null;
99 let control: PreviewStreamControl | null = null;
100 let resizeTimer: ReturnType<typeof setTimeout> | null = null;
101 let wheelFrame: number | null = null;
102 let pendingWheel: WheelInput | null = null;
103 let mouseFrame: number | null = null;
104 let pendingMouse: MouseInput | null = null;
105 let mouseButtons = 0;
106 const mouseClicks = { left: 1, middle: 1, right: 1, none: 1 };
107 let probe: {
108 readonly x: number;
109 readonly y: number;
110 readonly clientX: number;
111 readonly clientY: number;
112 editable: boolean | null;
113 end: "touching" | "acted" | "late" | "panned";
114 } | null = null;
115 let probeCache: {
116 readonly clientX: number;
117 readonly clientY: number;
118 readonly editable: boolean;
119 readonly time: number;
120 } | null = null;
121 let keySent = false;
122 let touch: {
123 start: PointerEvent;
124 last: PointerEvent;
125 panning: boolean;
126 } | null = null;
127 let lastTap: {
128 time: number;
129 x: number;
130 y: number;
131 count: number;
132 button: MouseInput["button"];
133 } | null = null;
134
135 const reportStatus = (status: "connecting" | "streaming") => {
136 streaming = status === "streaming";
137 post({ type: "status", status });
138 };
139 const painter = createPreviewFramePainter(canvas, () => {
140 if (!streaming && !stopped) reportStatus("streaming");
141 sendPictureInPictureFrame();
142 });
143 const send = (message: PreviewStreamInput) => client?.send(message) ?? false;
144 const clearInput = () => {
145 if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
146 if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
147 wheelFrame = mouseFrame = null;
148 pendingWheel = pendingMouse = null;
149 mouseButtons = 0;
150 Object.assign(mouseClicks, { left: 1, middle: 1, right: 1, none: 1 });
151 probe = probeCache = touch = lastTap = null;
152 keySent = false;
153 input.value = SENTINEL;
154 input.disabled = true;
155 input.blur();
156 };
157
158 const connect = () => {
159 if (client) reportStatus("connecting");
160 client?.stop();
161 control = null;
162 clearInput();
163 if (!cap || stopped) return;
164 const next = createPreviewStreamClient(
165 {
166 access: configuration.access,
167 threadId: configuration.threadId,
168 tabId: configuration.tabId,
169 maxWidth: cap.width,
170 maxHeight: cap.height,
171 interactive,
172 },
173 {
174 onFrame: (jpeg) => painter.paint(jpeg),
175 onPointer: showAgentCursor,
176 onClipboard: (text) => post({ type: "clipboard", text }),
177 onDownload: (download) => post({ type: "download", ...download }),
178 onFileChooser: (chooser) => post({ type: "fileChooser", chooser }),
179 onViewport: (page) => {
180 if (viewport?.width === page.width && viewport.height === page.height) return;
181 viewport = page;
182 post({ type: "viewport", width: page.width, height: page.height });
183 },
184 onProbe: (result) => {
185 if (control?.controller !== "you") return;
186 const current = probe;
187 if (!current || current.x !== result.x || current.y !== result.y) return;
188 probeCache = {
189 clientX: current.clientX,
190 clientY: current.clientY,
191 editable: result.editable,
192 time: performance.now(),
193 };
194 if (current.end === "touching") {
195 current.editable = result.editable;
196 return;
197 }
198 probe = null;
199 if (current.end === "panned") return;
200 if (current.end === "acted" && current.editable === result.editable) return;
201 // Late answers can raise Android's keyboard; iOS needs another tap.
202 if (result.editable) input.focus({ preventScroll: true });
203 else input.blur();
204 },
205 onControl: (nextControl) => {
206 const previous = control;
207 control = nextControl;
208 post({ type: "control", ...nextControl });
209 if (nextControl.controller === "you") agentCursor.style.opacity = "0";
210 if (nextControl.controller !== "you") clearInput();
211 else {
212 input.disabled = !interactive;
213 if (interactive && size && previous?.controller !== "you")
214 next.send({ type: "resize", ...size });
215 }
216 },
217 onConnectedChange: (connected) => {
218 if (!connected) {
219 control = null;
220 clearInput();
221 if (streaming) reportStatus("connecting");
222 return;
223 }
224 },
225 onUnauthorized: () => post({ type: "unauthorized" }),
226 onGone: () => post({ type: "gone" }),
227 onHostSetup: (setup) => post({ type: "hostSetup", ...setup }),
228 },
229 );
230 client = next;
231 };
232
233 const measure = () => {
234 resizeTimer = null;
235 const rect = container.getBoundingClientRect();
236 if (rect.width < 1 || rect.height < 1) return;
237 const next = { width: Math.round(rect.width), height: Math.round(rect.height) };
238 if (size?.width === next.width && size.height === next.height) return;
239 size = next;
240 if (interactive) send({ type: "resize", ...next });
241 const ratio = Math.min(window.devicePixelRatio || 1, MAX_PIXEL_RATIO);
242 const side = Math.max(next.width, next.height);
243 const width = Math.round((interactive ? next.width : side) * ratio);
244 const height = Math.round((interactive ? next.height : side) * ratio);
245 if (cap && cap.width >= width && cap.height >= height) return;
246 cap = { width: Math.max(width, cap?.width ?? 0), height: Math.max(height, cap?.height ?? 0) };
247 connect();
248 };
249 const observer = new ResizeObserver(() => {
250 if (resizeTimer !== null) clearTimeout(resizeTimer);
251 resizeTimer = setTimeout(measure, size === null ? 0 : RESIZE_DEBOUNCE_MS);
252 });
253 observer.observe(container);
254
255 // iOS keeps its layout viewport under the keyboard; resize to the visual viewport.
256 const visualViewport = window.visualViewport;
257 const followVisualViewport = () => {
258 if (!visualViewport) return;
259 container.style.top = `${visualViewport.offsetTop}px`;
260 container.style.height = `${visualViewport.height}px`;
261 };
262 if (interactive) {
263 visualViewport?.addEventListener("resize", followVisualViewport);
264 visualViewport?.addEventListener("scroll", followVisualViewport);
265 }
266
267 let agentCursorTimer: ReturnType<typeof setTimeout> | null = null;
268 const showAgentCursor = (pointer: PreviewStreamPointer) => {
269 if (!viewport || canvas.width === 0 || canvas.height === 0 || control?.controller === "you")
270 return;
271 const box = { width: canvas.clientWidth, height: canvas.clientHeight };
272 const fit = Math.min(box.width / canvas.width, box.height / canvas.height);
273 const width = canvas.width * fit;
274 const height = canvas.height * fit;
275 const left = (box.width - width) / 2 + (pointer.x * width) / viewport.width;
276 const top = (box.height - height) / 2 + (pointer.y * height) / viewport.height;
277 agentCursor.style.transform = `translate3d(${left}px, ${top}px, 0)`;
278 agentCursor.style.opacity = "1";
279 if (pointer.phase === "click") {
280 agentCursor.querySelector(".ping")?.remove();
281 const ping = document.createElement("span");
282 ping.className = "ping";
283 agentCursor.prepend(ping);
284 }
285 if (agentCursorTimer !== null) clearTimeout(agentCursorTimer);
286 agentCursorTimer = setTimeout(() => {
287 agentCursor.style.opacity = control?.controller === "agent" ? "0.35" : "0";
288 }, AGENT_CURSOR_ACTIVE_MS);
289 };
290
291 const pagePoint = (clientX: number, clientY: number, clamp: boolean) => {
292 if (!viewport || canvas.width === 0 || canvas.height === 0 || control?.controller !== "you")
293 return null;
294 const rect = canvas.getBoundingClientRect();
295 const fit = Math.min(rect.width / canvas.width, rect.height / canvas.height);
296 const width = canvas.width * fit;
297 const height = canvas.height * fit;
298 if (!(width > 0 && height > 0)) return null;
299 const scale = viewport.width / width;
300 const x = (clientX - rect.left - (rect.width - width) / 2) * scale;
301 const y = (clientY - rect.top - (rect.height - height) / 2) * (viewport.height / height);
302 const inside = x >= 0 && y >= 0 && x <= viewport.width && y <= viewport.height;
303 if (!inside && !clamp) return null;
304 return {
305 x: Math.min(Math.max(x, 0), viewport.width),
306 y: Math.min(Math.max(y, 0), viewport.height),
307 scale,
308 };
309 };
310
311 let scrolled = 0;
312 const flushWheel = () => {
313 if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
314 wheelFrame = null;
315 const wheel = pendingWheel;
316 pendingWheel = null;
317 if (!wheel) return;
318 send(wheel);
319 // The app shrinks its bar while the page scrolls down, so it only needs the direction.
320 scrolled =
321 Math.sign(scrolled) === Math.sign(wheel.deltaY) ? scrolled + wheel.deltaY : wheel.deltaY;
322 if (Math.abs(scrolled) < SCROLL_REPORT_PX) return;
323 post({ type: "scroll", direction: scrolled > 0 ? "down" : "up" });
324 scrolled = 0;
325 };
326 const queueWheel = (point: { x: number; y: number }, deltaX: number, deltaY: number) => {
327 pendingWheel = {
328 type: "wheel",
329 x: point.x,
330 y: point.y,
331 deltaX: (pendingWheel?.deltaX ?? 0) + deltaX,
332 deltaY: (pendingWheel?.deltaY ?? 0) + deltaY,
333 modifiers: 0,
334 };
335 wheelFrame ??= requestAnimationFrame(flushWheel);
336 };
337
338 const flushMouse = () => {
339 if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
340 mouseFrame = null;
341 if (pendingMouse) send(pendingMouse);
342 pendingMouse = null;
343 };
344 const countClick = (event: PointerEvent, button: MouseInput["button"]) => {
345 const last = lastTap;
346 const count =
347 last &&
348 last.button === button &&
349 event.timeStamp - last.time < MULTI_CLICK_MS &&
350 Math.hypot(event.clientX - last.x, event.clientY - last.y) < MULTI_CLICK_SLOP_PX
351 ? last.count + 1
352 : 1;
353 lastTap = { time: event.timeStamp, x: event.clientX, y: event.clientY, count, button };
354 return count;
355 };
356 const mouseInput = (
357 event: PointerEvent,
358 point: { x: number; y: number },
359 action: MouseInput["action"],
360 button: MouseInput["button"],
361 buttons = event.buttons,
362 clickCount = mouseClicks[button],
363 ): MouseInput => ({
364 type: "mouse",
365 action,
366 x: point.x,
367 y: point.y,
368 button,
369 buttons,
370 clickCount,
371 modifiers: previewStreamModifiers(event),
372 });
373 const onPointerDown = (event: PointerEvent) => {
374 if (control?.controller !== "you") return;
375 if (!event.isPrimary) return;
376 event.preventDefault();
377 if (event.pointerType !== "touch") {
378 const point = pagePoint(event.clientX, event.clientY, false);
379 if (!point) return;
380 canvas.setPointerCapture(event.pointerId);
381 input.focus({ preventScroll: true });
382 flushMouse();
383 flushWheel();
384 mouseButtons = event.buttons & 7;
385 const button = mouseButton(event.button);
386 mouseClicks[button] = countClick(event, button);
387 send(mouseInput(event, point, "down", button));
388 return;
389 }
390 touch = { start: event, last: event, panning: false };
391 const point = pagePoint(event.clientX, event.clientY, false);
392 if (!point) {
393 probe = null;
394 return;
395 }
396 const cached = probeCache;
397 const reuse =
398 cached !== null &&
399 performance.now() - cached.time < PROBE_REUSE_MS &&
400 Math.hypot(event.clientX - cached.clientX, event.clientY - cached.clientY) <= PROBE_REUSE_PX;
401 probe = {
402 x: point.x,
403 y: point.y,
404 clientX: event.clientX,
405 clientY: event.clientY,
406 editable: reuse ? cached.editable : null,
407 end: "touching",
408 };
409 send({ type: "probe", x: point.x, y: point.y });
410 };
411 const onPointerMove = (event: PointerEvent) => {
412 if (control?.controller !== "you") return;
413 if (event.pointerType !== "touch") {
414 const point = pagePoint(event.clientX, event.clientY, mouseButtons !== 0);
415 if (!point) return;
416 // Chorded presses and releases arrive as pointermove while another button is held.
417 const changed = mouseButtons ^ (event.buttons & 7);
418 if (mouseButtons !== 0 && changed !== 0) {
419 flushMouse();
420 flushWheel();
421 for (const bit of [1, 2, 4]) {
422 if (!(changed & bit)) continue;
423 const button = bit === 1 ? "left" : bit === 2 ? "right" : "middle";
424 const down = (event.buttons & bit) !== 0;
425 mouseButtons ^= bit;
426 if (down) mouseClicks[button] = countClick(event, button);
427 send(mouseInput(event, point, down ? "down" : "up", button, mouseButtons));
428 }
429 }
430 const button =
431 mouseButtons & 1
432 ? "left"
433 : mouseButtons & 2
434 ? "right"
435 : mouseButtons & 4
436 ? "middle"
437 : "none";
438 pendingMouse = mouseInput(event, point, "move", button, event.buttons, 0);
439 mouseFrame ??= requestAnimationFrame(flushMouse);
440 return;
441 }
442 if (!touch || touch.start.pointerId !== event.pointerId) return;
443 if (
444 !touch.panning &&
445 Math.hypot(event.clientX - touch.start.clientX, event.clientY - touch.start.clientY) <
446 TAP_SLOP_PX
447 ) {
448 return;
449 }
450 touch.panning = true;
451 const point = pagePoint(event.clientX, event.clientY, true);
452 if (point) {
453 queueWheel(
454 point,
455 (touch.last.clientX - event.clientX) * point.scale,
456 (touch.last.clientY - event.clientY) * point.scale,
457 );
458 }
459 touch.last = event;
460 };
461 const releaseMouse = (event: PointerEvent, cancelled: boolean) => {
462 let buttons = mouseButtons;
463 if (buttons === 0) return;
464 mouseButtons = cancelled ? 0 : event.buttons & 7;
465 flushMouse();
466 flushWheel();
467 const point = pagePoint(event.clientX, event.clientY, true);
468 if (!point) return;
469 for (const bit of [1, 2, 4]) {
470 if (!(buttons & bit) || (mouseButtons & bit) !== 0) continue;
471 buttons &= ~bit;
472 const button = bit === 1 ? "left" : bit === 2 ? "right" : "middle";
473 send(mouseInput(event, point, "up", button, buttons));
474 }
475 };
476 const onPointerEnd = (event: PointerEvent) => {
477 const cancelled = event.type === "pointercancel";
478 if (event.pointerType !== "touch") {
479 releaseMouse(event, cancelled);
480 return;
481 }
482 const ended = touch;
483 if (!ended || ended.start.pointerId !== event.pointerId) return;
484 touch = null;
485 const answered = probe;
486 if (cancelled || ended.panning) {
487 if (answered) answered.end = "panned";
488 return;
489 }
490 const point = pagePoint(event.clientX, event.clientY, false);
491 if (!point) return;
492 // Focusing inside the tap's user activation is what lets iOS raise the keyboard.
493 if (answered?.editable === true) input.focus({ preventScroll: true });
494 else if (answered?.editable === false) input.blur();
495 if (answered) answered.end = answered.editable === null ? "late" : "acted";
496 flushWheel();
497 const clickCount = countClick(event, "left");
498 const at = { x: point.x, y: point.y, modifiers: 0 };
499 send({ type: "mouse", action: "move", ...at, button: "none", buttons: 0, clickCount: 0 });
500 send({ type: "mouse", action: "down", ...at, button: "left", buttons: 1, clickCount });
501 send({ type: "mouse", action: "up", ...at, button: "left", buttons: 0, clickCount });
502 };
503 const onWheel = (event: WheelEvent) => {
504 const point = pagePoint(event.clientX, event.clientY, false);
505 if (!point) return;
506 event.preventDefault();
507 const unit =
508 event.deltaMode === WheelEvent.DOM_DELTA_LINE
509 ? WHEEL_LINE_PX
510 : event.deltaMode === WheelEvent.DOM_DELTA_PAGE
511 ? (viewport?.height ?? 0)
512 : 1;
513 queueWheel(point, event.deltaX * unit, event.deltaY * unit);
514 };
515 const preventDefault = (event: Event) => event.preventDefault();
516
517 const onKey = (action: "down" | "up", event: KeyboardEvent) => {
518 if (control?.controller !== "you") return;
519 if (
520 event.isComposing ||
521 event.keyCode === 229 ||
522 event.key === "Process" ||
523 event.key === "Unidentified"
524 ) {
525 return;
526 }
527 const shortcut = event.ctrlKey || event.metaKey;
528 if (shortcut && event.key.toLowerCase() === "v") return;
529 if (event.shiftKey && event.key === "Insert") return;
530 // Enter carries "\r" like Puppeteer's key table, so forms submit and textareas break lines.
531 const text = shortcut
532 ? undefined
533 : [...event.key].length === 1
534 ? event.key
535 : event.key === "Enter"
536 ? "\r"
537 : undefined;
538 send({
539 type: "key",
540 action,
541 key: event.key,
542 code: event.code,
543 keyCode: event.keyCode,
544 ...(action === "down" && text !== undefined ? { text } : {}),
545 modifiers: previewStreamModifiers(event),
546 });
547 // Some Android keyboards edit the textarea even when keydown is prevented.
548 keySent =
549 action === "down" &&
550 !shortcut &&
551 (text !== undefined || event.key === "Backspace" || event.key === "Delete");
552 if (!shortcut) event.preventDefault();
553 };
554 const resetInput = () => {
555 input.value = SENTINEL;
556 input.setSelectionRange(SENTINEL.length, SENTINEL.length);
557 };
558 const pressKey = (key: "Backspace" | "Delete", keyCode: number) => {
559 for (const action of ["down", "up"] as const) {
560 send({ type: "key", action, key, code: key, keyCode, modifiers: 0 });
561 }
562 };
563 const onInput = (event: Event) => {
564 if (event instanceof InputEvent && event.isComposing) return;
565 const inputType = event instanceof InputEvent ? event.inputType : "";
566 if (keySent) keySent = false;
567 else if (inputType === "deleteContentBackward") pressKey("Backspace", 8);
568 else if (inputType === "deleteContentForward") pressKey("Delete", 46);
569 else {
570 const value = input.value;
571 const text = value.startsWith(SENTINEL) ? value.slice(SENTINEL.length) : value;
572 if (text) send({ type: "text", text });
573 }
574 resetInput();
575 };
576 const onCompositionEnd = (event: CompositionEvent) => {
577 const text = event.data.startsWith(SENTINEL) ? event.data.slice(SENTINEL.length) : event.data;
578 if (text) send({ type: "text", text });
579 resetInput();
580 };
581
582 if (interactive) {
583 canvas.addEventListener("pointerdown", onPointerDown);
584 canvas.addEventListener("pointermove", onPointerMove);
585 canvas.addEventListener("pointerup", onPointerEnd);
586 canvas.addEventListener("pointercancel", onPointerEnd);
587 canvas.addEventListener("wheel", onWheel, { passive: false });
588 canvas.addEventListener("mousedown", preventDefault);
589 canvas.addEventListener("contextmenu", preventDefault);
590 input.addEventListener("keydown", (event) => onKey("down", event));
591 input.addEventListener("keyup", (event) => onKey("up", event));
592 input.addEventListener("input", onInput);
593 input.addEventListener("compositionend", onCompositionEnd);
594 input.addEventListener("focus", resetInput);
595 // Copying the input would put its sentinel on this device's clipboard.
596 input.addEventListener("copy", preventDefault);
597 input.addEventListener("cut", preventDefault);
598 resetInput();
599 }
600
601 // WebKit reports no support on a video that has not loaded yet; check the prototype.
602 const videoPrototype: PresentationVideo = HTMLVideoElement.prototype;
603 const pictureInPictureSupported =
604 interactive &&
605 typeof canvas.captureStream === "function" &&
606 typeof CanvasCaptureMediaStreamTrack === "function" &&
607 ((document.pictureInPictureEnabled === true &&
608 typeof videoPrototype.requestPictureInPicture === "function") ||
609 typeof videoPrototype.webkitSetPresentationMode === "function");
610 // WebKit revokes activation after return; start picture in picture before awaiting. Prepare paused video before the tap.
611 let video: PresentationVideo | null = null;
612 let track: CanvasCaptureMediaStreamTrack | null = null;
613 const pictureInPictureActive = () =>
614 video !== null &&
615 (document.pictureInPictureElement === video ||
616 video.webkitPresentationMode === "picture-in-picture");
617 const reportPictureInPicture = (detail?: string) =>
618 post({
619 type: "pictureInPicture",
620 supported: pictureInPictureSupported,
621 active: pictureInPictureActive(),
622 ...(detail ? { detail } : {}),
623 });
624 // Called after each paint. A frame drawn before the player attached is lost, so retry.
625 const sendPictureInPictureFrame = () => {
626 if (video && (video.readyState === HTMLMediaElement.HAVE_NOTHING || pictureInPictureActive())) {
627 track?.requestFrame();
628 }
629 };
630 if (pictureInPictureSupported) {
631 const stream = canvas.captureStream(0);
632 const [captured] = stream.getVideoTracks();
633 const element: PresentationVideo = document.createElement("video");
634 element.muted = true;
635 element.playsInline = true;
636 // Under the canvas at full size: WebKit pauses muted video it considers off screen.
637 element.srcObject = stream;
638 for (const name of [
639 "enterpictureinpicture",
640 "leavepictureinpicture",
641 "webkitpresentationmodechanged",
642 ]) {
643 element.addEventListener(name, () => {
644 if (!pictureInPictureActive()) element.pause();
645 reportPictureInPicture();
646 });
647 }
648 container.prepend(element);
649 canvas.style.background = configuration.background;
650 video = element;
651 track = captured instanceof CanvasCaptureMediaStreamTrack ? captured : null;
652 }
653 const togglePictureInPicture = () => {
654 const element = video;
655 if (!element) return;
656 if (pictureInPictureActive()) {
657 if (document.pictureInPictureElement) void document.exitPictureInPicture().catch(() => {});
658 else element.webkitSetPresentationMode?.("inline");
659 return;
660 }
661 if (element.readyState === HTMLMediaElement.HAVE_NOTHING) {
662 reportPictureInPicture(configuration.labels.pictureInPictureNotReady);
663 return;
664 }
665 // The page may have changed since the video's last frame. WebKit captures only on a draw.
666 track?.requestFrame();
667 canvas.getContext("2d")?.drawImage(canvas, 0, 0);
668 const fail = (error: unknown) => {
669 element.pause();
670 reportPictureInPicture(
671 error instanceof Error ? error.message : configuration.labels.pictureInPictureUnavailable,
672 );
673 };
674 if (document.pictureInPictureEnabled) {
675 const entering = element.requestPictureInPicture();
676 void Promise.all([entering, element.play()]).catch(fail);
677 } else {
678 element.webkitSetPresentationMode?.("picture-in-picture");
679 void element.play().catch(fail);
680 }
681 };
682
683 activeViewer = {
684 stop: () => {
685 stopped = true;
686 observer.disconnect();
687 visualViewport?.removeEventListener("resize", followVisualViewport);
688 visualViewport?.removeEventListener("scroll", followVisualViewport);
689 if (resizeTimer !== null) clearTimeout(resizeTimer);
690 if (wheelFrame !== null) cancelAnimationFrame(wheelFrame);
691 if (mouseFrame !== null) cancelAnimationFrame(mouseFrame);
692 if (agentCursorTimer !== null) clearTimeout(agentCursorTimer);
693 painter.stop();
694 client?.stop();
695 client = null;
696 track?.stop();
697 if (video) video.srcObject = null;
698 },
699 command: (message) => {
700 send(message);
701 },
702 togglePictureInPicture,
703 };
704 window.addEventListener("pagehide", stop, { once: true });
705 reportStatus("connecting");
706 if (interactive) reportPictureInPicture();
707 }