@samitouri / QOSami_Agents_Zero-2 / commits / 91b3bc96

Slim browser viewer frame transport

Negotiate binary and slim screencast frames so updated clients avoid base64-in-JSON and repeated viewer metadata on every frame. Clamp screencast capture to the subscriber viewport and DPR, lower streaming JPEG quality while preserving snapshot quality, and teach the client to manage binary frame URLs safely. Document the browser viewer performance plan and update regression coverage for the new transport contract.

Alessandro committed Jul 3, 2026 at 01:49 UTC 91b3bc9613c7ea314623fc04c634929106d99415
6 files changed +417 -107
docs/AGENTS.md
+1
@@ -9,6 +9,7 @@
9
10 - `README.md`, `quickstart.md`, `guides/`, and `setup/` cover user-facing setup and workflows.
11 - `developer/` covers compact developer references and source handoffs.
12 +- `plans/` covers implementation plans, migration notes, and staged technical roadmaps.
13 - `res/` contains documentation images and other documentation assets.
14
15 ## Local Contracts
plugins/_browser/AGENTS.md
+1
@@ -19,6 +19,7 @@
19 - Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
20 - Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
21 - Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
22 +- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback.
23 - Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
24 - Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
25 - Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback.
plugins/_browser/api/ws_browser.py
+200 -44
@@ -1,6 +1,7 @@
1 from __future__ import annotations
2
3 import asyncio
4 +import base64
5 import contextlib
6 import time
7 from typing import Any, ClassVar
@@ -15,7 +16,8 @@ FRAME_READ_TIMEOUT_SECONDS = 0.5
16 FRAME_RETRY_DELAY_SECONDS = 0.5
17 FRAME_STATE_REFRESH_SECONDS = 0.75
18 SNAPSHOT_STATE_POLL_SECONDS = 0.75
18 -SCREENCAST_QUALITY = 92
19 +SCREENCAST_STREAM_QUALITY = 80
20 +SCREENSHOT_QUALITY = 92
21 VIEWER_TRANSPORT_SCREENCAST = "screencast"
22 VIEWER_TRANSPORT_SNAPSHOT = "snapshot"
23 VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT}
@@ -97,10 +99,21 @@ class WsBrowser(WsHandler):
99 existing.cancel()
100 viewer_id = str(data.get("viewer_id") or "")
101 viewer_transport = self._viewer_transport(data)
102 + binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames")))
103 + slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames)))
104 + capture_scale = self._capture_scale_from_data(data)
105 snapshot = None
106 if runtime:
107 if viewer_transport == VIEWER_TRANSPORT_SCREENCAST:
103 - stream_task = self._stream_frames(sid, context_id, active_id, viewer_id)
108 + stream_task = self._stream_frames(
109 + sid,
110 + context_id,
111 + active_id,
112 + viewer_id,
113 + binary_frames=binary_frames,
114 + slim_frames=slim_frames,
115 + capture_scale=capture_scale,
116 + )
117 else:
118 stream_task = self._stream_state(sid, context_id, active_id, viewer_id)
119 self._streams[stream_key] = asyncio.create_task(stream_task)
@@ -115,6 +128,8 @@ class WsBrowser(WsHandler):
128 "all_browsers": True,
129 "viewer_id": viewer_id,
130 "viewer_transport": viewer_transport,
131 + "binary_frames": binary_frames,
132 + "slim_frames": slim_frames,
133 }
134
135 def _unsubscribe(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
@@ -157,9 +172,9 @@ class WsBrowser(WsHandler):
172 snapshot = None
173 if active_id:
174 try:
160 - quality = int(data.get("quality") or SCREENCAST_QUALITY)
175 + quality = int(data.get("quality") or SCREENSHOT_QUALITY)
176 except (TypeError, ValueError):
162 - quality = SCREENCAST_QUALITY
177 + quality = SCREENSHOT_QUALITY
178 with contextlib.suppress(Exception):
179 snapshot = await runtime.call(
180 "screenshot",
@@ -347,7 +362,7 @@ class WsBrowser(WsHandler):
362 if not browser_id:
363 return None
364 with contextlib.suppress(Exception):
350 - return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
365 + return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)
366 return None
367
368 async def _snapshot_for_browser(
@@ -358,7 +373,7 @@ class WsBrowser(WsHandler):
373 if not browser_id:
374 return None
375 with contextlib.suppress(Exception):
361 - return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
376 + return await runtime.call("screenshot", browser_id, quality=SCREENSHOT_QUALITY)
377 return None
378
379 async def _all_browser_tabs(self) -> list[dict[str, Any]]:
@@ -377,6 +392,10 @@ class WsBrowser(WsHandler):
392 context_id: str,
393 browser_id: int | str | None,
394 viewer_id: str = "",
395 + *,
396 + binary_frames: bool = False,
397 + slim_frames: bool = False,
398 + capture_scale: float = 1.0,
399 ) -> None:
400 runtime = None
401 stream_id = None
@@ -397,12 +416,14 @@ class WsBrowser(WsHandler):
416 browsers = listing.get("browsers") or []
417 active_id = self._active_browser_id(listing, browser_id)
418 if not active_id:
400 - await self._emit_empty_frame(
419 + await self._emit_viewer_state(
420 sid,
421 context_id,
422 + active_id,
423 browsers=browsers,
424 viewer_id=viewer_id,
405 - frame_source=VIEWER_TRANSPORT_SCREENCAST,
425 + state=None,
426 + viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
427 )
428 await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
429 continue
@@ -410,29 +431,26 @@ class WsBrowser(WsHandler):
431 screencast = await runtime.call(
432 "start_screencast",
433 active_id,
413 - quality=SCREENCAST_QUALITY,
434 + quality=SCREENCAST_STREAM_QUALITY,
435 every_nth_frame=1,
436 + capture_scale=capture_scale,
437 )
438 stream_id = screencast["stream_id"]
439 active_id = screencast["browser_id"]
440 state = screencast.get("state")
419 - await self.emit_to(
441 + await self._emit_viewer_state(
442 sid,
421 - "browser_viewer_frame",
422 - {
423 - "context_id": context_id,
424 - "viewer_id": viewer_id,
425 - "browser_id": active_id,
426 - "browsers": browsers,
427 - "image": "",
428 - "mime": "",
429 - "state": state,
430 - "frame_source": "state",
431 - "viewer_transport": VIEWER_TRANSPORT_SCREENCAST,
432 - },
443 + context_id,
444 + active_id,
445 + browsers=browsers,
446 + viewer_id=viewer_id,
447 + state=state,
448 + viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
449 )
450
451 last_state_refresh = 0.0
452 + last_state_signature = self._state_signature(active_id, browsers)
453 + frame_sequence = 0
454 while True:
455 now = time.monotonic()
456 if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
@@ -442,6 +460,18 @@ class WsBrowser(WsHandler):
460 if str(active_id) not in browser_ids:
461 break
462 state = self._state_for_browser(browsers, active_id, state)
463 + state_signature = self._state_signature(active_id, browsers)
464 + if state_signature != last_state_signature:
465 + await self._emit_viewer_state(
466 + sid,
467 + context_id,
468 + active_id,
469 + browsers=browsers,
470 + viewer_id=viewer_id,
471 + state=state,
472 + viewer_transport=VIEWER_TRANSPORT_SCREENCAST,
473 + )
474 + last_state_signature = state_signature
475 last_state_refresh = now
476
477 try:
@@ -455,14 +485,19 @@ class WsBrowser(WsHandler):
485 except TimeoutError:
486 continue
487
458 - frame["context_id"] = context_id
459 - frame["viewer_id"] = viewer_id
460 - frame["browser_id"] = active_id
461 - frame["browsers"] = browsers
462 - frame["state"] = state
463 - frame["frame_source"] = VIEWER_TRANSPORT_SCREENCAST
464 - frame["viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST
465 - await self.emit_to(sid, "browser_viewer_frame", frame)
488 + frame_sequence += 1
489 + payload = self._frame_payload(
490 + frame,
491 + context_id=context_id,
492 + viewer_id=viewer_id,
493 + browser_id=active_id,
494 + sequence=frame_sequence,
495 + binary_frames=binary_frames,
496 + )
497 + if not slim_frames:
498 + payload["browsers"] = browsers
499 + payload["state"] = state
500 + await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
501 except asyncio.CancelledError:
502 raise
503 except Exception:
@@ -515,20 +550,14 @@ class WsBrowser(WsHandler):
550 ),
551 )
552 if signature != last_signature:
518 - await self.emit_to(
553 + await self._emit_viewer_state(
554 sid,
520 - "browser_viewer_frame",
521 - {
522 - "context_id": context_id,
523 - "viewer_id": viewer_id,
524 - "browser_id": active_id,
525 - "browsers": browsers,
526 - "image": "",
527 - "mime": "",
528 - "state": state,
529 - "frame_source": VIEWER_TRANSPORT_SNAPSHOT,
530 - "viewer_transport": VIEWER_TRANSPORT_SNAPSHOT,
531 - },
555 + context_id,
556 + active_id,
557 + browsers=browsers,
558 + viewer_id=viewer_id,
559 + state=state,
560 + viewer_transport=VIEWER_TRANSPORT_SNAPSHOT,
561 )
562 last_signature = signature
563 await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS)
@@ -567,6 +596,105 @@ class WsBrowser(WsHandler):
596 return browser
597 return current_state
598
599 + @staticmethod
600 + def _state_signature(
601 + active_id: int | str | None,
602 + browsers: list[dict[str, Any]],
603 + ) -> tuple[str, tuple[tuple[str, str, str, str, bool], ...]]:
604 + return (
605 + str(active_id or ""),
606 + tuple(
607 + (
608 + str(browser.get("context_id") or ""),
609 + str(browser.get("id") or ""),
610 + str(browser.get("currentUrl") or ""),
611 + str(browser.get("title") or ""),
612 + bool(browser.get("loading")),
613 + )
614 + for browser in browsers
615 + ),
616 + )
617 +
618 + @staticmethod
619 + def _frame_payload(
620 + frame: dict[str, Any],
621 + *,
622 + context_id: str,
623 + viewer_id: str,
624 + browser_id: int | str,
625 + sequence: int,
626 + binary_frames: bool,
627 + ) -> dict[str, Any]:
628 + image = str(frame.get("image") or "")
629 + payload: dict[str, Any] = {
630 + "context_id": context_id,
631 + "viewer_id": viewer_id,
632 + "browser_id": browser_id,
633 + "seq": sequence,
634 + "mime": frame.get("mime") or "image/jpeg",
635 + "frame_source": VIEWER_TRANSPORT_SCREENCAST,
636 + "viewer_transport": VIEWER_TRANSPORT_SCREENCAST,
637 + }
638 + dimensions = WsBrowser._frame_dimensions(frame.get("metadata"))
639 + if dimensions:
640 + payload.update(dimensions)
641 + if binary_frames:
642 + try:
643 + payload["image"] = base64.b64decode(image, validate=False)
644 + payload["encoding"] = "binary"
645 + except Exception:
646 + payload["image"] = image
647 + payload["encoding"] = "base64"
648 + else:
649 + payload["image"] = image
650 + payload["encoding"] = "base64"
651 + return payload
652 +
653 + @staticmethod
654 + def _frame_dimensions(metadata: Any) -> dict[str, int]:
655 + if not isinstance(metadata, dict):
656 + return {}
657 + for width_key, height_key in (
658 + ("expectedWidth", "expectedHeight"),
659 + ("deviceWidth", "deviceHeight"),
660 + ("jpegWidth", "jpegHeight"),
661 + ):
662 + try:
663 + width = int(metadata.get(width_key) or 0)
664 + height = int(metadata.get(height_key) or 0)
665 + except (TypeError, ValueError):
666 + continue
667 + if width > 0 and height > 0:
668 + return {"width": width, "height": height}
669 + return {}
670 +
671 + async def _emit_viewer_state(
672 + self,
673 + sid: str,
674 + context_id: str,
675 + browser_id: int | str | None,
676 + *,
677 + browsers: list[dict[str, Any]] | None = None,
678 + viewer_id: str = "",
679 + state: dict[str, Any] | None = None,
680 + viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT,
681 + ) -> None:
682 + await self._emit_to_connected_viewer(
683 + sid,
684 + "browser_viewer_state",
685 + {
686 + "context_id": context_id,
687 + "active_browser_context_id": context_id,
688 + "viewer_id": viewer_id,
689 + "browser_id": browser_id,
690 + "active_browser_id": browser_id,
691 + "browsers": browsers or [],
692 + "state": state,
693 + "all_browsers": False,
694 + "viewer_transport": viewer_transport,
695 + },
696 + )
697 +
698 async def _emit_empty_frame(
699 self,
700 sid: str,
@@ -576,7 +704,7 @@ class WsBrowser(WsHandler):
704 viewer_id: str = "",
705 frame_source: str = "",
706 ) -> None:
579 - await self.emit_to(
707 + await self._emit_to_connected_viewer(
708 sid,
709 "browser_viewer_frame",
710 {
@@ -592,6 +720,20 @@ class WsBrowser(WsHandler):
720 },
721 )
722
723 + async def _emit_to_connected_viewer(
724 + self,
725 + sid: str,
726 + event: str,
727 + data: dict[str, Any],
728 + ) -> None:
729 + manager = getattr(self, "_manager", None)
730 + if manager is not None:
731 + with manager.lock:
732 + connected = (getattr(self, "namespace", "/ws"), sid) in manager.connections
733 + if not connected:
734 + raise asyncio.CancelledError()
735 + await self.emit_to(sid, event, data)
736 +
737 @staticmethod
738 def _viewer_transport(data: dict[str, Any]) -> str:
739 raw = (
@@ -623,6 +765,20 @@ class WsBrowser(WsHandler):
765 "height": max(200, min(4096, height)),
766 }
767
768 + @staticmethod
769 + def _capture_scale_from_data(data: dict[str, Any]) -> float:
770 + try:
771 + scale = float(
772 + data.get("device_pixel_ratio")
773 + or data.get("devicePixelRatio")
774 + or data.get("pixel_ratio")
775 + or data.get("pixelRatio")
776 + or 1
777 + )
778 + except (TypeError, ValueError):
779 + return 1.0
780 + return max(1.0, min(2.0, scale))
781 +
782 @staticmethod
783 def _context_id(data: dict[str, Any]) -> str:
784 return str(data.get("context_id") or data.get("context") or "").strip()
plugins/_browser/helpers/runtime.py
+8 -2
@@ -329,10 +329,14 @@ class _BrowserScreencast:
329 quality: int,
330 every_nth_frame: int,
331 viewport: dict[str, int],
332 + capture_scale: float = 1.0,
333 ) -> None:
334 self.session.on("Page.screencastFrame", self._on_frame)
335 width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))
336 height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
337 + scale = max(1.0, min(2.0, float(capture_scale or 1.0)))
338 + max_width = max(320, min(SCREENCAST_MAX_WIDTH, int(round(width * scale))))
339 + max_height = max(200, min(SCREENCAST_MAX_HEIGHT, int(round(height * scale))))
340 self._expected_width = width
341 self._expected_height = height
342 with contextlib.suppress(Exception):
@@ -343,8 +347,8 @@ class _BrowserScreencast:
347 {
348 "format": "jpeg",
349 "quality": max(20, min(95, int(quality))),
346 - "maxWidth": SCREENCAST_MAX_WIDTH,
347 - "maxHeight": SCREENCAST_MAX_HEIGHT,
350 + "maxWidth": max_width,
351 + "maxHeight": max_height,
352 "everyNthFrame": max(1, int(every_nth_frame)),
353 },
354 )
@@ -1616,6 +1620,7 @@ class _BrowserRuntimeCore:
1620 *,
1621 quality: int = 78,
1622 every_nth_frame: int = 1,
1623 + capture_scale: float = 1.0,
1624 ) -> dict[str, Any]:
1625 await self.ensure_started()
1626 resolved_id = self._resolve_browser_id(browser_id)
@@ -1634,6 +1639,7 @@ class _BrowserRuntimeCore:
1639 quality=quality,
1640 every_nth_frame=every_nth_frame,
1641 viewport=page.viewport_size or DEFAULT_VIEWPORT,
1642 + capture_scale=capture_scale,
1643 )
1644 except Exception:
1645 self.screencasts.pop(stream_id, None)
plugins/_browser/webui/browser-store.js
+152 -53
@@ -34,6 +34,8 @@ const ANNOTATION_DOM_LIMIT = 1200;
34 const ANNOTATION_TRAY_MARGIN = 10;
35 const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]);
36 const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
37 +const BROWSER_BINARY_FRAMES_SUPPORTED = typeof Blob === "function"
38 + && typeof globalThis.URL?.createObjectURL === "function";
39
40 function makeViewerToken() {
41 return globalThis.crypto?.randomUUID?.()
@@ -114,6 +116,31 @@ function loadFrameDimensions(src) {
116 });
117 }
118
119 +function frameImageSource(data = {}) {
120 + const image = data?.image;
121 + if (!image) return null;
122 + const mime = data.mime || "image/jpeg";
123 + const isArrayBuffer = typeof ArrayBuffer !== "undefined" && image instanceof ArrayBuffer;
124 + const isView = typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView?.(image);
125 + const isBlob = typeof Blob !== "undefined" && image instanceof Blob;
126 + if (data.encoding === "binary" || isArrayBuffer || isView || isBlob) {
127 + if (!BROWSER_BINARY_FRAMES_SUPPORTED) return null;
128 + const blob = isBlob ? image : new Blob([image], { type: mime });
129 + const src = globalThis.URL.createObjectURL(blob);
130 + return {
131 + src,
132 + objectUrl: src,
133 + cleanup: () => globalThis.URL.revokeObjectURL(src),
134 + };
135 + }
136 + if (typeof image !== "string") return null;
137 + return {
138 + src: `data:${mime};base64,${image}`,
139 + objectUrl: "",
140 + cleanup: null,
141 + };
142 +}
143 +
144 const model = {
145 loading: true,
146 error: "",
@@ -146,6 +173,7 @@ const model = {
173 _lastFrameDimensions: null,
174 _pendingFrameSrc: "",
175 _pendingFrameOptions: null,
176 + _frameObjectUrl: "",
177 _frameRenderHandle: null,
178 _frameRenderCancel: null,
179 _frameRenderSequence: 0,
@@ -442,7 +470,7 @@ const model = {
470 this.setActiveBrowserId(null);
471 this.address = "";
472 this.frameState = null;
445 - this.frameSrc = "";
473 + this.clearFrameSrc();
474 if (this.contextId) {
475 await this.connectViewer();
476 }
@@ -833,7 +861,7 @@ const model = {
861
862 resetRenderedFrame() {
863 this.cancelFrameRender();
836 - this.frameSrc = "";
864 + this.clearFrameSrc();
865 this._lastFrameDimensions = null;
866 this._lastFrameAt = 0;
867 },
@@ -915,6 +943,25 @@ const model = {
943 return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST;
944 },
945
946 + supportsBinaryFrames() {
947 + return BROWSER_BINARY_FRAMES_SUPPORTED;
948 + },
949 +
950 + captureDevicePixelRatio() {
951 + const value = Number(globalThis.devicePixelRatio || 1);
952 + if (!Number.isFinite(value) || value <= 1) return 1;
953 + return Math.min(2, value);
954 + },
955 +
956 + frameDimensionsFromData(data = null) {
957 + const width = Number(data?.width || 0);
958 + const height = Number(data?.height || 0);
959 + if (Number.isFinite(width) && Number.isFinite(height) && width > 0 && height > 0) {
960 + return { width, height };
961 + }
962 + return this.frameDimensionsFromMetadata(data?.metadata);
963 + },
964 +
965 frameDimensionsFromMetadata(metadata = null) {
966 if (!metadata || typeof metadata !== "object") return null;
967 const width = Number(metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth || 0);
@@ -986,6 +1033,9 @@ const model = {
1033 viewer_id: viewerToken,
1034 create_browser: Boolean(options.createBrowser || options.create_browser),
1035 viewer_transport: this.requestedViewerTransport(),
1036 + binary_frames: this.supportsBinaryFrames(),
1037 + slim_frames: true,
1038 + device_pixel_ratio: this.captureDevicePixelRatio(),
1039 viewport_width: initialViewport?.width,
1040 viewport_height: initialViewport?.height,
1041 },
@@ -1014,9 +1064,9 @@ const model = {
1064 data.active_browser_context_id || contextId,
1065 );
1066 this.applySnapshot(data.snapshot);
1017 - this.connected = true;
1018 - this.browserInstallExpected = false;
1019 - },
1067 + this.connected = true;
1068 + this.browserInstallExpected = false;
1069 + },
1070
1071 async _bindSocketEvents() {
1072 if (!this._frameOff) {
@@ -1028,7 +1078,9 @@ const model = {
1078 }
1079 const incomingContextId = this.normalizeContextId(data.context_id || this.contextId);
1080 const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id);
1031 - this.applyBrowserListing(data.browsers || [], incomingContextId, { replaceContext: true });
1081 + if (Array.isArray(data.browsers)) {
1082 + this.applyBrowserListing(data.browsers, incomingContextId, { replaceContext: true });
1083 + }
1084 if (incomingBrowserId && !this.activeBrowserId) {
1085 this.setActiveBrowserId(incomingBrowserId, incomingContextId);
1086 }
@@ -1046,11 +1098,15 @@ const model = {
1098 this.address = data.state.currentUrl;
1099 }
1100 if (data.image) {
1101 + const frameImage = frameImageSource(data);
1102 + if (!frameImage?.src) return;
1103 const frameBrowserId = incomingBrowserId || this.activeBrowserId;
1050 - this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {
1104 + this.queueFrameRender(frameImage.src, {
1105 browserId: frameBrowserId,
1106 contextId: incomingContextId,
1053 - dimensions: this.frameDimensionsFromMetadata(data.metadata),
1107 + dimensions: this.frameDimensionsFromData(data),
1108 + objectUrl: frameImage.objectUrl,
1109 + cleanup: frameImage.cleanup,
1110 onAccepted: () => {
1111 if (
1112 this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
@@ -1063,13 +1119,13 @@ const model = {
1119 });
1120 } else if (!data.state) {
1121 this.cancelFrameRender();
1066 - this.frameSrc = "";
1122 + this.clearFrameSrc();
1123 }
1124 if (!data.image && !data.state) {
1125 if (!this.activeBrowserId) {
1126 this.setActiveBrowserId(null, "");
1127 this.frameState = null;
1072 - this.frameSrc = "";
1128 + this.clearFrameSrc();
1129 }
1130 }
1131 this._lastFrameAt = Date.now();
@@ -1077,15 +1133,20 @@ const model = {
1133 await websocket.on("browser_viewer_frame", frameHandler);
1134 this._frameOff = () => websocket.off("browser_viewer_frame", frameHandler);
1135 }
1080 - if (!this._stateOff) {
1081 - const stateHandler = ({ data }) => {
1082 - if (data?.context_id !== this.contextId) return;
1083 - if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
1084 - if (data?.viewer_transport) {
1085 - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1086 - }
1136 + if (!this._stateOff) {
1137 + const stateHandler = ({ data }) => {
1138 + if (data?.context_id !== this.contextId) return;
1139 + if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
1140 + if (data?.viewer_transport) {
1141 + this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1142 + }
1143 const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId);
1088 - this.applyBrowserListing(data.browsers || [], commandContextId, { replaceAll: Boolean(data.all_browsers) });
1144 + if (Array.isArray(data.browsers)) {
1145 + this.applyBrowserListing(data.browsers, commandContextId, {
1146 + replaceAll: Boolean(data.all_browsers),
1147 + replaceContext: !data.all_browsers,
1148 + });
1149 + }
1150 const command = String(data.command || "").toLowerCase();
1151 const commandBrowserId = this.normalizeBrowserId(data.browser_id);
1152 const result = data.result || {};
@@ -1102,23 +1163,40 @@ const model = {
1163 || this.activeBrowserId
1164 || this.firstBrowserId(resultContextId)
1165 );
1105 - if (
1106 - !this.activeBrowserId
1107 - || command === "open"
1108 - || command === "close"
1109 - || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
1110 - ) {
1111 - this.setActiveBrowserId(preferredBrowserId, resultContextId);
1112 - }
1113 - this.applyActiveFrameState(resultState || this.browserById(this.activeBrowserId, this.activeBrowserContextId));
1114 - this.applySnapshot(data.snapshot);
1115 - };
1166 + const stateBrowserId = this.normalizeBrowserId(data.active_browser_id || data.browser_id || data.state?.id);
1167 + if (
1168 + stateBrowserId
1169 + && (
1170 + !this.activeBrowserId
1171 + || this.sameBrowserTab(stateBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
1172 + )
1173 + ) {
1174 + this.setActiveBrowserId(stateBrowserId, commandContextId);
1175 + }
1176 + if (
1177 + !this.activeBrowserId
1178 + || command === "open"
1179 + || command === "close"
1180 + || this.sameBrowserTab(commandBrowserId, commandContextId, this.activeBrowserId, this.activeBrowserContextId)
1181 + ) {
1182 + this.setActiveBrowserId(preferredBrowserId, resultContextId);
1183 + }
1184 + this.applyActiveFrameState(
1185 + resultState
1186 + || data.state
1187 + || this.browserById(this.activeBrowserId, this.activeBrowserContextId)
1188 + );
1189 + this.applySnapshot(data.snapshot);
1190 + };
1191 await websocket.on("browser_viewer_state", stateHandler);
1192 this._stateOff = () => websocket.off("browser_viewer_state", stateHandler);
1193 }
1194 },
1195
1196 queueFrameRender(frameSrc, options = {}) {
1197 + if (this._pendingFrameSrc) {
1198 + this.releasePendingFrame();
1199 + }
1200 this._pendingFrameSrc = frameSrc;
1201 this._pendingFrameOptions = options || null;
1202 if (this._frameRenderHandle) return;
@@ -1148,22 +1226,26 @@ const model = {
1226 async renderDecodedFrame(frameSrc, options = {}, sequence = 0, surfaceSequence = this._surfaceOpenSequence) {
1227 if (!frameSrc) {
1228 if (sequence === this._frameRenderSequence) {
1151 - this.frameSrc = "";
1229 + this.clearFrameSrc();
1230 }
1231 return;
1232 }
1233 const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc);
1234 if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
1235 + options?.cleanup?.();
1236 return;
1237 }
1238 const viewport = this.currentViewportSize() || this._lastViewport;
1239 if (!this.frameMatchesViewport(dimensions, viewport)) {
1240 this.requestViewportSyncAfterRejectedFrame();
1241 if (!this.shouldAcceptMismatchedFrame(dimensions)) {
1242 + options?.cleanup?.();
1243 return;
1244 }
1245 }
1246 + this.releaseRenderedFrameUrl(frameSrc);
1247 this.frameSrc = frameSrc;
1248 + this._frameObjectUrl = options?.objectUrl || "";
1249 this._lastFrameDimensions = dimensions;
1250 this._lastFrameAt = Date.now();
1251 options?.onAccepted?.();
@@ -1262,9 +1344,26 @@ const model = {
1344 }
1345 this._frameRenderHandle = null;
1346 this._frameRenderCancel = null;
1347 + this.releasePendingFrame();
1348 + this._frameRenderSequence += 1;
1349 + },
1350 +
1351 + releasePendingFrame() {
1352 + this._pendingFrameOptions?.cleanup?.();
1353 this._pendingFrameSrc = "";
1354 this._pendingFrameOptions = null;
1267 - this._frameRenderSequence += 1;
1355 + },
1356 +
1357 + releaseRenderedFrameUrl(nextSrc = "") {
1358 + if (this._frameObjectUrl && this._frameObjectUrl !== nextSrc) {
1359 + globalThis.URL?.revokeObjectURL?.(this._frameObjectUrl);
1360 + this._frameObjectUrl = "";
1361 + }
1362 + },
1363 +
1364 + clearFrameSrc() {
1365 + this.releaseRenderedFrameUrl("");
1366 + this.frameSrc = "";
1367 },
1368
1369 beginCommand() {
@@ -1329,7 +1428,7 @@ const model = {
1428 );
1429 if (!this.activeBrowserId) {
1430 this.frameState = null;
1332 - this.frameSrc = "";
1431 + this.clearFrameSrc();
1432 }
1433 if (result.state?.currentUrl || result.currentUrl) {
1434 this.address = result.state?.currentUrl || result.currentUrl;
@@ -1438,7 +1537,7 @@ const model = {
1537 this.error = "";
1538 this.switchingBrowserId = targetId;
1539 this.cancelFrameRender();
1441 - this.frameSrc = "";
1540 + this.clearFrameSrc();
1541 this.frameState = browser || null;
1542 if (!this.addressFocused && browser?.currentUrl) {
1543 this.address = browser.currentUrl;
@@ -1643,35 +1742,35 @@ const model = {
1742 }
1743 },
1744
1646 - applySnapshot(snapshot = null) {
1647 - if (!snapshot?.image) return;
1648 - const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);
1745 + applySnapshot(snapshot = null) {
1746 + if (!snapshot?.image) return;
1747 + const snapshotId = this.normalizeBrowserId(snapshot.browser_id || snapshot.state?.id);
1748 const snapshotContextId = this.normalizeContextId(snapshot.context_id || snapshot.state?.context_id || this.activeBrowserContextId);
1650 - if (
1749 + if (
1750 snapshotId
1751 && this.activeBrowserId
1752 && !this.sameBrowserTab(snapshotId, snapshotContextId, this.activeBrowserId, this.activeBrowserContextId)
1753 ) {
1655 - return;
1656 - }
1657 - if (snapshot.state) {
1658 - this.applyActiveFrameState(snapshot.state);
1659 - }
1660 - const frameBrowserId = snapshotId || this.activeBrowserId;
1661 - this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
1662 - browserId: frameBrowserId,
1754 + return;
1755 + }
1756 + if (snapshot.state) {
1757 + this.applyActiveFrameState(snapshot.state);
1758 + }
1759 + const frameBrowserId = snapshotId || this.activeBrowserId;
1760 + this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
1761 + browserId: frameBrowserId,
1762 contextId: snapshotContextId,
1664 - onAccepted: () => {
1665 - if (
1763 + onAccepted: () => {
1764 + if (
1765 this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
1766 && this.normalizeContextId(this.activeBrowserContextId) === snapshotContextId
1767 ) {
1669 - this.switchingBrowserId = null;
1670 - }
1671 - this._surfaceSwitching = false;
1672 - },
1673 - });
1674 - },
1768 + this.switchingBrowserId = null;
1769 + }
1770 + this._surfaceSwitching = false;
1771 + },
1772 + });
1773 + },
1774
1775 isSwitchingBrowser() {
1776 return Boolean(
tests/test_browser_agent_regressions.py
+55 -8
@@ -1158,7 +1158,8 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1158 assert "def _viewer_transport(data: dict[str, Any])" in ws_browser
1159 assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
1160 assert "self._stream_state" in ws_browser
1161 - assert "SCREENCAST_QUALITY = 92" in ws_browser
1161 + assert "SCREENCAST_STREAM_QUALITY = 80" in ws_browser
1162 + assert "SCREENSHOT_QUALITY = 92" in ws_browser
1163 assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
1164 assert '"set_viewport"' in ws_browser
1165 assert "start_screencast" in ws_browser
@@ -1169,7 +1170,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1170 assert "stop_screencast" in ws_browser
1171 assert "viewer_transport == VIEWER_TRANSPORT_SCREENCAST" in ws_browser
1172 assert '"viewer_transport": viewer_transport' in ws_browser
1172 - assert '"viewer_transport": VIEWER_TRANSPORT_SNAPSHOT' in ws_browser
1173 + assert "viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
1174 assert '"Page.startScreencast"' in runtime
1175 assert '"Page.screencastFrame"' in runtime
1176 assert '"Page.screencastFrameAck"' in runtime
@@ -1180,6 +1181,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1181 assert "await self._stop_screencasts_for_browser(resolved_id)" in runtime
1182 assert "queueFrameRender" in browser_store
1183 assert "requestAnimationFrame" in browser_store
1184 + assert "function frameImageSource(data = {})" in browser_store
1185 assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
1186 assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
1187 assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
@@ -1187,11 +1189,17 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1189 assert "requestedViewerTransport()" in browser_store
1190 assert "normalizeViewerTransport(value = \"\")" in browser_store
1191 assert "usesScreencastTransport()" in browser_store
1192 + assert "supportsBinaryFrames()" in browser_store
1193 + assert "captureDevicePixelRatio()" in browser_store
1194 + assert "frameDimensionsFromData(data = null)" in browser_store
1195 assert "frameDimensionsFromMetadata(metadata = null)" in browser_store
1196 assert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store
1192 - assert "dimensions: this.frameDimensionsFromMetadata(data.metadata)" in browser_store
1197 + assert "dimensions: this.frameDimensionsFromData(data)" in browser_store
1198 assert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_store
1199 assert "viewer_transport: this.requestedViewerTransport()" in browser_store
1200 + assert "binary_frames: this.supportsBinaryFrames()" in browser_store
1201 + assert "slim_frames: true" in browser_store
1202 + assert "device_pixel_ratio: this.captureDevicePixelRatio()" in browser_store
1203 assert "viewport_width: initialViewport?.width" in browser_store
1204 assert "viewport_height: initialViewport?.height" in browser_store
1205 assert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_store
@@ -1213,6 +1221,13 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1221 assert "this.applySnapshot(data.snapshot);" in browser_store
1222 assert "else if (!data.state)" in browser_store
1223 assert '"snapshot": snapshot' in ws_browser
1224 + assert '"binary_frames": binary_frames' in ws_browser
1225 + assert '"slim_frames": slim_frames' in ws_browser
1226 + assert "capture_scale=capture_scale" in ws_browser
1227 + assert "base64.b64decode(image, validate=False)" in ws_browser
1228 + assert '"encoding"] = "binary"' in ws_browser
1229 + assert "def _frame_payload(" in ws_browser
1230 + assert "def _emit_viewer_state(" in ws_browser
1231 assert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handler
1232 assert "staticScreenshotUri(kvps)" in browser_tool_handler
1233 assert "/components/modals/image-viewer/image-viewer-store.js" in browser_tool_handler
@@ -1223,9 +1238,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1238 assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handler
1239 assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handler
1240 assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
1226 - assert '"frame_source": "state"' in ws_browser
1227 - assert '"frame_source"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
1228 - assert '"viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
1241 + assert '"browser_viewer_state"' in ws_browser
1242 + assert '"frame_source": VIEWER_TRANSPORT_SCREENCAST' in ws_browser
1243 + assert '"viewer_transport": VIEWER_TRANSPORT_SCREENCAST' in ws_browser
1244 assert "fallback_screenshot" not in ws_browser
1245 assert "canvas_wheel_screenshot" not in ws_browser
1246 assert "surface_mode: this._mode" not in browser_store
@@ -1234,6 +1249,29 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1249 assert "image-rendering: auto;" in main_html
1250
1251
1252 +def test_browser_viewer_frame_payload_supports_binary_slim_frames():
1253 + payload = ws_browser_module.WsBrowser._frame_payload(
1254 + {
1255 + "image": SMALL_JPEG_10X10,
1256 + "mime": "image/jpeg",
1257 + "metadata": {"expectedWidth": 900, "expectedHeight": 600},
1258 + },
1259 + context_id="ctx",
1260 + viewer_id="viewer",
1261 + browser_id=1,
1262 + sequence=3,
1263 + binary_frames=True,
1264 + )
1265 +
1266 + assert payload["encoding"] == "binary"
1267 + assert payload["image"] == __import__("base64").b64decode(SMALL_JPEG_10X10)
1268 + assert payload["width"] == 900
1269 + assert payload["height"] == 600
1270 + assert payload["seq"] == 3
1271 + assert "browsers" not in payload
1272 + assert "state" not in payload
1273 +
1274 +
1275 def test_browser_navigation_errors_stay_inside_native_browser_page():
1276 runtime = (
1277 PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
@@ -1504,7 +1542,12 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
1542 mime="image/jpeg",
1543 )
1544
1507 - await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})
1545 + await screencast.start(
1546 + quality=92,
1547 + every_nth_frame=1,
1548 + viewport={"width": 1118, "height": 662},
1549 + capture_scale=2,
1550 + )
1551 session.handlers["Page.screencastFrame"](
1552 {"data": first_image, "metadata": {"deviceWidth": 10}, "sessionId": 1}
1553 )
@@ -1563,6 +1606,10 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
1606 for index, (method, _params) in enumerate(session.sent)
1607 if method == "Page.startScreencast"
1608 )
1609 + start_params = session.sent[start_index][1]
1610 + assert start_params["quality"] == 92
1611 + assert start_params["maxWidth"] == 2236
1612 + assert start_params["maxHeight"] == 1324
1613 cdp_viewport_indices = [
1614 index
1615 for index, (method, _params) in enumerate(session.sent)
@@ -2287,7 +2334,7 @@ async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch):
2334
2335 assert result["active_browser_id"] == 1
2336 assert result["snapshot"]["image"] == "jpeg-data"
2290 - assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls
2337 + assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENSHOT_QUALITY}) in calls
2338
2339 await handler.on_disconnect("sid-snapshot")
2340