@samitouri / QOSami_Agents_Zero-2 / commits / dccf017d

Redesign Browser viewer screencast transport and viewport fit

Replace the Browser viewer’s screenshot polling with CDP screencast streaming for much smoother navigation. The runtime now starts/stops CDP screencasts cleanly, acknowledges frames, drops stale frames, and keeps the WebSocket payload compatible with the existing viewer. Also fixes modal viewport sizing by sending the initial stage dimensions on subscribe, applying CDP emulation sizing before the first frame, avoiding image stretching, and increasing screencast JPEG quality to 92. Regression coverage was added for the screencast path, frame ack/drop behavior, viewport sizing, and UI rendering assumptions. -- Still needs thorough performance audit and optimization --

Alessandro committed Apr 26, 2026 at 02:28 UTC dccf017d2cf74bbec67151d03ba3e4282ea08c6b
5 files changed +637 -93
plugins/_browser/api/ws_browser.py
+151 -36
@@ -1,6 +1,8 @@
1 from __future__ import annotations
2
3 import asyncio
4 +import contextlib
5 +import time
6 from typing import Any, ClassVar
7
8 from agent import AgentContext
@@ -9,6 +11,12 @@ from helpers.ws_manager import WsResult
11 from plugins._browser.helpers.runtime import get_runtime
12
13
14 +FRAME_IDLE_TIMEOUT_SECONDS = 0.35
15 +FRAME_RETRY_DELAY_SECONDS = 0.5
16 +FRAME_STATE_REFRESH_SECONDS = 0.75
17 +SCREENCAST_QUALITY = 92
18 +
19 +
20 class WsBrowser(WsHandler):
21 _streams: ClassVar[dict[tuple[str, str], asyncio.Task[None]]] = {}
22
@@ -57,9 +65,17 @@ class WsBrowser(WsHandler):
65 browsers = listing.get("browsers") or []
66 if opened.get("id"):
67 listing["last_interacted_browser_id"] = opened.get("id")
60 - active_id = data.get("browser_id") or listing.get("last_interacted_browser_id")
61 - if not active_id and browsers:
62 - active_id = browsers[0].get("id")
68 + active_id = self._active_browser_id(listing, data.get("browser_id"))
69 + initial_viewport = self._viewport_from_data(data)
70 + if active_id and initial_viewport:
71 + await runtime.call(
72 + "set_viewport",
73 + active_id,
74 + initial_viewport["width"],
75 + initial_viewport["height"],
76 + )
77 + listing = await runtime.call("list")
78 + browsers = listing.get("browsers") or []
79
80 stream_key = (sid, context_id)
81 existing = self._streams.pop(stream_key, None)
@@ -187,46 +203,145 @@ class WsBrowser(WsHandler):
203 context_id: str,
204 browser_id: int | str | None,
205 ) -> None:
206 + runtime = None
207 + stream_id = None
208 while True:
209 try:
210 runtime = await get_runtime(context_id, create=False)
193 - if runtime:
194 - listing = await runtime.call("list")
195 - browsers = listing.get("browsers") or []
196 - browser_ids = {str(browser.get("id")) for browser in browsers}
197 - requested_id = str(browser_id or "") if browser_id else ""
198 - active_id = (
199 - browser_id
200 - if requested_id and requested_id in browser_ids
201 - else listing.get("last_interacted_browser_id")
202 - )
203 - if active_id and str(active_id) not in browser_ids:
204 - active_id = None
205 - if not active_id and browsers:
206 - active_id = browsers[0].get("id")
207 - if active_id:
208 - frame = await runtime.call("screenshot", active_id)
209 - frame["context_id"] = context_id
210 - frame["browsers"] = browsers
211 - await self.emit_to(sid, "browser_viewer_frame", frame)
212 - else:
213 - await self.emit_to(
214 - sid,
215 - "browser_viewer_frame",
216 - {
217 - "context_id": context_id,
218 - "browser_id": None,
219 - "browsers": browsers,
220 - "image": "",
221 - "mime": "",
222 - "state": None,
223 - },
211 + if not runtime:
212 + await self._emit_empty_frame(sid, context_id)
213 + await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
214 + continue
215 +
216 + listing = await runtime.call("list")
217 + browsers = listing.get("browsers") or []
218 + active_id = self._active_browser_id(listing, browser_id)
219 + if not active_id:
220 + await self._emit_empty_frame(sid, context_id, browsers=browsers)
221 + await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
222 + continue
223 +
224 + screencast = await runtime.call(
225 + "start_screencast",
226 + active_id,
227 + quality=SCREENCAST_QUALITY,
228 + every_nth_frame=1,
229 + )
230 + stream_id = screencast["stream_id"]
231 + active_id = screencast["browser_id"]
232 + state = screencast.get("state")
233 + await self.emit_to(
234 + sid,
235 + "browser_viewer_frame",
236 + {
237 + "context_id": context_id,
238 + "browser_id": active_id,
239 + "browsers": browsers,
240 + "image": "",
241 + "mime": "",
242 + "state": state,
243 + },
244 + )
245 +
246 + last_state_refresh = 0.0
247 + while True:
248 + now = time.monotonic()
249 + if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
250 + listing = await runtime.call("list")
251 + browsers = listing.get("browsers") or []
252 + browser_ids = {str(browser.get("id")) for browser in browsers}
253 + if str(active_id) not in browser_ids:
254 + break
255 + state = self._state_for_browser(browsers, active_id, state)
256 + last_state_refresh = now
257 +
258 + try:
259 + frame = await runtime.call(
260 + "read_screencast_frame",
261 + stream_id,
262 + timeout=FRAME_IDLE_TIMEOUT_SECONDS,
263 )
225 - await asyncio.sleep(0.75)
264 + except TimeoutError:
265 + continue
266 +
267 + frame["context_id"] = context_id
268 + frame["browser_id"] = active_id
269 + frame["browsers"] = browsers
270 + frame["state"] = state
271 + await self.emit_to(sid, "browser_viewer_frame", frame)
272 except asyncio.CancelledError:
273 raise
274 except Exception:
229 - await asyncio.sleep(1.5)
275 + await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
276 + finally:
277 + if runtime and stream_id:
278 + with contextlib.suppress(Exception):
279 + await runtime.call("stop_screencast", stream_id)
280 + stream_id = None
281 +
282 + @staticmethod
283 + def _active_browser_id(
284 + listing: dict[str, Any],
285 + requested_browser_id: int | str | None,
286 + ) -> int | str | None:
287 + browsers = listing.get("browsers") or []
288 + browser_ids = {str(browser.get("id")) for browser in browsers}
289 + requested_id = str(requested_browser_id or "") if requested_browser_id else ""
290 + active_id = (
291 + requested_browser_id
292 + if requested_id and requested_id in browser_ids
293 + else listing.get("last_interacted_browser_id")
294 + )
295 + if active_id and str(active_id) not in browser_ids:
296 + active_id = None
297 + if not active_id and browsers:
298 + active_id = browsers[0].get("id")
299 + return active_id
300 +
301 + @staticmethod
302 + def _state_for_browser(
303 + browsers: list[dict[str, Any]],
304 + browser_id: int | str,
305 + current_state: dict[str, Any] | None,
306 + ) -> dict[str, Any] | None:
307 + for browser in browsers:
308 + if str(browser.get("id")) == str(browser_id):
309 + return browser
310 + return current_state
311 +
312 + async def _emit_empty_frame(
313 + self,
314 + sid: str,
315 + context_id: str,
316 + *,
317 + browsers: list[dict[str, Any]] | None = None,
318 + ) -> None:
319 + await self.emit_to(
320 + sid,
321 + "browser_viewer_frame",
322 + {
323 + "context_id": context_id,
324 + "browser_id": None,
325 + "browsers": browsers or [],
326 + "image": "",
327 + "mime": "",
328 + "state": None,
329 + },
330 + )
331 +
332 + @staticmethod
333 + def _viewport_from_data(data: dict[str, Any]) -> dict[str, int] | None:
334 + try:
335 + width = int(data.get("viewport_width") or data.get("width") or 0)
336 + height = int(data.get("viewport_height") or data.get("height") or 0)
337 + except (TypeError, ValueError):
338 + return None
339 + if width < 80 or height < 80:
340 + return None
341 + return {
342 + "width": max(320, min(4096, width)),
343 + "height": max(200, min(4096, height)),
344 + }
345
346 @staticmethod
347 def _context_id(data: dict[str, Any]) -> str:
plugins/_browser/helpers/runtime.py
+270 -1
@@ -3,12 +3,14 @@ from __future__ import annotations
3 import atexit
4 import asyncio
5 import base64
6 +import contextlib
7 import os
8 import re
9 import shutil
10 import signal
11 import threading
12 import time
13 +import uuid
14 from dataclasses import dataclass
15 from pathlib import Path
16 from typing import Any
@@ -27,6 +29,8 @@ CONTENT_HELPER_PATH = PLUGIN_DIR / "assets" / "browser-page-content.js"
29 RUNTIME_DATA_KEY = "_browser_runtime"
30 DEFAULT_VIEWPORT = {"width": 1024, "height": 768}
31 CHROME_SINGLETON_FILES = ("SingletonLock", "SingletonCookie", "SingletonSocket")
32 +SCREENCAST_MAX_WIDTH = 4096
33 +SCREENCAST_MAX_HEIGHT = 4096
34
35 _SPECIAL_SCHEME_RE = re.compile(r"^(?:about|blob|data|file|mailto|tel):", re.I)
36 _URL_SCHEME_RE = re.compile(r"^[a-z][a-z\d+\-.]*://", re.I)
@@ -84,6 +88,195 @@ class BrowserPage:
88 page: Any
89
90
91 +class _BrowserScreencast:
92 + def __init__(
93 + self,
94 + *,
95 + stream_id: str,
96 + browser_id: int,
97 + session: Any,
98 + mime: str,
99 + ):
100 + self.id = stream_id
101 + self.browser_id = browser_id
102 + self.session = session
103 + self.mime = mime
104 + self.queue = asyncio.Queue(maxsize=1)
105 + self.stopped = False
106 + self._ack_tasks: set[asyncio.Task] = set()
107 + self._expected_width = 0
108 + self._expected_height = 0
109 + self._dimension_mismatches = 0
110 +
111 + async def start(
112 + self,
113 + *,
114 + quality: int,
115 + every_nth_frame: int,
116 + viewport: dict[str, int],
117 + ) -> None:
118 + self.session.on("Page.screencastFrame", self._on_frame)
119 + width = max(320, min(4096, int(viewport.get("width") or DEFAULT_VIEWPORT["width"])))
120 + height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
121 + self._expected_width = width
122 + self._expected_height = height
123 + self._dimension_mismatches = 0
124 + with contextlib.suppress(Exception):
125 + await self.session.send("Page.enable")
126 + await self.session.send(
127 + "Emulation.setDeviceMetricsOverride",
128 + {
129 + "width": width,
130 + "height": height,
131 + "deviceScaleFactor": 1,
132 + "mobile": False,
133 + "dontSetVisibleSize": True,
134 + },
135 + )
136 + with contextlib.suppress(Exception):
137 + await self.session.send(
138 + "Emulation.setVisibleSize",
139 + {
140 + "width": width,
141 + "height": height,
142 + },
143 + )
144 + await self.session.send(
145 + "Page.startScreencast",
146 + {
147 + "format": "jpeg",
148 + "quality": max(20, min(95, int(quality))),
149 + "maxWidth": SCREENCAST_MAX_WIDTH,
150 + "maxHeight": SCREENCAST_MAX_HEIGHT,
151 + "everyNthFrame": max(1, int(every_nth_frame)),
152 + },
153 + )
154 +
155 + async def next_frame(self, timeout: float = 1.0) -> dict[str, Any]:
156 + frame = await asyncio.wait_for(self.queue.get(), timeout=max(0.1, float(timeout)))
157 + if frame is None:
158 + raise RuntimeError("Browser screencast stopped.")
159 + return frame
160 +
161 + async def stop(self) -> None:
162 + if self.stopped:
163 + return
164 + self.stopped = True
165 + self._drop_queued_frames()
166 + with contextlib.suppress(asyncio.QueueFull):
167 + self.queue.put_nowait(None)
168 + with contextlib.suppress(Exception):
169 + await self.session.send("Page.stopScreencast")
170 + for task in list(self._ack_tasks):
171 + task.cancel()
172 + if self._ack_tasks:
173 + await asyncio.gather(*self._ack_tasks, return_exceptions=True)
174 + self._ack_tasks.clear()
175 + with contextlib.suppress(Exception):
176 + await self.session.detach()
177 +
178 + def _on_frame(self, params: dict[str, Any]) -> None:
179 + if self.stopped:
180 + return
181 + task = asyncio.create_task(self._handle_frame(params or {}))
182 + self._ack_tasks.add(task)
183 + task.add_done_callback(self._ack_tasks.discard)
184 +
185 + async def _handle_frame(self, params: dict[str, Any]) -> None:
186 + try:
187 + data = params.get("data") or ""
188 + if data and self._frame_matches_viewport(data):
189 + self._queue_latest(
190 + {
191 + "browser_id": self.browser_id,
192 + "mime": self.mime,
193 + "image": data,
194 + "metadata": params.get("metadata") or {},
195 + }
196 + )
197 + finally:
198 + session_id = params.get("sessionId")
199 + if session_id is not None and not self.stopped:
200 + with contextlib.suppress(Exception):
201 + await self.session.send(
202 + "Page.screencastFrameAck",
203 + {"sessionId": int(session_id)},
204 + )
205 +
206 + def _queue_latest(self, frame: dict[str, Any]) -> None:
207 + self._drop_queued_frames()
208 + with contextlib.suppress(asyncio.QueueFull):
209 + self.queue.put_nowait(frame)
210 +
211 + def _frame_matches_viewport(self, data: str) -> bool:
212 + if not self._expected_width or not self._expected_height:
213 + return True
214 + size = self._jpeg_size(data)
215 + if not size:
216 + return True
217 + width, height = size
218 + if abs(width - self._expected_width) <= 2 and abs(height - self._expected_height) <= 2:
219 + return True
220 + self._dimension_mismatches += 1
221 + return self._dimension_mismatches > 10
222 +
223 + @staticmethod
224 + def _jpeg_size(data: str) -> tuple[int, int] | None:
225 + try:
226 + raw = base64.b64decode(data, validate=False)
227 + except Exception:
228 + return None
229 + if len(raw) < 10 or raw[:2] != b"\xff\xd8":
230 + return None
231 + index = 2
232 + standalone_markers = {0x01, *range(0xD0, 0xD8)}
233 + size_markers = {
234 + 0xC0,
235 + 0xC1,
236 + 0xC2,
237 + 0xC3,
238 + 0xC5,
239 + 0xC6,
240 + 0xC7,
241 + 0xC9,
242 + 0xCA,
243 + 0xCB,
244 + 0xCD,
245 + 0xCE,
246 + 0xCF,
247 + }
248 + while index < len(raw) - 9:
249 + if raw[index] != 0xFF:
250 + index += 1
251 + continue
252 + while index < len(raw) and raw[index] == 0xFF:
253 + index += 1
254 + if index >= len(raw):
255 + return None
256 + marker = raw[index]
257 + index += 1
258 + if marker in standalone_markers:
259 + continue
260 + if index + 2 > len(raw):
261 + return None
262 + segment_length = int.from_bytes(raw[index : index + 2], "big")
263 + if segment_length < 2 or index + segment_length > len(raw):
264 + return None
265 + if marker in size_markers and segment_length >= 7:
266 + height = int.from_bytes(raw[index + 3 : index + 5], "big")
267 + width = int.from_bytes(raw[index + 5 : index + 7], "big")
268 + return width, height
269 + index += segment_length
270 + return None
271 +
272 + def _drop_queued_frames(self) -> None:
273 + while True:
274 + try:
275 + self.queue.get_nowait()
276 + except asyncio.QueueEmpty:
277 + return
278 +
279 +
280 class BrowserRuntime:
281 def __init__(self, context_id: str):
282 self.context_id = str(context_id)
@@ -118,6 +311,7 @@ class _BrowserRuntimeCore:
311 self.playwright = None
312 self.context = None
313 self.pages: dict[int, BrowserPage] = {}
314 + self.screencasts: dict[str, _BrowserScreencast] = {}
315 self.next_browser_id = 1
316 self.last_interacted_browser_id: int | None = None
317 self._content_helper_source: str | None = None
@@ -378,6 +572,7 @@ class _BrowserRuntimeCore:
572 async def close_browser(self, browser_id: int | str | None = None) -> dict[str, Any]:
573 await self.ensure_started()
574 resolved_id = self._resolve_browser_id(browser_id)
575 + await self._stop_screencasts_for_browser(resolved_id)
576 page = self._page(resolved_id)
577 await page.close()
578 self.pages.pop(resolved_id, None)
@@ -387,6 +582,7 @@ class _BrowserRuntimeCore:
582
583 async def close_all_browsers(self) -> dict[str, Any]:
584 await self.ensure_started()
585 + await self._stop_all_screencasts()
586 for browser_id in list(self.pages):
587 try:
588 await self.pages[browser_id].page.close()
@@ -413,6 +609,58 @@ class _BrowserRuntimeCore:
609 "state": await self._state(resolved_id),
610 }
611
612 + async def start_screencast(
613 + self,
614 + browser_id: int | str | None = None,
615 + *,
616 + quality: int = 78,
617 + every_nth_frame: int = 1,
618 + ) -> dict[str, Any]:
619 + await self.ensure_started()
620 + resolved_id = self._resolve_browser_id(browser_id)
621 + page = self._page(resolved_id)
622 + stream_id = uuid.uuid4().hex
623 + session = await self.context.new_cdp_session(page)
624 + screencast = _BrowserScreencast(
625 + stream_id=stream_id,
626 + browser_id=resolved_id,
627 + session=session,
628 + mime="image/jpeg",
629 + )
630 + self.screencasts[stream_id] = screencast
631 + try:
632 + await screencast.start(
633 + quality=quality,
634 + every_nth_frame=every_nth_frame,
635 + viewport=page.viewport_size or DEFAULT_VIEWPORT,
636 + )
637 + except Exception:
638 + self.screencasts.pop(stream_id, None)
639 + await screencast.stop()
640 + raise
641 + self.last_interacted_browser_id = resolved_id
642 + return {
643 + "stream_id": stream_id,
644 + "browser_id": resolved_id,
645 + "state": await self._state(resolved_id),
646 + }
647 +
648 + async def read_screencast_frame(
649 + self,
650 + stream_id: str,
651 + *,
652 + timeout: float = 1.0,
653 + ) -> dict[str, Any]:
654 + screencast = self.screencasts.get(str(stream_id or ""))
655 + if not screencast:
656 + raise KeyError("Browser screencast is not active.")
657 + return await screencast.next_frame(timeout=timeout)
658 +
659 + async def stop_screencast(self, stream_id: str) -> None:
660 + screencast = self.screencasts.pop(str(stream_id or ""), None)
661 + if screencast:
662 + await screencast.stop()
663 +
664 async def set_viewport(
665 self,
666 browser_id: int | str | None,
@@ -426,7 +674,14 @@ class _BrowserRuntimeCore:
674 "width": max(320, min(4096, int(width or DEFAULT_VIEWPORT["width"]))),
675 "height": max(200, min(4096, int(height or DEFAULT_VIEWPORT["height"]))),
676 }
429 - await page.set_viewport_size(viewport)
677 + current_viewport = page.viewport_size or {}
678 + changed = (
679 + int(current_viewport.get("width") or 0) != viewport["width"]
680 + or int(current_viewport.get("height") or 0) != viewport["height"]
681 + )
682 + if changed:
683 + await page.set_viewport_size(viewport)
684 + await self._stop_screencasts_for_browser(resolved_id)
685 self.last_interacted_browser_id = resolved_id
686 return {"state": await self._state(resolved_id), "viewport": viewport}
687
@@ -490,6 +745,7 @@ class _BrowserRuntimeCore:
745 return await self._state(resolved_id)
746
747 async def close(self, delete_profile: bool = False) -> None:
748 + await self._stop_all_screencasts()
749 for browser_id in list(self.pages):
750 try:
751 await self.pages[browser_id].page.close()
@@ -618,6 +874,19 @@ class _BrowserRuntimeCore:
874 def _page(self, browser_id: int) -> Any:
875 return self.pages[int(browser_id)].page
876
877 + async def _stop_screencasts_for_browser(self, browser_id: int) -> None:
878 + stream_ids = [
879 + stream_id
880 + for stream_id, screencast in self.screencasts.items()
881 + if screencast.browser_id == int(browser_id)
882 + ]
883 + for stream_id in stream_ids:
884 + await self.stop_screencast(stream_id)
885 +
886 + async def _stop_all_screencasts(self) -> None:
887 + for stream_id in list(self.screencasts):
888 + await self.stop_screencast(stream_id)
889 +
890 async def _ensure_content_helper(self, page: Any) -> None:
891 has_helper = await page.evaluate(
892 "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture)"
plugins/_browser/webui/browser-store.js
+80 -37
@@ -40,6 +40,9 @@ const model = {
40 _frameOff: null,
41 _stateOff: null,
42 _lastFrameAt: 0,
43 + _pendingFrameSrc: "",
44 + _frameRenderHandle: null,
45 + _frameRenderCancel: null,
46 _floatingCleanup: null,
47 _stageElement: null,
48 _stageResizeObserver: null,
@@ -298,21 +301,24 @@ const model = {
301 }
302 },
303
301 - async connectViewer() {
302 - if (!this.contextId) {
303 - this.connected = false;
304 - this.error = "No active chat context is selected.";
305 - return;
306 - }
307 - this.error = "";
308 - await this._bindSocketEvents();
309 - const response = await websocket.request(
310 - "browser_viewer_subscribe",
311 - {
312 - context_id: this.contextId,
313 - browser_id: this.activeBrowserId,
314 - },
315 - {
304 + async connectViewer() {
305 + if (!this.contextId) {
306 + this.connected = false;
307 + this.error = "No active chat context is selected.";
308 + return;
309 + }
310 + this.error = "";
311 + await this._bindSocketEvents();
312 + const initialViewport = this.currentViewportSize();
313 + const response = await websocket.request(
314 + "browser_viewer_subscribe",
315 + {
316 + context_id: this.contextId,
317 + browser_id: this.activeBrowserId,
318 + viewport_width: initialViewport?.width,
319 + viewport_height: initialViewport?.height,
320 + },
321 + {
322 timeoutMs: this.browserInstallExpected
323 ? BROWSER_FIRST_INSTALL_TIMEOUT_MS
324 : BROWSER_SUBSCRIBE_TIMEOUT_MS,
@@ -329,18 +335,25 @@ const model = {
335 async _bindSocketEvents() {
336 if (!this._frameOff) {
337 const frameHandler = ({ data }) => {
332 - if (data?.context_id !== this.contextId) return;
333 - this.browsers = data.browsers || this.browsers;
334 - this.setActiveBrowserId(data.browser_id || data.state?.id || this.activeBrowserId);
335 - this.frameState = data.state || null;
336 - if (!this.addressFocused && data.state?.currentUrl) {
337 - this.address = data.state.currentUrl;
338 - }
339 - this.frameSrc = data.image ? `data:${data.mime || "image/jpeg"};base64,${data.image}` : "";
340 - if (!data.image && !data.state) {
341 - this.setActiveBrowserId(null);
342 - this.frameState = null;
343 - this.frameSrc = "";
338 + if (data?.context_id !== this.contextId) return;
339 + this.browsers = data.browsers || this.browsers;
340 + this.setActiveBrowserId(data.browser_id || data.state?.id || this.activeBrowserId);
341 + if (data.state) {
342 + this.frameState = data.state;
343 + }
344 + if (!this.addressFocused && data.state?.currentUrl) {
345 + this.address = data.state.currentUrl;
346 + }
347 + if (data.image) {
348 + this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`);
349 + } else {
350 + this.cancelFrameRender();
351 + this.frameSrc = "";
352 + }
353 + if (!data.image && !data.state) {
354 + this.setActiveBrowserId(null);
355 + this.frameState = null;
356 + this.frameSrc = "";
357 }
358 this._lastFrameAt = Date.now();
359 };
@@ -356,12 +369,41 @@ const model = {
369 };
370 await websocket.on("browser_viewer_state", stateHandler);
371 this._stateOff = () => websocket.off("browser_viewer_state", stateHandler);
359 - }
360 - },
361 -
362 - async command(command, extra = {}) {
363 - this.error = "";
364 - const previousActiveBrowserId = this.activeBrowserId;
372 + }
373 + },
374 +
375 + queueFrameRender(frameSrc) {
376 + this._pendingFrameSrc = frameSrc;
377 + if (this._frameRenderHandle) return;
378 + const schedule = globalThis.requestAnimationFrame?.bind(globalThis);
379 + if (schedule) {
380 + this._frameRenderCancel = globalThis.cancelAnimationFrame?.bind(globalThis) || null;
381 + this._frameRenderHandle = schedule(() => this.flushFrameRender());
382 + return;
383 + }
384 + this._frameRenderCancel = globalThis.clearTimeout?.bind(globalThis) || null;
385 + this._frameRenderHandle = globalThis.setTimeout(() => this.flushFrameRender(), 16);
386 + },
387 +
388 + flushFrameRender() {
389 + this._frameRenderHandle = null;
390 + this._frameRenderCancel = null;
391 + this.frameSrc = this._pendingFrameSrc || "";
392 + this._pendingFrameSrc = "";
393 + },
394 +
395 + cancelFrameRender() {
396 + if (this._frameRenderHandle && this._frameRenderCancel) {
397 + this._frameRenderCancel(this._frameRenderHandle);
398 + }
399 + this._frameRenderHandle = null;
400 + this._frameRenderCancel = null;
401 + this._pendingFrameSrc = "";
402 + },
403 +
404 + async command(command, extra = {}) {
405 + this.error = "";
406 + const previousActiveBrowserId = this.activeBrowserId;
407 try {
408 const response = await websocket.request(
409 "browser_viewer_command",
@@ -577,10 +619,11 @@ const model = {
619 } catch {}
620 }
621 this._frameOff?.();
580 - this._stateOff?.();
581 - this._frameOff = null;
582 - this._stateOff = null;
583 - this._floatingCleanup?.();
622 + this._stateOff?.();
623 + this._frameOff = null;
624 + this._stateOff = null;
625 + this.cancelFrameRender();
626 + this._floatingCleanup?.();
627 this._floatingCleanup = null;
628 this._stageResizeObserver?.disconnect?.();
629 this._stageResizeObserver = null;
plugins/_browser/webui/main.html
+20 -18
@@ -782,24 +782,26 @@
782 color: #9f1239;
783 }
784
785 - .browser-stage {
786 - flex: 1 1 auto;
787 - display: flex;
788 - flex-direction: column;
789 - min-height: 0;
790 - overflow: auto;
791 - background: #fff;
792 - outline: none;
793 - }
785 + .browser-stage {
786 + flex: 1 1 auto;
787 + display: flex;
788 + flex-direction: column;
789 + min-height: 0;
790 + overflow: hidden;
791 + background: #fff;
792 + outline: none;
793 + }
794
795 - .browser-frame {
796 - flex: 0 0 auto;
797 - display: block;
798 - width: 100%;
799 - height: auto;
800 - user-select: none;
801 - background: #fff;
802 - }
795 + .browser-frame {
796 + flex: 0 0 auto;
797 + display: block;
798 + width: 100%;
799 + height: auto;
800 + min-width: 0;
801 + image-rendering: auto;
802 + user-select: none;
803 + background: #fff;
804 + }
805
806 .browser-status,
807 .browser-error,
@@ -884,4 +886,4 @@
886 </style>
887 </body>
888
887 -</html>
\ No newline at end of file
889 +</html>
tests/test_browser_agent_regressions.py
+116 -1
@@ -1,3 +1,4 @@
1 +import asyncio
2 import sys
3 import threading
4 from pathlib import Path
@@ -26,7 +27,11 @@ from plugins._browser.helpers.extension_manager import (
27 parse_chrome_web_store_extension_id,
28 )
29 import plugins._browser.helpers.extension_manager as browser_extension_manager_module
29 -from plugins._browser.helpers.runtime import _BrowserRuntimeCore, normalize_url
30 +from plugins._browser.helpers.runtime import (
31 + _BrowserRuntimeCore,
32 + _BrowserScreencast,
33 + normalize_url,
34 +)
35 import plugins._browser.helpers.runtime as browser_runtime_module
36 from plugins._browser.helpers.playwright import (
37 get_playwright_binary,
@@ -320,6 +325,116 @@ def test_browser_viewer_uses_tabs_for_session_switching():
325 assert "Using ${this.mainModelSummary}" in browser_store
326
327
328 +def test_browser_viewer_uses_cdp_screencast_transport():
329 + ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(
330 + encoding="utf-8"
331 + )
332 + main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
333 + encoding="utf-8"
334 + )
335 + runtime = (
336 + PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
337 + ).read_text(encoding="utf-8")
338 + browser_store = (
339 + PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
340 + ).read_text(encoding="utf-8")
341 +
342 + assert 'runtime.call("screenshot"' not in ws_browser
343 + assert "SCREENCAST_QUALITY = 92" in ws_browser
344 + assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
345 + assert '"set_viewport"' in ws_browser
346 + assert "start_screencast" in ws_browser
347 + assert "read_screencast_frame" in ws_browser
348 + assert "stop_screencast" in ws_browser
349 + assert '"Page.startScreencast"' in runtime
350 + assert '"Page.screencastFrame"' in runtime
351 + assert '"Page.screencastFrameAck"' in runtime
352 + assert '"Page.stopScreencast"' in runtime
353 + assert '"Emulation.setDeviceMetricsOverride"' in runtime
354 + assert '"Emulation.setVisibleSize"' in runtime
355 + assert "asyncio.Queue(maxsize=1)" in runtime
356 + assert "await self._stop_screencasts_for_browser(resolved_id)" in runtime
357 + assert "queueFrameRender" in browser_store
358 + assert "requestAnimationFrame" in browser_store
359 + assert "viewport_width: initialViewport?.width" in browser_store
360 + assert "viewport_height: initialViewport?.height" in browser_store
361 + assert "this.frameState = data.state || null" not in browser_store
362 + assert "overflow: hidden;" in main_html
363 + assert "object-fit: fill;" not in main_html
364 + assert "height: auto;" in main_html
365 + assert "image-rendering: auto;" in main_html
366 +
367 +
368 +@pytest.mark.asyncio
369 +async def test_browser_screencast_acknowledges_and_drops_stale_frames():
370 + first_image = (
371 + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
372 + "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"
373 + "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"
374 + "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"
375 + "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"
376 + "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"
377 + "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"
378 + "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"
379 + "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"
380 + "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"
381 + "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="
382 + )
383 +
384 + class FakeSession:
385 + def __init__(self):
386 + self.handlers = {}
387 + self.sent = []
388 + self.detached = False
389 +
390 + def on(self, event, handler):
391 + self.handlers[event] = handler
392 +
393 + async def send(self, method, params=None):
394 + self.sent.append((method, params or {}))
395 +
396 + async def detach(self):
397 + self.detached = True
398 +
399 + session = FakeSession()
400 + screencast = _BrowserScreencast(
401 + stream_id="stream",
402 + browser_id=7,
403 + session=session,
404 + mime="image/jpeg",
405 + )
406 +
407 + await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})
408 + session.handlers["Page.screencastFrame"](
409 + {"data": first_image, "metadata": {"deviceWidth": 10}, "sessionId": 1}
410 + )
411 + session.handlers["Page.screencastFrame"](
412 + {"data": "second", "metadata": {"deviceWidth": 200}, "sessionId": 2}
413 + )
414 + await asyncio.sleep(0)
415 +
416 + frame = await screencast.next_frame(timeout=0.1)
417 +
418 + assert frame["browser_id"] == 7
419 + assert frame["image"] == "second"
420 + assert frame["metadata"]["deviceWidth"] == 200
421 + assert ("Emulation.setDeviceMetricsOverride", {
422 + "width": 1118,
423 + "height": 662,
424 + "deviceScaleFactor": 1,
425 + "mobile": False,
426 + "dontSetVisibleSize": True,
427 + }) in session.sent
428 + assert ("Emulation.setVisibleSize", {"width": 1118, "height": 662}) in session.sent
429 + assert ("Page.screencastFrameAck", {"sessionId": 1}) in session.sent
430 + assert ("Page.screencastFrameAck", {"sessionId": 2}) in session.sent
431 +
432 + await screencast.stop()
433 +
434 + assert ("Page.stopScreencast", {}) in session.sent
435 + assert session.detached is True
436 +
437 +
438 def test_browser_docker_installs_full_chromium_to_persistent_cache():
439 script = (
440 PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_playwright.sh"