@samitouri / QOSami_Agents_Zero-2 / commits / decb05a6

Stabilize browser viewer viewport rendering

Decode browser frames before display and only render frames that match the active viewer viewport, avoiding stretched stale screencast images during startup and resize. Keep rejecting mismatched CDP screencast frames on the backend, extend canvas viewport settling, and cover the behavior with browser regression tests. Include small browser panel CSS polish.

Alessandro committed Apr 28, 2026 at 03:50 UTC decb05a6823b4a04fddafa2357ea8155c766e638
4 files changed +177 -34
plugins/_browser/helpers/runtime.py
+5 -5
@@ -111,7 +111,6 @@ class _BrowserScreencast:
111 self._ack_tasks: set[asyncio.Task] = set()
112 self._expected_width = 0
113 self._expected_height = 0
114 - self._dimension_mismatches = 0
114
115 async def start(
116 self,
@@ -125,7 +124,6 @@ class _BrowserScreencast:
124 height = max(200, min(4096, int(viewport.get("height") or DEFAULT_VIEWPORT["height"])))
125 self._expected_width = width
126 self._expected_height = height
128 - self._dimension_mismatches = 0
127 with contextlib.suppress(Exception):
128 await self.session.send("Page.enable")
129 await self.session.send(
@@ -229,10 +227,12 @@ class _BrowserScreencast:
227 if not size:
228 return True
229 width, height = size
232 - if abs(width - self._expected_width) <= 2 and abs(height - self._expected_height) <= 2:
230 + if (
231 + abs(width - self._expected_width) <= VIEWPORT_SIZE_TOLERANCE
232 + and abs(height - self._expected_height) <= VIEWPORT_SIZE_TOLERANCE
233 + ):
234 return True
234 - self._dimension_mismatches += 1
235 - return self._dimension_mismatches > 10
235 + return False
236
237 @staticmethod
238 def _jpeg_size(data: str) -> tuple[int, int] | None:
plugins/_browser/webui/browser-panel.html
+1 -1
@@ -620,7 +620,7 @@
620 align-items: end;
621 gap: 6px;
622 min-width: 0;
623 - padding-bottom: 1px;
623 + padding-bottom: 7px;
624 }
625
626 .browser-annotate-toggle {
plugins/_browser/webui/browser-store.js
+113 -14
@@ -13,8 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
13 const BROWSER_CONFIG_REFRESH_MS = 15000;
14 const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
15 const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
16 -const CANVAS_VIEWPORT_SETTLE_MS = 260;
17 -const SURFACE_VIEWPORT_STABLE_FRAMES = 2;
16 +const CANVAS_VIEWPORT_SETTLE_MS = 520;
17 +const SURFACE_VIEWPORT_STABLE_FRAMES = 4;
18 +const FRAME_REJECT_SYNC_COOLDOWN_MS = 600;
19 const ANNOTATION_DRAG_THRESHOLD = 6;
20 const ANNOTATION_MAX_COMMENTS = 24;
21 const ANNOTATION_DOM_LIMIT = 1200;
@@ -55,6 +56,34 @@ function nextAnimationFrame() {
56 });
57 }
58
59 +function loadFrameDimensions(src) {
60 + return new Promise((resolve) => {
61 + if (!src) {
62 + resolve(null);
63 + return;
64 + }
65 +
66 + const image = new Image();
67 + let settled = false;
68 + const finish = (dimensions) => {
69 + if (settled) return;
70 + settled = true;
71 + resolve(dimensions);
72 + };
73 +
74 + image.onload = () => finish({
75 + width: image.naturalWidth || 0,
76 + height: image.naturalHeight || 0,
77 + });
78 + image.onerror = () => finish(null);
79 + image.src = src;
80 +
81 + if (image.complete) {
82 + image.onload();
83 + }
84 + });
85 +}
86 +
87 const model = {
88 loading: true,
89 error: "",
@@ -79,13 +108,17 @@ const model = {
108 _frameOff: null,
109 _stateOff: null,
110 _lastFrameAt: 0,
111 + _lastFrameDimensions: null,
112 _pendingFrameSrc: "",
113 + _pendingFrameOptions: null,
114 _frameRenderHandle: null,
115 _frameRenderCancel: null,
116 + _frameRenderSequence: 0,
117 _floatingCleanup: null,
118 _stageElement: null,
119 _stageResizeObserver: null,
120 _viewportSyncTimer: null,
121 + _lastFrameRejectSyncAt: 0,
122 _lastViewportKey: "",
123 _lastViewport: null,
124 _annotationPointer: null,
@@ -466,6 +499,7 @@ const model = {
499 resetRenderedFrame() {
500 this.cancelFrameRender();
501 this.frameSrc = "";
502 + this._lastFrameDimensions = null;
503 this._lastFrameAt = 0;
504 },
505
@@ -599,11 +633,16 @@ const model = {
633 this.address = data.state.currentUrl;
634 }
635 if (data.image) {
602 - this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`);
603 - if (this.sameBrowserId(this.switchingBrowserId, incomingBrowserId || this.activeBrowserId)) {
604 - this.switchingBrowserId = null;
605 - }
606 - this._surfaceSwitching = false;
636 + const frameBrowserId = incomingBrowserId || this.activeBrowserId;
637 + this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {
638 + browserId: frameBrowserId,
639 + onAccepted: () => {
640 + if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) {
641 + this.switchingBrowserId = null;
642 + }
643 + this._surfaceSwitching = false;
644 + },
645 + });
646 } else {
647 this.cancelFrameRender();
648 if (!data.state) {
@@ -654,8 +693,9 @@ const model = {
693 }
694 },
695
657 - queueFrameRender(frameSrc) {
696 + queueFrameRender(frameSrc, options = {}) {
697 this._pendingFrameSrc = frameSrc;
698 + this._pendingFrameOptions = options || null;
699 if (this._frameRenderHandle) return;
700 const schedule = globalThis.requestAnimationFrame?.bind(globalThis);
701 if (schedule) {
@@ -670,8 +710,59 @@ const model = {
710 flushFrameRender() {
711 this._frameRenderHandle = null;
712 this._frameRenderCancel = null;
673 - this.frameSrc = this._pendingFrameSrc || "";
713 + const frameSrc = this._pendingFrameSrc || "";
714 + const options = this._pendingFrameOptions || {};
715 this._pendingFrameSrc = "";
716 + this._pendingFrameOptions = null;
717 + const sequence = this._frameRenderSequence + 1;
718 + this._frameRenderSequence = sequence;
719 + void this.renderDecodedFrame(frameSrc, options, sequence);
720 + },
721 +
722 + async renderDecodedFrame(frameSrc, options = {}, sequence = 0) {
723 + if (!frameSrc) {
724 + if (sequence === this._frameRenderSequence) {
725 + this.frameSrc = "";
726 + }
727 + return;
728 + }
729 + const dimensions = await loadFrameDimensions(frameSrc);
730 + if (sequence !== this._frameRenderSequence) return;
731 + const viewport = this.currentViewportSize() || this._lastViewport;
732 + if (!this.frameMatchesViewport(dimensions, viewport)) {
733 + this.requestViewportSyncAfterRejectedFrame();
734 + return;
735 + }
736 + this.frameSrc = frameSrc;
737 + this._lastFrameDimensions = dimensions;
738 + this._lastFrameAt = Date.now();
739 + options?.onAccepted?.();
740 + },
741 +
742 + frameMatchesViewport(dimensions = null, viewport = null) {
743 + if (!dimensions?.width || !dimensions?.height || !viewport?.width || !viewport?.height) {
744 + return false;
745 + }
746 + return Math.abs(Number(dimensions.width) - Number(viewport.width)) <= VIEWPORT_SYNC_SIZE_TOLERANCE
747 + && Math.abs(Number(dimensions.height) - Number(viewport.height)) <= VIEWPORT_SYNC_SIZE_TOLERANCE;
748 + },
749 +
750 + requestViewportSyncAfterRejectedFrame() {
751 + const now = Date.now();
752 + if (now - this._lastFrameRejectSyncAt < FRAME_REJECT_SYNC_COOLDOWN_MS) return;
753 + this._lastFrameRejectSyncAt = now;
754 + this.queueViewportSync(true);
755 + },
756 +
757 + clearRenderedFrameIfViewportChanged() {
758 + const viewport = this.currentViewportSize();
759 + if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return;
760 + if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return;
761 + this.resetRenderedFrame();
762 + if (this.activeBrowserId) {
763 + this._surfaceSwitching = true;
764 + this.switchingBrowserId = this.activeBrowserId;
765 + }
766 },
767
768 cancelFrameRender() {
@@ -681,6 +772,8 @@ const model = {
772 this._frameRenderHandle = null;
773 this._frameRenderCancel = null;
774 this._pendingFrameSrc = "";
775 + this._pendingFrameOptions = null;
776 + this._frameRenderSequence += 1;
777 },
778
779 async command(command, extra = {}) {
@@ -865,11 +958,16 @@ const model = {
958 if (snapshot.state) {
959 this.applyActiveFrameState(snapshot.state);
960 }
868 - this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`);
869 - if (this.sameBrowserId(this.switchingBrowserId, snapshotId || this.activeBrowserId)) {
870 - this.switchingBrowserId = null;
871 - }
872 - this._surfaceSwitching = false;
961 + const frameBrowserId = snapshotId || this.activeBrowserId;
962 + this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
963 + browserId: frameBrowserId,
964 + onAccepted: () => {
965 + if (this.sameBrowserId(this.switchingBrowserId, frameBrowserId)) {
966 + this.switchingBrowserId = null;
967 + }
968 + this._surfaceSwitching = false;
969 + },
970 + });
971 },
972
973 isSwitchingBrowser() {
@@ -1371,6 +1469,7 @@ const model = {
1469 },
1470
1471 queueViewportSync(force = false) {
1472 + this.clearRenderedFrameIfViewportChanged();
1473 if (this._viewportSyncTimer) {
1474 globalThis.clearTimeout(this._viewportSyncTimer);
1475 }
tests/test_browser_agent_regressions.py
+58 -14
@@ -110,6 +110,21 @@ import plugins._browser.tools.browser as browser_tool_module
110 import plugins._browser.api.ws_browser as ws_browser_module
111
112
113 +SMALL_JPEG_10X10 = (
114 + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
115 + "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"
116 + "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"
117 + "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"
118 + "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"
119 + "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"
120 + "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"
121 + "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"
122 + "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"
123 + "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"
124 + "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="
125 +)
126 +
127 +
128 def test_browser_url_normalization_matches_address_bar_hosts():
129 assert normalize_url("localhost:3000") == "http://localhost:3000/"
130 assert normalize_url("127.0.0.1:8000/path") == "http://127.0.0.1:8000/path"
@@ -365,7 +380,8 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():
380 encoding="utf-8"
381 )
382
368 - assert "const CANVAS_VIEWPORT_SETTLE_MS = 260;" in js
383 + assert "const CANVAS_VIEWPORT_SETTLE_MS = 520;" in js
384 + assert "const SURFACE_VIEWPORT_STABLE_FRAMES = 4;" in js
385 assert "surfaceViewportMeasurement()" in js
386 assert "rawWidth" in js
387 assert "rawHeight" in js
@@ -454,6 +470,9 @@ def test_browser_viewer_uses_cdp_screencast_transport():
470 assert "viewport_width: initialViewport?.width" in browser_store
471 assert "viewport_height: initialViewport?.height" in browser_store
472 assert "this.frameState = data.state || null" not in browser_store
473 + assert "function loadFrameDimensions(src)" in browser_store
474 + assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store
475 + assert "requestViewportSyncAfterRejectedFrame()" in browser_store
476 assert "overflow: hidden;" in main_html
477 assert "object-fit: fill;" in main_html
478 assert "image-rendering: auto;" in main_html
@@ -504,19 +523,7 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():
523
524 @pytest.mark.anyio
525 async def test_browser_screencast_acknowledges_and_drops_stale_frames():
507 - first_image = (
508 - "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
509 - "DBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/"
510 - "2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIy"
511 - "MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAAKAAoDASIAAhEBAxEB/8QAFQAB"
512 - "AAAAAAAAAAAAAAAAAAAACf/EABQQAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhADE"
513 - "AAAAKf/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAEFAqf/xAAUEQEAAAAAAAA"
514 - "AAAAAAAAAAAAA/9oACAEDAQE/ASP/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oACAECA"
515 - "QE/ASP/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/9oACAEBAAY/Aqf/xAAUEAEAAAAAAA"
516 - "AAAAAAAAAAAAAA/9oACAEBAAE/ISf/2gAMAwEAAgADAAAAEP/EABQRAQAAAAAAAAA"
517 - "AAAAAAAAAAP/aAAgBAwEBPxAk/8QAFBEBAAAAAAAAAAAAAAAAAAAAAP/aAAgBAgEB"
518 - "PxAk/8QAFBABAAAAAAAAAAAAAAAAAAAAAP/aAAgBAQABPxAn/9k="
519 - )
526 + first_image = SMALL_JPEG_10X10
527
528 class FakeSession:
529 def __init__(self):
@@ -572,6 +579,43 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
579 assert session.detached is True
580
581
582 +@pytest.mark.anyio
583 +async def test_browser_screencast_keeps_rejecting_wrong_viewport_frames():
584 + class FakeSession:
585 + def __init__(self):
586 + self.handlers = {}
587 + self.sent = []
588 +
589 + def on(self, event, handler):
590 + self.handlers[event] = handler
591 +
592 + async def send(self, method, params=None):
593 + self.sent.append((method, params or {}))
594 +
595 + async def detach(self):
596 + pass
597 +
598 + session = FakeSession()
599 + screencast = _BrowserScreencast(
600 + stream_id="stream",
601 + browser_id=7,
602 + session=session,
603 + mime="image/jpeg",
604 + )
605 +
606 + await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 1118, "height": 662})
607 + for session_id in range(1, 14):
608 + session.handlers["Page.screencastFrame"](
609 + {"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": session_id}
610 + )
611 + await asyncio.sleep(0)
612 +
613 + assert await screencast.pop_frame() is None
614 + assert ("Page.screencastFrameAck", {"sessionId": 13}) in session.sent
615 +
616 + await screencast.stop()
617 +
618 +
619 def test_browser_docker_installs_full_chromium_to_persistent_cache():
620 script = (
621 PROJECT_ROOT / "docker" / "run" / "fs" / "ins" / "install_playwright.sh"