Keep WebUI browser frames on base64 fallback
Disable WebUI binary-frame negotiation until its Socket.IO client reconstructs frame attachments as real Blob, ArrayBuffer, or typed-array values. Reject placeholder binary payloads instead of wrapping them into broken image blobs, preserving the slim-frame path through base64 frames. Document the transport constraint and pin the fallback behavior in browser regression coverage.
Alessandro committed
Jul 3, 2026 at 02:15 UTC
727a840e10c1d877d5d54ce68e3e2ffa0ddef485
3 files changed
+10
-5
plugins/_browser/AGENTS.md
+1
-1
@@ -19,7 +19,7 @@
19
- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
20
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
21
- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
22
-- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback.
22
+- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values.
23
- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
24
- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
25
- Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback.
plugins/_browser/webui/browser-store.js
+6
-4
@@ -34,7 +34,8 @@ const ANNOTATION_DOM_LIMIT = 1200;
34
const ANNOTATION_TRAY_MARGIN = 10;
35
const BROWSER_VISUAL_SHORTCUT_KEYS = new Set(["a", "c", "insert", "v", "x", "y", "z"]);
36
const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
37
-const BROWSER_BINARY_FRAMES_SUPPORTED = typeof Blob === "function"
37
+const BROWSER_BINARY_FRAME_REQUESTS_ENABLED = false;
38
+const BROWSER_BINARY_PAYLOADS_SUPPORTED = typeof Blob === "function"
39
&& typeof globalThis.URL?.createObjectURL === "function";
40
41
function makeViewerToken() {
@@ -123,8 +124,8 @@ function frameImageSource(data = {}) {
124
const isArrayBuffer = typeof ArrayBuffer !== "undefined" && image instanceof ArrayBuffer;
125
const isView = typeof ArrayBuffer !== "undefined" && ArrayBuffer.isView?.(image);
126
const isBlob = typeof Blob !== "undefined" && image instanceof Blob;
126
- if (data.encoding === "binary" || isArrayBuffer || isView || isBlob) {
127
- if (!BROWSER_BINARY_FRAMES_SUPPORTED) return null;
127
+ if (isArrayBuffer || isView || isBlob) {
128
+ if (!BROWSER_BINARY_PAYLOADS_SUPPORTED) return null;
129
const blob = isBlob ? image : new Blob([image], { type: mime });
130
const src = globalThis.URL.createObjectURL(blob);
131
return {
@@ -133,6 +134,7 @@ function frameImageSource(data = {}) {
134
cleanup: () => globalThis.URL.revokeObjectURL(src),
135
};
136
}
137
+ if (data.encoding === "binary") return null;
138
if (typeof image !== "string") return null;
139
return {
140
src: `data:${mime};base64,${image}`,
@@ -944,7 +946,7 @@ const model = {
946
},
947
948
supportsBinaryFrames() {
947
- return BROWSER_BINARY_FRAMES_SUPPORTED;
949
+ return BROWSER_BINARY_FRAME_REQUESTS_ENABLED && BROWSER_BINARY_PAYLOADS_SUPPORTED;
950
},
951
952
captureDevicePixelRatio() {
tests/test_browser_agent_regressions.py
+3
@@ -1182,6 +1182,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1182
assert "queueFrameRender" in browser_store
1183
assert "requestAnimationFrame" in browser_store
1184
assert "function frameImageSource(data = {})" in browser_store
1185
+ assert "const BROWSER_BINARY_FRAME_REQUESTS_ENABLED = false;" in browser_store
1186
+ assert "const BROWSER_BINARY_PAYLOADS_SUPPORTED = typeof Blob" in browser_store
1187
+ assert "if (data.encoding === \"binary\") return null;" in browser_store
1188
assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
1189
assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
1190
assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store