Make Browser viewer CDP-live by default
Default the WebUI Browser surface back to live CDP screencast while keeping snapshot/state transport as a fallback. Carry viewer transport through the WebSocket API and Browser store, use screencast metadata to avoid extra frame dimension decode, and include frame-chain action metadata in Browser content results. Update Browser prompts, skill guidance, DOX, and focused regressions so agent actions prefer DOM/CDP refs while the visible Browser remains responsive.
Alessandro committed
Jun 17, 2026 at 16:15 UTC
d398656d337739e41adbbbab70bec7e26d8b1112
7 files changed
+211
-20
plugins/_browser/AGENTS.md
+5
-1
@@ -16,12 +16,16 @@
16
## Local Contracts
17
18
- Keep browser actions safe around external pages, credentials, and user data.
19
-- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility.
19
+- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
20
+- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
21
+- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
22
+- Prefer DOM/CDP browser actions with refs, selectors, frame-chain refs, and screenshots over viewport coordinate input. Coordinates remain a visual fallback.
23
- Do not hardcode user-specific browser paths or secrets.
24
25
## Work Guidance
26
27
- Coordinate tool, helper, and panel changes so browser state shown in the UI matches tool behavior.
28
+- Do not depend on nested Electron `<webview>` support or launcher-specific preload bridges unless the launcher exposes that bridge as an explicit contract.
29
- Keep `prompts/agent.system.tool.browser.md` as a compact callable contract; move detailed browser workflows into `skills/browser-automation/SKILL.md`.
30
- Keep `skills/browser-automation/SKILL.md` frontmatter triggers current with rendered browsing, host-browser, screenshot, and web-interaction user phrasing so relevant-skill recall can surface the skill before the full browser workflow is needed.
31
- Keep fragile form guidance progressively disclosed through `skills/browser-form-workflows/SKILL.md`, linked from the browser prompt through `browser-automation`.
plugins/_browser/api/ws_browser.py
+113
-6
@@ -14,7 +14,11 @@ from plugins._browser.helpers.runtime import get_runtime, list_runtime_sessions
14
FRAME_IDLE_POLL_SECONDS = 0.05
15
FRAME_RETRY_DELAY_SECONDS = 0.5
16
FRAME_STATE_REFRESH_SECONDS = 0.75
17
+SNAPSHOT_STATE_POLL_SECONDS = 0.75
18
SCREENCAST_QUALITY = 92
19
+VIEWER_TRANSPORT_SCREENCAST = "screencast"
20
+VIEWER_TRANSPORT_SNAPSHOT = "snapshot"
21
+VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT}
22
23
24
class WsBrowser(WsHandler):
@@ -92,11 +96,14 @@ class WsBrowser(WsHandler):
96
if existing:
97
existing.cancel()
98
viewer_id = str(data.get("viewer_id") or "")
99
+ viewer_transport = self._viewer_transport(data)
100
snapshot = None
101
if runtime:
97
- self._streams[stream_key] = asyncio.create_task(
98
- self._stream_frames(sid, context_id, active_id, viewer_id)
99
- )
102
+ if viewer_transport == VIEWER_TRANSPORT_SCREENCAST:
103
+ stream_task = self._stream_frames(sid, context_id, active_id, viewer_id)
104
+ else:
105
+ stream_task = self._stream_state(sid, context_id, active_id, viewer_id)
106
+ self._streams[stream_key] = asyncio.create_task(stream_task)
107
snapshot = await self._snapshot_for_browser(runtime, active_id)
108
109
return {
@@ -107,6 +114,7 @@ class WsBrowser(WsHandler):
114
"browsers": await self._all_browser_tabs(),
115
"all_browsers": True,
116
"viewer_id": viewer_id,
117
+ "viewer_transport": viewer_transport,
118
}
119
120
def _unsubscribe(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
@@ -215,6 +223,7 @@ class WsBrowser(WsHandler):
223
"browsers": all_browsers,
224
"all_browsers": True,
225
"last_interacted_browser_id": last_interacted_browser_id,
226
+ "viewer_transport": self._viewer_transport(data),
227
},
228
correlation_id=data.get("correlationId"),
229
)
@@ -228,6 +237,7 @@ class WsBrowser(WsHandler):
237
"command": command,
238
"browser_id": browser_id,
239
"viewer_id": viewer_id,
240
+ "viewer_transport": self._viewer_transport(data),
241
}
242
243
async def _input(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
@@ -374,7 +384,12 @@ class WsBrowser(WsHandler):
384
try:
385
runtime = await get_runtime(context_id, create=False)
386
if not runtime:
377
- await self._emit_empty_frame(sid, context_id, viewer_id=viewer_id)
387
+ await self._emit_empty_frame(
388
+ sid,
389
+ context_id,
390
+ viewer_id=viewer_id,
391
+ frame_source=VIEWER_TRANSPORT_SCREENCAST,
392
+ )
393
await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
394
continue
395
@@ -382,7 +397,13 @@ class WsBrowser(WsHandler):
397
browsers = listing.get("browsers") or []
398
active_id = self._active_browser_id(listing, browser_id)
399
if not active_id:
385
- await self._emit_empty_frame(sid, context_id, browsers=browsers, viewer_id=viewer_id)
400
+ await self._emit_empty_frame(
401
+ sid,
402
+ context_id,
403
+ browsers=browsers,
404
+ viewer_id=viewer_id,
405
+ frame_source=VIEWER_TRANSPORT_SCREENCAST,
406
+ )
407
await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
408
continue
409
@@ -407,6 +428,7 @@ class WsBrowser(WsHandler):
428
"mime": "",
429
"state": state,
430
"frame_source": "state",
431
+ "viewer_transport": VIEWER_TRANSPORT_SCREENCAST,
432
},
433
)
434
@@ -435,7 +457,8 @@ class WsBrowser(WsHandler):
457
frame["browser_id"] = active_id
458
frame["browsers"] = browsers
459
frame["state"] = state
438
- frame["frame_source"] = "screencast"
460
+ frame["frame_source"] = VIEWER_TRANSPORT_SCREENCAST
461
+ frame["viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST
462
await self.emit_to(sid, "browser_viewer_frame", frame)
463
except asyncio.CancelledError:
464
raise
@@ -447,6 +470,70 @@ class WsBrowser(WsHandler):
470
await runtime.call("stop_screencast", stream_id)
471
stream_id = None
472
473
+ async def _stream_state(
474
+ self,
475
+ sid: str,
476
+ context_id: str,
477
+ browser_id: int | str | None,
478
+ viewer_id: str = "",
479
+ ) -> None:
480
+ last_signature = None
481
+ while True:
482
+ try:
483
+ runtime = await get_runtime(context_id, create=False)
484
+ if not runtime:
485
+ signature = (None, ())
486
+ if signature != last_signature:
487
+ await self._emit_empty_frame(
488
+ sid,
489
+ context_id,
490
+ viewer_id=viewer_id,
491
+ frame_source=VIEWER_TRANSPORT_SNAPSHOT,
492
+ )
493
+ last_signature = signature
494
+ await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
495
+ continue
496
+
497
+ listing = await runtime.call("list")
498
+ browsers = listing.get("browsers") or []
499
+ active_id = self._active_browser_id(listing, browser_id)
500
+ state = self._state_for_browser(browsers, active_id, None) if active_id else None
501
+ signature = (
502
+ str(active_id or ""),
503
+ tuple(
504
+ (
505
+ str(browser.get("context_id") or context_id),
506
+ str(browser.get("id") or ""),
507
+ str(browser.get("currentUrl") or ""),
508
+ str(browser.get("title") or ""),
509
+ bool(browser.get("loading")),
510
+ )
511
+ for browser in browsers
512
+ ),
513
+ )
514
+ if signature != last_signature:
515
+ await self.emit_to(
516
+ sid,
517
+ "browser_viewer_frame",
518
+ {
519
+ "context_id": context_id,
520
+ "viewer_id": viewer_id,
521
+ "browser_id": active_id,
522
+ "browsers": browsers,
523
+ "image": "",
524
+ "mime": "",
525
+ "state": state,
526
+ "frame_source": VIEWER_TRANSPORT_SNAPSHOT,
527
+ "viewer_transport": VIEWER_TRANSPORT_SNAPSHOT,
528
+ },
529
+ )
530
+ last_signature = signature
531
+ await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS)
532
+ except asyncio.CancelledError:
533
+ raise
534
+ except Exception:
535
+ await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
536
+
537
@staticmethod
538
def _active_browser_id(
539
listing: dict[str, Any],
@@ -484,6 +571,7 @@ class WsBrowser(WsHandler):
571
*,
572
browsers: list[dict[str, Any]] | None = None,
573
viewer_id: str = "",
574
+ frame_source: str = "",
575
) -> None:
576
await self.emit_to(
577
sid,
@@ -496,9 +584,28 @@ class WsBrowser(WsHandler):
584
"image": "",
585
"mime": "",
586
"state": None,
587
+ "frame_source": frame_source,
588
+ "viewer_transport": frame_source or VIEWER_TRANSPORT_SNAPSHOT,
589
},
590
)
591
592
+ @staticmethod
593
+ def _viewer_transport(data: dict[str, Any]) -> str:
594
+ raw = (
595
+ data.get("viewer_transport")
596
+ or data.get("viewerTransport")
597
+ or data.get("surface_transport")
598
+ or data.get("surfaceTransport")
599
+ or data.get("transport")
600
+ or ""
601
+ )
602
+ normalized = str(raw or "").strip().lower().replace("-", "_")
603
+ if normalized == "live":
604
+ normalized = VIEWER_TRANSPORT_SCREENCAST
605
+ if normalized in VIEWER_TRANSPORTS:
606
+ return normalized
607
+ return VIEWER_TRANSPORT_SNAPSHOT
608
+
609
@staticmethod
610
def _viewport_from_data(data: dict[str, Any]) -> dict[str, int] | None:
611
try:
plugins/_browser/assets/browser-page-content.js
+6
-1
@@ -1,7 +1,7 @@
1
(() => {
2
const GLOBAL_KEY = "__spaceBrowserPageContent__";
3
const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";
4
- const VERSION = "12";
4
+ const VERSION = "13";
5
const REQUIRED_API_NAMES = Object.freeze([
6
"annotate",
7
"boundingBoxFor",
@@ -3103,8 +3103,12 @@
3103
3104
function buildActionResult(entry, extra = {}) {
3105
return {
3106
+ actionStrategy: entry.helperBacked ? "frame_chain_ref" : "dom_ref",
3107
captureId: state.captureId,
3108
descriptorTags: Array.isArray(entry?.descriptorTags) ? entry.descriptorTags.slice() : [],
3109
+ frameChain: Array.isArray(entry?.frameChain) ? entry.frameChain.slice() : [],
3110
+ frameId: entry.frameId || "",
3111
+ nodeId: entry.nodeId || "",
3112
referenceId: entry.referenceId,
3113
semanticTags: Array.isArray(entry?.semanticTags) ? entry.semanticTags.slice() : [],
3114
state: entry.state || collectElementStateMetadata(entry.element, state.captureOptions),
@@ -3116,6 +3120,7 @@
3120
3121
function buildHelperBackedActionResult(entry, helperResult, extra = {}) {
3122
return {
3123
+ actionStrategy: "frame_chain_ref",
3124
captureId: state.captureId,
3125
descriptorTags: Array.isArray(helperResult?.descriptorTags) ? helperResult.descriptorTags : (entry.descriptorTags || []),
3126
frameChain: entry.frameChain.slice(),
plugins/_browser/prompts/agent.system.tool.browser.md
+1
-1
@@ -11,7 +11,7 @@ Actions: tabs `open`, `list`, `state`, `set_active`, `navigate`, `back`, `forwar
11
12
Rules:
13
- If the user asks for an existing tab, page title, or already-open URL, call `list` first, match by `title` or `currentUrl`, then use `set_active` or `navigate` on that `browser_id`.
14
-- Use refs from the latest `content`; same-page controls may use `selector`.
14
+- Prefer DOM/CDP actions: use refs from the latest `content`, including frame-chain refs, or same-page `selector`; use coordinates only when refs/selectors are unavailable or the user explicitly asks for visual manual control.
15
- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned `vision_load.tool_args.paths` value before reasoning visually.
16
- Keep the tab set small; close pages after extracting what you need.
17
- `multi` is only a browser action: use `tool_name: "browser"` with `tool_args.action: "multi"`. Never use `tool_name: "multi"`.
plugins/_browser/skills/browser-automation/SKILL.md
+4
-3
@@ -30,7 +30,7 @@ For fragile forms, load `browser-form-workflows` with `skills_tool:load` before
30
31
1. `open` creates a browser tab and returns a `browser_id`.
32
2. `content` returns readable markdown plus typed refs like `[link 3]`, `[button 6]`, `[input text 8]`.
33
-3. Interact with refs using `click`, `type`, `submit`, `scroll`, etc.
33
+3. Interact with refs using `click`, `type`, `submit`, `scroll`, etc.; iframe/shadow targets may return frame-chain metadata in action results.
34
4. Use `navigate` on an existing `browser_id` for serial browsing.
35
5. Keep only a small working tab set; close pages when finished.
36
6. If the user asks for an existing tab, page title, or already-open URL, call `list` first, match by `title` or `currentUrl`, then use `set_active` or `navigate` on that `browser_id` instead of opening a new tab.
@@ -63,8 +63,9 @@ Screenshot args include `quality`, `full_page`, and optional `path`. Without `pa
63
64
## Pointer And Keyboard
65
66
-- `hover`, `double_click`, `right_click`, and `drag` accept refs or viewport coordinates.
67
-- Coordinates are Chromium viewport CSS pixels and match screenshots.
66
+- Prefer refs/selectors and DOM/CDP actions over viewport coordinates.
67
+- `hover`, `double_click`, `right_click`, and `drag` accept refs or viewport coordinates when no reliable ref exists.
68
+- Coordinates are Chromium viewport CSS pixels and match screenshots; treat them as visual fallback, not the default interaction path.
69
- `key_chord` presses keys in order and releases in reverse.
70
- `clipboard` actions are copy, cut, or paste.
71
- `set_viewport` resizes the page viewport.
plugins/_browser/webui/browser-store.js
+52
-3
@@ -20,6 +20,8 @@ const EXTENSIONS_ROOT = "/a0/usr/_browser/extensions";
20
const BROWSER_SUBSCRIBE_TIMEOUT_MS = 60000;
21
const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
22
const BROWSER_CONFIG_REFRESH_MS = 15000;
23
+const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";
24
+const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";
25
const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
26
const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
27
const CANVAS_VIEWPORT_SETTLE_MS = 520;
@@ -123,6 +125,8 @@ const model = {
125
address: "",
126
frameSrc: "",
127
frameState: null,
128
+ viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST,
129
+ liveScreencastEnabled: true,
130
annotating: false,
131
annotationComments: [],
132
annotationDraft: null,
@@ -885,12 +889,42 @@ const model = {
889
if (!this.isCurrentSurfaceOpen(sequence)) {
890
return;
891
}
888
- await this.syncViewport(true, { restartStream: this._mode === "canvas" });
892
+ await this.syncViewport(true, {
893
+ restartStream: this._mode === "canvas" && this.usesScreencastTransport(),
894
+ });
895
if (this._mode !== "canvas") return;
896
this.scheduleViewportSyncForSurface(sequence, 240);
897
this.scheduleViewportSyncForSurface(sequence, 520);
898
},
899
900
+ requestedViewerTransport() {
901
+ return this.liveScreencastEnabled
902
+ ? BROWSER_VIEWER_TRANSPORT_SCREENCAST
903
+ : BROWSER_VIEWER_TRANSPORT_SNAPSHOT;
904
+ },
905
+
906
+ normalizeViewerTransport(value = "") {
907
+ const normalized = String(value || "").trim().toLowerCase().replace("-", "_");
908
+ if (normalized === BROWSER_VIEWER_TRANSPORT_SCREENCAST) {
909
+ return BROWSER_VIEWER_TRANSPORT_SCREENCAST;
910
+ }
911
+ return BROWSER_VIEWER_TRANSPORT_SNAPSHOT;
912
+ },
913
+
914
+ usesScreencastTransport() {
915
+ return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST;
916
+ },
917
+
918
+ frameDimensionsFromMetadata(metadata = null) {
919
+ if (!metadata || typeof metadata !== "object") return null;
920
+ const width = Number(metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth || 0);
921
+ const height = Number(metadata.expectedHeight || metadata.deviceHeight || metadata.jpegHeight || 0);
922
+ if (!Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0) {
923
+ return null;
924
+ }
925
+ return { width, height };
926
+ },
927
+
928
scheduleViewportSyncForSurface(sequence, delayMs = 0) {
929
globalThis.setTimeout?.(() => {
930
if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== "canvas") {
@@ -951,6 +985,7 @@ const model = {
985
browser_id: requestedBrowserId,
986
viewer_id: viewerToken,
987
create_browser: Boolean(options.createBrowser || options.create_browser),
988
+ viewer_transport: this.requestedViewerTransport(),
989
viewport_width: initialViewport?.width,
990
viewport_height: initialViewport?.height,
991
},
@@ -973,6 +1008,7 @@ const model = {
1008
}
1009
const data = firstOk(response);
1010
this.applyBrowserListing(data.browsers || [], contextId, { replaceAll: Boolean(data.all_browsers) });
1011
+ this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1012
this.setActiveBrowserId(
1013
data.active_browser_id || requestedBrowserId || this.activeBrowserId || null,
1014
data.active_browser_context_id || contextId,
@@ -987,6 +1023,9 @@ const model = {
1023
const frameHandler = ({ data }) => {
1024
if (data?.context_id !== this.contextId) return;
1025
if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
1026
+ if (data?.viewer_transport) {
1027
+ this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1028
+ }
1029
const incomingContextId = this.normalizeContextId(data.context_id || this.contextId);
1030
const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id);
1031
this.applyBrowserListing(data.browsers || [], incomingContextId, { replaceContext: true });
@@ -1011,6 +1050,7 @@ const model = {
1050
this.queueFrameRender(`data:${data.mime || "image/jpeg"};base64,${data.image}`, {
1051
browserId: frameBrowserId,
1052
contextId: incomingContextId,
1053
+ dimensions: this.frameDimensionsFromMetadata(data.metadata),
1054
onAccepted: () => {
1055
if (
1056
this.sameBrowserId(this.switchingBrowserId, frameBrowserId)
@@ -1041,6 +1081,9 @@ const model = {
1081
const stateHandler = ({ data }) => {
1082
if (data?.context_id !== this.contextId) return;
1083
if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
1084
+ if (data?.viewer_transport) {
1085
+ this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1086
+ }
1087
const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId);
1088
this.applyBrowserListing(data.browsers || [], commandContextId, { replaceAll: Boolean(data.all_browsers) });
1089
const command = String(data.command || "").toLowerCase();
@@ -1109,7 +1152,7 @@ const model = {
1152
}
1153
return;
1154
}
1112
- const dimensions = await loadFrameDimensions(frameSrc);
1155
+ const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc);
1156
if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
1157
return;
1158
}
@@ -1254,12 +1297,14 @@ const model = {
1297
context_id: targetContextId,
1298
browser_id: targetBrowserId,
1299
viewer_id: this._viewerToken,
1300
+ viewer_transport: this.requestedViewerTransport(),
1301
command,
1302
},
1303
{ timeoutMs: 20000 },
1304
);
1305
const data = firstOk(response);
1306
this.applyBrowserListing(data.browsers || [], targetContextId, { replaceAll: Boolean(data.all_browsers) });
1307
+ this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1308
const result = data.result || {};
1309
const resultContextId = this.normalizeContextId(
1310
result.context_id
@@ -1317,6 +1362,9 @@ const model = {
1362
},
1363
1364
async restartCanvasStreamAfterPageChange() {
1365
+ if (!this.usesScreencastTransport()) {
1366
+ return;
1367
+ }
1368
const surfaceSequence = this._surfaceOpenSequence;
1369
if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
1370
return;
@@ -2340,7 +2388,7 @@ const model = {
2388
input_type: "viewport",
2389
width: viewport.width,
2390
height: viewport.height,
2343
- restart_stream: restartStream,
2391
+ restart_stream: restartStream && this.usesScreencastTransport(),
2392
});
2393
this._lastViewportKey = key;
2394
this._lastViewport = {
@@ -2516,6 +2564,7 @@ const model = {
2564
this._connectSequence += 1;
2565
this._viewerToken = "";
2566
this.switchingBrowserId = null;
2567
+ this.viewerTransport = this.requestedViewerTransport();
2568
this._surfaceMounted = false;
2569
this._surfaceSwitching = false;
2570
this.commandInFlight = false;
tests/test_browser_agent_regressions.py
+30
-5
@@ -688,6 +688,7 @@ def test_browser_canvas_restarts_stream_after_page_navigation():
688
)
689
690
assert "async restartCanvasStreamAfterPageChange()" in js
691
+ assert "if (!this.usesScreencastTransport())" in js
692
assert '["navigate", "back", "forward", "reload"].includes(commandName)' in js
693
assert "await this.restartCanvasStreamAfterPageChange();" in js
694
assert "await this.waitForSurfaceViewport({ sequence: surfaceSequence });" in js
@@ -1089,7 +1090,7 @@ def test_browser_tabs_close_without_confirmation_or_busy_lock():
1090
assert "_commandInFlightCount" in browser_store
1091
1092
1092
-def test_browser_viewer_uses_cdp_screencast_transport():
1093
+def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1094
ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(
1095
encoding="utf-8"
1096
)
@@ -1107,12 +1108,20 @@ def test_browser_viewer_uses_cdp_screencast_transport():
1108
).read_text(encoding="utf-8")
1109
1110
assert 'runtime.call("screenshot"' in ws_browser
1111
+ assert 'VIEWER_TRANSPORT_SNAPSHOT = "snapshot"' in ws_browser
1112
+ assert 'VIEWER_TRANSPORT_SCREENCAST = "screencast"' in ws_browser
1113
+ assert "def _viewer_transport(data: dict[str, Any])" in ws_browser
1114
+ assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
1115
+ assert "self._stream_state" in ws_browser
1116
assert "SCREENCAST_QUALITY = 92" in ws_browser
1117
assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
1118
assert '"set_viewport"' in ws_browser
1119
assert "start_screencast" in ws_browser
1120
assert "pop_screencast_frame" in ws_browser
1121
assert "stop_screencast" in ws_browser
1122
+ assert "viewer_transport == VIEWER_TRANSPORT_SCREENCAST" in ws_browser
1123
+ assert '"viewer_transport": viewer_transport' in ws_browser
1124
+ assert '"viewer_transport": VIEWER_TRANSPORT_SNAPSHOT' in ws_browser
1125
assert '"Page.startScreencast"' in runtime
1126
assert '"Page.screencastFrame"' in runtime
1127
assert '"Page.screencastFrameAck"' in runtime
@@ -1123,9 +1132,21 @@ def test_browser_viewer_uses_cdp_screencast_transport():
1132
assert "await self._stop_screencasts_for_browser(resolved_id)" in runtime
1133
assert "queueFrameRender" in browser_store
1134
assert "requestAnimationFrame" in browser_store
1135
+ assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
1136
+ assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
1137
+ assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
1138
+ assert "liveScreencastEnabled: true" in browser_store
1139
+ assert "requestedViewerTransport()" in browser_store
1140
+ assert "normalizeViewerTransport(value = \"\")" in browser_store
1141
+ assert "usesScreencastTransport()" in browser_store
1142
+ assert "frameDimensionsFromMetadata(metadata = null)" in browser_store
1143
+ assert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store
1144
+ assert "dimensions: this.frameDimensionsFromMetadata(data.metadata)" in browser_store
1145
+ assert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_store
1146
+ assert "viewer_transport: this.requestedViewerTransport()" in browser_store
1147
assert "viewport_width: initialViewport?.width" in browser_store
1148
assert "viewport_height: initialViewport?.height" in browser_store
1128
- assert "restart_stream: restartStream" in browser_store
1149
+ assert "restart_stream: restartStream && this.usesScreencastTransport()" in browser_store
1150
assert 'restart_screencast=bool(data.get("restart_stream"))' in ws_browser
1151
assert "restart_screencast: bool = False" in runtime
1152
assert "should_remount_viewport = changed or restart_screencast" in runtime
@@ -1135,7 +1156,7 @@ def test_browser_viewer_uses_cdp_screencast_transport():
1156
assert "await self._remount_viewport(page, viewport)" in runtime
1157
assert "await asyncio.sleep(VIEWPORT_REMOUNT_PAUSE_SECONDS)" in runtime
1158
assert "def _nudged_viewport(viewport: dict[str, int])" in runtime
1138
- assert 'await this.syncViewport(true, { restartStream: this._mode === "canvas" });' in browser_store
1159
+ assert 'restartStream: this._mode === "canvas" && this.usesScreencastTransport()' in browser_store
1160
assert "this.frameState = data.state || null" not in browser_store
1161
assert "function loadFrameDimensions(src)" in browser_store
1162
assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store
@@ -1155,7 +1176,8 @@ def test_browser_viewer_uses_cdp_screencast_transport():
1176
assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handler
1177
assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
1178
assert '"frame_source": "state"' in ws_browser
1158
- assert '"frame_source"] = "screencast"' in ws_browser
1179
+ assert '"frame_source"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
1180
+ assert '"viewer_transport"] = VIEWER_TRANSPORT_SCREENCAST' in ws_browser
1181
assert "fallback_screenshot" not in ws_browser
1182
assert "canvas_wheel_screenshot" not in ws_browser
1183
assert "surface_mode: this._mode" not in browser_store
@@ -1279,7 +1301,7 @@ def test_browser_content_helper_keeps_label_wrapped_controls_referenceable():
1301
PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"
1302
).read_text(encoding="utf-8")
1303
1282
- assert 'const VERSION = "12"' in helper
1304
+ assert 'const VERSION = "13"' in helper
1305
assert "function patchOpenShadowDom" in helper
1306
assert "Element.prototype.attachShadow = patched" in helper
1307
assert "const REQUIRED_API_NAMES = Object.freeze([" in helper
@@ -1294,6 +1316,9 @@ def test_browser_content_helper_keeps_label_wrapped_controls_referenceable():
1316
assert "function isGlobalOrDelegatedEventBinding" in helper
1317
assert 'parts.includes("window")' in helper
1318
assert 'parts.includes("outside")' in helper
1319
+ assert 'actionStrategy: entry.helperBacked ? "frame_chain_ref" : "dom_ref"' in helper
1320
+ assert 'actionStrategy: "frame_chain_ref"' in helper
1321
+ assert "frameChain: Array.isArray(entry?.frameChain)" in helper
1322
1323
1324
def test_browser_panel_exposes_agent_friendly_address_input():