Smooth browser viewer rendering and frame push
Render live browser screencast frames through a canvas/ImageBitmap path while keeping the existing image/data-url path for snapshots and fallback rendering. Attach WebSocket streams as runtime screencast consumers so delivered frames are pushed to the server loop and acknowledged after emit, removing the per-frame read_screencast_frame runtime round trip.
Alessandro committed
Jul 3, 2026 at 11:06 UTC
69fb0fca2aa3405a5a8154d572e763c165ff9441
6 files changed
+296
-46
plugins/_browser/AGENTS.md
+3
@@ -19,6 +19,8 @@
19
- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
20
- Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
21
- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
22
+- Paint live screencast frames through the Browser panel canvas/ImageBitmap path when available; keep the `<img>`/data URL path for snapshots and fallback rendering.
23
+- Push internal screencast frames from the runtime to the WebSocket consumer after subscription; keep `read/pop_screencast_frame` as fallback/tooling APIs, not the WebUI hot path.
24
- Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values.
25
- Keep narrow WebUI Browser controls usable by grouping navigation with Annotate/settings above a full-width address bar.
26
- Browser URL-intent handling must only claim web URL schemes and leave custom Agent Zero schemes to their owning surfaces.
@@ -36,6 +38,7 @@
38
## Verification
39
40
- Smoke-test browser launch, navigation, DOM capture, and WebUI viewer after runtime changes.
41
+- For viewer render-path changes, verify the live Browser panel paints a screencast frame on canvas with `frameSrc` empty and snapshots still falling back to the image path.
42
- Run browser prompt/skill regression tests after changing browser prompt or Browser plugin skills.
43
44
## Child DOX Index
plugins/_browser/api/ws_browser.py
+31
-20
@@ -451,7 +451,37 @@ class WsBrowser(WsHandler):
451
last_state_refresh = 0.0
452
last_state_signature = self._state_signature(active_id, browsers)
453
frame_sequence = 0
454
+ server_loop = asyncio.get_running_loop()
455
+ stop_event = asyncio.Event()
456
+
457
+ async def emit_frame(frame: dict[str, Any]) -> None:
458
+ nonlocal frame_sequence
459
+ try:
460
+ frame_sequence += 1
461
+ payload = self._frame_payload(
462
+ frame,
463
+ context_id=context_id,
464
+ viewer_id=viewer_id,
465
+ browser_id=active_id,
466
+ sequence=frame_sequence,
467
+ binary_frames=binary_frames,
468
+ )
469
+ if not slim_frames:
470
+ payload["browsers"] = browsers
471
+ payload["state"] = state
472
+ await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
473
+ except BaseException:
474
+ stop_event.set()
475
+ raise
476
+
477
+ def frame_consumer(frame: dict[str, Any]):
478
+ return asyncio.run_coroutine_threadsafe(emit_frame(frame), server_loop)
479
+
480
+ await runtime.call("attach_screencast_consumer", stream_id, frame_consumer)
481
+
482
while True:
483
+ if stop_event.is_set():
484
+ break
485
now = time.monotonic()
486
if now - last_state_refresh >= FRAME_STATE_REFRESH_SECONDS:
487
listing = await runtime.call("list")
@@ -475,29 +505,10 @@ class WsBrowser(WsHandler):
505
last_state_refresh = now
506
507
try:
478
- frame = await runtime.call(
479
- "read_screencast_frame",
480
- stream_id,
481
- timeout=FRAME_READ_TIMEOUT_SECONDS,
482
- )
483
- except KeyError:
508
+ await asyncio.wait_for(stop_event.wait(), timeout=FRAME_READ_TIMEOUT_SECONDS)
509
break
510
except TimeoutError:
511
continue
487
-
488
- frame_sequence += 1
489
- payload = self._frame_payload(
490
- frame,
491
- context_id=context_id,
492
- viewer_id=viewer_id,
493
- browser_id=active_id,
494
- sequence=frame_sequence,
495
- binary_frames=binary_frames,
496
- )
497
- if not slim_frames:
498
- payload["browsers"] = browsers
499
- payload["state"] = state
500
- await self._emit_to_connected_viewer(sid, "browser_viewer_frame", payload)
512
except asyncio.CancelledError:
513
raise
514
except Exception:
plugins/_browser/helpers/runtime.py
+32
-1
@@ -317,6 +317,7 @@ class _BrowserScreencast:
317
self.browser_id = browser_id
318
self.session = session
319
self.mime = mime
320
+ self.frame_consumer: Any | None = None
321
self.queue = asyncio.Queue(maxsize=1)
322
self.stopped = False
323
self._ack_tasks: set[asyncio.Task] = set()
@@ -398,6 +399,12 @@ class _BrowserScreencast:
399
raise RuntimeError("Browser screencast stopped.")
400
return frame
401
402
+ async def attach_consumer(self, frame_consumer: Any) -> None:
403
+ self.frame_consumer = frame_consumer
404
+ frame = await self.pop_frame()
405
+ if frame:
406
+ await self._deliver_frame(frame)
407
+
408
async def stop(self) -> None:
409
if self.stopped:
410
return
@@ -423,6 +430,7 @@ class _BrowserScreencast:
430
task.add_done_callback(self._ack_tasks.discard)
431
432
async def _handle_frame(self, params: dict[str, Any]) -> None:
433
+ stop_after_ack = False
434
try:
435
data = params.get("data") or ""
436
if data:
@@ -432,7 +440,7 @@ class _BrowserScreencast:
440
metadata["jpegWidth"], metadata["jpegHeight"] = size
441
metadata["expectedWidth"] = self._expected_width
442
metadata["expectedHeight"] = self._expected_height
435
- self._queue_latest(
443
+ await self._deliver_frame(
444
{
445
"browser_id": self.browser_id,
446
"mime": self.mime,
@@ -440,6 +448,13 @@ class _BrowserScreencast:
448
"metadata": metadata,
449
}
450
)
451
+ except asyncio.CancelledError:
452
+ stop_after_ack = True
453
+ except Exception:
454
+ if self.frame_consumer:
455
+ stop_after_ack = True
456
+ else:
457
+ raise
458
finally:
459
session_id = params.get("sessionId")
460
if session_id is not None and not self.stopped:
@@ -448,6 +463,16 @@ class _BrowserScreencast:
463
"Page.screencastFrameAck",
464
{"sessionId": int(session_id)},
465
)
466
+ if stop_after_ack:
467
+ self.stopped = True
468
+
469
+ async def _deliver_frame(self, frame: dict[str, Any]) -> None:
470
+ if not self.frame_consumer:
471
+ self._queue_latest(frame)
472
+ return
473
+ future = self.frame_consumer(frame)
474
+ if future is not None:
475
+ await asyncio.wrap_future(future)
476
477
def _queue_latest(self, frame: dict[str, Any]) -> None:
478
self._drop_queued_frames()
@@ -1669,6 +1694,12 @@ class _BrowserRuntimeCore:
1694
raise KeyError("Browser screencast is not active.")
1695
return await screencast.pop_frame()
1696
1697
+ async def attach_screencast_consumer(self, stream_id: str, frame_consumer: Any) -> None:
1698
+ screencast = self.screencasts.get(str(stream_id or ""))
1699
+ if not screencast:
1700
+ raise KeyError("Browser screencast is not active.")
1701
+ await screencast.attach_consumer(frame_consumer)
1702
+
1703
async def stop_screencast(self, stream_id: str) -> None:
1704
screencast = self.screencasts.pop(str(stream_id or ""), None)
1705
if screencast:
plugins/_browser/webui/browser-panel.html
+8
-3
@@ -187,12 +187,17 @@
187
<div class="browser-stage" tabindex="0" @click="$el.focus()"
188
:class="{ 'is-annotating': $store.browserPage.annotating }"
189
@wheel.prevent="$store.browserPage.handleStageWheel($event)">
190
+ <canvas class="browser-frame browser-frame-canvas"
191
+ x-show="$store.browserPage.frameCanvasReady"
192
+ x-init="$store.browserPage.attachFrameCanvas($el)"
193
+ @click="$store.browserPage.sendMouse('click', $event)"
194
+ @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)"></canvas>
195
<template x-if="$store.browserPage.frameSrc">
191
- <img class="browser-frame" :src="$store.browserPage.frameSrc"
196
+ <img class="browser-frame browser-frame-image" :src="$store.browserPage.frameSrc"
197
@click="$store.browserPage.sendMouse('click', $event)"
198
@mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)" draggable="false" />
199
</template>
195
- <template x-if="$store.browserPage.annotating && $store.browserPage.frameSrc">
200
+ <template x-if="$store.browserPage.annotating && $store.browserPage.hasFrame()">
201
<div class="browser-annotation-layer"
202
:class="{ 'is-busy': $store.browserPage.annotationBusy }"
203
@pointerdown.stop.prevent="$store.browserPage.startAnnotationSelection($event)"
@@ -267,7 +272,7 @@
272
@click="$store.browserPage.sendAnnotationsToChat()">Send now</button>
273
</div>
274
</div>
270
- <template x-if="!$store.browserPage.frameSrc && !$store.browserPage.isBusy()">
275
+ <template x-if="!$store.browserPage.hasFrame() && !$store.browserPage.isBusy()">
276
<div class="browser-empty">
277
<span class="material-symbols-outlined">captive_portal</span>
278
<button class="btn btn-field" @click="$store.browserPage.command('open')">Open
plugins/_browser/webui/browser-store.js
+104
-15
@@ -37,6 +37,7 @@ const LOCAL_EDITABLE_SELECTOR = "input, textarea, select, [contenteditable]";
37
const BROWSER_BINARY_FRAME_REQUESTS_ENABLED = false;
38
const BROWSER_BINARY_PAYLOADS_SUPPORTED = typeof Blob === "function"
39
&& typeof globalThis.URL?.createObjectURL === "function";
40
+const BROWSER_CANVAS_FRAMES_SUPPORTED = typeof globalThis.createImageBitmap === "function";
41
42
function makeViewerToken() {
43
return globalThis.crypto?.randomUUID?.()
@@ -130,6 +131,7 @@ function frameImageSource(data = {}) {
131
const src = globalThis.URL.createObjectURL(blob);
132
return {
133
src,
134
+ blob,
135
objectUrl: src,
136
cleanup: () => globalThis.URL.revokeObjectURL(src),
137
};
@@ -143,6 +145,16 @@ function frameImageSource(data = {}) {
145
};
146
}
147
148
+async function loadFrameBitmap(src, options = {}) {
149
+ if (!BROWSER_CANVAS_FRAMES_SUPPORTED || !src) return null;
150
+ try {
151
+ const blob = options.blob || await fetch(src).then((response) => response.blob());
152
+ return await globalThis.createImageBitmap(blob);
153
+ } catch {
154
+ return null;
155
+ }
156
+}
157
+
158
const model = {
159
loading: true,
160
error: "",
@@ -153,6 +165,7 @@ const model = {
165
activeBrowserContextId: "",
166
address: "",
167
frameSrc: "",
168
+ frameCanvasReady: false,
169
frameState: null,
170
viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST,
171
liveScreencastEnabled: true,
@@ -179,6 +192,7 @@ const model = {
192
_frameRenderHandle: null,
193
_frameRenderCancel: null,
194
_frameRenderSequence: 0,
195
+ _frameCanvas: null,
196
_floatingCleanup: null,
197
_stageElement: null,
198
_stageResizeObserver: null,
@@ -473,6 +487,7 @@ const model = {
487
this.address = "";
488
this.frameState = null;
489
this.clearFrameSrc();
490
+ this.clearFrameCanvas();
491
if (this.contextId) {
492
await this.connectViewer();
493
}
@@ -784,11 +799,13 @@ const model = {
799
},
800
801
releaseSurfaceBindings() {
802
+ this.freezeCanvasFrameToImage();
803
this._floatingCleanup?.();
804
this._floatingCleanup = null;
805
this._stageResizeObserver?.disconnect?.();
806
this._stageResizeObserver = null;
807
this._stageElement = null;
808
+ this._frameCanvas = null;
809
},
810
811
isCanvasSurfaceVisible(element = null) {
@@ -843,7 +860,7 @@ const model = {
860
this._surfaceMounted = true;
861
this._surfaceOpenedAt = Date.now();
862
this._lastViewportKey = "";
846
- if (this.frameSrc && !targetChanged) {
863
+ if (this.hasFrame() && !targetChanged) {
864
this._surfaceSwitching = false;
865
this.switchingBrowserId = null;
866
return;
@@ -864,12 +881,13 @@ const model = {
881
resetRenderedFrame() {
882
this.cancelFrameRender();
883
this.clearFrameSrc();
884
+ this.clearFrameCanvas();
885
this._lastFrameDimensions = null;
886
this._lastFrameAt = 0;
887
},
888
889
resetRenderedFrameIfViewportChanged(viewport = null, requestedBrowserId = null, requestedContextId = "") {
872
- if (!viewport || !this.frameSrc || !this._lastViewport) return;
890
+ if (!viewport || !this.hasFrame() || !this._lastViewport) return;
891
const targetBrowserId = requestedBrowserId || this.activeBrowserId || this.firstBrowserId();
892
const targetContextId = this.normalizeContextId(requestedContextId || this.contextIdForBrowserId(targetBrowserId) || this.activeBrowserContextId);
893
if (!this.sameBrowserTab(this._lastViewport.browserId, this._lastViewport.contextId, targetBrowserId, targetContextId)) return;
@@ -1107,7 +1125,9 @@ const model = {
1125
browserId: frameBrowserId,
1126
contextId: incomingContextId,
1127
dimensions: this.frameDimensionsFromData(data),
1128
+ blob: frameImage.blob,
1129
objectUrl: frameImage.objectUrl,
1130
+ useCanvas: true,
1131
cleanup: frameImage.cleanup,
1132
onAccepted: () => {
1133
if (
@@ -1122,12 +1142,14 @@ const model = {
1142
} else if (!data.state) {
1143
this.cancelFrameRender();
1144
this.clearFrameSrc();
1145
+ this.clearFrameCanvas();
1146
}
1147
if (!data.image && !data.state) {
1148
if (!this.activeBrowserId) {
1149
this.setActiveBrowserId(null, "");
1150
this.frameState = null;
1151
this.clearFrameSrc();
1152
+ this.clearFrameCanvas();
1153
}
1154
}
1155
this._lastFrameAt = Date.now();
@@ -1229,11 +1251,21 @@ const model = {
1251
if (!frameSrc) {
1252
if (sequence === this._frameRenderSequence) {
1253
this.clearFrameSrc();
1254
+ this.clearFrameCanvas();
1255
}
1256
return;
1257
}
1235
- const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc);
1258
+ let bitmap = null;
1259
+ let dimensions = options?.dimensions || null;
1260
+ if (options?.useCanvas && this.canUseCanvasFrames()) {
1261
+ bitmap = await loadFrameBitmap(frameSrc, options);
1262
+ if (bitmap) {
1263
+ dimensions ||= { width: bitmap.width || 0, height: bitmap.height || 0 };
1264
+ }
1265
+ }
1266
+ dimensions ||= await loadFrameDimensions(frameSrc);
1267
if (sequence !== this._frameRenderSequence || surfaceSequence !== this._surfaceOpenSequence) {
1268
+ bitmap?.close?.();
1269
options?.cleanup?.();
1270
return;
1271
}
@@ -1241,13 +1273,21 @@ const model = {
1273
if (!this.frameMatchesViewport(dimensions, viewport)) {
1274
this.requestViewportSyncAfterRejectedFrame();
1275
if (!this.shouldAcceptMismatchedFrame(dimensions)) {
1276
+ bitmap?.close?.();
1277
options?.cleanup?.();
1278
return;
1279
}
1280
}
1248
- this.releaseRenderedFrameUrl(frameSrc);
1249
- this.frameSrc = frameSrc;
1250
- this._frameObjectUrl = options?.objectUrl || "";
1281
+ if (bitmap && this.paintFrameBitmap(bitmap)) {
1282
+ this.clearFrameSrc();
1283
+ options?.cleanup?.();
1284
+ } else {
1285
+ this.clearFrameCanvas();
1286
+ this.releaseRenderedFrameUrl(frameSrc);
1287
+ this.frameSrc = frameSrc;
1288
+ this._frameObjectUrl = options?.objectUrl || "";
1289
+ }
1290
+ bitmap?.close?.();
1291
this._lastFrameDimensions = dimensions;
1292
this._lastFrameAt = Date.now();
1293
options?.onAccepted?.();
@@ -1259,7 +1299,7 @@ const model = {
1299
return Boolean(
1300
dimensions?.width
1301
&& dimensions?.height
1262
- && (!this.frameSrc || this._surfaceSwitching || this.isSwitchingBrowser())
1302
+ && (!this.hasFrame() || this._surfaceSwitching || this.isSwitchingBrowser())
1303
);
1304
},
1305
@@ -1330,7 +1370,7 @@ const model = {
1370
1371
clearRenderedFrameIfViewportChanged() {
1372
const viewport = this.currentViewportSize();
1333
- if (!this.frameSrc || !this._lastFrameDimensions || !viewport) return;
1373
+ if (!this.hasFrame() || !this._lastFrameDimensions || !viewport) return;
1374
if (this.frameMatchesViewport(this._lastFrameDimensions, viewport)) return;
1375
this.cancelFrameRender();
1376
this.resetViewportTracking();
@@ -1368,6 +1408,55 @@ const model = {
1408
this.frameSrc = "";
1409
},
1410
1411
+ attachFrameCanvas(canvas = null) {
1412
+ this._frameCanvas = canvas || null;
1413
+ },
1414
+
1415
+ canUseCanvasFrames() {
1416
+ return Boolean(BROWSER_CANVAS_FRAMES_SUPPORTED && this._frameCanvas?.getContext);
1417
+ },
1418
+
1419
+ hasFrame() {
1420
+ return Boolean(this.frameSrc || this.frameCanvasReady);
1421
+ },
1422
+
1423
+ paintFrameBitmap(bitmap) {
1424
+ const canvas = this._frameCanvas;
1425
+ if (!canvas || !bitmap?.width || !bitmap?.height) return false;
1426
+ canvas.width = bitmap.width;
1427
+ canvas.height = bitmap.height;
1428
+ const context = canvas.getContext("2d");
1429
+ if (!context) return false;
1430
+ context.drawImage(bitmap, 0, 0);
1431
+ this.frameCanvasReady = true;
1432
+ return true;
1433
+ },
1434
+
1435
+ clearFrameCanvas() {
1436
+ const canvas = this._frameCanvas;
1437
+ if (canvas?.width && canvas?.height) {
1438
+ canvas.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height);
1439
+ }
1440
+ this.frameCanvasReady = false;
1441
+ },
1442
+
1443
+ freezeCanvasFrameToImage() {
1444
+ if (!this.frameCanvasReady || !this._frameCanvas) return;
1445
+ try {
1446
+ this.frameSrc = this._frameCanvas.toDataURL("image/jpeg", 0.86);
1447
+ } catch {
1448
+ this.frameSrc = "";
1449
+ }
1450
+ this.clearFrameCanvas();
1451
+ },
1452
+
1453
+ frameElement() {
1454
+ if (this.frameCanvasReady && this._frameCanvas) {
1455
+ return this._frameCanvas;
1456
+ }
1457
+ return this._stageElement?.querySelector?.(".browser-frame-image") || null;
1458
+ },
1459
+
1460
beginCommand() {
1461
this._commandInFlightCount += 1;
1462
this.commandInFlight = true;
@@ -1431,6 +1520,7 @@ const model = {
1520
if (!this.activeBrowserId) {
1521
this.frameState = null;
1522
this.clearFrameSrc();
1523
+ this.clearFrameCanvas();
1524
}
1525
if (result.state?.currentUrl || result.currentUrl) {
1526
this.address = result.state?.currentUrl || result.currentUrl;
@@ -1540,6 +1630,7 @@ const model = {
1630
this.switchingBrowserId = targetId;
1631
this.cancelFrameRender();
1632
this.clearFrameSrc();
1633
+ this.clearFrameCanvas();
1634
this.frameState = browser || null;
1635
if (!this.addressFocused && browser?.currentUrl) {
1636
this.address = browser.currentUrl;
@@ -1811,8 +1902,8 @@ const model = {
1902
const target = element || event?.currentTarget;
1903
if (!target) return null;
1904
const rect = target.getBoundingClientRect();
1814
- const naturalWidth = target.naturalWidth || rect.width;
1815
- const naturalHeight = target.naturalHeight || rect.height;
1905
+ const naturalWidth = target.naturalWidth || target.width || rect.width;
1906
+ const naturalHeight = target.naturalHeight || target.height || rect.height;
1907
let contentLeft = rect.left;
1908
let contentTop = rect.top;
1909
let contentWidth = rect.width;
@@ -1963,7 +2054,7 @@ const model = {
2054
},
2055
2056
canAnnotate() {
1966
- return Boolean(this.activeBrowserId && this.frameSrc && !this.isBusy());
2057
+ return Boolean(this.activeBrowserId && this.hasFrame() && !this.isBusy());
2058
},
2059
2060
activeAnnotationUrl() {
@@ -2124,8 +2215,7 @@ const model = {
2215
},
2216
2217
stagePointForEvent(event) {
2127
- const image = this._stageElement?.querySelector?.(".browser-frame") || null;
2128
- return this.pointerCoordinatesFor(event, image);
2218
+ return this.pointerCoordinatesFor(event, this.frameElement());
2219
},
2220
2221
normalizeAnnotationRect(start = {}, end = {}) {
@@ -2538,8 +2628,7 @@ const model = {
2628
async sendWheel(event) {
2629
const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
2630
if (!contextId || !this.activeBrowserId || !event) return;
2541
- const image = event.currentTarget?.querySelector?.(".browser-frame") || event.target?.closest?.(".browser-frame");
2542
- const pointer = this.pointerCoordinatesFor(event, image);
2631
+ const pointer = this.pointerCoordinatesFor(event, this.frameElement());
2632
if (!pointer) return;
2633
const payload = {
2634
context_id: contextId,
tests/test_browser_agent_regressions.py
+118
-7
@@ -1,4 +1,5 @@
1
import asyncio
2
+import concurrent.futures
3
import json
4
import re
5
import sys
@@ -609,7 +610,7 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():
610
assert "isCanvasSurfaceVisible(element)" in js
611
assert "scheduleViewportSyncForSurface" in js
612
assert "const targetChanged = Boolean(" in js
612
- assert "if (this.frameSrc && !targetChanged)" in js
613
+ assert "if (this.hasFrame() && !targetChanged)" in js
614
assert "this.cancelFrameRender();" in js
615
assert "this.resetRenderedFrame();" in js
616
@@ -626,7 +627,7 @@ def test_browser_surface_handoffs_keep_existing_frame_until_replacement_arrives(
627
clear_block = js[clear_start: js.index("beginCommand()", clear_start)]
628
629
assert "modeChanged" not in prepare_block
629
- assert "if (this.frameSrc && !targetChanged)" in prepare_block
630
+ assert "if (this.hasFrame() && !targetChanged)" in prepare_block
631
assert "this.resetRenderedFrame();" in prepare_block
632
assert "this.cancelFrameRender();" in viewport_block
633
assert "this.resetRenderedFrame();" not in viewport_block
@@ -674,12 +675,12 @@ def test_browser_canvas_nudges_width_after_first_accepted_frame():
675
assert 'canvas.activeSurfaceId !== "browser"' in js
676
assert "canvas.setWidth?.(nudgedWidth, { persist: false })" in js
677
assert "this.queueViewportSync(true)" in js
677
- frame_assignment_index = js.index("this.frameSrc = frameSrc;")
678
+ frame_accept_index = js.index("this._lastFrameDimensions = dimensions;")
679
frame_nudge_schedule_index = js.index(
680
"this.scheduleCanvasWidthNudgeAfterFirstFrame();",
680
- frame_assignment_index,
681
+ frame_accept_index,
682
)
682
- assert frame_assignment_index < frame_nudge_schedule_index
683
+ assert frame_accept_index < frame_nudge_schedule_index
684
685
686
def test_browser_canvas_restarts_stream_after_page_navigation():
@@ -1163,7 +1164,10 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1164
assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
1165
assert '"set_viewport"' in ws_browser
1166
assert "start_screencast" in ws_browser
1166
- assert "read_screencast_frame" in ws_browser
1167
+ assert '"attach_screencast_consumer"' in ws_browser
1168
+ assert "asyncio.run_coroutine_threadsafe" in ws_browser
1169
+ assert 'runtime.call(\n "read_screencast_frame"' not in ws_browser
1170
+ assert "read_screencast_frame" in runtime
1171
assert "FRAME_READ_TIMEOUT_SECONDS" in ws_browser
1172
assert "FRAME_IDLE_POLL_SECONDS" not in ws_browser
1173
assert "except TimeoutError:" in ws_browser
@@ -1174,6 +1178,10 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1178
assert '"Page.startScreencast"' in runtime
1179
assert '"Page.screencastFrame"' in runtime
1180
assert '"Page.screencastFrameAck"' in runtime
1181
+ assert "async def attach_screencast_consumer" in runtime
1182
+ assert "async def attach_consumer" in runtime
1183
+ assert "await self._deliver_frame(" in runtime
1184
+ assert "await asyncio.wrap_future(future)" in runtime
1185
assert '"Page.stopScreencast"' in runtime
1186
assert '"Emulation.setDeviceMetricsOverride"' in runtime
1187
assert '"Emulation.setVisibleSize"' in runtime
@@ -1184,7 +1192,10 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1192
assert "function frameImageSource(data = {})" in browser_store
1193
assert "const BROWSER_BINARY_FRAME_REQUESTS_ENABLED = false;" in browser_store
1194
assert "const BROWSER_BINARY_PAYLOADS_SUPPORTED = typeof Blob" in browser_store
1195
+ assert "const BROWSER_CANVAS_FRAMES_SUPPORTED = typeof globalThis.createImageBitmap" in browser_store
1196
assert "if (data.encoding === \"binary\") return null;" in browser_store
1197
+ assert "async function loadFrameBitmap(src, options = {})" in browser_store
1198
+ assert "return await globalThis.createImageBitmap(blob);" in browser_store
1199
assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
1200
assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
1201
assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
@@ -1198,7 +1209,16 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1209
assert "frameDimensionsFromMetadata(metadata = null)" in browser_store
1210
assert "metadata.expectedWidth || metadata.deviceWidth || metadata.jpegWidth" in browser_store
1211
assert "dimensions: this.frameDimensionsFromData(data)" in browser_store
1201
- assert "const dimensions = options?.dimensions || await loadFrameDimensions(frameSrc)" in browser_store
1212
+ assert "useCanvas: true" in browser_store
1213
+ assert "let dimensions = options?.dimensions || null" in browser_store
1214
+ assert "dimensions ||= await loadFrameDimensions(frameSrc)" in browser_store
1215
+ assert "paintFrameBitmap(bitmap)" in browser_store
1216
+ assert "freezeCanvasFrameToImage()" in browser_store
1217
+ assert 'this._frameCanvas.toDataURL("image/jpeg", 0.86)' in browser_store
1218
+ assert "this.frameSrc = frameSrc;" in browser_store
1219
+ assert "hasFrame()" in browser_store
1220
+ assert "frameCanvasReady: false" in browser_store
1221
+ assert "attachFrameCanvas(canvas = null)" in browser_store
1222
assert "viewer_transport: this.requestedViewerTransport()" in browser_store
1223
assert "binary_frames: this.supportsBinaryFrames()" in browser_store
1224
assert "slim_frames: true" in browser_store
@@ -1247,6 +1267,10 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1267
assert "fallback_screenshot" not in ws_browser
1268
assert "canvas_wheel_screenshot" not in ws_browser
1269
assert "surface_mode: this._mode" not in browser_store
1270
+ assert '<canvas class="browser-frame browser-frame-canvas"' in main_html
1271
+ assert 'x-init="$store.browserPage.attachFrameCanvas($el)"' in main_html
1272
+ assert '<img class="browser-frame browser-frame-image"' in main_html
1273
+ assert "$store.browserPage.hasFrame()" in main_html
1274
assert "overflow: hidden;" in main_html
1275
assert "object-fit: contain;" in main_html
1276
assert "image-rendering: auto;" in main_html
@@ -1628,6 +1652,93 @@ async def test_browser_screencast_acknowledges_and_drops_stale_frames():
1652
assert session.detached is True
1653
1654
1655
+@pytest.mark.anyio
1656
+async def test_browser_screencast_acks_after_consumer_settles():
1657
+ class FakeSession:
1658
+ def __init__(self):
1659
+ self.handlers = {}
1660
+ self.sent = []
1661
+
1662
+ def on(self, event, handler):
1663
+ self.handlers[event] = handler
1664
+
1665
+ async def send(self, method, params=None):
1666
+ self.sent.append((method, params or {}))
1667
+
1668
+ async def detach(self):
1669
+ pass
1670
+
1671
+ session = FakeSession()
1672
+ delivered = []
1673
+ delivery = concurrent.futures.Future()
1674
+ screencast = _BrowserScreencast(
1675
+ stream_id="stream",
1676
+ browser_id=7,
1677
+ session=session,
1678
+ mime="image/jpeg",
1679
+ )
1680
+ screencast.frame_consumer = lambda frame: delivered.append(frame) or delivery
1681
+
1682
+ await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 640, "height": 480})
1683
+ session.handlers["Page.screencastFrame"](
1684
+ {"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": 21}
1685
+ )
1686
+ await asyncio.sleep(0)
1687
+
1688
+ assert delivered
1689
+ assert ("Page.screencastFrameAck", {"sessionId": 21}) not in session.sent
1690
+
1691
+ delivery.set_result(None)
1692
+ await asyncio.sleep(0)
1693
+ await asyncio.sleep(0)
1694
+
1695
+ assert ("Page.screencastFrameAck", {"sessionId": 21}) in session.sent
1696
+
1697
+ await screencast.stop()
1698
+
1699
+
1700
+@pytest.mark.anyio
1701
+async def test_browser_screencast_attach_consumer_flushes_queued_frame():
1702
+ class FakeSession:
1703
+ def __init__(self):
1704
+ self.handlers = {}
1705
+ self.sent = []
1706
+
1707
+ def on(self, event, handler):
1708
+ self.handlers[event] = handler
1709
+
1710
+ async def send(self, method, params=None):
1711
+ self.sent.append((method, params or {}))
1712
+
1713
+ async def detach(self):
1714
+ pass
1715
+
1716
+ session = FakeSession()
1717
+ delivered = []
1718
+ delivery = concurrent.futures.Future()
1719
+ delivery.set_result(None)
1720
+ screencast = _BrowserScreencast(
1721
+ stream_id="stream",
1722
+ browser_id=7,
1723
+ session=session,
1724
+ mime="image/jpeg",
1725
+ )
1726
+
1727
+ await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 640, "height": 480})
1728
+ session.handlers["Page.screencastFrame"](
1729
+ {"data": SMALL_JPEG_10X10, "metadata": {}, "sessionId": 31}
1730
+ )
1731
+ await asyncio.sleep(0)
1732
+
1733
+ await screencast.attach_consumer(lambda frame: delivered.append(frame) or delivery)
1734
+
1735
+ assert delivered
1736
+ assert delivered[0]["image"] == SMALL_JPEG_10X10
1737
+ assert await screencast.pop_frame() is None
1738
+
1739
+ await screencast.stop()
1740
+
1741
+
1742
@pytest.mark.anyio
1743
async def test_browser_screencast_passes_wrong_viewport_frames_to_frontend_validator():
1744
class FakeSession: