Fix Browser viewer input after stream restarts
Notify screencast consumers when a stream is stopped so the WebSocket frame supervisor restarts after viewport-driven screencast resets. Keep the active canvas as the frame/input surface and avoid replacing live screencasts with click snapshots, preserving hover, click, and wheel feedback. Tested: conda run -n a0 pytest tests/test_browser_agent_regressions.py -q
Alessandro committed
Jul 4, 2026 at 16:21 UTC
9bfa3b5c81d7ee87fcaba983a7d106f734dfe194
3 files changed
+60
-9
plugins/_browser/helpers/runtime.py
+3
@@ -411,8 +411,11 @@ class _BrowserScreencast:
411
async def stop(self) -> None:
412
if self._closed:
413
return
414
+ was_stopped = self.stopped
415
self._closed = True
416
self.stopped = True
417
+ if not was_stopped:
418
+ self._notify_stopped()
419
self._drop_queued_frames()
420
with contextlib.suppress(asyncio.QueueFull):
421
self.queue.put_nowait(None)
plugins/_browser/webui/browser-store.js
+19
-8
@@ -1412,8 +1412,15 @@ const model = {
1412
this._frameCanvas = canvas || null;
1413
},
1414
1415
+ currentFrameCanvas() {
1416
+ const stageCanvas = this._stageElement?.querySelector?.(".browser-frame-canvas");
1417
+ if (stageCanvas?.isConnected) return stageCanvas;
1418
+ if (this._frameCanvas?.isConnected) return this._frameCanvas;
1419
+ return null;
1420
+ },
1421
+
1422
canUseCanvasFrames() {
1416
- return Boolean(BROWSER_CANVAS_FRAMES_SUPPORTED && this._frameCanvas?.getContext);
1423
+ return Boolean(BROWSER_CANVAS_FRAMES_SUPPORTED && this.currentFrameCanvas()?.getContext);
1424
},
1425
1426
hasFrame() {
@@ -1421,7 +1428,7 @@ const model = {
1428
},
1429
1430
paintFrameBitmap(bitmap) {
1424
- const canvas = this._frameCanvas;
1431
+ const canvas = this.currentFrameCanvas();
1432
if (!canvas || !bitmap?.width || !bitmap?.height) return false;
1433
if (canvas.width !== bitmap.width) canvas.width = bitmap.width;
1434
if (canvas.height !== bitmap.height) canvas.height = bitmap.height;
@@ -1433,7 +1440,7 @@ const model = {
1440
},
1441
1442
clearFrameCanvas() {
1436
- const canvas = this._frameCanvas;
1443
+ const canvas = this.currentFrameCanvas();
1444
if (canvas?.width && canvas?.height) {
1445
canvas.getContext("2d")?.clearRect(0, 0, canvas.width, canvas.height);
1446
}
@@ -1441,9 +1448,10 @@ const model = {
1448
},
1449
1450
freezeCanvasFrameToImage() {
1444
- if (!this.frameCanvasReady || !this._frameCanvas) return;
1451
+ const canvas = this.currentFrameCanvas();
1452
+ if (!this.frameCanvasReady || !canvas) return;
1453
try {
1446
- this.frameSrc = this._frameCanvas.toDataURL("image/jpeg", 0.86);
1454
+ this.frameSrc = canvas.toDataURL("image/jpeg", 0.86);
1455
} catch {
1456
this.frameSrc = "";
1457
}
@@ -1451,8 +1459,9 @@ const model = {
1459
},
1460
1461
frameElement() {
1454
- if (this.frameCanvasReady && this._frameCanvas) {
1455
- return this._frameCanvas;
1462
+ if (this.frameCanvasReady) {
1463
+ const canvas = this.currentFrameCanvas();
1464
+ if (canvas) return canvas;
1465
}
1466
return this._stageElement?.querySelector?.(".browser-frame-image") || null;
1467
},
@@ -2616,7 +2625,9 @@ const model = {
2625
const response = await websocket.request("browser_viewer_input", payload, { timeoutMs: 10000 });
2626
const data = firstOk(response);
2627
this.applyActiveFrameState(data.state);
2619
- this.applySnapshot(data.snapshot);
2628
+ if (!this.frameCanvasReady || !this.usesScreencastTransport()) {
2629
+ this.applySnapshot(data.snapshot);
2630
+ }
2631
} catch (error) {
2632
this.error = error instanceof Error ? error.message : String(error);
2633
}
tests/test_browser_agent_regressions.py
+38
-1
@@ -1218,7 +1218,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1218
assert "paintFrameBitmap(bitmap)" in browser_store
1219
assert "if (canvas.width !== bitmap.width) canvas.width = bitmap.width;" in browser_store
1220
assert "freezeCanvasFrameToImage()" in browser_store
1221
- assert 'this._frameCanvas.toDataURL("image/jpeg", 0.86)' in browser_store
1221
+ assert "currentFrameCanvas()" in browser_store
1222
+ assert 'this._stageElement?.querySelector?.(".browser-frame-canvas")' in browser_store
1223
+ assert 'canvas.toDataURL("image/jpeg", 0.86)' in browser_store
1224
assert "this.frameSrc = frameSrc;" in browser_store
1225
assert "hasFrame()" in browser_store
1226
assert "frameCanvasReady: false" in browser_store
@@ -1246,6 +1248,7 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1248
assert "shouldAcceptMismatchedFrame(dimensions = null)" in browser_store
1249
assert "requestViewportSyncAfterRejectedFrame()" in browser_store
1250
assert "this.applySnapshot(data.snapshot);" in browser_store
1251
+ assert "if (!this.frameCanvasReady || !this.usesScreencastTransport())" in browser_store
1252
assert "else if (!data.state)" in browser_store
1253
assert '"snapshot": snapshot' in ws_browser
1254
assert '"binary_frames": binary_frames' in ws_browser
@@ -1755,6 +1758,40 @@ async def test_browser_screencast_notifies_consumer_when_frame_task_stops_before
1758
assert session.detached is True
1759
1760
1761
+@pytest.mark.anyio
1762
+async def test_browser_screencast_stop_notifies_consumer_once():
1763
+ class FakeSession:
1764
+ def __init__(self):
1765
+ self.handlers = {}
1766
+ self.sent = []
1767
+
1768
+ def on(self, event, handler):
1769
+ self.handlers[event] = handler
1770
+
1771
+ async def send(self, method, params=None):
1772
+ self.sent.append((method, params or {}))
1773
+
1774
+ async def detach(self):
1775
+ pass
1776
+
1777
+ session = FakeSession()
1778
+ stopped = []
1779
+ screencast = _BrowserScreencast(
1780
+ stream_id="stream",
1781
+ browser_id=7,
1782
+ session=session,
1783
+ mime="image/jpeg",
1784
+ )
1785
+
1786
+ await screencast.start(quality=92, every_nth_frame=1, viewport={"width": 640, "height": 480})
1787
+ await screencast.attach_consumer(lambda frame: None, lambda: stopped.append(True))
1788
+ await screencast.stop()
1789
+ await screencast.stop()
1790
+
1791
+ assert stopped == [True]
1792
+ assert ("Page.stopScreencast", {}) in session.sent
1793
+
1794
+
1795
@pytest.mark.anyio
1796
async def test_browser_screencast_attach_consumer_flushes_queued_frame():
1797
class FakeSession: