@samitouri / QOSami_Agents_Zero-1 / commits / 005b366b

Add interactive internal Browser viewport

Render the existing Patchright page through an isolated, authenticated Xpra session with CDP screencast and snapshot fallbacks. Keep Chromium out of fullscreen, synchronize native viewport resizing across canvas and modal handoffs, and remove Xpra decoration and shadow-cursor artifacts. Move loading feedback into each Browser tab while reserving the footer for actual errors.

Alessandro committed Aug 16, 2026 at 12:58 UTC 005b366b5147f35c5fe66c44c864f74e1ec4633d
12 files changed +1163 -211
helpers/virtual_desktop.py
+33 -22
@@ -115,29 +115,38 @@ def get_registry() -> VirtualDesktopRegistry:
115 return _registry
116
117
118 -def session_url(token: str, *, title: str = "Desktop") -> str:
118 +def session_url(
119 + token: str,
120 + *,
121 + title: str = "Desktop",
122 + encoding: str = "jpeg",
123 + quality: int = 85,
124 + speed: int = 80,
125 + file_transfer: bool = True,
126 + printing: bool = True,
127 +) -> str:
128 quoted_token = quote(str(token), safe="")
129 base_path = f"{SESSION_PATH}/{quoted_token}/"
121 - query = urlencode(
122 - {
123 - "path": base_path,
124 - "title": title,
125 - "encoding": "jpeg",
126 - "quality": "85",
127 - "speed": "80",
128 - "sharing": "true",
129 - "clipboard": "true",
130 - "clipboard_direction": "both",
131 - "clipboard_poll": "true",
132 - "clipboard_preferred_format": "text/plain",
133 - "printing": "true",
134 - "file_transfer": "true",
135 - "sound": "false",
136 - "offscreen": "true",
137 - "floating_menu": "false",
138 - "xpramenu": "false",
139 - },
140 - )
130 + options = {
131 + "path": base_path,
132 + "title": title,
133 + "quality": str(max(0, min(100, int(quality)))),
134 + "speed": str(max(0, min(100, int(speed)))),
135 + "sharing": "true",
136 + "clipboard": "true",
137 + "clipboard_direction": "both",
138 + "clipboard_poll": "true",
139 + "clipboard_preferred_format": "text/plain",
140 + "printing": str(bool(printing)).lower(),
141 + "file_transfer": str(bool(file_transfer)).lower(),
142 + "sound": "false",
143 + "offscreen": "true",
144 + "floating_menu": "false",
145 + "xpramenu": "false",
146 + }
147 + if encoding:
148 + options["encoding"] = str(encoding)
149 + query = urlencode(options)
150 return f"{base_path}index.html?{query}"
151
152
@@ -264,6 +273,7 @@ def resize_display(
273 keys: tuple[str, ...] = (),
274 xauthority: str = "",
275 home: str = "",
276 + settle_seconds: float = 0.15,
277 ) -> dict[str, Any]:
278 target_width, target_height = normalize_size(width, height, max_width=max_width, max_height=max_height)
279 xrandr = shutil.which("xrandr")
@@ -296,7 +306,8 @@ def resize_display(
306 timeout=4,
307 env=env,
308 )
299 - time.sleep(0.15)
309 + if settle_seconds > 0:
310 + time.sleep(settle_seconds)
311 current = current_display_size(display, xauthority=xauthority, home=home)
312 ok = current == (target_width, target_height)
313 if ok:
helpers/virtual_desktop.py.dox.md
+4 -2
@@ -23,13 +23,13 @@
23 - `proxy_for_token(token: str) -> VirtualDesktopEndpoint | None`
24 - `resize_session(token: str, width: int, height: int) -> dict[str, Any]`
25 - `get_registry() -> VirtualDesktopRegistry`
26 -- `session_url(token: str, title: str=...) -> str`
26 +- `session_url(token: str, title: str=..., encoding: str=..., quality: int=..., speed: int=..., file_transfer: bool=..., printing: bool=...) -> str`
27 - `collect_status() -> dict[str, Any]`
28 - `find_xpra_html_root() -> Path | None`
29 - `_package_installed(package: str) -> bool`
30 - `normalize_size(width: int | float | str, height: int | float | str, max_width: int=..., max_height: int=..., min_width: int=..., min_height: int=...) -> tuple[int, int]`
31 - `normalize_desktop_display_size(width: int | float | str, height: int | float | str, max_width: int=..., max_height: int=..., min_width: int=..., min_height: int=..., min_aspect_ratio: float=...) -> tuple[int, int]`
32 -- `resize_display(display: int, width: int, height: int, max_width: int=..., max_height: int=..., window_class: str=..., keys: tuple[str, ...]=..., xauthority: str=..., home: str=...) -> dict[str, Any]`
32 +- `resize_display(display: int, width: int, height: int, max_width: int=..., max_height: int=..., window_class: str=..., keys: tuple[str, ...]=..., xauthority: str=..., home: str=..., settle_seconds: float=...) -> dict[str, Any]`
33 - `_ensure_xrandr_mode(env: dict[str, str], width: int, height: int) -> None`
34 - `_select_xrandr_mode(env: dict[str, str], width: int, height: int) -> subprocess.CompletedProcess[str]`
35 - `_xrandr_output_modes(env: dict[str, str]) -> tuple[str, set[str]]`
@@ -54,6 +54,8 @@
54
55 - Important called helpers/classes observed in the source: `Path`, `files.get_abs_path`, `get_registry.register`, `get_registry.unregister`, `get_registry.proxy_for_token`, `get_registry.resize`, `quote`, `urlencode`, `find_xpra_html_root`, `subprocess.run`, `normalize_size`, `shutil.which`, `_display_env`, `current_display_size`, `_ensure_xrandr_mode`, `_select_xrandr_mode`, `time.sleep`, `strip`, `_xrandr_output_modes`, `result.stdout.splitlines`.
56 - Keep request/response, tool, or helper semantics documented here at the same time as source changes.
57 +- Session URLs keep Desktop's JPEG, printing, and file-transfer defaults while allowing restricted viewers such as Browser to negotiate encoding and disable unrelated capabilities.
58 +- Display resizing keeps the Desktop settle delay by default; latency-sensitive callers may skip it when they immediately verify the XRandR size.
59
60 ## Work Guidance
61
plugins/_browser/AGENTS.md
+13 -5
@@ -2,23 +2,30 @@
2
3 ## Purpose
4
5 -- Own the built-in Playwright browser tool and WebUI browser viewer.
5 +- Own the built-in Patchright browser tool and WebUI browser viewer.
6 - Bridge browser automation, page inspection helpers, and browser panel UI.
7
8 ## Ownership
9
10 - `plugin.yaml` and `default_config.yaml` own metadata and browser settings defaults.
11 - `tools/browser.py` owns the agent-facing browser tool.
12 -- `helpers/` owns Playwright runtime, selectors, URL helpers, extension management, and connector runtime logic.
12 +- `helpers/` owns the Patchright runtime, private interactive display, selectors, URL helpers, extension management, and connector runtime logic.
13 - `api/` owns status, extension, and browser WebSocket handlers.
14 - `assets/`, `prompts/`, `skills/`, `extensions/`, and `webui/` own browser scripts, prompts, skill guidance, hook contributions, and UI.
15
16 ## Local Contracts
17
18 - Keep browser actions safe around external pages, credentials, and user data.
19 -- Preserve Playwright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
19 +- Preserve Patchright lifecycle cleanup and WebSocket viewer compatibility across regular host browsers and Electron WebContentsView embedding.
20 - Keep the WebUI Browser inside its own modal/canvas affordance; do not replace it with page-level navigation.
21 -- Default the visible WebUI Browser to live CDP screencast for responsiveness. Keep lightweight CDP/DOM state snapshots as the fallback transport.
21 +- Default the visible WebUI Browser to the authenticated Xpra HTML5 viewer for its existing Patchright page. Keep live CDP screencast and lightweight snapshots as automatic fallbacks.
22 +- Keep headful Chromium in a normal window with its own toolbar clipped above the private display; do not use browser fullscreen, which shows Chromium's exit warning.
23 +- Throttle interactive resize updates throughout a drag and let the native-sized Chromium viewport follow the private display; do not defer all layout updates until resizing stops.
24 +- Keep exactly one interactive viewer iframe connected during canvas/modal handoff so hidden surfaces cannot compete to resize the same display.
25 +- Notify the active Xpra client of its new frame geometry before resizing the backing display; after an interactive canvas/modal handoff, reconcile once after Xpra's deferred resize so Chromium cannot retain the previous surface size.
26 +- Present the Xpra shadow window as the raw browser canvas: remove its HTML decoration and shadow pointer while preserving exact viewport geometry.
27 +- Give every internal Browser runtime its own Xvfb display and unguessable Xpra gateway token; never expose another chat context's display through a shared viewer.
28 +- Bind Browser Xpra endpoints to loopback, route them through the authenticated virtual-desktop gateway, and keep file transfer, URL opening, printing, and audio disabled.
29 - Paint live screencast frames through the Browser panel canvas/ImageBitmap path when available; keep the `<img>`/data URL path for snapshots and fallback rendering.
30 - Push internal screencast frames from the runtime to the WebSocket consumer after subscription; keep `read/pop_screencast_frame` as fallback/tooling APIs, not the WebUI hot path.
31 - Keep Browser viewer frame transport capability-negotiated: updated clients may request binary/slim screencast frames, while older clients must keep the base64/full-metadata fallback. Do not let the WebUI advertise binary frames unless its Socket.IO client reconstructs attachments as real `Blob`, `ArrayBuffer`, or typed-array values.
@@ -35,6 +42,7 @@
42 - Run internal Chromium headful through Patchright on the private virtual display; do not add user-agent or header spoofing on top of the patched driver.
43 - Browser startup and on-demand launch must converge on the Chromium revision declared by Patchright; let its installer select the host architecture rather than hardcoding x64 or ARM downloads.
44 - `hooks.prepare_playwright_cache()` owns reconciliation of the pinned Patchright package and Chromium binary so repository self-updates and fresh images use the same setup path.
45 +- Browser startup must install the shared virtual-desktop route hook itself; do not make Browser depend on the Desktop plugin being enabled.
46
47 ## Work Guidance
48
@@ -47,7 +55,7 @@
55 ## Verification
56
57 - Smoke-test browser launch, navigation, DOM capture, and WebUI viewer after runtime changes.
50 -- For viewer render-path changes, verify the live Browser panel paints a screencast frame on canvas with `frameSrc` empty and snapshots still falling back to the image path.
58 +- For viewer render-path changes, verify direct iframe interaction reaches the same page controlled by Patchright, separate contexts use separate displays, and an unavailable Xpra runtime falls back to CDP screencast/snapshot rendering.
59 - Run browser prompt/skill regression tests after changing browser prompt or Browser plugin skills.
60
61 ## Child DOX Index
plugins/_browser/api/status.py
+2
@@ -1,5 +1,6 @@
1 from helpers.api import ApiHandler, Request
2 from plugins._browser.helpers.config import build_browser_launch_config, get_browser_config
3 +from plugins._browser.helpers.interactive_view import collect_status as collect_interactive_status
4 from plugins._browser.helpers.playwright import (
5 get_playwright_binary,
6 get_playwright_cache_dir,
@@ -37,5 +38,6 @@ class Status(ApiHandler):
38 "requires_full_browser": launch_config["requires_full_browser"],
39 },
40 "host_browser": host_browser,
41 + "interactive_view": collect_interactive_status(),
42 "contexts": known_context_ids(),
43 }
plugins/_browser/api/ws_browser.py
+83 -8
@@ -25,7 +25,12 @@ SCREENCAST_STREAM_QUALITY = 80
25 SCREENSHOT_QUALITY = 92
26 VIEWER_TRANSPORT_SCREENCAST = "screencast"
27 VIEWER_TRANSPORT_SNAPSHOT = "snapshot"
28 -VIEWER_TRANSPORTS = {VIEWER_TRANSPORT_SCREENCAST, VIEWER_TRANSPORT_SNAPSHOT}
28 +VIEWER_TRANSPORT_INTERACTIVE = "interactive"
29 +VIEWER_TRANSPORTS = {
30 + VIEWER_TRANSPORT_INTERACTIVE,
31 + VIEWER_TRANSPORT_SCREENCAST,
32 + VIEWER_TRANSPORT_SNAPSHOT,
33 +}
34
35
36 class WsBrowser(WsHandler):
@@ -87,8 +92,19 @@ class WsBrowser(WsHandler):
92 if opened.get("id"):
93 listing["last_interacted_browser_id"] = opened.get("id")
94 active_id = self._active_browser_id(listing, data.get("browser_id"))
95 + requested_transport = self._viewer_transport(data)
96 + viewer_transport, interactive_view = await self._effective_viewer(
97 + runtime,
98 + active_id,
99 + data,
100 + )
101 initial_viewport = self._viewport_from_data(data)
91 - if runtime and active_id and initial_viewport:
102 + if (
103 + runtime
104 + and active_id
105 + and initial_viewport
106 + and viewer_transport != VIEWER_TRANSPORT_INTERACTIVE
107 + ):
108 await runtime.call(
109 "set_viewport",
110 active_id,
@@ -103,7 +119,6 @@ class WsBrowser(WsHandler):
119 if existing:
120 existing.cancel()
121 viewer_id = str(data.get("viewer_id") or "")
106 - viewer_transport = self._viewer_transport(data)
122 binary_frames = self._bool(data.get("binary_frames", data.get("binaryFrames")))
123 slim_frames = self._bool(data.get("slim_frames", data.get("slimFrames", binary_frames)))
124 capture_scale = self._capture_scale_from_data(data)
@@ -120,7 +135,13 @@ class WsBrowser(WsHandler):
135 capture_scale=capture_scale,
136 )
137 else:
123 - stream_task = self._stream_state(sid, context_id, active_id, viewer_id)
138 + stream_task = self._stream_state(
139 + sid,
140 + context_id,
141 + active_id,
142 + viewer_id,
143 + viewer_transport=viewer_transport,
144 + )
145 self._streams[stream_key] = asyncio.create_task(stream_task)
146 snapshot = await self._snapshot_for_browser(runtime, active_id)
147
@@ -136,6 +157,14 @@ class WsBrowser(WsHandler):
157 "tab_scope": tab_scope,
158 "viewer_id": viewer_id,
159 "viewer_transport": viewer_transport,
160 + "interactive_view": interactive_view,
161 + "viewer_fallback_reason": (
162 + str(interactive_view.get("error") or "")
163 + if requested_transport == VIEWER_TRANSPORT_INTERACTIVE
164 + and interactive_view
165 + and not interactive_view.get("available")
166 + else ""
167 + ),
168 "binary_frames": binary_frames,
169 "slim_frames": slim_frames,
170 }
@@ -253,6 +282,15 @@ class WsBrowser(WsHandler):
282
283 listing = await runtime.call("list")
284 last_interacted_browser_id = listing.get("last_interacted_browser_id")
285 + active_id = self._active_browser_id(
286 + listing,
287 + self._result_browser_id(result) or browser_id,
288 + )
289 + viewer_transport, interactive_view = await self._effective_viewer(
290 + runtime,
291 + active_id,
292 + data,
293 + )
294 snapshot = await self._snapshot_for_result(runtime, result)
295 browsers, all_browsers, tab_scope = await self._tabs_for_scope(
296 context_id,
@@ -273,7 +311,8 @@ class WsBrowser(WsHandler):
311 "all_browsers": all_browsers,
312 "tab_scope": tab_scope,
313 "last_interacted_browser_id": last_interacted_browser_id,
276 - "viewer_transport": self._viewer_transport(data),
314 + "viewer_transport": viewer_transport,
315 + "interactive_view": interactive_view,
316 },
317 correlation_id=data.get("correlationId"),
318 )
@@ -288,7 +327,8 @@ class WsBrowser(WsHandler):
327 "command": command,
328 "browser_id": browser_id,
329 "viewer_id": viewer_id,
291 - "viewer_transport": self._viewer_transport(data),
330 + "viewer_transport": viewer_transport,
331 + "interactive_view": interactive_view,
332 }
333
334 async def _input(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
@@ -326,12 +366,15 @@ class WsBrowser(WsHandler):
366 text=str(data.get("text") or ""),
367 )
368 elif input_type == "viewport":
369 + viewer_transport = self._viewer_transport(data)
370 result = await runtime.call(
371 "set_viewport",
372 browser_id,
373 int(data.get("width") or 0),
374 int(data.get("height") or 0),
375 restart_screencast=bool(data.get("restart_stream")),
376 + resize_interactive=viewer_transport == VIEWER_TRANSPORT_INTERACTIVE,
377 + include_state=viewer_transport != VIEWER_TRANSPORT_INTERACTIVE,
378 )
379 elif input_type == "wheel":
380 result = await runtime.call(
@@ -582,6 +625,8 @@ class WsBrowser(WsHandler):
625 context_id: str,
626 browser_id: int | str | None,
627 viewer_id: str = "",
628 + *,
629 + viewer_transport: str = VIEWER_TRANSPORT_SNAPSHOT,
630 ) -> None:
631 last_signature = None
632 while True:
@@ -594,7 +639,7 @@ class WsBrowser(WsHandler):
639 sid,
640 context_id,
641 viewer_id=viewer_id,
597 - frame_source=VIEWER_TRANSPORT_SNAPSHOT,
642 + frame_source=viewer_transport,
643 )
644 last_signature = signature
645 await asyncio.sleep(FRAME_RETRY_DELAY_SECONDS)
@@ -625,7 +670,7 @@ class WsBrowser(WsHandler):
670 browsers=browsers,
671 viewer_id=viewer_id,
672 state=state,
628 - viewer_transport=VIEWER_TRANSPORT_SNAPSHOT,
673 + viewer_transport=viewer_transport,
674 )
675 last_signature = signature
676 await asyncio.sleep(SNAPSHOT_STATE_POLL_SECONDS)
@@ -653,6 +698,36 @@ class WsBrowser(WsHandler):
698 active_id = browsers[0].get("id")
699 return active_id
700
701 + async def _effective_viewer(
702 + self,
703 + runtime: Any,
704 + browser_id: int | str | None,
705 + data: dict[str, Any],
706 + ) -> tuple[str, dict[str, Any] | None]:
707 + requested = self._viewer_transport(data)
708 + if requested != VIEWER_TRANSPORT_INTERACTIVE or not runtime or not browser_id:
709 + return requested, None
710 + viewport = self._viewport_from_data(data) or {}
711 + try:
712 + viewer = await runtime.call(
713 + "interactive_viewer",
714 + browser_id,
715 + width=int(viewport.get("width") or 0),
716 + height=int(viewport.get("height") or 0),
717 + )
718 + except Exception as exc:
719 + viewer = {"available": False, "error": str(exc)}
720 + if viewer.get("available"):
721 + return VIEWER_TRANSPORT_INTERACTIVE, viewer
722 + return VIEWER_TRANSPORT_SCREENCAST, viewer
723 +
724 + @staticmethod
725 + def _result_browser_id(result: Any) -> int | str | None:
726 + if not isinstance(result, dict):
727 + return None
728 + state = result.get("state") if isinstance(result.get("state"), dict) else result
729 + return state.get("id") if isinstance(state, dict) else None
730 +
731 @staticmethod
732 def _state_for_browser(
733 browsers: list[dict[str, Any]],
plugins/_browser/extensions/python/startup_migration/_20_browser_playwright_cache.py
+2
@@ -3,6 +3,7 @@ from __future__ import annotations
3 import threading
4 from typing import Any
5
6 +from helpers import virtual_desktop_routes
7 from helpers.extension import Extension
8 from helpers.print_style import PrintStyle
9 from plugins._browser import hooks
@@ -13,6 +14,7 @@ _startup_migration_thread: threading.Thread | None = None
14
15 class BrowserPlaywrightCacheMigration(Extension):
16 def execute(self, **kwargs):
17 + virtual_desktop_routes.install_route_hooks()
18 _start_background_cache_migration()
19
20
plugins/_browser/helpers/interactive_view.py new
+293
@@ -0,0 +1,293 @@
1 +from __future__ import annotations
2 +
3 +import os
4 +import select
5 +import shutil
6 +import socket
7 +import subprocess
8 +import threading
9 +import time
10 +import uuid
11 +from pathlib import Path
12 +from typing import Any
13 +
14 +from helpers import files, virtual_desktop
15 +from helpers.print_style import PrintStyle
16 +
17 +
18 +DEFAULT_WIDTH = 1024
19 +DEFAULT_HEIGHT = 768
20 +START_TIMEOUT_SECONDS = 15.0
21 +
22 +
23 +def collect_status() -> dict[str, Any]:
24 + binaries = {
25 + name: shutil.which(name) or ""
26 + for name in ("Xvfb", "xpra", "xrandr")
27 + }
28 + html_root = virtual_desktop.find_xpra_html_root()
29 + missing = [name for name, path in binaries.items() if not path]
30 + if not html_root:
31 + missing.append("xpra-html5")
32 + return {
33 + "available": not missing,
34 + "missing": missing,
35 + "binaries": binaries,
36 + "xpra_html_root": str(html_root or ""),
37 + }
38 +
39 +
40 +class BrowserInteractiveView:
41 + """Own one private X display and its optional Xpra viewer."""
42 +
43 + def __init__(self, context_id: str) -> None:
44 + self.context_id = str(context_id)
45 + self.token = f"browser-{uuid.uuid4().hex}"
46 + self.state_dir = Path(files.get_abs_path("tmp", "browser", "displays", self.token))
47 + self.display: int | None = None
48 + self.port = 0
49 + self.width = DEFAULT_WIDTH
50 + self.height = DEFAULT_HEIGHT
51 + self._xvfb: subprocess.Popen[Any] | None = None
52 + self._xpra: subprocess.Popen[Any] | None = None
53 + self._lock = threading.RLock()
54 +
55 + @property
56 + def display_name(self) -> str:
57 + return f":{self.display}" if self.display is not None else ""
58 +
59 + def ensure_display(self) -> str:
60 + with self._lock:
61 + if self._running(self._xvfb) and self.display is not None:
62 + return self.display_name
63 +
64 + self._stop_locked()
65 + xvfb = shutil.which("Xvfb")
66 + if not xvfb:
67 + return ""
68 +
69 + self.state_dir.mkdir(parents=True, exist_ok=True)
70 + self.state_dir.chmod(0o700)
71 + read_fd, write_fd = os.pipe()
72 + try:
73 + process = subprocess.Popen(
74 + [
75 + xvfb,
76 + "-displayfd",
77 + str(write_fd),
78 + "-screen",
79 + "0",
80 + f"{virtual_desktop.MAX_WIDTH}x{virtual_desktop.MAX_HEIGHT}x24",
81 + "+extension",
82 + "GLX",
83 + "+extension",
84 + "RANDR",
85 + "+extension",
86 + "RENDER",
87 + "+extension",
88 + "Composite",
89 + "-nolisten",
90 + "tcp",
91 + "-noreset",
92 + "-ac",
93 + ],
94 + stdin=subprocess.DEVNULL,
95 + stdout=subprocess.DEVNULL,
96 + stderr=subprocess.DEVNULL,
97 + pass_fds=(write_fd,),
98 + )
99 + except OSError:
100 + os.close(read_fd)
101 + return ""
102 + finally:
103 + os.close(write_fd)
104 +
105 + try:
106 + ready, _, _ = select.select([read_fd], [], [], 5)
107 + display_number = os.read(read_fd, 32).decode().strip() if ready else ""
108 + finally:
109 + os.close(read_fd)
110 +
111 + if not display_number.isdigit() or process.poll() is not None:
112 + self._terminate(process)
113 + return ""
114 +
115 + self._xvfb = process
116 + self.display = int(display_number)
117 + self.resize(self.width, self.height)
118 + return self.display_name
119 +
120 + def ensure_viewer(self, width: int = 0, height: int = 0) -> dict[str, Any]:
121 + with self._lock:
122 + display_name = self.ensure_display()
123 + if not display_name:
124 + return self._unavailable("Xvfb is unavailable.")
125 +
126 + status = collect_status()
127 + if not status["available"]:
128 + return self._unavailable(
129 + f"Interactive Browser runtime needs: {', '.join(status['missing'])}."
130 + )
131 +
132 + self.resize(width or self.width, height or self.height)
133 + if not self._running(self._xpra):
134 + try:
135 + self._start_xpra(str(status["binaries"]["xpra"]))
136 + except Exception as exc:
137 + PrintStyle.warning(f"Interactive Browser viewer failed to start: {exc}")
138 + self._terminate(self._xpra)
139 + self._xpra = None
140 + self.port = 0
141 + virtual_desktop.unregister_session(self.token)
142 + return self._unavailable(str(exc))
143 +
144 + virtual_desktop.register_session(
145 + token=self.token,
146 + host="127.0.0.1",
147 + port=self.port,
148 + owner="browser",
149 + title="Browser",
150 + resize=self.resize,
151 + )
152 + return {
153 + "available": True,
154 + "token": self.token,
155 + "url": virtual_desktop.session_url(
156 + self.token,
157 + title="Browser",
158 + encoding="",
159 + quality=90,
160 + speed=90,
161 + file_transfer=False,
162 + printing=False,
163 + ),
164 + "width": self.width,
165 + "height": self.height,
166 + }
167 +
168 + def resize(self, width: int, height: int) -> dict[str, Any]:
169 + with self._lock:
170 + target_width, target_height = virtual_desktop.normalize_size(width, height)
171 + self.width = target_width
172 + self.height = target_height
173 + if self.display is None or not self._running(self._xvfb):
174 + return {
175 + "ok": False,
176 + "error": "Browser display is unavailable.",
177 + "width": target_width,
178 + "height": target_height,
179 + }
180 + result = virtual_desktop.resize_display(
181 + display=self.display,
182 + width=target_width,
183 + height=target_height,
184 + settle_seconds=0,
185 + )
186 + return result
187 +
188 + def close(self) -> None:
189 + with self._lock:
190 + self._stop_locked()
191 + shutil.rmtree(self.state_dir, ignore_errors=True)
192 +
193 + def _start_xpra(self, xpra: str) -> None:
194 + self.port = self._free_port()
195 + runtime_dir = self.state_dir / "runtime"
196 + socket_dir = self.state_dir / "sockets"
197 + runtime_dir.mkdir(parents=True, exist_ok=True)
198 + socket_dir.mkdir(parents=True, exist_ok=True)
199 + runtime_dir.chmod(0o700)
200 + env = {
201 + **os.environ,
202 + "DISPLAY": self.display_name,
203 + "XDG_RUNTIME_DIR": str(runtime_dir),
204 + }
205 + self._xpra = subprocess.Popen(
206 + [
207 + xpra,
208 + "shadow",
209 + self.display_name,
210 + "--daemon=no",
211 + "--mdns=no",
212 + "--html=on",
213 + "--tray=no",
214 + "--system-tray=no",
215 + "--notifications=no",
216 + "--clipboard=yes",
217 + "--clipboard-direction=both",
218 + "--file-transfer=no",
219 + "--open-files=no",
220 + "--open-url=no",
221 + "--printing=no",
222 + "--audio=no",
223 + "--speaker=off",
224 + "--microphone=off",
225 + "--sharing=yes",
226 + "--resize-display=yes",
227 + "--encoding=auto",
228 + "--quality=90",
229 + "--speed=90",
230 + f"--bind-tcp=127.0.0.1:{self.port}",
231 + f"--socket-dir={socket_dir}",
232 + f"--log-dir={self.state_dir}",
233 + "--log-file=xpra.log",
234 + ],
235 + stdin=subprocess.DEVNULL,
236 + stdout=subprocess.DEVNULL,
237 + stderr=subprocess.DEVNULL,
238 + env=env,
239 + )
240 + self._wait_for_port(self._xpra, self.port)
241 +
242 + def _stop_locked(self) -> None:
243 + virtual_desktop.unregister_session(self.token)
244 + self._terminate(self._xpra)
245 + self._terminate(self._xvfb)
246 + self._xpra = None
247 + self._xvfb = None
248 + self.port = 0
249 + self.display = None
250 +
251 + def _unavailable(self, error: str) -> dict[str, Any]:
252 + return {
253 + "available": False,
254 + "error": str(error or "Interactive Browser viewer is unavailable."),
255 + }
256 +
257 + @staticmethod
258 + def _running(process: subprocess.Popen[Any] | None) -> bool:
259 + return bool(process and process.poll() is None)
260 +
261 + @staticmethod
262 + def _terminate(process: subprocess.Popen[Any] | None) -> None:
263 + if not process or process.poll() is not None:
264 + return
265 + process.terminate()
266 + try:
267 + process.wait(timeout=2)
268 + except subprocess.TimeoutExpired:
269 + process.kill()
270 + process.wait(timeout=2)
271 +
272 + @staticmethod
273 + def _free_port() -> int:
274 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
275 + probe.bind(("127.0.0.1", 0))
276 + return int(probe.getsockname()[1])
277 +
278 + @staticmethod
279 + def _wait_for_port(
280 + process: subprocess.Popen[Any],
281 + port: int,
282 + timeout: float = START_TIMEOUT_SECONDS,
283 + ) -> None:
284 + deadline = time.monotonic() + timeout
285 + while time.monotonic() < deadline:
286 + if process.poll() is not None:
287 + raise RuntimeError("Xpra exited before its Browser endpoint was ready.")
288 + try:
289 + with socket.create_connection(("127.0.0.1", port), timeout=0.2):
290 + return
291 + except OSError:
292 + time.sleep(0.1)
293 + raise TimeoutError("Timed out waiting for the interactive Browser endpoint.")
plugins/_browser/helpers/playwright.py
-85
@@ -1,9 +1,6 @@
1 -import atexit
1 import json
2 import os
3 import re
5 -import select
6 -import shutil
4 import subprocess
5 import sys
6 import threading
@@ -23,9 +20,6 @@ RETIRED_PLAYWRIGHT_CACHE_DIRS = (
20 ("usr", "browser", "playwright"),
21 )
22 _INSTALL_LOCK = threading.Lock()
26 -_DISPLAY_LOCK = threading.Lock()
27 -_DISPLAY_PROCESS: subprocess.Popen | None = None
28 -_DISPLAY_NAME = ""
23
24
25 def _primary_cache_dir() -> Path:
@@ -64,82 +58,6 @@ def configure_playwright_env() -> str:
58 return cache_dir
59
60
67 -def ensure_browser_display() -> str:
68 - global _DISPLAY_NAME, _DISPLAY_PROCESS
69 -
70 - with _DISPLAY_LOCK:
71 - if _DISPLAY_PROCESS and _DISPLAY_PROCESS.poll() is None:
72 - return _DISPLAY_NAME
73 -
74 - xvfb = shutil.which("Xvfb")
75 - if not xvfb:
76 - return ""
77 -
78 - read_fd, write_fd = os.pipe()
79 - try:
80 - process = subprocess.Popen(
81 - [
82 - xvfb,
83 - "-displayfd",
84 - str(write_fd),
85 - "-screen",
86 - "0",
87 - "1365x768x24",
88 - "+extension",
89 - "GLX",
90 - "-nolisten",
91 - "tcp",
92 - "-noreset",
93 - "-ac",
94 - ],
95 - stdin=subprocess.DEVNULL,
96 - stdout=subprocess.DEVNULL,
97 - stderr=subprocess.DEVNULL,
98 - pass_fds=(write_fd,),
99 - )
100 - except OSError:
101 - os.close(read_fd)
102 - return ""
103 - finally:
104 - os.close(write_fd)
105 -
106 - try:
107 - ready, _, _ = select.select([read_fd], [], [], 5)
108 - display_number = os.read(read_fd, 32).decode().strip() if ready else ""
109 - finally:
110 - os.close(read_fd)
111 -
112 - if not display_number.isdigit() or process.poll() is not None:
113 - _terminate_browser_display(process)
114 - return ""
115 -
116 - _DISPLAY_PROCESS = process
117 - _DISPLAY_NAME = f":{display_number}"
118 - return _DISPLAY_NAME
119 -
120 -
121 -def close_browser_display() -> None:
122 - global _DISPLAY_NAME, _DISPLAY_PROCESS
123 -
124 - with _DISPLAY_LOCK:
125 - process = _DISPLAY_PROCESS
126 - _DISPLAY_PROCESS = None
127 - _DISPLAY_NAME = ""
128 - if process:
129 - _terminate_browser_display(process)
130 -
131 -
132 -def _terminate_browser_display(process: subprocess.Popen) -> None:
133 - if process.poll() is not None:
134 - return
135 - process.terminate()
136 - try:
137 - process.wait(timeout=2)
138 - except subprocess.TimeoutExpired:
139 - process.kill()
140 - process.wait(timeout=2)
141 -
142 -
61 def find_playwright_binary(cache_dir: Path, revision: str = "") -> Path | None:
62 prefix = f"chromium-{revision}" if revision.isdigit() else "chromium-*"
63 binaries = [
@@ -197,6 +115,3 @@ def ensure_playwright_binary() -> Path:
115 if not binary:
116 raise RuntimeError("Patchright Chromium binary not found after installation")
117 return binary
200 -
201 -
202 -atexit.register(close_browser_display)
plugins/_browser/helpers/runtime.py
+172 -12
@@ -27,9 +27,7 @@ from plugins._browser.helpers.config import (
27 build_browser_launch_config,
28 get_browser_config,
29 )
30 -from plugins._browser.helpers.playwright import (
31 - ensure_browser_display,
32 -)
30 +from plugins._browser.helpers.interactive_view import BrowserInteractiveView
31 from plugins._browser.helpers.url import normalize_url
32
33
@@ -558,7 +556,10 @@ class BrowserRuntime:
556 await self.call("close", delete_profile=delete_profile)
557 finally:
558 self._closed = True
561 - self._worker.kill(terminate_thread=True)
559 + try:
560 + self._worker.kill(terminate_thread=True)
561 + finally:
562 + self._core.interactive_view.close()
563
564
565 class _BrowserRuntimeCore:
@@ -594,6 +595,11 @@ class _BrowserRuntimeCore:
595 self._pending_popups: list[asyncio.Future[int]] = []
596 self._background_popup_pages: set[int] = set()
597 self._bootstrap_page: Any | None = None
598 + self._browser_chrome_height: int | None = None
599 + self._browser_window_page: Any | None = None
600 + self._browser_window_session: Any | None = None
601 + self._browser_window_id: int | None = None
602 + self.interactive_view = BrowserInteractiveView(context_id)
603
604 def _ensure_registry_lock(self) -> asyncio.Lock:
605 if self._registry_lock is None:
@@ -777,6 +783,10 @@ class _BrowserRuntimeCore:
783 self._pending_popups.clear()
784 self._background_popup_pages.clear()
785 self._bootstrap_page = None
786 + self._browser_chrome_height = None
787 + self._browser_window_page = None
788 + self._browser_window_session = None
789 + self._browser_window_id = None
790 self.pages.clear()
791 self.last_interacted_browser_id = None
792 for screencast in self.screencasts.values():
@@ -815,7 +825,15 @@ class _BrowserRuntimeCore:
825 browser_config = get_browser_config()
826 launch_config = build_browser_launch_config(browser_config)
827 browser_binary = Path(preparation["binary"])
818 - browser_display = ensure_browser_display()
828 + browser_display = self.interactive_view.ensure_display()
829 + launch_args = list(launch_config["args"])
830 + if browser_display:
831 + launch_args.extend(
832 + [
833 + "--window-position=0,0",
834 + f"--window-size={self.interactive_view.width},{self.interactive_view.height}",
835 + ]
836 + )
837
838 self.playwright = await async_playwright().start()
839 launch_kwargs: dict[str, Any] = {
@@ -823,13 +841,17 @@ class _BrowserRuntimeCore:
841 "headless": not bool(browser_display),
842 "accept_downloads": True,
843 "downloads_path": str(self.downloads_dir),
826 - "viewport": DEFAULT_VIEWPORT,
827 - "screen": DEFAULT_VIEWPORT,
828 - "no_viewport": False,
829 - "args": launch_config["args"],
844 + "args": launch_args,
845 }
846 if browser_display:
847 launch_kwargs["env"] = {**os.environ, "DISPLAY": browser_display}
848 + launch_kwargs["no_viewport"] = True
849 + else:
850 + launch_kwargs.update(
851 + viewport=DEFAULT_VIEWPORT,
852 + screen=DEFAULT_VIEWPORT,
853 + no_viewport=False,
854 + )
855 if launch_config["channel"]:
856 launch_kwargs["channel"] = launch_config["channel"]
857 else:
@@ -857,6 +879,7 @@ class _BrowserRuntimeCore:
879 if page.url == "about:blank":
880 if browser_display and self._bootstrap_page is None:
881 self._bootstrap_page = page
882 + await self._fit_browser_window(page)
883 continue
884 try:
885 await page.close()
@@ -1236,8 +1259,12 @@ class _BrowserRuntimeCore:
1259 async def set_active(self, browser_id: int | str | None) -> dict[str, Any]:
1260 await self.ensure_started()
1261 resolved_id = self._resolve_browser_id(browser_id)
1262 + page = self._page(resolved_id)
1263 # Explicit focus change — bypass _maybe_promote.
1264 self.last_interacted_browser_id = int(resolved_id)
1265 + with contextlib.suppress(Exception):
1266 + await page.bring_to_front()
1267 + await self._fit_browser_window(page)
1268 return await self._state(resolved_id)
1269
1270 async def state(self, browser_id: int | str | None = None) -> dict[str, Any]:
@@ -1703,7 +1730,7 @@ class _BrowserRuntimeCore:
1730 await screencast.start(
1731 quality=quality,
1732 every_nth_frame=every_nth_frame,
1706 - viewport=page.viewport_size or DEFAULT_VIEWPORT,
1733 + viewport=await self._page_viewport(page),
1734 capture_scale=capture_scale,
1735 )
1736 except Exception:
@@ -1717,6 +1744,61 @@ class _BrowserRuntimeCore:
1744 "state": await self._state(resolved_id),
1745 }
1746
1747 + @staticmethod
1748 + async def _page_viewport(page: Any) -> dict[str, int]:
1749 + viewport = getattr(page, "viewport_size", None)
1750 + if viewport:
1751 + return {
1752 + "width": int(viewport.get("width") or DEFAULT_VIEWPORT["width"]),
1753 + "height": int(viewport.get("height") or DEFAULT_VIEWPORT["height"]),
1754 + }
1755 + try:
1756 + measured = await page.evaluate(
1757 + "() => ({ width: globalThis.innerWidth, height: globalThis.innerHeight })",
1758 + isolated_context=False,
1759 + )
1760 + return {
1761 + "width": int(measured.get("width") or DEFAULT_VIEWPORT["width"]),
1762 + "height": int(measured.get("height") or DEFAULT_VIEWPORT["height"]),
1763 + }
1764 + except Exception:
1765 + return dict(DEFAULT_VIEWPORT)
1766 +
1767 + async def interactive_viewer(
1768 + self,
1769 + browser_id: int | str | None = None,
1770 + *,
1771 + width: int = 0,
1772 + height: int = 0,
1773 + ) -> dict[str, Any]:
1774 + await self.ensure_started()
1775 + resolved_id = self._resolve_browser_id(browser_id)
1776 + page = self._page(resolved_id)
1777 + current_viewport = await self._page_viewport(page)
1778 + viewer = self.interactive_view.ensure_viewer(
1779 + width or int(current_viewport.get("width") or DEFAULT_VIEWPORT["width"]),
1780 + height or int(current_viewport.get("height") or DEFAULT_VIEWPORT["height"]),
1781 + )
1782 + if not viewer.get("available"):
1783 + return viewer
1784 +
1785 + await self._stop_screencasts_for_browser(resolved_id)
1786 + with contextlib.suppress(Exception):
1787 + await page.bring_to_front()
1788 + viewport_result = await self.set_viewport(
1789 + resolved_id,
1790 + int(viewer.get("width") or width or DEFAULT_VIEWPORT["width"]),
1791 + int(viewer.get("height") or height or DEFAULT_VIEWPORT["height"]),
1792 + resize_interactive=True,
1793 + )
1794 + self.last_interacted_browser_id = int(resolved_id)
1795 + return {
1796 + **viewer,
1797 + "browser_id": resolved_id,
1798 + "state": viewport_result["state"],
1799 + "viewport": viewport_result["viewport"],
1800 + }
1801 +
1802 async def read_screencast_frame(
1803 self,
1804 stream_id: str,
@@ -1756,15 +1838,30 @@ class _BrowserRuntimeCore:
1838 width: int,
1839 height: int,
1840 restart_screencast: bool = False,
1841 + resize_interactive: bool = False,
1842 + include_state: bool = True,
1843 ) -> dict[str, Any]:
1844 await self.ensure_started()
1845 resolved_id = self._resolve_browser_id(browser_id)
1846 page = self._page(resolved_id)
1847 + if resize_interactive:
1848 + resized = self.interactive_view.resize(width, height)
1849 + viewport = {
1850 + "width": int(resized.get("width") or self.interactive_view.width),
1851 + "height": int(resized.get("height") or self.interactive_view.height),
1852 + }
1853 + await self._fit_browser_window(page)
1854 + self._maybe_promote(resolved_id)
1855 + return {
1856 + "state": await self._state(resolved_id) if include_state else None,
1857 + "viewport": viewport,
1858 + }
1859 +
1860 viewport = {
1861 "width": max(320, min(4096, int(width or DEFAULT_VIEWPORT["width"]))),
1862 "height": max(200, min(4096, int(height or DEFAULT_VIEWPORT["height"]))),
1863 }
1767 - current_viewport = page.viewport_size or {}
1864 + current_viewport = await self._page_viewport(page)
1865 changed = (
1866 abs(int(current_viewport.get("width") or 0) - viewport["width"])
1867 > VIEWPORT_SIZE_TOLERANCE
@@ -2182,6 +2279,7 @@ class _BrowserRuntimeCore:
2279 self._pending_popups.clear()
2280 self._background_popup_pages.clear()
2281 await self._stop_all_screencasts()
2282 + await self._reset_browser_window_session()
2283 for browser_id in list(self.pages):
2284 try:
2285 await self.pages[browser_id].page.close()
@@ -2315,7 +2413,67 @@ class _BrowserRuntimeCore:
2413 async def _register_page(self, page: Any) -> BrowserPage:
2414 lock = self._ensure_registry_lock()
2415 async with lock:
2318 - return self._register_page_locked(page)
2416 + browser_page = self._register_page_locked(page)
2417 + await self._fit_browser_window(page)
2418 + return browser_page
2419 +
2420 + async def _fit_browser_window(self, page: Any) -> None:
2421 + if getattr(self.interactive_view, "display", None) is None or not self.context:
2422 + return
2423 + try:
2424 + if self._browser_window_session is None or self._browser_window_page is not page:
2425 + await self._reset_browser_window_session()
2426 + self._browser_window_page = page
2427 + self._browser_window_session = await self.context.new_cdp_session(page)
2428 + target = await self._browser_window_session.send("Browser.getWindowForTarget")
2429 + self._browser_window_id = target.get("windowId")
2430 + if self._browser_window_id is None:
2431 + await self._reset_browser_window_session()
2432 + return
2433 + current = await self._browser_window_session.send(
2434 + "Browser.getWindowBounds",
2435 + {"windowId": self._browser_window_id},
2436 + )
2437 + if current.get("bounds", {}).get("windowState") != "normal":
2438 + await self._browser_window_session.send(
2439 + "Browser.setWindowBounds",
2440 + {
2441 + "windowId": self._browser_window_id,
2442 + "bounds": {"windowState": "normal"},
2443 + },
2444 + )
2445 + if self._browser_chrome_height is None:
2446 + chrome_height = await page.evaluate(
2447 + "() => Math.max(0, globalThis.outerHeight - globalThis.innerHeight)",
2448 + isolated_context=False,
2449 + )
2450 + self._browser_chrome_height = max(0, min(256, int(chrome_height or 0)))
2451 + chrome_height = self._browser_chrome_height
2452 + await self._browser_window_session.send(
2453 + "Browser.setWindowBounds",
2454 + {
2455 + "windowId": self._browser_window_id,
2456 + "bounds": {
2457 + "windowState": "normal",
2458 + "left": 0,
2459 + "top": -chrome_height,
2460 + "width": self.interactive_view.width,
2461 + "height": self.interactive_view.height + chrome_height,
2462 + },
2463 + },
2464 + )
2465 + except Exception as exc:
2466 + await self._reset_browser_window_session()
2467 + PrintStyle.warning(f"Interactive Browser window fit failed: {exc}")
2468 +
2469 + async def _reset_browser_window_session(self) -> None:
2470 + session = self._browser_window_session
2471 + self._browser_window_page = None
2472 + self._browser_window_session = None
2473 + self._browser_window_id = None
2474 + if session:
2475 + with contextlib.suppress(Exception):
2476 + await session.detach()
2477
2478 async def _unregister_page_async(self, browser_id: int) -> None:
2479 try:
@@ -2372,6 +2530,8 @@ class _BrowserRuntimeCore:
2530 if close_over_limit:
2531 with contextlib.suppress(Exception):
2532 await page.close()
2533 + else:
2534 + await self._fit_browser_window(page)
2535 except Exception as exc:
2536 PrintStyle.warning(f"Popup registration failed: {exc}")
2537
plugins/_browser/webui/browser-panel.html
+55 -39
@@ -20,10 +20,13 @@
20 <div class="browser-tab-shell" :class="{ 'is-active': $store.browserPage.isActiveBrowser(browser) }">
21 <button type="button" class="browser-tab" role="tab"
22 :aria-selected="$store.browserPage.isActiveBrowser(browser).toString()"
23 + :aria-busy="$store.browserPage.isBrowserLoading(browser).toString()"
24 :title="$store.browserPage.browserTabTooltip(browser)"
25 @click="$store.browserPage.selectBrowser(browser.id, browser.context_id)">
26 <x-icon class="browser-tab-icon" aria-hidden="true" name="language"></x-icon>
27 <span class="browser-tab-title" x-text="$store.browserPage.browserTabTitle(browser)"></span>
28 + <x-icon class="browser-tab-loading spinning" aria-hidden="true"
29 + x-show="$store.browserPage.isBrowserLoading(browser)" name="progress_activity"></x-icon>
30 </button>
31 <button type="button" class="browser-tab-close"
32 :title="'Close ' + $store.browserPage.browserTabLabel(browser)"
@@ -185,15 +188,22 @@
188 </form>
189 </div>
190
188 - <div class="browser-stage" tabindex="0" @click="$el.focus()"
189 - :class="{ 'is-annotating': $store.browserPage.annotating }"
190 - @wheel.prevent="$store.browserPage.handleStageWheel($event)">
191 - <canvas class="browser-frame browser-frame-canvas"
192 - x-show="$store.browserPage.frameCanvasReady"
193 - x-init="$store.browserPage.attachFrameCanvas($el)"
194 - @click="$store.browserPage.sendMouse('click', $event)"
195 - @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)"></canvas>
196 - <template x-if="$store.browserPage.frameSrc">
191 + <div class="browser-stage" tabindex="0" @click="$el.focus()"
192 + :class="{ 'is-annotating': $store.browserPage.annotating }"
193 + @wheel.prevent="$store.browserPage.handleStageWheel($event)">
194 + <template x-if="$store.browserPage.isInteractiveSurface($el.parentElement)">
195 + <iframe class="browser-interactive-frame"
196 + :src="$store.browserPage.interactiveViewUrl"
197 + aria-label="Browser viewport"
198 + allow="clipboard-read; clipboard-write"
199 + @load="$store.browserPage.onInteractiveViewLoad()"></iframe>
200 + </template>
201 + <canvas class="browser-frame browser-frame-canvas"
202 + x-show="!$store.browserPage.usesInteractiveTransport() && $store.browserPage.frameCanvasReady"
203 + x-init="$store.browserPage.attachFrameCanvas($el)"
204 + @click="$store.browserPage.sendMouse('click', $event)"
205 + @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)"></canvas>
206 + <template x-if="!$store.browserPage.usesInteractiveTransport() && $store.browserPage.frameSrc">
207 <img class="browser-frame browser-frame-image" :src="$store.browserPage.frameSrc"
208 @click="$store.browserPage.sendMouse('click', $event)"
209 @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)" draggable="false" />
@@ -282,18 +292,11 @@
292 </template>
293 </div>
294
285 - <div class="browser-bottom-status"
286 - :class="{ 'is-active': $store.browserPage.isBusy() || $store.browserPage.error, 'has-error': $store.browserPage.error && !$store.browserPage.isBusy() }">
287 - <template x-if="$store.browserPage.isBusy()">
288 - <div class="browser-status" :title="$store.browserPage.loadingMessage()">
289 - <x-icon class="spinning" name="progress_activity"></x-icon>
290 - <span x-text="$store.browserPage.loadingMessage()">Loading</span>
291 - </div>
292 - </template>
293 - <template x-if="!$store.browserPage.isBusy() && $store.browserPage.error">
295 + <template x-if="$store.browserPage.error">
296 + <div class="browser-bottom-status has-error">
297 <div class="browser-error" x-text="$store.browserPage.error" :title="$store.browserPage.error"></div>
295 - </template>
296 - </div>
298 + </div>
299 + </template>
300 </div>
301 </template>
302 </div>
@@ -603,6 +606,13 @@
606 font-weight: 600;
607 }
608
609 + .browser-tab-loading {
610 + flex: 0 0 auto;
611 + color: color-mix(in srgb, var(--color-text) 66%, var(--color-primary) 34%);
612 + font-size: 0.96rem;
613 + line-height: 1;
614 + }
615 +
616 .browser-tab-close {
617 display: inline-flex;
618 align-items: center;
@@ -997,19 +1007,31 @@
1007 cursor: crosshair;
1008 }
1009
1000 - .browser-frame {
1001 - display: block;
1002 - position: absolute;
1003 - inset: 0;
1004 - width: 100%;
1005 - height: 100%;
1006 - min-width: 0;
1007 - min-height: 0;
1008 - object-fit: contain;
1009 - image-rendering: auto;
1010 - user-select: none;
1011 - background: #fff;
1012 - }
1010 + .browser-frame {
1011 + display: block;
1012 + position: absolute;
1013 + inset: 0;
1014 + width: 100%;
1015 + height: 100%;
1016 + min-width: 0;
1017 + min-height: 0;
1018 + object-fit: contain;
1019 + image-rendering: auto;
1020 + user-select: none;
1021 + background: #fff;
1022 + }
1023 +
1024 + .browser-interactive-frame {
1025 + display: block;
1026 + position: absolute;
1027 + inset: 0;
1028 + z-index: 1;
1029 + box-sizing: border-box;
1030 + width: 100%;
1031 + height: 100%;
1032 + border: 0;
1033 + background: #fff;
1034 + }
1035
1036 .browser-annotation-layer {
1037 position: absolute;
@@ -1239,7 +1261,6 @@
1261 background: color-mix(in srgb, #7f1d1d 22%, var(--color-background));
1262 }
1263
1242 - .browser-status,
1264 .browser-error {
1265 display: inline-flex;
1266 align-items: center;
@@ -1250,17 +1271,12 @@
1271 white-space: nowrap;
1272 }
1273
1253 - .browser-status span:not(.material-symbols-outlined),
1274 .browser-error {
1275 min-width: 0;
1276 overflow: hidden;
1277 text-overflow: ellipsis;
1278 }
1279
1260 - .browser-status .material-symbols-outlined {
1261 - font-size: 15px;
1262 - }
1263 -
1280 .browser-modal .spinning,
1281 .browser-panel .spinning {
1282 display: inline-block;
plugins/_browser/webui/browser-store.js
+155 -32
@@ -21,11 +21,13 @@ const BROWSER_SUBSCRIBE_TIMEOUT_MS = 60000;
21 const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
22 const BROWSER_COMMAND_TIMEOUT_MS = 45000;
23 const BROWSER_CONFIG_REFRESH_MS = 15000;
24 +const BROWSER_VIEWER_TRANSPORT_INTERACTIVE = "interactive";
25 const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";
26 const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";
26 -const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
27 +const VIEWPORT_SYNC_INTERVAL_MS = 50;
28 const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
29 const CANVAS_VIEWPORT_SETTLE_MS = 520;
30 +const INTERACTIVE_VIEWPORT_SETTLE_MS = 320;
31 const SURFACE_VIEWPORT_STABLE_FRAMES = 4;
32 const SURFACE_VIEWPORT_MAX_WAIT_MS = 1200;
33 const FRAME_REJECT_SYNC_COOLDOWN_MS = 600;
@@ -168,9 +170,10 @@ const model = {
170 frameSrc: "",
171 frameCanvasReady: false,
172 frameState: null,
171 - viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST,
173 + viewerTransport: BROWSER_VIEWER_TRANSPORT_INTERACTIVE,
174 + interactiveViewUrl: "",
175 + viewerFallbackReason: "",
176 tabScope: "per_context",
173 - liveScreencastEnabled: true,
177 annotating: false,
178 annotationComments: [],
179 annotationDraft: null,
@@ -216,6 +219,7 @@ const model = {
219 _openSignature: "",
220 _connectSequence: 0,
221 _viewerToken: "",
222 + _subscribedViewerTransport: BROWSER_VIEWER_TRANSPORT_INTERACTIVE,
223 _contextCreatePromise: null,
224 _lastSelectedContextId: "",
225 _sessionRefreshPromise: null,
@@ -879,6 +883,7 @@ const model = {
883
884 resetRenderedFrame() {
885 this.cancelFrameRender();
886 + this.interactiveViewUrl = "";
887 this.clearFrameSrc();
888 this.clearFrameCanvas();
889 this._lastFrameDimensions = null;
@@ -932,6 +937,7 @@ const model = {
937
938 async syncViewportAfterSurfaceOpen(sequence = this._surfaceOpenSequence) {
939 if (!this.connected || !this.activeBrowserId) return;
940 + const surfaceMode = this._mode;
941 await this.waitForSurfaceViewport({ sequence });
942 if (!this.isCurrentSurfaceOpen(sequence)) {
943 return;
@@ -939,19 +945,24 @@ const model = {
945 await this.syncViewport(true, {
946 restartStream: this._mode === "canvas" && this.usesScreencastTransport(),
947 });
942 - if (this._mode !== "canvas") return;
943 - this.scheduleViewportSyncForSurface(sequence, 240);
944 - this.scheduleViewportSyncForSurface(sequence, 520);
948 + if (surfaceMode === "modal" && this.usesInteractiveTransport()) {
949 + this.scheduleViewportSyncForSurface(sequence, INTERACTIVE_VIEWPORT_SETTLE_MS, surfaceMode);
950 + return;
951 + }
952 + if (surfaceMode !== "canvas") return;
953 + this.scheduleViewportSyncForSurface(sequence, 240, surfaceMode);
954 + this.scheduleViewportSyncForSurface(sequence, 520, surfaceMode);
955 },
956
957 requestedViewerTransport() {
948 - return this.liveScreencastEnabled
949 - ? BROWSER_VIEWER_TRANSPORT_SCREENCAST
950 - : BROWSER_VIEWER_TRANSPORT_SNAPSHOT;
958 + return BROWSER_VIEWER_TRANSPORT_INTERACTIVE;
959 },
960
961 normalizeViewerTransport(value = "") {
962 const normalized = String(value || "").trim().toLowerCase().replace("-", "_");
963 + if (normalized === BROWSER_VIEWER_TRANSPORT_INTERACTIVE) {
964 + return BROWSER_VIEWER_TRANSPORT_INTERACTIVE;
965 + }
966 if (normalized === BROWSER_VIEWER_TRANSPORT_SCREENCAST) {
967 return BROWSER_VIEWER_TRANSPORT_SCREENCAST;
968 }
@@ -974,6 +985,101 @@ const model = {
985 return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_SCREENCAST;
986 },
987
988 + usesInteractiveTransport() {
989 + return this.viewerTransport === BROWSER_VIEWER_TRANSPORT_INTERACTIVE
990 + && Boolean(this.interactiveViewUrl);
991 + },
992 +
993 + isInteractiveSurface(stage = null) {
994 + return this.usesInteractiveTransport() && stage === this._stageElement;
995 + },
996 +
997 + prepareInteractiveViewFrame(frame = null) {
998 + const target = frame || this._stageElement?.querySelector?.(".browser-interactive-frame");
999 + const remoteWindow = target?.contentWindow;
1000 + if (!remoteWindow) return false;
1001 + try {
1002 + const remoteDocument = target.contentDocument || remoteWindow.document;
1003 + if (!remoteDocument) return false;
1004 + if (!remoteDocument.getElementById("a0-xpra-browser-frame-css")) {
1005 + const style = remoteDocument.createElement("style");
1006 + style.id = "a0-xpra-browser-frame-css";
1007 + style.textContent = `
1008 + #shadow_pointer {
1009 + display: none !important;
1010 + visibility: hidden !important;
1011 + opacity: 0 !important;
1012 + }
1013 + .window canvas,
1014 + .undecorated canvas {
1015 + display: block !important;
1016 + margin: 0 !important;
1017 + }
1018 + `;
1019 + remoteDocument.head?.appendChild(style);
1020 + }
1021 +
1022 + const normalizeWindows = () => {
1023 + const windows = Object.values(remoteWindow.client?.id_to_window || {});
1024 + for (const xpraWindow of windows) {
1025 + xpraWindow.resizable = false;
1026 + xpraWindow.decorations = false;
1027 + xpraWindow.decorated = false;
1028 + xpraWindow.metadata = { ...(xpraWindow.metadata || {}), decorations: false };
1029 + xpraWindow._set_decorated?.(false);
1030 + xpraWindow.configure_border_class?.();
1031 + xpraWindow.leftoffset = 0;
1032 + xpraWindow.rightoffset = 0;
1033 + xpraWindow.topoffset = 0;
1034 + xpraWindow.bottomoffset = 0;
1035 + xpraWindow.updateCSSGeometry?.();
1036 + }
1037 + return windows.length > 0;
1038 + };
1039 +
1040 + const screen = remoteDocument.querySelector?.("#screen");
1041 + if (screen && !remoteWindow.__a0BrowserFrameObserver && remoteWindow.MutationObserver) {
1042 + const observer = new remoteWindow.MutationObserver(normalizeWindows);
1043 + observer.observe(screen, { childList: true });
1044 + remoteWindow.__a0BrowserFrameObserver = observer;
1045 + }
1046 + return normalizeWindows();
1047 + } catch {
1048 + return false;
1049 + }
1050 + },
1051 +
1052 + syncInteractiveViewSize() {
1053 + if (!this.usesInteractiveTransport()) return;
1054 + const frame = this._stageElement?.querySelector?.(".browser-interactive-frame");
1055 + try {
1056 + this.prepareInteractiveViewFrame(frame);
1057 + frame?.contentWindow?.client?._screen_resized?.();
1058 + } catch {}
1059 + },
1060 +
1061 + applyViewer(data = {}) {
1062 + if (data?.viewer_transport) {
1063 + this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1064 + }
1065 + if (Object.prototype.hasOwnProperty.call(data || {}, "interactive_view")) {
1066 + const viewer = data.interactive_view;
1067 + this.interactiveViewUrl = viewer?.available && viewer?.url ? String(viewer.url) : "";
1068 + this.viewerFallbackReason = String(data.viewer_fallback_reason || viewer?.error || "");
1069 + }
1070 + if (this.viewerTransport !== BROWSER_VIEWER_TRANSPORT_INTERACTIVE) {
1071 + this.interactiveViewUrl = "";
1072 + }
1073 + },
1074 +
1075 + onInteractiveViewLoad() {
1076 + if (!this.usesInteractiveTransport()) return;
1077 + this.prepareInteractiveViewFrame();
1078 + this.switchingBrowserId = null;
1079 + this._surfaceSwitching = false;
1080 + this.queueViewportSync(true);
1081 + },
1082 +
1083 supportsBinaryFrames() {
1084 return BROWSER_BINARY_FRAME_REQUESTS_ENABLED && BROWSER_BINARY_PAYLOADS_SUPPORTED;
1085 },
@@ -1003,9 +1109,9 @@ const model = {
1109 return { width, height };
1110 },
1111
1006 - scheduleViewportSyncForSurface(sequence, delayMs = 0) {
1112 + scheduleViewportSyncForSurface(sequence, delayMs = 0, mode = this._mode) {
1113 globalThis.setTimeout?.(() => {
1008 - if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== "canvas") {
1114 + if (!this.isCurrentSurfaceOpen(sequence) || this._mode !== mode) {
1115 return;
1116 }
1117 this.queueViewportSync(true);
@@ -1093,7 +1199,8 @@ const model = {
1199 replaceAll: Boolean(data.all_browsers),
1200 replaceContext: !data.all_browsers,
1201 });
1096 - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1202 + this.applyViewer(data);
1203 + this._subscribedViewerTransport = this.viewerTransport;
1204 this.setActiveBrowserId(
1205 data.active_browser_id || requestedBrowserId || this.activeBrowserId || null,
1206 data.active_browser_context_id || contextId,
@@ -1108,9 +1215,7 @@ const model = {
1215 const frameHandler = ({ data }) => {
1216 if (data?.context_id !== this.contextId) return;
1217 if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
1111 - if (data?.viewer_transport) {
1112 - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1113 - }
1218 + this.applyViewer(data);
1219 this.applyTabScope(data);
1220 const incomingContextId = this.normalizeContextId(data.context_id || this.contextId);
1221 const incomingBrowserId = this.normalizeBrowserId(data.browser_id || data.state?.id);
@@ -1180,9 +1285,7 @@ const model = {
1285 const stateHandler = ({ data }) => {
1286 if (data?.context_id !== this.contextId) return;
1287 if (data?.viewer_id && data.viewer_id !== this._viewerToken) return;
1183 - if (data?.viewer_transport) {
1184 - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1185 - }
1288 + this.applyViewer(data);
1289 this.applyTabScope(data);
1290 const commandContextId = this.normalizeContextId(data.active_browser_context_id || data.context_id || this.contextId);
1291 if (Array.isArray(data.browsers)) {
@@ -1384,7 +1487,7 @@ const model = {
1487 },
1488
1489 hasFrame() {
1387 - return Boolean(this.frameSrc || this.frameCanvasReady);
1490 + return Boolean(this.interactiveViewUrl || this.frameSrc || this.frameCanvasReady);
1491 },
1492
1493 paintFrameBitmap(bitmap) {
@@ -1419,6 +1522,10 @@ const model = {
1522 },
1523
1524 frameElement() {
1525 + if (this.usesInteractiveTransport()) {
1526 + const iframe = this._stageElement?.querySelector?.(".browser-interactive-frame");
1527 + if (iframe) return iframe;
1528 + }
1529 if (this.frameCanvasReady) {
1530 const canvas = this.currentFrameCanvas();
1531 if (canvas) return canvas;
@@ -1467,7 +1574,7 @@ const model = {
1574 replaceAll: Boolean(data.all_browsers),
1575 replaceContext: !data.all_browsers,
1576 });
1470 - this.viewerTransport = this.normalizeViewerTransport(data.viewer_transport);
1577 + this.applyViewer(data);
1578 const result = data.result || {};
1579 const resultContextId = this.normalizeContextId(
1580 result.context_id
@@ -1510,7 +1617,12 @@ const model = {
1617 previousActiveBrowserId,
1618 previousActiveContextId,
1619 );
1513 - if ((commandName === "open" || commandName === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
1620 + const viewerTransportChanged = this._subscribedViewerTransport !== this.viewerTransport;
1621 + if (
1622 + (commandName === "open" || commandName === "close" || activeChanged || viewerTransportChanged)
1623 + && this.contextId
1624 + && this.activeBrowserId
1625 + ) {
1626 await this.connectViewer({
1627 browserId: this.activeBrowserId,
1628 contextId: this.activeBrowserContextId,
@@ -1670,6 +1782,10 @@ const model = {
1782 return this.sameBrowserTab(browser?.id, browser?.context_id, this.activeBrowserId, this.activeBrowserContextId);
1783 },
1784
1785 + isBrowserLoading(browser) {
1786 + return Boolean(browser?.loading || (this.isActiveBrowser(browser) && this.isBusy()));
1787 + },
1788 +
1789 browserTabTitle(browser) {
1790 const title = String(browser?.title || "").trim();
1791 const url = String(browser?.currentUrl || "").trim();
@@ -1831,6 +1947,7 @@ const model = {
1947 if (snapshot.state) {
1948 this.applyActiveFrameState(snapshot.state);
1949 }
1950 + if (this.usesInteractiveTransport()) return;
1951 const frameBrowserId = snapshotId || this.activeBrowserId;
1952 this.queueFrameRender(`data:${snapshot.mime || "image/jpeg"};base64,${snapshot.image}`, {
1953 browserId: frameBrowserId,
@@ -2521,13 +2638,21 @@ const model = {
2638
2639 queueViewportSync(force = false) {
2640 this.clearRenderedFrameIfViewportChanged();
2641 + if (force) {
2642 + if (this._viewportSyncTimer) {
2643 + globalThis.clearTimeout(this._viewportSyncTimer);
2644 + this._viewportSyncTimer = null;
2645 + }
2646 + void this.syncViewport(true);
2647 + return;
2648 + }
2649 if (this._viewportSyncTimer) {
2525 - globalThis.clearTimeout(this._viewportSyncTimer);
2650 + return;
2651 }
2652 this._viewportSyncTimer = globalThis.setTimeout(() => {
2653 this._viewportSyncTimer = null;
2529 - void this.syncViewport(force);
2530 - }, force ? 0 : VIEWPORT_SYNC_DEBOUNCE_MS);
2654 + void this.syncViewport(false);
2655 + }, VIEWPORT_SYNC_INTERVAL_MS);
2656 },
2657
2658 async syncViewport(force = false, options = {}) {
@@ -2542,7 +2667,7 @@ const model = {
2667 }
2668 const key = `${contextId}:${this.activeBrowserId}:${viewport.width}x${viewport.height}`;
2669 if (
2545 - (!restartStream && this._lastViewportKey === key)
2670 + (!force && !restartStream && this._lastViewportKey === key)
2671 || (
2672 !force
2673 && !restartStream
@@ -2555,11 +2680,13 @@ const model = {
2680 return;
2681 }
2682 try {
2683 + this.syncInteractiveViewSize();
2684 await websocket.emit("browser_viewer_input", {
2685 context_id: contextId,
2686 browser_id: this.activeBrowserId,
2687 viewer_id: this._viewerToken,
2688 input_type: "viewport",
2689 + viewer_transport: this.viewerTransport,
2690 width: viewport.width,
2691 height: viewport.height,
2692 restart_stream: restartStream && this.usesScreencastTransport(),
@@ -2739,6 +2866,9 @@ const model = {
2866 this._viewerToken = "";
2867 this.switchingBrowserId = null;
2868 this.viewerTransport = this.requestedViewerTransport();
2869 + this._subscribedViewerTransport = this.viewerTransport;
2870 + this.interactiveViewUrl = "";
2871 + this.viewerFallbackReason = "";
2872 this.tabScope = "per_context";
2873 this._surfaceMounted = false;
2874 this._surfaceSwitching = false;
@@ -3010,13 +3140,6 @@ const model = {
3140 return this.frameState?.currentUrl || this.address || "about:blank";
3141 },
3142
3013 - loadingMessage() {
3014 - if (this.browserInstallExpected) {
3015 - const cacheDir = this.status?.playwright?.cache_dir || "/a0/tmp/playwright";
3016 - return `Installing Chromium for the first Browser run. This can take a few minutes; future starts reuse ${cacheDir}.`;
3017 - }
3018 - return "Loading";
3019 - },
3143 };
3144
3145 export const store = createStore("browserPage", model);
tests/test_browser_agent_regressions.py
+351 -6
@@ -82,7 +82,7 @@ sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_conf
82 def anyio_backend():
83 return "asyncio"
84
85 -from helpers import ephemeral_images
85 +from helpers import ephemeral_images, virtual_desktop
86 from helpers.errors import RepairableException
87 from plugins._browser.helpers.config import (
88 build_browser_launch_config,
@@ -109,6 +109,8 @@ from plugins._browser.helpers.runtime import (
109 normalize_url,
110 )
111 import plugins._browser.helpers.runtime as browser_runtime_module
112 +from plugins._browser.helpers.interactive_view import BrowserInteractiveView
113 +import plugins._browser.helpers.interactive_view as browser_interactive_view_module
114 from plugins._browser.helpers.playwright import (
115 ensure_playwright_binary,
116 get_playwright_binary,
@@ -833,7 +835,8 @@ def test_browser_viewer_allows_slow_extension_startup():
835 assert "const BROWSER_COMMAND_TIMEOUT_MS = 45000;" in js
836 assert "? BROWSER_FIRST_INSTALL_TIMEOUT_MS" in js
837 assert ": BROWSER_SUBSCRIBE_TIMEOUT_MS" in js
836 - assert "Installing Chromium for the first Browser run" in js
838 + assert "browserInstallExpected" in js
839 + assert "Installing Chromium for the first Browser run" not in js
840
841
842 def test_browser_viewer_creates_chat_when_no_context_is_selected():
@@ -880,6 +883,20 @@ def test_browser_canvas_startup_waits_for_raw_viewport_settle():
883 assert "this.resetRenderedFrame();" in js
884
885
886 +def test_browser_interactive_modal_handoff_reconciles_after_xpra_resize():
887 + js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(
888 + encoding="utf-8"
889 + )
890 +
891 + assert "const INTERACTIVE_VIEWPORT_SETTLE_MS = 320;" in js
892 + assert "const surfaceMode = this._mode;" in js
893 + assert 'surfaceMode === "modal" && this.usesInteractiveTransport()' in js
894 + assert "this.scheduleViewportSyncForSurface(sequence, INTERACTIVE_VIEWPORT_SETTLE_MS, surfaceMode);" in js
895 + assert "scheduleViewportSyncForSurface(sequence, delayMs = 0, mode = this._mode)" in js
896 + assert "this._mode !== mode" in js
897 + assert "(!force && !restartStream && this._lastViewportKey === key)" in js
898 +
899 +
900 def test_browser_surface_handoffs_keep_existing_frame_until_replacement_arrives():
901 js = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js").read_text(
902 encoding="utf-8"
@@ -1393,6 +1410,12 @@ def test_browser_viewer_uses_tabs_for_session_switching():
1410 assert ':key="$store.browserPage.browserTabKey(browser)"' in main_html
1411 assert "browser.context_id" in main_html
1412 assert ':title="$store.browserPage.browserTabTooltip(browser)"' in main_html
1413 + assert ':aria-busy="$store.browserPage.isBrowserLoading(browser).toString()"' in main_html
1414 + assert 'class="browser-tab-loading spinning"' in main_html
1415 + assert 'x-show="$store.browserPage.isBrowserLoading(browser)"' in main_html
1416 + assert "<span x-text=\"$store.browserPage.loadingMessage()\">Loading</span>" not in main_html
1417 + assert "isBrowserLoading(browser)" in browser_store
1418 + assert "loadingMessage()" not in browser_store
1419 assert "browser-tab-context" not in main_html
1420 assert 'handleSelectedContextChange($store.chats?.selected)' in main_html
1421 assert "activeBrowserContextId" in browser_store
@@ -1445,7 +1468,7 @@ def test_browser_tabs_close_without_confirmation_or_busy_lock():
1468 assert "_commandInFlightCount" in browser_store
1469
1470
1448 -def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1471 +def test_browser_viewer_defaults_to_interactive_with_screencast_and_snapshot_fallbacks():
1472 ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(
1473 encoding="utf-8"
1474 )
@@ -1465,6 +1488,9 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1488 assert 'runtime.call("screenshot"' in ws_browser
1489 assert 'VIEWER_TRANSPORT_SNAPSHOT = "snapshot"' in ws_browser
1490 assert 'VIEWER_TRANSPORT_SCREENCAST = "screencast"' in ws_browser
1491 + assert 'VIEWER_TRANSPORT_INTERACTIVE = "interactive"' in ws_browser
1492 + assert "async def _effective_viewer(" in ws_browser
1493 + assert "return VIEWER_TRANSPORT_SCREENCAST, viewer" in ws_browser
1494 assert "def _viewer_transport(data: dict[str, Any])" in ws_browser
1495 assert "return VIEWER_TRANSPORT_SNAPSHOT" in ws_browser
1496 assert "self._stream_state" in ws_browser
@@ -1510,8 +1536,20 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1536 assert "return await globalThis.createImageBitmap(blob);" in browser_store
1537 assert 'const BROWSER_VIEWER_TRANSPORT_SNAPSHOT = "snapshot";' in browser_store
1538 assert 'const BROWSER_VIEWER_TRANSPORT_SCREENCAST = "screencast";' in browser_store
1513 - assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_SCREENCAST" in browser_store
1514 - assert "liveScreencastEnabled: true" in browser_store
1539 + assert 'const BROWSER_VIEWER_TRANSPORT_INTERACTIVE = "interactive";' in browser_store
1540 + assert "const VIEWPORT_SYNC_INTERVAL_MS = 50;" in browser_store
1541 + assert "viewerTransport: BROWSER_VIEWER_TRANSPORT_INTERACTIVE" in browser_store
1542 + assert "return BROWSER_VIEWER_TRANSPORT_INTERACTIVE;" in browser_store
1543 + assert "usesInteractiveTransport()" in browser_store
1544 + assert "isInteractiveSurface(stage = null)" in browser_store
1545 + assert "prepareInteractiveViewFrame(frame = null)" in browser_store
1546 + assert 'style.id = "a0-xpra-browser-frame-css";' in browser_store
1547 + assert "#shadow_pointer" in browser_store
1548 + assert "xpraWindow._set_decorated?.(false);" in browser_store
1549 + assert "xpraWindow.updateCSSGeometry?.();" in browser_store
1550 + assert "syncInteractiveViewSize()" in browser_store
1551 + assert "frame?.contentWindow?.client?._screen_resized?.();" in browser_store
1552 + assert "applyViewer(data = {})" in browser_store
1553 assert "requestedViewerTransport()" in browser_store
1554 assert "normalizeViewerTransport(value = \"\")" in browser_store
1555 assert "usesScreencastTransport()" in browser_store
@@ -1583,6 +1621,11 @@ def test_browser_viewer_defaults_to_live_screencast_with_snapshot_fallback():
1621 assert "canvas_wheel_screenshot" not in ws_browser
1622 assert "surface_mode: this._mode" not in browser_store
1623 assert '<canvas class="browser-frame browser-frame-canvas"' in main_html
1624 + assert '<iframe class="browser-interactive-frame"' in main_html
1625 + assert 'x-if="$store.browserPage.isInteractiveSurface($el.parentElement)"' in main_html
1626 + assert 'aria-label="Browser viewport"' in main_html
1627 + assert 'title="Interactive Browser viewport"' not in main_html
1628 + assert 'allow="clipboard-read; clipboard-write"' in main_html
1629 assert 'x-init="$store.browserPage.attachFrameCanvas($el)"' in main_html
1630 assert '<img class="browser-frame browser-frame-image"' in main_html
1631 assert "$store.browserPage.hasFrame()" in main_html
@@ -2207,7 +2250,12 @@ def test_browser_docker_installs_full_chromium_to_tmp_cache():
2250 ).read_text(encoding="utf-8")
2251 assert '"headless": not bool(browser_display)' in runtime
2252 assert 'launch_kwargs["env"] = {**os.environ, "DISPLAY": browser_display}' in runtime
2253 + assert 'launch_kwargs["no_viewport"] = True' in runtime
2254 + assert '"windowState": "fullscreen"' not in runtime
2255 + assert "self.interactive_view.ensure_display()" in runtime
2256 assert " xvfb \\" in install_additional
2257 + assert "XPRA_PACKAGES=(xpra xpra-x11 xpra-html5)" in install_additional
2258 + assert " xdotool \\" in install_additional
2259
2260
2261 def test_browser_startup_migration_prepares_current_playwright_binary():
@@ -2222,10 +2270,248 @@ def test_browser_startup_migration_prepares_current_playwright_binary():
2270 ).read_text(encoding="utf-8")
2271
2272 assert "class BrowserPlaywrightCacheMigration(Extension)" in extension
2273 + assert "virtual_desktop_routes.install_route_hooks()" in extension
2274 assert "hooks.prepare_playwright_cache()" in extension
2275 assert "PrintStyle.warning" in extension
2276
2277
2278 +def test_browser_interactive_views_use_isolated_loopback_sessions(monkeypatch, tmp_path):
2279 + class FakeProcess:
2280 + def __init__(self):
2281 + self.returncode = None
2282 +
2283 + def poll(self):
2284 + return self.returncode
2285 +
2286 + def terminate(self):
2287 + self.returncode = 0
2288 +
2289 + def wait(self, timeout=None):
2290 + return self.returncode
2291 +
2292 + next_display = iter((71, 72))
2293 + next_port = iter((44001, 44002))
2294 +
2295 + def fake_ensure_display(view):
2296 + view.display = next(next_display)
2297 + view._xvfb = FakeProcess()
2298 + return view.display_name
2299 +
2300 + def fake_resize(view, width, height):
2301 + view.width, view.height = virtual_desktop.normalize_size(width, height)
2302 + return {"ok": True, "width": view.width, "height": view.height}
2303 +
2304 + def fake_start_xpra(view, xpra):
2305 + assert xpra == "/usr/bin/xpra"
2306 + view.port = next(next_port)
2307 + view._xpra = FakeProcess()
2308 +
2309 + monkeypatch.setattr(
2310 + browser_interactive_view_module.files,
2311 + "get_abs_path",
2312 + lambda *parts: str(tmp_path.joinpath(*parts)),
2313 + )
2314 + monkeypatch.setattr(BrowserInteractiveView, "ensure_display", fake_ensure_display)
2315 + monkeypatch.setattr(BrowserInteractiveView, "resize", fake_resize)
2316 + monkeypatch.setattr(BrowserInteractiveView, "_start_xpra", fake_start_xpra)
2317 + monkeypatch.setattr(
2318 + browser_interactive_view_module,
2319 + "collect_status",
2320 + lambda: {
2321 + "available": True,
2322 + "missing": [],
2323 + "binaries": {"xpra": "/usr/bin/xpra"},
2324 + "xpra_html_root": "/usr/share/xpra/www",
2325 + },
2326 + )
2327 +
2328 + views = [BrowserInteractiveView("ctx-a"), BrowserInteractiveView("ctx-b")]
2329 + try:
2330 + viewers = [view.ensure_viewer(1200, 700) for view in views]
2331 +
2332 + assert views[0].token != views[1].token
2333 + assert views[0].display != views[1].display
2334 + assert all(viewer["available"] for viewer in viewers)
2335 + for view, viewer in zip(views, viewers, strict=True):
2336 + endpoint = virtual_desktop.proxy_for_token(view.token)
2337 + assert endpoint is not None
2338 + assert endpoint.host == "127.0.0.1"
2339 + assert endpoint.owner == "browser"
2340 + assert endpoint.port == view.port
2341 + assert viewer["token"] == view.token
2342 + assert "file_transfer=false" in viewer["url"]
2343 + assert "printing=false" in viewer["url"]
2344 + finally:
2345 + for view in views:
2346 + view.close()
2347 +
2348 + assert all(virtual_desktop.proxy_for_token(view.token) is None for view in views)
2349 +
2350 +
2351 +@pytest.mark.anyio
2352 +async def test_browser_interactive_viewer_reuses_the_automated_page():
2353 + class FakePage:
2354 + viewport_size = {"width": 1024, "height": 768}
2355 +
2356 + class FakeInteractiveView:
2357 + def __init__(self):
2358 + self.ensure_calls = []
2359 +
2360 + def ensure_viewer(self, width, height):
2361 + self.ensure_calls.append((width, height))
2362 + return {
2363 + "available": True,
2364 + "token": "browser-token",
2365 + "url": "/desktop/session/browser-token/",
2366 + "width": width,
2367 + "height": height,
2368 + }
2369 +
2370 + page = FakePage()
2371 + interactive_view = FakeInteractiveView()
2372 + viewport_calls = []
2373 + stopped = []
2374 + core = _BrowserRuntimeCore("ctx")
2375 + core.context = object()
2376 + core.pages[7] = BrowserPage(id=7, page=page)
2377 + core.interactive_view = interactive_view
2378 +
2379 + async def ensure_started():
2380 + return None
2381 +
2382 + async def stop_screencasts(browser_id):
2383 + stopped.append(browser_id)
2384 +
2385 + async def set_viewport(browser_id, width, height, **kwargs):
2386 + viewport_calls.append((browser_id, width, height, kwargs))
2387 + return {"state": {"id": browser_id}, "viewport": {"width": width, "height": height}}
2388 +
2389 + core.ensure_started = ensure_started
2390 + core._stop_screencasts_for_browser = stop_screencasts
2391 + core.set_viewport = set_viewport
2392 +
2393 + result = await core.interactive_viewer(7, width=1200, height=700)
2394 +
2395 + assert core.pages[7].page is page
2396 + assert interactive_view.ensure_calls == [(1200, 700)]
2397 + assert stopped == [7]
2398 + assert viewport_calls == [(7, 1200, 700, {"resize_interactive": True})]
2399 + assert core.last_interacted_browser_id == 7
2400 + assert result["url"] == "/desktop/session/browser-token/"
2401 +
2402 +
2403 +@pytest.mark.anyio
2404 +async def test_browser_interactive_window_clips_chrome_without_fullscreen():
2405 + commands = []
2406 +
2407 + class FakeSession:
2408 + async def send(self, method, params=None):
2409 + commands.append((method, params))
2410 + if method == "Browser.getWindowForTarget":
2411 + return {"windowId": 7}
2412 + if method == "Browser.getWindowBounds":
2413 + return {
2414 + "bounds": {
2415 + "windowState": "normal",
2416 + "left": 0,
2417 + "top": 0,
2418 + "width": 1024,
2419 + "height": 768,
2420 + }
2421 + }
2422 + return {}
2423 +
2424 + async def detach(self):
2425 + commands.append(("detach", None))
2426 +
2427 + class FakeContext:
2428 + async def new_cdp_session(self, page):
2429 + assert page is fake_page
2430 + return FakeSession()
2431 +
2432 + class FakeInteractiveView:
2433 + display = 0
2434 + width = 1200
2435 + height = 700
2436 +
2437 + class FakePage:
2438 + async def evaluate(self, script, **kwargs):
2439 + commands.append(("evaluate", script))
2440 + return 87
2441 +
2442 + fake_page = FakePage()
2443 + core = _BrowserRuntimeCore("ctx")
2444 + core.context = FakeContext()
2445 + core.interactive_view = FakeInteractiveView()
2446 +
2447 + await core._fit_browser_window(fake_page)
2448 + core.interactive_view.width = 1280
2449 + core.interactive_view.height = 720
2450 + await core._fit_browser_window(fake_page)
2451 + await core._reset_browser_window_session()
2452 +
2453 + assert commands == [
2454 + ("Browser.getWindowForTarget", None),
2455 + ("Browser.getWindowBounds", {"windowId": 7}),
2456 + (
2457 + "evaluate",
2458 + "() => Math.max(0, globalThis.outerHeight - globalThis.innerHeight)",
2459 + ),
2460 + (
2461 + "Browser.setWindowBounds",
2462 + {
2463 + "windowId": 7,
2464 + "bounds": {
2465 + "windowState": "normal",
2466 + "left": 0,
2467 + "top": -87,
2468 + "width": 1200,
2469 + "height": 787,
2470 + },
2471 + },
2472 + ),
2473 + (
2474 + "Browser.setWindowBounds",
2475 + {
2476 + "windowId": 7,
2477 + "bounds": {
2478 + "windowState": "normal",
2479 + "left": 0,
2480 + "top": -87,
2481 + "width": 1280,
2482 + "height": 807,
2483 + },
2484 + },
2485 + ),
2486 + ("detach", None),
2487 + ]
2488 +
2489 +
2490 +@pytest.mark.anyio
2491 +async def test_browser_viewer_falls_back_when_interactive_runtime_is_unavailable():
2492 + calls = []
2493 +
2494 + class FakeRuntime:
2495 + async def call(self, method, *args, **kwargs):
2496 + calls.append((method, args, kwargs))
2497 + return {"available": False, "error": "xpra unavailable"}
2498 +
2499 + handler = ws_browser_module.WsBrowser(SimpleNamespace(), threading.RLock(), manager=None)
2500 + transport, viewer = await handler._effective_viewer(
2501 + FakeRuntime(),
2502 + 7,
2503 + {
2504 + "viewer_transport": "interactive",
2505 + "viewport_width": 1200,
2506 + "viewport_height": 700,
2507 + },
2508 + )
2509 +
2510 + assert transport == ws_browser_module.VIEWER_TRANSPORT_SCREENCAST
2511 + assert viewer == {"available": False, "error": "xpra unavailable"}
2512 + assert calls == [("interactive_viewer", (7,), {"width": 1200, "height": 700})]
2513 +
2514 +
2515 def test_browser_runtime_removes_stale_profile_singletons(monkeypatch, tmp_path):
2516 monkeypatch.setattr(
2517 browser_runtime_module.files,
@@ -3245,6 +3531,7 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
3531 "width": 1280,
3532 "height": 720,
3533 "restart_stream": True,
3534 + "viewer_transport": "interactive",
3535 },
3536 "sid-1",
3537 )
@@ -3254,10 +3541,68 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
3541 "snapshot": None,
3542 }
3543 assert calls == [
3257 - ("set_viewport", (7, 1280, 720), {"restart_screencast": True})
3544 + (
3545 + "set_viewport",
3546 + (7, 1280, 720),
3547 + {
3548 + "restart_screencast": True,
3549 + "resize_interactive": True,
3550 + "include_state": False,
3551 + },
3552 + )
3553 ]
3554
3555
3556 +@pytest.mark.anyio
3557 +async def test_browser_interactive_resize_uses_native_window_viewport():
3558 + fitted = []
3559 +
3560 + class FakePage:
3561 + viewport_size = None
3562 +
3563 + async def set_viewport_size(self, viewport):
3564 + raise AssertionError("Interactive native viewport must not enable emulation")
3565 +
3566 + class FakeInteractiveView:
3567 + display = 0
3568 + width = 1024
3569 + height = 768
3570 +
3571 + def resize(self, width, height):
3572 + self.width = width
3573 + self.height = height
3574 + return {"ok": True, "width": width, "height": height}
3575 +
3576 + page = FakePage()
3577 + core = _BrowserRuntimeCore("ctx")
3578 + core.context = object()
3579 + core.pages[7] = browser_runtime_module.BrowserPage(id=7, page=page)
3580 + core.interactive_view = FakeInteractiveView()
3581 +
3582 + async def fake_fit(fitted_page):
3583 + fitted.append(fitted_page)
3584 +
3585 + async def fake_state(browser_id):
3586 + return {"id": browser_id}
3587 +
3588 + core._fit_browser_window = fake_fit
3589 + core._state = fake_state
3590 +
3591 + result = await core.set_viewport(
3592 + 7,
3593 + 1280,
3594 + 720,
3595 + resize_interactive=True,
3596 + include_state=False,
3597 + )
3598 +
3599 + assert result == {
3600 + "state": None,
3601 + "viewport": {"width": 1280, "height": 720},
3602 + }
3603 + assert fitted == [page]
3604 +
3605 +
3606 @pytest.mark.anyio
3607 async def test_browser_runtime_restarts_screencast_without_resizing_same_viewport():
3608 viewport_calls = []