Harden browser multi-tab focus handling
Alessandro committed
May 2, 2026 at 15:49 UTC
12b96ae41e072bd1f0be0d1216c6b857439d04b6
5 files changed
+177
-65
plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
+22
-13
@@ -14,7 +14,7 @@ import {
14
15
const BROWSER_MODAL = "/plugins/_browser/webui/main.html";
16
const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;
17
-const autoOpenedBrowsers = new Set();
17
+const syncedBrowserCanvases = new Set();
18
19
export default async function registerBrowserToolHandler(extData) {
20
if (extData?.tool_name === "browser") {
@@ -86,8 +86,17 @@ function isFreshToolMessage(timestamp) {
86
return Math.abs(Date.now() - messageMs) <= AUTO_OPEN_WINDOW_MS;
87
}
88
89
-// Allowlist: only these actions cause the viewer to follow. Background work
90
-// (evaluate, click, type, key_chord, mouse, multi, ...) does not steal focus.
89
+function isBrowserCanvasAlreadyOpen() {
90
+ return Boolean(
91
+ rightCanvasStore?.isOpen
92
+ && rightCanvasStore?.activeSurfaceId === "browser"
93
+ && !rightCanvasStore?.isMobileMode,
94
+ );
95
+}
96
+
97
+// Allowlist: only these actions sync an already-open viewer to the target tab.
98
+// Background work (evaluate, click, type, key_chord, mouse, multi, ...) does
99
+// not steal focus.
100
const FOCUS_ACTIONS = new Set([
101
"open",
102
"navigate",
@@ -97,28 +106,28 @@ const FOCUS_ACTIONS = new Set([
106
"focus",
107
]);
108
100
-function shouldAutoOpenBrowser(args, result) {
109
+function shouldSyncOpenBrowserCanvas(args, result) {
110
+ if (!isBrowserCanvasAlreadyOpen()) return false;
111
if (!isFreshToolMessage(args?.timestamp)) return false;
112
const action = String(args?.kvps?.action || "").trim().toLowerCase().replace("-", "_");
113
if (!FOCUS_ACTIONS.has(action)) return false;
114
return Boolean(browserIdFromResult(result, args?.kvps || {}));
115
}
116
107
-function autoOpenBrowserCanvas(args, result) {
108
- if (!shouldAutoOpenBrowser(args, result)) return;
117
+function syncOpenBrowserCanvas(args, result) {
118
+ if (!shouldSyncOpenBrowserCanvas(args, result)) return;
119
const kvps = args?.kvps || {};
120
const browserId = browserIdFromResult(result, kvps);
121
const key = `${args.id || ""}:${kvps.action || ""}:${browserId || ""}:${result.currentUrl || result.state?.currentUrl || kvps.url || ""}`;
112
- const persistedKey = `a0.browser.autoOpened.${key}`;
113
- if (autoOpenedBrowsers.has(key) || sessionStorage.getItem(persistedKey)) return;
114
- autoOpenedBrowsers.add(key);
115
- sessionStorage.setItem(persistedKey, "1");
122
+ if (syncedBrowserCanvases.has(key)) return;
123
+ syncedBrowserCanvases.add(key);
124
requestAnimationFrame(async () => {
125
+ if (!isBrowserCanvasAlreadyOpen()) return;
126
if (!(await browserAllowsToolAutofocus())) return;
118
- void openBrowserCanvas({
127
+ void rightCanvasStore.open("browser", {
128
browserId,
129
contextId: browserContextIdFromResult(result, kvps),
121
- source: "tool",
130
+ source: "tool-sync",
131
});
132
});
133
}
@@ -178,6 +187,6 @@ function drawBrowserTool({
187
actionButtons: actionButtons.filter(Boolean),
188
log: args,
189
});
181
- autoOpenBrowserCanvas(args, browserResult);
190
+ syncOpenBrowserCanvas(args, browserResult);
191
return result;
192
}
plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
+25
-25
@@ -2,18 +2,18 @@ import { store as rightCanvasStore } from "/components/canvas/right-canvas-store
2
import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";
3
4
const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;
5
-const BROWSER_MODAL = "/plugins/_browser/webui/main.html";
6
-const autoOpenedBrowsers = new Set();
5
+const syncedBrowserCanvases = new Set();
6
8
-export default async function autoOpenBrowserResults(context) {
7
+export default async function syncBrowserResultsIntoOpenCanvas(context) {
8
if (!context?.results?.length || context.historyEmpty) return;
9
+ if (!isBrowserCanvasAlreadyOpen()) return;
10
11
for (const { args } of context.results) {
12
const payload = getToolResultPayload(args);
13
if (getToolName(payload) !== "browser") continue;
14
15
const result = parseMaybeJson(payload.tool_result) || {};
16
- if (!shouldAutoOpen(args, payload, result)) continue;
16
+ if (!shouldSyncOpenBrowserCanvas(args, payload, result)) continue;
17
18
const browserId = getBrowserId(payload, result);
19
const contextId = getBrowserContextId(payload, result);
@@ -22,12 +22,13 @@ export default async function autoOpenBrowserResults(context) {
22
browserId || "",
23
result.currentUrl || result.state?.currentUrl || payload.url || "",
24
].join(":");
25
- const persistedKey = `a0.browser.autoOpened.${key}`;
25
+ const persistedKey = `a0.browser.synced.${key}`;
26
if (hasOpened(key, persistedKey)) continue;
27
28
requestAnimationFrame(async () => {
29
+ if (!isBrowserCanvasAlreadyOpen()) return;
30
if (!(await browserAllowsToolAutofocus())) return;
30
- void openBrowserCanvas({ browserId, contextId, source: "tool-result" });
31
+ void syncOpenBrowserCanvas({ browserId, contextId, source: "tool-result-sync" });
32
});
33
}
34
}
@@ -79,9 +80,9 @@ function parseMaybeJson(value) {
80
}
81
}
82
82
-// Actions that should bring the viewer to the targeted tab. Everything else
83
-// (read, click, type, evaluate, key_chord, mouse, multi, ...) leaves the
84
-// viewer where it is so cross-tab work doesn't steal user focus.
83
+// Actions that should sync an already-open viewer to the targeted tab.
84
+// Everything else (read, click, type, evaluate, key_chord, mouse, multi, ...)
85
+// leaves the viewer where it is so cross-tab work doesn't steal user focus.
86
const FOCUS_ACTIONS = new Set([
87
"open",
88
"navigate",
@@ -91,7 +92,8 @@ const FOCUS_ACTIONS = new Set([
92
"focus",
93
]);
94
94
-function shouldAutoOpen(args = {}, payload = {}, result = {}) {
95
+function shouldSyncOpenBrowserCanvas(args = {}, payload = {}, result = {}) {
96
+ if (!isBrowserCanvasAlreadyOpen()) return false;
97
if (!isFresh(args.timestamp, payload.last_modified || result.last_modified)) return false;
98
const action = String(payload.action || "").trim().toLowerCase().replace("-", "_");
99
if (!FOCUS_ACTIONS.has(action)) return false;
@@ -139,32 +141,30 @@ function toMs(value) {
141
}
142
143
function hasOpened(key, persistedKey) {
142
- if (autoOpenedBrowsers.has(key)) return true;
143
- autoOpenedBrowsers.add(key);
144
+ if (syncedBrowserCanvases.has(key)) return true;
145
+ syncedBrowserCanvases.add(key);
146
147
try {
148
if (sessionStorage.getItem(persistedKey)) return true;
149
sessionStorage.setItem(persistedKey, "1");
150
} catch {
149
- // Best-effort persistence; the in-memory guard still prevents repeat opens.
151
+ // Best-effort persistence; the in-memory guard still prevents repeat syncs.
152
}
153
154
return false;
155
}
156
155
-async function openBrowserCanvas(payload = {}) {
156
- if (rightCanvasStore?.open) {
157
- await rightCanvasStore.open("browser", payload);
158
- return;
159
- }
157
+async function syncOpenBrowserCanvas(payload = {}) {
158
+ if (!isBrowserCanvasAlreadyOpen()) return;
159
+ await rightCanvasStore.open("browser", payload);
160
+}
161
161
- if (window.ensureModalOpen) {
162
- await window.ensureModalOpen(BROWSER_MODAL);
163
- return;
164
- }
165
- if (window.openModal) {
166
- await window.openModal(BROWSER_MODAL);
167
- }
162
+function isBrowserCanvasAlreadyOpen() {
163
+ return Boolean(
164
+ rightCanvasStore?.isOpen
165
+ && rightCanvasStore?.activeSurfaceId === "browser"
166
+ && !rightCanvasStore?.isMobileMode,
167
+ );
168
}
169
170
async function browserAllowsToolAutofocus() {
plugins/_browser/helpers/runtime.py
+64
-21
@@ -369,6 +369,49 @@ class _BrowserRuntimeCore:
369
if current is None or current == resolved_id:
370
self.last_interacted_browser_id = int(resolved_id)
371
372
+ def _background_focus_target(
373
+ self,
374
+ previous_focus: int | None,
375
+ fallback_id: int,
376
+ ) -> int | None:
377
+ if previous_focus in self.pages:
378
+ return int(previous_focus)
379
+ if fallback_id in self.pages:
380
+ return int(fallback_id)
381
+ return next(iter(sorted(self.pages)), None)
382
+
383
+ def _normalize_modifiers(self, modifiers: list[str] | str | None) -> list[str] | None:
384
+ if modifiers is None:
385
+ return None
386
+ if isinstance(modifiers, str):
387
+ raw = [modifiers]
388
+ elif isinstance(modifiers, list):
389
+ raw = modifiers
390
+ else:
391
+ raise ValueError("modifiers must be a string or list")
392
+ normalized = [str(modifier).strip() for modifier in raw if str(modifier).strip()]
393
+ if not normalized:
394
+ return None
395
+ bad = set(normalized) - self._VALID_MODIFIERS
396
+ if bad:
397
+ raise ValueError(
398
+ f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
399
+ )
400
+ return normalized
401
+
402
+ @staticmethod
403
+ def _multi_group_key(call: dict[str, Any]) -> Any:
404
+ value = call.get("browser_id")
405
+ if value is None or str(value).strip() == "":
406
+ return None
407
+ raw = str(value).strip()
408
+ if raw.startswith("browser-"):
409
+ raw = raw.split("-", 1)[1]
410
+ try:
411
+ return int(raw)
412
+ except ValueError:
413
+ return raw
414
+
415
@property
416
def profile_dir(self) -> Path:
417
return Path(files.get_abs_path("tmp/browser/sessions", self.safe_context_id))
@@ -560,7 +603,7 @@ class _BrowserRuntimeCore:
603
for idx, call in enumerate(calls):
604
if not isinstance(call, dict):
605
raise ValueError(f"calls[{idx}] is not an object")
563
- key = call.get("browser_id")
606
+ key = self._multi_group_key(call)
607
groups.setdefault(key, []).append((idx, call))
608
609
results: list[dict[str, Any] | None] = [None] * len(calls)
@@ -615,7 +658,7 @@ class _BrowserRuntimeCore:
658
raise ValueError("click requires ref")
659
return await self.click(
660
bid, ref,
618
- modifiers=call.get("modifiers"),
661
+ modifiers=self._normalize_modifiers(call.get("modifiers")),
662
focus_popup=call.get("focus_popup"),
663
)
664
if action == "type":
@@ -650,7 +693,7 @@ class _BrowserRuntimeCore:
693
bid, call.get("event_type") or "click",
694
float(call.get("x") or 0), float(call.get("y") or 0),
695
button=call.get("button") or "left",
653
- modifiers=call.get("modifiers"),
696
+ modifiers=self._normalize_modifiers(call.get("modifiers")),
697
)
698
if action == "close":
699
return await self.close_browser(bid)
@@ -760,9 +803,10 @@ class _BrowserRuntimeCore:
803
self,
804
browser_id: int | str | None,
805
reference_id: int | str,
763
- modifiers: list[str] | None = None,
806
+ modifiers: list[str] | str | None = None,
807
focus_popup: bool | None = None,
808
) -> dict[str, Any]:
809
+ modifiers = self._normalize_modifiers(modifiers)
810
if modifiers:
811
return await self._modifier_click(browser_id, reference_id, modifiers, focus_popup)
812
return await self._reference_action("click", browser_id, reference_id)
@@ -774,13 +818,9 @@ class _BrowserRuntimeCore:
818
modifiers: list[str],
819
focus_popup: bool | None,
820
) -> dict[str, Any]:
777
- bad = set(modifiers) - self._VALID_MODIFIERS
778
- if bad:
779
- raise ValueError(
780
- f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
781
- )
821
await self.ensure_started()
822
resolved_id = self._resolve_browser_id(browser_id)
823
+ previous_focus = self.last_interacted_browser_id
824
page = self._page(resolved_id)
825
await self._ensure_content_helper(page)
826
@@ -843,17 +883,24 @@ class _BrowserRuntimeCore:
883
waiter.cancel()
884
885
if opened_id is not None and background:
846
- self._background_popup_pages.add(opened_id)
886
if self.last_interacted_browser_id == opened_id:
848
- # Force focus back to origin tab — popup hook had promoted.
849
- self.last_interacted_browser_id = int(resolved_id)
887
+ # Force focus back to the tab that was active before the
888
+ # background click; the popup hook may have promoted.
889
+ self.last_interacted_browser_id = self._background_focus_target(
890
+ previous_focus,
891
+ resolved_id,
892
+ )
893
finally:
894
if waiter in self._pending_popups:
895
self._pending_popups.remove(waiter)
896
897
if background:
855
- # Background-mode click: explicitly keep focus on origin.
856
- self.last_interacted_browser_id = int(resolved_id)
898
+ # Background-mode click: preserve the pre-click focus even when
899
+ # the clicked tab itself was not active.
900
+ self.last_interacted_browser_id = self._background_focus_target(
901
+ previous_focus,
902
+ resolved_id,
903
+ )
904
return {
905
"action": {
906
"ref": reference_id,
@@ -1062,17 +1109,13 @@ class _BrowserRuntimeCore:
1109
x: float,
1110
y: float,
1111
button: str = "left",
1065
- modifiers: list[str] | None = None,
1112
+ modifiers: list[str] | str | None = None,
1113
) -> dict[str, Any]:
1114
event_type_lower = str(event_type or "click").lower()
1115
+ modifiers = self._normalize_modifiers(modifiers)
1116
if modifiers:
1117
if event_type_lower != "click":
1118
raise ValueError("modifiers are only valid for event_type='click'")
1071
- bad = set(modifiers) - self._VALID_MODIFIERS
1072
- if bad:
1073
- raise ValueError(
1074
- f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
1075
- )
1119
await self.ensure_started()
1120
resolved_id = self._resolve_browser_id(browser_id)
1121
page = self._page(resolved_id)
@@ -1342,7 +1385,7 @@ class _BrowserRuntimeCore:
1385
1386
async def _ensure_content_helper(self, page: Any) -> None:
1387
has_helper = await page.evaluate(
1345
- "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate)"
1388
+ "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate && globalThis.__spaceBrowserPageContent__?.boundingBoxFor)"
1389
)
1390
if has_helper:
1391
return
plugins/_browser/prompts/agent.system.tool.browser.md
+3
-1
@@ -1,9 +1,11 @@
1
### browser
2
-direct Playwright browser control with visible WebUI viewer
2
+direct Playwright browser control with optional visible WebUI viewer
3
use for web browsing, page inspection, forms, downloads, and browser-only tasks
4
state stays open per chat context
5
refs come from content as typed markers: [link 3], [button 6], [image 1], [input text 8]
6
7
+Browser tool actions must not open the right canvas automatically. Use the tool headlessly unless the user opens the Browser canvas or explicitly asks for a visible browser view; if the Browser canvas is already open, it may reflect the active page.
8
+
9
actions: open list state set_active navigate back forward reload content detail click type submit type_submit scroll evaluate key_chord mouse multi close close_all
10
common args: action browser_id url ref text selector selectors script modifiers keys include_content focus_popup event_type x y button calls
11
tests/test_browser_agent_regressions.py
+63
-5
@@ -539,12 +539,25 @@ def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():
539
assert "releaseSurfaceBindings()" in browser_store
540
assert "this.releaseSurfaceBindings();" in browser_store
541
542
- for js in (tool_handler, after_loop_handler):
543
- assert "async function openBrowserCanvas" in js
544
- assert "openBrowserModal" not in js
545
- assert js.index('await rightCanvasStore.open("browser", payload);') < js.index("if (window.ensureModalOpen)")
542
+ assert "async function openBrowserCanvas" in tool_handler
543
+ assert "openBrowserModal" not in tool_handler
544
+ assert tool_handler.index('await rightCanvasStore.open("browser", payload);') < tool_handler.index("if (window.ensureModalOpen)")
545
+ assert "function syncOpenBrowserCanvas" in tool_handler
546
+ assert "function autoOpenBrowserCanvas" not in tool_handler
547
+ assert "function shouldSyncOpenBrowserCanvas" in tool_handler
548
+ assert "function isBrowserCanvasAlreadyOpen" in tool_handler
549
+
550
+ assert "async function syncOpenBrowserCanvas" in after_loop_handler
551
+ assert "syncBrowserResultsIntoOpenCanvas" in after_loop_handler
552
+ assert "openBrowserCanvas" not in after_loop_handler
553
+ assert "BROWSER_MODAL" not in after_loop_handler
554
547
- assert "function autoOpenBrowserCanvas" in tool_handler
555
+ for js in (tool_handler, after_loop_handler):
556
+ assert "syncedBrowserCanvases" in js
557
+ assert "autoOpenedBrowsers" not in js
558
+ assert 'rightCanvasStore?.activeSurfaceId === "browser"' in js
559
+ assert "const FOCUS_ACTIONS = new Set" in js
560
+ assert "FOCUS_ACTIONS.has(action)" in js
561
562
for js in (tool_handler, after_loop_handler, register_js, browser_store, modals_js):
563
assert "globalThis.Alpine" not in js
@@ -552,6 +565,16 @@ def test_browser_entry_points_prefer_canvas_and_modal_dock_handoff():
565
assert "Alpine.store" not in js
566
567
568
+def test_browser_tool_does_not_auto_open_canvas_policy_is_documented():
569
+ prompt = (
570
+ PROJECT_ROOT / "plugins" / "_browser" / "prompts" / "agent.system.tool.browser.md"
571
+ ).read_text(encoding="utf-8")
572
+
573
+ assert "optional visible WebUI viewer" in prompt
574
+ assert "must not open the right canvas automatically" in prompt
575
+ assert "Use the tool headlessly unless the user opens the Browser canvas" in prompt
576
+
577
+
578
def test_browser_canvas_uses_plain_panel_without_debug_probe():
579
panel_html = (
580
PROJECT_ROOT
@@ -787,6 +810,14 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():
810
assert "password" in helper
811
812
813
+def test_browser_runtime_requires_current_content_helper_for_modifier_clicks():
814
+ runtime = (
815
+ PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
816
+ ).read_text(encoding="utf-8")
817
+
818
+ assert "__spaceBrowserPageContent__?.boundingBoxFor" in runtime
819
+
820
+
821
@pytest.mark.anyio
822
async def test_browser_screencast_acknowledges_and_drops_stale_frames():
823
first_image = SMALL_JPEG_10X10
@@ -1449,6 +1480,33 @@ async def test_browser_viewer_annotation_dispatches_runtime(monkeypatch):
1480
assert calls == [("annotation_target", (4, payload), {})]
1481
1482
1483
+def test_browser_runtime_normalizes_multi_group_ids_and_modifiers():
1484
+ core = _BrowserRuntimeCore("ctx")
1485
+
1486
+ assert core._multi_group_key({"browser_id": 7}) == 7
1487
+ assert core._multi_group_key({"browser_id": "7"}) == 7
1488
+ assert core._multi_group_key({"browser_id": "browser-7"}) == 7
1489
+ assert core._multi_group_key({"browser_id": ""}) is None
1490
+ assert core._normalize_modifiers("Control") == ["Control"]
1491
+ assert core._normalize_modifiers(["Control", " Shift "]) == ["Control", "Shift"]
1492
+ assert core._normalize_modifiers([]) is None
1493
+
1494
+ with pytest.raises(ValueError):
1495
+ core._normalize_modifiers("Ctrl")
1496
+
1497
+
1498
+def test_browser_runtime_background_focus_restores_previous_active_tab():
1499
+ core = _BrowserRuntimeCore("ctx")
1500
+ core.pages[1] = browser_runtime_module.BrowserPage(id=1, page=object())
1501
+ core.pages[2] = browser_runtime_module.BrowserPage(id=2, page=object())
1502
+
1503
+ assert core._background_focus_target(previous_focus=1, fallback_id=2) == 1
1504
+
1505
+ core.pages.pop(1)
1506
+
1507
+ assert core._background_focus_target(previous_focus=1, fallback_id=2) == 2
1508
+
1509
+
1510
def test_browser_cleanup_extensions_follow_extensible_path_layout():
1511
extension = __import__("helpers.extension", fromlist=["_get_extension_classes"])
1512
remove_classes = extension._get_extension_classes( # type: ignore[attr-defined]