@samitouri / QOSami_Agents_Zero-2 / commits / 5012dd31

feat(browser): multi-tab awareness + modifier-key click

- Auto-register tabs opened by site (window.open, target=_blank, ctrl-click) via context.on("page",...) with registry lock and closing-state guard. - Modifier-key click via Playwright trusted input: keyboard.down/up around mouse.click for coord-based path; locator.click(modifiers=...) selector fallback for off-screen / hidden elements. Chrome focus rule: ctrl/meta-click keeps focus on origin tab; override via focus_popup arg. - key_chord action: presses keys in order, releases in reverse; guarantees release on exception. Supports Ctrl+A/C/V style chords. - mouse modifiers click-only (raises ValueError for non-click events). - list(include_content=true) bulk read across all tabs in parallel via asyncio.gather (was sequential). - multi action: batched sub-calls. Different browser_id groups run concurrently; same browser_id sequentially. Returns array of {ok, result|error} matching input order. Lets the agent fan out reads or coordinated mutations across tabs in one tool call. - Cross-tab work no longer steals viewer focus. last_interacted_browser_id promotes only on open / set_active / same-tab work / Chrome popup rule. WebUI auto-open allowlist tightened to open|navigate|set_active so background actions don't drag the viewer. - New set_active action for explicit focus switch. - JS helper bumps VERSION to force re-injection on cached pages; exports boundingBoxFor returning {x,y,w,h,selector} for the trusted-input modifier-click paths. Backwards-compatible: every new arg is optional with safe defaults. No removed actions; existing call shapes preserved. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

TerminallyLazy committed Apr 29, 2026 at 06:37 UTC 5012dd3128aa6218cc55f6cbce8be42b2db2fee4
6 files changed +607 -44
plugins/_browser/assets/browser-page-content.js
+72 -1
@@ -1,7 +1,7 @@
1 (() => {
2 const GLOBAL_KEY = "__spaceBrowserPageContent__";
3 const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";
4 - const VERSION = "7";
4 + const VERSION = "9";
5 const BLOCK_TAGS = new Set([
6 "ADDRESS",
7 "ARTICLE",
@@ -2026,6 +2026,76 @@
2026 return entry;
2027 }
2028
2029 + function computeStableSelector(el) {
2030 + if (!el || el.nodeType !== 1) return null;
2031 + const doc = el.ownerDocument || document;
2032 + if (el.id && /^[A-Za-z_][\w-]*$/.test(el.id)) {
2033 + const sel = "#" + (typeof CSS !== "undefined" && CSS.escape ? CSS.escape(el.id) : el.id);
2034 + try {
2035 + if (doc.querySelectorAll(sel).length === 1) return sel;
2036 + } catch (_) {}
2037 + }
2038 + const parts = [];
2039 + let node = el;
2040 + while (node && node.nodeType === 1 && node !== doc.documentElement) {
2041 + let part = node.tagName.toLowerCase();
2042 + if (node.id && /^[A-Za-z_][\w-]*$/.test(node.id)) {
2043 + const idSel = "#" + (typeof CSS !== "undefined" && CSS.escape ? CSS.escape(node.id) : node.id);
2044 + try {
2045 + if (doc.querySelectorAll(idSel).length === 1) {
2046 + parts.unshift(idSel);
2047 + break;
2048 + }
2049 + } catch (_) {}
2050 + }
2051 + const parent = node.parentElement;
2052 + if (parent) {
2053 + const sibs = parent.children;
2054 + let idx = 0;
2055 + let sameTag = 0;
2056 + for (let i = 0; i < sibs.length; i++) {
2057 + if (sibs[i].tagName === node.tagName) {
2058 + sameTag++;
2059 + if (sibs[i] === node) idx = sameTag;
2060 + }
2061 + }
2062 + if (sameTag > 1) part += ":nth-of-type(" + idx + ")";
2063 + }
2064 + parts.unshift(part);
2065 + node = parent;
2066 + }
2067 + const sel = parts.join(" > ");
2068 + if (!sel) return null;
2069 + try {
2070 + if (doc.querySelectorAll(sel).length === 1) return sel;
2071 + } catch (_) {}
2072 + return null;
2073 + }
2074 +
2075 + function boundingBoxFor(referenceId) {
2076 + const entry = requireReferenceEntry(referenceId, {
2077 + actionLabel: "boundingBox",
2078 + requireConnected: false
2079 + });
2080 + if (entry.helperBacked || !entry.element) return null;
2081 + const el = entry.element;
2082 + if (typeof el.getBoundingClientRect !== "function") return null;
2083 + try {
2084 + el.scrollIntoView({ block: "center", inline: "center", behavior: "instant" });
2085 + } catch (_) {}
2086 + const r = el.getBoundingClientRect();
2087 + const selector = computeStableSelector(el);
2088 + const hasBox = r && r.width > 0 && r.height > 0;
2089 + if (!hasBox && !selector) return null;
2090 + return {
2091 + x: hasBox ? r.left : 0,
2092 + y: hasBox ? r.top : 0,
2093 + width: hasBox ? r.width : 0,
2094 + height: hasBox ? r.height : 0,
2095 + selector: selector || null
2096 + };
2097 + }
2098 +
2099 function refreshReferenceEntry(entry) {
2100 if (!entry || entry.helperBacked || !entry.element) {
2101 return entry;
@@ -3253,6 +3323,7 @@
3323 typeSubmit(referenceId, value) {
3324 return typeAndSubmit(referenceId, value);
3325 },
3326 + boundingBoxFor,
3327 version: VERSION
3328 };
3329 })();
plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
+13 -2
@@ -86,11 +86,22 @@ function isFreshToolMessage(timestamp) {
86 return Math.abs(Date.now() - messageMs) <= AUTO_OPEN_WINDOW_MS;
87 }
88
89 +// Allowlist: only these actions cause the viewer to follow. Background work
90 +// (evaluate, click, type, key_chord, mouse, multi, ...) does not steal focus.
91 +const FOCUS_ACTIONS = new Set([
92 + "open",
93 + "navigate",
94 + "set_active",
95 + "setactive",
96 + "activate",
97 + "focus",
98 +]);
99 +
100 function shouldAutoOpenBrowser(args, result) {
101 if (!isFreshToolMessage(args?.timestamp)) return false;
102 const action = String(args?.kvps?.action || "").trim().toLowerCase().replace("-", "_");
92 - if (["list", "content", "detail", "close", "close_all"].includes(action)) return false;
93 - return Boolean(browserIdFromResult(result, args?.kvps || {}) || action === "open" || action === "navigate");
103 + if (!FOCUS_ACTIONS.has(action)) return false;
104 + return Boolean(browserIdFromResult(result, args?.kvps || {}));
105 }
106
107 function autoOpenBrowserCanvas(args, result) {
plugins/_browser/extensions/webui/set_messages_after_loop/auto-open-browser-results.js
+14 -10
@@ -79,19 +79,23 @@ function parseMaybeJson(value) {
79 }
80 }
81
82 +// Actions that should bring the viewer to the targeted tab. Everything else
83 +// (read, click, type, evaluate, key_chord, mouse, multi, ...) leaves the
84 +// viewer where it is so cross-tab work doesn't steal user focus.
85 +const FOCUS_ACTIONS = new Set([
86 + "open",
87 + "navigate",
88 + "set_active",
89 + "setactive",
90 + "activate",
91 + "focus",
92 +]);
93 +
94 function shouldAutoOpen(args = {}, payload = {}, result = {}) {
95 if (!isFresh(args.timestamp, payload.last_modified || result.last_modified)) return false;
84 -
96 const action = String(payload.action || "").trim().toLowerCase().replace("-", "_");
86 - if (["list", "content", "detail", "close", "close_all"].includes(action)) return false;
87 -
88 - return Boolean(
89 - getBrowserId(payload, result)
90 - || action === "open"
91 - || action === "navigate"
92 - || result.currentUrl
93 - || result.state?.currentUrl,
94 - );
97 + if (!FOCUS_ACTIONS.has(action)) return false;
98 + return Boolean(getBrowserId(payload, result) || result.currentUrl || result.state?.currentUrl);
99 }
100
101 function getBrowserId(payload = {}, result = {}) {
plugins/_browser/helpers/runtime.py
+390 -26
@@ -337,6 +337,9 @@ class BrowserRuntime:
337
338
339 class _BrowserRuntimeCore:
340 + _VALID_MODIFIERS = {"Control", "Shift", "Alt", "Meta"}
341 + _POPUP_WAIT_SECONDS = 2.0
342 +
343 def __init__(self, context_id: str):
344 self.context_id = context_id
345 self.safe_context_id = _safe_context_id(context_id)
@@ -348,6 +351,23 @@ class _BrowserRuntimeCore:
351 self.last_interacted_browser_id: int | None = None
352 self._content_helper_source: str | None = None
353 self._start_lock: asyncio.Lock | None = None
354 + self._registry_lock: asyncio.Lock | None = None
355 + self._closing = False
356 + self._pending_popups: list[asyncio.Future[int]] = []
357 + self._background_popup_pages: set[int] = set()
358 +
359 + def _ensure_registry_lock(self) -> asyncio.Lock:
360 + if self._registry_lock is None:
361 + self._registry_lock = asyncio.Lock()
362 + return self._registry_lock
363 +
364 + def _maybe_promote(self, resolved_id: int) -> None:
365 + # Promote only if the target IS the current active tab or no tab is
366 + # active yet. Cross-tab work on a backgrounded tab does not steal
367 + # viewer focus.
368 + current = self.last_interacted_browser_id
369 + if current is None or current == resolved_id:
370 + self.last_interacted_browser_id = int(resolved_id)
371
372 @property
373 def profile_dir(self) -> Path:
@@ -411,6 +431,7 @@ class _BrowserRuntimeCore:
431 raise
432 self.context.set_default_timeout(30000)
433 self.context.set_default_navigation_timeout(30000)
434 + self.context.on("page", self._on_new_page_sync)
435 await self.context.add_init_script(self._shadow_dom_script())
436 await self.context.add_init_script(path=str(CONTENT_HELPER_PATH))
437
@@ -421,7 +442,7 @@ class _BrowserRuntimeCore:
442 except Exception:
443 pass
444 continue
424 - self._register_page(page)
445 + await self._register_page(page)
446
447 def _release_orphaned_profile_singleton(self) -> None:
448 lock_path = self.profile_dir / "SingletonLock"
@@ -486,7 +507,7 @@ class _BrowserRuntimeCore:
507 async def open(self, url: str = "") -> dict[str, Any]:
508 await self.ensure_started()
509 page = await self.context.new_page()
489 - browser_page = self._register_page(page)
510 + browser_page = await self._register_page(page)
511 self.last_interacted_browser_id = browser_page.id
512 target_url = self._initial_url(url)
513 if target_url and target_url != "about:blank":
@@ -501,13 +522,149 @@ class _BrowserRuntimeCore:
522 return raw_url
523 return str(get_browser_config().get(DEFAULT_HOMEPAGE_KEY) or "about:blank").strip() or "about:blank"
524
504 - async def list(self) -> dict[str, Any]:
525 + async def list(self, include_content: bool = False) -> dict[str, Any]:
526 await self.ensure_started()
527 + ids = sorted(self.pages)
528 + if not ids:
529 + return {
530 + "browsers": [],
531 + "last_interacted_browser_id": self.last_interacted_browser_id,
532 + }
533 + states_task = asyncio.gather(*(self._state(bid) for bid in ids))
534 + if include_content:
535 + contents_task = asyncio.gather(
536 + *(self.content(bid) for bid in ids),
537 + return_exceptions=True,
538 + )
539 + states, contents = await asyncio.gather(states_task, contents_task)
540 + out: list[dict[str, Any]] = []
541 + for idx, bid in enumerate(ids):
542 + entry = states[idx]
543 + c = contents[idx]
544 + if isinstance(c, Exception):
545 + entry["content_error"] = str(c)
546 + else:
547 + entry["content"] = c
548 + out.append(entry)
549 + else:
550 + out = await states_task
551 return {
507 - "browsers": [await self._state(browser_id) for browser_id in sorted(self.pages)],
552 + "browsers": out,
553 "last_interacted_browser_id": self.last_interacted_browser_id,
554 }
555
556 + async def multi(self, calls: list[dict[str, Any]]) -> list[dict[str, Any]]:
557 + if not isinstance(calls, list) or not calls:
558 + raise ValueError("multi requires a non-empty list of calls")
559 + groups: dict[Any, list[tuple[int, dict[str, Any]]]] = {}
560 + for idx, call in enumerate(calls):
561 + if not isinstance(call, dict):
562 + raise ValueError(f"calls[{idx}] is not an object")
563 + key = call.get("browser_id")
564 + groups.setdefault(key, []).append((idx, call))
565 +
566 + results: list[dict[str, Any] | None] = [None] * len(calls)
567 +
568 + async def run_group(group: list[tuple[int, dict[str, Any]]]) -> None:
569 + for idx, call in group:
570 + try:
571 + out = await self._dispatch_call(call)
572 + results[idx] = {"ok": True, "result": out}
573 + except Exception as exc:
574 + results[idx] = {"ok": False, "error": str(exc)}
575 +
576 + await asyncio.gather(*(run_group(g) for g in groups.values()))
577 + return [r if r is not None else {"ok": False, "error": "missing"} for r in results]
578 +
579 + async def _dispatch_call(self, call: dict[str, Any]) -> Any:
580 + action = str(call.get("action") or "").strip().lower().replace("-", "_")
581 + bid = call.get("browser_id")
582 + if action == "open":
583 + return await self.open(call.get("url") or "")
584 + if action == "list":
585 + return await self.list(include_content=bool(call.get("include_content")))
586 + if action == "state":
587 + return await self.state(bid)
588 + if action in {"set_active", "setactive", "activate", "focus"}:
589 + return await self.set_active(bid)
590 + if action == "navigate":
591 + return await self.navigate(bid, call.get("url") or "")
592 + if action == "back":
593 + return await self.back(bid)
594 + if action == "forward":
595 + return await self.forward(bid)
596 + if action == "reload":
597 + return await self.reload(bid)
598 + if action == "content":
599 + payload = None
600 + sels = call.get("selectors")
601 + sel = call.get("selector")
602 + if sels:
603 + payload = {"selectors": sels}
604 + elif sel:
605 + payload = {"selector": sel}
606 + return await self.content(bid, payload)
607 + if action == "detail":
608 + ref = call.get("ref")
609 + if ref is None:
610 + raise ValueError("detail requires ref")
611 + return await self.detail(bid, ref)
612 + if action == "click":
613 + ref = call.get("ref")
614 + if ref is None:
615 + raise ValueError("click requires ref")
616 + return await self.click(
617 + bid, ref,
618 + modifiers=call.get("modifiers"),
619 + focus_popup=call.get("focus_popup"),
620 + )
621 + if action == "type":
622 + ref = call.get("ref")
623 + if ref is None:
624 + raise ValueError("type requires ref")
625 + return await self.type(bid, ref, call.get("text") or "")
626 + if action == "submit":
627 + ref = call.get("ref")
628 + if ref is None:
629 + raise ValueError("submit requires ref")
630 + return await self.submit(bid, ref)
631 + if action in {"type_submit", "typesubmit"}:
632 + ref = call.get("ref")
633 + if ref is None:
634 + raise ValueError("type_submit requires ref")
635 + return await self.type_submit(bid, ref, call.get("text") or "")
636 + if action == "scroll":
637 + ref = call.get("ref")
638 + if ref is None:
639 + raise ValueError("scroll requires ref")
640 + return await self.scroll(bid, ref)
641 + if action == "evaluate":
642 + return await self.evaluate(bid, call.get("script") or "")
643 + if action in {"key_chord", "keychord"}:
644 + keys = call.get("keys") or []
645 + if not keys:
646 + raise ValueError("key_chord requires non-empty keys")
647 + return await self.key_chord(bid, list(keys))
648 + if action == "mouse":
649 + return await self.mouse(
650 + bid, call.get("event_type") or "click",
651 + float(call.get("x") or 0), float(call.get("y") or 0),
652 + button=call.get("button") or "left",
653 + modifiers=call.get("modifiers"),
654 + )
655 + if action == "close":
656 + return await self.close_browser(bid)
657 + if action == "close_all":
658 + return await self.close_all_browsers()
659 + raise ValueError(f"unknown action: {action}")
660 +
661 + async def set_active(self, browser_id: int | str | None) -> dict[str, Any]:
662 + await self.ensure_started()
663 + resolved_id = self._resolve_browser_id(browser_id)
664 + # Explicit focus change — bypass _maybe_promote.
665 + self.last_interacted_browser_id = int(resolved_id)
666 + return await self._state(resolved_id)
667 +
668 async def state(self, browser_id: int | str | None = None) -> dict[str, Any]:
669 await self.ensure_started()
670 return await self._state(self._resolve_browser_id(browser_id))
@@ -517,7 +674,7 @@ class _BrowserRuntimeCore:
674 resolved_id = self._resolve_browser_id(browser_id)
675 page = self._page(resolved_id)
676 await self._goto(page, normalize_url(url))
520 - self.last_interacted_browser_id = resolved_id
677 + self._maybe_promote(resolved_id)
678 return await self._state(resolved_id)
679
680 async def back(self, browser_id: int | str | None = None) -> dict[str, Any]:
@@ -526,7 +683,7 @@ class _BrowserRuntimeCore:
683 page = self._page(resolved_id)
684 await page.go_back(wait_until="domcontentloaded", timeout=10000)
685 await self._settle(page)
529 - self.last_interacted_browser_id = resolved_id
686 + self._maybe_promote(resolved_id)
687 return await self._state(resolved_id)
688
689 async def forward(self, browser_id: int | str | None = None) -> dict[str, Any]:
@@ -535,7 +692,7 @@ class _BrowserRuntimeCore:
692 page = self._page(resolved_id)
693 await page.go_forward(wait_until="domcontentloaded", timeout=10000)
694 await self._settle(page)
538 - self.last_interacted_browser_id = resolved_id
695 + self._maybe_promote(resolved_id)
696 return await self._state(resolved_id)
697
698 async def reload(self, browser_id: int | str | None = None) -> dict[str, Any]:
@@ -544,7 +701,7 @@ class _BrowserRuntimeCore:
701 page = self._page(resolved_id)
702 await page.reload(wait_until="domcontentloaded", timeout=15000)
703 await self._settle(page)
547 - self.last_interacted_browser_id = resolved_id
704 + self._maybe_promote(resolved_id)
705 return await self._state(resolved_id)
706
707 async def content(
@@ -560,7 +717,7 @@ class _BrowserRuntimeCore:
717 "(payload) => globalThis.__spaceBrowserPageContent__.capture(payload || null)",
718 payload or None,
719 )
563 - self.last_interacted_browser_id = resolved_id
720 + self._maybe_promote(resolved_id)
721 return result or {}
722
723 async def detail(self, browser_id: int | str | None, reference_id: int | str) -> dict[str, Any]:
@@ -572,7 +729,7 @@ class _BrowserRuntimeCore:
729 "(ref) => globalThis.__spaceBrowserPageContent__.detail(ref)",
730 reference_id,
731 )
575 - self.last_interacted_browser_id = resolved_id
732 + self._maybe_promote(resolved_id)
733 return result or {}
734
735 async def annotation_target(
@@ -588,7 +745,7 @@ class _BrowserRuntimeCore:
745 "(payload) => globalThis.__spaceBrowserPageContent__.annotate(payload || null)",
746 payload or None,
747 )
591 - self.last_interacted_browser_id = resolved_id
748 + self._maybe_promote(resolved_id)
749 return result or {}
750
751 async def evaluate(self, browser_id: int | str | None, script: str) -> dict[str, Any]:
@@ -596,12 +753,140 @@ class _BrowserRuntimeCore:
753 resolved_id = self._resolve_browser_id(browser_id)
754 page = self._page(resolved_id)
755 result = await page.evaluate(str(script or "undefined"))
599 - self.last_interacted_browser_id = resolved_id
756 + self._maybe_promote(resolved_id)
757 return {"result": result, "state": await self._state(resolved_id)}
758
602 - async def click(self, browser_id: int | str | None, reference_id: int | str) -> dict[str, Any]:
759 + async def click(
760 + self,
761 + browser_id: int | str | None,
762 + reference_id: int | str,
763 + modifiers: list[str] | None = None,
764 + focus_popup: bool | None = None,
765 + ) -> dict[str, Any]:
766 + if modifiers:
767 + return await self._modifier_click(browser_id, reference_id, modifiers, focus_popup)
768 return await self._reference_action("click", browser_id, reference_id)
769
770 + async def _modifier_click(
771 + self,
772 + browser_id: int | str | None,
773 + reference_id: int | str,
774 + modifiers: list[str],
775 + focus_popup: bool | None,
776 + ) -> dict[str, Any]:
777 + bad = set(modifiers) - self._VALID_MODIFIERS
778 + if bad:
779 + raise ValueError(
780 + f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
781 + )
782 + await self.ensure_started()
783 + resolved_id = self._resolve_browser_id(browser_id)
784 + page = self._page(resolved_id)
785 + await self._ensure_content_helper(page)
786 +
787 + box = await page.evaluate(
788 + "(ref) => globalThis.__spaceBrowserPageContent__.boundingBoxFor(ref)",
789 + reference_id,
790 + )
791 +
792 + background = focus_popup is False or (
793 + focus_popup is None and bool({"Control", "Meta"} & set(modifiers))
794 + )
795 +
796 + loop = asyncio.get_running_loop()
797 + waiter: asyncio.Future[int] = loop.create_future()
798 + self._pending_popups.append(waiter)
799 +
800 + warning: str | None = None
801 + opened_id: int | None = None
802 + try:
803 + box_has_geometry = bool(box and box.get("width") and box.get("height"))
804 + box_selector = box.get("selector") if box else None
805 + if box_has_geometry:
806 + cx = box["x"] + box["width"] / 2
807 + cy = box["y"] + box["height"] / 2
808 + # Mouse.click does not accept modifiers; hold them via keyboard.
809 + pressed: list[str] = []
810 + try:
811 + for mod in modifiers:
812 + await page.keyboard.down(mod)
813 + pressed.append(mod)
814 + await page.mouse.click(cx, cy)
815 + finally:
816 + for mod in reversed(pressed):
817 + with contextlib.suppress(Exception):
818 + await page.keyboard.up(mod)
819 + await self._settle(page, short=False)
820 + elif box_selector:
821 + try:
822 + await page.locator(box_selector).click(
823 + modifiers=list(modifiers), force=True, timeout=5000
824 + )
825 + await self._settle(page, short=False)
826 + except Exception as exc:
827 + await self._reference_action("click", browser_id, reference_id)
828 + warning = f"modifiers ignored: locator click failed ({exc})"
829 + else:
830 + await self._reference_action("click", browser_id, reference_id)
831 + warning = "modifiers ignored: target geometry unavailable"
832 +
833 + try:
834 + opened_id = await asyncio.wait_for(
835 + asyncio.shield(waiter), timeout=self._POPUP_WAIT_SECONDS
836 + )
837 + except asyncio.TimeoutError:
838 + opened_id = None
839 + finally:
840 + if waiter in self._pending_popups:
841 + self._pending_popups.remove(waiter)
842 + if not waiter.done():
843 + waiter.cancel()
844 +
845 + if opened_id is not None and background:
846 + self._background_popup_pages.add(opened_id)
847 + if self.last_interacted_browser_id == opened_id:
848 + # Force focus back to origin tab — popup hook had promoted.
849 + self.last_interacted_browser_id = int(resolved_id)
850 + finally:
851 + if waiter in self._pending_popups:
852 + self._pending_popups.remove(waiter)
853 +
854 + if background:
855 + # Background-mode click: explicitly keep focus on origin.
856 + self.last_interacted_browser_id = int(resolved_id)
857 + return {
858 + "action": {
859 + "ref": reference_id,
860 + "modifiers": list(modifiers),
861 + "opened_browser_ids": [opened_id] if opened_id is not None else [],
862 + **({"warning": warning} if warning else {}),
863 + },
864 + "state": await self._state(resolved_id),
865 + }
866 +
867 + async def key_chord(
868 + self,
869 + browser_id: int | str | None,
870 + keys: list[str],
871 + ) -> dict[str, Any]:
872 + if not keys:
873 + raise ValueError("key_chord requires at least one key")
874 + await self.ensure_started()
875 + resolved_id = self._resolve_browser_id(browser_id)
876 + page = self._page(resolved_id)
877 + pressed: list[str] = []
878 + try:
879 + for k in keys:
880 + await page.keyboard.down(k)
881 + pressed.append(k)
882 + finally:
883 + for k in reversed(pressed):
884 + with contextlib.suppress(Exception):
885 + await page.keyboard.up(k)
886 + await self._settle(page, short=True)
887 + self._maybe_promote(resolved_id)
888 + return await self._state(resolved_id)
889 +
890 async def submit(self, browser_id: int | str | None, reference_id: int | str) -> dict[str, Any]:
891 return await self._reference_action("submit", browser_id, reference_id)
892
@@ -693,7 +978,7 @@ class _BrowserRuntimeCore:
978 self.screencasts.pop(stream_id, None)
979 await screencast.stop()
980 raise
696 - self.last_interacted_browser_id = resolved_id
981 + self._maybe_promote(resolved_id)
982 return {
983 "stream_id": stream_id,
984 "browser_id": resolved_id,
@@ -752,7 +1037,7 @@ class _BrowserRuntimeCore:
1037 await self._remount_viewport(page, viewport)
1038 if should_remount_viewport:
1039 await self._settle(page, short=True)
755 - self.last_interacted_browser_id = resolved_id
1040 + self._maybe_promote(resolved_id)
1041 return {"state": await self._state(resolved_id), "viewport": viewport}
1042
1043 async def _apply_viewport_with_remount(self, page: Any, viewport: dict[str, int]) -> None:
@@ -777,21 +1062,40 @@ class _BrowserRuntimeCore:
1062 x: float,
1063 y: float,
1064 button: str = "left",
1065 + modifiers: list[str] | None = None,
1066 ) -> dict[str, Any]:
1067 + event_type_lower = str(event_type or "click").lower()
1068 + if modifiers:
1069 + if event_type_lower != "click":
1070 + raise ValueError("modifiers are only valid for event_type='click'")
1071 + bad = set(modifiers) - self._VALID_MODIFIERS
1072 + if bad:
1073 + raise ValueError(
1074 + f"unsupported modifiers: {sorted(bad)}; allowed: {sorted(self._VALID_MODIFIERS)}"
1075 + )
1076 await self.ensure_started()
1077 resolved_id = self._resolve_browser_id(browser_id)
1078 page = self._page(resolved_id)
784 - event_type = str(event_type or "click").lower()
785 - if event_type == "move":
1079 + if event_type_lower == "move":
1080 await page.mouse.move(float(x), float(y))
787 - elif event_type == "down":
1081 + elif event_type_lower == "down":
1082 await page.mouse.down(button=button)
789 - elif event_type == "up":
1083 + elif event_type_lower == "up":
1084 await page.mouse.up(button=button)
1085 else:
792 - await page.mouse.click(float(x), float(y), button=button)
1086 + pressed: list[str] = []
1087 + try:
1088 + if modifiers:
1089 + for mod in modifiers:
1090 + await page.keyboard.down(mod)
1091 + pressed.append(mod)
1092 + await page.mouse.click(float(x), float(y), button=button)
1093 + finally:
1094 + for mod in reversed(pressed):
1095 + with contextlib.suppress(Exception):
1096 + await page.keyboard.up(mod)
1097 await self._settle(page, short=True)
794 - self.last_interacted_browser_id = resolved_id
1098 + self._maybe_promote(resolved_id)
1099 return await self._state(resolved_id)
1100
1101 async def wheel(
@@ -807,7 +1111,7 @@ class _BrowserRuntimeCore:
1111 page = self._page(resolved_id)
1112 await page.mouse.move(float(x), float(y))
1113 await page.mouse.wheel(float(delta_x), float(delta_y))
810 - self.last_interacted_browser_id = resolved_id
1114 + self._maybe_promote(resolved_id)
1115 return await self._state(resolved_id)
1116
1117 async def keyboard(
@@ -825,10 +1129,16 @@ class _BrowserRuntimeCore:
1129 elif key:
1130 await page.keyboard.press(str(key))
1131 await self._settle(page, short=True)
828 - self.last_interacted_browser_id = resolved_id
1132 + self._maybe_promote(resolved_id)
1133 return await self._state(resolved_id)
1134
1135 async def close(self, delete_profile: bool = False) -> None:
1136 + self._closing = True
1137 + for waiter in self._pending_popups:
1138 + if not waiter.done():
1139 + waiter.set_exception(RuntimeError("Browser runtime is closing."))
1140 + self._pending_popups.clear()
1141 + self._background_popup_pages.clear()
1142 await self._stop_all_screencasts()
1143 for browser_id in list(self.pages):
1144 try:
@@ -873,7 +1183,7 @@ class _BrowserRuntimeCore:
1183 {"method": helper_method, "ref": reference_id, "text": text},
1184 )
1185 await self._settle(page, short=False)
876 - self.last_interacted_browser_id = resolved_id
1186 + self._maybe_promote(resolved_id)
1187 return {"action": action or {}, "state": await self._state(resolved_id)}
1188
1189 async def _goto(self, page: Any, url: str) -> None:
@@ -924,7 +1234,7 @@ class _BrowserRuntimeCore:
1234 "loading": False,
1235 }
1236
927 - def _register_page(self, page: Any) -> BrowserPage:
1237 + def _register_page_locked(self, page: Any) -> BrowserPage:
1238 existing = self._browser_id_for_page(page)
1239 if existing is not None:
1240 return self.pages[existing]
@@ -934,11 +1244,65 @@ class _BrowserRuntimeCore:
1244 self.pages[browser_id] = browser_page
1245
1246 def on_close() -> None:
937 - self.pages.pop(browser_id, None)
1247 + try:
1248 + asyncio.create_task(self._unregister_page_async(browser_id))
1249 + except RuntimeError:
1250 + # No running loop (e.g., during shutdown). Best-effort sync pop.
1251 + self.pages.pop(browser_id, None)
1252
1253 page.on("close", on_close)
1254 return browser_page
1255
1256 + async def _register_page(self, page: Any) -> BrowserPage:
1257 + lock = self._ensure_registry_lock()
1258 + async with lock:
1259 + return self._register_page_locked(page)
1260 +
1261 + async def _unregister_page_async(self, browser_id: int) -> None:
1262 + try:
1263 + lock = self._ensure_registry_lock()
1264 + async with lock:
1265 + self.pages.pop(browser_id, None)
1266 + if self.last_interacted_browser_id == browser_id:
1267 + self.last_interacted_browser_id = next(iter(sorted(self.pages)), None)
1268 + self._background_popup_pages.discard(browser_id)
1269 + except Exception as exc:
1270 + PrintStyle.warning(f"Page unregister failed: {exc}")
1271 +
1272 + def _on_new_page_sync(self, page: Any) -> None:
1273 + if self._closing or self.context is None:
1274 + return
1275 + try:
1276 + asyncio.create_task(self._on_new_page_async(page))
1277 + except RuntimeError:
1278 + return
1279 +
1280 + async def _on_new_page_async(self, page: Any) -> None:
1281 + try:
1282 + with contextlib.suppress(Exception):
1283 + await page.wait_for_load_state("domcontentloaded", timeout=2000)
1284 + if self._closing or page.is_closed():
1285 + return
1286 + lock = self._ensure_registry_lock()
1287 + async with lock:
1288 + if self._closing:
1289 + return
1290 + if self._browser_id_for_page(page) is not None:
1291 + return
1292 + browser_page = self._register_page_locked(page)
1293 + new_id = browser_page.id
1294 + while self._pending_popups:
1295 + waiter = self._pending_popups.pop(0)
1296 + if not waiter.done():
1297 + waiter.set_result(new_id)
1298 + break
1299 + if new_id not in self._background_popup_pages:
1300 + self.last_interacted_browser_id = new_id
1301 + else:
1302 + self._background_popup_pages.discard(new_id)
1303 + except Exception as exc:
1304 + PrintStyle.warning(f"Popup registration failed: {exc}")
1305 +
1306 def _browser_id_for_page(self, page: Any) -> int | None:
1307 for browser_id, browser_page in self.pages.items():
1308 if browser_page.page == page:
plugins/_browser/prompts/agent.system.tool.browser.md
+81 -3
@@ -4,8 +4,8 @@ use for web browsing, page inspection, forms, downloads, and browser-only tasks
4 state stays open per chat context
5 refs come from content as typed markers: [link 3], [button 6], [image 1], [input text 8]
6
7 -actions: open list state navigate back forward reload content detail click type submit type_submit scroll evaluate close close_all
8 -common args: action browser_id url ref text selector selectors script
7 +actions: open list state set_active navigate back forward reload content detail click type submit type_submit scroll evaluate key_chord mouse multi close close_all
8 +common args: action browser_id url ref text selector selectors script modifiers keys include_content focus_popup event_type x y button calls
9
10 workflow:
11 - open creates a new browser and returns id/state
@@ -13,7 +13,38 @@ workflow:
13 - detail inspects one ref, including link/image/input/button metadata
14 - click/type/type_submit/submit/scroll use refs from latest content capture and return {action,state}
15 - navigate/back/forward/reload return fresh state
16 -- list shows open browsers
16 +- list shows open browsers; pass include_content: true for one-call bulk read
17 +
18 +modifier clicks:
19 +- click accepts modifiers like ["Control"], ["Shift"], ["Alt"], ["Meta"]
20 +- ctrl/meta-click opens link in new tab in background (Chrome rule)
21 +- override with focus_popup: true (focus follows new tab) or false (always background)
22 +- the new tab id is reported in action.opened_browser_ids; list shows all tabs
23 +
24 +popup awareness:
25 +- tabs opened by site (window.open, target=_blank, ctrl-click) auto-register
26 +- list returns every tab; last_interacted_browser_id tracks current focus
27 +
28 +background work (do not steal focus):
29 +- operations on a non-active tab (read, click, type, evaluate, etc.) target that tab WITHOUT moving focus
30 +- last_interacted_browser_id (and the WebUI viewer that follows it) only changes on:
31 + - open (new tab created)
32 + - explicit set_active action
33 + - action on the already-active tab
34 + - chrome popup-focus rule (plain click on target=_blank → follow; ctrl-click → stay)
35 +- to switch focus deliberately: {"action":"set_active","browser_id":N}
36 +
37 +key_chord:
38 +- presses keys in order, releases in reverse; safe across exceptions
39 +- example: {"action":"key_chord","keys":["Control","a"]} selects all
40 +
41 +multi (parallel batch):
42 +- run many actions concurrently across tabs in one tool call
43 +- pass calls: array of action objects (each has its own action+args)
44 +- different browser_ids run in parallel; same browser_id runs in submit order
45 +- returns array of {"ok":true,"result":...} or {"ok":false,"error":"..."} matching input order
46 +- ideal for: scrape N tabs at once, fan-out reads, parallel evaluate
47 +- avoid mutating same tab twice in one batch unless serial order is intended
48
49 examples:
50 ~~~json
@@ -46,3 +77,50 @@ examples:
77 }
78 }
79 ~~~
80 +
81 +~~~json
82 +{
83 + "tool_name": "browser",
84 + "tool_args": {
85 + "action": "click",
86 + "browser_id": 1,
87 + "ref": 3,
88 + "modifiers": ["Control"]
89 + }
90 +}
91 +~~~
92 +
93 +~~~json
94 +{
95 + "tool_name": "browser",
96 + "tool_args": {
97 + "action": "key_chord",
98 + "browser_id": 1,
99 + "keys": ["Control", "a"]
100 + }
101 +}
102 +~~~
103 +
104 +~~~json
105 +{
106 + "tool_name": "browser",
107 + "tool_args": {
108 + "action": "list",
109 + "include_content": true
110 + }
111 +}
112 +~~~
113 +
114 +~~~json
115 +{
116 + "tool_name": "browser",
117 + "tool_args": {
118 + "action": "multi",
119 + "calls": [
120 + {"action": "content", "browser_id": 1},
121 + {"action": "content", "browser_id": 2},
122 + {"action": "evaluate", "browser_id": 3, "script": "document.title"}
123 + ]
124 + }
125 +}
126 +~~~
plugins/_browser/tools/browser.py
+37 -2
@@ -18,18 +18,34 @@ class Browser(Tool):
18 selector: str = "",
19 selectors: list[str] | None = None,
20 script: str = "",
21 + modifiers: list[str] | str | None = None,
22 + keys: list[str] | None = None,
23 + include_content: bool = False,
24 + focus_popup: bool | None = None,
25 + event_type: str = "",
26 + x: float = 0.0,
27 + y: float = 0.0,
28 + button: str = "left",
29 + calls: list[dict[str, Any]] | None = None,
30 **kwargs: Any,
31 ) -> Response:
32 action = str(action or self.method or "state").strip().lower().replace("-", "_")
33 runtime = await get_runtime(self.agent.context.id)
34
35 + if isinstance(modifiers, str):
36 + modifiers = [modifiers] if modifiers else None
37 + elif isinstance(modifiers, list) and not modifiers:
38 + modifiers = None
39 +
40 try:
41 if action == "open":
42 result = await runtime.call("open", url or "")
43 elif action == "list":
30 - result = await runtime.call("list")
44 + result = await runtime.call("list", include_content=bool(include_content))
45 elif action == "state":
46 result = await runtime.call("state", browser_id)
47 + elif action in {"set_active", "setactive", "activate", "focus"}:
48 + result = await runtime.call("set_active", browser_id)
49 elif action == "navigate":
50 result = await runtime.call("navigate", browser_id, url)
51 elif action == "back":
@@ -44,7 +60,13 @@ class Browser(Tool):
60 elif action == "detail":
61 result = await runtime.call("detail", browser_id, self._require_ref(ref))
62 elif action == "click":
47 - result = await runtime.call("click", browser_id, self._require_ref(ref))
63 + if modifiers:
64 + result = await runtime.call(
65 + "click", browser_id, self._require_ref(ref),
66 + modifiers=modifiers, focus_popup=focus_popup,
67 + )
68 + else:
69 + result = await runtime.call("click", browser_id, self._require_ref(ref))
70 elif action == "type":
71 result = await runtime.call("type", browser_id, self._require_ref(ref), text)
72 elif action == "submit":
@@ -60,6 +82,19 @@ class Browser(Tool):
82 result = await runtime.call("scroll", browser_id, self._require_ref(ref))
83 elif action == "evaluate":
84 result = await runtime.call("evaluate", browser_id, script)
85 + elif action in {"key_chord", "keychord"}:
86 + if not keys:
87 + raise ValueError("key_chord requires non-empty 'keys' list")
88 + result = await runtime.call("key_chord", browser_id, list(keys))
89 + elif action == "mouse":
90 + result = await runtime.call(
91 + "mouse", browser_id, event_type or "click", x, y,
92 + button=button or "left", modifiers=modifiers,
93 + )
94 + elif action == "multi":
95 + if not calls:
96 + raise ValueError("multi requires non-empty 'calls' list")
97 + result = await runtime.call("multi", list(calls))
98 elif action == "close":
99 result = await runtime.call("close_browser", browser_id)
100 elif action == "close_all":