@samitouri / QOSami_Agents_Zero-2 / commits / 7b1c84ae

Improve browser tool ergonomics for agent UI control

Teach the Browser content helper to ignore global/delegated framework event bindings so snapshots surface the actual actionable controls instead of broad wrapper elements. Add an accessible name to the Browser address bar for more reliable capture output. Allow agents to use selector-based reference actions, coordinate click fallbacks, focused-field typing, and string key chords such as CTRL+A across the browser tool, container runtime, and host connector runtime. Cover the behavior with browser regression and host connector tests.

Alessandro committed May 12, 2026 at 09:41 UTC 7b1c84aeca74d0fa60cdfe0999aade593adb9270

8 files changed +363 -22

plugins/_browser/assets/browser-page-content.js

+20 -1

@@ -1,7 +1,7 @@

1 (() => {

2 const GLOBAL_KEY = "__spaceBrowserPageContent__";

3 const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";

4 - const VERSION = "11";

4 + const VERSION = "12";

5 const REQUIRED_API_NAMES = Object.freeze([

6 "annotate",

7 "boundingBoxFor",

@@ -646,6 +646,19 @@

646 .split(/[.:]/u, 1)[0];

647 }

648

649 + function isGlobalOrDelegatedEventBinding(value) {

650 + const parts = String(value || "")

651 + .trim()

652 + .toLowerCase()

653 + .split(/[.:]/u)

654 + .map((part) => part.trim())

655 + .filter(Boolean);

656 + return parts.includes("window")

657 + || parts.includes("document")

658 + || parts.includes("outside")

659 + || parts.includes("away");

660 + }

661 +

662 function isInteractiveEventName(value) {

663 return INTERACTIVE_EVENT_NAMES.has(normalizeInteractiveEventName(value));

664 }

@@ -657,10 +670,16 @@

670 }

671

672 if (normalizedName.startsWith("@")) {

673 + if (isGlobalOrDelegatedEventBinding(normalizedName.slice(1))) {

674 + return false;

675 + }

676 return isInteractiveEventName(normalizedName.slice(1));

677 }

678

679 if (normalizedName.startsWith("x-on:") || normalizedName.startsWith("v-on:")) {

680 + if (isGlobalOrDelegatedEventBinding(normalizedName.slice(5))) {

681 + return false;

682 + }

683 return isInteractiveEventName(normalizedName.slice(5));

684 }

685

plugins/_browser/helpers/connector_runtime.py

+47 -1

@@ -63,6 +63,18 @@ get_browser_config = browser_config.get_browser_config

63 _LOCAL_PROVIDERS = {"ollama", "lm_studio"}

64 _LOCAL_HOSTS = {"localhost", "127.0.0.1", "::1", "host.docker.internal"}

65 _SENSITIVE_ACTIONS = {"content", "detail", "evaluate", "screenshot", "screenshot_file"}

66 +_KEY_ALIASES = {

67 + "cmd": "Meta",

68 + "command": "Meta",

69 + "control": "Control",

70 + "ctrl": "Control",

71 + "escape": "Escape",

72 + "esc": "Escape",

73 + "meta": "Meta",

74 + "option": "Alt",

75 + "return": "Enter",

76 + "space": "Space",

77 +}

78 _REQUIRED_API_NAMES_RE = re.compile(

79 r"const\s+REQUIRED_API_NAMES\s*=\s*Object\.freeze$\[(?P<body>.*?)\]$;",

80 re.S,

@@ -151,7 +163,7 @@ class ConnectorBrowserRuntime:

163 payload["text"] = kwargs.get("text", "")

164 elif action == "key_chord":

165 payload["browser_id"] = args[0] if args else None

154 - payload["keys"] = args[1] if len(args) > 1 else []

166 + payload["keys"] = self._normalize_keys(args[1] if len(args) > 1 else [])

167 elif action == "clipboard":

168 payload["browser_id"] = args[0] if args else None

169 payload["clipboard_action"] = kwargs.get("action", "")

@@ -196,11 +208,45 @@ class ConnectorBrowserRuntime:

208 normalized["url"] = cls._normalize_open_url(normalized.get("url"))

209 elif action == "navigate":

210 normalized["url"] = normalize_url(normalized.get("url", ""))

211 + elif action == "click" and not normalized.get("ref") and (

212 + normalized.get("x") or normalized.get("y")

213 + ):

214 + normalized["action"] = "mouse"

215 + normalized.setdefault("event_type", "click")

216 + normalized.setdefault("button", "left")

217 + elif action == "type" and not normalized.get("ref"):

218 + normalized["action"] = "keyboard"

219 + normalized.setdefault("key", "")

220 + elif action in {"key_chord", "keychord"}:

221 + normalized["keys"] = cls._normalize_keys(normalized.get("keys"))

222 elif action == "multi" or isinstance(normalized.get("calls"), list):

223 normalized["calls"] = cls._normalize_multi_calls(normalized.get("calls", []))

224 normalized_calls.append(normalized)

225 return normalized_calls

226

227 + @staticmethod

228 + def _normalize_keys(keys: Any) -> list[str]:

229 + if keys is None:

230 + return []

231 + if isinstance(keys, str):

232 + raw = re.split(r"\s*\+\s*|\s*,\s*", keys.strip())

233 + elif isinstance(keys, list):

234 + raw = keys

235 + else:

236 + raw = [str(keys)]

237 + normalized: list[str] = []

238 + for key in raw:

239 + value = str(key or "").strip()

240 + if not value:

241 + continue

242 + normalized.append(

243 + _KEY_ALIASES.get(

244 + value.lower(),

245 + value.upper() if len(value) == 1 and value.isalpha() else value,

246 + )

247 + )

248 + return normalized

249 +

250 async def _dispatch(self, payload: dict[str, Any]) -> Any:

251 payload.setdefault("profile_mode", self._host_browser_profile_mode())

252 self._enforce_privacy(payload)

plugins/_browser/helpers/runtime.py

+51 -3

@@ -535,6 +535,18 @@ class BrowserRuntime:

535

536 class _BrowserRuntimeCore:

537 _VALID_MODIFIERS = {"Control", "Shift", "Alt", "Meta"}

538 + _KEY_ALIASES = {

539 + "cmd": "Meta",

540 + "command": "Meta",

541 + "control": "Control",

542 + "ctrl": "Control",

543 + "escape": "Escape",

544 + "esc": "Escape",

545 + "meta": "Meta",

546 + "option": "Alt",

547 + "return": "Enter",

548 + "space": "Space",

549 + }

550 _POPUP_WAIT_SECONDS = 2.0

551

552 def __init__(self, context_id: str):

@@ -596,6 +608,29 @@ class _BrowserRuntimeCore:

608 )

609 return normalized

610

611 + @classmethod

612 + def _normalize_keys(cls, keys: list[str] | str | None) -> list[str]:

613 + if keys is None:

614 + return []

615 + if isinstance(keys, str):

616 + raw = re.split(r"\s*\+\s*|\s*,\s*", keys.strip())

617 + elif isinstance(keys, list):

618 + raw = keys

619 + else:

620 + raw = [str(keys)]

621 + normalized: list[str] = []

622 + for key in raw:

623 + value = str(key or "").strip()

624 + if not value:

625 + continue

626 + normalized.append(

627 + cls._KEY_ALIASES.get(

628 + value.lower(),

629 + value.upper() if len(value) == 1 and value.isalpha() else value,

630 + )

631 + )

632 + return normalized

633 +

634 @staticmethod

635 def _has_reference(reference_id: int | str | None) -> bool:

636 return reference_id is not None and str(reference_id).strip() != ""

@@ -980,6 +1015,15 @@ class _BrowserRuntimeCore:

1015 return await self.detail(bid, ref)

1016 if action == "click":

1017 ref = call.get("ref")

1018 + if ref is None and (call.get("x") or call.get("y")):

1019 + return await self.mouse(

1020 + bid,

1021 + "click",

1022 + float(call.get("x") or 0),

1023 + float(call.get("y") or 0),

1024 + button=call.get("button") or "left",

1025 + modifiers=self._normalize_modifiers(call.get("modifiers")),

1026 + )

1027 if ref is None:

1028 raise ValueError("click requires ref")

1029 return await self.click(

@@ -990,7 +1034,11 @@ class _BrowserRuntimeCore:

1034 if action == "type":

1035 ref = call.get("ref")

1036 if ref is None:

993 - raise ValueError("type requires ref")

1037 + return await self.keyboard(

1038 + bid,

1039 + key="",

1040 + text=str(call.get("text") or ""),

1041 + )

1042 return await self.type(bid, ref, call.get("text") or "")

1043 if action == "submit":

1044 ref = call.get("ref")

@@ -1010,10 +1058,10 @@ class _BrowserRuntimeCore:

1058 if action == "evaluate":

1059 return await self.evaluate(bid, call.get("script") or "")

1060 if action in {"key_chord", "keychord"}:

1013 - keys = call.get("keys") or []

1061 + keys = self._normalize_keys(call.get("keys"))

1062 if not keys:

1063 raise ValueError("key_chord requires non-empty keys")

1016 - return await self.key_chord(bid, list(keys))

1064 + return await self.key_chord(bid, keys)

1065 if action == "mouse":

1066 return await self.mouse(

1067 bid, call.get("event_type") or "click",

plugins/_browser/prompts/agent.system.tool.browser.md

@@ -15,6 +15,8 @@ Workflow:

15 - `open` creates a tab and returns id/state.

16 - `content` returns markdown with refs like `[link 3]`, `[button 6]`, `[input text 8]`.

17 - Interactions use refs from the latest `content` capture.

18 +- For same-page controls that are easier to identify structurally, `click`, `type`, `submit`, `type_submit`, `scroll`, `select_option`, `set_checked`, and `upload_file` may use `selector` instead of `ref`; the tool resolves the selector through `content` first.

19 +- `click` with `x`/`y` and no `ref` is treated as a coordinate mouse click. `type` with text and no `ref` types into the currently focused element. `key_chord` accepts either `["Control", "A"]` or `"CTRL+A"`.

20 - `navigate` reuses an existing `browser_id` and is preferred for serial browsing.

21 - Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned path before reasoning visually.

22 - Keep the tab set small; close pages after extracting what you need.

plugins/_browser/tools/browser.py

+129 -12

@@ -1,6 +1,7 @@

1 from __future__ import annotations

3 import json

4 +import re

5 import time

6 import uuid

7 from pathlib import Path

@@ -82,6 +83,7 @@ class Browser(Tool):

83 modifiers = [modifiers] if modifiers else None

84 elif isinstance(modifiers, list) and not modifiers:

85 modifiers = None

86 + keys = self._normalize_keys(keys)

88 try:

89 if action == "open":

@@ -114,34 +116,70 @@ class Browser(Tool):

116 payload = self._selector_payload(selector, selectors)

117 result = await runtime.call("content", browser_id, payload)

118 elif action == "detail":

117 - result = await runtime.call("detail", browser_id, self._require_ref(ref))

119 + result = await runtime.call(

120 + "detail",

121 + browser_id,

122 + await self._resolve_ref(runtime, browser_id, ref, selector, action),

123 + )

124 elif action == "click":

119 - if modifiers:

125 + resolved_ref = await self._resolve_ref(

126 + runtime,

127 + browser_id,

128 + ref,

129 + selector,

130 + action,

131 + required=not self._has_coordinates(x, y),

132 + )

133 + if resolved_ref is None and self._has_coordinates(x, y):

134 result = await runtime.call(

121 - "click", browser_id, self._require_ref(ref),

135 + "mouse", browser_id, "click", x, y,

136 + button=button or "left", modifiers=modifiers,

137 + )

138 + elif modifiers:

139 + result = await runtime.call(

140 + "click", browser_id, resolved_ref,

141 modifiers=modifiers, focus_popup=focus_popup,

142 )

143 else:

125 - result = await runtime.call("click", browser_id, self._require_ref(ref))

144 + result = await runtime.call("click", browser_id, resolved_ref)

145 elif action == "type":

127 - result = await runtime.call("type", browser_id, self._require_ref(ref), text)

146 + resolved_ref = await self._resolve_ref(

147 + runtime,

148 + browser_id,

149 + ref,

150 + selector,

151 + action,

152 + required=False,

153 + )

154 + if resolved_ref is None:

155 + result = await runtime.call("keyboard", browser_id, key="", text=text)

156 + else:

157 + result = await runtime.call("type", browser_id, resolved_ref, text)

158 elif action == "submit":

129 - result = await runtime.call("submit", browser_id, self._require_ref(ref))

159 + result = await runtime.call(

160 + "submit",

161 + browser_id,

162 + await self._resolve_ref(runtime, browser_id, ref, selector, action),

163 + )

164 elif action in {"type_submit", "typesubmit"}:

165 result = await runtime.call(

166 "type_submit",

167 browser_id,

134 - self._require_ref(ref),

168 + await self._resolve_ref(runtime, browser_id, ref, selector, action),

169 text,

170 )

171 elif action == "scroll":

138 - result = await runtime.call("scroll", browser_id, self._require_ref(ref))

172 + result = await runtime.call(

173 + "scroll",

174 + browser_id,

175 + await self._resolve_ref(runtime, browser_id, ref, selector, action),

176 + )

177 elif action == "evaluate":

178 result = await runtime.call("evaluate", browser_id, script)

179 elif action in {"key_chord", "keychord"}:

180 if not keys:

181 raise ValueError("key_chord requires non-empty 'keys' list")

144 - result = await runtime.call("key_chord", browser_id, list(keys))

182 + result = await runtime.call("key_chord", browser_id, keys)

183 elif action == "hover":

184 result = await runtime.call(

185 "hover",

@@ -232,7 +270,7 @@ class Browser(Tool):

270 result = await runtime.call(

271 "select_option",

272 browser_id,

235 - self._require_ref(ref),

273 + await self._resolve_ref(runtime, browser_id, ref, selector, action),

274 value=value,

275 values=values,

276 )

@@ -240,14 +278,14 @@ class Browser(Tool):

278 result = await runtime.call(

279 "set_checked",

280 browser_id,

243 - self._require_ref(ref),

281 + await self._resolve_ref(runtime, browser_id, ref, selector, action),

282 checked=True if checked is None else bool(checked),

283 )

284 elif action == "upload_file":

285 result = await runtime.call(

286 "upload_file",

287 browser_id,

250 - self._require_ref(ref),

288 + await self._resolve_ref(runtime, browser_id, ref, selector, action),

289 path=path,

290 paths=paths,

291 )

@@ -290,6 +328,85 @@ class Browser(Tool):

328 raise ValueError("ref is required for this browser action")

329 return ref

330

331 + @staticmethod

332 + def _has_ref(ref: int | str | None) -> bool:

333 + return ref is not None and str(ref).strip() != ""

334 +

335 + @staticmethod

336 + def _has_coordinates(x: float, y: float) -> bool:

337 + return bool(float(x or 0) or float(y or 0))

338 +

339 + @classmethod

340 + async def _resolve_ref(

341 + cls,

342 + runtime: Any,

343 + browser_id: int | str | None,

344 + ref: int | str | None,

345 + selector: str = "",

346 + action: str = "action",

347 + *,

348 + required: bool = True,

349 + ) -> int | str | None:

350 + if cls._has_ref(ref):

351 + return ref

352 +

353 + selector = str(selector or "").strip()

354 + if selector:

355 + content = await runtime.call("content", browser_id, {"selector": selector})

356 + resolved = cls._first_ref_from_content(content, selector)

357 + if resolved is not None:

358 + return resolved

359 + raise ValueError(

360 + f"{action} could not resolve selector {selector!r} to a browser ref"

361 + )

362 +

363 + if required:

364 + return cls._require_ref(ref)

365 + return None

366 +

367 + @staticmethod

368 + def _first_ref_from_content(content: Any, selector: str = "") -> str | None:

369 + if isinstance(content, dict):

370 + values: list[Any] = []

371 + if selector and selector in content:

372 + values.append(content.get(selector))

373 + values.extend(value for key, value in content.items() if key != selector)

374 + text = "\n".join(str(value or "") for value in values)

375 + else:

376 + text = str(content or "")

377 + match = re.search(r"\[[^\]\n]*?\b(\d+)\]", text)

378 + return match.group(1) if match else None

379 +

380 + @staticmethod

381 + def _normalize_keys(keys: list[str] | str | None) -> list[str]:

382 + if keys is None:

383 + return []

384 + if isinstance(keys, str):

385 + raw = re.split(r"\s*\+\s*|\s*,\s*", keys.strip())

386 + elif isinstance(keys, list):

387 + raw = keys

388 + else:

389 + raw = [str(keys)]

390 + aliases = {

391 + "cmd": "Meta",

392 + "command": "Meta",

393 + "control": "Control",

394 + "ctrl": "Control",

395 + "escape": "Escape",

396 + "esc": "Escape",

397 + "meta": "Meta",

398 + "option": "Alt",

399 + "return": "Enter",

400 + "space": "Space",

401 + }

402 + normalized: list[str] = []

403 + for key in raw:

404 + value = str(key or "").strip()

405 + if not value:

406 + continue

407 + normalized.append(aliases.get(value.lower(), value.upper() if len(value) == 1 and value.isalpha() else value))

408 + return normalized

409 +

410 @staticmethod

411 def _selector_payload(selector: str = "", selectors: list[str] | None = None) -> dict | None:

412 if selectors:

plugins/_browser/webui/browser-panel.html

+2 -2

@@ -173,9 +173,9 @@

173 </button>

174 </div>

175

176 - <form class="browser-address-form" @submit.prevent="$store.browserPage.go()">

176 + <form class="browser-address-form" aria-label="Browser navigation" @submit.prevent="$store.browserPage.go()">

177 <span class="material-symbols-outlined browser-address-icon">language</span>

178 - <input class="browser-address" x-model="$store.browserPage.address"

178 + <input class="browser-address" aria-label="Browser address" name="browser_address" x-model="$store.browserPage.address"

179 @focus="$store.browserPage.onAddressFocus()" @blur="$store.browserPage.onAddressBlur()"

180 :disabled="$store.browserPage.isBusy()"

181 placeholder="https://example.com" autocomplete="off" />

tests/test_browser_agent_regressions.py

+91 -1

@@ -1237,7 +1237,7 @@ def test_browser_content_helper_keeps_label_wrapped_controls_referenceable():

1237 PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"

1238 ).read_text(encoding="utf-8")

1239

1240 - assert 'const VERSION = "11"' in helper

1240 + assert 'const VERSION = "12"' in helper

1241 assert "function patchOpenShadowDom" in helper

1242 assert "Element.prototype.attachShadow = patched" in helper

1243 assert "const REQUIRED_API_NAMES = Object.freeze([" in helper

@@ -1249,6 +1249,18 @@ def test_browser_content_helper_keeps_label_wrapped_controls_referenceable():

1249 assert "getLabelElementText(labelElement, element)" in helper

1250 assert "return renderControlLabelReferences(node, context);" in helper

1251 assert "return renderControlLabelReferences(element, context);" in helper

1252 + assert "function isGlobalOrDelegatedEventBinding" in helper

1253 + assert 'parts.includes("window")' in helper

1254 + assert 'parts.includes("outside")' in helper

1255 +

1256 +

1257 +def test_browser_panel_exposes_agent_friendly_address_input():

1258 + panel = (

1259 + PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html"

1260 + ).read_text(encoding="utf-8")

1261 +

1262 + assert 'class="browser-address-form" aria-label="Browser navigation"' in panel

1263 + assert 'class="browser-address" aria-label="Browser address" name="browser_address"' in panel

1264

1265

1266 def test_browser_runtime_requires_current_content_helper_for_modifier_clicks():

@@ -1637,6 +1649,9 @@ async def test_browser_tool_dispatches_v1_agent_actions(monkeypatch):

1649 await execute(action="select_option", browser_id=1, ref=6, value="CA")

1650 await execute(action="set_checked", browser_id=1, ref=7, checked=False)

1651 await execute(action="upload_file", browser_id=1, ref=8, paths=["/tmp/a.txt"])

1652 + await execute(action="key_chord", browser_id=1, keys="CTRL+A")

1653 + await execute(action="click", browser_id=1, x=10, y=20)

1654 + await execute(action="type", browser_id=1, text="agent-zero.ai")

1655

1656 assert calls == [

1657 ("screenshot_file", (1,), {"quality": 91, "full_page": True, "path": "/tmp/a.jpg"}),

@@ -1690,6 +1705,81 @@ async def test_browser_tool_dispatches_v1_agent_actions(monkeypatch):

1705 ("select_option", (1, 6), {"value": "CA", "values": None}),

1706 ("set_checked", (1, 7), {"checked": False}),

1707 ("upload_file", (1, 8), {"path": "", "paths": ["/tmp/a.txt"]}),

1708 + ("key_chord", (1, ["Control", "A"]), {}),

1709 + ("mouse", (1, "click", 10, 20), {"button": "left", "modifiers": None}),

1710 + ("keyboard", (1,), {"key": "", "text": "agent-zero.ai"}),

1711 + ]

1712 +

1713 +

1714 +@pytest.mark.anyio

1715 +async def test_browser_tool_resolves_selector_for_reference_actions(monkeypatch):

1716 + calls = []

1717 +

1718 + class FakeRuntime:

1719 + async def call(self, method, *args, **kwargs):

1720 + calls.append((method, args, kwargs))

1721 + if method == "content":

1722 + return {"input.browser-address": "[input text 31] Browser address"}

1723 + return {"ok": True, "method": method, "args": args, "kwargs": kwargs}

1724 +

1725 + async def fake_get_runtime(context_id, create=True, agent=None):

1726 + del create, agent

1727 + assert context_id == "ctx"

1728 + return FakeRuntime()

1729 +

1730 + monkeypatch.setattr(browser_tool_module, "get_runtime", fake_get_runtime)

1731 + tool = browser_tool_module.Browser(

1732 + agent=SimpleNamespace(context=SimpleNamespace(id="ctx")),

1733 + name="browser",

1734 + method=None,

1735 + args={},

1736 + message="",

1737 + loop_data=None,

1738 + )

1739 +

1740 + response = await tool.execute(

1741 + action="type_submit",

1742 + browser_id=1,

1743 + selector="input.browser-address",

1744 + text="agent-zero.ai",

1745 + )

1746 +

1747 + assert response.break_loop is False

1748 + assert calls == [

1749 + ("content", (1, {"selector": "input.browser-address"}), {}),

1750 + ("type_submit", (1, "31", "agent-zero.ai"), {}),

1751 + ]

1752 +

1753 +

1754 +@pytest.mark.anyio

1755 +async def test_browser_runtime_multi_accepts_human_shaped_input_calls():

1756 + calls = []

1757 + core = _BrowserRuntimeCore("ctx")

1758 +

1759 + async def fake_mouse(browser_id, event_type, x, y, *, button="left", modifiers=None):

1760 + calls.append(("mouse", browser_id, event_type, x, y, button, modifiers))

1761 + return {"ok": True}

1762 +

1763 + async def fake_keyboard(browser_id, *, key="", text=""):

1764 + calls.append(("keyboard", browser_id, key, text))

1765 + return {"ok": True}

1766 +

1767 + async def fake_key_chord(browser_id, keys):

1768 + calls.append(("key_chord", browser_id, keys))

1769 + return {"ok": True}

1770 +

1771 + core.mouse = fake_mouse

1772 + core.keyboard = fake_keyboard

1773 + core.key_chord = fake_key_chord

1774 +

1775 + assert await core._dispatch_call({"action": "click", "browser_id": 1, "x": 10, "y": 20}) == {"ok": True}

1776 + assert await core._dispatch_call({"action": "type", "browser_id": 1, "text": "agent-zero.ai"}) == {"ok": True}

1777 + assert await core._dispatch_call({"action": "key_chord", "browser_id": 1, "keys": "CTRL+A"}) == {"ok": True}

1778 +

1779 + assert calls == [

1780 + ("mouse", 1, "click", 10.0, 20.0, "left", None),

1781 + ("keyboard", 1, "", "agent-zero.ai"),

1782 + ("key_chord", 1, ["Control", "A"]),

1783 ]

1784

1785

tests/test_host_browser_connector.py

+21 -2

@@ -245,6 +245,9 @@ def test_connector_runtime_normalizes_host_navigation_payloads(monkeypatch):

245 [

246 {"action": "open", "url": "example.com"},

247 {"action": "navigate", "browser_id": 1, "url": "127.0.0.1:8000/path"},

248 + {"action": "click", "browser_id": 1, "x": 12, "y": 34},

249 + {"action": "type", "browser_id": 1, "text": "agent-zero.ai"},

250 + {"action": "key_chord", "browser_id": 1, "keys": "CTRL+A"},

251 {

252 "action": "multi",

253 "calls": [{"action": "open", "url": "nested.example"}],

@@ -258,9 +261,25 @@ def test_connector_runtime_normalizes_host_navigation_payloads(monkeypatch):

261 assert navigate_payload["url"] == "https://novinky.cz/"

262 assert multi_payload["calls"][0]["url"] == "https://example.com/"

263 assert multi_payload["calls"][1]["url"] == "http://127.0.0.1:8000/path"

261 - assert multi_payload["calls"][2]["calls"][0]["url"] == "https://nested.example/"

262 - assert multi_payload["calls"][3] == {"action": "content", "browser_id": 1}

264 + assert multi_payload["calls"][2] == {

265 + "action": "mouse",

266 + "browser_id": 1,

267 + "x": 12,

268 + "y": 34,

269 + "event_type": "click",

270 + "button": "left",

271 + }

272 + assert multi_payload["calls"][3] == {

273 + "action": "keyboard",

274 + "browser_id": 1,

275 + "text": "agent-zero.ai",

276 + "key": "",

277 + }

278 + assert multi_payload["calls"][4]["keys"] == ["Control", "A"]

279 + assert multi_payload["calls"][5]["calls"][0]["url"] == "https://nested.example/"

280 + assert multi_payload["calls"][6] == {"action": "content", "browser_id": 1}

281 assert open_payload["profile_mode"] == "existing"

282 + assert runtime._payload_for_call("key_chord", 1, "CTRL+A")["keys"] == ["Control", "A"]

283

284

285 def test_connector_runtime_forwards_host_profile_mode(monkeypatch):