Expose computer_use_remote as a runtime-checked tool
Add the standard tool prompt contract so the model can call computer_use_remote in live sessions. Keep availability, CLI enablement, trust mode, and re-arm enforcement as runtime checks instead of prompt-loader gating. Update connector prompt and prompt-budget tests to cover the new exposure path.
Alessandro committed
May 23, 2026 at 09:40 UTC
60c36d16d85732b7d844e64e41b8b37b718e9d00
5 files changed
+100
-30
plugins/_a0_connector/extensions/python/_functions/extensions/python/system_prompt/_11_tools_prompt/build_prompt/end/_70_include_remote_tool_stubs.py
+22
-1
@@ -5,6 +5,27 @@ from typing import Any
5
from helpers.extension import Extension
6
7
8
+_COMPUTER_USE_PROMPT = "agent.system.tool.computer_use_remote.md"
9
+_COMPUTER_USE_TOOL_MARKER = '"tool_name": "computer_use_remote"'
10
+
11
+
12
class IncludeRemoteToolStubs(Extension):
13
def execute(self, data: dict[str, Any] = {}, **kwargs: Any) -> None:
10
- return
14
+ if self.agent is None:
15
+ return
16
+ if not isinstance(data, dict):
17
+ return
18
+ result = data.get("result")
19
+ if not isinstance(result, str):
20
+ return
21
+ if _COMPUTER_USE_TOOL_MARKER in result:
22
+ return
23
+
24
+ try:
25
+ prompt = self.agent.read_prompt(_COMPUTER_USE_PROMPT).strip()
26
+ except Exception:
27
+ return
28
+ if not prompt:
29
+ return
30
+
31
+ data["result"] = f"{result.rstrip()}\n\n{prompt}"
plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md
new
+31
@@ -0,0 +1,31 @@
1
+### computer_use_remote
2
+
3
+Runtime-gated beta desktop control through a connected A0 CLI on the user's host machine. The callable contract is available in the tool prompt. Availability, backend support, and trust mode are checked when the tool runs, together with CLI presence, local enablement, and re-arm state. Computer Use enablement is scoped to the current CLI session, not scoped to a single chat context.
4
+
5
+Use this for native host desktop UI inspection, screenshots, clicking, scrolling, typing, key presses, and status checks. Do not use it for ordinary web-page navigation or host-browser control; use the browser tool for web pages unless browser automation cannot express the task. For complex desktop workflows, load and follow skill `host-computer-use` before proceeding.
6
+
7
+If the tool reports no CLI, disabled computer use, or `COMPUTER_USE_REARM_REQUIRED`, stop and tell the user to run `/computer-use on` in A0 CLI and approve any host permission prompt.
8
+
9
+Call `start_session` before screen-driven tasks. Use `status` for state only, `capture` for screenshots without an action, and `stop_session` when the desktop task is complete. Interactive actions should use normalized global-screen coordinates from the most recent capture.
10
+
11
+```json
12
+{
13
+ "tool_name": "computer_use_remote",
14
+ "tool_args": {
15
+ "action": "status"
16
+ }
17
+}
18
+```
19
+
20
+Required argument:
21
+- `action`: one of `start_session`, `status`, `capture`, `move`, `click`, `scroll`, `key`, `type`, `stop_session`
22
+
23
+Optional arguments by action:
24
+- `session_id`: session returned by `start_session`
25
+- `x`, `y`: normalized `[0,1]` global-screen coordinates for `move` and `click`
26
+- `button`: `left`, `right`, or `middle` for `click`
27
+- `count`: click count for `click`
28
+- `dx`, `dy`: scroll amounts for `scroll`
29
+- `key` or `keys`: key press value for `key`
30
+- `text`: text to type for `type`
31
+- `submit`: boolean Enter-after-type flag for `type`
tests/test_a0_connector_prompt_gating.py
+33
-24
@@ -95,31 +95,32 @@ def _apply_gate(context_id: str) -> str:
95
return data["result"]
96
97
98
-def _subscribe(
99
- context_id: str,
100
- *,
101
- remote_files: dict | None = None,
102
- remote_exec: dict | None = None,
103
- computer_use: dict | None = None,
104
-) -> str:
105
- sid = _sid()
106
- ws_runtime.register_sid(sid)
107
- ws_runtime.subscribe_sid_to_context(sid, context_id)
108
- if remote_files is not None:
109
- ws_runtime.store_sid_remote_file_metadata(sid, remote_files)
110
- if remote_exec is not None:
111
- ws_runtime.store_sid_remote_exec_metadata(sid, remote_exec)
112
- if computer_use is not None:
113
- ws_runtime.store_sid_computer_use_metadata(sid, computer_use)
114
- return sid
115
-
116
-
117
-def test_legacy_dynamic_remote_tool_gate_is_noop():
98
+def test_remote_tool_gate_includes_runtime_checked_computer_use_contract():
99
prompt = _apply_gate(_context_id())
100
101
assert "text_editor_remote tool" not in prompt
102
assert "code_execution_remote tool" not in prompt
122
- assert "computer_use_remote tool" not in prompt
103
+ assert '"tool_name": "computer_use_remote"' in prompt
104
+ assert "### computer_use_remote" in prompt
105
+ assert "checked when the tool runs" in prompt
106
+
107
+
108
+def test_computer_use_remote_prompt_is_cli_session_wide_not_context_scoped():
109
+ prompt = _apply_gate(_context_id())
110
+
111
+ assert "### computer_use_remote" in prompt
112
+ assert '"tool_name": "computer_use_remote"' in prompt
113
+ assert "scoped to the current CLI session" in prompt
114
+ assert "not scoped to a single chat context" in prompt
115
+
116
+
117
+def test_computer_use_remote_prompt_keeps_runtime_failures_actionable():
118
+ prompt = _apply_gate(_context_id())
119
+
120
+ assert "no CLI" in prompt
121
+ assert "disabled computer use" in prompt
122
+ assert "COMPUTER_USE_REARM_REQUIRED" in prompt
123
+ assert "/computer-use on" in prompt
124
125
126
def test_remote_file_and_exec_tools_are_standard_tool_prompts_independent_from_context():
@@ -132,7 +133,7 @@ def test_remote_file_and_exec_tools_are_standard_tool_prompts_independent_from_c
133
assert "Availability and permissions are checked when the tool runs" in exec_stub
134
135
135
-def test_beta_computer_use_remote_is_skill_only_not_standard_tool_prompt():
136
+def test_computer_use_remote_is_standard_prompt_with_runtime_checks():
137
skill = (
138
PROJECT_ROOT
139
/ "plugins"
@@ -141,8 +142,12 @@ def test_beta_computer_use_remote_is_skill_only_not_standard_tool_prompt():
142
/ "host-computer-use"
143
/ "SKILL.md"
144
)
145
+ standard_prompt = PROMPT_ROOT / "agent.system.tool.computer_use_remote.md"
146
145
- assert not (PROMPT_ROOT / "agent.system.tool.computer_use_remote.md").exists()
147
+ assert not (PROMPT_ROOT / "agent.system.runtime_tool.computer_use_remote.md").exists()
148
+ assert standard_prompt.exists()
149
+ assert '"tool_name": "computer_use_remote"' in standard_prompt.read_text(encoding="utf-8")
150
+ assert "checked when the tool runs" in standard_prompt.read_text(encoding="utf-8")
151
assert '"tool_name": "computer_use_remote"' in skill.read_text(encoding="utf-8")
152
153
@@ -329,6 +334,7 @@ def test_remote_affordance_skills_parse():
334
def test_remote_tool_stubs_are_self_contained_and_reference_per_tool_skills():
335
text_stub = (PROMPT_ROOT / "agent.system.tool.text_editor_remote.md").read_text(encoding="utf-8")
336
exec_stub = (PROMPT_ROOT / "agent.system.tool.code_execution_remote.md").read_text(encoding="utf-8")
337
+ computer_stub = (PROMPT_ROOT / "agent.system.tool.computer_use_remote.md").read_text(encoding="utf-8")
338
computer_skill = (
339
PROJECT_ROOT
340
/ "plugins"
@@ -342,13 +348,16 @@ def test_remote_tool_stubs_are_self_contained_and_reference_per_tool_skills():
348
assert "optionally load skill `host-code-execution`" in exec_stub
349
assert '"tool_name": "text_editor_remote"' in text_stub
350
assert '"tool_name": "code_execution_remote"' in exec_stub
351
+ assert '"tool_name": "computer_use_remote"' in computer_stub
352
+ assert "load and follow skill `host-computer-use`" in computer_stub
353
assert '"tool_name": "computer_use_remote"' in computer_skill
346
- assert "Availability, backend support, and trust mode are checked when the tool runs" in computer_skill
354
+ assert "Availability, backend support, and trust mode are checked when the tool runs" in computer_stub
355
assert "not `code_execution_tool`" in exec_stub
356
assert "not to" in exec_stub
357
assert "Docker/server/container execution" in exec_stub
358
assert "a0-cli-remote-workflows" not in text_stub
359
assert "a0-cli-remote-workflows" not in exec_stub
360
+ assert "a0-cli-remote-workflows" not in computer_stub
361
assert "a0-cli-remote-workflows" not in computer_skill
362
363
tests/test_default_prompt_budget.py
+3
-2
@@ -50,7 +50,7 @@ async def test_default_agent0_prompt_budget_and_guardrails():
50
# surface plus skill metadata. Keep the guardrail close to the observed
51
# budget so prompt creep remains visible without pretending this surface is
52
# a tiny single-tool prompt.
53
- assert tokens.approximate_tokens(system_text) <= 8500
53
+ assert tokens.approximate_tokens(system_text) <= 10500
54
assert "`tool_name` must be one listed tool name" in system_text
55
assert "- tool_args: key value pairs tool arguments" in system_text
56
assert '"tool_name": "call_subordinate"' in system_text
@@ -62,7 +62,8 @@ async def test_default_agent0_prompt_budget_and_guardrails():
62
assert "informative but tight" in system_text
63
assert '"tool_name": "code_execution_remote"' in system_text
64
assert '"tool_name": "text_editor_remote"' in system_text
65
- assert '"tool_name": "computer_use_remote"' not in system_text
65
+ assert '"tool_name": "computer_use_remote"' in system_text
66
+ assert "Computer Use enablement is scoped to the current CLI session" in system_text
67
assert "host-computer-use" in system_text
68
69
tests/test_tool_action_contracts.py
+11
-3
@@ -548,17 +548,25 @@ def test_corrected_tool_prompts_only_teach_action_contract():
548
assert "Open Document, or Desktop edit actions" not in text
549
550
551
-def test_computer_use_remote_is_skill_gated():
551
+def test_computer_use_remote_is_runtime_checked_standard_tool():
552
project_root = Path(__file__).resolve().parents[1]
553
- prompt_path = (
553
+ standard_prompt_path = (
554
project_root
555
/ "plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md"
556
)
557
+ standard_prompt_text = standard_prompt_path.read_text(encoding="utf-8")
558
skill_text = (
559
project_root
560
/ "plugins/_a0_connector/skills/host-computer-use/SKILL.md"
561
).read_text(encoding="utf-8")
562
562
- assert not prompt_path.exists()
563
+ assert standard_prompt_path.exists()
564
+ assert not (
565
+ project_root
566
+ / "plugins/_a0_connector/prompts/agent.system.runtime_tool.computer_use_remote.md"
567
+ ).exists()
568
+ assert '"tool_name": "computer_use_remote"' in standard_prompt_text
569
+ assert "not scoped to a single chat context" in standard_prompt_text
570
+ assert "checked when the tool runs" in standard_prompt_text
571
assert '"tool_name": "computer_use_remote"' in skill_text
572
assert "Beta desktop control" in skill_text