Expose computer-use captures as vision messages
Store computer-use screenshots as standalone RawMessage entries after the textual tool result, matching the existing vision_load path so the model receives a real multimodal message. Prefer shared screenshot file paths over base64 artifacts when available, and tighten host computer-use guidance so agents stop instead of proceeding from unverified state when a screenshot is not visible.
Alessandro committed
May 23, 2026 at 11:51 UTC
cee9abfde44fd909a35f387329063e5480b9e0d8
5 files changed
+29
-7
plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md
+1
-1
@@ -10,7 +10,7 @@ If the tool reports no CLI, disabled computer use, or `COMPUTER_USE_REARM_REQUIR
10
11
Call `start_session` before screen-driven tasks. Use `status` for state only, `capture` for screenshots without an action, and `stop_session` when the desktop task is complete. Interactive actions should use normalized global-screen coordinates from the most recent capture.
12
13
-State-changing actions automatically attach a fresh screen after they run. Treat key presses, clicks, scrolling, typing, and window-manager shortcuts as attempts, not success: inspect the latest attached screen, or one explicit `capture` if it is unclear or unchanged, before saying the requested outcome happened. For Ubuntu/GNOME/Wayland hide-window tasks, prefer `Super+H` (`{"action":"key","keys":["Super","H"]}`) for the active window; do not use `Alt+F9` as the primary hide/minimize shortcut because it often leaves the window visible. A `type` result only proves keystrokes were sent; it does not prove the window was hidden or that text landed in the intended place.
13
+State-changing actions automatically attach a fresh screen after they run. Treat key presses, clicks, scrolling, typing, and window-manager shortcuts as attempts, not success: inspect the latest attached screen, or one explicit `capture` if it is unclear or unchanged, before saying the requested outcome happened. If the tool says a screen was attached but you cannot actually inspect the image, stop and report that visual verification is unavailable; do not continue by assuming the host state. For Ubuntu/GNOME/Wayland hide-window tasks, prefer `Super+H` (`{"action":"key","keys":["Super","H"]}`) for the active window; do not use `Alt+F9` as the primary hide/minimize shortcut because it often leaves the window visible. A `type` result only proves keystrokes were sent; it does not prove the window was hidden or that text landed in the intended place.
14
15
```json
16
{
plugins/_a0_connector/skills/host-computer-use/SKILL.md
+1
@@ -80,6 +80,7 @@ If any tool result contains `COMPUTER_USE_REARM_REQUIRED` or `status=rearm requi
80
## Operating Rules
81
82
- Only the latest screenshot or a definitive tool result counts as evidence.
83
+- If a tool result says a screenshot was attached but you cannot actually see the image, stop and report that visual verification is unavailable. Do not continue with another action from an assumed host state.
84
- The current API uses normalized global screen coordinates; do not assume window ids, element indexes, background-safe input, or semantic click targets unless the runtime explicitly advertises them.
85
- Prefer accessibility and semantic UI paths first: shortcuts, command palettes, menu accelerators, address/search bars, focus traversal, and other keyboard-accessible controls.
86
- Prefer `key` and `type` over pointer actions whenever a reliable keyboard path exists.
plugins/_a0_connector/tools/computer_use_remote.py
+23
-6
@@ -6,6 +6,7 @@ from pathlib import Path
6
import uuid
7
from typing import Any
8
9
+from helpers import history
10
from helpers.print_style import PrintStyle
11
from helpers.tool import Response, Tool
12
from helpers.ws import NAMESPACE
@@ -24,7 +25,9 @@ COMPUTER_USE_OP_EVENT = "connector_computer_use_op"
25
CAPTURE_TOKENS_ESTIMATE = 1500
26
MAX_CAPTURE_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024
27
CAPTURE_VERIFICATION_NOTE = (
27
- "Do not claim success unless this screen visibly confirms the requested outcome."
28
+ "Inspect the attached screenshot before the next action; do not claim or proceed "
29
+ "from assumed state. If you cannot see the screenshot, stop and report that visual "
30
+ "verification is unavailable."
31
)
32
REARM_REQUIRED_DEFAULT_MESSAGE = (
33
"Computer use is configured, but the installed desktop-control backend is not armed."
@@ -144,16 +147,21 @@ class ComputerUseRemote(Tool):
147
148
text = _sanitize_tool_text(response.message.strip())
149
additional = dict(response.additional)
150
+ raw_content = additional.pop("raw_content", None)
151
+ preview = str(additional.pop("preview", "") or "").strip() or text
152
token_estimate = self._coerce_token_estimate(additional.pop("_tokens", CAPTURE_TOKENS_ESTIMATE))
153
log_id = str(getattr(getattr(self, "log", None), "id", "") or "")
149
- message = self.agent.hist_add_tool_result(
154
+ self.agent.hist_add_tool_result(
155
self.name,
156
text,
157
id=log_id,
158
**additional,
159
)
155
- if hasattr(message, "tokens"):
156
- message.tokens = token_estimate
160
+ self.agent.hist_add_message(
161
+ False,
162
+ content=history.RawMessage(raw_content=raw_content, preview=preview),
163
+ tokens=token_estimate,
164
+ )
165
166
agent_name = str(getattr(self.agent, "agent_name", "Agent Zero") or "Agent Zero")
167
PrintStyle(
@@ -472,6 +480,14 @@ class ComputerUseRemote(Tool):
480
message.tokens = message.calculate_tokens()
481
482
def _resolve_capture_ref(self, data: dict[str, Any]) -> tuple[str, str]:
483
+ path_error: FileNotFoundError | None = None
484
+ try:
485
+ image_path, display_path = self._resolve_capture_path(data)
486
+ except FileNotFoundError as exc:
487
+ path_error = exc
488
+ else:
489
+ return display_path, image_path.stem
490
+
491
artifact = data.get("artifact")
492
if isinstance(artifact, dict) and str(artifact.get("encoding", "")).strip().lower() == "base64":
493
encoded = str(artifact.get("data") or "")
@@ -488,8 +504,9 @@ class ComputerUseRemote(Tool):
504
filename = _safe_filename(str(artifact.get("filename") or "computer-use-capture.png"))
505
return f"data:{mime};base64,{encoded}", Path(filename).stem
506
491
- image_path, display_path = self._resolve_capture_path(data)
492
- return display_path, image_path.stem
507
+ if path_error is not None:
508
+ raise path_error
509
+ raise FileNotFoundError("Capture artifact was not found in the tool response.")
510
511
def _collect_capture_messages(self, history_obj: Any) -> list[Any]:
512
messages: list[Any] = []
tests/test_a0_connector_prompt_gating.py
+3
@@ -136,9 +136,12 @@ def test_computer_use_remote_prompt_requires_visual_verification_after_actions()
136
137
assert "Treat key presses, clicks, scrolling, typing" in prompt
138
assert "attempts, not success" in prompt
139
+ assert "visual verification is unavailable" in prompt
140
+ assert "do not continue by assuming the host state" in prompt
141
assert "Super+H" in prompt
142
assert '["Super","H"]' in prompt
143
assert "Alt+F9" in prompt
144
+ assert "cannot actually see the image" in skill
145
assert "Do not use `Alt+F9` as the primary hide/minimize shortcut" in skill
146
assert "A `type` tool result only confirms keystrokes were sent" in skill
147
assert "do not type follow-up text into the active field" in skill
tests/test_tool_action_contracts.py
+1
@@ -568,5 +568,6 @@ def test_computer_use_remote_is_runtime_checked_standard_tool():
568
assert '"tool_name": "computer_use_remote"' in standard_prompt_text
569
assert "not scoped to a single chat context" in standard_prompt_text
570
assert "checked when the tool runs" in standard_prompt_text
571
+ assert "visual verification is unavailable" in standard_prompt_text
572
assert '"tool_name": "computer_use_remote"' in skill_text
573
assert "Beta desktop control" in skill_text