Require visual verification for computer-use captures
Sanitize embedded image data URLs from prompt token estimates so screenshot attachments do not explode context accounting.\n\nStrengthen computer_use_remote prompt, skill, and capture-result text so state-changing desktop actions are treated as attempts until a fresh screen visibly confirms the requested outcome.
Alessandro committed
May 23, 2026 at 10:32 UTC
1f34b87c00edacbc5eeea9d6a41bc16f2d7f062d
8 files changed
+62
-5
agent.py
+1
-1
@@ -584,7 +584,7 @@ class Agent:
584
Agent.DATA_NAME_CTX_WINDOW,
585
{
586
"text": full_text,
587
- "tokens": tokens.approximate_tokens(full_text),
587
+ "tokens": tokens.approximate_prompt_tokens(full_text),
588
},
589
)
590
helpers/tokens.py
+19
@@ -1,8 +1,13 @@
1
+import re
2
from typing import Literal
3
import tiktoken
4
5
APPROX_BUFFER = 1.1
6
TRIM_BUFFER = 0.8
7
+EMBEDDED_IMAGE_DATA_PLACEHOLDER = "[embedded image data omitted from token estimate]"
8
+_EMBEDDED_IMAGE_DATA_URL_PATTERN = re.compile(
9
+ r"data:(image/[A-Za-z0-9.+-]+(?:;[A-Za-z0-9.+-]+=[A-Za-z0-9.+/=_-]+)*);base64,[A-Za-z0-9+/=_-]+"
10
+)
11
12
13
def count_tokens(text: str, encoding_name="cl100k_base") -> int:
@@ -25,6 +30,20 @@ def approximate_tokens(
30
return int(count_tokens(text) * APPROX_BUFFER)
31
32
33
+def sanitize_embedded_image_data_urls(text: str) -> str:
34
+ if not text:
35
+ return text
36
+
37
+ return _EMBEDDED_IMAGE_DATA_URL_PATTERN.sub(
38
+ f"data:\\1;base64,{EMBEDDED_IMAGE_DATA_PLACEHOLDER}",
39
+ text,
40
+ )
41
+
42
+
43
+def approximate_prompt_tokens(text: str) -> int:
44
+ return approximate_tokens(sanitize_embedded_image_data_urls(text))
45
+
46
+
47
def trim_to_tokens(
48
text: str,
49
max_tokens: int,
plugins/_a0_connector/api/v1/token_status.py
+1
-1
@@ -49,7 +49,7 @@ class TokenStatus(connector_base.ProtectedConnectorApiHandler):
49
history_output, ai_label="assistant", human_label="user"
50
)
51
if full_text.strip():
52
- token_count = tokens_helper.approximate_tokens(full_text)
52
+ token_count = tokens_helper.approximate_prompt_tokens(full_text)
53
except Exception:
54
token_count = None
55
plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md
+2
@@ -8,6 +8,8 @@ If the tool reports no CLI, disabled computer use, or `COMPUTER_USE_REARM_REQUIR
8
9
Call `start_session` before screen-driven tasks. Use `status` for state only, `capture` for screenshots without an action, and `stop_session` when the desktop task is complete. Interactive actions should use normalized global-screen coordinates from the most recent capture.
10
11
+State-changing actions automatically attach a fresh screen after they run. Treat key presses, clicks, scrolling, typing, and window-manager shortcuts as attempts, not success: inspect the latest attached screen, or one explicit `capture` if it is unclear or unchanged, before saying the requested outcome happened. This is mandatory for Ubuntu/Wayland shortcuts such as `Alt+F9`.
12
+
13
```json
14
{
15
"tool_name": "computer_use_remote",
plugins/_a0_connector/skills/host-computer-use/SKILL.md
+3
-1
@@ -53,7 +53,7 @@ If any tool result contains `COMPUTER_USE_REARM_REQUIRED` or `status=rearm requi
53
54
1. Call `start_session` first.
55
2. Decide from the latest screenshot, not from memory.
56
-3. Interactive actions (`move`, `click`, `scroll`, `key`, `type`) already attach a fresh screenshot after they run.
56
+3. Interactive actions (`move`, `click`, `scroll`, `key`, `type`) already attach a fresh screenshot after they run; inspect it before claiming the requested outcome succeeded.
57
4. Use `status` for state without starting a session.
58
5. Use `capture` only when you need another screenshot without taking an action.
59
@@ -67,6 +67,8 @@ If any tool result contains `COMPUTER_USE_REARM_REQUIRED` or `status=rearm requi
67
- If a click dismisses a menu or popup without producing the expected next UI, treat that attempt as failed.
68
- If the same approach has already failed twice without visible progress, switch strategy instead of repeating it.
69
- Do not infer focus or task completion from chat logs, sidebars, tool summaries, or status text.
70
+- Never claim a window was hidden, minimized, moved, text was submitted, or navigation completed until the latest screenshot visibly confirms it.
71
+- Treat Ubuntu/Wayland window-manager shortcuts such as `Alt+F9` as attempts only; verify the result from the fresh screenshot before deciding what happened.
72
- For browser-navigation tasks done through this tool, only claim success if the browser content area visibly shows the destination page or result.
73
- If the attached screenshot appears unchanged after a state-changing action, use one explicit `capture` to verify before repeating the same action.
74
- Use `type(..., submit=true)` only for URL or navigation-style entry where Enter should fire immediately after typing.
plugins/_a0_connector/tools/computer_use_remote.py
+5
-2
@@ -23,6 +23,9 @@ COMPUTER_USE_OP_TIMEOUT = 180.0
23
COMPUTER_USE_OP_EVENT = "connector_computer_use_op"
24
CAPTURE_TOKENS_ESTIMATE = 1500
25
MAX_CAPTURE_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024
26
+CAPTURE_VERIFICATION_NOTE = (
27
+ "Do not claim success unless this screen visibly confirms the requested outcome."
28
+)
29
REARM_REQUIRED_DEFAULT_MESSAGE = (
30
"Computer use is configured, but the installed desktop-control backend is not armed."
31
)
@@ -204,7 +207,7 @@ class ComputerUseRemote(Tool):
207
summary = self._record_capture(capture_data)
208
except Exception as exc:
209
return f"Automatic screen refresh failed: {exc}"
207
- return f"Latest screen attached: {summary}"
210
+ return f"Latest screen attached: {summary} {CAPTURE_VERIFICATION_NOTE}"
211
212
def _auto_capture_settle_seconds(self, action: str) -> float:
213
if action == "start_session":
@@ -288,7 +291,7 @@ class ComputerUseRemote(Tool):
291
292
if action == "capture":
293
summary = self._record_capture(data)
291
- return f"Current screen attached: {summary}"
294
+ return f"Current screen attached: {summary} {CAPTURE_VERIFICATION_NOTE}"
295
if action == "status":
296
return self._format_status(data)
297
if action == "start_session":
tests/test_a0_connector_prompt_gating.py
+18
@@ -123,6 +123,24 @@ def test_computer_use_remote_prompt_keeps_runtime_failures_actionable():
123
assert "/computer-use on" in prompt
124
125
126
+def test_computer_use_remote_prompt_requires_visual_verification_after_actions():
127
+ prompt = _apply_gate(_context_id())
128
+ skill = (
129
+ PROJECT_ROOT
130
+ / "plugins"
131
+ / "_a0_connector"
132
+ / "skills"
133
+ / "host-computer-use"
134
+ / "SKILL.md"
135
+ ).read_text(encoding="utf-8")
136
+
137
+ assert "Treat key presses, clicks, scrolling, typing" in prompt
138
+ assert "attempts, not success" in prompt
139
+ assert "Alt+F9" in prompt
140
+ assert "visibly confirms" in skill
141
+ assert "Ubuntu/Wayland" in skill
142
+
143
+
144
def test_remote_file_and_exec_tools_are_standard_tool_prompts_independent_from_context():
145
text_stub = (PROMPT_ROOT / "agent.system.tool.text_editor_remote.md").read_text(encoding="utf-8")
146
exec_stub = (PROMPT_ROOT / "agent.system.tool.code_execution_remote.md").read_text(encoding="utf-8")
tests/test_default_prompt_budget.py
+13
@@ -74,3 +74,16 @@ def test_a0_small_profile_removed_and_prompt_text_generic():
74
75
for path in _iter_prompt_files():
76
assert "a0_small" not in path.read_text(encoding="utf-8")
77
+
78
+
79
+def test_prompt_token_estimate_omits_embedded_image_data_urls():
80
+ embedded_png = "data:image/png;base64," + ("ABCDabcd0123+/==" * 20_000)
81
+ prompt_text = f"user: please inspect this screenshot {embedded_png}"
82
+
83
+ sanitized = tokens.sanitize_embedded_image_data_urls(prompt_text)
84
+
85
+ assert "ABCDabcd0123+/==" not in sanitized
86
+ assert "data:image/png;base64," in sanitized
87
+ assert tokens.EMBEDDED_IMAGE_DATA_PLACEHOLDER in sanitized
88
+ assert tokens.approximate_prompt_tokens(prompt_text) < 100
89
+ assert tokens.approximate_prompt_tokens(prompt_text) < tokens.approximate_tokens(prompt_text) / 100