Preserve computer-use capability metadata
Store and return the connector computer-use contract_version and nested capabilities payload in Agent Zero, surface the contract version in tool status text, and teach the remote computer-use prompt/skill to prefer the structured background dispatch contract over OS-name assumptions.
Alessandro committed
May 30, 2026 at 22:39 UTC
d8710c3f8145731d8bfa0900f0c3b2b2707d0e20
5 files changed
+82
-9
plugins/_a0_connector/helpers/ws_runtime.py
+13
@@ -1,6 +1,7 @@
1
from __future__ import annotations
2
3
import asyncio
4
+import copy
5
import threading
6
import time
7
from dataclasses import dataclass
@@ -58,6 +59,8 @@ class ComputerUseMetadata:
59
backend_id: str
60
backend_family: str
61
features: tuple[str, ...]
62
+ contract_version: int
63
+ capabilities: dict[str, Any]
64
support_reason: str
65
updated_at: float
66
@@ -337,6 +340,12 @@ def store_sid_computer_use_metadata(sid: str, payload: dict[str, Any]) -> Comput
340
features = tuple(str(item).strip() for item in features_value if str(item).strip())
341
else:
342
features = ()
343
+ capabilities_value = payload.get("capabilities")
344
+ capabilities = copy.deepcopy(capabilities_value) if isinstance(capabilities_value, dict) else {}
345
+ try:
346
+ contract_version = int(payload.get("contract_version") or 0)
347
+ except (TypeError, ValueError):
348
+ contract_version = 0
349
metadata = ComputerUseMetadata(
350
supported=bool(payload.get("supported")),
351
enabled=bool(payload.get("supported")) and bool(payload.get("enabled")),
@@ -348,6 +357,8 @@ def store_sid_computer_use_metadata(sid: str, payload: dict[str, Any]) -> Comput
357
backend_id=str(payload.get("backend_id", "") or "").strip(),
358
backend_family=str(payload.get("backend_family", "") or "").strip(),
359
features=features,
360
+ contract_version=contract_version,
361
+ capabilities=capabilities,
362
support_reason=str(payload.get("support_reason", "") or "").strip(),
363
updated_at=time.time(),
364
)
@@ -377,6 +388,8 @@ def computer_use_metadata_for_sid(sid: str) -> dict[str, Any] | None:
388
"backend_id": metadata.backend_id,
389
"backend_family": metadata.backend_family,
390
"features": list(metadata.features),
391
+ "contract_version": metadata.contract_version,
392
+ "capabilities": copy.deepcopy(metadata.capabilities),
393
"support_reason": metadata.support_reason,
394
"updated_at": metadata.updated_at,
395
}
plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md
+3
-1
@@ -8,7 +8,7 @@ This is the only desktop-control path for the user's connected host/local comput
8
9
If the tool reports no CLI, disabled computer use, or `COMPUTER_USE_REARM_REQUIRED`, stop and tell the user to run `/computer-use on` in A0 CLI and approve any host permission prompt.
10
11
-Call `start_session` before screen-driven tasks. Use `status` for state only, `capture` for screenshots without an action, and `stop_session` when the desktop task is complete. When the backend advertises native window and element-index features, prefer `list_windows` -> `get_window_state` -> `element_action` with `dispatch: "background"` before using global coordinates. Interactive coordinate actions should use normalized global-screen coordinates from the most recent capture.
11
+Call `start_session` before screen-driven tasks. Use `status` for state only, `capture` for screenshots without an action, and `stop_session` when the desktop task is complete. Read `backend_id`, `backend_family`, `features`, and the structured `capabilities` object in status/session results. When capabilities report native windows, window state, element indexes, and background dispatch, prefer `list_windows` -> `get_window_state` -> `element_action` with `dispatch: "background"` before using global coordinates. Interactive coordinate actions should use normalized global-screen coordinates from the most recent capture.
12
13
Some actions are backend-specific and intentionally documented only in backend skills. If `status` or `start_session` reports backend-specific features or tells you to load a backend skill, load and follow that skill before using those backend-only actions. For structural targeting details, load and follow the backend-specific skill such as `host-computer-use-macos` or `host-computer-use-windows`; do not apply one backend's guidance to another backend.
14
@@ -39,3 +39,5 @@ Optional arguments by action:
39
- `key` or `keys`: key press value for `key`
40
- `text`: text to type for `type`
41
- `submit`: boolean Enter-after-type flag for `type`
42
+
43
+Status/session results may include `contract_version` and `capabilities`. Treat `capabilities.identity.pid`, `capabilities.identity.window_id`, `capabilities.identity.element_index`, and `capabilities.dispatch.background` as the authoritative cross-platform contract for whether the native background loop is available. Use `features` for backend-specific refinements and skill selection.
plugins/_a0_connector/skills/host-computer-use/SKILL.md
+9
-8
@@ -75,14 +75,15 @@ If any tool result contains `COMPUTER_USE_REARM_REQUIRED` or `status=rearm requi
75
## Core Loop
76
77
1. Call `start_session` first.
78
-2. Read the returned `backend_id`, `backend_family`, and `features`; load a backend-specific Computer Use skill when the task needs backend-only affordances.
79
-3. If the backend advertises `native-window-list`, call `list_windows` before using coordinates.
80
-4. If the backend advertises `window-state` and `element-index-targeting`, call `get_window_state` for the target `pid`/`window_id`, then use `element_action` with `dispatch: "background"` by default.
81
-5. If `element_action` reports `background_unavailable`, use `dispatch: "auto"` or `dispatch: "foreground"` only when foreground control is acceptable for the user/task.
82
-6. Decide final success from the latest screenshot or a definitive structural result, not from memory.
83
-7. Interactive actions already attach a fresh screenshot after they run; inspect it before claiming the requested outcome succeeded.
84
-8. Use `status` for state without starting a session.
85
-9. Use `capture` only when you need another screenshot without taking an action.
78
+2. Read the returned `backend_id`, `backend_family`, `features`, `contract_version`, and `capabilities`; load a backend-specific Computer Use skill when the task needs backend-only affordances.
79
+3. Prefer the structured `capabilities` object over guessing from OS names. Use `capabilities.identity.pid`, `capabilities.identity.window_id`, `capabilities.identity.element_index`, and `capabilities.dispatch.background` as the portable contract for the native background loop.
80
+4. If the backend advertises native window listing through capabilities or `native-window-list`, call `list_windows` before using coordinates.
81
+5. If the backend advertises window state and element-index targeting through capabilities or features, call `get_window_state` for the target `pid`/`window_id`, then use `element_action` with `dispatch: "background"` by default.
82
+6. If `element_action` reports `background_unavailable`, use `dispatch: "auto"` or `dispatch: "foreground"` only when foreground control is acceptable for the user/task.
83
+7. Decide final success from the latest screenshot or a definitive structural result, not from memory.
84
+8. Interactive actions already attach a fresh screenshot after they run; inspect it before claiming the requested outcome succeeded.
85
+9. Use `status` for state without starting a session.
86
+10. Use `capture` only when you need another screenshot without taking an action.
87
88
## Backend Skills
89
plugins/_a0_connector/tools/computer_use_remote.py
+5
@@ -505,6 +505,11 @@ class ComputerUseRemote(Tool):
505
if backend_family:
506
backend_text = f"{backend_text}/{backend_family}"
507
parts.append(f"backend={backend_text}")
508
+ contract_version = data.get("contract_version")
509
+ if not contract_version and isinstance(data.get("capabilities"), dict):
510
+ contract_version = data["capabilities"].get("contract_version")
511
+ if contract_version:
512
+ parts.append(f"contract=v{contract_version}")
513
if features:
514
parts.append(f"features={', '.join(features)}")
515
return ", ".join(parts)
tests/test_a0_connector_computer_use_metadata.py
new
+52
@@ -0,0 +1,52 @@
1
+from __future__ import annotations
2
+
3
+from plugins._a0_connector.helpers import ws_runtime
4
+
5
+
6
+def test_computer_use_metadata_preserves_structured_capabilities() -> None:
7
+ sid = "sid-capabilities"
8
+ ws_runtime.clear_sid_computer_use_metadata(sid)
9
+
10
+ ws_runtime.store_sid_computer_use_metadata(
11
+ sid,
12
+ {
13
+ "supported": True,
14
+ "enabled": True,
15
+ "trust_mode": "allow",
16
+ "status": "active",
17
+ "last_error": "",
18
+ "restore_token_present": True,
19
+ "artifact_root": "/a0/tmp/_a0_connector/computer_use",
20
+ "backend_id": "windows",
21
+ "backend_family": "windows",
22
+ "features": ["native-window-list", "element-index-targeting"],
23
+ "contract_version": 1,
24
+ "capabilities": {
25
+ "contract_version": 1,
26
+ "identity": {
27
+ "pid": True,
28
+ "window_id": True,
29
+ "element_index": True,
30
+ },
31
+ "dispatch": {
32
+ "default": "background",
33
+ "background": True,
34
+ },
35
+ },
36
+ "support_reason": "Windows UIA backend is available.",
37
+ },
38
+ )
39
+
40
+ metadata = ws_runtime.computer_use_metadata_for_sid(sid)
41
+
42
+ assert metadata is not None
43
+ assert metadata["contract_version"] == 1
44
+ assert metadata["capabilities"]["identity"]["element_index"] is True
45
+ assert metadata["capabilities"]["dispatch"]["default"] == "background"
46
+
47
+ metadata["capabilities"]["dispatch"]["default"] = "foreground"
48
+ fresh_metadata = ws_runtime.computer_use_metadata_for_sid(sid)
49
+ assert fresh_metadata is not None
50
+ assert fresh_metadata["capabilities"]["dispatch"]["default"] == "background"
51
+
52
+ ws_runtime.clear_sid_computer_use_metadata(sid)