@samitouri / QOSami_Agents_Zero / commits / bdf9cad4

add backend-aware computer-use and inline capture support

- extend `_a0_connector` computer-use metadata handling to retain `backend_id`, `backend_family`, `features`, and `support_reason` from the CLI hello payload - update `computer_use_remote` to prefer inline `png_base64` screenshots for capture and auto-refresh flows, while keeping filesystem-path fallback for migration/debug cases - include backend information in status formatting so remote computer-use sessions are easier to inspect across Wayland and Windows backends - align the builtin Agent Zero plugin with the new multi-backend computer-use transport used by `a0` 1.5 - replaced heavy CU instructions with a SKILL.md

Alessandro committed Apr 17, 2026 at 02:10 UTC bdf9cad447bc171fabe3fb378982346b51c09884

8 files changed +755 -127

plugins/_a0_connector/api/v1/capabilities.py

@@ -21,6 +21,7 @@ _BASE_FEATURES = [

21 "projects",

22 "text_editor_remote",

23 "code_execution_remote",

24 + "computer_use_remote",

25 "remote_file_tree",

26 "token_status",

27 ]

plugins/_a0_connector/api/ws_connector.py

+42 -2

@@ -8,16 +8,20 @@ from helpers.print_style import PrintStyle

8 from helpers.ws import WsHandler

9 from helpers.ws_manager import WsResult

11 -from plugins._a0_connector.helpers.event_bridge import get_context_log_entries

11 from plugins._a0_connector.helpers.exec_config import build_exec_config

12 +from plugins._a0_connector.helpers.event_bridge import get_context_log_entries

13 from plugins._a0_connector.helpers.ws_runtime import (

14 clear_remote_tree_snapshot,

15 + clear_sid_computer_use_metadata,

16 + fail_pending_computer_use_ops_for_sid,

17 fail_pending_file_ops_for_sid,

18 fail_pending_exec_ops_for_sid,

19 register_sid,

18 - resolve_pending_file_op,

20 + resolve_pending_computer_use_op,

21 resolve_pending_exec_op,

22 + resolve_pending_file_op,

23 store_remote_tree_snapshot,

24 + store_sid_computer_use_metadata,

25 subscribe_sid_to_context,

26 subscribed_contexts_for_sid,

27 subscribed_sids_for_context,

@@ -36,6 +40,7 @@ WS_FEATURES = [

40 "text_editor_remote",

41 "remote_file_tree",

42 "code_execution_remote",

43 + "computer_use_remote",

44 ]

@@ -71,6 +76,11 @@ class WsConnector(WsHandler):

76 sid,

77 error="CLI disconnected before completing the requested remote execution",

78 )

79 + fail_pending_computer_use_ops_for_sid(

80 + sid,

81 + error="CLI disconnected before completing the requested computer-use operation",

82 + )

83 + clear_sid_computer_use_metadata(sid)

84 PrintStyle.debug(f"[a0-connector] /ws disconnected: {sid}")

86 async def process(

@@ -80,6 +90,11 @@ class WsConnector(WsHandler):

90 sid: str,

91 ) -> dict[str, Any] | WsResult | None:

92 if event == "connector_hello":

93 + computer_use = data.get("computer_use")

94 + if isinstance(computer_use, dict):

95 + store_sid_computer_use_metadata(sid, computer_use)

96 + else:

97 + clear_sid_computer_use_metadata(sid)

98 return {

99 "protocol": PROTOCOL_VERSION,

100 "features": WS_FEATURES,

@@ -104,6 +119,9 @@ class WsConnector(WsHandler):

119 if event == "connector_exec_op_result":

120 return self._handle_exec_op_result(data, sid)

121

122 + if event == "connector_computer_use_op_result":

123 + return self._handle_computer_use_op_result(data, sid)

124 +

125 if event.startswith("connector_"):

126 return WsResult.error(

127 code="UNKNOWN_EVENT",

@@ -347,6 +365,28 @@ class WsConnector(WsHandler):

365

366 return {"op_id": op_id, "accepted": True}

367

368 + def _handle_computer_use_op_result(

369 + self,

370 + data: dict[str, Any],

371 + sid: str,

372 + ) -> dict[str, Any] | WsResult:

373 + op_id = str(data.get("op_id", "")).strip()

374 + if not op_id:

375 + return WsResult.error(

376 + code="MISSING_OP_ID",

377 + message="op_id is required",

378 + correlation_id=data.get("correlationId"),

379 + )

380 +

381 + if not resolve_pending_computer_use_op(op_id, sid=sid, payload=data):

382 + return WsResult.error(

383 + code="UNKNOWN_OP_ID",

384 + message=f"No pending computer-use operation for op_id '{op_id}'",

385 + correlation_id=data.get("correlationId"),

386 + )

387 +

388 + return {"op_id": op_id, "accepted": True}

389 +

390 async def _resolve_context(

391 self,

392 *,

plugins/_a0_connector/helpers/exec_config.py

+20 -80

@@ -2,95 +2,35 @@ from __future__ import annotations

3 from typing import Any

5 -_TIMEOUT_KEYS = (

6 - "first_output_timeout",

7 - "between_output_timeout",

8 - "max_exec_timeout",

9 - "dialog_timeout",

10 -)

12 -_DEFAULT_CODE_EXEC_TIMEOUTS = {

13 - "first_output_timeout": 30,

14 - "between_output_timeout": 15,

15 - "max_exec_timeout": 180,

16 - "dialog_timeout": 5,

17 -}

18 -_DEFAULT_OUTPUT_TIMEOUTS = {

19 - "first_output_timeout": 90,

20 - "between_output_timeout": 45,

21 - "max_exec_timeout": 300,

22 - "dialog_timeout": 5,

23 -}

24 -_DEFAULT_PROMPT_PATTERNS = [

25 - r"($venv$).+[$#] ?$",

26 - r"root@[^:]+:[^#]+# ?$",

27 - r"[a-zA-Z0-9_.-]+@[^:]+:[^$#]+[$#] ?$",

28 - r"$?.*$?\s*PS\s+[^>]+> ?$",

29 -]

30 -_DEFAULT_DIALOG_PATTERNS = [

31 - r"Y/N",

32 - r"yes/no",

33 - r":\s*$",

34 - r"\?\s*$",

35 -]

6 +_TIMEOUT_KEYS = ("first_output_timeout", "between_output_timeout", "max_exec_timeout", "dialog_timeout")

38 -def _coerce_timeout_group(raw: Any, defaults: dict[str, int]) -> dict[str, int]:

39 - group = raw if isinstance(raw, dict) else {}

40 - result: dict[str, int] = {}

41 - for key in _TIMEOUT_KEYS:

42 - value = group.get(key, defaults[key])

43 - try:

44 - result[key] = int(value)

45 - except (TypeError, ValueError):

46 - result[key] = defaults[key]

47 - return result

48 -

49 -

50 -def _pattern_lines(raw: Any, defaults: list[str]) -> list[str]:

51 - if isinstance(raw, list):

52 - values = raw

53 - elif isinstance(raw, str):

54 - values = raw.splitlines()

9 +def _parse_patterns(value: object) -> list[str]:

10 + if isinstance(value, str):

11 + items = value.splitlines()

12 + elif isinstance(value, (list, tuple)):

13 + items = value

14 else:

56 - values = defaults

15 + return []

16 + return [str(item).strip() for item in items if str(item).strip()]

17 +

58 - patterns = [str(value).strip() for value in values if str(value).strip()]

59 - return patterns or list(defaults)

19 +def _parse_timeouts(cfg: dict[str, Any], prefix: str, defaults: tuple[int, ...]) -> dict[str, int]:

20 + return {

21 + key: int(cfg.get(f"{prefix}_{key}", default))

22 + for key, default in zip(_TIMEOUT_KEYS, defaults)

23 + }

62 -def build_exec_config() -> dict[str, Any]:

26 +def build_exec_config(*, agent: object | None = None) -> dict[str, Any]:

27 from helpers import plugins

65 - try:

66 - config = plugins.get_plugin_config("_code_execution") or {}

67 - except Exception:

68 - config = {}

69 -

29 + cfg = plugins.get_plugin_config("_code_execution", agent=agent) or {}

30 return {

31 "version": 1,

72 - "code_exec_timeouts": _coerce_timeout_group(

73 - config.get("code_exec_timeouts") or {

74 - key: config.get(f"code_exec_{key}")

75 - for key in _TIMEOUT_KEYS

76 - if f"code_exec_{key}" in config

77 - },

78 - _DEFAULT_CODE_EXEC_TIMEOUTS,

79 - ),

80 - "output_timeouts": _coerce_timeout_group(

81 - config.get("output_timeouts") or {

82 - key: config.get(f"output_{key}")

83 - for key in _TIMEOUT_KEYS

84 - if f"output_{key}" in config

85 - },

86 - _DEFAULT_OUTPUT_TIMEOUTS,

87 - ),

88 - "prompt_patterns": _pattern_lines(

89 - config.get("prompt_patterns"),

90 - _DEFAULT_PROMPT_PATTERNS,

91 - ),

92 - "dialog_patterns": _pattern_lines(

93 - config.get("dialog_patterns"),

94 - _DEFAULT_DIALOG_PATTERNS,

95 - ),

32 + "code_exec_timeouts": _parse_timeouts(cfg, "code_exec", (30, 15, 180, 5)),

33 + "output_timeouts": _parse_timeouts(cfg, "output", (90, 45, 300, 5)),

34 + "prompt_patterns": _parse_patterns(cfg.get("prompt_patterns", "")),

35 + "dialog_patterns": _parse_patterns(cfg.get("dialog_patterns", "")),

36 }

plugins/_a0_connector/helpers/ws_runtime.py

+174 -44

@@ -23,6 +23,14 @@ class PendingExecOperation:

23 context_id: str | None = None

26 +@dataclass

27 +class PendingComputerUseOperation:

28 + sid: str

29 + loop: asyncio.AbstractEventLoop

30 + future: asyncio.Future[dict[str, Any]]

31 + context_id: str | None = None

32 +

33 +

34 @dataclass(frozen=True)

35 class RemoteTreeSnapshot:

36 sid: str

@@ -30,11 +38,26 @@ class RemoteTreeSnapshot:

38 updated_at: float

41 +@dataclass(frozen=True)

42 +class ComputerUseMetadata:

43 + supported: bool

44 + enabled: bool

45 + trust_mode: str

46 + artifact_root: str

47 + backend_id: str

48 + backend_family: str

49 + features: tuple[str, ...]

50 + support_reason: str

51 + updated_at: float

52 +

53 +

54 _context_subscriptions: dict[str, set[str]] = {}

55 _sid_contexts: dict[str, set[str]] = {}

56 _pending_file_ops: dict[str, PendingFileOperation] = {}

57 _pending_exec_ops: dict[str, PendingExecOperation] = {}

58 +_pending_computer_use_ops: dict[str, PendingComputerUseOperation] = {}

59 _remote_tree_snapshots: dict[str, RemoteTreeSnapshot] = {}

60 +_sid_computer_use_metadata: dict[str, ComputerUseMetadata] = {}

61 _state_lock = threading.RLock()

@@ -47,6 +70,7 @@ def unregister_sid(sid: str) -> set[str]:

70 with _state_lock:

71 contexts = _sid_contexts.pop(sid, set())

72 _remote_tree_snapshots.pop(sid, None)

73 + _sid_computer_use_metadata.pop(sid, None)

74 for context_id in contexts:

75 subscribers = _context_subscriptions.get(context_id)

76 if not subscribers:

@@ -143,6 +167,61 @@ def select_target_sid(context_id: str) -> str | None:

167 return sorted(subscribers)[0]

168

169

170 +def store_sid_computer_use_metadata(sid: str, payload: dict[str, Any]) -> ComputerUseMetadata:

171 + features_value = payload.get("features")

172 + if isinstance(features_value, (list, tuple)):

173 + features = tuple(str(item).strip() for item in features_value if str(item).strip())

174 + else:

175 + features = ()

176 + metadata = ComputerUseMetadata(

177 + supported=bool(payload.get("supported")),

178 + enabled=bool(payload.get("supported")) and bool(payload.get("enabled")),

179 + trust_mode=str(payload.get("trust_mode", "") or "").strip(),

180 + artifact_root=str(payload.get("artifact_root", "") or "").strip(),

181 + backend_id=str(payload.get("backend_id", "") or "").strip(),

182 + backend_family=str(payload.get("backend_family", "") or "").strip(),

183 + features=features,

184 + support_reason=str(payload.get("support_reason", "") or "").strip(),

185 + updated_at=time.time(),

186 + )

187 + with _state_lock:

188 + _sid_computer_use_metadata[sid] = metadata

189 + return metadata

190 +

191 +

192 +def clear_sid_computer_use_metadata(sid: str) -> None:

193 + with _state_lock:

194 + _sid_computer_use_metadata.pop(sid, None)

195 +

196 +

197 +def computer_use_metadata_for_sid(sid: str) -> dict[str, Any] | None:

198 + with _state_lock:

199 + metadata = _sid_computer_use_metadata.get(sid)

200 + if metadata is None:

201 + return None

202 + return {

203 + "supported": metadata.supported,

204 + "enabled": metadata.enabled,

205 + "trust_mode": metadata.trust_mode,

206 + "artifact_root": metadata.artifact_root,

207 + "backend_id": metadata.backend_id,

208 + "backend_family": metadata.backend_family,

209 + "features": list(metadata.features),

210 + "support_reason": metadata.support_reason,

211 + "updated_at": metadata.updated_at,

212 + }

213 +

214 +

215 +def select_computer_use_target_sid(context_id: str) -> str | None:

216 + with _state_lock:

217 + subscribers = sorted(_context_subscriptions.get(context_id, set()))

218 + for sid in subscribers:

219 + metadata = _sid_computer_use_metadata.get(sid)

220 + if metadata and metadata.supported and metadata.enabled:

221 + return sid

222 + return None

223 +

224 +

225 def store_pending_file_op(

226 op_id: str,

227 *,

@@ -171,14 +250,7 @@ def resolve_pending_file_op(

250 sid: str,

251 payload: dict[str, Any],

252 ) -> bool:

174 - with _state_lock:

175 - pending = _pending_file_ops.get(op_id)

176 - if pending is None or pending.sid != sid:

177 - return False

178 - _pending_file_ops.pop(op_id, None)

179 -

180 - pending.loop.call_soon_threadsafe(_set_future_result, pending.future, dict(payload))

181 - return True

253 + return _resolve_pending(_pending_file_ops, op_id, sid=sid, payload=payload)

254

255

256 def fail_pending_file_op(

@@ -187,32 +259,11 @@ def fail_pending_file_op(

259 sid: str | None = None,

260 error: str,

261 ) -> bool:

190 - with _state_lock:

191 - pending = _pending_file_ops.get(op_id)

192 - if pending is None:

193 - return False

194 - if sid is not None and pending.sid != sid:

195 - return False

196 - _pending_file_ops.pop(op_id, None)

197 -

198 - payload = {"op_id": op_id, "ok": False, "error": error}

199 - pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)

200 - return True

262 + return _fail_pending(_pending_file_ops, op_id, sid=sid, error=error)

263

264

265 def fail_pending_file_ops_for_sid(sid: str, *, error: str) -> None:

204 - with _state_lock:

205 - matches = [

206 - (op_id, pending)

207 - for op_id, pending in _pending_file_ops.items()

208 - if pending.sid == sid

209 - ]

210 - for op_id, _pending in matches:

211 - _pending_file_ops.pop(op_id, None)

212 -

213 - for op_id, pending in matches:

214 - payload = {"op_id": op_id, "ok": False, "error": error}

215 - pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)

266 + _fail_pending_for_sid(_pending_file_ops, sid=sid, error=error)

267

268

269 def store_pending_exec_op(

@@ -243,48 +294,127 @@ def resolve_pending_exec_op(

294 sid: str,

295 payload: dict[str, Any],

296 ) -> bool:

297 + return _resolve_pending(_pending_exec_ops, op_id, sid=sid, payload=payload)

298 +

299 +

300 +def fail_pending_exec_op(

301 + op_id: str,

302 + *,

303 + sid: str | None = None,

304 + error: str,

305 +) -> bool:

306 + return _fail_pending(_pending_exec_ops, op_id, sid=sid, error=error)

307 +

308 +

309 +def fail_pending_exec_ops_for_sid(sid: str, *, error: str) -> None:

310 + _fail_pending_for_sid(_pending_exec_ops, sid=sid, error=error)

311 +

312 +

313 +def store_pending_computer_use_op(

314 + op_id: str,

315 + *,

316 + sid: str,

317 + future: asyncio.Future[dict[str, Any]],

318 + loop: asyncio.AbstractEventLoop,

319 + context_id: str | None = None,

320 +) -> None:

321 + with _state_lock:

322 + _pending_computer_use_ops[op_id] = PendingComputerUseOperation(

323 + sid=sid,

324 + loop=loop,

325 + future=future,

326 + context_id=context_id,

327 + )

328 +

329 +

330 +def clear_pending_computer_use_op(op_id: str) -> None:

331 with _state_lock:

247 - pending = _pending_exec_ops.get(op_id)

332 + _pending_computer_use_ops.pop(op_id, None)

333 +

334 +

335 +def resolve_pending_computer_use_op(

336 + op_id: str,

337 + *,

338 + sid: str,

339 + payload: dict[str, Any],

340 +) -> bool:

341 + return _resolve_pending(_pending_computer_use_ops, op_id, sid=sid, payload=payload)

342 +

343 +

344 +def fail_pending_computer_use_op(

345 + op_id: str,

346 + *,

347 + sid: str | None = None,

348 + error: str,

349 +) -> bool:

350 + return _fail_pending(_pending_computer_use_ops, op_id, sid=sid, error=error)

351 +

352 +

353 +def fail_pending_computer_use_ops_for_sid(sid: str, *, error: str) -> None:

354 + _fail_pending_for_sid(_pending_computer_use_ops, sid=sid, error=error)

355 +

356 +

357 +def _resolve_pending(

358 + registry: dict[str, PendingFileOperation | PendingExecOperation | PendingComputerUseOperation],

359 + op_id: str,

360 + *,

361 + sid: str,

362 + payload: dict[str, Any],

363 +) -> bool:

364 + with _state_lock:

365 + pending = registry.get(op_id)

366 if pending is None or pending.sid != sid:

367 return False

250 - _pending_exec_ops.pop(op_id, None)

368 + registry.pop(op_id, None)

369

370 pending.loop.call_soon_threadsafe(_set_future_result, pending.future, dict(payload))

371 return True

372

373

256 -def fail_pending_exec_op(

374 +def _fail_pending(

375 + registry: dict[str, PendingFileOperation | PendingExecOperation | PendingComputerUseOperation],

376 op_id: str,

377 *,

259 - sid: str | None = None,

378 + sid: str | None,

379 error: str,

380 ) -> bool:

381 with _state_lock:

263 - pending = _pending_exec_ops.get(op_id)

382 + pending = registry.get(op_id)

383 if pending is None:

384 return False

385 if sid is not None and pending.sid != sid:

386 return False

268 - _pending_exec_ops.pop(op_id, None)

387 + registry.pop(op_id, None)

388

270 - payload = {"op_id": op_id, "ok": False, "error": error}

271 - pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)

389 + pending.loop.call_soon_threadsafe(

390 + _set_future_result,

391 + pending.future,

392 + {"op_id": op_id, "ok": False, "error": error},

393 + )

394 return True

395

396

275 -def fail_pending_exec_ops_for_sid(sid: str, *, error: str) -> None:

397 +def _fail_pending_for_sid(

398 + registry: dict[str, PendingFileOperation | PendingExecOperation | PendingComputerUseOperation],

399 + *,

400 + sid: str,

401 + error: str,

402 +) -> None:

403 with _state_lock:

404 matches = [

405 (op_id, pending)

279 - for op_id, pending in _pending_exec_ops.items()

406 + for op_id, pending in registry.items()

407 if pending.sid == sid

408 ]

409 for op_id, _pending in matches:

283 - _pending_exec_ops.pop(op_id, None)

410 + registry.pop(op_id, None)

411

412 for op_id, pending in matches:

286 - payload = {"op_id": op_id, "ok": False, "error": error}

287 - pending.loop.call_soon_threadsafe(_set_future_result, pending.future, payload)

413 + pending.loop.call_soon_threadsafe(

414 + _set_future_result,

415 + pending.future,

416 + {"op_id": op_id, "ok": False, "error": error},

417 + )

418

419

420 def _set_future_result(

plugins/_a0_connector/plugin.yaml

+1 -1

@@ -1,7 +1,7 @@

1 name: _a0_connector

2 title: A0 Connector

3 description: Current Agent Zero connector plugin for HTTP plus /ws integration, using session auth and handler activation through auth.handlers.

4 -version: 1.4

4 +version: 1.5.0

5 settings_sections:

6 - external

7 - developer

plugins/_a0_connector/prompts/agent.system.tool.computer_use_remote.md new

+34

@@ -0,0 +1,34 @@

1 +# computer_use_remote tool

2 +

3 +Use the connected CLI host machine as a local desktop target.

4 +

5 +## Preferred Scope

6 +- Use this for local desktop and native UI tasks on the connected machine.

7 +- For ordinary website browsing, search, form filling, and web downloads, prefer `browser_agent`.

8 +- If the user is flexible and the task is browser-only, briefly guide them toward browser tools because they are usually more reliable and token-efficient than screenshot-driven computer use.

9 +- Before doing real computer-use work, load the `computer-use-remote` skill and follow it.

10 +

11 +## Requirements

12 +- A CLI client must be connected to this context via the shared `/ws` namespace.

13 +- The CLI must advertise `computer_use_remote` support and local computer use must be enabled there.

14 +- In `free_run`, do not expect a fresh approval prompt. If restore is no longer valid, the tool will surface `COMPUTER_USE_REARM_REQUIRED`.

15 +

16 +## Minimal Rules

17 +- Treat user interventions as high-priority control signals.

18 +- If the user says `stop`, `pause`, `abort`, `hold`, `don't continue`, or equivalent, halt immediately and do not use computer-use tools again until the user explicitly resumes.

19 +- Call `start_session` first. It automatically attaches the current screen.

20 +- Decide from the latest screenshot, not from memory.

21 +- Interactive actions (`move`, `click`, `scroll`, `key`, `type`) automatically attach a fresh screenshot after they run.

22 +- Use `capture` only when you need another screen refresh without taking an action.

23 +- Prefer keyboard actions over pointer actions whenever a reliable keyboard path exists.

24 +

25 +## Arguments

26 +- `action`: one of `start_session`, `status`, `capture`, `move`, `click`, `scroll`, `key`, `type`, `stop_session`

27 +- `session_id`: optional for actions after `start_session`

28 +

29 +Action-specific fields:

30 +- `move`: `x`, `y` normalized to `[0,1]`

31 +- `click`: optional `x`, `y`, plus optional `button` (`left`, `right`, `middle`) and `count`

32 +- `scroll`: `dx`, `dy`

33 +- `key`: `key` or `keys`

34 +- `type`: `text`, optional `submit` boolean

plugins/_a0_connector/tools/computer_use_remote.py new

+433

@@ -0,0 +1,433 @@

1 +"""computer_use_remote tool — drive the CLI host machine through the connected frontend."""

2 +from __future__ import annotations

3 +

4 +import asyncio

5 +import base64

6 +from pathlib import Path

7 +import uuid

8 +from typing import Any

9 +

10 +from helpers import history

11 +from helpers.tool import Response, Tool

12 +from helpers.ws import NAMESPACE

13 +from helpers.ws_manager import ConnectionNotFoundError, get_shared_ws_manager

14 +

15 +from plugins._a0_connector.helpers.ws_runtime import (

16 + clear_pending_computer_use_op,

17 + select_computer_use_target_sid,

18 + store_pending_computer_use_op,

19 +)

20 +

21 +

22 +COMPUTER_USE_OP_TIMEOUT = 180.0

23 +COMPUTER_USE_OP_EVENT = "connector_computer_use_op"

24 +CAPTURE_TOKENS_ESTIMATE = 1500

25 +_AUTO_CAPTURE_ACTIONS = {

26 + "start_session",

27 + "move",

28 + "click",

29 + "scroll",

30 + "key",

31 + "type",

32 +}

33 +_SETTLE_DELAY_START_SESSION = 0.2

34 +_SETTLE_DELAY_GLOBAL_FOCUS = 0.45

35 +_SETTLE_DELAY_PLAIN_ENTER = 0.3

36 +_SETTLE_DELAY_SUBMIT = 0.45

37 +_SUPPORTED_ACTIONS = {

38 + "start_session",

39 + "status",

40 + "capture",

41 + "move",

42 + "click",

43 + "scroll",

44 + "key",

45 + "type",

46 + "stop_session",

47 +}

48 +

49 +

50 +class ComputerUseRemote(Tool):

51 + async def execute(self, **kwargs: Any) -> Response:

52 + action = str(self.args.get("action") or "").strip().lower()

53 + if action not in _SUPPORTED_ACTIONS:

54 + return Response(

55 + message=(

56 + "action is required and must be one of: "

57 + "start_session, status, capture, move, click, scroll, key, type, stop_session"

58 + ),

59 + break_loop=False,

60 + )

61 +

62 + context_id = self.agent.context.id

63 + sid = select_computer_use_target_sid(context_id)

64 + if not sid:

65 + return Response(

66 + message=(

67 + "computer_use_remote: no subscribed CLI in this context currently advertises "

68 + "enabled local computer use. Enable it in the CLI with F2 and choose a trust mode first."

69 + ),

70 + break_loop=False,

71 + )

72 +

73 + try:

74 + payload = self._build_payload(op_id=str(uuid.uuid4()), context_id=context_id, action=action)

75 + result = await self._dispatch_payload(sid=sid, payload=payload)

76 + capture_note = await self._maybe_attach_latest_capture(

77 + action=action,

78 + sid=sid,

79 + context_id=context_id,

80 + result=result,

81 + )

82 + except ValueError as exc:

83 + return Response(

84 + message=f"computer_use_remote: {exc}",

85 + break_loop=False,

86 + )

87 + except ConnectionNotFoundError:

88 + return Response(

89 + message=(

90 + "computer_use_remote: the selected CLI disconnected before the request "

91 + "could be delivered."

92 + ),

93 + break_loop=False,

94 + )

95 + except asyncio.TimeoutError:

96 + return Response(

97 + message=f"computer_use_remote: timed out waiting for action={action!r}",

98 + break_loop=False,

99 + )

100 + except Exception as exc:

101 + return Response(

102 + message=f"computer_use_remote: error sending action={action!r}: {exc}",

103 + break_loop=False,

104 + )

105 +

106 + message = self._extract_result(action, result)

107 + if capture_note:

108 + message = f"{message} {capture_note}".strip()

109 +

110 + return Response(

111 + message=message,

112 + break_loop=False,

113 + )

114 +

115 + async def _dispatch_payload(self, *, sid: str, payload: dict[str, Any]) -> dict[str, Any]:

116 + op_id = str(payload.get("op_id") or "").strip()

117 + loop = asyncio.get_running_loop()

118 + future: asyncio.Future[dict[str, Any]] = loop.create_future()

119 + store_pending_computer_use_op(

120 + op_id,

121 + sid=sid,

122 + future=future,

123 + loop=loop,

124 + context_id=str(payload.get("context_id") or "").strip() or None,

125 + )

126 +

127 + try:

128 + await get_shared_ws_manager().emit_to(

129 + NAMESPACE,

130 + sid,

131 + COMPUTER_USE_OP_EVENT,

132 + payload,

133 + handler_id=f"{self.__class__.__module__}.{self.__class__.__name__}",

134 + )

135 + result = await asyncio.wait_for(future, timeout=COMPUTER_USE_OP_TIMEOUT)

136 + finally:

137 + clear_pending_computer_use_op(op_id)

138 +

139 + if isinstance(result, dict):

140 + return result

141 + raise RuntimeError(f"Unexpected response format from CLI: {result!r}")

142 +

143 + async def _maybe_attach_latest_capture(

144 + self,

145 + *,

146 + action: str,

147 + sid: str,

148 + context_id: str,

149 + result: dict[str, Any],

150 + ) -> str:

151 + if action not in _AUTO_CAPTURE_ACTIONS or not bool(result.get("ok")):

152 + return ""

153 +

154 + data = result.get("result")

155 + result_data = dict(data) if isinstance(data, dict) else {}

156 + session_id = str(result_data.get("session_id") or self.args.get("session_id") or "").strip()

157 + if not session_id:

158 + return ""

159 +

160 + settle_seconds = self._auto_capture_settle_seconds(action)

161 + if settle_seconds > 0:

162 + await asyncio.sleep(settle_seconds)

163 +

164 + capture_result = await self._dispatch_payload(

165 + sid=sid,

166 + payload={

167 + "op_id": str(uuid.uuid4()),

168 + "context_id": context_id,

169 + "action": "capture",

170 + "session_id": session_id,

171 + },

172 + )

173 + if not bool(capture_result.get("ok")):

174 + return f"Automatic screen refresh failed: {self._format_error(capture_result)}"

175 +

176 + capture_data = capture_result.get("result")

177 + if not isinstance(capture_data, dict):

178 + return "Automatic screen refresh failed: missing capture payload."

179 +

180 + self._record_capture(capture_data)

181 + return "Latest screen attached."

182 +

183 + def _auto_capture_settle_seconds(self, action: str) -> float:

184 + if action == "start_session":

185 + return _SETTLE_DELAY_START_SESSION

186 + if action == "type" and self._coerce_bool(self.args.get("submit")):

187 + return _SETTLE_DELAY_SUBMIT

188 + if action != "key":

189 + return 0.0

190 +

191 + keyset = {key.lower() for key in self._requested_keys()}

192 + if "super" in keyset or ("alt" in keyset and "tab" in keyset):

193 + return _SETTLE_DELAY_GLOBAL_FOCUS

194 + if keyset == {"enter"}:

195 + return _SETTLE_DELAY_PLAIN_ENTER

196 + return 0.0

197 +

198 + def _requested_keys(self) -> list[str]:

199 + keys_value = self.args.get("keys")

200 + if isinstance(keys_value, (list, tuple)):

201 + return [str(item).strip() for item in keys_value if str(item).strip()]

202 + raw = str(keys_value or self.args.get("key", "") or "").strip()

203 + if not raw:

204 + return []

205 + return [part.strip() for part in raw.split("+") if part.strip()]

206 +

207 + def _build_payload(self, *, op_id: str, context_id: str, action: str) -> dict[str, Any]:

208 + payload: dict[str, Any] = {

209 + "op_id": op_id,

210 + "context_id": context_id,

211 + "action": action,

212 + }

213 + session_id = str(self.args.get("session_id", "") or "").strip()

214 + if session_id:

215 + payload["session_id"] = session_id

216 +

217 + if action == "move":

218 + payload["x"] = self.args.get("x")

219 + payload["y"] = self.args.get("y")

220 + elif action == "click":

221 + if "x" in self.args:

222 + payload["x"] = self.args.get("x")

223 + if "y" in self.args:

224 + payload["y"] = self.args.get("y")

225 + payload["button"] = self.args.get("button", "left")

226 + payload["count"] = self._coerce_int(self.args.get("count", 1), name="count")

227 + elif action == "scroll":

228 + payload["dx"] = self._coerce_int(self.args.get("dx", self.args.get("delta_x", 0)), name="dx")

229 + payload["dy"] = self._coerce_int(self.args.get("dy", self.args.get("delta_y", 0)), name="dy")

230 + elif action == "key":

231 + if "keys" in self.args:

232 + payload["keys"] = self.args.get("keys")

233 + elif "key" in self.args:

234 + payload["key"] = self.args.get("key")

235 + elif action == "type":

236 + payload["text"] = self.args.get("text", "")

237 + if self._coerce_bool(self.args.get("submit")):

238 + payload["submit"] = True

239 +

240 + return payload

241 +

242 + def _extract_result(self, action: str, result: Any) -> str:

243 + if not isinstance(result, dict):

244 + return f"Unexpected response format from CLI: {result!r}"

245 +

246 + ok = bool(result.get("ok"))

247 + data = result.get("result")

248 +

249 + if not ok:

250 + return self._format_error(result)

251 +

252 + if not isinstance(data, dict):

253 + data = {}

254 +

255 + if action == "capture":

256 + self._record_capture(data)

257 + return "Current screen attached."

258 + if action == "status":

259 + return self._format_status(data)

260 + if action == "start_session":

261 + return (

262 + f"Computer-use session started: session_id={data.get('session_id', '?')} "

263 + f"size={data.get('width', '?')}x{data.get('height', '?')}"

264 + )

265 + if action == "stop_session":

266 + return "Computer-use session stopped."

267 + if action == "move":

268 + return f"Pointer moved to x={data.get('x')} y={data.get('y')}."

269 + if action == "click":

270 + return f"Clicked {data.get('button', 'left')} button {data.get('count', 1)} time(s)."

271 + if action == "scroll":

272 + return f"Scrolled dx={data.get('dx', 0)} dy={data.get('dy', 0)}."

273 + if action == "key":

274 + keys = data.get("keys") or []

275 + return f"Sent keys: {keys!r}."

276 + if action == "type":

277 + text = str(data.get("text", "") or "")

278 + if data.get("submitted"):

279 + return f"Typed {len(text)} character(s) and submitted."

280 + return f"Typed {len(text)} character(s)."

281 + return str(data)

282 +

283 + def _format_error(self, result: dict[str, Any]) -> str:

284 + error = str(result.get("error") or "Unknown error")

285 + code = str(result.get("code") or "")

286 + if code:

287 + return f"{code}: {error}"

288 + return error

289 +

290 + def _format_status(self, data: dict[str, Any]) -> str:

291 + status = str(data.get("status", "unknown") or "unknown")

292 + trust_mode = str(data.get("trust_mode", "") or "")

293 + backend_id = str(data.get("backend_id", "") or "").strip()

294 + backend_family = str(data.get("backend_family", "") or "").strip()

295 + active_contexts = data.get("active_contexts") or []

296 + active_text = ", ".join(str(item) for item in active_contexts) if active_contexts else "none"

297 + backend_text = ""

298 + if backend_id:

299 + backend_text = backend_id

300 + if backend_family:

301 + backend_text = f"{backend_text}/{backend_family}"

302 + if backend_text:

303 + return (

304 + f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, "

305 + f"backend={backend_text}, active_contexts={active_text}."

306 + )

307 + return f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, active_contexts={active_text}."

308 +

309 + def _record_capture(self, data: dict[str, Any]) -> str:

310 + image_b64 = self._capture_image_base64(data)

311 + width = data.get("width", "?")

312 + height = data.get("height", "?")

313 + summary = f"Computer-use capture {width}x{height}."

314 + content = [

315 + {"type": "text", "text": summary},

316 + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{image_b64}"}},

317 + ]

318 + raw_message = history.RawMessage(raw_content=content, preview=summary)

319 + self.agent.hist_add_message(False, content=raw_message, tokens=CAPTURE_TOKENS_ESTIMATE)

320 + self._prune_prior_capture_history()

321 + return summary

322 +

323 + def _prune_prior_capture_history(self) -> None:

324 + history_obj = getattr(self.agent, "history", None)

325 + if history_obj is None:

326 + return

327 +

328 + capture_messages = self._collect_capture_messages(history_obj)

329 + if len(capture_messages) <= 1:

330 + return

331 +

332 + latest = capture_messages[-1]

333 + for message in capture_messages[:-1]:

334 + if message is latest:

335 + continue

336 + preview = self._capture_preview_from_message(message)

337 + if not preview:

338 + continue

339 + message.content = f"{preview} [embedded image removed]"

340 + if hasattr(message, "summary"):

341 + message.summary = ""

342 + if hasattr(message, "calculate_tokens"):

343 + message.tokens = message.calculate_tokens()

344 +

345 + def _collect_capture_messages(self, history_obj: Any) -> list[Any]:

346 + messages: list[Any] = []

347 +

348 + def collect_topic(topic: Any) -> None:

349 + topic_messages = getattr(topic, "messages", None)

350 + if isinstance(topic_messages, list):

351 + for message in topic_messages:

352 + if self._capture_preview_from_message(message):

353 + messages.append(message)

354 +

355 + bulks = getattr(history_obj, "bulks", None)

356 + if isinstance(bulks, list):

357 + for bulk in bulks:

358 + self._collect_capture_messages_from_record(bulk, messages)

359 +

360 + topics = getattr(history_obj, "topics", None)

361 + if isinstance(topics, list):

362 + for topic in topics:

363 + collect_topic(topic)

364 +

365 + current = getattr(history_obj, "current", None)

366 + if current is not None:

367 + collect_topic(current)

368 +

369 + return messages

370 +

371 + def _collect_capture_messages_from_record(self, record: Any, messages: list[Any]) -> None:

372 + topic_messages = getattr(record, "messages", None)

373 + if isinstance(topic_messages, list):

374 + for message in topic_messages:

375 + if self._capture_preview_from_message(message):

376 + messages.append(message)

377 + return

378 +

379 + nested_records = getattr(record, "records", None)

380 + if isinstance(nested_records, list):

381 + for nested in nested_records:

382 + self._collect_capture_messages_from_record(nested, messages)

383 +

384 + def _capture_preview_from_message(self, message: Any) -> str:

385 + content = getattr(message, "content", None)

386 + if not isinstance(content, dict):

387 + return ""

388 + raw_content = content.get("raw_content")

389 + preview = content.get("preview")

390 + if raw_content is None or not isinstance(preview, str):

391 + return ""

392 + if preview.startswith("Computer-use capture "):

393 + return preview

394 + return ""

395 +

396 + def _capture_image_base64(self, data: dict[str, Any]) -> str:

397 + inline_payload = str(data.get("png_base64", "") or "").strip()

398 + if inline_payload:

399 + try:

400 + base64.b64decode(inline_payload, validate=True)

401 + except Exception:

402 + pass

403 + else:

404 + return inline_payload

405 +

406 + image_path, _display_path = self._resolve_capture_path(data)

407 + return base64.b64encode(image_path.read_bytes()).decode("utf-8")

408 +

409 + def _resolve_capture_path(self, data: dict[str, Any]) -> tuple[Path, str]:

410 + candidates = [

411 + str(data.get("capture_path", "") or "").strip(),

412 + str(data.get("container_path", "") or "").strip(),

413 + str(data.get("host_path", "") or "").strip(),

414 + ]

415 + for candidate in candidates:

416 + if candidate and Path(candidate).exists():

417 + return Path(candidate), candidate

418 + raise FileNotFoundError(

419 + f"Capture artifact was not found in any advertised path: {candidates!r}"

420 + )

421 +

422 + def _coerce_int(self, value: object, *, name: str) -> int:

423 + try:

424 + return int(value or 0)

425 + except (TypeError, ValueError) as exc:

426 + raise ValueError(f"{name} must be an integer") from exc

427 +

428 + def _coerce_bool(self, value: object) -> bool:

429 + if isinstance(value, bool):

430 + return value

431 + if isinstance(value, (int, float)):

432 + return bool(value)

433 + return str(value or "").strip().lower() in {"1", "true", "yes", "on"}

skills/computer-use-remote/SKILL.md new

+50

@@ -0,0 +1,50 @@

1 +---

2 +name: computer-use-remote

3 +description: Detailed operating guide for using computer_use_remote on the connected local machine. Load this skill before using computer_use_remote for desktop control, screenshots, menus, browser chrome, or other native UI tasks.

4 +version: 1.0.0

5 +author: Agent Zero Team

6 +tags: ["computer-use", "desktop", "local-ui", "screenshots", "native-ui"]

7 +trigger_patterns:

8 + - "computer use"

9 + - "computer-use"

10 + - "computer_use_remote"

11 + - "local desktop control"

12 + - "control local browser"

13 + - "click on screen"

14 + - "native ui"

15 +allowed_tools:

16 + - computer_use_remote

17 + - code_execution_remote

18 +---

19 +

20 +# Computer Use Remote

21 +

22 +## When to Use

23 +

24 +Load this skill before using `computer_use_remote` for local desktop and native UI tasks on the connected machine.

25 +

26 +For ordinary website browsing, search, form filling, and web downloads, prefer `browser_agent` instead. If the user is flexible and the task is browser-only, guide them toward browser tools because they are usually more reliable and token-efficient than screenshot-driven computer use.

27 +

28 +## Core Loop

29 +

30 +1. Call `start_session` first.

31 +2. Decide from the latest screenshot, not from memory.

32 +3. Interactive actions (`move`, `click`, `scroll`, `key`, `type`) already attach a fresh screenshot after they run.

33 +4. Use `capture` only when you need another screenshot without taking an action.

34 +

35 +## Operating Rules

36 +

37 +- Only the latest screenshot or a definitive tool result counts as evidence.

38 +- Prefer keyboard actions over pointer actions whenever a reliable keyboard path exists.

39 +- When a menu or popup is open, treat it as the active UI and prefer keyboard navigation over clicking small transient rows by coordinate.

40 +- If a click dismisses a menu or popup without producing the expected next UI, treat that attempt as failed.

41 +- If the same approach has already failed twice without visible progress, switch strategy instead of repeating it.

42 +- Do not infer focus or task completion from chat logs, sidebars, tool summaries, or status text.

43 +- For browser-navigation tasks done through this tool, only claim success if the browser content area visibly shows the destination page or result.

44 +- Use `type(..., submit=true)` only for URL or navigation-style entry where Enter should fire immediately after typing.

45 +- Do not use `submit=true` for ordinary text fields. Type first, then send `enter` separately if needed.

46 +

47 +## Control Signals

48 +

49 +- Treat user interventions as high-priority control signals.

50 +- If the user says `stop`, `pause`, `abort`, `hold`, `don't continue`, or equivalent, halt immediately and do not use computer-use tools again until the user explicitly resumes.