Avoid persisting computer-use capture artifacts
Attach base64 computer-use capture artifacts directly as data-image URLs in RawMessage content instead of materializing them under the connector temp capture directory. Keep legacy path-based captures as a fallback while preserving base64 compatibility for model adapters and avoiding durable screenshot files for artifact payloads.
Alessandro committed
May 22, 2026 at 05:09 UTC
e36cf19bfc6245fb93dd25204c04c1ae550cee2d
1 file changed
+24
-43
plugins/_a0_connector/tools/computer_use_remote.py
+24
-43
@@ -6,7 +6,7 @@ from pathlib import Path
6
import uuid
7
from typing import Any
8
9
-from helpers import files, history
9
+from helpers import history
10
from helpers.tool import Response, Tool
11
from helpers.ws import NAMESPACE
12
from helpers.ws_manager import ConnectionNotFoundError, get_shared_ws_manager
@@ -21,7 +21,6 @@ from plugins._a0_connector.helpers.ws_runtime import (
21
22
COMPUTER_USE_OP_TIMEOUT = 180.0
23
COMPUTER_USE_OP_EVENT = "connector_computer_use_op"
24
-COMPUTER_USE_CAPTURE_DIR = ("tmp", "_a0_connector", "computer_use", "captures")
24
CAPTURE_TOKENS_ESTIMATE = 1500
25
MAX_CAPTURE_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024
26
REARM_REQUIRED_DEFAULT_MESSAGE = (
@@ -367,11 +366,10 @@ class ComputerUseRemote(Tool):
366
)
367
368
def _record_capture(self, data: dict[str, Any]) -> str:
370
- data = self._materialize_capture_artifact(data)
371
- _image_path, display_path = self._resolve_capture_path(data)
369
+ display_ref, resolved_capture_id = self._resolve_capture_ref(data)
370
width = data.get("width", "?")
371
height = data.get("height", "?")
374
- capture_id = str(data.get("capture_id") or Path(display_path).stem or "?").strip()
372
+ capture_id = str(data.get("capture_id") or resolved_capture_id or "?").strip()
373
coordinate_space = str(data.get("coordinate_space") or "normalized_global_screen").strip()
374
summary = (
375
f"Computer-use capture id={capture_id} {width}x{height}, "
@@ -385,7 +383,7 @@ class ComputerUseRemote(Tool):
383
summary = f"{summary} Fresh capture requested."
384
content = [
385
{"type": "text", "text": summary},
388
- {"type": "image_url", "image_url": {"url": display_path}},
386
+ {"type": "image_url", "image_url": {"url": display_ref}},
387
]
388
raw_message = history.RawMessage(raw_content=content, preview=summary)
389
self.agent.hist_add_message(False, content=raw_message, tokens=CAPTURE_TOKENS_ESTIMATE)
@@ -414,6 +412,26 @@ class ComputerUseRemote(Tool):
412
if hasattr(message, "calculate_tokens"):
413
message.tokens = message.calculate_tokens()
414
415
+ def _resolve_capture_ref(self, data: dict[str, Any]) -> tuple[str, str]:
416
+ artifact = data.get("artifact")
417
+ if isinstance(artifact, dict) and str(artifact.get("encoding", "")).strip().lower() == "base64":
418
+ encoded = str(artifact.get("data") or "")
419
+ if encoded:
420
+ estimated_size = _estimated_base64_decoded_size(encoded)
421
+ if estimated_size > MAX_CAPTURE_ARTIFACT_SIZE_BYTES:
422
+ raise RuntimeError(
423
+ "Computer-use capture artifact is too large to attach safely "
424
+ f"({estimated_size} bytes, limit {MAX_CAPTURE_ARTIFACT_SIZE_BYTES} bytes)."
425
+ )
426
+ mime = str(artifact.get("mime") or "image/png").strip()
427
+ if not mime.startswith("image/"):
428
+ mime = "image/png"
429
+ filename = _safe_filename(str(artifact.get("filename") or "computer-use-capture.png"))
430
+ return f"data:{mime};base64,{encoded}", Path(filename).stem
431
+
432
+ image_path, display_path = self._resolve_capture_path(data)
433
+ return display_path, image_path.stem
434
+
435
def _collect_capture_messages(self, history_obj: Any) -> list[Any]:
436
messages: list[Any] = []
437
@@ -479,43 +497,6 @@ class ComputerUseRemote(Tool):
497
f"Capture artifact was not found in any advertised path: {candidates!r}"
498
)
499
482
- def _materialize_capture_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
483
- artifact = data.get("artifact")
484
- if not isinstance(artifact, dict):
485
- return data
486
- if str(artifact.get("encoding", "")).strip().lower() != "base64":
487
- return data
488
-
489
- encoded = str(artifact.get("data") or "")
490
- if not encoded:
491
- return data
492
-
493
- estimated_size = _estimated_base64_decoded_size(encoded)
494
- if estimated_size > MAX_CAPTURE_ARTIFACT_SIZE_BYTES:
495
- raise RuntimeError(
496
- "Computer-use capture artifact is too large to materialize safely "
497
- f"({estimated_size} bytes, limit {MAX_CAPTURE_ARTIFACT_SIZE_BYTES} bytes)."
498
- )
499
-
500
- filename = _safe_filename(str(artifact.get("filename") or "computer-use-capture.png"))
501
- context_id = str(getattr(getattr(self.agent, "context", None), "id", "") or "default")
502
- target_relative = str(Path(*COMPUTER_USE_CAPTURE_DIR, context_id, filename))
503
- target_path = Path(files.get_abs_path(target_relative))
504
- try:
505
- files.write_file_base64(target_relative, encoded)
506
- except Exception as exc:
507
- target_path.unlink(missing_ok=True)
508
- raise RuntimeError("Computer-use capture artifact could not be decoded.") from exc
509
-
510
- materialized = dict(data)
511
- materialized.pop("artifact", None)
512
- local_path = str(target_path)
513
- materialized["path"] = local_path
514
- materialized["a0_path"] = files.normalize_a0_path(local_path)
515
- materialized.setdefault("capture_path", local_path)
516
- materialized.setdefault("capture_id", target_path.stem)
517
- return materialized
518
-
500
def _coerce_int(self, value: object, *, name: str) -> int:
501
try:
502
return int(value or 0)