Store vision and computer-use images as path refs
Keep image payloads out of persistent agent history by storing vision and computer-use captures as file path references instead of inline base64 data. - update vision_load to attach image paths without compression or JPEG conversion - update computer_use_remote to attach shared capture artifact paths directly - serialize local image refs into provider-valid data URLs only at request prep - reject base64/data URL attachments on the connector WebSocket path - advertise path_or_url as the connector attachment mode
Alessandro committed
Apr 21, 2026 at 17:27 UTC
1993f6f86415a0352d49eda61d3c061eb4b5dcea
7 files changed
+128
-95
helpers/images.py
+70
-1
@@ -1,6 +1,75 @@
1
-from PIL import Image
1
+import base64
2
import io
3
import math
4
+import mimetypes
5
+from pathlib import Path
6
+from typing import Any
7
+from urllib.parse import unquote, urlparse
8
+
9
+from PIL import Image
10
+
11
+
12
+def prepare_content(content: Any) -> Any:
13
+ if isinstance(content, list):
14
+ return [prepare_content(item) for item in content]
15
+ if not isinstance(content, dict):
16
+ return content
17
+
18
+ if content.get("type") == "image_url":
19
+ image_url = content.get("image_url")
20
+ if isinstance(image_url, dict):
21
+ url = str(image_url.get("url", "") or "").strip()
22
+ if is_local_ref(url):
23
+ return {**content, "image_url": {**image_url, "url": to_data_url(url)}}
24
+ elif isinstance(image_url, str):
25
+ url = image_url.strip()
26
+ if is_local_ref(url):
27
+ return {**content, "image_url": {"url": to_data_url(url)}}
28
+
29
+ return {key: prepare_content(value) for key, value in content.items()}
30
+
31
+
32
+def is_local_ref(url: str) -> bool:
33
+ if not url:
34
+ return False
35
+ lowered = url.lower()
36
+ if lowered.startswith(("http://", "https://", "data:")):
37
+ return False
38
+ return lowered.startswith("file://") or url.startswith(("/", "./", "../", "~"))
39
+
40
+
41
+def to_data_url(url: str) -> str:
42
+ path = resolve_ref(url)
43
+ mime_type = mimetypes.guess_type(path.name)[0]
44
+ if not mime_type or not mime_type.startswith("image/"):
45
+ raise ValueError(f"Image attachment must have an image MIME type: {path}")
46
+ encoded = base64.b64encode(path.read_bytes()).decode("utf-8")
47
+ return f"data:{mime_type};base64,{encoded}"
48
+
49
+
50
+def resolve_ref(url: str) -> Path:
51
+ raw_path = unquote(urlparse(url).path) if url.lower().startswith("file://") else url
52
+ path = Path(raw_path).expanduser()
53
+ candidates = [path]
54
+ if raw_path.startswith("/a0/"):
55
+ from helpers import files
56
+
57
+ candidates.append(Path(files.fix_dev_path(raw_path)))
58
+ elif not path.is_absolute():
59
+ from helpers import files
60
+
61
+ candidates.append(Path(files.get_abs_path(raw_path)))
62
+
63
+ seen: set[str] = set()
64
+ for candidate in candidates:
65
+ key = str(candidate)
66
+ if key in seen:
67
+ continue
68
+ seen.add(key)
69
+ if candidate.exists() and candidate.is_file():
70
+ return candidate
71
+
72
+ raise FileNotFoundError(f"Image attachment path does not exist: {raw_path}")
73
74
75
def compress_image(image_data: bytes, *, max_pixels: int = 256_000, quality: int = 50) -> bytes:
models.py
+2
-2
@@ -20,7 +20,7 @@ import openai
20
from litellm.types.utils import ModelResponse
21
22
from helpers import dotenv
23
-from helpers import settings, dirty_json
23
+from helpers import settings, dirty_json, images
24
from helpers.dotenv import load_dotenv
25
from helpers.providers import ModelType as ProviderModelType, get_provider_config
26
from helpers.rate_limiter import RateLimiter
@@ -329,7 +329,7 @@ class LiteLLMChatWrapper(SimpleChatModel):
329
}
330
for m in messages:
331
role = role_mapping.get(m.type, m.type)
332
- message_dict = {"role": role, "content": m.content}
332
+ message_dict = {"role": role, "content": images.prepare_content(m.content)}
333
334
# Handle tool calls for AI messages
335
tool_calls = getattr(m, "tool_calls", None)
plugins/_a0_connector/api/v1/capabilities.py
+2
-1
@@ -80,7 +80,8 @@ class Capabilities(connector_base.PublicConnectorApiHandler):
80
"websocket_namespace": "/ws",
81
"websocket_handlers": ["plugins/_a0_connector/ws_connector"],
82
"attachments": {
83
- "mode": "base64",
83
+ "mode": "path_or_url",
84
+ "http_upload": "base64_to_file",
85
"max_files": 20,
86
},
87
"features": _feature_list(),
plugins/_a0_connector/api/ws_connector.py
+42
-8
@@ -217,20 +217,26 @@ class WsConnector(WsHandler):
217
from plugins._a0_connector.helpers.chat_context import ConnectorContextError
218
219
message = str(data.get("message", "")).strip()
220
- if not message:
221
- return WsResult.error(
222
- code="MISSING_MESSAGE",
223
- message="message is required",
224
- correlation_id=data.get("correlationId"),
225
- )
226
-
220
context_id = str(data.get("context_id", "")).strip() or None
221
current_context_id = (
222
str(data.get("current_context", data.get("current_context_id", ""))).strip()
223
or None
224
)
225
client_message_id = str(data.get("client_message_id", "")).strip()
233
- attachments = list(data.get("attachments", [])) if isinstance(data.get("attachments"), list) else []
226
+ raw_attachments = list(data.get("attachments", [])) if isinstance(data.get("attachments"), list) else []
227
+ attachments, attachment_error = self._normalize_attachment_refs(raw_attachments)
228
+ if attachment_error:
229
+ return WsResult.error(
230
+ code="INVALID_ATTACHMENTS",
231
+ message=attachment_error,
232
+ correlation_id=data.get("correlationId"),
233
+ )
234
+ if not message and not attachments:
235
+ return WsResult.error(
236
+ code="MISSING_MESSAGE",
237
+ message="message or attachments are required",
238
+ correlation_id=data.get("correlationId"),
239
+ )
240
project_name = str(data.get("project_name", "")).strip() or None
241
agent_profile = str(data.get("agent_profile", "")).strip() or None
242
@@ -299,6 +305,34 @@ class WsConnector(WsHandler):
305
"client_message_id": client_message_id or None,
306
}
307
308
+ def _normalize_attachment_refs(self, attachments: list[Any]) -> tuple[list[str], str]:
309
+ refs: list[str] = []
310
+ for attachment in attachments:
311
+ if isinstance(attachment, str):
312
+ ref = attachment.strip()
313
+ elif isinstance(attachment, dict):
314
+ if str(attachment.get("base64", "") or "").strip():
315
+ return [], (
316
+ "WebSocket attachments must be file paths or URLs. "
317
+ "Use the HTTP message_send upload path for base64 file uploads."
318
+ )
319
+ ref = str(
320
+ attachment.get("path")
321
+ or attachment.get("url")
322
+ or attachment.get("file")
323
+ or ""
324
+ ).strip()
325
+ else:
326
+ return [], "attachments must be file paths, URLs, or metadata objects with path/url"
327
+
328
+ if not ref:
329
+ continue
330
+ if ref.lower().startswith("data:"):
331
+ return [], "data URL attachments are not accepted; provide a file path or URL"
332
+ refs.append(ref)
333
+
334
+ return refs, ""
335
+
336
def _handle_file_op_result(
337
self,
338
data: dict[str, Any],
plugins/_a0_connector/plugin.yaml
+1
-1
@@ -1,7 +1,7 @@
1
name: _a0_connector
2
title: A0 Connector
3
description: Current Agent Zero connector plugin for HTTP plus /ws integration, using session auth and handler activation through auth.handlers.
4
-version: 1.5.0
4
+version: 1.5
5
settings_sections:
6
- external
7
- developer
plugins/_a0_connector/tools/computer_use_remote.py
+3
-47
@@ -2,15 +2,10 @@
2
from __future__ import annotations
3
4
import asyncio
5
-import base64
6
-import io
7
-import math
5
from pathlib import Path
6
import uuid
7
from typing import Any
8
12
-from PIL import Image
13
-
9
from helpers import history
10
from helpers.tool import Response, Tool
11
from helpers.ws import NAMESPACE
@@ -26,8 +21,6 @@ from plugins._a0_connector.helpers.ws_runtime import (
21
COMPUTER_USE_OP_TIMEOUT = 180.0
22
COMPUTER_USE_OP_EVENT = "connector_computer_use_op"
23
CAPTURE_TOKENS_ESTIMATE = 1500
29
-CAPTURE_MAX_PIXELS = 768_000
30
-CAPTURE_JPEG_QUALITY = 75
24
_AUTO_CAPTURE_ACTIONS = {
25
"start_session",
26
"move",
@@ -313,13 +306,13 @@ class ComputerUseRemote(Tool):
306
return f"Computer use status={status}, trust_mode={trust_mode or 'unknown'}, active_contexts={active_text}."
307
308
def _record_capture(self, data: dict[str, Any]) -> str:
316
- mime_type, image_b64 = self._capture_image_data(data)
309
+ _image_path, display_path = self._resolve_capture_path(data)
310
width = data.get("width", "?")
311
height = data.get("height", "?")
312
summary = f"Computer-use capture {width}x{height}."
313
content = [
314
{"type": "text", "text": summary},
322
- {"type": "image_url", "image_url": {"url": f"data:{mime_type};base64,{image_b64}"}},
315
+ {"type": "image_url", "image_url": {"url": display_path}},
316
]
317
raw_message = history.RawMessage(raw_content=content, preview=summary)
318
self.agent.hist_add_message(False, content=raw_message, tokens=CAPTURE_TOKENS_ESTIMATE)
@@ -342,7 +335,7 @@ class ComputerUseRemote(Tool):
335
preview = self._capture_preview_from_message(message)
336
if not preview:
337
continue
345
- message.content = f"{preview} [embedded image removed]"
338
+ message.content = f"{preview} [image reference superseded]"
339
if hasattr(message, "summary"):
340
message.summary = ""
341
if hasattr(message, "calculate_tokens"):
@@ -399,43 +392,6 @@ class ComputerUseRemote(Tool):
392
return preview
393
return ""
394
402
- def _capture_image_data(self, data: dict[str, Any]) -> tuple[str, str]:
403
- image_bytes = self._capture_image_bytes(data)
404
- optimized_bytes = self._optimize_capture_image(image_bytes)
405
- if optimized_bytes is not None:
406
- return "image/jpeg", base64.b64encode(optimized_bytes).decode("utf-8")
407
- return "image/png", base64.b64encode(image_bytes).decode("utf-8")
408
-
409
- def _capture_image_bytes(self, data: dict[str, Any]) -> bytes:
410
- inline_payload = str(data.get("png_base64", "") or "").strip()
411
- if inline_payload:
412
- try:
413
- return base64.b64decode(inline_payload, validate=True)
414
- except Exception:
415
- pass
416
-
417
- image_path, _display_path = self._resolve_capture_path(data)
418
- return image_path.read_bytes()
419
-
420
- def _optimize_capture_image(self, image_bytes: bytes) -> bytes | None:
421
- try:
422
- image = Image.open(io.BytesIO(image_bytes))
423
- current_pixels = image.width * image.height
424
- if current_pixels > CAPTURE_MAX_PIXELS:
425
- scale = math.sqrt(CAPTURE_MAX_PIXELS / current_pixels)
426
- resized = (
427
- max(1, int(image.width * scale)),
428
- max(1, int(image.height * scale)),
429
- )
430
- image = image.resize(resized, Image.Resampling.LANCZOS)
431
- if image.mode not in {"RGB", "L"}:
432
- image = image.convert("RGB")
433
- output = io.BytesIO()
434
- image.save(output, format="JPEG", quality=CAPTURE_JPEG_QUALITY, optimize=True)
435
- return output.getvalue()
436
- except Exception:
437
- return None
438
-
395
def _resolve_capture_path(self, data: dict[str, Any]) -> tuple[Path, str]:
396
candidates = [
397
str(data.get("capture_path", "") or "").strip(),
tools/vision_load.py
+8
-35
@@ -1,13 +1,10 @@
1
-import base64
1
from helpers.print_style import PrintStyle
2
from helpers.tool import Tool, Response
4
-from helpers import runtime, files, images, plugins
3
+from helpers import runtime, files, plugins
4
from mimetypes import guess_type
5
from helpers import history
6
8
-# image optimization and token estimation for context window
9
-MAX_PIXELS = 768_000
10
-QUALITY = 75
7
+# image token estimation for context window
8
TOKENS_ESTIMATE = 1500
9
10
@@ -17,7 +14,6 @@ class VisionLoad(Tool):
14
self.images_dict = {}
15
self.loaded_paths: list[str] = []
16
self.skipped_paths: list[str] = []
20
- template: list[dict[str, str]] = [] # type: ignore
17
18
max_embeds = self._get_max_embeds()
19
limited_paths = paths if max_embeds <= 0 else paths[-max_embeds:]
@@ -30,31 +26,8 @@ class VisionLoad(Tool):
26
if path not in self.images_dict:
27
mime_type, _ = guess_type(str(path))
28
if mime_type and mime_type.startswith("image/"):
33
- try:
34
- # Read binary file
35
- file_content = await runtime.call_development_function(
36
- files.read_file_base64, str(path)
37
- )
38
- file_content = base64.b64decode(file_content)
39
- # Compress and convert to JPEG
40
- compressed = images.compress_image(
41
- file_content, max_pixels=MAX_PIXELS, quality=QUALITY
42
- )
43
- # Encode as base64
44
- file_content_b64 = base64.b64encode(compressed).decode("utf-8")
45
-
46
- # DEBUG: Save compressed image
47
- # await runtime.call_development_function(
48
- # files.write_file_base64, str(path), file_content_b64
49
- # )
50
-
51
- # Construct the data URL (always JPEG after compression)
52
- self.images_dict[path] = file_content_b64
53
- self.loaded_paths.append(path)
54
- except Exception as e:
55
- self.images_dict[path] = None
56
- PrintStyle().error(f"Error processing image {path}: {e}")
57
- self.agent.context.log.log("warning", f"Error processing image {path}: {e}")
29
+ self.images_dict[path] = str(path)
30
+ self.loaded_paths.append(path)
31
32
return Response(message="dummy", break_loop=False)
33
@@ -80,12 +53,12 @@ class VisionLoad(Tool):
53
)
54
if self.images_dict:
55
self.agent.hist_add_tool_result(self.name, summary, id=self.log.id if self.log else "")
83
- for path, image in self.images_dict.items():
84
- if image:
56
+ for path, image_path in self.images_dict.items():
57
+ if image_path:
58
content.append(
59
{
60
"type": "image_url",
88
- "image_url": {"url": f"data:image/jpeg;base64,{image}"},
61
+ "image_url": {"url": image_path},
62
}
63
)
64
else:
@@ -96,7 +69,7 @@ class VisionLoad(Tool):
69
}
70
)
71
# append as raw message content for LLMs with vision tokens estimate
99
- msg = history.RawMessage(raw_content=content, preview="<Base64 encoded image data>")
72
+ msg = history.RawMessage(raw_content=content, preview="<Image attachments loaded by path>")
73
self.agent.hist_add_message(
74
False, content=msg, tokens=TOKENS_ESTIMATE * len(content)
75
)