@samitouri / QOSami_Agents_Zero-1 / commits / edd58a42

Fix durable screenshot artifacts and Xpra sizing

Materialize browser, desktop, computer-use, and vision-load screenshots into chat-scoped artifacts so historical image refs survive temporary screenshot pruning. Keep history serialization free of rescue assumptions, document durable screenshot behavior in tool prompts/skills, and size Xpra canvases from backend-normalized display dimensions to prevent stretched desktop views. Verified with focused pytest coverage plus live Docker checks for browser screenshot persistence and Xpra canvas dimensions.

Alessandro committed May 30, 2026 at 17:45 UTC edd58a42d2b27cc2e9e013609f64fb10652f168f
18 files changed +688 -81
helpers/chat_media.py new
+244
@@ -0,0 +1,244 @@
1 +from __future__ import annotations
2 +
3 +import time
4 +import uuid
5 +from dataclasses import dataclass
6 +from pathlib import Path
7 +from typing import Literal
8 +
9 +from helpers import files, media_artifacts
10 +
11 +
12 +DEFAULT_MAX_IMAGE_BYTES = media_artifacts.DEFAULT_MAX_ARTIFACT_SIZE_BYTES
13 +ImageCategory = Literal["images", "screenshots"]
14 +
15 +
16 +@dataclass(frozen=True)
17 +class ChatImage:
18 + path: str
19 + a0_path: str
20 + mime: str
21 + size: int
22 +
23 +
24 +def screenshot_dir(context_id: str, source: str) -> Path:
25 + return artifact_dir(context_id, category="screenshots", source=source)
26 +
27 +
28 +def artifact_dir(
29 + context_id: str,
30 + *,
31 + category: ImageCategory = "images",
32 + source: str = "vision-load",
33 +) -> Path:
34 + context_segment = files.safe_file_name(str(context_id or "default")).strip("._") or "default"
35 + safe_category = files.safe_file_name(category).strip("._") or "images"
36 + safe_source = files.safe_file_name(source).strip("._") or "vision-load"
37 +
38 + return Path(files.get_abs_path("usr/chats", context_segment)) / safe_category / safe_source
39 +
40 +
41 +def save_image_bytes(
42 + *,
43 + context_id: str,
44 + payload: bytes,
45 + mime_type: str = "image/png",
46 + category: ImageCategory = "images",
47 + source: str = "vision-load",
48 + preferred_name: str = "",
49 + max_bytes: int | None = DEFAULT_MAX_IMAGE_BYTES,
50 +) -> ChatImage:
51 + data = bytes(payload or b"")
52 + if not data:
53 + raise media_artifacts.EmptyBase64Data("image payload is empty")
54 + if max_bytes is not None and len(data) > max_bytes:
55 + raise media_artifacts.ArtifactTooLarge(len(data), max_bytes)
56 +
57 + safe_mime = media_artifacts.normalize_mime(
58 + mime_type,
59 + default="image/png",
60 + required_prefix="image/",
61 + )
62 + default_extension = media_artifacts.guess_extension(safe_mime, ".png")
63 + default_filename = f"{source or 'image'}{default_extension}"
64 + filename = media_artifacts.safe_filename(
65 + preferred_name,
66 + default=default_filename,
67 + default_extension=default_extension,
68 + )
69 + filename_path = Path(filename)
70 + stem = filename_path.stem or Path(default_filename).stem or "image"
71 + suffix = filename_path.suffix or default_extension
72 + timestamp = time.strftime("%Y%m%d-%H%M%S")
73 + path = artifact_dir(context_id, category=category, source=source) / (
74 + f"{stem}-{timestamp}-{uuid.uuid4().hex[:8]}{suffix}"
75 + )
76 + path.parent.mkdir(parents=True, exist_ok=True)
77 + path.write_bytes(data)
78 + return ChatImage(
79 + path=str(path),
80 + a0_path=files.normalize_a0_path(str(path)),
81 + mime=safe_mime,
82 + size=len(data),
83 + )
84 +
85 +
86 +def save_image_base64(
87 + *,
88 + context_id: str,
89 + data: str,
90 + mime_type: str = "image/png",
91 + category: ImageCategory = "images",
92 + source: str = "vision-load",
93 + preferred_name: str = "",
94 + max_bytes: int | None = DEFAULT_MAX_IMAGE_BYTES,
95 +) -> ChatImage:
96 + payload = media_artifacts.decode_base64_payload(data, max_bytes=max_bytes)
97 + return save_image_bytes(
98 + context_id=context_id,
99 + payload=payload.payload,
100 + mime_type=mime_type,
101 + category=category,
102 + source=source,
103 + preferred_name=preferred_name,
104 + max_bytes=max_bytes,
105 + )
106 +
107 +
108 +def save_image_file(
109 + *,
110 + context_id: str,
111 + path: str | Path,
112 + category: ImageCategory = "images",
113 + source: str = "vision-load",
114 + preferred_name: str = "",
115 + max_bytes: int | None = DEFAULT_MAX_IMAGE_BYTES,
116 +) -> ChatImage:
117 + image_path = Path(path)
118 + payload = image_path.read_bytes()
119 + mime = media_artifacts.normalize_mime(
120 + _guess_image_mime(image_path),
121 + default="image/png",
122 + required_prefix="image/",
123 + )
124 + return save_image_bytes(
125 + context_id=context_id,
126 + payload=payload,
127 + mime_type=mime,
128 + category=category,
129 + source=source,
130 + preferred_name=preferred_name or image_path.name,
131 + max_bytes=max_bytes,
132 + )
133 +
134 +
135 +def save_image_data_url(
136 + *,
137 + context_id: str,
138 + data_url: str,
139 + category: ImageCategory = "images",
140 + source: str = "vision-load",
141 + preferred_name: str = "",
142 + max_bytes: int | None = DEFAULT_MAX_IMAGE_BYTES,
143 +) -> ChatImage:
144 + header, encoded = _split_image_data_url(data_url)
145 + mime = header.removeprefix("data:").split(";", 1)[0] or "image/png"
146 + return save_image_base64(
147 + context_id=context_id,
148 + data=encoded,
149 + mime_type=mime,
150 + category=category,
151 + source=source,
152 + preferred_name=preferred_name,
153 + max_bytes=max_bytes,
154 + )
155 +
156 +
157 +def materialize_image_ref(
158 + *,
159 + context_id: str,
160 + url: str,
161 + source: str = "",
162 + preferred_name: str = "",
163 + max_bytes: int | None = DEFAULT_MAX_IMAGE_BYTES,
164 +) -> str:
165 + value = str(url or "").strip()
166 + if not value or not str(context_id or "").strip():
167 + return value
168 +
169 + resolved_source = source or infer_source(value, preferred_name)
170 + category = category_for_source(resolved_source)
171 + if _is_data_image_url(value):
172 + saved = save_image_data_url(
173 + context_id=context_id,
174 + data_url=value,
175 + category=category,
176 + source=resolved_source,
177 + preferred_name=preferred_name,
178 + max_bytes=max_bytes,
179 + )
180 + return saved.a0_path
181 +
182 + from helpers import images
183 +
184 + source_path = images.resolve_ref(value)
185 + if is_chat_scoped_path(context_id=context_id, path=source_path):
186 + return files.normalize_a0_path(str(source_path))
187 + saved = save_image_file(
188 + context_id=context_id,
189 + path=source_path,
190 + category=category,
191 + source=resolved_source,
192 + preferred_name=preferred_name or source_path.name,
193 + max_bytes=max_bytes,
194 + )
195 + return saved.a0_path
196 +
197 +
198 +def is_chat_scoped_path(*, context_id: str, path: str | Path) -> bool:
199 + if not str(context_id or "").strip():
200 + return False
201 + try:
202 + target = Path(path).resolve(strict=False)
203 + root = artifact_dir(context_id, category="images", source="vision-load").parents[1].resolve(strict=False)
204 + return target == root or root in target.parents
205 + except OSError:
206 + return False
207 +
208 +
209 +def infer_source(value: str = "", preferred_name: str = "") -> str:
210 + raw = f"{value or ''} {preferred_name or ''}".lower()
211 + if "computer-use" in raw or "computer_use" in raw or "_a0_connector/computer_use" in raw:
212 + return "computer-use"
213 + if "/desktop/screenshots/" in raw or "\\desktop\\screenshots\\" in raw or "desktop-" in raw:
214 + return "desktop"
215 + if (
216 + "/browser/screenshots/" in raw
217 + or "\\browser\\screenshots\\" in raw
218 + or "host-browser" in raw
219 + or "browser-" in raw
220 + ):
221 + return "browser"
222 + return "vision-load"
223 +
224 +
225 +def category_for_source(source: str) -> ImageCategory:
226 + return "screenshots" if source in {"desktop", "browser", "computer-use"} else "images"
227 +
228 +
229 +def _guess_image_mime(path: Path) -> str:
230 + import mimetypes
231 +
232 + return mimetypes.guess_type(path.name)[0] or "image/png"
233 +
234 +
235 +def _is_data_image_url(value: str) -> bool:
236 + normalized = str(value or "").strip().lower()
237 + return normalized.startswith("data:image/") and ";base64," in normalized
238 +
239 +
240 +def _split_image_data_url(data_url: str) -> tuple[str, str]:
241 + value = str(data_url or "").strip()
242 + if not _is_data_image_url(value) or "," not in value:
243 + raise ValueError("image data URL must be data:image/*;base64,...")
244 + return value.split(",", 1)
plugins/_a0_connector/tools/computer_use_remote.py
+20 -3
@@ -6,7 +6,7 @@ from pathlib import Path
6 import uuid
7 from typing import Any
8
9 -from helpers import history, media_artifacts
9 +from helpers import chat_media, history, media_artifacts
10 from helpers.print_style import PrintStyle
11 from helpers.tool import Response, Tool
12 from helpers.ws import NAMESPACE
@@ -744,7 +744,15 @@ class ComputerUseRemote(Tool):
744 except FileNotFoundError as exc:
745 path_error = exc
746 else:
747 - return display_path, image_path.stem
747 + saved = chat_media.save_image_file(
748 + context_id=self.agent.context.id,
749 + path=image_path,
750 + category="screenshots",
751 + source="computer-use",
752 + preferred_name=Path(display_path).name or image_path.name,
753 + max_bytes=MAX_CAPTURE_ARTIFACT_SIZE_BYTES,
754 + )
755 + return saved.a0_path, Path(saved.path).stem
756
757 artifact = data.get("artifact")
758 if isinstance(artifact, dict) and str(artifact.get("encoding", "")).strip().lower() == "base64":
@@ -764,7 +772,16 @@ class ComputerUseRemote(Tool):
772 default=f"computer-use-{uuid.uuid4().hex}.png",
773 default_extension=".png",
774 )
767 - return f"data:{mime};base64,{encoded}", Path(filename).stem
775 + saved = chat_media.save_image_base64(
776 + context_id=self.agent.context.id,
777 + data=encoded,
778 + mime_type=mime,
779 + category="screenshots",
780 + source="computer-use",
781 + preferred_name=filename,
782 + max_bytes=MAX_CAPTURE_ARTIFACT_SIZE_BYTES,
783 + )
784 + return saved.a0_path, Path(saved.path).stem
785
786 if path_error is not None:
787 raise path_error
plugins/_browser/helpers/connector_runtime.py
+14 -7
@@ -9,7 +9,7 @@ from pathlib import Path
9 from typing import Any
10 from urllib.parse import urlparse
11
12 -from helpers import ephemeral_images, media_artifacts
12 +from helpers import chat_media, media_artifacts
13
14 try:
15 from helpers.ws import NAMESPACE
@@ -451,12 +451,16 @@ class ConnectorBrowserRuntime:
451 default=f"host-browser-{uuid.uuid4().hex}.jpg",
452 default_extension=".jpg",
453 )
454 + mime = str(artifact.get("mime") or result.get("mime") or "image/jpeg")
455 try:
455 - ref = ephemeral_images.put_image(
456 + saved = chat_media.save_image_base64(
457 context_id=self.context_id,
457 - mime=str(artifact.get("mime") or result.get("mime") or "image/jpeg"),
458 data=data,
459 - name=filename,
459 + mime_type=mime,
460 + category="screenshots",
461 + source="browser",
462 + preferred_name=filename,
463 + max_bytes=MAX_ARTIFACT_SIZE_BYTES,
464 )
465 except Exception as exc:
466 raise RuntimeError("Host browser artifact could not be decoded.") from exc
@@ -466,11 +470,14 @@ class ConnectorBrowserRuntime:
470 materialized.pop("a0_path", None)
471 materialized.pop("host_path", None)
472 materialized.setdefault("context_id", self.context_id)
469 - materialized["ephemeral"] = True
470 - materialized["ephemeral_ref"] = ref
473 + materialized["path"] = saved.path
474 + materialized["a0_path"] = saved.a0_path
475 + materialized["mime"] = saved.mime
476 + materialized["ephemeral"] = False
477 + materialized["chat_scoped"] = True
478 materialized["vision_load"] = {
479 "tool_name": "vision_load",
473 - "tool_args": {"paths": [ref]},
480 + "tool_args": {"paths": [saved.a0_path]},
481 }
482 return materialized
483
plugins/_browser/helpers/runtime.py
+11 -7
@@ -15,7 +15,7 @@ from dataclasses import dataclass
15 from pathlib import Path
16 from typing import Any
17
18 -from helpers import ephemeral_images, files
18 +from helpers import chat_media, files
19 from helpers.defer import DeferredTask
20 from helpers.errors import RepairableException
21 from helpers.print_style import PrintStyle
@@ -1558,23 +1558,27 @@ class _BrowserRuntimeCore:
1558 quality=max(20, min(95, int(quality))),
1559 full_page=bool(full_page),
1560 )
1561 - ref = ephemeral_images.put_image_bytes(
1561 + saved = chat_media.save_image_bytes(
1562 context_id=self.context_id,
1563 - mime="image/jpeg",
1563 payload=image,
1565 - name=f"browser-{resolved_id}.jpg",
1564 + mime_type="image/jpeg",
1565 + category="screenshots",
1566 + source="browser",
1567 + preferred_name=f"browser-{resolved_id}.jpg",
1568 )
1569 return {
1570 "browser_id": resolved_id,
1571 "context_id": self.context_id,
1572 + "path": saved.path,
1573 + "a0_path": saved.a0_path,
1574 "mime": "image/jpeg",
1571 - "ephemeral": True,
1572 - "ephemeral_ref": ref,
1575 + "ephemeral": False,
1576 + "chat_scoped": True,
1577 "state": await self._state(resolved_id),
1578 "vision_load": {
1579 "tool_name": "vision_load",
1580 "tool_args": {
1577 - "paths": [ref],
1581 + "paths": [saved.a0_path],
1582 },
1583 },
1584 }
plugins/_browser/prompts/agent.system.tool.browser.md
+1 -1
@@ -20,7 +20,7 @@ Workflow:
20 - For same-page controls that are easier to identify structurally, `click`, `type`, `submit`, `type_submit`, `scroll`, `select_option`, `set_checked`, and `upload_file` may use `selector` instead of `ref`; the tool resolves the selector through `content` first.
21 - `click` with `x`/`y` and no `ref` is treated as a coordinate mouse click. `type` with text and no `ref` types into the currently focused element. `key_chord` accepts either `["Control", "A"]` or `"CTRL+A"`.
22 - `navigate` reuses an existing `browser_id` and is preferred for serial browsing.
23 -- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned `vision_load.tool_args.paths` value before reasoning visually. When no `path` is requested, browser screenshots are ephemeral refs rather than conserved files.
23 +- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned `vision_load.tool_args.paths` value before reasoning visually. When no `path` is requested, browser screenshots are saved as chat-scoped artifacts; explicit `path` requests remain user-owned files.
24 - Keep the tab set small; close pages after extracting what you need.
25
26 `multi` is only a browser action: use `tool_name: "browser"` with `tool_args.action: "multi"`. Never use `tool_name: "multi"`.
plugins/_browser/skills/browser-automation/SKILL.md
+1 -1
@@ -32,7 +32,7 @@ Screenshots are explicit only; the browser does not automatically load images in
32 2. Call `vision_load` with the returned `vision_load.tool_args.paths` value.
33 3. Reason from the latest loaded screenshot.
34
35 -Screenshot args include `quality`, `full_page`, and optional `path`. Without `path`, the screenshot is an ephemeral ref consumed by `vision_load`; with `path`, PNG is used when `path` ends with `.png`, otherwise JPEG is used.
35 +Screenshot args include `quality`, `full_page`, and optional `path`. Without `path`, the screenshot is saved as a chat-scoped artifact and returned through `vision_load.tool_args.paths`; with `path`, PNG is used when `path` ends with `.png`, otherwise JPEG is used.
36
37 ## Forms And Files
38
plugins/_browser/skills/browser-form-workflows/SKILL.md
+1 -1
@@ -11,7 +11,7 @@ Start with `browser:content` to capture current refs, then use `browser:detail`
11
12 Use `select_option`, `set_checked`, `upload_file`, `type`, `type_submit`, and `submit` for form interaction. Use coordinates only when no stable ref exists or the UI is intentionally canvas-like.
13
14 -Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history; no-path screenshots return ephemeral refs for `vision_load`.
14 +Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history; no-path screenshots return chat-scoped artifact paths for `vision_load`.
15
16 Verify after submission with `browser:content`, `browser:state`, or another explicit `browser:screenshot` plus `vision_load`.
17
plugins/_desktop/helpers/desktop_state.py
+70 -15
@@ -37,6 +37,19 @@ def context_screenshot_dir(context_id: str = "") -> Path:
37 return SCREENSHOT_DIR / _safe_context_id(context_id)
38
39
40 +def chat_screenshot_dir(context_id: str = "") -> Path:
41 + return BASE_DIR / "usr" / "chats" / _safe_context_id(context_id) / "screenshots" / "desktop"
42 +
43 +
44 +def normalize_a0_path(path: str | Path) -> str:
45 + candidate = Path(path)
46 + try:
47 + relative = candidate.resolve(strict=False).relative_to(BASE_DIR.resolve(strict=False))
48 + except ValueError:
49 + return str(candidate)
50 + return "/a0/" + str(relative).replace(os.sep, "/")
51 +
52 +
53 def _safe_context_id(context_id: str = "") -> str:
54 raw = str(context_id or os.environ.get("A0_DESKTOP_CONTEXT_ID") or "default")
55 return _SAFE_CONTEXT_RE.sub("_", raw).strip("._") or "default"
@@ -118,9 +131,11 @@ def capture_screenshot(
131 return {"ok": False, "path": "", "format": "", "captured_at": "", "error": message}
132
133 explicit_path = path is not None and str(path).strip() != ""
121 - ephemeral_ref = not explicit_path and str(transport or "").strip().lower() != "path"
122 - screenshot_dir = context_screenshot_dir(context_id)
123 - if not explicit_path:
134 + transport_mode = str(transport or "").strip().lower()
135 + chat_scoped = bool(not explicit_path and transport_mode == "path" and str(context_id or "").strip())
136 + ephemeral_ref = not explicit_path and transport_mode != "path"
137 + screenshot_dir = chat_screenshot_dir(context_id) if chat_scoped else context_screenshot_dir(context_id)
138 + if not explicit_path and not chat_scoped:
139 prune_context_screenshots(context_id=context_id)
140 screenshot_dir.mkdir(parents=True, exist_ok=True)
141 timestamp = time.strftime("%Y%m%d-%H%M%S")
@@ -138,15 +153,17 @@ def capture_screenshot(
153 return {"ok": False, "path": "", "format": "", "captured_at": "", "error": detail}
154
155 if target.suffix.lower() == ".xwd":
141 - if not explicit_path:
156 + if not explicit_path and not chat_scoped:
157 prune_context_screenshots(context_id=context_id, keep_path=raw_path)
158 return {
159 "ok": True,
160 "path": str(raw_path),
161 + "a0_path": normalize_a0_path(raw_path),
162 "format": "xwd",
163 "captured_at": iso_now(),
164 "recent": True,
149 - "ephemeral": not explicit_path,
165 + "ephemeral": not explicit_path and not chat_scoped,
166 + "chat_scoped": chat_scoped,
167 "context_id": safe_context,
168 "error": "",
169 }
@@ -167,17 +184,19 @@ def capture_screenshot(
184 width=width,
185 height=height,
186 )
170 - if not explicit_path:
187 + if not explicit_path and not chat_scoped:
188 prune_context_screenshots(context_id=context_id, keep_path=target)
189 return {
190 "ok": True,
191 "path": str(target),
192 + "a0_path": normalize_a0_path(target),
193 "format": target.suffix.lower().lstrip(".") or "png",
194 "width": width,
195 "height": height,
196 "captured_at": iso_now(),
197 "recent": True,
180 - "ephemeral": not explicit_path,
198 + "ephemeral": not explicit_path and not chat_scoped,
199 + "chat_scoped": chat_scoped,
200 "context_id": safe_context,
201 "error": "",
202 }
@@ -193,17 +212,19 @@ def capture_screenshot(
212 width=converted["width"],
213 height=converted["height"],
214 )
196 - if not explicit_path:
215 + if not explicit_path and not chat_scoped:
216 prune_context_screenshots(context_id=context_id, keep_path=target)
217 return {
218 "ok": True,
219 "path": str(target),
220 + "a0_path": normalize_a0_path(target),
221 "format": target.suffix.lower().lstrip(".") or "png",
222 "width": converted["width"],
223 "height": converted["height"],
224 "captured_at": iso_now(),
225 "recent": True,
206 - "ephemeral": not explicit_path,
226 + "ephemeral": not explicit_path and not chat_scoped,
227 + "chat_scoped": chat_scoped,
228 "context_id": safe_context,
229 "error": "",
230 }
@@ -226,10 +247,12 @@ def capture_screenshot(
247 return {
248 "ok": True,
249 "path": str(raw_path),
250 + "a0_path": normalize_a0_path(raw_path),
251 "format": "xwd",
252 "captured_at": iso_now(),
253 "recent": True,
232 - "ephemeral": not explicit_path,
254 + "ephemeral": not explicit_path and not chat_scoped,
255 + "chat_scoped": chat_scoped,
256 "context_id": safe_context,
257 "error": message,
258 }
@@ -575,8 +598,36 @@ def parse_xprop(output: str) -> dict[str, str]:
598
599
600 def latest_screenshot(*, context_id: str = "") -> dict[str, Any]:
601 + chat_dir = chat_screenshot_dir(context_id)
602 + chat_latest = _latest_screenshot_from_dir(
603 + chat_dir,
604 + context_id=context_id,
605 + ephemeral=False,
606 + chat_scoped=True,
607 + prune_older=False,
608 + )
609 + if chat_latest.get("ok"):
610 + return chat_latest
611 +
612 prune_context_screenshots(context_id=context_id, max_age_seconds=RECENT_SCREENSHOT_SECONDS)
613 screenshot_dir = context_screenshot_dir(context_id)
614 + return _latest_screenshot_from_dir(
615 + screenshot_dir,
616 + context_id=context_id,
617 + ephemeral=True,
618 + chat_scoped=False,
619 + prune_older=True,
620 + )
621 +
622 +
623 +def _latest_screenshot_from_dir(
624 + screenshot_dir: Path,
625 + *,
626 + context_id: str = "",
627 + ephemeral: bool,
628 + chat_scoped: bool,
629 + prune_older: bool,
630 +) -> dict[str, Any]:
631 if not screenshot_dir.exists():
632 return {"ok": False, "path": "", "format": "", "captured_at": "", "recent": False}
633 candidates = [
@@ -587,17 +638,20 @@ def latest_screenshot(*, context_id: str = "") -> dict[str, Any]:
638 if not candidates:
639 return {"ok": False, "path": "", "format": "", "captured_at": "", "recent": False}
640 latest = max(candidates, key=lambda item: item.stat().st_mtime)
590 - for candidate in candidates:
591 - if candidate != latest:
592 - candidate.unlink(missing_ok=True)
641 + if prune_older:
642 + for candidate in candidates:
643 + if candidate != latest:
644 + candidate.unlink(missing_ok=True)
645 age = max(0.0, time.time() - latest.stat().st_mtime)
646 return {
647 "ok": True,
648 "path": str(latest),
649 + "a0_path": normalize_a0_path(latest),
650 "format": latest.suffix.lower().lstrip("."),
651 "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(latest.stat().st_mtime)),
652 "recent": age <= RECENT_SCREENSHOT_SECONDS,
600 - "ephemeral": True,
653 + "ephemeral": ephemeral,
654 + "chat_scoped": chat_scoped,
655 "context_id": _safe_context_id(context_id),
656 }
657
@@ -660,7 +714,8 @@ def compact_prompt_context(state: dict[str, Any] | None = None) -> str:
714 screenshot = state.get("screenshot") or {}
715 if screenshot.get("recent") and screenshot.get("path"):
716 ephemeral = " ephemeral" if screenshot.get("ephemeral") else ""
663 - lines.append(f"- recent_screenshot={screenshot['path']}{ephemeral}")
717 + screenshot_ref = screenshot.get("a0_path") or screenshot["path"]
718 + lines.append(f"- recent_screenshot={screenshot_ref}{ephemeral}")
719 context_id = str(state.get("context_id") or "").strip()
720 if context_id:
721 lines.append(f"- screenshot_context={context_id}")
plugins/_desktop/skills/linux-desktop/SKILL.md
+2 -2
@@ -38,7 +38,7 @@ The Desktop is an observe-act-verify control surface. Use this decision hierarch
38 3. Prefer launcher commands, window focus, keyboard shortcuts, menus, paste, and save commands.
39 4. Use coordinate clicks only as a last resort, and only after a fresh Desktop observation.
40 5. After any GUI action, verify through Desktop state, active window titles, screenshots, saved file state, or exported output.
41 -6. For terminal or CLI-agent work, verify against a fresh final `observe --json --screenshot` captured after the command has finished or visibly returned to an input prompt. Agent-facing Desktop screenshots are ephemeral refs; `desktopctl` shell observations return temporary context paths. Do not report from an earlier screenshot path.
41 +6. For terminal or CLI-agent work, verify against a fresh final `observe --json --screenshot` captured after the command has finished or visibly returned to an input prompt. Agent-facing Desktop screenshots are ephemeral refs; `desktopctl` shell observations with `--context-id` return chat-scoped screenshot paths. Do not report from an earlier screenshot path.
42
43 Keep these standing rules:
44
@@ -68,7 +68,7 @@ $DESKTOP key ctrl+s
68
69 The script targets the persistent `agent-zero-desktop` X display, sets `DISPLAY`, `XAUTHORITY`, and `HOME` to the XFCE profile, then uses `xdotool` for input. Startup normally prepares this session. If `check` fails during explicit Desktop work, report that the Desktop runtime is not ready instead of installing packages ad hoc.
70
71 -If `observe --json --screenshot` shows a reachable display, visible Desktop/window entries, and a fresh screenshot, the Desktop is usable even when `active_window` is `null`; a bare XFCE desktop can have no active application window. Treat missing screenshots, missing display, or unavailable `xdotool`/`xwd` as blockers and stop with the specific readiness message instead of repeating clicks or inventing a fallback. Use any returned shell screenshot path promptly; only the latest temporary context screenshot is retained.
71 +If `observe --json --screenshot` shows a reachable display, visible Desktop/window entries, and a fresh screenshot, the Desktop is usable even when `active_window` is `null`; a bare XFCE desktop can have no active application window. Treat missing screenshots, missing display, or unavailable `xdotool`/`xwd` as blockers and stop with the specific readiness message instead of repeating clicks or inventing a fallback. Shell screenshots captured with `--context-id` live in the owning chat's screenshot folder; screenshots without a chat context remain temporary.
72
73 For direct app launches without coordinates:
74
plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh
+1 -1
@@ -60,7 +60,7 @@ Commands:
60 observe --json [--screenshot] [--context-id ID]
61 Return structured state, optionally with a fresh screenshot.
62 screenshot [PATH] [--context-id ID]
63 - Capture the Desktop to PATH, or to the temporary context screenshot directory.
63 + Capture the Desktop to PATH, or to the chat screenshot directory when context-id is set.
64 active-window Print the active window name.
65 geometry PATTERN Print the first matching visible window geometry.
66 wait-window PATTERN Wait for a visible matching window and print its id.
plugins/_desktop/webui/desktop-store.js
+48 -8
@@ -258,6 +258,7 @@ const model = {
258 _desktopFrameHost: null,
259 _desktopFrameLoadHandler: null,
260 _desktopKeepaliveHost: null,
261 + _desktopDisplaySizes: {},
262 _desktopIntentionalShutdown: false,
263
264 async init(element = null) {
@@ -1499,7 +1500,7 @@ const model = {
1500 this.stopXpraDesktopPrime();
1501 this._desktopPrimeAttempts = 0;
1502 }
1502 - if (this.applyXpraDesktopFrameMode(options.frame || null)) return;
1503 + if (this.applyXpraDesktopFrameMode(options.frame || null, options)) return;
1504 if (this._desktopPrimeAttempts >= XPRA_DESKTOP_PRIME_ATTEMPTS) return;
1505 this._desktopPrimeAttempts += 1;
1506 if (this._desktopPrimeTimer) globalThis.clearTimeout(this._desktopPrimeTimer);
@@ -1540,8 +1541,12 @@ const model = {
1541 const windows = Object.values(client.id_to_window || {});
1542 if (!client.connected || !windows.length) return false;
1543
1543 - const width = Math.round(container.clientWidth || remoteWindow.innerWidth || 0);
1544 - const height = Math.round(container.clientHeight || remoteWindow.innerHeight || 0);
1544 + const token = options.token || this.session?.desktop?.token || "";
1545 + const displaySize = options.displaySize || this.desktopDisplaySizeForToken(token);
1546 + const viewportWidth = Math.round(container.clientWidth || remoteWindow.innerWidth || 0);
1547 + const viewportHeight = Math.round(container.clientHeight || remoteWindow.innerHeight || 0);
1548 + const width = Math.round(displaySize?.width || viewportWidth || 0);
1549 + const height = Math.round(displaySize?.height || viewportHeight || 0);
1550 if (width > 0 && height > 0) {
1551 client.desktop_width = width;
1552 client.desktop_height = height;
@@ -1574,6 +1579,26 @@ const model = {
1579 }
1580 },
1581
1582 + desktopDisplaySizeForToken(token = "") {
1583 + const key = String(token || "").trim();
1584 + const size = key ? this._desktopDisplaySizes?.[key] : null;
1585 + const width = Math.round(Number(size?.width || 0));
1586 + const height = Math.round(Number(size?.height || 0));
1587 + return width > 0 && height > 0 ? { width, height } : null;
1588 + },
1589 +
1590 + rememberDesktopDisplaySize(token = "", width = 0, height = 0) {
1591 + const key = String(token || "").trim();
1592 + const normalizedWidth = Math.round(Number(width || 0));
1593 + const normalizedHeight = Math.round(Number(height || 0));
1594 + if (!key || normalizedWidth <= 0 || normalizedHeight <= 0) return null;
1595 + this._desktopDisplaySizes = {
1596 + ...(this._desktopDisplaySizes || {}),
1597 + [key]: { width: normalizedWidth, height: normalizedHeight },
1598 + };
1599 + return this._desktopDisplaySizes[key];
1600 + },
1601 +
1602 installXpraDesktopAgentBridge(frame, remoteWindow, remoteDocument, client, container) {
1603 if (!frame || !remoteWindow || !remoteDocument || !client) return null;
1604 const store = this;
@@ -1584,8 +1609,10 @@ const model = {
1609 const metrics = () => {
1610 const desktopWidth = Math.max(1, finite(client.desktop_width || container?.clientWidth || remoteWindow.innerWidth, 1));
1611 const desktopHeight = Math.max(1, finite(client.desktop_height || container?.clientHeight || remoteWindow.innerHeight, 1));
1587 - const clientWidth = Math.max(1, finite(container?.clientWidth || remoteWindow.innerWidth, desktopWidth));
1588 - const clientHeight = Math.max(1, finite(container?.clientHeight || remoteWindow.innerHeight, desktopHeight));
1612 + const primaryWindow = Object.values(client.id_to_window || {})[0];
1613 + const canvas = primaryWindow?.canvas;
1614 + const clientWidth = Math.max(1, finite(canvas?.clientWidth || canvas?.width || container?.clientWidth || remoteWindow.innerWidth, desktopWidth));
1615 + const clientHeight = Math.max(1, finite(canvas?.clientHeight || canvas?.height || container?.clientHeight || remoteWindow.innerHeight, desktopHeight));
1616 return {
1617 desktopWidth,
1618 desktopHeight,
@@ -1683,8 +1710,10 @@ const model = {
1710 },
1711
1712 fitXpraDesktopWindowElement(xpraWindow, width, height) {
1686 - const cssWidth = `${Math.max(1, Number(width || 0))}px`;
1687 - const cssHeight = `${Math.max(1, Number(height || 0))}px`;
1713 + const normalizedWidth = Math.max(1, Math.round(Number(width || 0)));
1714 + const normalizedHeight = Math.max(1, Math.round(Number(height || 0)));
1715 + const cssWidth = `${normalizedWidth}px`;
1716 + const cssHeight = `${normalizedHeight}px`;
1717 const windowElement = xpraWindow?.div;
1718 const canvas = xpraWindow?.canvas;
1719 windowElement?.style?.setProperty("left", "0px", "important");
@@ -1698,6 +1727,12 @@ const model = {
1727 canvas?.style?.setProperty("height", cssHeight, "important");
1728 canvas?.style?.setProperty("display", "block", "important");
1729 canvas?.style?.setProperty("margin", "0", "important");
1730 + if (canvas) {
1731 + if (canvas.width !== normalizedWidth) canvas.width = normalizedWidth;
1732 + if (canvas.height !== normalizedHeight) canvas.height = normalizedHeight;
1733 + canvas.setAttribute("width", String(normalizedWidth));
1734 + canvas.setAttribute("height", String(normalizedHeight));
1735 + }
1736 },
1737
1738 installXpraDesktopWheelBridge(remoteWindow, xpraWindow) {
@@ -2139,6 +2174,11 @@ const model = {
2174 const response = await fetch(`/desktop/resize?${params.toString()}`, { credentials: "same-origin" });
2175 if (response.ok) {
2176 const result = await response.json().catch(() => ({}));
2177 + const displaySize = this.rememberDesktopDisplaySize(
2178 + token,
2179 + result?.width || width,
2180 + result?.height || height,
2181 + );
2182 this._desktopResizeKey = key;
2183 const activeFrame = this.desktopFrame(frame);
2184 const activeTarget = activeFrame?.parentElement || activeFrame;
@@ -2153,7 +2193,7 @@ const model = {
2193 }
2194 }
2195 if (result?.reload) this.reloadDesktopFrame(activeFrame || frame);
2156 - this.primeXpraDesktopFrame({ reset: true, frame: activeFrame || frame });
2196 + this.primeXpraDesktopFrame({ reset: true, frame: activeFrame || frame, token, displaySize });
2197 }
2198 } catch (error) {
2199 console.warn("Desktop resize skipped", error);
tests/test_browser_agent_regressions.py
+31 -11
@@ -2477,7 +2477,7 @@ async def test_browser_runtime_remounts_initial_changed_viewport():
2477
2478
2479 @pytest.mark.anyio
2480 -async def test_browser_runtime_screenshot_file_defaults_to_ephemeral_ref(monkeypatch, tmp_path):
2480 +async def test_browser_runtime_screenshot_file_defaults_to_chat_scoped_artifact(monkeypatch, tmp_path):
2481 screenshot_calls = []
2482
2483 def fake_get_abs_path(*parts):
@@ -2512,15 +2512,15 @@ async def test_browser_runtime_screenshot_file_defaults_to_ephemeral_ref(monkeyp
2512
2513 result = await core.screenshot_file(5, quality=500)
2514
2515 - assert "path" not in result
2516 - assert "a0_path" not in result
2515 + assert Path(result["path"]).read_bytes() == b"image-bytes"
2516 + assert result["a0_path"].startswith("/a0/usr/chats/ctx_id/screenshots/browser/browser-5-")
2517 assert result["context_id"] == "ctx/id"
2518 assert result["mime"] == "image/jpeg"
2519 - assert result["ephemeral"] is True
2520 - assert result["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)
2519 + assert result["ephemeral"] is False
2520 + assert result["chat_scoped"] is True
2521 assert result["vision_load"] == {
2522 "tool_name": "vision_load",
2523 - "tool_args": {"paths": [result["ephemeral_ref"]]},
2523 + "tool_args": {"paths": [result["a0_path"]]},
2524 }
2525 assert "image" not in result
2526 assert not list((tmp_path / "tmp" / "browser" / "screenshots").rglob("*.jpg"))
@@ -2528,7 +2528,6 @@ async def test_browser_runtime_screenshot_file_defaults_to_ephemeral_ref(monkeyp
2528 assert screenshot_calls[-1]["quality"] == 95
2529 assert screenshot_calls[-1]["full_page"] is False
2530 assert "path" not in screenshot_calls[-1]
2531 - assert ephemeral_images.consume_image(result["ephemeral_ref"], context_id="ctx/id").data_url == "data:image/jpeg;base64,aW1hZ2UtYnl0ZXM="
2531
2532 png_path = tmp_path / "custom.png"
2533 png_result = await core.screenshot_file(5, quality=1, full_page=True, path=str(png_path))
@@ -2543,9 +2542,27 @@ async def test_browser_runtime_screenshot_file_defaults_to_ephemeral_ref(monkeyp
2542
2543
2544 @pytest.mark.anyio
2546 -async def test_vision_load_consumes_ephemeral_browser_refs(monkeypatch):
2545 +async def test_vision_load_materializes_ephemeral_browser_refs(monkeypatch, tmp_path):
2546 + monkeypatch.setitem(sys.modules, "helpers.tool", SimpleNamespace(Response=_TestResponse, Tool=_TestTool))
2547 + history_stub = ModuleType("helpers.history")
2548 +
2549 + class _RawMessage(dict):
2550 + def __init__(self, raw_content, preview):
2551 + super().__init__(raw_content=raw_content, preview=preview)
2552 +
2553 + history_stub.RawMessage = _RawMessage
2554 + monkeypatch.setitem(sys.modules, "helpers.history", history_stub)
2555 + monkeypatch.delitem(sys.modules, "tools.vision_load", raising=False)
2556 import tools.vision_load as vision_load_module
2557
2558 + def fake_get_abs_path(*parts):
2559 + return str(tmp_path.joinpath(*parts))
2560 +
2561 + def fake_normalize_a0_path(path):
2562 + return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/")
2563 +
2564 + monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path)
2565 + monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
2566 monkeypatch.setattr(
2567 vision_load_module.plugins,
2568 "get_plugin_config",
@@ -2561,7 +2578,7 @@ async def test_vision_load_consumes_ephemeral_browser_refs(monkeypatch):
2578 hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
2579 hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)),
2580 )
2564 - ref = ephemeral_images.put_image(
2581 + ref = vision_load_module.ephemeral_images.put_image(
2582 context_id="ctx-vision",
2583 mime="image/jpeg",
2584 data=SMALL_JPEG_10X10,
@@ -2580,10 +2597,13 @@ async def test_vision_load_consumes_ephemeral_browser_refs(monkeypatch):
2597 response = await tool.execute(paths=[ref])
2598 await tool.after_execution(response)
2599
2583 - assert ephemeral_images.get_image(ref, context_id="ctx-vision") is None
2600 + assert vision_load_module.ephemeral_images.get_image(ref, context_id="ctx-vision") is None
2601 assert tool.loaded_paths == ["browser-shot.jpg"]
2602 raw_message = messages[0][1]["content"]
2586 - assert raw_message.raw_content[0]["image_url"]["url"] == f"data:image/jpeg;base64,{SMALL_JPEG_10X10}"
2603 + stored_ref = raw_message["raw_content"][0]["image_url"]["url"]
2604 + assert stored_ref.startswith("/a0/usr/chats/ctx-vision/screenshots/browser/browser-shot-")
2605 + stored_path = tmp_path / stored_ref.removeprefix("/a0/")
2606 + assert stored_path.read_bytes() == __import__("base64").b64decode(SMALL_JPEG_10X10)
2607 assert updates[-1]["result"] == "1 images loaded, 0 skipped"
2608
2609
tests/test_host_browser_connector.py
+15 -11
@@ -12,8 +12,8 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
12 if str(PROJECT_ROOT) not in sys.path:
13 sys.path.insert(0, str(PROJECT_ROOT))
14
15 -from helpers import ephemeral_images
15 from plugins._a0_connector.helpers import ws_runtime
16 +from plugins._browser.helpers import connector_runtime as connector_runtime_module
17 from plugins._browser.helpers.connector_runtime import (
18 ConnectorBrowserRuntime,
19 _agent_uses_local_chat_model,
@@ -330,7 +330,15 @@ def test_connector_runtime_adds_docker_recovery_to_host_errors():
330 assert "/browser container" in message
331
332
333 -def test_host_browser_artifacts_become_context_scoped_ephemeral_refs(tmp_path):
333 +def test_host_browser_artifacts_become_chat_scoped_files(monkeypatch, tmp_path):
334 + def fake_get_abs_path(*parts):
335 + return str(tmp_path.joinpath(*parts))
336 +
337 + def fake_normalize_a0_path(path):
338 + return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/")
339 +
340 + monkeypatch.setattr(connector_runtime_module.chat_media.files, "get_abs_path", fake_get_abs_path)
341 + monkeypatch.setattr(connector_runtime_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
342 runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))
343
344 result = runtime._materialize_artifact(
@@ -352,19 +360,15 @@ def test_host_browser_artifacts_become_context_scoped_ephemeral_refs(tmp_path):
360
361 inner = result[0]["result"]
362 assert "artifact" not in inner
355 - assert "path" not in inner
356 - assert "a0_path" not in inner
363 + assert Path(inner["path"]).read_bytes() == b"fake"
364 + assert inner["a0_path"].startswith("/a0/usr/chats/ctx-host/screenshots/browser/shot-")
365 assert inner["context_id"] == "ctx-host"
358 - assert inner["ephemeral"] is True
359 - assert inner["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)
360 - assert inner["vision_load"]["tool_args"]["paths"] == [inner["ephemeral_ref"]]
361 - assert ephemeral_images.consume_image(inner["ephemeral_ref"], context_id="ctx-host").data_url == "data:image/jpeg;base64,ZmFrZQ=="
362 - assert not list(tmp_path.rglob("shot.jpg"))
366 + assert inner["ephemeral"] is False
367 + assert inner["chat_scoped"] is True
368 + assert inner["vision_load"]["tool_args"]["paths"] == [inner["a0_path"]]
369
370
371 def test_host_browser_artifact_materialization_rejects_oversized_payload(monkeypatch, tmp_path):
366 - import plugins._browser.helpers.connector_runtime as connector_runtime_module
367 -
372 monkeypatch.setattr(connector_runtime_module, "MAX_ARTIFACT_SIZE_BYTES", 2)
373 runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))
374
tests/test_office_canvas_setup.py
+8
@@ -264,6 +264,14 @@ def test_desktop_plugin_owns_routes_runtime_surface_and_state_paths():
264 assert "DESKTOP_RUNTIME_INSTALL_MESSAGE" in desktop_store
265 assert "openDesktopWhenRuntimeReady" in desktop_store
266 assert "isDesktopRuntimeInstalling" in desktop_store
267 + assert "_desktopDisplaySizes: {}" in desktop_store
268 + assert "desktopDisplaySizeForToken(token" in desktop_store
269 + assert "rememberDesktopDisplaySize(token" in desktop_store
270 + assert "options.displaySize || this.desktopDisplaySizeForToken(token)" in desktop_store
271 + assert "result?.width || width" in desktop_store
272 + assert "canvas.width = normalizedWidth" in desktop_store
273 + assert "canvas.height = normalizedHeight" in desktop_store
274 + assert "canvas?.clientWidth || canvas?.width" in desktop_store
275 assert "Installing Agent Zero Desktop runtime dependencies" in desktop_session
276 assert "__a0XpraOffsetWarnPatched" in desktop_store
277 assert "window does not fit in canvas, offsets" in desktop_store
tests/test_office_desktop_state.py
+8 -5
@@ -191,7 +191,8 @@ def test_desktop_state_screenshot_capture_uses_xwd_and_pillow_when_available(tmp
191
192
193 def test_desktop_state_shell_screenshot_path_is_context_scoped(tmp_path, monkeypatch):
194 - monkeypatch.setattr(desktop_state, "SCREENSHOT_DIR", tmp_path)
194 + monkeypatch.setattr(desktop_state, "BASE_DIR", tmp_path)
195 + monkeypatch.setattr(desktop_state, "SCREENSHOT_DIR", tmp_path / "tmp" / "desktop" / "screenshots")
196 capabilities = {"xwd": "/usr/bin/xwd"}
197 env = {"DISPLAY": ":120"}
198
@@ -222,7 +223,7 @@ def test_desktop_state_shell_screenshot_path_is_context_scoped(tmp_path, monkeyp
223 monkeypatch.setattr(desktop_state, "run", fake_run)
224 monkeypatch.setitem(sys.modules, "PIL", pil_module)
225 monkeypatch.setitem(sys.modules, "PIL.Image", image_module)
225 - stale_path = tmp_path / "ctx_id" / "stale.png"
226 + stale_path = tmp_path / "tmp" / "desktop" / "screenshots" / "ctx_id" / "stale.png"
227 stale_path.parent.mkdir(parents=True)
228 stale_path.write_bytes(b"stale")
229
@@ -236,12 +237,14 @@ def test_desktop_state_shell_screenshot_path_is_context_scoped(tmp_path, monkeyp
237
238 path = Path(screenshot["path"])
239 assert screenshot["ok"] is True
239 - assert screenshot["ephemeral"] is True
240 + assert screenshot["ephemeral"] is False
241 + assert screenshot["chat_scoped"] is True
242 assert screenshot["context_id"] == "ctx_id"
241 - assert path.parent == tmp_path / "ctx_id"
243 + assert screenshot["a0_path"].startswith("/a0/usr/chats/ctx_id/screenshots/desktop/desktop-")
244 + assert path.parent == tmp_path / "usr" / "chats" / "ctx_id" / "screenshots" / "desktop"
245 assert path.name.startswith("desktop-")
246 assert desktop_state.latest_screenshot(context_id="ctx/id")["path"] == str(path)
244 - assert not stale_path.exists()
247 + assert stale_path.exists()
248
249
250 def test_desktop_state_default_screenshot_returns_ephemeral_ref(tmp_path, monkeypatch):
tests/test_tool_action_contracts.py
+32
@@ -699,3 +699,35 @@ def test_computer_use_remote_start_session_reports_backend_features_and_windows_
699 assert "backend=windows/windows" in message
700 assert "features=uia-tree-snapshot, uia-structural-targeting" in message
701 assert "host-computer-use-windows" in message
702 +
703 +
704 +def test_computer_use_remote_capture_artifact_is_chat_scoped(monkeypatch, tmp_path: Path):
705 + module = _load_computer_use_remote_tool(monkeypatch)
706 +
707 + def fake_get_abs_path(*parts):
708 + return str(tmp_path.joinpath(*parts))
709 +
710 + def fake_normalize_a0_path(path):
711 + return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/")
712 +
713 + monkeypatch.setattr(module.chat_media.files, "get_abs_path", fake_get_abs_path)
714 + monkeypatch.setattr(module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
715 +
716 + tool = object.__new__(module.ComputerUseRemote)
717 + tool.agent = types.SimpleNamespace(context=types.SimpleNamespace(id="ctx-computer"))
718 +
719 + display_ref, capture_id = tool._resolve_capture_ref(
720 + {
721 + "artifact": {
722 + "filename": "capture.png",
723 + "mime": "image/png",
724 + "encoding": "base64",
725 + "data": "ZmFrZQ==",
726 + },
727 + }
728 + )
729 +
730 + assert display_ref.startswith("/a0/usr/chats/ctx-computer/screenshots/computer-use/capture-")
731 + stored_path = tmp_path / display_ref.removeprefix("/a0/")
732 + assert stored_path.read_bytes() == b"fake"
733 + assert capture_id == stored_path.stem
tests/test_vision_load_image_refs.py new
+123
@@ -0,0 +1,123 @@
1 +import types
2 +from types import SimpleNamespace
3 +import sys
4 +from pathlib import Path
5 +
6 +import pytest
7 +
8 +PROJECT_ROOT = Path(__file__).resolve().parents[1]
9 +if str(PROJECT_ROOT) not in sys.path:
10 + sys.path.insert(0, str(PROJECT_ROOT))
11 +
12 +from helpers import images
13 +
14 +
15 +class _TestResponse(SimpleNamespace):
16 + def __init__(self, message="", break_loop=False, **kwargs):
17 + super().__init__(message=message, break_loop=break_loop, **kwargs)
18 +
19 +
20 +class _TestTool:
21 + def __init__(
22 + self,
23 + agent=None,
24 + name="",
25 + method=None,
26 + args=None,
27 + message="",
28 + loop_data=None,
29 + **kwargs,
30 + ):
31 + self.agent = agent
32 + self.name = name
33 + self.method = method
34 + self.args = args or {}
35 + self.message = message
36 + self.loop_data = loop_data
37 +
38 +
39 +def _install_tool_stub(monkeypatch):
40 + tool_stub = types.ModuleType("helpers.tool")
41 + tool_stub.Response = _TestResponse
42 + tool_stub.Tool = _TestTool
43 + history_stub = types.ModuleType("helpers.history")
44 +
45 + class _RawMessage(dict):
46 + def __init__(self, raw_content, preview):
47 + super().__init__(raw_content=raw_content, preview=preview)
48 +
49 + history_stub.RawMessage = _RawMessage
50 + monkeypatch.setitem(sys.modules, "helpers.tool", tool_stub)
51 + monkeypatch.setitem(sys.modules, "helpers.history", history_stub)
52 + monkeypatch.delitem(sys.modules, "tools.vision_load", raising=False)
53 +
54 +
55 +def test_prepare_content_keeps_missing_local_image_refs_strict():
56 + missing_path = "/tmp/a0-missing-desktop-screenshot.png"
57 +
58 + with pytest.raises(FileNotFoundError):
59 + images.prepare_content(
60 + [{"type": "image_url", "image_url": {"url": missing_path}}]
61 + )
62 +
63 +
64 +@pytest.mark.anyio
65 +async def test_vision_load_materializes_local_image_to_chat_artifact(monkeypatch, tmp_path):
66 + _install_tool_stub(monkeypatch)
67 + import tools.vision_load as vision_load_module
68 +
69 + def fake_get_abs_path(*parts):
70 + return str(tmp_path.joinpath(*parts))
71 +
72 + def fake_normalize_a0_path(path):
73 + return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/")
74 +
75 + monkeypatch.setattr(vision_load_module.chat_media.files, "get_abs_path", fake_get_abs_path)
76 + monkeypatch.setattr(vision_load_module.chat_media.files, "normalize_a0_path", fake_normalize_a0_path)
77 + monkeypatch.setattr(
78 + vision_load_module.plugins,
79 + "get_plugin_config",
80 + lambda *args, **kwargs: {"chat_model": {"max_embeds": 10}},
81 + )
82 +
83 + async def direct_call(func, *args, **kwargs):
84 + return func(*args, **kwargs)
85 +
86 + monkeypatch.setattr(
87 + vision_load_module.runtime,
88 + "call_development_function",
89 + direct_call,
90 + )
91 +
92 + image_path = tmp_path / "sample-image.png"
93 + image_path.write_bytes(b"png-data")
94 +
95 + tool_results = []
96 + messages = []
97 + updates = []
98 + agent = SimpleNamespace(
99 + context=SimpleNamespace(id="ctx-vision"),
100 + agent_name="Agent 0",
101 + hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
102 + hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)),
103 + )
104 + tool = vision_load_module.VisionLoad(
105 + agent=agent,
106 + name="vision_load",
107 + method=None,
108 + args={"paths": [str(image_path)]},
109 + message="",
110 + loop_data=None,
111 + )
112 + tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: updates.append(kwargs))
113 +
114 + response = await tool.execute(paths=[str(image_path)])
115 + image_path.unlink()
116 + await tool.after_execution(response)
117 +
118 + raw_message = messages[0][1]["content"]
119 + stored_ref = raw_message["raw_content"][0]["image_url"]["url"]
120 + assert stored_ref.startswith("/a0/usr/chats/ctx-vision/images/vision-load/sample-image-")
121 + stored_path = tmp_path / stored_ref.removeprefix("/a0/")
122 + assert stored_path.read_bytes() == b"png-data"
123 + assert updates[-1]["result"] == "1 images loaded, 0 skipped"
tools/vision_load.py
+58 -8
@@ -1,6 +1,6 @@
1 from helpers.print_style import PrintStyle
2 from helpers.tool import Tool, Response
3 -from helpers import runtime, files, plugins, ephemeral_images
3 +from helpers import runtime, files, plugins, ephemeral_images, images, chat_media
4 from mimetypes import guess_type
5 from helpers import history
6
@@ -27,7 +27,7 @@ class VisionLoad(Tool):
27 else []
28 )
29
30 - for path, display_path in limited_paths:
30 + for idx, (path, display_path) in enumerate(limited_paths):
31 if not path:
32 continue
33 if ephemeral_images.is_ref(path):
@@ -38,12 +38,16 @@ class VisionLoad(Tool):
38 if image is None:
39 continue
40 display = image.display_name or display_path
41 - self.images_dict[display] = image.data_url
42 - self.loaded_paths.append(display)
41 + stored_ref = self._store_ephemeral_image(image)
42 + if stored_ref:
43 + self.images_dict[display] = stored_ref
44 + self.loaded_paths.append(display)
45 continue
46 if self._is_data_image_url(path):
45 - self.images_dict[display_path] = path
46 - self.loaded_paths.append(display_path)
47 + stored_ref = self._store_data_url(path, preferred_name=f"vision-load-{idx + 1}.png")
48 + if stored_ref:
49 + self.images_dict[display_path] = stored_ref
50 + self.loaded_paths.append(display_path)
51 continue
52 if not await runtime.call_development_function(files.exists, str(path)):
53 continue
@@ -51,8 +55,12 @@ class VisionLoad(Tool):
55 if path not in self.images_dict:
56 mime_type, _ = guess_type(str(path))
57 if mime_type and mime_type.startswith("image/"):
54 - self.images_dict[display_path] = str(path)
55 - self.loaded_paths.append(display_path)
58 + try:
59 + stored_ref = self._store_local_image(path, preferred_name=files.basename(path))
60 + self.images_dict[display_path] = stored_ref
61 + self.loaded_paths.append(display_path)
62 + except (FileNotFoundError, OSError, ValueError):
63 + continue
64
65 return Response(message="dummy", break_loop=False)
66
@@ -65,6 +73,48 @@ class VisionLoad(Tool):
73 def _context_id(self) -> str:
74 return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()
75
76 + def _store_ephemeral_image(self, image: ephemeral_images.EphemeralImage) -> str:
77 + context_id = self._context_id()
78 + if not context_id:
79 + return image.data_url
80 + source = chat_media.infer_source(image.ref, image.display_name)
81 + category = chat_media.category_for_source(source)
82 + saved = chat_media.save_image_base64(
83 + context_id=context_id,
84 + data=image.data,
85 + mime_type=image.mime,
86 + category=category,
87 + source=source,
88 + preferred_name=image.display_name,
89 + )
90 + return saved.a0_path
91 +
92 + def _store_data_url(self, data_url: str, *, preferred_name: str = "") -> str:
93 + context_id = self._context_id()
94 + if not context_id:
95 + return data_url
96 + source = chat_media.infer_source(data_url, preferred_name)
97 + category = chat_media.category_for_source(source)
98 + saved = chat_media.save_image_data_url(
99 + context_id=context_id,
100 + data_url=data_url,
101 + category=category,
102 + source=source,
103 + preferred_name=preferred_name,
104 + )
105 + return saved.a0_path
106 +
107 + def _store_local_image(self, path: str, *, preferred_name: str = "") -> str:
108 + context_id = self._context_id()
109 + if not context_id:
110 + return images.to_data_url(path)
111 + return chat_media.materialize_image_ref(
112 + context_id=context_id,
113 + url=path,
114 + source=chat_media.infer_source(path, preferred_name),
115 + preferred_name=preferred_name,
116 + )
117 +
118 @staticmethod
119 def _is_data_image_url(value: str) -> bool:
120 normalized = str(value or "").strip().lower()