@samitouri / QOSami_Agents_Zero / commits / 430c48d1

Make browser screenshots ephemeral and context scoped

Route no-path browser screenshots through an in-process ephemeral image registry that vision_load consumes into the existing data-url model boundary. Stop materializing host-browser artifacts into tmp/browser/host-screenshots, keep explicit path screenshots durable, and make browser log metadata point at the active chat/task context while preserving browser-context detail.

Alessandro committed May 22, 2026 at 09:50 UTC 430c48d1a5ae8cf0934426408c1d5b4083b1e6e2
11 files changed +385 -107
helpers/ephemeral_images.py new
+150
@@ -0,0 +1,150 @@
1 +from __future__ import annotations
2 +
3 +import base64
4 +import threading
5 +import time
6 +import uuid
7 +from dataclasses import dataclass
8 +
9 +
10 +REF_PREFIX = "a0-ephemeral-image://"
11 +DEFAULT_TTL_SECONDS = 15 * 60
12 +
13 +
14 +@dataclass(frozen=True)
15 +class EphemeralImage:
16 + ref: str
17 + context_id: str
18 + mime: str
19 + data: str
20 + name: str
21 + created_at: float
22 + expires_at: float
23 +
24 + @property
25 + def data_url(self) -> str:
26 + return f"data:{self.mime};base64,{self.data}"
27 +
28 + @property
29 + def display_name(self) -> str:
30 + return self.name or display_ref(self.ref)
31 +
32 +
33 +_store: dict[str, EphemeralImage] = {}
34 +_lock = threading.RLock()
35 +
36 +
37 +def put_image_bytes(
38 + *,
39 + context_id: str,
40 + mime: str,
41 + payload: bytes,
42 + name: str = "",
43 + ttl_seconds: float = DEFAULT_TTL_SECONDS,
44 +) -> str:
45 + data = base64.b64encode(bytes(payload or b"")).decode("ascii")
46 + return put_image(
47 + context_id=context_id,
48 + mime=mime,
49 + data=data,
50 + name=name,
51 + ttl_seconds=ttl_seconds,
52 + )
53 +
54 +
55 +def put_image(
56 + *,
57 + context_id: str,
58 + mime: str,
59 + data: str,
60 + name: str = "",
61 + ttl_seconds: float = DEFAULT_TTL_SECONDS,
62 +) -> str:
63 + compact_data = _compact_base64(data)
64 + if not compact_data:
65 + raise ValueError("ephemeral image data is empty")
66 + base64.b64decode(compact_data, validate=True)
67 +
68 + normalized_mime = _normalize_mime(mime)
69 + now = time.time()
70 + ref = f"{REF_PREFIX}{uuid.uuid4().hex}"
71 + image = EphemeralImage(
72 + ref=ref,
73 + context_id=str(context_id or "").strip(),
74 + mime=normalized_mime,
75 + data=compact_data,
76 + name=str(name or "").strip(),
77 + created_at=now,
78 + expires_at=now + max(1.0, float(ttl_seconds or DEFAULT_TTL_SECONDS)),
79 + )
80 + with _lock:
81 + _prune_expired_locked(now)
82 + _store[ref] = image
83 + return ref
84 +
85 +
86 +def is_ref(value: object) -> bool:
87 + return str(value or "").strip().startswith(REF_PREFIX)
88 +
89 +
90 +def display_ref(ref: str) -> str:
91 + value = str(ref or "").strip()
92 + if not is_ref(value):
93 + return value
94 + return f"{REF_PREFIX}<ephemeral>"
95 +
96 +
97 +def get_image(ref: str, *, context_id: str = "") -> EphemeralImage | None:
98 + return _resolve_image(ref, context_id=context_id, consume=False)
99 +
100 +
101 +def consume_image(ref: str, *, context_id: str = "") -> EphemeralImage | None:
102 + return _resolve_image(ref, context_id=context_id, consume=True)
103 +
104 +
105 +def delete_image(ref: str) -> None:
106 + with _lock:
107 + _store.pop(str(ref or "").strip(), None)
108 +
109 +
110 +def clear_context(context_id: str) -> None:
111 + normalized_context = str(context_id or "").strip()
112 + with _lock:
113 + for ref, image in list(_store.items()):
114 + if image.context_id == normalized_context:
115 + _store.pop(ref, None)
116 +
117 +
118 +def _resolve_image(ref: str, *, context_id: str = "", consume: bool) -> EphemeralImage | None:
119 + value = str(ref or "").strip()
120 + if not is_ref(value):
121 + return None
122 +
123 + now = time.time()
124 + with _lock:
125 + _prune_expired_locked(now)
126 + image = _store.get(value)
127 + if image is None:
128 + return None
129 + requested_context = str(context_id or "").strip()
130 + if requested_context and image.context_id and image.context_id != requested_context:
131 + return None
132 + if consume:
133 + _store.pop(value, None)
134 + return image
135 +
136 +
137 +def _compact_base64(data: str) -> str:
138 + return "".join(char for char in str(data or "") if not char.isspace())
139 +
140 +
141 +def _normalize_mime(mime: str) -> str:
142 + value = str(mime or "").strip().lower()
143 + return value if value.startswith("image/") else "image/jpeg"
144 +
145 +
146 +def _prune_expired_locked(now: float | None = None) -> None:
147 + current = time.time() if now is None else float(now)
148 + for ref, image in list(_store.items()):
149 + if image.expires_at <= current:
150 + _store.pop(ref, None)
plugins/_browser/helpers/connector_runtime.py
+15 -31
@@ -1,7 +1,6 @@
1 from __future__ import annotations
2
3 import asyncio
4 -import base64
4 import hashlib
5 import re
6 import uuid
@@ -10,7 +9,7 @@ from pathlib import Path
9 from typing import Any
10 from urllib.parse import urlparse
11
13 -from helpers import files
12 +from helpers import ephemeral_images
13
14 try:
15 from helpers.ws import NAMESPACE
@@ -40,10 +39,8 @@ from plugins._browser.helpers.url import normalize_url
39
40 BROWSER_OP_EVENT = "connector_browser_op"
41 BROWSER_OP_TIMEOUT = 120.0
43 -HOST_BROWSER_SCREENSHOT_DIR = ("tmp", "browser", "host-screenshots")
42 CONTENT_HELPER_PATH = Path(__file__).resolve().parents[1] / "assets" / "browser-page-content.js"
43 MAX_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024
46 -BASE64_DECODE_CHARS_PER_CHUNK = 64 * 1024
44 HOST_BROWSER_PRIVACY_POLICY_KEY = getattr(
45 browser_config,
46 "HOST_BROWSER_PRIVACY_POLICY_KEY",
@@ -431,26 +428,30 @@ class ConnectorBrowserRuntime:
428 estimated_size = _estimated_base64_decoded_size(data)
429 if estimated_size > MAX_ARTIFACT_SIZE_BYTES:
430 raise RuntimeError(
434 - "Host browser artifact is too large to materialize safely "
431 + "Host browser artifact is too large to attach safely "
432 f"({estimated_size} bytes, limit {MAX_ARTIFACT_SIZE_BYTES} bytes)."
433 )
434 filename = _safe_filename(str(artifact.get("filename") or "host-browser.jpg"))
438 - target_dir = Path(files.get_abs_path(*HOST_BROWSER_SCREENSHOT_DIR, self.context_id))
439 - target_dir.mkdir(parents=True, exist_ok=True)
440 - target_path = target_dir / filename
435 try:
442 - _write_base64_to_path(data, target_path)
436 + ref = ephemeral_images.put_image(
437 + context_id=self.context_id,
438 + mime=str(artifact.get("mime") or result.get("mime") or "image/jpeg"),
439 + data=data,
440 + name=filename,
441 + )
442 except Exception as exc:
444 - target_path.unlink(missing_ok=True)
443 raise RuntimeError("Host browser artifact could not be decoded.") from exc
444 materialized = dict(result)
445 materialized.pop("artifact", None)
448 - local_path = str(target_path)
449 - materialized["path"] = local_path
450 - materialized["a0_path"] = files.normalize_a0_path(local_path)
446 + materialized.pop("path", None)
447 + materialized.pop("a0_path", None)
448 + materialized.pop("host_path", None)
449 + materialized.setdefault("context_id", self.context_id)
450 + materialized["ephemeral"] = True
451 + materialized["ephemeral_ref"] = ref
452 materialized["vision_load"] = {
453 "tool_name": "vision_load",
453 - "tool_args": {"paths": [local_path]},
454 + "tool_args": {"paths": [ref]},
455 }
456 return materialized
457
@@ -569,20 +570,3 @@ def _safe_filename(value: str) -> str:
570 def _estimated_base64_decoded_size(data: str) -> int:
571 compact_length = sum(1 for char in data if not char.isspace())
572 return (compact_length * 3) // 4
572 -
573 -
574 -def _write_base64_to_path(data: str, target_path: Path) -> None:
575 - pending = ""
576 - with target_path.open("wb") as target:
577 - for offset in range(0, len(data), BASE64_DECODE_CHARS_PER_CHUNK):
578 - chunk = pending + "".join(
579 - char
580 - for char in data[offset : offset + BASE64_DECODE_CHARS_PER_CHUNK]
581 - if not char.isspace()
582 - )
583 - ready_length = (len(chunk) // 4) * 4
584 - if ready_length:
585 - target.write(base64.b64decode(chunk[:ready_length], validate=True))
586 - pending = chunk[ready_length:]
587 - if pending:
588 - target.write(base64.b64decode(pending, validate=True))
plugins/_browser/helpers/runtime.py
+30 -1
@@ -15,7 +15,7 @@ from dataclasses import dataclass
15 from pathlib import Path
16 from typing import Any
17
18 -from helpers import files
18 +from helpers import ephemeral_images, files
19 from helpers.defer import DeferredTask
20 from helpers.errors import RepairableException
21 from helpers.print_style import PrintStyle
@@ -1548,6 +1548,34 @@ class _BrowserRuntimeCore:
1548 await self.ensure_started()
1549 resolved_id = self._resolve_browser_id(browser_id)
1550 page = self._page(resolved_id)
1551 + raw_path = str(path or "").strip()
1552 + if not raw_path:
1553 + image = await page.screenshot(
1554 + type="jpeg",
1555 + quality=max(20, min(95, int(quality))),
1556 + full_page=bool(full_page),
1557 + )
1558 + ref = ephemeral_images.put_image_bytes(
1559 + context_id=self.context_id,
1560 + mime="image/jpeg",
1561 + payload=image,
1562 + name=f"browser-{resolved_id}.jpg",
1563 + )
1564 + return {
1565 + "browser_id": resolved_id,
1566 + "context_id": self.context_id,
1567 + "mime": "image/jpeg",
1568 + "ephemeral": True,
1569 + "ephemeral_ref": ref,
1570 + "state": await self._state(resolved_id),
1571 + "vision_load": {
1572 + "tool_name": "vision_load",
1573 + "tool_args": {
1574 + "paths": [ref],
1575 + },
1576 + },
1577 + }
1578 +
1579 output_path, image_type, mime = self._screenshot_output_path(resolved_id, path)
1580 output_path.parent.mkdir(parents=True, exist_ok=True)
1581 clamped_quality = max(20, min(95, int(quality)))
@@ -1562,6 +1590,7 @@ class _BrowserRuntimeCore:
1590 local_path = str(output_path)
1591 return {
1592 "browser_id": resolved_id,
1593 + "context_id": self.context_id,
1594 "path": local_path,
1595 "a0_path": files.normalize_a0_path(local_path),
1596 "mime": mime,
plugins/_browser/prompts/agent.system.tool.browser.md
+1 -1
@@ -20,7 +20,7 @@ Workflow:
20 - For same-page controls that are easier to identify structurally, `click`, `type`, `submit`, `type_submit`, `scroll`, `select_option`, `set_checked`, and `upload_file` may use `selector` instead of `ref`; the tool resolves the selector through `content` first.
21 - `click` with `x`/`y` and no `ref` is treated as a coordinate mouse click. `type` with text and no `ref` types into the currently focused element. `key_chord` accepts either `["Control", "A"]` or `"CTRL+A"`.
22 - `navigate` reuses an existing `browser_id` and is preferred for serial browsing.
23 -- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned path before reasoning visually.
23 +- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned `vision_load.tool_args.paths` value before reasoning visually. When no `path` is requested, browser screenshots are ephemeral refs rather than conserved files.
24 - Keep the tab set small; close pages after extracting what you need.
25
26 `multi` is only a browser action: use `tool_name: "browser"` with `tool_args.action: "multi"`. Never use `tool_name: "multi"`.
plugins/_browser/skills/browser-automation/SKILL.md
+2 -2
@@ -29,10 +29,10 @@ In host mode, page content and screenshots may be blocked by host-content policy
29 Screenshots are explicit only; the browser does not automatically load images into model context.
30
31 1. Call `browser` with `action: "screenshot"`.
32 -2. Call `vision_load` with the returned path.
32 +2. Call `vision_load` with the returned `vision_load.tool_args.paths` value.
33 3. Reason from the latest loaded screenshot.
34
35 -Screenshot args include `quality`, `full_page`, and optional `path`. PNG is used when `path` ends with `.png`; otherwise JPEG is used.
35 +Screenshot args include `quality`, `full_page`, and optional `path`. Without `path`, the screenshot is an ephemeral ref consumed by `vision_load`; with `path`, PNG is used when `path` ends with `.png`, otherwise JPEG is used.
36
37 ## Forms And Files
38
plugins/_browser/skills/browser-form-workflows/SKILL.md
+1 -1
@@ -11,7 +11,7 @@ Start with `browser:content` to capture current refs, then use `browser:detail`
11
12 Use `select_option`, `set_checked`, `upload_file`, `type`, `type_submit`, and `submit` for form interaction. Use coordinates only when no stable ref exists or the UI is intentionally canvas-like.
13
14 -Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history.
14 +Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history; no-path screenshots return ephemeral refs for `vision_load`.
15
16 Verify after submission with `browser:content`, `browser:state`, or another explicit `browser:screenshot` plus `vision_load`.
17
plugins/_browser/tools/browser.py
+60 -25
@@ -89,8 +89,6 @@ class Browser(Tool):
89 if action == "open":
90 result = await runtime.call("open", url or "")
91 elif action == "screenshot":
92 - if not path:
93 - path = self._history_screenshot_path(action)
92 result = await runtime.call(
93 "screenshot_file",
94 browser_id,
@@ -428,53 +426,65 @@ class Browser(Tool):
426 return
427
428 screenshot = result if action == "screenshot" and isinstance(result, dict) else None
431 - if not self._screenshot_has_path(screenshot):
429 + if not self._screenshot_has_reference(screenshot):
430 target_browser_id = self._browser_id_from_result(result) or requested_browser_id
433 - output_path = self._history_screenshot_path(action)
434 - if not output_path:
435 - return
431 try:
432 screenshot = await runtime.call(
433 "screenshot_file",
434 target_browser_id,
435 quality=HISTORY_SCREENSHOT_QUALITY,
436 full_page=False,
442 - path=output_path,
437 + path="",
438 )
439 except Exception as exc:
440 PrintStyle.debug(
441 "Browser history screenshot capture failed:",
442 f"browser_id={target_browser_id}",
443 f"quality={HISTORY_SCREENSHOT_QUALITY}",
449 - f"path={output_path}",
444 f"error={exc}",
445 )
446 return
447
454 - if not self._screenshot_has_path(screenshot):
448 + if not self._screenshot_has_reference(screenshot):
449 return
450
457 - local_path = str(screenshot.get("path") or files.fix_dev_path(str(screenshot.get("a0_path") or "")))
458 - if not local_path:
459 - return
460 - uri = f"img://{local_path}&t={time.time()}"
451 + a0_path = str(screenshot.get("a0_path") or "").strip()
452 + local_path = str(screenshot.get("path") or (files.fix_dev_path(a0_path) if a0_path else ""))
453 state = screenshot.get("state") if isinstance(screenshot.get("state"), dict) else {}
462 - self.log.update(
463 - Screenshot=uri,
464 - browser_snapshot={
465 - "uri": uri,
466 - "path": local_path,
467 - "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),
468 - "mime": screenshot.get("mime") or "image/jpeg",
469 - "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,
470 - "context_id": screenshot.get("context_id") or state.get("context_id") or "",
471 - },
472 - )
454 + chat_context_id = self._agent_context_id()
455 + browser_context_id = str(screenshot.get("context_id") or state.get("context_id") or "").strip()
456 + snapshot = {
457 + "mime": screenshot.get("mime") or "image/jpeg",
458 + "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,
459 + "context_id": chat_context_id or browser_context_id,
460 + "browser_context_id": browser_context_id,
461 + }
462 + update_payload: dict[str, Any] = {"browser_snapshot": snapshot}
463 + if local_path:
464 + uri = f"img://{local_path}&t={time.time()}"
465 + snapshot.update(
466 + {
467 + "uri": uri,
468 + "path": local_path,
469 + "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),
470 + "ephemeral": False,
471 + }
472 + )
473 + update_payload["Screenshot"] = uri
474 + else:
475 + ephemeral_ref = self._screenshot_ephemeral_ref(screenshot)
476 + snapshot.update(
477 + {
478 + "ephemeral": bool(ephemeral_ref),
479 + "ephemeral_ref": ephemeral_ref,
480 + }
481 + )
482 + self.log.update(**update_payload)
483
484 def _history_screenshot_path(self, action: str) -> str:
485 if not getattr(self, "agent", None) or not getattr(self.agent, "context", None):
486 return ""
477 - context_id = str(getattr(self.agent.context, "id", "") or "").strip()
487 + context_id = self._agent_context_id()
488 if not context_id:
489 return ""
490 from helpers import persist_chat
@@ -521,6 +531,31 @@ class Browser(Tool):
531 def _screenshot_has_path(screenshot: Any) -> bool:
532 return isinstance(screenshot, dict) and bool(screenshot.get("path") or screenshot.get("a0_path"))
533
534 + @classmethod
535 + def _screenshot_has_reference(cls, screenshot: Any) -> bool:
536 + return cls._screenshot_has_path(screenshot) or bool(cls._screenshot_ephemeral_ref(screenshot))
537 +
538 + @staticmethod
539 + def _screenshot_ephemeral_ref(screenshot: Any) -> str:
540 + if not isinstance(screenshot, dict):
541 + return ""
542 + ref = str(screenshot.get("ephemeral_ref") or "").strip()
543 + if ref:
544 + return ref
545 + vision_load = screenshot.get("vision_load")
546 + if isinstance(vision_load, dict):
547 + tool_args = vision_load.get("tool_args")
548 + if isinstance(tool_args, dict):
549 + paths = tool_args.get("paths")
550 + if isinstance(paths, list) and paths:
551 + first = str(paths[0] or "").strip()
552 + if first.startswith("a0-ephemeral-image://"):
553 + return first
554 + return ""
555 +
556 + def _agent_context_id(self) -> str:
557 + return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()
558 +
559 @staticmethod
560 def _format_result(action: str, result: Any) -> str:
561 if action == "content" and isinstance(result, dict):
prompts/agent.system.tools_vision.md
+1 -1
@@ -2,7 +2,7 @@
2
3 ### vision_load
4 load images into the model for visual reasoning
5 -args: `paths` list of absolute image paths
5 +args: `paths` list of absolute image paths or tool-returned ephemeral image refs
6 rules:
7 - load all relevant images in one call when comparing screenshots or pages
8 - use when the task depends on screenshots, diagrams, scanned documents, charts, or photos
tests/test_browser_agent_regressions.py
+67 -18
@@ -79,6 +79,7 @@ sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_conf
79 def anyio_backend():
80 return "asyncio"
81
82 +from helpers import ephemeral_images
83 from helpers.errors import RepairableException
84 from plugins._browser.helpers.config import (
85 build_browser_launch_config,
@@ -1808,12 +1809,10 @@ async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_p
1809 },
1810 }
1811 if method == "screenshot_file":
1811 - Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
1812 - Path(kwargs["path"]).write_bytes(b"jpeg")
1812 return {
1813 "browser_id": args[0],
1815 - "path": kwargs["path"],
1816 - "a0_path": "/a0/usr/chats/chat/browser/screenshots/open.jpg",
1814 + "ephemeral": True,
1815 + "ephemeral_ref": "a0-ephemeral-image://fake",
1816 "mime": "image/jpeg",
1817 "state": {"id": args[0], "context_id": "browser-context"},
1818 }
@@ -1861,11 +1860,13 @@ async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_p
1860 assert calls[1][1] == (1,)
1861 assert calls[1][2]["quality"] == browser_tool_module.HISTORY_SCREENSHOT_QUALITY
1862 assert calls[1][2]["full_page"] is False
1864 - assert Path(calls[1][2]["path"]).parent == tmp_path / "usr" / "chats" / "chat" / "browser" / "screenshots"
1865 - assert Path(calls[1][2]["path"]).read_bytes() == b"jpeg"
1866 - assert log.updates[-1]["Screenshot"].startswith("img://")
1863 + assert calls[1][2]["path"] == ""
1864 + assert "Screenshot" not in log.updates[-1]
1865 assert log.updates[-1]["browser_snapshot"]["browser_id"] == 1
1868 - assert log.updates[-1]["browser_snapshot"]["context_id"] == "browser-context"
1866 + assert log.updates[-1]["browser_snapshot"]["context_id"] == "chat"
1867 + assert log.updates[-1]["browser_snapshot"]["browser_context_id"] == "browser-context"
1868 + assert log.updates[-1]["browser_snapshot"]["ephemeral"] is True
1869 + assert log.updates[-1]["browser_snapshot"]["ephemeral_ref"] == "a0-ephemeral-image://fake"
1870
1871
1872 @pytest.mark.anyio
@@ -2393,7 +2394,7 @@ async def test_browser_runtime_remounts_initial_changed_viewport():
2394
2395
2396 @pytest.mark.anyio
2396 -async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch, tmp_path):
2397 +async def test_browser_runtime_screenshot_file_defaults_to_ephemeral_ref(monkeypatch, tmp_path):
2398 screenshot_calls = []
2399
2400 def fake_get_abs_path(*parts):
@@ -2411,8 +2412,9 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch
2412
2413 async def screenshot(self, **kwargs):
2414 screenshot_calls.append(kwargs)
2414 - Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
2415 - Path(kwargs["path"]).write_bytes(b"image-bytes")
2415 + if kwargs.get("path"):
2416 + Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
2417 + Path(kwargs["path"]).write_bytes(b"image-bytes")
2418 return b"image-bytes"
2419
2420 async def title(self):
@@ -2427,21 +2429,23 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch
2429
2430 result = await core.screenshot_file(5, quality=500)
2431
2430 - path = Path(result["path"])
2431 - assert path.exists()
2432 - assert path.parent == tmp_path / "tmp" / "browser" / "screenshots" / "ctx_id"
2433 - assert path.name.startswith("browser-5-")
2434 - assert path.suffix == ".jpg"
2435 - assert result["a0_path"].startswith("/a0/tmp/browser/screenshots/ctx_id/browser-5-")
2432 + assert "path" not in result
2433 + assert "a0_path" not in result
2434 + assert result["context_id"] == "ctx/id"
2435 assert result["mime"] == "image/jpeg"
2436 + assert result["ephemeral"] is True
2437 + assert result["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)
2438 assert result["vision_load"] == {
2439 "tool_name": "vision_load",
2439 - "tool_args": {"paths": [result["path"]]},
2440 + "tool_args": {"paths": [result["ephemeral_ref"]]},
2441 }
2442 assert "image" not in result
2443 + assert not list((tmp_path / "tmp" / "browser" / "screenshots").rglob("*.jpg"))
2444 assert screenshot_calls[-1]["type"] == "jpeg"
2445 assert screenshot_calls[-1]["quality"] == 95
2446 assert screenshot_calls[-1]["full_page"] is False
2447 + assert "path" not in screenshot_calls[-1]
2448 + assert ephemeral_images.consume_image(result["ephemeral_ref"], context_id="ctx/id").data_url == "data:image/jpeg;base64,aW1hZ2UtYnl0ZXM="
2449
2450 png_path = tmp_path / "custom.png"
2451 png_result = await core.screenshot_file(5, quality=1, full_page=True, path=str(png_path))
@@ -2455,6 +2459,51 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch
2459 }
2460
2461
2462 +@pytest.mark.anyio
2463 +async def test_vision_load_consumes_ephemeral_browser_refs(monkeypatch):
2464 + import tools.vision_load as vision_load_module
2465 +
2466 + monkeypatch.setattr(
2467 + vision_load_module.plugins,
2468 + "get_plugin_config",
2469 + lambda *args, **kwargs: {"chat_model": {"max_embeds": 10}},
2470 + )
2471 +
2472 + tool_results = []
2473 + messages = []
2474 + updates = []
2475 + agent = SimpleNamespace(
2476 + context=SimpleNamespace(id="ctx-vision"),
2477 + agent_name="Agent 0",
2478 + hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
2479 + hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)),
2480 + )
2481 + ref = ephemeral_images.put_image(
2482 + context_id="ctx-vision",
2483 + mime="image/jpeg",
2484 + data=SMALL_JPEG_10X10,
2485 + name="browser-shot.jpg",
2486 + )
2487 + tool = vision_load_module.VisionLoad(
2488 + agent=agent,
2489 + name="vision_load",
2490 + method=None,
2491 + args={"paths": [ref]},
2492 + message="",
2493 + loop_data=None,
2494 + )
2495 + tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: updates.append(kwargs))
2496 +
2497 + response = await tool.execute(paths=[ref])
2498 + await tool.after_execution(response)
2499 +
2500 + assert ephemeral_images.get_image(ref, context_id="ctx-vision") is None
2501 + assert tool.loaded_paths == ["browser-shot.jpg"]
2502 + raw_message = messages[0][1]["content"]
2503 + assert raw_message.raw_content[0]["image_url"]["url"] == f"data:image/jpeg;base64,{SMALL_JPEG_10X10}"
2504 + assert updates[-1]["result"] == "1 images loaded, 0 skipped"
2505 +
2506 +
2507 @pytest.mark.anyio
2508 async def test_browser_runtime_ref_point_resolution_applies_offsets():
2509 eval_payloads = []
tests/test_host_browser_connector.py
+10 -21
@@ -12,6 +12,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
12 if str(PROJECT_ROOT) not in sys.path:
13 sys.path.insert(0, str(PROJECT_ROOT))
14
15 +from helpers import ephemeral_images
16 from plugins._a0_connector.helpers import ws_runtime
17 from plugins._browser.helpers.connector_runtime import (
18 ConnectorBrowserRuntime,
@@ -329,19 +330,7 @@ def test_connector_runtime_adds_docker_recovery_to_host_errors():
330 assert "/browser container" in message
331
332
332 -def test_host_browser_artifacts_materialize_inside_multi_results(monkeypatch, tmp_path):
333 - import plugins._browser.helpers.connector_runtime as connector_runtime_module
334 -
335 - monkeypatch.setattr(
336 - connector_runtime_module.files,
337 - "get_abs_path",
338 - lambda *parts: str(tmp_path.joinpath(*parts)),
339 - )
340 - monkeypatch.setattr(
341 - connector_runtime_module.files,
342 - "normalize_a0_path",
343 - lambda path: "/a0/" + str(path).lstrip("/"),
344 - )
333 +def test_host_browser_artifacts_become_context_scoped_ephemeral_refs(tmp_path):
334 runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))
335
336 result = runtime._materialize_artifact(
@@ -363,19 +352,19 @@ def test_host_browser_artifacts_materialize_inside_multi_results(monkeypatch, tm
352
353 inner = result[0]["result"]
354 assert "artifact" not in inner
366 - assert inner["path"].endswith("shot.jpg")
367 - assert Path(inner["path"]).read_bytes() == b"fake"
368 - assert inner["vision_load"]["tool_args"]["paths"] == [inner["path"]]
355 + assert "path" not in inner
356 + assert "a0_path" not in inner
357 + assert inner["context_id"] == "ctx-host"
358 + assert inner["ephemeral"] is True
359 + assert inner["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)
360 + assert inner["vision_load"]["tool_args"]["paths"] == [inner["ephemeral_ref"]]
361 + assert ephemeral_images.consume_image(inner["ephemeral_ref"], context_id="ctx-host").data_url == "data:image/jpeg;base64,ZmFrZQ=="
362 + assert not list(tmp_path.rglob("shot.jpg"))
363
364
365 def test_host_browser_artifact_materialization_rejects_oversized_payload(monkeypatch, tmp_path):
366 import plugins._browser.helpers.connector_runtime as connector_runtime_module
367
374 - monkeypatch.setattr(
375 - connector_runtime_module.files,
376 - "get_abs_path",
377 - lambda *parts: str(tmp_path.joinpath(*parts)),
378 - )
368 monkeypatch.setattr(connector_runtime_module, "MAX_ARTIFACT_SIZE_BYTES", 2)
369 runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))
370
tools/vision_load.py
+48 -6
@@ -1,6 +1,6 @@
1 from helpers.print_style import PrintStyle
2 from helpers.tool import Tool, Response
3 -from helpers import runtime, files, plugins
3 +from helpers import runtime, files, plugins, ephemeral_images
4 from mimetypes import guess_type
5 from helpers import history
6
@@ -16,18 +16,43 @@ class VisionLoad(Tool):
16 self.skipped_paths: list[str] = []
17
18 max_embeds = self._get_max_embeds()
19 - limited_paths = paths if max_embeds <= 0 else paths[-max_embeds:]
20 - self.skipped_paths = paths[:-max_embeds] if max_embeds > 0 and len(paths) > max_embeds else []
19 + requested = [
20 + (str(path or "").strip(), self._display_input_path(str(path or "").strip(), idx + 1))
21 + for idx, path in enumerate(paths)
22 + ]
23 + limited_paths = requested if max_embeds <= 0 else requested[-max_embeds:]
24 + self.skipped_paths = (
25 + [display for _, display in requested[:-max_embeds]]
26 + if max_embeds > 0 and len(requested) > max_embeds
27 + else []
28 + )
29
22 - for path in limited_paths:
30 + for path, display_path in limited_paths:
31 + if not path:
32 + continue
33 + if ephemeral_images.is_ref(path):
34 + image = ephemeral_images.consume_image(
35 + path,
36 + context_id=self._context_id(),
37 + )
38 + if image is None:
39 + continue
40 + display = image.display_name or display_path
41 + self.images_dict[display] = image.data_url
42 + self.loaded_paths.append(display)
43 + continue
44 + if self._is_data_image_url(path):
45 + self.images_dict[display_path] = path
46 + self.loaded_paths.append(display_path)
47 + continue
48 if not await runtime.call_development_function(files.exists, str(path)):
49 continue
50
51 if path not in self.images_dict:
52 mime_type, _ = guess_type(str(path))
53 if mime_type and mime_type.startswith("image/"):
29 - self.images_dict[path] = str(path)
30 - self.loaded_paths.append(path)
54 + self.images_dict[display_path] = str(path)
55 + self.loaded_paths.append(display_path)
56
57 return Response(message="dummy", break_loop=False)
58
@@ -37,6 +62,23 @@ class VisionLoad(Tool):
62 max_embeds = chat_cfg.get("max_embeds", 10)
63 return int(max_embeds or 0)
64
65 + def _context_id(self) -> str:
66 + return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()
67 +
68 + @staticmethod
69 + def _is_data_image_url(value: str) -> bool:
70 + normalized = str(value or "").strip().lower()
71 + return normalized.startswith("data:image/") and ";base64," in normalized
72 +
73 + @classmethod
74 + def _display_input_path(cls, value: str, index: int) -> str:
75 + if ephemeral_images.is_ref(value):
76 + return ephemeral_images.display_ref(value)
77 + if cls._is_data_image_url(value):
78 + prefix = value.split(",", 1)[0]
79 + return f"{prefix},<ephemeral-image-{index}>"
80 + return value
81 +
82 async def after_execution(self, response: Response, **kwargs):
83
84 # build image data messages for LLMs, or error message