Make browser screenshots ephemeral and context scoped
Route no-path browser screenshots through an in-process ephemeral image registry that vision_load consumes into the existing data-url model boundary. Stop materializing host-browser artifacts into tmp/browser/host-screenshots, keep explicit path screenshots durable, and make browser log metadata point at the active chat/task context while preserving browser-context detail.
Alessandro committed
May 22, 2026 at 09:50 UTC
430c48d1a5ae8cf0934426408c1d5b4083b1e6e2
11 files changed
+385
-107
helpers/ephemeral_images.py
new
+150
@@ -0,0 +1,150 @@
1
+from __future__ import annotations
2
+
3
+import base64
4
+import threading
5
+import time
6
+import uuid
7
+from dataclasses import dataclass
8
+
9
+
10
+REF_PREFIX = "a0-ephemeral-image://"
11
+DEFAULT_TTL_SECONDS = 15 * 60
12
+
13
+
14
+@dataclass(frozen=True)
15
+class EphemeralImage:
16
+ ref: str
17
+ context_id: str
18
+ mime: str
19
+ data: str
20
+ name: str
21
+ created_at: float
22
+ expires_at: float
23
+
24
+ @property
25
+ def data_url(self) -> str:
26
+ return f"data:{self.mime};base64,{self.data}"
27
+
28
+ @property
29
+ def display_name(self) -> str:
30
+ return self.name or display_ref(self.ref)
31
+
32
+
33
+_store: dict[str, EphemeralImage] = {}
34
+_lock = threading.RLock()
35
+
36
+
37
+def put_image_bytes(
38
+ *,
39
+ context_id: str,
40
+ mime: str,
41
+ payload: bytes,
42
+ name: str = "",
43
+ ttl_seconds: float = DEFAULT_TTL_SECONDS,
44
+) -> str:
45
+ data = base64.b64encode(bytes(payload or b"")).decode("ascii")
46
+ return put_image(
47
+ context_id=context_id,
48
+ mime=mime,
49
+ data=data,
50
+ name=name,
51
+ ttl_seconds=ttl_seconds,
52
+ )
53
+
54
+
55
+def put_image(
56
+ *,
57
+ context_id: str,
58
+ mime: str,
59
+ data: str,
60
+ name: str = "",
61
+ ttl_seconds: float = DEFAULT_TTL_SECONDS,
62
+) -> str:
63
+ compact_data = _compact_base64(data)
64
+ if not compact_data:
65
+ raise ValueError("ephemeral image data is empty")
66
+ base64.b64decode(compact_data, validate=True)
67
+
68
+ normalized_mime = _normalize_mime(mime)
69
+ now = time.time()
70
+ ref = f"{REF_PREFIX}{uuid.uuid4().hex}"
71
+ image = EphemeralImage(
72
+ ref=ref,
73
+ context_id=str(context_id or "").strip(),
74
+ mime=normalized_mime,
75
+ data=compact_data,
76
+ name=str(name or "").strip(),
77
+ created_at=now,
78
+ expires_at=now + max(1.0, float(ttl_seconds or DEFAULT_TTL_SECONDS)),
79
+ )
80
+ with _lock:
81
+ _prune_expired_locked(now)
82
+ _store[ref] = image
83
+ return ref
84
+
85
+
86
+def is_ref(value: object) -> bool:
87
+ return str(value or "").strip().startswith(REF_PREFIX)
88
+
89
+
90
+def display_ref(ref: str) -> str:
91
+ value = str(ref or "").strip()
92
+ if not is_ref(value):
93
+ return value
94
+ return f"{REF_PREFIX}<ephemeral>"
95
+
96
+
97
+def get_image(ref: str, *, context_id: str = "") -> EphemeralImage | None:
98
+ return _resolve_image(ref, context_id=context_id, consume=False)
99
+
100
+
101
+def consume_image(ref: str, *, context_id: str = "") -> EphemeralImage | None:
102
+ return _resolve_image(ref, context_id=context_id, consume=True)
103
+
104
+
105
+def delete_image(ref: str) -> None:
106
+ with _lock:
107
+ _store.pop(str(ref or "").strip(), None)
108
+
109
+
110
+def clear_context(context_id: str) -> None:
111
+ normalized_context = str(context_id or "").strip()
112
+ with _lock:
113
+ for ref, image in list(_store.items()):
114
+ if image.context_id == normalized_context:
115
+ _store.pop(ref, None)
116
+
117
+
118
+def _resolve_image(ref: str, *, context_id: str = "", consume: bool) -> EphemeralImage | None:
119
+ value = str(ref or "").strip()
120
+ if not is_ref(value):
121
+ return None
122
+
123
+ now = time.time()
124
+ with _lock:
125
+ _prune_expired_locked(now)
126
+ image = _store.get(value)
127
+ if image is None:
128
+ return None
129
+ requested_context = str(context_id or "").strip()
130
+ if requested_context and image.context_id and image.context_id != requested_context:
131
+ return None
132
+ if consume:
133
+ _store.pop(value, None)
134
+ return image
135
+
136
+
137
+def _compact_base64(data: str) -> str:
138
+ return "".join(char for char in str(data or "") if not char.isspace())
139
+
140
+
141
+def _normalize_mime(mime: str) -> str:
142
+ value = str(mime or "").strip().lower()
143
+ return value if value.startswith("image/") else "image/jpeg"
144
+
145
+
146
+def _prune_expired_locked(now: float | None = None) -> None:
147
+ current = time.time() if now is None else float(now)
148
+ for ref, image in list(_store.items()):
149
+ if image.expires_at <= current:
150
+ _store.pop(ref, None)
plugins/_browser/helpers/connector_runtime.py
+15
-31
@@ -1,7 +1,6 @@
1
from __future__ import annotations
2
3
import asyncio
4
-import base64
4
import hashlib
5
import re
6
import uuid
@@ -10,7 +9,7 @@ from pathlib import Path
9
from typing import Any
10
from urllib.parse import urlparse
11
13
-from helpers import files
12
+from helpers import ephemeral_images
13
14
try:
15
from helpers.ws import NAMESPACE
@@ -40,10 +39,8 @@ from plugins._browser.helpers.url import normalize_url
39
40
BROWSER_OP_EVENT = "connector_browser_op"
41
BROWSER_OP_TIMEOUT = 120.0
43
-HOST_BROWSER_SCREENSHOT_DIR = ("tmp", "browser", "host-screenshots")
42
CONTENT_HELPER_PATH = Path(__file__).resolve().parents[1] / "assets" / "browser-page-content.js"
43
MAX_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024
46
-BASE64_DECODE_CHARS_PER_CHUNK = 64 * 1024
44
HOST_BROWSER_PRIVACY_POLICY_KEY = getattr(
45
browser_config,
46
"HOST_BROWSER_PRIVACY_POLICY_KEY",
@@ -431,26 +428,30 @@ class ConnectorBrowserRuntime:
428
estimated_size = _estimated_base64_decoded_size(data)
429
if estimated_size > MAX_ARTIFACT_SIZE_BYTES:
430
raise RuntimeError(
434
- "Host browser artifact is too large to materialize safely "
431
+ "Host browser artifact is too large to attach safely "
432
f"({estimated_size} bytes, limit {MAX_ARTIFACT_SIZE_BYTES} bytes)."
433
)
434
filename = _safe_filename(str(artifact.get("filename") or "host-browser.jpg"))
438
- target_dir = Path(files.get_abs_path(*HOST_BROWSER_SCREENSHOT_DIR, self.context_id))
439
- target_dir.mkdir(parents=True, exist_ok=True)
440
- target_path = target_dir / filename
435
try:
442
- _write_base64_to_path(data, target_path)
436
+ ref = ephemeral_images.put_image(
437
+ context_id=self.context_id,
438
+ mime=str(artifact.get("mime") or result.get("mime") or "image/jpeg"),
439
+ data=data,
440
+ name=filename,
441
+ )
442
except Exception as exc:
444
- target_path.unlink(missing_ok=True)
443
raise RuntimeError("Host browser artifact could not be decoded.") from exc
444
materialized = dict(result)
445
materialized.pop("artifact", None)
448
- local_path = str(target_path)
449
- materialized["path"] = local_path
450
- materialized["a0_path"] = files.normalize_a0_path(local_path)
446
+ materialized.pop("path", None)
447
+ materialized.pop("a0_path", None)
448
+ materialized.pop("host_path", None)
449
+ materialized.setdefault("context_id", self.context_id)
450
+ materialized["ephemeral"] = True
451
+ materialized["ephemeral_ref"] = ref
452
materialized["vision_load"] = {
453
"tool_name": "vision_load",
453
- "tool_args": {"paths": [local_path]},
454
+ "tool_args": {"paths": [ref]},
455
}
456
return materialized
457
@@ -569,20 +570,3 @@ def _safe_filename(value: str) -> str:
570
def _estimated_base64_decoded_size(data: str) -> int:
571
compact_length = sum(1 for char in data if not char.isspace())
572
return (compact_length * 3) // 4
572
-
573
-
574
-def _write_base64_to_path(data: str, target_path: Path) -> None:
575
- pending = ""
576
- with target_path.open("wb") as target:
577
- for offset in range(0, len(data), BASE64_DECODE_CHARS_PER_CHUNK):
578
- chunk = pending + "".join(
579
- char
580
- for char in data[offset : offset + BASE64_DECODE_CHARS_PER_CHUNK]
581
- if not char.isspace()
582
- )
583
- ready_length = (len(chunk) // 4) * 4
584
- if ready_length:
585
- target.write(base64.b64decode(chunk[:ready_length], validate=True))
586
- pending = chunk[ready_length:]
587
- if pending:
588
- target.write(base64.b64decode(pending, validate=True))
plugins/_browser/helpers/runtime.py
+30
-1
@@ -15,7 +15,7 @@ from dataclasses import dataclass
15
from pathlib import Path
16
from typing import Any
17
18
-from helpers import files
18
+from helpers import ephemeral_images, files
19
from helpers.defer import DeferredTask
20
from helpers.errors import RepairableException
21
from helpers.print_style import PrintStyle
@@ -1548,6 +1548,34 @@ class _BrowserRuntimeCore:
1548
await self.ensure_started()
1549
resolved_id = self._resolve_browser_id(browser_id)
1550
page = self._page(resolved_id)
1551
+ raw_path = str(path or "").strip()
1552
+ if not raw_path:
1553
+ image = await page.screenshot(
1554
+ type="jpeg",
1555
+ quality=max(20, min(95, int(quality))),
1556
+ full_page=bool(full_page),
1557
+ )
1558
+ ref = ephemeral_images.put_image_bytes(
1559
+ context_id=self.context_id,
1560
+ mime="image/jpeg",
1561
+ payload=image,
1562
+ name=f"browser-{resolved_id}.jpg",
1563
+ )
1564
+ return {
1565
+ "browser_id": resolved_id,
1566
+ "context_id": self.context_id,
1567
+ "mime": "image/jpeg",
1568
+ "ephemeral": True,
1569
+ "ephemeral_ref": ref,
1570
+ "state": await self._state(resolved_id),
1571
+ "vision_load": {
1572
+ "tool_name": "vision_load",
1573
+ "tool_args": {
1574
+ "paths": [ref],
1575
+ },
1576
+ },
1577
+ }
1578
+
1579
output_path, image_type, mime = self._screenshot_output_path(resolved_id, path)
1580
output_path.parent.mkdir(parents=True, exist_ok=True)
1581
clamped_quality = max(20, min(95, int(quality)))
@@ -1562,6 +1590,7 @@ class _BrowserRuntimeCore:
1590
local_path = str(output_path)
1591
return {
1592
"browser_id": resolved_id,
1593
+ "context_id": self.context_id,
1594
"path": local_path,
1595
"a0_path": files.normalize_a0_path(local_path),
1596
"mime": mime,
plugins/_browser/prompts/agent.system.tool.browser.md
+1
-1
@@ -20,7 +20,7 @@ Workflow:
20
- For same-page controls that are easier to identify structurally, `click`, `type`, `submit`, `type_submit`, `scroll`, `select_option`, `set_checked`, and `upload_file` may use `selector` instead of `ref`; the tool resolves the selector through `content` first.
21
- `click` with `x`/`y` and no `ref` is treated as a coordinate mouse click. `type` with text and no `ref` types into the currently focused element. `key_chord` accepts either `["Control", "A"]` or `"CTRL+A"`.
22
- `navigate` reuses an existing `browser_id` and is preferred for serial browsing.
23
-- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned path before reasoning visually.
23
+- Screenshots are explicit only; the browser does not automatically load screenshots. Call `vision_load` with the returned `vision_load.tool_args.paths` value before reasoning visually. When no `path` is requested, browser screenshots are ephemeral refs rather than conserved files.
24
- Keep the tab set small; close pages after extracting what you need.
25
26
`multi` is only a browser action: use `tool_name: "browser"` with `tool_args.action: "multi"`. Never use `tool_name: "multi"`.
plugins/_browser/skills/browser-automation/SKILL.md
+2
-2
@@ -29,10 +29,10 @@ In host mode, page content and screenshots may be blocked by host-content policy
29
Screenshots are explicit only; the browser does not automatically load images into model context.
30
31
1. Call `browser` with `action: "screenshot"`.
32
-2. Call `vision_load` with the returned path.
32
+2. Call `vision_load` with the returned `vision_load.tool_args.paths` value.
33
3. Reason from the latest loaded screenshot.
34
35
-Screenshot args include `quality`, `full_page`, and optional `path`. PNG is used when `path` ends with `.png`; otherwise JPEG is used.
35
+Screenshot args include `quality`, `full_page`, and optional `path`. Without `path`, the screenshot is an ephemeral ref consumed by `vision_load`; with `path`, PNG is used when `path` ends with `.png`, otherwise JPEG is used.
36
37
## Forms And Files
38
plugins/_browser/skills/browser-form-workflows/SKILL.md
+1
-1
@@ -11,7 +11,7 @@ Start with `browser:content` to capture current refs, then use `browser:detail`
11
12
Use `select_option`, `set_checked`, `upload_file`, `type`, `type_submit`, and `submit` for form interaction. Use coordinates only when no stable ref exists or the UI is intentionally canvas-like.
13
14
-Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history.
14
+Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history; no-path screenshots return ephemeral refs for `vision_load`.
15
16
Verify after submission with `browser:content`, `browser:state`, or another explicit `browser:screenshot` plus `vision_load`.
17
plugins/_browser/tools/browser.py
+60
-25
@@ -89,8 +89,6 @@ class Browser(Tool):
89
if action == "open":
90
result = await runtime.call("open", url or "")
91
elif action == "screenshot":
92
- if not path:
93
- path = self._history_screenshot_path(action)
92
result = await runtime.call(
93
"screenshot_file",
94
browser_id,
@@ -428,53 +426,65 @@ class Browser(Tool):
426
return
427
428
screenshot = result if action == "screenshot" and isinstance(result, dict) else None
431
- if not self._screenshot_has_path(screenshot):
429
+ if not self._screenshot_has_reference(screenshot):
430
target_browser_id = self._browser_id_from_result(result) or requested_browser_id
433
- output_path = self._history_screenshot_path(action)
434
- if not output_path:
435
- return
431
try:
432
screenshot = await runtime.call(
433
"screenshot_file",
434
target_browser_id,
435
quality=HISTORY_SCREENSHOT_QUALITY,
436
full_page=False,
442
- path=output_path,
437
+ path="",
438
)
439
except Exception as exc:
440
PrintStyle.debug(
441
"Browser history screenshot capture failed:",
442
f"browser_id={target_browser_id}",
443
f"quality={HISTORY_SCREENSHOT_QUALITY}",
449
- f"path={output_path}",
444
f"error={exc}",
445
)
446
return
447
454
- if not self._screenshot_has_path(screenshot):
448
+ if not self._screenshot_has_reference(screenshot):
449
return
450
457
- local_path = str(screenshot.get("path") or files.fix_dev_path(str(screenshot.get("a0_path") or "")))
458
- if not local_path:
459
- return
460
- uri = f"img://{local_path}&t={time.time()}"
451
+ a0_path = str(screenshot.get("a0_path") or "").strip()
452
+ local_path = str(screenshot.get("path") or (files.fix_dev_path(a0_path) if a0_path else ""))
453
state = screenshot.get("state") if isinstance(screenshot.get("state"), dict) else {}
462
- self.log.update(
463
- Screenshot=uri,
464
- browser_snapshot={
465
- "uri": uri,
466
- "path": local_path,
467
- "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),
468
- "mime": screenshot.get("mime") or "image/jpeg",
469
- "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,
470
- "context_id": screenshot.get("context_id") or state.get("context_id") or "",
471
- },
472
- )
454
+ chat_context_id = self._agent_context_id()
455
+ browser_context_id = str(screenshot.get("context_id") or state.get("context_id") or "").strip()
456
+ snapshot = {
457
+ "mime": screenshot.get("mime") or "image/jpeg",
458
+ "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,
459
+ "context_id": chat_context_id or browser_context_id,
460
+ "browser_context_id": browser_context_id,
461
+ }
462
+ update_payload: dict[str, Any] = {"browser_snapshot": snapshot}
463
+ if local_path:
464
+ uri = f"img://{local_path}&t={time.time()}"
465
+ snapshot.update(
466
+ {
467
+ "uri": uri,
468
+ "path": local_path,
469
+ "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),
470
+ "ephemeral": False,
471
+ }
472
+ )
473
+ update_payload["Screenshot"] = uri
474
+ else:
475
+ ephemeral_ref = self._screenshot_ephemeral_ref(screenshot)
476
+ snapshot.update(
477
+ {
478
+ "ephemeral": bool(ephemeral_ref),
479
+ "ephemeral_ref": ephemeral_ref,
480
+ }
481
+ )
482
+ self.log.update(**update_payload)
483
484
def _history_screenshot_path(self, action: str) -> str:
485
if not getattr(self, "agent", None) or not getattr(self.agent, "context", None):
486
return ""
477
- context_id = str(getattr(self.agent.context, "id", "") or "").strip()
487
+ context_id = self._agent_context_id()
488
if not context_id:
489
return ""
490
from helpers import persist_chat
@@ -521,6 +531,31 @@ class Browser(Tool):
531
def _screenshot_has_path(screenshot: Any) -> bool:
532
return isinstance(screenshot, dict) and bool(screenshot.get("path") or screenshot.get("a0_path"))
533
534
+ @classmethod
535
+ def _screenshot_has_reference(cls, screenshot: Any) -> bool:
536
+ return cls._screenshot_has_path(screenshot) or bool(cls._screenshot_ephemeral_ref(screenshot))
537
+
538
+ @staticmethod
539
+ def _screenshot_ephemeral_ref(screenshot: Any) -> str:
540
+ if not isinstance(screenshot, dict):
541
+ return ""
542
+ ref = str(screenshot.get("ephemeral_ref") or "").strip()
543
+ if ref:
544
+ return ref
545
+ vision_load = screenshot.get("vision_load")
546
+ if isinstance(vision_load, dict):
547
+ tool_args = vision_load.get("tool_args")
548
+ if isinstance(tool_args, dict):
549
+ paths = tool_args.get("paths")
550
+ if isinstance(paths, list) and paths:
551
+ first = str(paths[0] or "").strip()
552
+ if first.startswith("a0-ephemeral-image://"):
553
+ return first
554
+ return ""
555
+
556
+ def _agent_context_id(self) -> str:
557
+ return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()
558
+
559
@staticmethod
560
def _format_result(action: str, result: Any) -> str:
561
if action == "content" and isinstance(result, dict):
prompts/agent.system.tools_vision.md
+1
-1
@@ -2,7 +2,7 @@
2
3
### vision_load
4
load images into the model for visual reasoning
5
-args: `paths` list of absolute image paths
5
+args: `paths` list of absolute image paths or tool-returned ephemeral image refs
6
rules:
7
- load all relevant images in one call when comparing screenshots or pages
8
- use when the task depends on screenshots, diagrams, scanned documents, charts, or photos
tests/test_browser_agent_regressions.py
+67
-18
@@ -79,6 +79,7 @@ sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_conf
79
def anyio_backend():
80
return "asyncio"
81
82
+from helpers import ephemeral_images
83
from helpers.errors import RepairableException
84
from plugins._browser.helpers.config import (
85
build_browser_launch_config,
@@ -1808,12 +1809,10 @@ async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_p
1809
},
1810
}
1811
if method == "screenshot_file":
1811
- Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
1812
- Path(kwargs["path"]).write_bytes(b"jpeg")
1812
return {
1813
"browser_id": args[0],
1815
- "path": kwargs["path"],
1816
- "a0_path": "/a0/usr/chats/chat/browser/screenshots/open.jpg",
1814
+ "ephemeral": True,
1815
+ "ephemeral_ref": "a0-ephemeral-image://fake",
1816
"mime": "image/jpeg",
1817
"state": {"id": args[0], "context_id": "browser-context"},
1818
}
@@ -1861,11 +1860,13 @@ async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_p
1860
assert calls[1][1] == (1,)
1861
assert calls[1][2]["quality"] == browser_tool_module.HISTORY_SCREENSHOT_QUALITY
1862
assert calls[1][2]["full_page"] is False
1864
- assert Path(calls[1][2]["path"]).parent == tmp_path / "usr" / "chats" / "chat" / "browser" / "screenshots"
1865
- assert Path(calls[1][2]["path"]).read_bytes() == b"jpeg"
1866
- assert log.updates[-1]["Screenshot"].startswith("img://")
1863
+ assert calls[1][2]["path"] == ""
1864
+ assert "Screenshot" not in log.updates[-1]
1865
assert log.updates[-1]["browser_snapshot"]["browser_id"] == 1
1868
- assert log.updates[-1]["browser_snapshot"]["context_id"] == "browser-context"
1866
+ assert log.updates[-1]["browser_snapshot"]["context_id"] == "chat"
1867
+ assert log.updates[-1]["browser_snapshot"]["browser_context_id"] == "browser-context"
1868
+ assert log.updates[-1]["browser_snapshot"]["ephemeral"] is True
1869
+ assert log.updates[-1]["browser_snapshot"]["ephemeral_ref"] == "a0-ephemeral-image://fake"
1870
1871
1872
@pytest.mark.anyio
@@ -2393,7 +2394,7 @@ async def test_browser_runtime_remounts_initial_changed_viewport():
2394
2395
2396
@pytest.mark.anyio
2396
-async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch, tmp_path):
2397
+async def test_browser_runtime_screenshot_file_defaults_to_ephemeral_ref(monkeypatch, tmp_path):
2398
screenshot_calls = []
2399
2400
def fake_get_abs_path(*parts):
@@ -2411,8 +2412,9 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch
2412
2413
async def screenshot(self, **kwargs):
2414
screenshot_calls.append(kwargs)
2414
- Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
2415
- Path(kwargs["path"]).write_bytes(b"image-bytes")
2415
+ if kwargs.get("path"):
2416
+ Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
2417
+ Path(kwargs["path"]).write_bytes(b"image-bytes")
2418
return b"image-bytes"
2419
2420
async def title(self):
@@ -2427,21 +2429,23 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch
2429
2430
result = await core.screenshot_file(5, quality=500)
2431
2430
- path = Path(result["path"])
2431
- assert path.exists()
2432
- assert path.parent == tmp_path / "tmp" / "browser" / "screenshots" / "ctx_id"
2433
- assert path.name.startswith("browser-5-")
2434
- assert path.suffix == ".jpg"
2435
- assert result["a0_path"].startswith("/a0/tmp/browser/screenshots/ctx_id/browser-5-")
2432
+ assert "path" not in result
2433
+ assert "a0_path" not in result
2434
+ assert result["context_id"] == "ctx/id"
2435
assert result["mime"] == "image/jpeg"
2436
+ assert result["ephemeral"] is True
2437
+ assert result["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)
2438
assert result["vision_load"] == {
2439
"tool_name": "vision_load",
2439
- "tool_args": {"paths": [result["path"]]},
2440
+ "tool_args": {"paths": [result["ephemeral_ref"]]},
2441
}
2442
assert "image" not in result
2443
+ assert not list((tmp_path / "tmp" / "browser" / "screenshots").rglob("*.jpg"))
2444
assert screenshot_calls[-1]["type"] == "jpeg"
2445
assert screenshot_calls[-1]["quality"] == 95
2446
assert screenshot_calls[-1]["full_page"] is False
2447
+ assert "path" not in screenshot_calls[-1]
2448
+ assert ephemeral_images.consume_image(result["ephemeral_ref"], context_id="ctx/id").data_url == "data:image/jpeg;base64,aW1hZ2UtYnl0ZXM="
2449
2450
png_path = tmp_path / "custom.png"
2451
png_result = await core.screenshot_file(5, quality=1, full_page=True, path=str(png_path))
@@ -2455,6 +2459,51 @@ async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch
2459
}
2460
2461
2462
+@pytest.mark.anyio
2463
+async def test_vision_load_consumes_ephemeral_browser_refs(monkeypatch):
2464
+ import tools.vision_load as vision_load_module
2465
+
2466
+ monkeypatch.setattr(
2467
+ vision_load_module.plugins,
2468
+ "get_plugin_config",
2469
+ lambda *args, **kwargs: {"chat_model": {"max_embeds": 10}},
2470
+ )
2471
+
2472
+ tool_results = []
2473
+ messages = []
2474
+ updates = []
2475
+ agent = SimpleNamespace(
2476
+ context=SimpleNamespace(id="ctx-vision"),
2477
+ agent_name="Agent 0",
2478
+ hist_add_tool_result=lambda *args, **kwargs: tool_results.append((args, kwargs)),
2479
+ hist_add_message=lambda *args, **kwargs: messages.append((args, kwargs)),
2480
+ )
2481
+ ref = ephemeral_images.put_image(
2482
+ context_id="ctx-vision",
2483
+ mime="image/jpeg",
2484
+ data=SMALL_JPEG_10X10,
2485
+ name="browser-shot.jpg",
2486
+ )
2487
+ tool = vision_load_module.VisionLoad(
2488
+ agent=agent,
2489
+ name="vision_load",
2490
+ method=None,
2491
+ args={"paths": [ref]},
2492
+ message="",
2493
+ loop_data=None,
2494
+ )
2495
+ tool.log = SimpleNamespace(id="vision-log", update=lambda **kwargs: updates.append(kwargs))
2496
+
2497
+ response = await tool.execute(paths=[ref])
2498
+ await tool.after_execution(response)
2499
+
2500
+ assert ephemeral_images.get_image(ref, context_id="ctx-vision") is None
2501
+ assert tool.loaded_paths == ["browser-shot.jpg"]
2502
+ raw_message = messages[0][1]["content"]
2503
+ assert raw_message.raw_content[0]["image_url"]["url"] == f"data:image/jpeg;base64,{SMALL_JPEG_10X10}"
2504
+ assert updates[-1]["result"] == "1 images loaded, 0 skipped"
2505
+
2506
+
2507
@pytest.mark.anyio
2508
async def test_browser_runtime_ref_point_resolution_applies_offsets():
2509
eval_payloads = []
tests/test_host_browser_connector.py
+10
-21
@@ -12,6 +12,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
12
if str(PROJECT_ROOT) not in sys.path:
13
sys.path.insert(0, str(PROJECT_ROOT))
14
15
+from helpers import ephemeral_images
16
from plugins._a0_connector.helpers import ws_runtime
17
from plugins._browser.helpers.connector_runtime import (
18
ConnectorBrowserRuntime,
@@ -329,19 +330,7 @@ def test_connector_runtime_adds_docker_recovery_to_host_errors():
330
assert "/browser container" in message
331
332
332
-def test_host_browser_artifacts_materialize_inside_multi_results(monkeypatch, tmp_path):
333
- import plugins._browser.helpers.connector_runtime as connector_runtime_module
334
-
335
- monkeypatch.setattr(
336
- connector_runtime_module.files,
337
- "get_abs_path",
338
- lambda *parts: str(tmp_path.joinpath(*parts)),
339
- )
340
- monkeypatch.setattr(
341
- connector_runtime_module.files,
342
- "normalize_a0_path",
343
- lambda path: "/a0/" + str(path).lstrip("/"),
344
- )
333
+def test_host_browser_artifacts_become_context_scoped_ephemeral_refs(tmp_path):
334
runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))
335
336
result = runtime._materialize_artifact(
@@ -363,19 +352,19 @@ def test_host_browser_artifacts_materialize_inside_multi_results(monkeypatch, tm
352
353
inner = result[0]["result"]
354
assert "artifact" not in inner
366
- assert inner["path"].endswith("shot.jpg")
367
- assert Path(inner["path"]).read_bytes() == b"fake"
368
- assert inner["vision_load"]["tool_args"]["paths"] == [inner["path"]]
355
+ assert "path" not in inner
356
+ assert "a0_path" not in inner
357
+ assert inner["context_id"] == "ctx-host"
358
+ assert inner["ephemeral"] is True
359
+ assert inner["ephemeral_ref"].startswith(ephemeral_images.REF_PREFIX)
360
+ assert inner["vision_load"]["tool_args"]["paths"] == [inner["ephemeral_ref"]]
361
+ assert ephemeral_images.consume_image(inner["ephemeral_ref"], context_id="ctx-host").data_url == "data:image/jpeg;base64,ZmFrZQ=="
362
+ assert not list(tmp_path.rglob("shot.jpg"))
363
364
365
def test_host_browser_artifact_materialization_rejects_oversized_payload(monkeypatch, tmp_path):
366
import plugins._browser.helpers.connector_runtime as connector_runtime_module
367
374
- monkeypatch.setattr(
375
- connector_runtime_module.files,
376
- "get_abs_path",
377
- lambda *parts: str(tmp_path.joinpath(*parts)),
378
- )
368
monkeypatch.setattr(connector_runtime_module, "MAX_ARTIFACT_SIZE_BYTES", 2)
369
runtime = ConnectorBrowserRuntime("ctx-host", _agent("ctx-host"))
370
tools/vision_load.py
+48
-6
@@ -1,6 +1,6 @@
1
from helpers.print_style import PrintStyle
2
from helpers.tool import Tool, Response
3
-from helpers import runtime, files, plugins
3
+from helpers import runtime, files, plugins, ephemeral_images
4
from mimetypes import guess_type
5
from helpers import history
6
@@ -16,18 +16,43 @@ class VisionLoad(Tool):
16
self.skipped_paths: list[str] = []
17
18
max_embeds = self._get_max_embeds()
19
- limited_paths = paths if max_embeds <= 0 else paths[-max_embeds:]
20
- self.skipped_paths = paths[:-max_embeds] if max_embeds > 0 and len(paths) > max_embeds else []
19
+ requested = [
20
+ (str(path or "").strip(), self._display_input_path(str(path or "").strip(), idx + 1))
21
+ for idx, path in enumerate(paths)
22
+ ]
23
+ limited_paths = requested if max_embeds <= 0 else requested[-max_embeds:]
24
+ self.skipped_paths = (
25
+ [display for _, display in requested[:-max_embeds]]
26
+ if max_embeds > 0 and len(requested) > max_embeds
27
+ else []
28
+ )
29
22
- for path in limited_paths:
30
+ for path, display_path in limited_paths:
31
+ if not path:
32
+ continue
33
+ if ephemeral_images.is_ref(path):
34
+ image = ephemeral_images.consume_image(
35
+ path,
36
+ context_id=self._context_id(),
37
+ )
38
+ if image is None:
39
+ continue
40
+ display = image.display_name or display_path
41
+ self.images_dict[display] = image.data_url
42
+ self.loaded_paths.append(display)
43
+ continue
44
+ if self._is_data_image_url(path):
45
+ self.images_dict[display_path] = path
46
+ self.loaded_paths.append(display_path)
47
+ continue
48
if not await runtime.call_development_function(files.exists, str(path)):
49
continue
50
51
if path not in self.images_dict:
52
mime_type, _ = guess_type(str(path))
53
if mime_type and mime_type.startswith("image/"):
29
- self.images_dict[path] = str(path)
30
- self.loaded_paths.append(path)
54
+ self.images_dict[display_path] = str(path)
55
+ self.loaded_paths.append(display_path)
56
57
return Response(message="dummy", break_loop=False)
58
@@ -37,6 +62,23 @@ class VisionLoad(Tool):
62
max_embeds = chat_cfg.get("max_embeds", 10)
63
return int(max_embeds or 0)
64
65
+ def _context_id(self) -> str:
66
+ return str(getattr(getattr(self.agent, "context", None), "id", "") or "").strip()
67
+
68
+ @staticmethod
69
+ def _is_data_image_url(value: str) -> bool:
70
+ normalized = str(value or "").strip().lower()
71
+ return normalized.startswith("data:image/") and ";base64," in normalized
72
+
73
+ @classmethod
74
+ def _display_input_path(cls, value: str, index: int) -> str:
75
+ if ephemeral_images.is_ref(value):
76
+ return ephemeral_images.display_ref(value)
77
+ if cls._is_data_image_url(value):
78
+ prefix = value.split(",", 1)[0]
79
+ return f"{prefix},<ephemeral-image-{index}>"
80
+ return value
81
+
82
async def after_execution(self, response: Response, **kwargs):
83
84
# build image data messages for LLMs, or error message