Persist browser history screenshots
Save a static JPEG for each Browser tool call in the chat history folder and render that immutable image in transcript screenshot previews. Keep live Browser surface attachment available through stored browser/context metadata, and ignore generated Playwright CLI artifacts.
Alessandro committed
May 8, 2026 at 19:24 UTC
44d5e1ccf7245b755c091e363661e8bcf196ab18
4 files changed
+243
-6
.gitignore
+1
@@ -50,3 +50,4 @@ agent_history.gif
50
51
.agent/**
52
.claude/**
53
+.playwright-cli/
plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
+39
-6
@@ -14,6 +14,7 @@ import {
14
} from "/js/messages.js";
15
16
const BROWSER_SCREENSHOT_KVP_KEY = "Screenshot";
17
+const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";
18
const BROWSER_SCREENSHOT_STYLE_ID = "a0-browser-screenshot-kvp-style";
19
const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;
20
const PREVIEW_REFRESH_MS = 2500;
@@ -62,7 +63,14 @@ function normalizeBrowserAction(kvps = {}) {
63
return String(kvps.action || "").trim().toLowerCase().replace("-", "_");
64
}
65
66
+function browserSnapshotMeta(kvps = {}) {
67
+ return kvps?.[BROWSER_SNAPSHOT_META_KEY] && typeof kvps[BROWSER_SNAPSHOT_META_KEY] === "object"
68
+ ? kvps[BROWSER_SNAPSHOT_META_KEY]
69
+ : {};
70
+}
71
+
72
function browserIdFromResult(result = {}, kvps = {}) {
73
+ const snapshotMeta = browserSnapshotMeta(kvps);
74
const browsers = Array.isArray(result.browsers) ? result.browsers : [];
75
const lastInteractedId = result.last_interacted_browser_id;
76
const listedBrowser = lastInteractedId
@@ -74,12 +82,14 @@ function browserIdFromResult(result = {}, kvps = {}) {
82
|| result.state?.id
83
|| result.last_interacted_browser_id
84
|| listedBrowser?.id
85
+ || snapshotMeta.browser_id
86
|| kvps.browser_id
87
|| null
88
);
89
}
90
91
function browserContextIdFromResult(result = {}, kvps = {}) {
92
+ const snapshotMeta = browserSnapshotMeta(kvps);
93
const browserId = browserIdFromResult(result, kvps);
94
const browsers = Array.isArray(result.browsers) ? result.browsers : [];
95
const listedBrowser = browserId
@@ -89,6 +99,7 @@ function browserContextIdFromResult(result = {}, kvps = {}) {
99
result.context_id
100
|| result.state?.context_id
101
|| listedBrowser?.context_id
102
+ || snapshotMeta.context_id
103
|| kvps.context_id
104
|| kvps.contextId
105
|| null
@@ -185,6 +196,16 @@ function snapshotToObjectUrl(snapshot) {
196
return URL.createObjectURL(new Blob(chunks, { type: snapshot.mime || "image/jpeg" }));
197
}
198
199
+function staticScreenshotUri(kvps = {}) {
200
+ const metaUri = browserSnapshotMeta(kvps).uri;
201
+ const value = kvps?.[BROWSER_SCREENSHOT_KVP_KEY] || metaUri || "";
202
+ return typeof value === "string" && value.startsWith("img://") ? value : "";
203
+}
204
+
205
+function staticScreenshotSrc(uri = "") {
206
+ return uri ? uri.replace("img://", "/api/image_get?path=") : "";
207
+}
208
+
209
function releaseLiveScreenshotFrame(viewerId) {
210
const release = liveScreenshotFrames.get(viewerId);
211
if (!release) return;
@@ -299,7 +320,7 @@ function ensureBrowserScreenshotStyles() {
320
document.head.appendChild(style);
321
}
322
302
-function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label) {
323
+function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label, staticUri = "") {
324
if (!kvpsTable || typeof resolveBrowserPayload !== "function") return;
325
ensureBrowserScreenshotStyles();
326
@@ -334,11 +355,17 @@ function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label) {
355
356
cell.textContent = "";
357
cell.appendChild(button);
337
- cell.__browserScreenshotCleanup = startBrowserScreenshotPreview(
338
- button,
339
- button.querySelector(".browser-screenshot-kvp-image"),
340
- resolveBrowserPayload,
341
- );
358
+ const image = button.querySelector(".browser-screenshot-kvp-image");
359
+ if (staticUri) {
360
+ image.src = staticScreenshotSrc(staticUri);
361
+ button.classList.add("has-frame");
362
+ cell.__browserScreenshotCleanup = () => {
363
+ image.removeAttribute("src");
364
+ button.classList.remove("has-frame");
365
+ };
366
+ return;
367
+ }
368
+ cell.__browserScreenshotCleanup = startBrowserScreenshotPreview(button, image, resolveBrowserPayload);
369
}
370
371
function startBrowserScreenshotPreview(button, image, resolveBrowserPayload) {
@@ -447,6 +474,7 @@ function drawBrowserTool({
474
].filter(Boolean);
475
const contentText = String(content ?? "");
476
const browserResult = parseBrowserResult(contentText);
477
+ const screenshotUri = staticScreenshotUri(kvps);
478
const browserId = browserIdFromResult(browserResult, kvps);
479
const browserCanvasPayload = buildBrowserCanvasPayload(browserResult, kvps);
480
const browserPreviewLabel = browserId
@@ -455,6 +483,10 @@ function drawBrowserTool({
483
if (shouldRenderBrowserScreenshotKvp(browserResult, kvps)) {
484
displayKvps[BROWSER_SCREENSHOT_KVP_KEY] = "";
485
}
486
+ if (screenshotUri) {
487
+ displayKvps[BROWSER_SCREENSHOT_KVP_KEY] = screenshotUri;
488
+ }
489
+ delete displayKvps[BROWSER_SNAPSHOT_META_KEY];
490
const browserButton = createActionButton(
491
"visibility",
492
"Browser",
@@ -503,6 +535,7 @@ function drawBrowserTool({
535
|| buildBrowserCanvasPayload({}, kvps)
536
),
537
browserPreviewLabel,
538
+ screenshotUri,
539
);
540
syncOpenBrowserCanvas(args, browserResult);
541
return result;
plugins/_browser/tools/browser.py
+118
@@ -1,12 +1,21 @@
1
from __future__ import annotations
2
3
import json
4
+import time
5
+import uuid
6
+from pathlib import Path
7
from typing import Any
8
9
+from helpers import files
10
+from helpers.print_style import PrintStyle
11
from helpers.tool import Response, Tool
12
from plugins._browser.helpers.selector import get_tool_runtime
13
14
15
+HISTORY_SCREENSHOT_QUALITY = 62
16
+HISTORY_SCREENSHOT_ACTION_DENYLIST = {"close", "close_all"}
17
+
18
+
19
async def get_runtime(context_id: str, create: bool = True, agent: Any | None = None):
20
if agent is not None:
21
return await get_tool_runtime(agent)
@@ -78,6 +87,8 @@ class Browser(Tool):
87
if action == "open":
88
result = await runtime.call("open", url or "")
89
elif action == "screenshot":
90
+ if not path:
91
+ path = self._history_screenshot_path(action)
92
result = await runtime.call(
93
"screenshot_file",
94
browser_id,
@@ -258,6 +269,7 @@ class Browser(Tool):
269
message=f"Unknown browser action: {action}",
270
break_loop=False,
271
)
272
+ await self._record_history_screenshot(runtime, action, result, browser_id)
273
except Exception as exc:
274
return Response(message=f"Browser {action} failed: {exc}", break_loop=False)
275
@@ -286,6 +298,112 @@ class Browser(Tool):
298
return {"selector": selector}
299
return None
300
301
+ async def _record_history_screenshot(
302
+ self,
303
+ runtime: Any,
304
+ action: str,
305
+ result: Any,
306
+ requested_browser_id: int | str | None = None,
307
+ ) -> None:
308
+ if not getattr(self, "log", None):
309
+ return
310
+ if action in HISTORY_SCREENSHOT_ACTION_DENYLIST:
311
+ return
312
+
313
+ screenshot = result if action == "screenshot" and isinstance(result, dict) else None
314
+ if not self._screenshot_has_path(screenshot):
315
+ target_browser_id = self._browser_id_from_result(result) or requested_browser_id
316
+ output_path = self._history_screenshot_path(action)
317
+ if not output_path:
318
+ return
319
+ try:
320
+ screenshot = await runtime.call(
321
+ "screenshot_file",
322
+ target_browser_id,
323
+ quality=HISTORY_SCREENSHOT_QUALITY,
324
+ full_page=False,
325
+ path=output_path,
326
+ )
327
+ except Exception as exc:
328
+ PrintStyle.debug(
329
+ "Browser history screenshot capture failed:",
330
+ f"browser_id={target_browser_id}",
331
+ f"quality={HISTORY_SCREENSHOT_QUALITY}",
332
+ f"path={output_path}",
333
+ f"error={exc}",
334
+ )
335
+ return
336
+
337
+ if not self._screenshot_has_path(screenshot):
338
+ return
339
+
340
+ local_path = str(screenshot.get("path") or files.fix_dev_path(str(screenshot.get("a0_path") or "")))
341
+ if not local_path:
342
+ return
343
+ uri = f"img://{local_path}&t={time.time()}"
344
+ state = screenshot.get("state") if isinstance(screenshot.get("state"), dict) else {}
345
+ self.log.update(
346
+ Screenshot=uri,
347
+ browser_snapshot={
348
+ "uri": uri,
349
+ "path": local_path,
350
+ "a0_path": screenshot.get("a0_path") or files.normalize_a0_path(local_path),
351
+ "mime": screenshot.get("mime") or "image/jpeg",
352
+ "browser_id": screenshot.get("browser_id") or state.get("id") or requested_browser_id,
353
+ "context_id": screenshot.get("context_id") or state.get("context_id") or "",
354
+ },
355
+ )
356
+
357
+ def _history_screenshot_path(self, action: str) -> str:
358
+ if not getattr(self, "agent", None) or not getattr(self.agent, "context", None):
359
+ return ""
360
+ context_id = str(getattr(self.agent.context, "id", "") or "").strip()
361
+ if not context_id:
362
+ return ""
363
+ from helpers import persist_chat
364
+
365
+ token = str(getattr(getattr(self, "log", None), "id", "") or uuid.uuid4())
366
+ safe_action = files.safe_file_name(str(action or "browser"))
367
+ safe_token = files.safe_file_name(token)
368
+ timestamp = time.strftime("%Y%m%d-%H%M%S")
369
+ return str(
370
+ Path(persist_chat.get_chat_folder_path(context_id))
371
+ / "browser"
372
+ / "screenshots"
373
+ / f"{timestamp}-{safe_action}-{safe_token}.jpg"
374
+ )
375
+
376
+ @staticmethod
377
+ def _browser_id_from_result(result: Any) -> Any:
378
+ if not isinstance(result, dict):
379
+ return None
380
+ browsers = result.get("browsers") if isinstance(result.get("browsers"), list) else []
381
+ last_interacted_id = result.get("last_interacted_browser_id")
382
+ listed_browser = None
383
+ if last_interacted_id is not None:
384
+ listed_browser = next(
385
+ (
386
+ browser
387
+ for browser in browsers
388
+ if isinstance(browser, dict) and str(browser.get("id")) == str(last_interacted_id)
389
+ ),
390
+ None,
391
+ )
392
+ if listed_browser is None and browsers:
393
+ listed_browser = next((browser for browser in browsers if isinstance(browser, dict)), None)
394
+ state = result.get("state") if isinstance(result.get("state"), dict) else {}
395
+ return (
396
+ result.get("id")
397
+ or result.get("browser_id")
398
+ or state.get("id")
399
+ or last_interacted_id
400
+ or (listed_browser or {}).get("id")
401
+ )
402
+
403
+ @staticmethod
404
+ def _screenshot_has_path(screenshot: Any) -> bool:
405
+ return isinstance(screenshot, dict) and bool(screenshot.get("path") or screenshot.get("a0_path"))
406
+
407
@staticmethod
408
def _format_result(action: str, result: Any) -> str:
409
if action == "content" and isinstance(result, dict):
tests/test_browser_agent_regressions.py
+85
@@ -1057,6 +1057,9 @@ def test_browser_viewer_uses_cdp_screencast_transport():
1057
browser_store = (
1058
PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
1059
).read_text(encoding="utf-8")
1060
+ browser_tool_handler = (
1061
+ PROJECT_ROOT / "plugins" / "_browser" / "extensions" / "webui" / "get_tool_message_handler" / "browser-tool-handler.js"
1062
+ ).read_text(encoding="utf-8")
1063
1064
assert 'runtime.call("screenshot"' in ws_browser
1065
assert "SCREENCAST_QUALITY = 92" in ws_browser
@@ -1096,6 +1099,10 @@ def test_browser_viewer_uses_cdp_screencast_transport():
1099
assert "this.applySnapshot(data.snapshot);" in browser_store
1100
assert "else if (!data.state)" in browser_store
1101
assert '"snapshot": snapshot' in ws_browser
1102
+ assert 'const BROWSER_SNAPSHOT_META_KEY = "browser_snapshot";' in browser_tool_handler
1103
+ assert "staticScreenshotUri(kvps)" in browser_tool_handler
1104
+ assert "delete displayKvps[BROWSER_SNAPSHOT_META_KEY];" in browser_tool_handler
1105
+ assert "startBrowserScreenshotPreview(button, image, resolveBrowserPayload)" in browser_tool_handler
1106
assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
1107
assert '"frame_source": "state"' in ws_browser
1108
assert '"frame_source"] = "screencast"' in ws_browser
@@ -1667,6 +1674,84 @@ async def test_browser_tool_dispatches_v1_agent_actions(monkeypatch):
1674
]
1675
1676
1677
+@pytest.mark.anyio
1678
+async def test_browser_tool_records_static_history_screenshot(monkeypatch, tmp_path):
1679
+ calls = []
1680
+
1681
+ class FakeRuntime:
1682
+ async def call(self, method, *args, **kwargs):
1683
+ calls.append((method, args, kwargs))
1684
+ if method == "open":
1685
+ return {
1686
+ "id": 1,
1687
+ "state": {
1688
+ "id": 1,
1689
+ "context_id": "browser-context",
1690
+ "currentUrl": "https://example.com/",
1691
+ "title": "Example Domain",
1692
+ },
1693
+ }
1694
+ if method == "screenshot_file":
1695
+ Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
1696
+ Path(kwargs["path"]).write_bytes(b"jpeg")
1697
+ return {
1698
+ "browser_id": args[0],
1699
+ "path": kwargs["path"],
1700
+ "a0_path": "/a0/usr/chats/chat/browser/screenshots/open.jpg",
1701
+ "mime": "image/jpeg",
1702
+ "state": {"id": args[0], "context_id": "browser-context"},
1703
+ }
1704
+ raise AssertionError(method)
1705
+
1706
+ async def fake_get_runtime(context_id, create=True, agent=None):
1707
+ del create, agent
1708
+ assert context_id == "chat"
1709
+ return FakeRuntime()
1710
+
1711
+ class FakeLog:
1712
+ id = "tool-log-id"
1713
+
1714
+ def __init__(self):
1715
+ self.updates = []
1716
+
1717
+ def update(self, **kwargs):
1718
+ self.updates.append(kwargs)
1719
+
1720
+ monkeypatch.setattr(browser_tool_module, "get_runtime", fake_get_runtime)
1721
+ monkeypatch.setitem(
1722
+ sys.modules,
1723
+ "helpers.persist_chat",
1724
+ SimpleNamespace(
1725
+ get_chat_folder_path=lambda context_id: str(tmp_path / "usr" / "chats" / context_id)
1726
+ ),
1727
+ )
1728
+
1729
+ log = FakeLog()
1730
+ tool = browser_tool_module.Browser(
1731
+ agent=SimpleNamespace(context=SimpleNamespace(id="chat")),
1732
+ name="browser",
1733
+ method=None,
1734
+ args={"action": "open", "url": "https://example.com"},
1735
+ message="",
1736
+ loop_data=None,
1737
+ )
1738
+ tool.log = log
1739
+
1740
+ response = await tool.execute(action="open", url="https://example.com")
1741
+
1742
+ assert response.break_loop is False
1743
+ assert calls[0] == ("open", ("https://example.com",), {})
1744
+ assert calls[1][0] == "screenshot_file"
1745
+ assert calls[1][1] == (1,)
1746
+ assert calls[1][2]["quality"] == browser_tool_module.HISTORY_SCREENSHOT_QUALITY
1747
+ assert calls[1][2]["full_page"] is False
1748
+ assert Path(calls[1][2]["path"]).parent == tmp_path / "usr" / "chats" / "chat" / "browser" / "screenshots"
1749
+ assert Path(calls[1][2]["path"]).read_bytes() == b"jpeg"
1750
+ assert log.updates[-1]["Screenshot"].startswith("img://")
1751
+ assert log.updates[-1]["browser_snapshot"]["browser_id"] == 1
1752
+ assert log.updates[-1]["browser_snapshot"]["context_id"] == "browser-context"
1753
+
1754
+
1755
@pytest.mark.anyio
1756
async def test_browser_multi_dispatch_accepts_v1_actions():
1757
calls = []