Add browser screenshot previews to tool messages
Render Browser tool Screenshot KVPs as clickable live thumbnails that open the Browser canvas while preserving the existing lower-row Browser action.\n\nAdd a lightweight websocket snapshot endpoint for existing browser runtimes and keep preview frame memory bounded with revocable object URLs.
Alessandro committed
May 2, 2026 at 16:48 UTC
c2fb2c3c94e1e1c85b783252332b3fc003f39f2b
2 files changed
+382
-5
plugins/_browser/api/ws_browser.py
+45
@@ -38,6 +38,8 @@ class WsBrowser(WsHandler):
38
return await self._subscribe(data, sid)
39
if event == "browser_viewer_unsubscribe":
40
return self._unsubscribe(data, sid)
41
+ if event == "browser_viewer_snapshot":
42
+ return await self._snapshot(data)
43
if event == "browser_viewer_sessions":
44
return await self._sessions(data)
45
if event == "browser_viewer_command":
@@ -115,6 +117,49 @@ class WsBrowser(WsHandler):
117
"all_browsers": True,
118
}
119
120
+ async def _snapshot(self, data: dict[str, Any]) -> dict[str, Any] | WsResult:
121
+ context_id = self._context_id(data)
122
+ if not context_id:
123
+ return self._error("MISSING_CONTEXT", "context_id is required", data)
124
+ if not AgentContext.get(context_id):
125
+ return self._error("CONTEXT_NOT_FOUND", f"Context '{context_id}' was not found", data)
126
+
127
+ runtime = await get_runtime(context_id, create=False)
128
+ if not runtime:
129
+ return {
130
+ "context_id": context_id,
131
+ "active_browser_context_id": context_id,
132
+ "active_browser_id": None,
133
+ "snapshot": None,
134
+ "browsers": await self._all_browser_tabs(),
135
+ "all_browsers": True,
136
+ }
137
+
138
+ listing = await runtime.call("list")
139
+ browsers = listing.get("browsers") or []
140
+ active_id = self._active_browser_id(listing, data.get("browser_id"))
141
+ snapshot = None
142
+ if active_id:
143
+ try:
144
+ quality = int(data.get("quality") or SCREENCAST_QUALITY)
145
+ except (TypeError, ValueError):
146
+ quality = SCREENCAST_QUALITY
147
+ with contextlib.suppress(Exception):
148
+ snapshot = await runtime.call(
149
+ "screenshot",
150
+ active_id,
151
+ quality=quality,
152
+ )
153
+
154
+ return {
155
+ "context_id": context_id,
156
+ "active_browser_context_id": context_id,
157
+ "active_browser_id": active_id,
158
+ "snapshot": snapshot,
159
+ "browsers": await self._all_browser_tabs(),
160
+ "all_browsers": True,
161
+ }
162
+
163
async def _command(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
164
context_id = self._context_id(data)
165
if not context_id:
plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
+337
-5
@@ -6,6 +6,7 @@ import { store as stepDetailStore } from "/components/modals/process-step-detail
6
import { store as speechStore } from "/components/chat/speech/speech-store.js";
7
import { store as rightCanvasStore } from "/components/canvas/right-canvas-store.js";
8
import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";
9
+import { getNamespacedClient } from "/js/websocket.js";
10
import {
11
buildDetailPayload,
12
cleanStepTitle,
@@ -13,8 +14,19 @@ import {
14
} from "/js/messages.js";
15
16
const BROWSER_MODAL = "/plugins/_browser/webui/main.html";
17
+const BROWSER_SCREENSHOT_KVP_KEY = "Screenshot";
18
+const BROWSER_SCREENSHOT_STYLE_ID = "a0-browser-screenshot-kvp-style";
19
const AUTO_OPEN_WINDOW_MS = 10 * 60 * 1000;
20
+const PREVIEW_REFRESH_MS = 2500;
21
+const PREVIEW_RETRY_MS = 5000;
22
+const PREVIEW_SNAPSHOT_TIMEOUT_MS = 10000;
23
+const PREVIEW_QUALITY = 62;
24
+const PREVIEW_FRAME_LIMIT = 16;
25
+const NO_SCREENSHOT_ACTIONS = new Set(["close", "close_all"]);
26
const syncedBrowserCanvases = new Set();
27
+const liveScreenshotFrames = new Map();
28
+const websocket = getNamespacedClient("/ws");
29
+websocket.addHandlers(["ws_webui"]);
30
31
export default async function registerBrowserToolHandler(extData) {
32
if (extData?.tool_name === "browser") {
@@ -58,21 +70,37 @@ function parseBrowserResult(content) {
70
}
71
}
72
73
+function normalizeBrowserAction(kvps = {}) {
74
+ return String(kvps.action || "").trim().toLowerCase().replace("-", "_");
75
+}
76
+
77
function browserIdFromResult(result = {}, kvps = {}) {
78
+ const browsers = Array.isArray(result.browsers) ? result.browsers : [];
79
+ const lastInteractedId = result.last_interacted_browser_id;
80
+ const listedBrowser = lastInteractedId
81
+ ? browsers.find((browser) => String(browser?.id) === String(lastInteractedId))
82
+ : browsers[0];
83
return (
84
result.id
85
|| result.browser_id
86
|| result.state?.id
87
|| result.last_interacted_browser_id
88
+ || listedBrowser?.id
89
|| kvps.browser_id
90
|| null
91
);
92
}
93
94
function browserContextIdFromResult(result = {}, kvps = {}) {
95
+ const browserId = browserIdFromResult(result, kvps);
96
+ const browsers = Array.isArray(result.browsers) ? result.browsers : [];
97
+ const listedBrowser = browserId
98
+ ? browsers.find((browser) => String(browser?.id) === String(browserId))
99
+ : browsers[0];
100
return (
101
result.context_id
102
|| result.state?.context_id
103
+ || listedBrowser?.context_id
104
|| kvps.context_id
105
|| kvps.contextId
106
|| null
@@ -132,6 +160,290 @@ function syncOpenBrowserCanvas(args, result) {
160
});
161
}
162
163
+function currentBrowserContextId(result = {}, kvps = {}) {
164
+ return (
165
+ browserContextIdFromResult(result, kvps)
166
+ || browserStore?.resolveContextId?.()
167
+ || ""
168
+ );
169
+}
170
+
171
+function buildBrowserCanvasPayload(result = {}, kvps = {}, source = "tool-kvp") {
172
+ const browserId = browserIdFromResult(result, kvps);
173
+ const contextId = currentBrowserContextId(result, kvps);
174
+ if (!browserId && !contextId) return null;
175
+ return {
176
+ browserId: browserId || null,
177
+ contextId,
178
+ source,
179
+ };
180
+}
181
+
182
+function makeViewerToken() {
183
+ return globalThis.crypto?.randomUUID?.()
184
+ || `${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
185
+}
186
+
187
+function snapshotToObjectUrl(snapshot) {
188
+ if (!snapshot?.image) return null;
189
+ const binary = globalThis.atob(snapshot.image);
190
+ const chunks = [];
191
+ const chunkSize = 8192;
192
+ for (let offset = 0; offset < binary.length; offset += chunkSize) {
193
+ const slice = binary.slice(offset, offset + chunkSize);
194
+ const bytes = new Uint8Array(slice.length);
195
+ for (let index = 0; index < slice.length; index += 1) {
196
+ bytes[index] = slice.charCodeAt(index);
197
+ }
198
+ chunks.push(bytes);
199
+ }
200
+ return URL.createObjectURL(new Blob(chunks, { type: snapshot.mime || "image/jpeg" }));
201
+}
202
+
203
+function releaseLiveScreenshotFrame(viewerId) {
204
+ const release = liveScreenshotFrames.get(viewerId);
205
+ if (!release) return;
206
+ liveScreenshotFrames.delete(viewerId);
207
+ release();
208
+}
209
+
210
+function rememberLiveScreenshotFrame(viewerId, release) {
211
+ liveScreenshotFrames.delete(viewerId);
212
+ liveScreenshotFrames.set(viewerId, release);
213
+
214
+ while (liveScreenshotFrames.size > PREVIEW_FRAME_LIMIT) {
215
+ releaseLiveScreenshotFrame(liveScreenshotFrames.keys().next().value);
216
+ }
217
+}
218
+
219
+function firstOk(response) {
220
+ const result = response?.results?.find((item) => item?.ok);
221
+ if (result) return result.data || {};
222
+ const error = response?.results?.find((item) => !item?.ok)?.error;
223
+ if (error) throw new Error(error.error || error.code || "Browser request failed");
224
+ return {};
225
+}
226
+
227
+function browserResultFromRenderedStep(step) {
228
+ if (!step) return {};
229
+ const contentText = step.querySelector(".process-step-detail-content")?.textContent || "";
230
+ return parseBrowserResult(contentText);
231
+}
232
+
233
+function shouldRenderBrowserScreenshotKvp(result = {}, kvps = {}) {
234
+ if (buildBrowserCanvasPayload(result, kvps)) return true;
235
+ const action = normalizeBrowserAction(kvps);
236
+ return Boolean(action && !NO_SCREENSHOT_ACTIONS.has(action));
237
+}
238
+
239
+function ensureBrowserScreenshotStyles() {
240
+ if (document.getElementById(BROWSER_SCREENSHOT_STYLE_ID)) return;
241
+ const style = document.createElement("style");
242
+ style.id = BROWSER_SCREENSHOT_STYLE_ID;
243
+ style.textContent = `
244
+ .browser-screenshot-kvp-button {
245
+ padding: 0;
246
+ width: min(12rem, 100%);
247
+ aspect-ratio: 16 / 10;
248
+ position: relative;
249
+ display: inline-flex;
250
+ align-items: center;
251
+ justify-content: center;
252
+ border: 1px solid color-mix(in srgb, var(--color-border) 74%, transparent);
253
+ border-radius: 7px;
254
+ background: color-mix(in srgb, var(--color-panel) 88%, var(--color-chat-background));
255
+ color: var(--color-message-text);
256
+ cursor: pointer;
257
+ overflow: hidden;
258
+ box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.08);
259
+ transition: border-color 0.16s ease, color 0.16s ease, transform 0.16s ease, background 0.16s ease;
260
+ }
261
+
262
+ .browser-screenshot-kvp-button:hover {
263
+ border-color: color-mix(in srgb, var(--color-primary) 68%, var(--color-border));
264
+ color: var(--color-text);
265
+ transform: translateY(-1px);
266
+ }
267
+
268
+ .browser-screenshot-kvp-button:focus-visible {
269
+ outline: 2px solid color-mix(in srgb, var(--color-primary) 72%, white);
270
+ outline-offset: 2px;
271
+ }
272
+
273
+ .browser-screenshot-kvp-image {
274
+ width: 100%;
275
+ height: 100%;
276
+ display: block;
277
+ object-fit: cover;
278
+ object-position: top left;
279
+ background: var(--color-chat-background);
280
+ }
281
+
282
+ .browser-screenshot-kvp-button:not(.has-frame) .browser-screenshot-kvp-image {
283
+ display: none;
284
+ }
285
+
286
+ .browser-screenshot-kvp-placeholder {
287
+ position: absolute;
288
+ inset: 0;
289
+ display: flex;
290
+ align-items: center;
291
+ justify-content: center;
292
+ background:
293
+ linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.06), transparent),
294
+ color-mix(in srgb, var(--color-panel) 84%, black 16%);
295
+ background-size: 200% 100%, auto;
296
+ animation: browser-screenshot-kvp-pulse 1.6s ease-in-out infinite;
297
+ }
298
+
299
+ .browser-screenshot-kvp-button.has-frame .browser-screenshot-kvp-placeholder {
300
+ display: none;
301
+ }
302
+
303
+ .browser-screenshot-kvp-button .material-symbols-outlined {
304
+ font-size: 1.15rem;
305
+ line-height: 1;
306
+ font-variation-settings: 'FILL' 0, 'wght' 420, 'GRAD' 0, 'opsz' 20;
307
+ }
308
+
309
+ @keyframes browser-screenshot-kvp-pulse {
310
+ 0% { background-position: 160% 0, 0 0; }
311
+ 100% { background-position: -160% 0, 0 0; }
312
+ }
313
+ `;
314
+ document.head.appendChild(style);
315
+}
316
+
317
+function renderBrowserScreenshotKvp(kvpsTable, resolveBrowserPayload, label) {
318
+ if (!kvpsTable || typeof resolveBrowserPayload !== "function") return;
319
+ ensureBrowserScreenshotStyles();
320
+
321
+ const rows = Array.from(kvpsTable.querySelectorAll(".kvps-row"));
322
+ const row = rows.find((candidate) => {
323
+ const keyText = candidate.querySelector(".kvps-key")?.textContent || "";
324
+ return keyText.trim().toLowerCase() === BROWSER_SCREENSHOT_KVP_KEY.toLowerCase();
325
+ });
326
+ const cell = row?.querySelector(".kvps-val") || row?.cells?.[1];
327
+ if (!cell) return;
328
+
329
+ cell.__browserScreenshotCleanup?.();
330
+
331
+ const button = document.createElement("button");
332
+ button.type = "button";
333
+ button.className = "browser-screenshot-kvp-button";
334
+ button.title = label;
335
+ button.setAttribute("aria-label", label);
336
+ button.innerHTML = `
337
+ <img class="browser-screenshot-kvp-image" alt="" draggable="false">
338
+ <span class="browser-screenshot-kvp-placeholder" aria-hidden="true">
339
+ <span class="material-symbols-outlined">image_search</span>
340
+ </span>
341
+ `;
342
+ button.addEventListener("click", async (event) => {
343
+ event.preventDefault();
344
+ event.stopPropagation();
345
+ const canvasPayload = resolveBrowserPayload();
346
+ if (!canvasPayload) return;
347
+ await openBrowserCanvas(canvasPayload);
348
+ });
349
+
350
+ cell.textContent = "";
351
+ cell.appendChild(button);
352
+ cell.__browserScreenshotCleanup = startBrowserScreenshotPreview(
353
+ button,
354
+ button.querySelector(".browser-screenshot-kvp-image"),
355
+ resolveBrowserPayload,
356
+ );
357
+}
358
+
359
+function startBrowserScreenshotPreview(button, image, resolveBrowserPayload) {
360
+ const viewerId = makeViewerToken();
361
+ let stopped = false;
362
+ let timerId = null;
363
+ let refreshInFlight = false;
364
+ let frameUrl = null;
365
+
366
+ const releaseFrame = () => {
367
+ liveScreenshotFrames.delete(viewerId);
368
+ if (frameUrl) {
369
+ URL.revokeObjectURL(frameUrl);
370
+ frameUrl = null;
371
+ }
372
+ image.removeAttribute("src");
373
+ button.classList.remove("has-frame");
374
+ };
375
+
376
+ const setFrame = (snapshot) => {
377
+ const nextUrl = snapshotToObjectUrl(snapshot);
378
+ if (!nextUrl) {
379
+ releaseFrame();
380
+ return false;
381
+ }
382
+ if (frameUrl) {
383
+ URL.revokeObjectURL(frameUrl);
384
+ }
385
+ frameUrl = nextUrl;
386
+ image.src = frameUrl;
387
+ button.classList.add("has-frame");
388
+ rememberLiveScreenshotFrame(viewerId, releaseFrame);
389
+ return true;
390
+ };
391
+
392
+ const cleanup = () => {
393
+ stopped = true;
394
+ if (timerId) {
395
+ globalThis.clearTimeout(timerId);
396
+ timerId = null;
397
+ }
398
+ releaseFrame();
399
+ };
400
+
401
+ const schedule = (delay = PREVIEW_REFRESH_MS) => {
402
+ if (stopped || timerId) return;
403
+ timerId = globalThis.setTimeout(refresh, delay);
404
+ };
405
+
406
+ const refresh = async () => {
407
+ timerId = null;
408
+ if (stopped || refreshInFlight) return;
409
+ if (!button.isConnected) {
410
+ cleanup();
411
+ return;
412
+ }
413
+
414
+ const payload = resolveBrowserPayload();
415
+ if (!payload?.contextId) {
416
+ schedule(PREVIEW_RETRY_MS);
417
+ return;
418
+ }
419
+
420
+ refreshInFlight = true;
421
+ try {
422
+ const response = await websocket.request(
423
+ "browser_viewer_snapshot",
424
+ {
425
+ context_id: payload.contextId,
426
+ browser_id: payload.browserId,
427
+ viewer_id: viewerId,
428
+ quality: PREVIEW_QUALITY,
429
+ },
430
+ { timeoutMs: PREVIEW_SNAPSHOT_TIMEOUT_MS },
431
+ );
432
+ const data = firstOk(response);
433
+ const hasFrame = setFrame(data.snapshot);
434
+ schedule(hasFrame ? PREVIEW_REFRESH_MS : PREVIEW_RETRY_MS);
435
+ } catch (_error) {
436
+ schedule(PREVIEW_RETRY_MS);
437
+ } finally {
438
+ refreshInFlight = false;
439
+ }
440
+ };
441
+
442
+ schedule(0);
443
+
444
+ return cleanup;
445
+}
446
+
447
function drawBrowserTool({
448
id,
449
type,
@@ -150,14 +462,25 @@ function drawBrowserTool({
462
].filter(Boolean);
463
const contentText = String(content ?? "");
464
const browserResult = parseBrowserResult(contentText);
465
+ const browserId = browserIdFromResult(browserResult, kvps);
466
+ const browserCanvasPayload = buildBrowserCanvasPayload(browserResult, kvps);
467
+ const browserPreviewLabel = browserId
468
+ ? `Open Browser canvas for Browser ${browserId}`
469
+ : "Open Browser canvas from screenshot";
470
+ if (shouldRenderBrowserScreenshotKvp(browserResult, kvps)) {
471
+ displayKvps[BROWSER_SCREENSHOT_KVP_KEY] = "";
472
+ }
473
const browserButton = createActionButton(
474
"visibility",
475
"Browser",
156
- () => openBrowserCanvas({
157
- browserId: browserIdFromResult(browserResult, kvps),
158
- contextId: browserContextIdFromResult(browserResult, kvps),
159
- source: "tool",
160
- }),
476
+ () => openBrowserCanvas(
477
+ buildBrowserCanvasPayload(browserResult, kvps, "tool")
478
+ || {
479
+ browserId,
480
+ contextId: browserContextIdFromResult(browserResult, kvps),
481
+ source: "tool",
482
+ },
483
+ ),
484
);
485
browserButton.setAttribute("title", "Open Browser");
486
browserButton.setAttribute("aria-label", "Open Browser");
@@ -187,6 +510,15 @@ function drawBrowserTool({
510
actionButtons: actionButtons.filter(Boolean),
511
log: args,
512
});
513
+ renderBrowserScreenshotKvp(
514
+ result.kvpsTable,
515
+ () => (
516
+ buildBrowserCanvasPayload(browserResultFromRenderedStep(result.step), kvps)
517
+ || browserCanvasPayload
518
+ || buildBrowserCanvasPayload({}, kvps)
519
+ ),
520
+ browserPreviewLabel,
521
+ );
522
syncOpenBrowserCanvas(args, browserResult);
523
return result;
524
}