Fix canvas attachment for browser and documents
Attach the Browser canvas to active Docker sessions by returning an initial snapshot on subscribe and preserving valid frames through state-only updates. Route Markdown document opens through the right-canvas Desktop editor instead of the legacy office modal. Skip automatic office document response affordances for subordinate agents so delegated reviews keep their actual content.
Alessandro committed
May 8, 2026 at 19:08 UTC
bb2432693ee640e0bdc07c061417ae390495adb8
10 files changed
+161
-38
plugins/_browser/api/ws_browser.py
+14
@@ -92,15 +92,18 @@ class WsBrowser(WsHandler):
92
if existing:
93
existing.cancel()
94
viewer_id = str(data.get("viewer_id") or "")
95
+ snapshot = None
96
if runtime:
97
self._streams[stream_key] = asyncio.create_task(
98
self._stream_frames(sid, context_id, active_id, viewer_id)
99
)
100
+ snapshot = await self._snapshot_for_browser(runtime, active_id)
101
102
return {
103
"context_id": context_id,
104
"active_browser_context_id": context_id,
105
"active_browser_id": active_id,
106
+ "snapshot": snapshot,
107
"browsers": await self._all_browser_tabs(),
108
"all_browsers": True,
109
"viewer_id": viewer_id,
@@ -337,6 +340,17 @@ class WsBrowser(WsHandler):
340
return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
341
return None
342
343
+ async def _snapshot_for_browser(
344
+ self,
345
+ runtime: Any,
346
+ browser_id: int | str | None,
347
+ ) -> dict[str, Any] | None:
348
+ if not browser_id:
349
+ return None
350
+ with contextlib.suppress(Exception):
351
+ return await runtime.call("screenshot", browser_id, quality=SCREENCAST_QUALITY)
352
+ return None
353
+
354
async def _all_browser_tabs(self) -> list[dict[str, Any]]:
355
browsers: list[dict[str, Any]] = []
356
for session in await list_runtime_sessions():
plugins/_browser/webui/browser-store.js
+14
-5
@@ -973,6 +973,7 @@ const model = {
973
data.active_browser_id || requestedBrowserId || this.activeBrowserId || null,
974
data.active_browser_context_id || contextId,
975
);
976
+ this.applySnapshot(data.snapshot);
977
this.connected = true;
978
this.browserInstallExpected = false;
979
},
@@ -1016,11 +1017,9 @@ const model = {
1017
this._surfaceSwitching = false;
1018
},
1019
});
1019
- } else {
1020
+ } else if (!data.state) {
1021
this.cancelFrameRender();
1021
- if (!data.state) {
1022
- this.frameSrc = "";
1023
- }
1022
+ this.frameSrc = "";
1023
}
1024
if (!data.image && !data.state) {
1025
if (!this.activeBrowserId) {
@@ -1113,7 +1112,9 @@ const model = {
1112
const viewport = this.currentViewportSize() || this._lastViewport;
1113
if (!this.frameMatchesViewport(dimensions, viewport)) {
1114
this.requestViewportSyncAfterRejectedFrame();
1116
- return;
1115
+ if (!this.shouldAcceptMismatchedFrame(dimensions)) {
1116
+ return;
1117
+ }
1118
}
1119
this.frameSrc = frameSrc;
1120
this._lastFrameDimensions = dimensions;
@@ -1123,6 +1124,14 @@ const model = {
1124
this.scheduleCanvasWidthNudgeAfterFirstFrame();
1125
},
1126
1127
+ shouldAcceptMismatchedFrame(dimensions = null) {
1128
+ return Boolean(
1129
+ dimensions?.width
1130
+ && dimensions?.height
1131
+ && (!this.frameSrc || this._surfaceSwitching || this.isSwitchingBrowser())
1132
+ );
1133
+ },
1134
+
1135
scheduleCanvasWidthNudgeAfterFirstFrame() {
1136
const surfaceSequence = this._surfaceOpenSequence;
1137
if (this._mode !== "canvas" || !this.isCurrentSurfaceOpen(surfaceSequence) || !this.activeBrowserId) {
plugins/_desktop/api/desktop_session.py
+23
-1
@@ -2,7 +2,7 @@ from __future__ import annotations
2
3
from helpers.api import ApiHandler, Request
4
from plugins._desktop.helpers import desktop_session
5
-from plugins._office.helpers import document_store
5
+from plugins._office.helpers import document_store, markdown_sessions
6
from plugins._office.helpers import libreoffice
7
8
@@ -74,6 +74,8 @@ class DesktopSession(ApiHandler):
74
return {"ok": False, "error": str(exc)}
75
76
ext = str(doc.get("extension") or "").lower()
77
+ if ext == "md":
78
+ return self._open_markdown(doc, input, request)
79
if ext not in desktop_session.OFFICIAL_EXTENSIONS:
80
return {"ok": False, "error": f".{ext} documents do not use the Desktop surface."}
81
@@ -108,6 +110,26 @@ class DesktopSession(ApiHandler):
110
"mode": "edit",
111
}
112
113
+ def _open_markdown(self, doc: dict, input: dict, request: Request) -> dict:
114
+ mode = "edit" if str(input.get("mode") or "edit").lower() == "edit" else "view"
115
+ store_session = document_store.create_session(
116
+ doc["file_id"],
117
+ user_id=str(input.get("user_id") or "agent-zero-user"),
118
+ permission="write" if mode == "edit" else "read",
119
+ origin=self._origin(request),
120
+ )
121
+ try:
122
+ editor = markdown_sessions.get_manager().open(doc, sid="")
123
+ except ValueError as exc:
124
+ document_store.close_session(session_id=store_session["session_id"])
125
+ return {"ok": False, "error": str(exc)}
126
+ return {
127
+ **editor,
128
+ "store_session_id": store_session["session_id"],
129
+ "session_id": editor["session_id"],
130
+ "mode": mode,
131
+ }
132
+
133
def _save(self, input: dict) -> dict:
134
session_id = str(input.get("desktop_session_id") or input.get("session_id") or "").strip()
135
if not session_id:
plugins/_office/extensions/python/tool_execute_after/_20_document_response_affordance.py
+2
@@ -22,6 +22,8 @@ class DocumentResponseAffordance(Extension):
22
):
23
if not self.agent or response is None:
24
return
25
+ if document_affordance.is_subordinate_agent(self.agent):
26
+ return
27
28
if tool_name == "document_artifact":
29
if (response.additional or {}).get("file_id"):
plugins/_office/extensions/webui/lib/document-actions.js
+1
-13
@@ -2,9 +2,7 @@ import {
2
createActionButton,
3
copyToClipboard,
4
} from "/components/messages/action-buttons/simple-action-buttons.js";
5
-import { ensureModalOpen } from "/js/modals.js";
5
import { open as openSurface } from "/js/surfaces.js";
7
-import { store as officeStore } from "/plugins/_office/webui/office-store.js";
6
7
function basename(path = "") {
8
const value = String(path || "").split("?")[0].split("#")[0];
@@ -46,17 +44,7 @@ export async function openDocumentInDesktop(kvps = {}) {
44
}
45
46
export async function openDocumentArtifact(kvps = {}) {
49
- if (usesDesktop(kvps)) {
50
- await openDocumentInDesktop(kvps);
51
- return;
52
- }
53
- await ensureModalOpen("/plugins/_office/webui/main.html");
54
- await officeStore.openSession?.({
55
- path: kvps.path || "",
56
- file_id: kvps.file_id || "",
57
- refresh: true,
58
- source: "message-action",
59
- });
47
+ await openDocumentInDesktop(kvps);
48
}
49
50
function usesDesktop(doc = {}) {
plugins/_office/extensions/webui/set_messages_after_loop/auto-open-document-results.js
+1
-18
@@ -1,9 +1,7 @@
1
import { store as officeStore } from "/plugins/_office/webui/office-store.js";
2
-import { ensureModalOpen } from "/js/modals.js";
2
import { open as openSurface } from "/js/surfaces.js";
3
4
const SYNC_WINDOW_MS = 10 * 60 * 1000;
6
-const DESKTOP_DOCUMENT_EXTENSIONS = new Set(["odt", "ods", "odp", "docx", "xlsx", "pptx"]);
5
const syncedDocumentResults = new Set();
6
7
export default async function syncDocumentResultsIntoOpenOfficeModal(context) {
@@ -112,18 +110,7 @@ function isExplicitDocumentUiRequest(payload = {}) {
110
}
111
112
async function openDocumentUiFromResult(target = {}, payload = {}, document = {}) {
115
- if (isDesktopDocument(payload, document)) {
116
- await openSurface("desktop", {
117
- path: target.path || "",
118
- file_id: target.file_id || "",
119
- refresh: true,
120
- source: "tool-result-open",
121
- });
122
- return;
123
- }
124
-
125
- await ensureModalOpen("/plugins/_office/webui/main.html");
126
- await officeStore.openSession?.({
113
+ await openSurface("desktop", {
114
path: target.path || "",
115
file_id: target.file_id || "",
116
refresh: true,
@@ -131,10 +118,6 @@ async function openDocumentUiFromResult(target = {}, payload = {}, document = {}
118
});
119
}
120
134
-function isDesktopDocument(payload = {}, document = {}) {
135
- return DESKTOP_DOCUMENT_EXTENSIONS.has(documentExtension(payload, document));
136
-}
137
-
121
function documentExtension(payload = {}, document = {}) {
122
return String(
123
payload.format
plugins/_office/helpers/document_affordance.py
+29
@@ -415,3 +415,32 @@ def format_created_response(basename: str, path: str) -> str:
415
f"Created **{basename}**.\n\n"
416
f"Path: `{path}`"
417
)
418
+
419
+
420
+def is_subordinate_agent(agent: Any) -> bool:
421
+ number = getattr(agent, "number", None)
422
+ if number is not None:
423
+ try:
424
+ return int(number) > 0
425
+ except (TypeError, ValueError):
426
+ pass
427
+
428
+ agent_name = str(getattr(agent, "agent_name", "") or "").strip().lower()
429
+ if agent_name.startswith("a") and agent_name[1:].isdigit():
430
+ return int(agent_name[1:]) > 0
431
+ if agent_name.isdigit():
432
+ return int(agent_name) > 0
433
+
434
+ get_data = getattr(agent, "get_data", None)
435
+ if callable(get_data):
436
+ try:
437
+ if get_data("_superior") is not None:
438
+ return True
439
+ except Exception:
440
+ pass
441
+
442
+ data = getattr(agent, "data", None)
443
+ if isinstance(data, dict) and data.get("_superior") is not None:
444
+ return True
445
+
446
+ return False
tests/test_browser_agent_regressions.py
+62
@@ -1091,7 +1091,11 @@ def test_browser_viewer_uses_cdp_screencast_transport():
1091
assert "this.frameState = data.state || null" not in browser_store
1092
assert "function loadFrameDimensions(src)" in browser_store
1093
assert "frameMatchesViewport(dimensions = null, viewport = null)" in browser_store
1094
+ assert "shouldAcceptMismatchedFrame(dimensions = null)" in browser_store
1095
assert "requestViewportSyncAfterRejectedFrame()" in browser_store
1096
+ assert "this.applySnapshot(data.snapshot);" in browser_store
1097
+ assert "else if (!data.state)" in browser_store
1098
+ assert '"snapshot": snapshot' in ws_browser
1099
assert "FRAME_FALLBACK_SCREENSHOT_SECONDS" not in ws_browser
1100
assert '"frame_source": "state"' in ws_browser
1101
assert '"frame_source"] = "screencast"' in ws_browser
@@ -1830,6 +1834,64 @@ async def test_browser_viewer_subscribe_can_create_blank_tab_when_requested(monk
1834
await handler.on_disconnect("sid-create")
1835
1836
1837
+@pytest.mark.anyio
1838
+async def test_browser_viewer_subscribe_returns_initial_snapshot(monkeypatch):
1839
+ calls = []
1840
+
1841
+ class FakeRuntime:
1842
+ async def call(self, method, *args, **kwargs):
1843
+ calls.append((method, args, kwargs))
1844
+ if method == "list":
1845
+ return {
1846
+ "browsers": [{"id": 1, "context_id": "ctx", "currentUrl": "https://example.com/"}],
1847
+ "last_interacted_browser_id": 1,
1848
+ }
1849
+ if method == "set_viewport":
1850
+ return {"state": {"id": args[0], "currentUrl": "https://example.com/"}}
1851
+ if method == "screenshot":
1852
+ return {
1853
+ "browser_id": args[0],
1854
+ "mime": "image/jpeg",
1855
+ "image": "jpeg-data",
1856
+ "state": {"id": args[0], "context_id": "ctx", "currentUrl": "https://example.com/"},
1857
+ }
1858
+ raise AssertionError(method)
1859
+
1860
+ async def fake_get_runtime(context_id, create=True):
1861
+ assert context_id == "ctx"
1862
+ assert create is False
1863
+ return FakeRuntime()
1864
+
1865
+ async def fake_all_browser_tabs():
1866
+ return [{"id": 1, "context_id": "ctx", "currentUrl": "https://example.com/"}]
1867
+
1868
+ monkeypatch.setattr(ws_browser_module, "get_runtime", fake_get_runtime)
1869
+ monkeypatch.setattr(ws_browser_module, "list_runtime_sessions", fake_all_browser_tabs)
1870
+ monkeypatch.setattr(
1871
+ ws_browser_module.AgentContext,
1872
+ "get",
1873
+ staticmethod(lambda context_id: SimpleNamespace(id=context_id)),
1874
+ )
1875
+
1876
+ handler = ws_browser_module.WsBrowser(
1877
+ SimpleNamespace(),
1878
+ threading.RLock(),
1879
+ manager=None,
1880
+ )
1881
+
1882
+ result = await handler.process(
1883
+ "browser_viewer_subscribe",
1884
+ {"context_id": "ctx", "browser_id": 1, "viewport_width": 900, "viewport_height": 600},
1885
+ "sid-snapshot",
1886
+ )
1887
+
1888
+ assert result["active_browser_id"] == 1
1889
+ assert result["snapshot"]["image"] == "jpeg-data"
1890
+ assert ("screenshot", (1,), {"quality": ws_browser_module.SCREENCAST_QUALITY}) in calls
1891
+
1892
+ await handler.on_disconnect("sid-snapshot")
1893
+
1894
+
1895
@pytest.mark.anyio
1896
async def test_browser_viewer_subscribe_without_runtime_does_not_create_runtime(monkeypatch):
1897
async def fake_get_runtime(context_id, create=True):
tests/test_office_canvas_setup.py
+6
-1
@@ -166,6 +166,9 @@ def test_desktop_plugin_owns_routes_runtime_surface_and_state_paths():
166
167
assert "virtual_desktop_routes.install_route_hooks()" in desktop_startup
168
assert 'action in {"open_document", "document"}' in desktop_api
169
+ assert "markdown_sessions" in desktop_api
170
+ assert 'if ext == "md":' in desktop_api
171
+ assert "return self._open_markdown(doc, input, request)" in desktop_api
172
assert '"status": desktop.get("status") or {}' in desktop_api
173
assert 'callJsonApi("/plugins/_desktop/desktop_session"' in desktop_store
174
assert 'callDesktop("open_document"' in desktop_store
@@ -233,7 +236,9 @@ def test_document_artifacts_only_open_desktop_from_explicit_document_ui_requests
236
assert "officeStore" in auto_open
237
assert "openDocumentInDesktop" in document_actions
238
assert "openDocumentArtifact" in document_actions
236
- assert "ensureModalOpen" in document_actions
239
+ assert "await openDocumentInDesktop(kvps);" in document_actions
240
+ assert 'ensureModalOpen("/plugins/_office/webui/main.html")' not in document_actions
241
+ assert 'ensureModalOpen("/plugins/_office/webui/main.html")' not in auto_open
242
assert "Open Document" in document_actions
243
assert 'openSurface("desktop"' in document_actions
244
assert "Edit in Writer" in document_actions
tests/test_office_document_affordance.py
+9
@@ -2,6 +2,7 @@ from __future__ import annotations
2
3
import sys
4
from pathlib import Path
5
+from types import SimpleNamespace
6
7
8
PROJECT_ROOT = Path(__file__).resolve().parents[1]
@@ -163,3 +164,11 @@ def test_created_response_does_not_claim_canvas_was_opened():
164
assert "Created **Project Brief.md**." in message
165
assert "opened" not in message.lower()
166
assert "Path: `/a0/usr/workdir/Project Brief.md`" in message
167
+
168
+
169
+def test_document_response_affordance_only_runs_for_primary_agent():
170
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(number=0, agent_name="A0")) is False
171
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(number=1, agent_name="A1")) is True
172
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(agent_name="A2")) is True
173
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(agent_name="0")) is False
174
+ assert document_affordance.is_subordinate_agent(SimpleNamespace(data={"_superior": object()})) is True