@samitouri / QOSami_Agents_Zero-1 / commits / 78570e56

Improve Linux Desktop state controls

Add a desktop_state helper, expanded desktopctl observe-act-verify commands, backend desktop_state support, Extra prompt state, and Xpra bridge diagnostics for the built-in Linux Desktop. Update the Linux Desktop skill so agents prefer structured/app-native/keyboard workflows, treat coordinate clicks as last resort, and verify terminal or CLI-agent work with fresh final screenshots. Cover the behavior with focused Office desktop state, canvas setup, and office_session tests.

Alessandro committed May 5, 2026 at 11:20 UTC 78570e5689403b33fe576c3282ae6802e2e730ce
11 files changed +1398 -13
plugins/_office/api/office_session.py
+6
@@ -61,6 +61,8 @@ class OfficeSession(ApiHandler):
61 return self._desktop_save(input)
62 if action == "desktop_sync":
63 return self._desktop_sync(input)
64 + if action == "desktop_state":
65 + return self._desktop_state(input)
66 return {"ok": False, "error": f"Unsupported office session action: {action}"}
67
68 async def _open_document(self, doc: dict, input: dict, request: Request) -> dict:
@@ -190,6 +192,10 @@ class OfficeSession(ApiHandler):
192 file_id=str(input.get("file_id") or ""),
193 )
194
195 + def _desktop_state(self, input: dict) -> dict:
196 + include_screenshot = bool(input.get("include_screenshot") is True)
197 + return libreoffice_desktop.get_manager().state(include_screenshot=include_screenshot)
198 +
199 def _origin(self, request: Request) -> str:
200 origin = request.headers.get("Origin") or request.host_url.rstrip("/")
201 return origin.rstrip("/")
plugins/_office/helpers/canvas_context.py
+20 -1
@@ -2,13 +2,15 @@ from __future__ import annotations
2
3 from typing import Any
4
5 +from plugins._office.helpers import desktop_state
6 from plugins._office.helpers import document_store
7
8
9 def build_context(max_items: int = 6) -> str:
10 documents = document_store.get_open_documents(limit=max_items)
11 + desktop_context = build_desktop_context()
12 if not documents:
11 - return ""
13 + return desktop_context
14
15 lines = [
16 "These document artifacts have active canvas sessions. Content is omitted; load skill `office-artifacts` for edit workflow, then use document_artifact:read before content-sensitive edits.",
@@ -18,6 +20,8 @@ def build_context(max_items: int = 6) -> str:
20 lines.append(
21 "Use document_artifact:edit with file_id or path for saved edits; tool results refresh the document canvas."
22 )
23 + if desktop_context:
24 + lines.extend(["", desktop_context])
25 return "\n".join(lines)
26
27
@@ -29,3 +33,18 @@ def format_document_line(doc: dict[str, Any]) -> str:
33 f"size={doc.get('size', 0)} bytes, last_modified={doc.get('last_modified', '')}, "
34 f"open_sessions={doc.get('open_sessions', 1)})"
35 )
36 +
37 +
38 +def build_desktop_context() -> str:
39 + if not desktop_state.session_manifest_exists():
40 + return ""
41 + try:
42 + return desktop_state.compact_prompt_context(
43 + desktop_state.collect_state(include_screenshot=False),
44 + )
45 + except Exception as exc:
46 + return (
47 + "[DESKTOP STATE]\n"
48 + f"- unavailable={exc}\n"
49 + "- next=Open the Desktop canvas manually, then run plugins/_office/skills/linux-desktop/scripts/desktopctl.sh observe --json."
50 + )
plugins/_office/helpers/desktop_state.py new
+680
@@ -0,0 +1,680 @@
1 +from __future__ import annotations
2 +
3 +import argparse
4 +import json
5 +import os
6 +import re
7 +import shutil
8 +import subprocess
9 +import time
10 +from pathlib import Path
11 +from typing import Any
12 +
13 +PROJECT_ROOT = Path(__file__).resolve().parents[3]
14 +
15 +
16 +SESSION_ID = "agent-zero-desktop"
17 +BASE_DIR = Path(os.environ.get("A0_BASE_DIR") or ("/a0" if Path("/a0").exists() else PROJECT_ROOT))
18 +STATE_DIR = BASE_DIR / "tmp" / "_office" / "desktop"
19 +SESSION_DIR = STATE_DIR / "sessions"
20 +PROFILE_DIR = STATE_DIR / "profiles"
21 +SCREENSHOT_DIR = STATE_DIR / "screenshots"
22 +RECENT_SCREENSHOT_SECONDS = 600
23 +
24 +
25 +def session_manifest_path(session_id: str = SESSION_ID) -> Path:
26 + return Path(os.environ.get("A0_DESKTOP_MANIFEST") or SESSION_DIR / f"{session_id}.json")
27 +
28 +
29 +def session_manifest_exists(session_id: str = SESSION_ID) -> bool:
30 + return session_manifest_path(session_id).exists()
31 +
32 +
33 +def collect_state(*, include_screenshot: bool = False, screenshot_path: str | Path | None = None) -> dict[str, Any]:
34 + errors: list[str] = []
35 + env_info = resolve_environment(errors=errors)
36 + display = env_info["display"]
37 + profile_dir = env_info["profile_dir"]
38 + env = display_env(display=display, profile_dir=profile_dir)
39 +
40 + capabilities = collect_capabilities()
41 + for name in ("xdotool", "xrandr", "xwininfo", "xprop"):
42 + if not capabilities.get(name):
43 + errors.append(f"{name} is not installed; install Office runtime dependencies through the _office plugin hook.")
44 +
45 + size = collect_display_size(env, capabilities, errors)
46 + pointer = collect_pointer(env, capabilities, errors)
47 + active_window = collect_active_window(env, capabilities, errors)
48 + windows = collect_windows(env, capabilities, errors)
49 + screenshot = latest_screenshot()
50 +
51 + if include_screenshot:
52 + screenshot = capture_screenshot(env, capabilities, path=screenshot_path, errors=errors)
53 +
54 + return stable_state(
55 + display=display,
56 + profile_dir=profile_dir,
57 + size=size,
58 + pointer=pointer,
59 + active_window=active_window,
60 + windows=windows,
61 + screenshot=screenshot,
62 + capabilities=capabilities,
63 + errors=errors,
64 + )
65 +
66 +
67 +def capture_screenshot(
68 + env: dict[str, str] | None = None,
69 + capabilities: dict[str, str] | None = None,
70 + *,
71 + path: str | Path | None = None,
72 + errors: list[str] | None = None,
73 +) -> dict[str, Any]:
74 + local_errors = errors if errors is not None else []
75 + capabilities = capabilities or collect_capabilities()
76 + if not env:
77 + env_errors: list[str] = []
78 + env_info = resolve_environment(errors=env_errors)
79 + local_errors.extend(env_errors)
80 + env = display_env(display=env_info["display"], profile_dir=env_info["profile_dir"])
81 +
82 + xwd = capabilities.get("xwd") or shutil.which("xwd") or ""
83 + if not xwd:
84 + message = "xwd is not installed; install x11-apps through the _office plugin hook."
85 + local_errors.append(message)
86 + return {"ok": False, "path": "", "format": "", "captured_at": "", "error": message}
87 +
88 + SCREENSHOT_DIR.mkdir(parents=True, exist_ok=True)
89 + timestamp = time.strftime("%Y%m%d-%H%M%S")
90 + target = Path(path) if path else SCREENSHOT_DIR / f"desktop-{timestamp}.png"
91 + target.parent.mkdir(parents=True, exist_ok=True)
92 + raw_path = target.with_suffix(".xwd")
93 +
94 + result = run([xwd, "-root", "-silent", "-out", str(raw_path)], env=env, timeout=8)
95 + if result.returncode != 0:
96 + detail = command_output(result) or "xwd screenshot capture failed."
97 + local_errors.append(detail)
98 + raw_path.unlink(missing_ok=True)
99 + return {"ok": False, "path": "", "format": "", "captured_at": "", "error": detail}
100 +
101 + if target.suffix.lower() == ".xwd":
102 + return {
103 + "ok": True,
104 + "path": str(raw_path),
105 + "format": "xwd",
106 + "captured_at": iso_now(),
107 + "recent": True,
108 + "error": "",
109 + }
110 +
111 + try:
112 + from PIL import Image
113 +
114 + with Image.open(raw_path) as image:
115 + image.save(target)
116 + width = int(image.width)
117 + height = int(image.height)
118 + raw_path.unlink(missing_ok=True)
119 + return {
120 + "ok": True,
121 + "path": str(target),
122 + "format": target.suffix.lower().lstrip(".") or "png",
123 + "width": width,
124 + "height": height,
125 + "captured_at": iso_now(),
126 + "recent": True,
127 + "error": "",
128 + }
129 + except Exception as exc:
130 + try:
131 + converted = convert_xwd_to_image(raw_path, target)
132 + raw_path.unlink(missing_ok=True)
133 + return {
134 + "ok": True,
135 + "path": str(target),
136 + "format": target.suffix.lower().lstrip(".") or "png",
137 + "width": converted["width"],
138 + "height": converted["height"],
139 + "captured_at": iso_now(),
140 + "recent": True,
141 + "error": "",
142 + }
143 + except Exception as fallback_exc:
144 + message = f"Pillow could not convert the XWD screenshot: {exc}; fallback parser failed: {fallback_exc}"
145 + local_errors.append(message)
146 + return {
147 + "ok": True,
148 + "path": str(raw_path),
149 + "format": "xwd",
150 + "captured_at": iso_now(),
151 + "recent": True,
152 + "error": message,
153 + }
154 +
155 +
156 +def convert_xwd_to_image(raw_path: Path, target: Path) -> dict[str, int]:
157 + from PIL import Image
158 +
159 + data = raw_path.read_bytes()
160 + header, endian = parse_xwd_header(data)
161 + width = header["pixmap_width"]
162 + height = header["pixmap_height"]
163 + bytes_per_line = header["bytes_per_line"]
164 + bits_per_pixel = header["bits_per_pixel"]
165 + image_byte_order = "little" if header["byte_order"] == 0 else "big"
166 + color_table_size = header["ncolors"] * 12
167 + pixel_offset = header["header_size"] + color_table_size
168 + bytes_per_pixel = max((bits_per_pixel + 7) // 8, 1)
169 + if width > 0 and bytes_per_line % width == 0:
170 + bytes_per_pixel = max(bytes_per_pixel, bytes_per_line // width)
171 + if width <= 0 or height <= 0 or bytes_per_line <= 0:
172 + raise ValueError("invalid XWD dimensions")
173 + if pixel_offset + (height * bytes_per_line) > len(data):
174 + raise ValueError("truncated XWD pixel data")
175 +
176 + red_mask = header["red_mask"]
177 + green_mask = header["green_mask"]
178 + blue_mask = header["blue_mask"]
179 + red_shift, red_bits = mask_shift_and_bits(red_mask)
180 + green_shift, green_bits = mask_shift_and_bits(green_mask)
181 + blue_shift, blue_bits = mask_shift_and_bits(blue_mask)
182 + if min(red_bits, green_bits, blue_bits) <= 0:
183 + raise ValueError("unsupported XWD visual masks")
184 +
185 + pixels: list[tuple[int, int, int]] = []
186 + for row in range(height):
187 + row_start = pixel_offset + (row * bytes_per_line)
188 + for column in range(width):
189 + start = row_start + (column * bytes_per_pixel)
190 + pixel_bytes = data[start : start + bytes_per_pixel]
191 + if len(pixel_bytes) < bytes_per_pixel:
192 + raise ValueError("truncated XWD pixel")
193 + pixel = int.from_bytes(pixel_bytes, image_byte_order, signed=False)
194 + pixels.append(
195 + (
196 + scale_channel((pixel & red_mask) >> red_shift, red_bits),
197 + scale_channel((pixel & green_mask) >> green_shift, green_bits),
198 + scale_channel((pixel & blue_mask) >> blue_shift, blue_bits),
199 + ),
200 + )
201 +
202 + image = Image.new("RGB", (width, height))
203 + image.putdata(pixels)
204 + image.save(target)
205 + return {"width": width, "height": height}
206 +
207 +
208 +def parse_xwd_header(data: bytes) -> tuple[dict[str, int], str]:
209 + if len(data) < 100:
210 + raise ValueError("XWD header is too short")
211 + field_names = (
212 + "header_size",
213 + "file_version",
214 + "pixmap_format",
215 + "pixmap_depth",
216 + "pixmap_width",
217 + "pixmap_height",
218 + "xoffset",
219 + "byte_order",
220 + "bitmap_unit",
221 + "bitmap_bit_order",
222 + "bitmap_pad",
223 + "bits_per_pixel",
224 + "bytes_per_line",
225 + "visual_class",
226 + "red_mask",
227 + "green_mask",
228 + "blue_mask",
229 + "bits_per_rgb",
230 + "colormap_entries",
231 + "ncolors",
232 + "window_width",
233 + "window_height",
234 + "window_x",
235 + "window_y",
236 + "window_bdrwidth",
237 + )
238 + for endian in ("big", "little"):
239 + values = [int.from_bytes(data[index : index + 4], endian, signed=False) for index in range(0, 100, 4)]
240 + header = dict(zip(field_names, values, strict=True))
241 + if 100 <= header["header_size"] <= len(data) and header["file_version"] == 7:
242 + return header, endian
243 + raise ValueError("unsupported XWD header")
244 +
245 +
246 +def mask_shift_and_bits(mask: int) -> tuple[int, int]:
247 + if mask <= 0:
248 + return 0, 0
249 + shift = 0
250 + value = mask
251 + while value and value & 1 == 0:
252 + shift += 1
253 + value >>= 1
254 + bits = 0
255 + while value & 1:
256 + bits += 1
257 + value >>= 1
258 + return shift, bits
259 +
260 +
261 +def scale_channel(value: int, bits: int) -> int:
262 + if bits >= 8:
263 + return max(0, min(255, value >> (bits - 8)))
264 + max_value = (1 << bits) - 1
265 + return 0 if max_value <= 0 else round((value / max_value) * 255)
266 +
267 +
268 +def resolve_environment(*, errors: list[str] | None = None, session_id: str = SESSION_ID) -> dict[str, str]:
269 + local_errors = errors if errors is not None else []
270 + manifest = session_manifest_path(session_id)
271 + payload: dict[str, Any] = {}
272 + if manifest.exists():
273 + try:
274 + payload = json.loads(manifest.read_text(encoding="utf-8"))
275 + except Exception as exc:
276 + local_errors.append(f"Desktop session manifest is unreadable: {exc}")
277 + elif not (os.environ.get("A0_DESKTOP_DISPLAY") or os.environ.get("DISPLAY")):
278 + local_errors.append(f"Desktop session manifest not found at {manifest}; open the Desktop canvas before GUI control.")
279 +
280 + display_value = str(
281 + os.environ.get("A0_DESKTOP_DISPLAY")
282 + or payload.get("display")
283 + or os.environ.get("DISPLAY")
284 + or ""
285 + ).strip()
286 + if display_value.startswith(":"):
287 + display = display_value
288 + elif display_value:
289 + display = f":{display_value}"
290 + else:
291 + display = ""
292 + local_errors.append("Desktop DISPLAY is unavailable; the persistent Desktop session is not running.")
293 +
294 + profile_dir = str(
295 + os.environ.get("A0_DESKTOP_PROFILE")
296 + or os.environ.get("A0_DESKTOP_HOME")
297 + or payload.get("profile_dir")
298 + or os.environ.get("HOME")
299 + or PROFILE_DIR / session_id
300 + )
301 +
302 + return {
303 + "display": display,
304 + "profile_dir": profile_dir,
305 + "manifest": str(manifest),
306 + }
307 +
308 +
309 +def display_env(*, display: str, profile_dir: str) -> dict[str, str]:
310 + env = {
311 + **os.environ,
312 + "HOME": profile_dir,
313 + "XDG_CONFIG_HOME": os.environ.get("XDG_CONFIG_HOME") or str(Path(profile_dir) / ".config"),
314 + "XDG_DATA_HOME": os.environ.get("XDG_DATA_HOME") or str(Path(profile_dir) / ".local" / "share"),
315 + "XDG_CACHE_HOME": os.environ.get("XDG_CACHE_HOME") or str(Path(profile_dir) / ".cache"),
316 + "XDG_CURRENT_DESKTOP": os.environ.get("XDG_CURRENT_DESKTOP") or "XFCE",
317 + }
318 + if display:
319 + env["DISPLAY"] = display
320 + xauthority = os.environ.get("A0_DESKTOP_XAUTHORITY") or str(Path(profile_dir) / ".Xauthority")
321 + if Path(xauthority).exists():
322 + env["XAUTHORITY"] = xauthority
323 + return env
324 +
325 +
326 +def collect_capabilities() -> dict[str, str]:
327 + return {
328 + name: shutil.which(name) or ""
329 + for name in (
330 + "xdotool",
331 + "xrandr",
332 + "xwininfo",
333 + "xprop",
334 + "xwd",
335 + "xclip",
336 + )
337 + }
338 +
339 +
340 +def collect_display_size(env: dict[str, str], capabilities: dict[str, str], errors: list[str]) -> dict[str, int]:
341 + if not capabilities.get("xrandr"):
342 + return {"width": 0, "height": 0}
343 + result = run([capabilities["xrandr"], "-q"], env=env, timeout=4)
344 + if result.returncode != 0:
345 + errors.append(command_output(result) or "xrandr could not read the Desktop display.")
346 + return {"width": 0, "height": 0}
347 + match = re.search(r"\bcurrent\s+(\d+)\s+x\s+(\d+)", result.stdout)
348 + if not match:
349 + errors.append("xrandr output did not include the current Desktop size.")
350 + return {"width": 0, "height": 0}
351 + return {"width": int(match.group(1)), "height": int(match.group(2))}
352 +
353 +
354 +def collect_pointer(env: dict[str, str], capabilities: dict[str, str], errors: list[str]) -> dict[str, int]:
355 + if not capabilities.get("xdotool"):
356 + return {"x": 0, "y": 0, "screen": 0, "window": 0}
357 + result = run([capabilities["xdotool"], "getmouselocation", "--shell"], env=env, timeout=3)
358 + if result.returncode != 0:
359 + errors.append(command_output(result) or "xdotool could not read the pointer location.")
360 + return {"x": 0, "y": 0, "screen": 0, "window": 0}
361 + values = parse_shell_values(result.stdout)
362 + return {
363 + "x": int_value(values.get("X")),
364 + "y": int_value(values.get("Y")),
365 + "screen": int_value(values.get("SCREEN")),
366 + "window": int_value(values.get("WINDOW")),
367 + }
368 +
369 +
370 +def collect_active_window(env: dict[str, str], capabilities: dict[str, str], errors: list[str]) -> dict[str, Any] | None:
371 + if not capabilities.get("xdotool"):
372 + return None
373 + result = run([capabilities["xdotool"], "getactivewindow"], env=env, timeout=3)
374 + if result.returncode != 0:
375 + errors.append(command_output(result) or "xdotool could not read the active window.")
376 + return None
377 + window_id = result.stdout.strip().splitlines()[0] if result.stdout.strip() else ""
378 + if not window_id:
379 + return None
380 + return collect_window(env, capabilities, window_id, errors)
381 +
382 +
383 +def collect_windows(env: dict[str, str], capabilities: dict[str, str], errors: list[str]) -> list[dict[str, Any]]:
384 + if not capabilities.get("xdotool"):
385 + return []
386 + result = run([capabilities["xdotool"], "search", "--onlyvisible", "--name", "."], env=env, timeout=4)
387 + if result.returncode != 0:
388 + detail = command_output(result)
389 + if detail:
390 + errors.append(detail)
391 + return []
392 + windows: list[dict[str, Any]] = []
393 + seen: set[str] = set()
394 + for window_id in result.stdout.splitlines():
395 + window_id = window_id.strip()
396 + if not window_id or window_id in seen:
397 + continue
398 + seen.add(window_id)
399 + windows.append(collect_window(env, capabilities, window_id, errors))
400 + return windows
401 +
402 +
403 +def collect_window(
404 + env: dict[str, str],
405 + capabilities: dict[str, str],
406 + window_id: str,
407 + errors: list[str],
408 +) -> dict[str, Any]:
409 + props = collect_window_props(env, capabilities, window_id)
410 + geometry = collect_window_geometry(env, capabilities, window_id)
411 + return {
412 + "id": str(window_id),
413 + "title": props.get("title", ""),
414 + "class": props.get("class", ""),
415 + "name": props.get("name", ""),
416 + "pid": int_value(props.get("pid")),
417 + "geometry": geometry,
418 + }
419 +
420 +
421 +def collect_window_geometry(env: dict[str, str], capabilities: dict[str, str], window_id: str) -> dict[str, int]:
422 + geometry = {"x": 0, "y": 0, "width": 0, "height": 0}
423 + if not capabilities.get("xwininfo"):
424 + return geometry
425 + result = run([capabilities["xwininfo"], "-id", str(window_id)], env=env, timeout=3)
426 + if result.returncode != 0:
427 + return geometry
428 + patterns = {
429 + "x": r"Absolute upper-left X:\s*(-?\d+)",
430 + "y": r"Absolute upper-left Y:\s*(-?\d+)",
431 + "width": r"Width:\s*(\d+)",
432 + "height": r"Height:\s*(\d+)",
433 + }
434 + for key, pattern in patterns.items():
435 + match = re.search(pattern, result.stdout)
436 + if match:
437 + geometry[key] = int(match.group(1))
438 + return geometry
439 +
440 +
441 +def collect_window_props(env: dict[str, str], capabilities: dict[str, str], window_id: str) -> dict[str, str]:
442 + props = {"title": "", "class": "", "name": "", "pid": ""}
443 + xdotool = capabilities.get("xdotool")
444 + if xdotool:
445 + result = run([xdotool, "getwindowname", str(window_id)], env=env, timeout=3)
446 + if result.returncode == 0:
447 + props["title"] = result.stdout.strip()
448 + xprop = capabilities.get("xprop")
449 + if not xprop:
450 + return props
451 + result = run([xprop, "-id", str(window_id), "WM_CLASS", "WM_NAME", "_NET_WM_NAME", "_NET_WM_PID"], env=env, timeout=3)
452 + if result.returncode != 0:
453 + return props
454 + parsed = parse_xprop(result.stdout)
455 + title = parsed.get("_NET_WM_NAME") or parsed.get("WM_NAME") or props["title"]
456 + props["title"] = title
457 + props["class"] = parsed.get("WM_CLASS_CLASS", "")
458 + props["name"] = parsed.get("WM_CLASS_NAME", "")
459 + props["pid"] = parsed.get("_NET_WM_PID", "")
460 + return props
461 +
462 +
463 +def parse_xprop(output: str) -> dict[str, str]:
464 + values: dict[str, str] = {}
465 + for line in output.splitlines():
466 + if "=" not in line:
467 + continue
468 + key, raw_value = line.split("=", 1)
469 + key = key.strip().split("(", 1)[0]
470 + raw_value = raw_value.strip()
471 + quoted = re.findall(r'"([^"]*)"', raw_value)
472 + if key == "WM_CLASS" and quoted:
473 + values["WM_CLASS_NAME"] = quoted[0]
474 + values["WM_CLASS_CLASS"] = quoted[-1]
475 + continue
476 + if quoted:
477 + values[key] = quoted[-1]
478 + continue
479 + match = re.search(r"-?\d+", raw_value)
480 + values[key] = match.group(0) if match else raw_value
481 + return values
482 +
483 +
484 +def latest_screenshot() -> dict[str, Any]:
485 + if not SCREENSHOT_DIR.exists():
486 + return {"ok": False, "path": "", "format": "", "captured_at": "", "recent": False}
487 + candidates = [
488 + path
489 + for path in SCREENSHOT_DIR.iterdir()
490 + if path.is_file() and path.suffix.lower() in {".png", ".jpg", ".jpeg", ".xwd"}
491 + ]
492 + if not candidates:
493 + return {"ok": False, "path": "", "format": "", "captured_at": "", "recent": False}
494 + latest = max(candidates, key=lambda item: item.stat().st_mtime)
495 + age = max(0.0, time.time() - latest.stat().st_mtime)
496 + return {
497 + "ok": True,
498 + "path": str(latest),
499 + "format": latest.suffix.lower().lstrip("."),
500 + "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(latest.stat().st_mtime)),
501 + "recent": age <= RECENT_SCREENSHOT_SECONDS,
502 + }
503 +
504 +
505 +def stable_state(
506 + *,
507 + display: str,
508 + profile_dir: str,
509 + size: dict[str, int] | None = None,
510 + pointer: dict[str, int] | None = None,
511 + active_window: dict[str, Any] | None = None,
512 + windows: list[dict[str, Any]] | None = None,
513 + screenshot: dict[str, Any] | None = None,
514 + capabilities: dict[str, str] | None = None,
515 + errors: list[str] | None = None,
516 +) -> dict[str, Any]:
517 + clean_errors = [str(error) for error in errors or [] if str(error)]
518 + return {
519 + "ok": not clean_errors,
520 + "display": display,
521 + "profile_dir": profile_dir,
522 + "size": size or {"width": 0, "height": 0},
523 + "pointer": pointer or {"x": 0, "y": 0, "screen": 0, "window": 0},
524 + "active_window": active_window,
525 + "windows": windows or [],
526 + "screenshot": screenshot or {"ok": False, "path": "", "format": "", "captured_at": "", "recent": False},
527 + "capabilities": capabilities or collect_capabilities(),
528 + "errors": clean_errors,
529 + }
530 +
531 +
532 +def compact_prompt_context(state: dict[str, Any] | None = None) -> str:
533 + state = state if state is not None else collect_state(include_screenshot=False)
534 + if not state.get("display"):
535 + return ""
536 + lines = ["[DESKTOP STATE]"]
537 + size = state.get("size") or {}
538 + pointer = state.get("pointer") or {}
539 + lines.append(
540 + f"- display={state.get('display', '')} size={size.get('width', 0)}x{size.get('height', 0)} "
541 + f"pointer={pointer.get('x', 0)},{pointer.get('y', 0)}"
542 + )
543 + active = state.get("active_window") or {}
544 + if active:
545 + lines.append(
546 + f"- active={active.get('title', '') or '<untitled>'} "
547 + f"class={active.get('class', '') or active.get('name', '')}"
548 + )
549 + visible = []
550 + for window in state.get("windows") or []:
551 + title = window.get("title") or "<untitled>"
552 + window_class = window.get("class") or window.get("name") or ""
553 + visible.append(f"{title} ({window_class})" if window_class else title)
554 + if len(visible) >= 5:
555 + break
556 + if visible:
557 + lines.append("- visible=" + "; ".join(visible))
558 + screenshot = state.get("screenshot") or {}
559 + if screenshot.get("recent") and screenshot.get("path"):
560 + lines.append(f"- recent_screenshot={screenshot['path']}")
561 + lines.append(
562 + "- next=plugins/_office/skills/linux-desktop/scripts/desktopctl.sh observe --json --screenshot "
563 + "before any coordinate action; prefer focus/key/paste/save/app-native helpers first."
564 + )
565 + lines.append(
566 + "- verify=for terminal/CLI-agent output, use the screenshot path from a fresh final "
567 + "observe --json --screenshot captured after the response appears."
568 + )
569 + if state.get("errors"):
570 + lines.append("- errors=" + "; ".join(str(item) for item in state["errors"][:2]))
571 + return "\n".join(lines)
572 +
573 +
574 +def parse_shell_values(output: str) -> dict[str, str]:
575 + values: dict[str, str] = {}
576 + for line in output.splitlines():
577 + if "=" not in line:
578 + continue
579 + key, value = line.split("=", 1)
580 + values[key.strip()] = value.strip().strip('"')
581 + return values
582 +
583 +
584 +def int_value(value: Any, default: int = 0) -> int:
585 + try:
586 + return int(str(value).strip())
587 + except (TypeError, ValueError):
588 + return default
589 +
590 +
591 +def run(command: list[str], *, env: dict[str, str], timeout: float) -> subprocess.CompletedProcess[str]:
592 + try:
593 + return subprocess.run(
594 + command,
595 + check=False,
596 + capture_output=True,
597 + text=True,
598 + timeout=timeout,
599 + env=env,
600 + )
601 + except OSError as exc:
602 + return subprocess.CompletedProcess(command, 127, "", str(exc))
603 + except subprocess.TimeoutExpired as exc:
604 + stdout = exc.stdout.decode("utf-8", errors="replace") if isinstance(exc.stdout, bytes) else (exc.stdout or "")
605 + stderr = exc.stderr.decode("utf-8", errors="replace") if isinstance(exc.stderr, bytes) else (exc.stderr or "")
606 + return subprocess.CompletedProcess(command, 124, stdout, stderr or "command timed out")
607 +
608 +
609 +def command_output(result: subprocess.CompletedProcess[str]) -> str:
610 + return (result.stderr or result.stdout or "").strip()
611 +
612 +
613 +def image_width(path: Path) -> int:
614 + try:
615 + from PIL import Image
616 +
617 + with Image.open(path) as image:
618 + return int(image.width)
619 + except Exception:
620 + return 0
621 +
622 +
623 +def image_height(path: Path) -> int:
624 + try:
625 + from PIL import Image
626 +
627 + with Image.open(path) as image:
628 + return int(image.height)
629 + except Exception:
630 + return 0
631 +
632 +
633 +def iso_now() -> str:
634 + return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
635 +
636 +
637 +def main(argv: list[str] | None = None) -> int:
638 + parser = argparse.ArgumentParser(description="Observe the Agent Zero persistent Linux Desktop state.")
639 + subparsers = parser.add_subparsers(dest="command")
640 +
641 + state_parser = subparsers.add_parser("state")
642 + state_parser.add_argument("--json", action="store_true")
643 + state_parser.add_argument("--screenshot", action="store_true")
644 +
645 + observe_parser = subparsers.add_parser("observe")
646 + observe_parser.add_argument("--json", action="store_true")
647 + observe_parser.add_argument("--screenshot", action="store_true")
648 +
649 + screenshot_parser = subparsers.add_parser("screenshot")
650 + screenshot_parser.add_argument("path", nargs="?")
651 + screenshot_parser.add_argument("--json", action="store_true")
652 +
653 + args = parser.parse_args(argv)
654 + command = args.command or "state"
655 + if command in {"state", "observe"}:
656 + payload = collect_state(include_screenshot=bool(args.screenshot))
657 + print(json.dumps(payload, sort_keys=True))
658 + return 0 if payload.get("ok") else 1
659 +
660 + if command == "screenshot":
661 + errors: list[str] = []
662 + env_info = resolve_environment(errors=errors)
663 + payload = capture_screenshot(
664 + display_env(display=env_info["display"], profile_dir=env_info["profile_dir"]),
665 + collect_capabilities(),
666 + path=args.path,
667 + errors=errors,
668 + )
669 + if args.json:
670 + print(json.dumps(payload, sort_keys=True))
671 + else:
672 + print(payload.get("path") or payload.get("error") or "")
673 + return 0 if payload.get("ok") else 1
674 +
675 + parser.print_help()
676 + return 2
677 +
678 +
679 +if __name__ == "__main__":
680 + raise SystemExit(main())
plugins/_office/helpers/libreoffice_desktop.py
+7 -1
@@ -17,7 +17,7 @@ from pathlib import Path
17 from typing import Any
18
19 from helpers import files, virtual_desktop
20 -from plugins._office.helpers import document_store, libreoffice
20 +from plugins._office.helpers import desktop_state, document_store, libreoffice
21
22
23 OFFICIAL_EXTENSIONS = {"odt", "ods", "odp", "docx", "xlsx", "pptx"}
@@ -213,6 +213,11 @@ class LibreOfficeDesktopManager:
213 "url_intents": url_intents,
214 }
215
216 + def state(self, *, include_screenshot: bool = False) -> dict[str, Any]:
217 + with self._lock:
218 + self._reap_dead_locked()
219 + return desktop_state.collect_state(include_screenshot=include_screenshot)
220 +
221 def claim_url_intents(self, session_id: str = SYSTEM_SESSION_ID) -> list[dict[str, Any]]:
222 session = self.get(session_id) or self.get(SYSTEM_SESSION_ID)
223 if not session:
@@ -1069,6 +1074,7 @@ fi
1074 "path": session.path,
1075 "display": session.display,
1076 "xpra_port": session.xpra_port,
1077 + "profile_dir": str(session.profile_dir),
1078 "owner_pid": os.getpid(),
1079 "pids": {name: process.pid for name, process in session.processes.items()},
1080 }
plugins/_office/hooks.py
+4
@@ -60,9 +60,13 @@ RUNTIME_PACKAGES = (
60 "libglib2.0-bin",
61 "xfce4-terminal",
62 "x11-xserver-utils",
63 + "x11-utils",
64 + "x11-apps",
65 "xdotool",
66 + "xclip",
67 "xauth",
68 "dbus-x11",
69 + "python3-pil",
70 "fonts-dejavu",
71 "fonts-liberation",
72 "fonts-crosextra-caladea",
plugins/_office/skills/linux-desktop/SKILL.md
+43 -10
@@ -23,14 +23,24 @@ Use the Desktop as a full Linux GUI when the user explicitly needs a visual work
23
24 ## Operating Model
25
26 -1. Prefer `document_artifact` for creating, reading, and editing Markdown, ODT, ODS, ODP, DOCX, XLSX, and PPTX files.
27 -2. Treat Markdown as first-class. For writing, notes, reports, and drafts with no explicit binary Office requirement, create Markdown and use the custom Markdown editor when the user opens the canvas.
28 -3. Treat ODF as first-class for LibreOffice office work: ODT in Writer, ODS in Calc, ODP in Impress. Use DOCX/XLSX/PPTX only for explicit Microsoft compatibility.
29 -4. Use the Desktop only when the user asks for the Desktop, a GUI app, binary Office visual work, or visual confirmation.
30 -5. Never open the Desktop/canvas automatically from a tool result if the user has not opened it. Offer the explicit Open in canvas action instead.
31 -6. Launch common apps from the Desktop icons, the header buttons, or `scripts/desktopctl.sh`.
32 -7. Use the external Agent Zero Browser for web browsing. Do not launch an operating-system browser in this version.
33 -8. Verify GUI work by observing the desktop state, checking window titles, and saving the file before reporting success.
26 +The Desktop is an observe-act-verify control surface. Use this decision hierarchy:
27 +
28 +1. Prefer structured tools such as `document_artifact` for deterministic file creation, reads, and edits.
29 +2. Prefer app-native helpers for visible live edits, such as `desktopctl.sh calc-set-cell` for Calc/UNO spreadsheet changes.
30 +3. Prefer launcher commands, window focus, keyboard shortcuts, menus, paste, and save commands.
31 +4. Use coordinate clicks only as a last resort, and only after a fresh Desktop observation.
32 +5. After any GUI action, verify through Desktop state, active window titles, screenshots, saved file state, or exported output.
33 +6. For terminal or CLI-agent work, verify against a fresh final `observe --json --screenshot` captured after the command has finished or visibly returned to an input prompt. Do not report from an earlier screenshot path.
34 +
35 +Keep these standing rules:
36 +
37 +1. Treat Markdown as first-class. For writing, notes, reports, and drafts with no explicit binary Office requirement, create Markdown and use the custom Markdown editor when the user opens the canvas.
38 +2. Treat ODF as first-class for LibreOffice office work: ODT in Writer, ODS in Calc, ODP in Impress. Use DOCX/XLSX/PPTX only for explicit Microsoft compatibility.
39 +3. Use the Desktop only when the user asks for the Desktop, a GUI app, binary Office visual work, or visual confirmation.
40 +4. Never open the Desktop/canvas automatically from a tool result if the user has not opened it. Offer the explicit Open in canvas action instead.
41 +5. Launch common apps from the Desktop icons, the header buttons, or `scripts/desktopctl.sh`.
42 +6. Use the external Agent Zero Browser for web browsing. Do not launch an operating-system browser in this version.
43 +7. Verify GUI work by observing the desktop state, checking window titles, and saving the file before reporting success. If exact terminal text matters, load or inspect the screenshot path returned by the final observation, not a screenshot captured before the text appeared.
44
45 ## Control Flow
46
@@ -38,7 +48,10 @@ Use the helper script when the Desktop is already open and you need reliable app
48
49 ```bash
50 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh check
51 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh state --json
52 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh observe --json --screenshot
53 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh launch calc
54 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh wait-window LibreOffice
55 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh windows LibreOffice
56 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh focus LibreOffice
57 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh key ctrl+s
@@ -55,6 +68,9 @@ plugins/_office/skills/linux-desktop/scripts/desktopctl.sh launch impress
68 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh launch terminal
69 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh launch settings
70 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh open-path /a0/usr/workdir
71 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh focus "LibreOffice"
72 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh paste-text "Text to insert"
73 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh key ctrl+s
74 ```
75
76 For live spreadsheet coworking, use the Calc helper instead of hand-written UNO snippets:
@@ -65,13 +81,17 @@ plugins/_office/skills/linux-desktop/scripts/desktopctl.sh calc-set-cell /a0/usr
81
82 This opens the workbook in the visible Desktop Calc session if needed, changes the cell through LibreOffice, saves the workbook, and verifies the `.xlsx` on disk. Because the edit happens through the running LibreOffice session, the user can see the sheet update without refreshing the Desktop surface.
83
68 -For coordinate actions after observing the Desktop:
84 +For coordinate actions, clicks are explicitly last resort. First try `launch`, `open-path`, `wait-window`, `focus`, `key`, `paste-text`, `save`, or an app-native helper. If a coordinate action is still necessary, base it on a fresh screenshot observation and verify immediately afterward:
85
86 ```bash
87 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh observe --json --screenshot
88 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh click 120 180
89 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh dblclick 120 180
90 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh right-click 120 180
91 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh drag 120 180 400 180
92 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh scroll down 3
93 plugins/_office/skills/linux-desktop/scripts/desktopctl.sh type "Text to enter"
74 -plugins/_office/skills/linux-desktop/scripts/desktopctl.sh location
94 +plugins/_office/skills/linux-desktop/scripts/desktopctl.sh observe --json
95 ```
96
97 When browser automation is available, the higher-level QA flow is:
@@ -82,6 +102,19 @@ When browser automation is available, the higher-level QA flow is:
102 4. Cross-check with `desktopctl.sh location` and `desktopctl.sh windows PATTERN`.
103 5. Capture the browser screenshot as visual evidence.
104
105 +## Terminal And CLI Agent Verification
106 +
107 +Terminal apps are visual state, not structured logs. When the task depends on exact terminal output, follow this stricter loop:
108 +
109 +1. Run `desktopctl.sh observe --json --screenshot` immediately before acting to record the starting window and screenshot path.
110 +2. Use `focus`, `paste-text` or `type`, and `key Return` to drive the terminal. Prefer CLI-native commands and keyboard input over clicks.
111 +3. Wait until the CLI has visibly produced a response or returned to an input prompt.
112 +4. Run a new final `desktopctl.sh observe --json --screenshot`.
113 +5. Verify exact text only from the screenshot path returned by that final observation, or from a newer screenshot. Never use an earlier screenshot path as final evidence.
114 +6. If the final screenshot is cropped, stale, or unreadable, capture another screenshot or report the result as unverified with that specific reason.
115 +
116 +For nested CLI agents, a successful proof requires both the input prompt and the nested agent's visible response in the final screenshot, or another deterministic saved transcript produced by the CLI itself.
117 +
118 ## Desktop Locations
119
120 The Desktop exposes stable folders for common user work:
plugins/_office/skills/linux-desktop/scripts/desktopctl.sh
+227
@@ -6,6 +6,8 @@ BASE_DIR="${A0_BASE_DIR:-/a0}"
6 PROFILE_DIR="${A0_DESKTOP_PROFILE:-$BASE_DIR/tmp/_office/desktop/profiles/$SESSION}"
7 MANIFEST="${A0_DESKTOP_MANIFEST:-$BASE_DIR/tmp/_office/desktop/sessions/$SESSION.json}"
8 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
9 +DESKTOP_STATE_HELPER="$SCRIPT_DIR/../../../helpers/desktop_state.py"
10 +DESKTOP_STATE_PYTHON="${A0_DESKTOP_STATE_PYTHON:-$(command -v /usr/bin/python3 || command -v python3 || true)}"
11
12 display_from_manifest() {
13 if [ ! -f "$MANIFEST" ] || ! command -v python3 >/dev/null 2>&1; then
@@ -34,6 +36,10 @@ esac
36
37 export XAUTHORITY="${A0_DESKTOP_XAUTHORITY:-$PROFILE_DIR/.Xauthority}"
38 export HOME="${A0_DESKTOP_HOME:-$PROFILE_DIR}"
39 +export A0_DESKTOP_SESSION="$SESSION"
40 +export A0_DESKTOP_MANIFEST="$MANIFEST"
41 +export A0_DESKTOP_PROFILE="$PROFILE_DIR"
42 +export A0_DESKTOP_DISPLAY="$DISPLAY"
43 export XDG_CONFIG_HOME="${XDG_CONFIG_HOME:-$HOME/.config}"
44 export XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}"
45 export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$HOME/.cache}"
@@ -49,9 +55,21 @@ Usage: desktopctl.sh <command> [args]
55 Commands:
56 env Print the X11 environment used for the Desktop.
57 check Verify that xdotool can reach the Desktop display.
58 + state --json Return structured Desktop state as JSON.
59 + observe --json [--screenshot]
60 + Return structured state, optionally with a fresh screenshot.
61 + screenshot [PATH] Capture the Desktop to PATH, or to the default screenshot directory.
62 + active-window Print the active window name.
63 + geometry PATTERN Print the first matching visible window geometry.
64 + wait-window PATTERN Wait for a visible matching window and print its id.
65 location Print the current X pointer location.
66 windows [PATTERN] List visible window names matching PATTERN.
67 focus PATTERN Focus the first visible window matching PATTERN.
68 + scroll DIRECTION [UNITS] Scroll up, down, left, or right; UNITS defaults to 5 clicks.
69 + drag X1 Y1 X2 Y2 Drag from X1,Y1 to X2,Y2 in Desktop coordinates.
70 + right-click X Y Move and right-click at X,Y in Desktop coordinates.
71 + paste-text TEXT Put TEXT on the Desktop clipboard and paste it with an app-native shortcut.
72 + sequence FILE|- Run a newline-delimited command sequence.
73 key KEY... Send one or more xdotool key names.
74 type TEXT Type text into the focused window.
75 click X Y Move and click at X,Y in Desktop coordinates.
@@ -79,6 +97,14 @@ ensure_display() {
97 fi
98 }
99
100 +desktop_state() {
101 + if [ ! -f "$DESKTOP_STATE_HELPER" ]; then
102 + echo "Desktop state helper not found: $DESKTOP_STATE_HELPER" >&2
103 + exit 2
104 + fi
105 + "$DESKTOP_STATE_PYTHON" "$DESKTOP_STATE_HELPER" "$@"
106 +}
107 +
108 run_detached() {
109 ( "$@" >/tmp/a0-desktopctl.log 2>&1 & )
110 }
@@ -98,6 +124,128 @@ first_window() {
124 xdotool search --onlyvisible --name "$pattern" 2>/dev/null | head -n 1 || true
125 }
126
127 +active_window_id() {
128 + xdotool getactivewindow 2>/dev/null || true
129 +}
130 +
131 +active_window_class() {
132 + window_id="$(active_window_id)"
133 + if [ -z "$window_id" ]; then
134 + return 0
135 + fi
136 + if command -v xprop >/dev/null 2>&1; then
137 + xprop -id "$window_id" WM_CLASS 2>/dev/null | awk -F'"' '/WM_CLASS/ { print $(NF - 1); exit }'
138 + fi
139 +}
140 +
141 +active_window_class_lower() {
142 + active_window_class | tr '[:upper:]' '[:lower:]'
143 +}
144 +
145 +active_window_is_terminal() {
146 + window_class="$(active_window_class_lower)"
147 + case "$window_class" in
148 + *terminal*|xterm|uxterm|rxvt|urxvt|kitty|alacritty|wezterm|konsole)
149 + return 0
150 + ;;
151 + *)
152 + return 1
153 + ;;
154 + esac
155 +}
156 +
157 +paste_key_for_active_window() {
158 + printf '%s\n' "${A0_DESKTOP_PASTE_KEY:-ctrl+v}"
159 +}
160 +
161 +window_geometry() {
162 + window_id="$1"
163 + if command -v xwininfo >/dev/null 2>&1; then
164 + xwininfo -id "$window_id" 2>/dev/null | awk '
165 + /Absolute upper-left X:/ { x=$4 }
166 + /Absolute upper-left Y:/ { y=$4 }
167 + /Width:/ { w=$2 }
168 + /Height:/ { h=$2 }
169 + END { if (w != "") printf "X=%s\nY=%s\nWIDTH=%s\nHEIGHT=%s\n", x, y, w, h }
170 + '
171 + else
172 + xdotool getwindowgeometry --shell "$window_id"
173 + fi
174 +}
175 +
176 +wait_window() {
177 + pattern="$1"
178 + timeout="${2:-15}"
179 + end=$((SECONDS + timeout))
180 + while [ "$SECONDS" -le "$end" ]; do
181 + window_id="$(first_window "$pattern")"
182 + if [ -n "$window_id" ]; then
183 + printf '%s\n' "$window_id"
184 + return 0
185 + fi
186 + sleep 0.25
187 + done
188 + echo "Timed out waiting for visible window: $pattern" >&2
189 + return 1
190 +}
191 +
192 +scroll_desktop() {
193 + direction="$1"
194 + units="${2:-5}"
195 + case "$direction" in
196 + up) button=4 ;;
197 + down) button=5 ;;
198 + left) button=6 ;;
199 + right) button=7 ;;
200 + *)
201 + echo "scroll direction must be up, down, left, or right." >&2
202 + exit 2
203 + ;;
204 + esac
205 + xdotool click --repeat "$units" "$button"
206 +}
207 +
208 +paste_text() {
209 + text="$*"
210 + if active_window_is_terminal; then
211 + xdotool type --delay "${A0_DESKTOP_PASTE_TYPE_DELAY_MS:-${A0_DESKTOP_TYPE_DELAY_MS:-4}}" -- "$text"
212 + return
213 + fi
214 + if command -v xclip >/dev/null 2>&1; then
215 + printf '%s' "$text" | xclip -selection clipboard
216 + xdotool key --clearmodifiers "$(paste_key_for_active_window)"
217 + return
218 + fi
219 + xdotool type --delay "${A0_DESKTOP_TYPE_DELAY_MS:-1}" -- "$text"
220 +}
221 +
222 +run_sequence_line() {
223 + line="$1"
224 + [ -z "$line" ] && return 0
225 + case "$line" in
226 + \#*) return 0 ;;
227 + esac
228 + # shellcheck disable=SC2086
229 + "$0" $line
230 +}
231 +
232 +run_sequence() {
233 + source_file="$1"
234 + if [ "$source_file" = "-" ]; then
235 + while IFS= read -r line; do
236 + run_sequence_line "$line"
237 + done
238 + return
239 + fi
240 + if [ ! -f "$source_file" ]; then
241 + echo "sequence requires an existing FILE or - for stdin." >&2
242 + exit 2
243 + fi
244 + while IFS= read -r line || [ -n "$line" ]; do
245 + run_sequence_line "$line"
246 + done < "$source_file"
247 +}
248 +
249 launch_app() {
250 app="${1:-}"
251 soffice="${SOFFICE:-$(command -v soffice || true)}"
@@ -142,6 +290,55 @@ case "$command_name" in
290 ensure_display
291 xdotool getmouselocation --shell
292 ;;
293 + state)
294 + if [ "${1:-}" != "--json" ]; then
295 + echo "state currently requires --json." >&2
296 + exit 2
297 + fi
298 + desktop_state state --json
299 + ;;
300 + observe)
301 + if [ "${1:-}" != "--json" ]; then
302 + echo "observe currently requires --json." >&2
303 + exit 2
304 + fi
305 + shift
306 + desktop_state observe --json "$@"
307 + ;;
308 + screenshot)
309 + if [ "${1:-}" = "--json" ]; then
310 + shift
311 + desktop_state screenshot --json "$@"
312 + elif [ "$#" -gt 0 ]; then
313 + desktop_state screenshot "$1"
314 + else
315 + desktop_state screenshot
316 + fi
317 + ;;
318 + active-window)
319 + ensure_display
320 + window_id="$(active_window_id)"
321 + if [ -z "$window_id" ]; then
322 + echo "No active window." >&2
323 + exit 1
324 + fi
325 + xdotool getwindowname "$window_id"
326 + ;;
327 + geometry)
328 + ensure_display
329 + pattern="${1:?geometry requires a window name pattern}"
330 + window_id="$(first_window "$pattern")"
331 + if [ -z "$window_id" ]; then
332 + echo "No visible window matched: $pattern" >&2
333 + exit 1
334 + fi
335 + window_geometry "$window_id"
336 + ;;
337 + wait-window)
338 + ensure_display
339 + pattern="${1:?wait-window requires a window name pattern}"
340 + wait_window "$pattern" "${2:-15}"
341 + ;;
342 location)
343 ensure_display
344 xdotool getmouselocation --shell
@@ -161,6 +358,36 @@ case "$command_name" in
358 fi
359 xdotool windowactivate --sync "$window_id"
360 ;;
361 + scroll)
362 + ensure_display
363 + scroll_desktop "${1:?scroll requires DIRECTION}" "${2:-5}"
364 + ;;
365 + drag)
366 + ensure_display
367 + x1="${1:?drag requires X1}"
368 + y1="${2:?drag requires Y1}"
369 + x2="${3:?drag requires X2}"
370 + y2="${4:?drag requires Y2}"
371 + xdotool mousemove --sync "$x1" "$y1" mousedown 1 mousemove --sync "$x2" "$y2" mouseup 1
372 + ;;
373 + right-click)
374 + ensure_display
375 + x="${1:?right-click requires X}"
376 + y="${2:?right-click requires Y}"
377 + xdotool mousemove --sync "$x" "$y" click 3
378 + ;;
379 + paste-text)
380 + ensure_display
381 + if [ "$#" -eq 0 ]; then
382 + echo "paste-text requires TEXT." >&2
383 + exit 2
384 + fi
385 + paste_text "$@"
386 + ;;
387 + sequence)
388 + source_file="${1:?sequence requires FILE or -}"
389 + run_sequence "$source_file"
390 + ;;
391 key)
392 ensure_display
393 if [ "$#" -eq 0 ]; then
plugins/_office/webui/office-store.js
+97 -1
@@ -228,6 +228,9 @@ const model = {
228 _desktopPrimeTimer: null,
229 _desktopPrimeAttempts: 0,
230 _desktopKeyboardActive: false,
231 + _desktopBridgeReady: false,
232 + _desktopKeyboardCaptureState: { ready: false, active: false, capture: false, focused: false },
233 + _desktopLastState: null,
234 _desktopKeyboardCleanup: null,
235 _desktopClipboardCleanup: null,
236 _desktopStarting: null,
@@ -1007,7 +1010,9 @@ const model = {
1010 } catch {
1011 target.focus?.({ preventScroll: true });
1012 }
1010 - return Boolean(document.activeElement === target || target.contentDocument?.hasFocus?.());
1013 + const focused = Boolean(document.activeElement === target || target.contentDocument?.hasFocus?.());
1014 + this.updateDesktopKeyboardCaptureState(target);
1015 + return focused;
1016 },
1017
1018 updateDesktopMonitor() {
@@ -1015,6 +1020,8 @@ const model = {
1020 this.stopDesktopMonitor();
1021 this.stopDesktopResizeObserver();
1022 this._desktopKeyboardActive = false;
1023 + this._desktopBridgeReady = false;
1024 + this.updateDesktopKeyboardCaptureState();
1025 return;
1026 }
1027 const sessionId = this.session?.desktop_session_id || this.session?.session_id || "";
@@ -1216,6 +1223,7 @@ const model = {
1223 this.installXpraDesktopWheelBridge(remoteWindow, xpraWindow);
1224 if (requestRefresh && xpraWindow.wid != null) client.request_refresh?.(xpraWindow.wid);
1225 }
1226 + this.installXpraDesktopAgentBridge(frame, remoteWindow, remoteDocument, client, container);
1227 return true;
1228 } catch (error) {
1229 console.warn("Xpra desktop viewport prime skipped", error);
@@ -1223,6 +1231,94 @@ const model = {
1231 }
1232 },
1233
1234 + installXpraDesktopAgentBridge(frame, remoteWindow, remoteDocument, client, container) {
1235 + if (!frame || !remoteWindow || !remoteDocument || !client) return null;
1236 + const store = this;
1237 + const finite = (value, fallback = 0) => {
1238 + const number = Number(value);
1239 + return Number.isFinite(number) ? number : fallback;
1240 + };
1241 + const metrics = () => {
1242 + const desktopWidth = Math.max(1, finite(client.desktop_width || container?.clientWidth || remoteWindow.innerWidth, 1));
1243 + const desktopHeight = Math.max(1, finite(client.desktop_height || container?.clientHeight || remoteWindow.innerHeight, 1));
1244 + const clientWidth = Math.max(1, finite(container?.clientWidth || remoteWindow.innerWidth, desktopWidth));
1245 + const clientHeight = Math.max(1, finite(container?.clientHeight || remoteWindow.innerHeight, desktopHeight));
1246 + return {
1247 + desktopWidth,
1248 + desktopHeight,
1249 + clientWidth,
1250 + clientHeight,
1251 + scaleX: clientWidth / desktopWidth,
1252 + scaleY: clientHeight / desktopHeight,
1253 + };
1254 + };
1255 + const bridge = frame.__agentZeroDesktopBridge || {};
1256 + Object.assign(bridge, {
1257 + ready: true,
1258 + state: async (options = {}) => {
1259 + const result = await callOffice("desktop_state", {
1260 + include_screenshot: options.includeScreenshot === true || options.include_screenshot === true,
1261 + });
1262 + store._desktopLastState = result;
1263 + return result;
1264 + },
1265 + focus: (options = {}) => store.focusDesktopFrame(frame, { ...options, arm: options.arm !== false }),
1266 + requestRefresh: () => {
1267 + for (const xpraWindow of Object.values(client.id_to_window || {})) {
1268 + if (xpraWindow?.wid != null) client.request_refresh?.(xpraWindow.wid);
1269 + }
1270 + return true;
1271 + },
1272 + desktopToClient: (x, y) => {
1273 + const value = metrics();
1274 + return {
1275 + x: Math.round(finite(x) * value.scaleX),
1276 + y: Math.round(finite(y) * value.scaleY),
1277 + scale_x: value.scaleX,
1278 + scale_y: value.scaleY,
1279 + };
1280 + },
1281 + clientToDesktop: (x, y) => {
1282 + const value = metrics();
1283 + return {
1284 + x: Math.round(finite(x) / value.scaleX),
1285 + y: Math.round(finite(y) / value.scaleY),
1286 + scale_x: value.scaleX,
1287 + scale_y: value.scaleY,
1288 + };
1289 + },
1290 + diagnostics: () => store.desktopBridgeDiagnostics(frame),
1291 + });
1292 + frame.agentZeroDesktop = bridge;
1293 + frame.__agentZeroDesktopBridge = bridge;
1294 + remoteWindow.agentZeroDesktop = bridge;
1295 + remoteWindow.__agentZeroDesktopBridge = bridge;
1296 + this._desktopBridgeReady = true;
1297 + this.updateDesktopKeyboardCaptureState(frame);
1298 + return bridge;
1299 + },
1300 +
1301 + desktopBridgeDiagnostics(frame = null) {
1302 + return {
1303 + ready: this._desktopBridgeReady,
1304 + keyboard: this.updateDesktopKeyboardCaptureState(frame),
1305 + lastStateOk: this._desktopLastState?.ok ?? null,
1306 + };
1307 + },
1308 +
1309 + updateDesktopKeyboardCaptureState(frame = null) {
1310 + const target = this.desktopFrame(frame);
1311 + const client = target?.contentWindow?.client;
1312 + const state = {
1313 + ready: Boolean(target?.__agentZeroDesktopBridge || target?.contentWindow?.__agentZeroDesktopBridge),
1314 + active: Boolean(this._desktopKeyboardActive),
1315 + capture: Boolean(client?.capture_keyboard),
1316 + focused: Boolean(target && (document.activeElement === target || target.contentDocument?.hasFocus?.())),
1317 + };
1318 + this._desktopKeyboardCaptureState = state;
1319 + return state;
1320 + },
1321 +
1322 normalizeXpraDesktopWindow(xpraWindow, width, height) {
1323 if (!xpraWindow) return;
1324 const normalizedWidth = Math.max(1, Math.round(Number(width || 0)));
tests/test_office_canvas_setup.py
+62
@@ -72,6 +72,14 @@ def test_document_canvas_uses_markdown_editor_and_official_libreoffice_desktop_f
72 assert "primeXpraDesktopFrame" in store
73 assert "normalizeXpraDesktopWindow" in store
74 assert "installXpraDesktopWheelBridge" in store
75 + assert "installXpraDesktopAgentBridge" in store
76 + assert "agentZeroDesktop" in store
77 + assert 'callOffice("desktop_state"' in store
78 + assert "desktopToClient" in store
79 + assert "clientToDesktop" in store
80 + assert "requestRefresh" in store
81 + assert "_desktopBridgeReady" in store
82 + assert "_desktopKeyboardCaptureState" in store
83 assert "installXpraDesktopKeyboardBridge" in store
84 assert "focusDesktopFrame" in store
85 assert "_desktopKeyboardActive" in store
@@ -227,6 +235,10 @@ def test_official_libreoffice_desktop_route_and_packages_are_declared():
235 linux_desktopctl = (
236 PROJECT_ROOT / "plugins" / "_office" / "skills" / "linux-desktop" / "scripts" / "desktopctl.sh"
237 ).read_text(encoding="utf-8")
238 + desktop_state_helper = (
239 + PROJECT_ROOT / "plugins" / "_office" / "helpers" / "desktop_state.py"
240 + ).read_text(encoding="utf-8")
241 + hooks_py = (PROJECT_ROOT / "plugins" / "_office" / "hooks.py").read_text(encoding="utf-8")
242 linux_calc_helper = (
243 PROJECT_ROOT / "plugins" / "_office" / "skills" / "linux-desktop" / "scripts" / "calc_set_cell.py"
244 ).read_text(encoding="utf-8")
@@ -320,10 +332,42 @@ def test_official_libreoffice_desktop_route_and_packages_are_declared():
332 assert "/a0/usr/projects" in linux_desktop_skill
333 assert "desktopctl.sh" in linux_desktop_skill
334 assert "calc-set-cell" in linux_desktop_skill
335 + assert "Clicks are explicitly last resort" in linux_desktop_skill or "clicks are explicitly last resort" in linux_desktop_skill
336 + assert "fresh Desktop observation" in linux_desktop_skill
337 + assert "observe --json --screenshot" in linux_desktop_skill
338 + assert "Terminal And CLI Agent Verification" in linux_desktop_skill
339 + assert "Do not report from an earlier screenshot path" in linux_desktop_skill
340 + assert "screenshot path returned by that final observation" in linux_desktop_skill
341 assert "xdotool" in linux_desktopctl
342 assert "agent-zero-desktop" in linux_desktopctl
343 assert "launch_app" in linux_desktopctl
344 + assert "paste_key_for_active_window" in linux_desktopctl
345 + assert "active_window_is_terminal" in linux_desktopctl
346 + assert "WM_CLASS" in linux_desktopctl
347 + for command in (
348 + "state)",
349 + "observe)",
350 + "screenshot)",
351 + "active-window)",
352 + "geometry)",
353 + "wait-window)",
354 + "scroll)",
355 + "drag)",
356 + "right-click)",
357 + "paste-text)",
358 + "sequence)",
359 + ):
360 + assert command in linux_desktopctl
361 assert "calc_set_cell.py" in linux_desktopctl
362 + assert "collect_state" in desktop_state_helper
363 + assert "compact_prompt_context" in desktop_state_helper
364 + assert "fresh final" in desktop_state_helper
365 + assert "xwd" in desktop_state_helper
366 + assert "PIL" in desktop_state_helper
367 + assert '"x11-utils"' in hooks_py
368 + assert '"x11-apps"' in hooks_py
369 + assert '"xclip"' in hooks_py
370 + assert '"python3-pil"' in hooks_py
371 assert "wait_for_document" in linux_calc_helper
372 assert "document.store()" in linux_calc_helper
373 assert "read_xlsx_cell" in linux_calc_helper
@@ -419,6 +463,8 @@ def test_office_skills_preserve_markdown_first_and_opt_in_desktop_policy():
463 assert "Download and Open in canvas actions" in office_skill
464 assert "method: \"create\"" in office_skill
465 assert "The Desktop is opt-in" in desktop_skill
466 + assert "coordinate clicks only as a last resort" in desktop_skill
467 + assert "After any GUI action, verify" in desktop_skill
468 assert "custom Markdown editor" in desktop_skill
469 assert "Never open the Desktop/canvas automatically" in desktop_skill
470 assert "persistent Desktop runtime during initial startup" in desktop_skill
@@ -432,3 +478,19 @@ def test_office_skills_preserve_markdown_first_and_opt_in_desktop_policy():
478 assert "must not open the canvas automatically" in excel_skill
479 assert '"format": "odp"' in presentation_skill
480 assert "must not open the canvas automatically" in presentation_skill
481 +
482 +
483 +def test_office_extra_prompt_includes_existing_desktop_state_without_opening_canvas():
484 + canvas_context = (
485 + PROJECT_ROOT / "plugins" / "_office" / "helpers" / "canvas_context.py"
486 + ).read_text(encoding="utf-8")
487 + prompt = (
488 + PROJECT_ROOT / "plugins" / "_office" / "prompts" / "agent.extras.office_canvas.md"
489 + ).read_text(encoding="utf-8")
490 +
491 + assert "build_desktop_context" in canvas_context
492 + assert "session_manifest_exists" in canvas_context
493 + assert "collect_state(include_screenshot=False)" in canvas_context
494 + assert "compact_prompt_context" in canvas_context
495 + assert "ensure_system_desktop" not in canvas_context
496 + assert "[DOCUMENT CANVAS]" in prompt
tests/test_office_desktop_state.py new
+201
@@ -0,0 +1,201 @@
1 +from __future__ import annotations
2 +
3 +import subprocess
4 +import struct
5 +import sys
6 +import types
7 +from pathlib import Path
8 +
9 +PROJECT_ROOT = Path(__file__).resolve().parents[1]
10 +if str(PROJECT_ROOT) not in sys.path:
11 + sys.path.insert(0, str(PROJECT_ROOT))
12 +
13 +from plugins._office.helpers import desktop_state
14 +
15 +
16 +def _completed(command, returncode=0, stdout="", stderr=""):
17 + return subprocess.CompletedProcess(command, returncode, stdout, stderr)
18 +
19 +
20 +def test_desktop_state_collects_x11_state_from_mocked_tools(tmp_path, monkeypatch):
21 + session_dir = tmp_path / "sessions"
22 + profile_dir = tmp_path / "profiles" / desktop_state.SESSION_ID
23 + session_dir.mkdir(parents=True)
24 + profile_dir.mkdir(parents=True)
25 + (session_dir / f"{desktop_state.SESSION_ID}.json").write_text(
26 + '{"display": 120, "profile_dir": "%s"}' % profile_dir,
27 + encoding="utf-8",
28 + )
29 +
30 + monkeypatch.setattr(desktop_state, "SESSION_DIR", session_dir)
31 + monkeypatch.setattr(desktop_state, "PROFILE_DIR", tmp_path / "profiles")
32 + monkeypatch.setattr(desktop_state, "SCREENSHOT_DIR", tmp_path / "screenshots")
33 + monkeypatch.setattr(
34 + desktop_state.shutil,
35 + "which",
36 + lambda name: f"/usr/bin/{name}"
37 + if name in {"xdotool", "xrandr", "xwininfo", "xprop", "xwd", "xclip"}
38 + else "",
39 + )
40 +
41 + def fake_run(command, **kwargs):
42 + name = Path(command[0]).name
43 + if name == "xrandr":
44 + return _completed(command, stdout="Screen 0: current 1440 x 900, maximum 1920 x 1080\n")
45 + if name == "xdotool" and command[1:3] == ["getmouselocation", "--shell"]:
46 + return _completed(command, stdout="X=12\nY=34\nSCREEN=0\nWINDOW=111\n")
47 + if name == "xdotool" and command[1] == "getactivewindow":
48 + return _completed(command, stdout="111\n")
49 + if name == "xdotool" and command[1] == "search":
50 + return _completed(command, stdout="111\n222\n")
51 + if name == "xdotool" and command[1] == "getwindowname":
52 + return _completed(command, stdout={"111": "LibreOffice Calc", "222": "Terminal"}[command[2]] + "\n")
53 + if name == "xwininfo":
54 + geometry = {
55 + "111": (5, 7, 800, 600),
56 + "222": (20, 30, 640, 480),
57 + }[command[2]]
58 + return _completed(
59 + command,
60 + stdout=(
61 + f" Absolute upper-left X: {geometry[0]}\n"
62 + f" Absolute upper-left Y: {geometry[1]}\n"
63 + f" Width: {geometry[2]}\n"
64 + f" Height: {geometry[3]}\n"
65 + ),
66 + )
67 + if name == "xprop":
68 + window_id = command[2]
69 + if window_id == "111":
70 + return _completed(
71 + command,
72 + stdout='WM_CLASS(STRING) = "libreoffice", "libreoffice-calc"\n_NET_WM_PID(CARDINAL) = 4242\n',
73 + )
74 + return _completed(
75 + command,
76 + stdout='WM_CLASS(STRING) = "xfce4-terminal", "Xfce4-terminal"\n_NET_WM_PID(CARDINAL) = 4343\n',
77 + )
78 + raise AssertionError(f"unexpected command: {command}")
79 +
80 + monkeypatch.setattr(desktop_state.subprocess, "run", fake_run)
81 +
82 + state = desktop_state.collect_state()
83 +
84 + assert state["ok"] is True
85 + assert state["display"] == ":120"
86 + assert state["profile_dir"] == str(profile_dir)
87 + assert state["size"] == {"width": 1440, "height": 900}
88 + assert state["pointer"]["x"] == 12
89 + assert state["active_window"]["title"] == "LibreOffice Calc"
90 + assert state["active_window"]["class"] == "libreoffice-calc"
91 + assert state["active_window"]["geometry"]["width"] == 800
92 + assert [window["title"] for window in state["windows"]] == ["LibreOffice Calc", "Terminal"]
93 +
94 +
95 +def test_desktop_state_screenshot_capture_uses_xwd_and_pillow_when_available(tmp_path, monkeypatch):
96 + monkeypatch.setattr(desktop_state, "SCREENSHOT_DIR", tmp_path)
97 + capabilities = {"xwd": "/usr/bin/xwd"}
98 + env = {"DISPLAY": ":120"}
99 +
100 + def fake_run(command, *, env, timeout):
101 + raw_path = Path(command[command.index("-out") + 1])
102 + raw_path.write_bytes(b"xwd")
103 + return _completed(command)
104 +
105 + image_module = types.ModuleType("PIL.Image")
106 +
107 + class FakeImage:
108 + width = 320
109 + height = 240
110 +
111 + def __enter__(self):
112 + return self
113 +
114 + def __exit__(self, *_args):
115 + return False
116 +
117 + def save(self, target):
118 + Path(target).write_bytes(b"png")
119 +
120 + image_module.open = lambda _path: FakeImage()
121 + pil_module = types.ModuleType("PIL")
122 + pil_module.Image = image_module
123 +
124 + monkeypatch.setattr(desktop_state, "run", fake_run)
125 + monkeypatch.setitem(sys.modules, "PIL", pil_module)
126 + monkeypatch.setitem(sys.modules, "PIL.Image", image_module)
127 +
128 + screenshot = desktop_state.capture_screenshot(env, capabilities, path=tmp_path / "shot.png", errors=[])
129 +
130 + assert screenshot["ok"] is True
131 + assert screenshot["path"] == str(tmp_path / "shot.png")
132 + assert screenshot["format"] == "png"
133 + assert (tmp_path / "shot.png").read_bytes() == b"png"
134 + assert not (tmp_path / "shot.xwd").exists()
135 +
136 +
137 +def test_xwd_fallback_parser_handles_truecolor_pixels(tmp_path, monkeypatch):
138 + raw_path = tmp_path / "shot.xwd"
139 + target = tmp_path / "shot.png"
140 + header_values = [
141 + 100, # header_size
142 + 7, # file_version
143 + 2, # pixmap_format
144 + 24, # pixmap_depth
145 + 2, # pixmap_width
146 + 1, # pixmap_height
147 + 0, # xoffset
148 + 1, # byte_order: MSBFirst for pixel bytes
149 + 32, # bitmap_unit
150 + 1, # bitmap_bit_order
151 + 32, # bitmap_pad
152 + 32, # bits_per_pixel
153 + 8, # bytes_per_line
154 + 4, # visual_class: TrueColor
155 + 0x00FF0000, # red_mask
156 + 0x0000FF00, # green_mask
157 + 0x000000FF, # blue_mask
158 + 8, # bits_per_rgb
159 + 256, # colormap_entries
160 + 0, # ncolors
161 + 2, # window_width
162 + 1, # window_height
163 + 0, # window_x
164 + 0, # window_y
165 + 0, # window_bdrwidth
166 + ]
167 + raw_path.write_bytes(
168 + struct.pack(">25I", *header_values)
169 + + bytes.fromhex("00ff0000")
170 + + bytes.fromhex("0000ff00")
171 + )
172 +
173 + captured: dict[str, object] = {}
174 + image_module = types.ModuleType("PIL.Image")
175 +
176 + class FakeOutputImage:
177 + def putdata(self, pixels):
178 + captured["pixels"] = list(pixels)
179 +
180 + def save(self, path):
181 + Path(path).write_bytes(b"fallback-png")
182 +
183 + def fake_new(mode, size):
184 + captured["mode"] = mode
185 + captured["size"] = size
186 + return FakeOutputImage()
187 +
188 + image_module.new = fake_new
189 + pil_module = types.ModuleType("PIL")
190 + pil_module.Image = image_module
191 +
192 + monkeypatch.setitem(sys.modules, "PIL", pil_module)
193 + monkeypatch.setitem(sys.modules, "PIL.Image", image_module)
194 +
195 + converted = desktop_state.convert_xwd_to_image(raw_path, target)
196 +
197 + assert converted == {"width": 2, "height": 1}
198 + assert captured["mode"] == "RGB"
199 + assert captured["size"] == (2, 1)
200 + assert captured["pixels"] == [(255, 0, 0), (0, 255, 0)]
201 + assert target.read_bytes() == b"fallback-png"
tests/test_office_document_store.py
+51
@@ -503,6 +503,57 @@ def test_official_libreoffice_desktop_status_and_url_contract(tmp_path, monkeypa
503 assert "printing=true" in url
504
505
506 +def test_office_session_desktop_state_action_defaults_without_screenshot(monkeypatch):
507 + api_module = types.ModuleType("helpers.api")
508 +
509 + class ApiHandler:
510 + def __init__(self, app=None, thread_lock=None):
511 + self.app = app
512 + self.thread_lock = thread_lock
513 +
514 + api_module.ApiHandler = ApiHandler
515 + api_module.Request = object
516 + monkeypatch.setitem(sys.modules, "helpers.api", api_module)
517 + monkeypatch.delitem(sys.modules, "plugins._office.api.office_session", raising=False)
518 +
519 + from plugins._office.api import office_session
520 +
521 + calls = []
522 +
523 + class FakeManager:
524 + def state(self, *, include_screenshot=False):
525 + calls.append(include_screenshot)
526 + return {
527 + "ok": True,
528 + "display": ":120",
529 + "profile_dir": "/a0/tmp/_office/desktop/profiles/agent-zero-desktop",
530 + "size": {"width": 1440, "height": 900},
531 + "pointer": {"x": 0, "y": 0, "screen": 0, "window": 0},
532 + "active_window": None,
533 + "windows": [],
534 + "screenshot": {"ok": False, "path": ""},
535 + "capabilities": {},
536 + "errors": [],
537 + }
538 +
539 + monkeypatch.setattr(office_session.libreoffice_desktop, "get_manager", lambda: FakeManager())
540 + handler = office_session.OfficeSession(app=None, thread_lock=None)
541 + request = types.SimpleNamespace(headers={}, host_url="http://localhost:32080")
542 +
543 + default_result = asyncio.run(handler.process({"action": "desktop_state"}, request))
544 + screenshot_result = asyncio.run(
545 + handler.process({"action": "desktop_state", "include_screenshot": True}, request),
546 + )
547 +
548 + assert default_result["ok"] is True
549 + assert screenshot_result["ok"] is True
550 + assert calls == [False, True]
551 + monkeypatch.delitem(sys.modules, "plugins._office.api.office_session", raising=False)
552 + api_package = sys.modules.get("plugins._office.api")
553 + if api_package is not None:
554 + monkeypatch.delattr(api_package, "office_session", raising=False)
555 +
556 +
557 def test_official_libreoffice_desktop_manager_opens_binary_session(office_state, tmp_path, monkeypatch):
558 class FakeProcess:
559 pid = 4242