@samitouri / QOSami_Agents_Zero-1 / commits / c1bdde05

Make Desktop screenshots ephemeral by default

Route in-process Xpra/Desktop screenshot observations through context-scoped ephemeral image refs with vision_load payloads, matching the privacy posture of computer-use and browser screenshots. Keep desktopctl shell observations path-based with aggressive pruning so image payloads are not printed into shell logs, and preserve explicit screenshot paths as durable user-owned artifacts.

Alessandro committed May 22, 2026 at 10:21 UTC c1bdde057c58adc04dd63ff511bd2eb5f4160015

8 files changed +222 -36

plugins/_desktop/extensions/python/message_loop_prompts_after/_55_include_desktop_state.py

+2 -1

@@ -7,7 +7,8 @@ from plugins._desktop.helpers import prompt_context

8 class IncludeDesktopState(Extension):

9 async def execute(self, loop_data: LoopData = LoopData(), **kwargs):

10 - context = prompt_context.build_context()

10 + context_id = str(getattr(getattr(self.agent, "context", None), "id", "") or "")

11 + context = prompt_context.build_context(context_id=context_id)

12 if not context:

13 loop_data.extras_temporary.pop("desktop_state", None)

14 return

plugins/_desktop/helpers/desktop_state.py

+148 -6

@@ -6,11 +6,14 @@ import os

6 import re

7 import shutil

8 import subprocess

9 +import sys

10 import time

11 from pathlib import Path

12 from typing import Any

14 PROJECT_ROOT = Path(__file__).resolve().parents[3]

15 +if str(PROJECT_ROOT) not in sys.path:

16 + sys.path.insert(0, str(PROJECT_ROOT))

19 SESSION_ID = "agent-zero-desktop"

@@ -20,9 +23,10 @@ STATE_DIR = BASE_DIR / "usr" / "plugins" / PLUGIN_NAME

23 RETIRED_STATE_DIR = BASE_DIR / "usr" / PLUGIN_NAME

24 SESSION_DIR = STATE_DIR / "sessions"

25 PROFILE_DIR = STATE_DIR / "profiles"

23 -SCREENSHOT_DIR = STATE_DIR / "screenshots"

26 +SCREENSHOT_DIR = Path(os.environ.get("A0_DESKTOP_SCREENSHOT_DIR") or BASE_DIR / "tmp" / "desktop" / "screenshots")

27 RECENT_SCREENSHOT_SECONDS = 600

28 _SAFE_CONTEXT_RE = re.compile(r"[^a-zA-Z0-9_.-]+")

29 +_SCREENSHOT_SUFFIXES = {".png", ".jpg", ".jpeg", ".xwd"}

32 def session_manifest_path(session_id: str = SESSION_ID) -> Path:

@@ -47,6 +51,7 @@ def collect_state(

51 include_screenshot: bool = False,

52 screenshot_path: str | Path | None = None,

53 context_id: str = "",

54 + screenshot_transport: str = "ephemeral",

55 ) -> dict[str, Any]:

56 errors: list[str] = []

57 env_info = resolve_environment(errors=errors)

@@ -72,9 +77,11 @@ def collect_state(

77 path=screenshot_path,

78 errors=errors,

79 context_id=context_id,

80 + transport=screenshot_transport,

81 )

83 return stable_state(

84 + context_id=context_id,

85 display=display,

86 profile_dir=profile_dir,

87 size=size,

@@ -94,6 +101,7 @@ def capture_screenshot(

101 path: str | Path | None = None,

102 errors: list[str] | None = None,

103 context_id: str = "",

104 + transport: str = "ephemeral",

105 ) -> dict[str, Any]:

106 local_errors = errors if errors is not None else []

107 capabilities = capabilities or collect_capabilities()

@@ -109,12 +117,18 @@ def capture_screenshot(

117 local_errors.append(message)

118 return {"ok": False, "path": "", "format": "", "captured_at": "", "error": message}

119

120 + explicit_path = path is not None and str(path).strip() != ""

121 + ephemeral_ref = not explicit_path and str(transport or "").strip().lower() != "path"

122 screenshot_dir = context_screenshot_dir(context_id)

113 - screenshot_dir.mkdir(parents=True, exist_ok=True)

123 + if not explicit_path:

124 + prune_context_screenshots(context_id=context_id)

125 + screenshot_dir.mkdir(parents=True, exist_ok=True)

126 timestamp = time.strftime("%Y%m%d-%H%M%S")

115 - target = Path(path) if path else screenshot_dir / f"desktop-{timestamp}.png"

127 + millis = int((time.time() % 1) * 1000)

128 + target = Path(path) if explicit_path else screenshot_dir / f"desktop-{timestamp}-{millis:03d}.png"

129 target.parent.mkdir(parents=True, exist_ok=True)

130 raw_path = target.with_suffix(".xwd")

131 + safe_context = _safe_context_id(context_id)

132

133 result = run([xwd, "-root", "-silent", "-out", str(raw_path)], env=env, timeout=8)

134 if result.returncode != 0:

@@ -124,12 +138,16 @@ def capture_screenshot(

138 return {"ok": False, "path": "", "format": "", "captured_at": "", "error": detail}

139

140 if target.suffix.lower() == ".xwd":

141 + if not explicit_path:

142 + prune_context_screenshots(context_id=context_id, keep_path=raw_path)

143 return {

144 "ok": True,

145 "path": str(raw_path),

146 "format": "xwd",

147 "captured_at": iso_now(),

148 "recent": True,

149 + "ephemeral": not explicit_path,

150 + "context_id": safe_context,

151 "error": "",

152 }

153

@@ -141,6 +159,16 @@ def capture_screenshot(

159 width = int(image.width)

160 height = int(image.height)

161 raw_path.unlink(missing_ok=True)

162 + if ephemeral_ref:

163 + return ephemeral_screenshot_result(

164 + target,

165 + context_id=context_id,

166 + image_format=target.suffix.lower().lstrip(".") or "png",

167 + width=width,

168 + height=height,

169 + )

170 + if not explicit_path:

171 + prune_context_screenshots(context_id=context_id, keep_path=target)

172 return {

173 "ok": True,

174 "path": str(target),

@@ -149,12 +177,24 @@ def capture_screenshot(

177 "height": height,

178 "captured_at": iso_now(),

179 "recent": True,

180 + "ephemeral": not explicit_path,

181 + "context_id": safe_context,

182 "error": "",

183 }

184 except Exception as exc:

185 try:

186 converted = convert_xwd_to_image(raw_path, target)

187 raw_path.unlink(missing_ok=True)

188 + if ephemeral_ref:

189 + return ephemeral_screenshot_result(

190 + target,

191 + context_id=context_id,

192 + image_format=target.suffix.lower().lstrip(".") or "png",

193 + width=converted["width"],

194 + height=converted["height"],

195 + )

196 + if not explicit_path:

197 + prune_context_screenshots(context_id=context_id, keep_path=target)

198 return {

199 "ok": True,

200 "path": str(target),

@@ -163,17 +203,34 @@ def capture_screenshot(

203 "height": converted["height"],

204 "captured_at": iso_now(),

205 "recent": True,

206 + "ephemeral": not explicit_path,

207 + "context_id": safe_context,

208 "error": "",

209 }

210 except Exception as fallback_exc:

211 message = f"Pillow could not convert the XWD screenshot: {exc}; fallback parser failed: {fallback_exc}"

212 local_errors.append(message)

213 + if ephemeral_ref:

214 + raw_path.unlink(missing_ok=True)

215 + target.unlink(missing_ok=True)

216 + return {

217 + "ok": False,

218 + "path": "",

219 + "format": "",

220 + "captured_at": iso_now(),

221 + "recent": False,

222 + "ephemeral": True,

223 + "context_id": safe_context,

224 + "error": message,

225 + }

226 return {

227 "ok": True,

228 "path": str(raw_path),

229 "format": "xwd",

230 "captured_at": iso_now(),

231 "recent": True,

232 + "ephemeral": not explicit_path,

233 + "context_id": safe_context,

234 "error": message,

235 }

236

@@ -518,17 +575,21 @@ def parse_xprop(output: str) -> dict[str, str]:

575

576

577 def latest_screenshot(*, context_id: str = "") -> dict[str, Any]:

578 + prune_context_screenshots(context_id=context_id, max_age_seconds=RECENT_SCREENSHOT_SECONDS)

579 screenshot_dir = context_screenshot_dir(context_id)

580 if not screenshot_dir.exists():

581 return {"ok": False, "path": "", "format": "", "captured_at": "", "recent": False}

582 candidates = [

583 path

584 for path in screenshot_dir.iterdir()

527 - if path.is_file() and path.suffix.lower() in {".png", ".jpg", ".jpeg", ".xwd"}

585 + if path.is_file() and path.suffix.lower() in _SCREENSHOT_SUFFIXES

586 ]

587 if not candidates:

588 return {"ok": False, "path": "", "format": "", "captured_at": "", "recent": False}

589 latest = max(candidates, key=lambda item: item.stat().st_mtime)

590 + for candidate in candidates:

591 + if candidate != latest:

592 + candidate.unlink(missing_ok=True)

593 age = max(0.0, time.time() - latest.stat().st_mtime)

594 return {

595 "ok": True,

@@ -536,6 +597,8 @@ def latest_screenshot(*, context_id: str = "") -> dict[str, Any]:

597 "format": latest.suffix.lower().lstrip("."),

598 "captured_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(latest.stat().st_mtime)),

599 "recent": age <= RECENT_SCREENSHOT_SECONDS,

600 + "ephemeral": True,

601 + "context_id": _safe_context_id(context_id),

602 }

603

604

@@ -543,6 +606,7 @@ def stable_state(

606 *,

607 display: str,

608 profile_dir: str,

609 + context_id: str = "",

610 size: dict[str, int] | None = None,

611 pointer: dict[str, int] | None = None,

612 active_window: dict[str, Any] | None = None,

@@ -554,6 +618,7 @@ def stable_state(

618 clean_errors = [str(error) for error in errors or [] if str(error)]

619 return {

620 "ok": not clean_errors,

621 + "context_id": _safe_context_id(context_id),

622 "display": display,

623 "profile_dir": profile_dir,

624 "size": size or {"width": 0, "height": 0},

@@ -594,9 +659,15 @@ def compact_prompt_context(state: dict[str, Any] | None = None) -> str:

659 lines.append("- visible=" + "; ".join(visible))

660 screenshot = state.get("screenshot") or {}

661 if screenshot.get("recent") and screenshot.get("path"):

597 - lines.append(f"- recent_screenshot={screenshot['path']}")

662 + ephemeral = " ephemeral" if screenshot.get("ephemeral") else ""

663 + lines.append(f"- recent_screenshot={screenshot['path']}{ephemeral}")

664 + context_id = str(state.get("context_id") or "").strip()

665 + if context_id:

666 + lines.append(f"- screenshot_context={context_id}")

667 + context_arg = f" --context-id {context_id}" if context_id else ""

668 lines.append(

599 - "- next=plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh observe --json --screenshot "

669 + "- next=plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh observe --json --screenshot"

670 + f"{context_arg} "

671 "before any coordinate action; prefer focus/key/paste/save/app-native helpers first."

672 )

673 lines.append(

@@ -667,6 +738,75 @@ def image_height(path: Path) -> int:

738 return 0

739

740

741 +def ephemeral_screenshot_result(

742 + path: Path,

743 + *,

744 + context_id: str = "",

745 + image_format: str = "png",

746 + width: int = 0,

747 + height: int = 0,

748 +) -> dict[str, Any]:

749 + from helpers import ephemeral_images

750 +

751 + mime = "image/jpeg" if image_format.lower() in {"jpg", "jpeg"} else "image/png"

752 + safe_context = _safe_context_id(context_id)

753 + ref = ephemeral_images.put_image_bytes(

754 + context_id=str(context_id or "").strip(),

755 + mime=mime,

756 + payload=path.read_bytes(),

757 + name=path.name,

758 + )

759 + path.unlink(missing_ok=True)

760 + prune_context_screenshots(context_id=context_id)

761 + return {

762 + "ok": True,

763 + "path": "",

764 + "format": image_format,

765 + "mime": mime,

766 + "width": width,

767 + "height": height,

768 + "captured_at": iso_now(),

769 + "recent": True,

770 + "ephemeral": True,

771 + "ephemeral_ref": ref,

772 + "context_id": safe_context,

773 + "vision_load": {

774 + "tool_name": "vision_load",

775 + "tool_args": {"paths": [ref]},

776 + },

777 + "error": "",

778 + }

779 +

780 +

781 +def prune_context_screenshots(

782 + *,

783 + context_id: str = "",

784 + keep_path: Path | None = None,

785 + max_age_seconds: float | None = None,

786 +) -> None:

787 + screenshot_dir = context_screenshot_dir(context_id)

788 + if not screenshot_dir.exists():

789 + return

790 + keep = keep_path.resolve(strict=False) if keep_path else None

791 + now = time.time()

792 + for candidate in screenshot_dir.iterdir():

793 + if not candidate.is_file() or candidate.suffix.lower() not in _SCREENSHOT_SUFFIXES:

794 + continue

795 + if keep is not None and candidate.resolve(strict=False) == keep:

796 + continue

797 + if max_age_seconds is not None:

798 + try:

799 + if now - candidate.stat().st_mtime <= max_age_seconds:

800 + continue

801 + except OSError:

802 + pass

803 + candidate.unlink(missing_ok=True)

804 + try:

805 + screenshot_dir.rmdir()

806 + except OSError:

807 + pass

808 +

809 +

810 def iso_now() -> str:

811 return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())

812

@@ -696,6 +836,7 @@ def main(argv: list[str] | None = None) -> int:

836 payload = collect_state(

837 include_screenshot=bool(args.screenshot),

838 context_id=str(args.context_id or ""),

839 + screenshot_transport="path",

840 )

841 print(json.dumps(payload, sort_keys=True))

842 return 0 if payload.get("ok") else 1

@@ -709,6 +850,7 @@ def main(argv: list[str] | None = None) -> int:

850 path=args.path,

851 errors=errors,

852 context_id=str(args.context_id or ""),

853 + transport="path",

854 )

855 if args.json:

856 print(json.dumps(payload, sort_keys=True))

plugins/_desktop/helpers/prompt_context.py

+2 -2

@@ -3,12 +3,12 @@ from __future__ import annotations

3 from plugins._desktop.helpers import desktop_state

6 -def build_context() -> str:

6 +def build_context(context_id: str = "") -> str:

7 if not desktop_state.session_manifest_exists():

8 return ""

9 try:

10 return desktop_state.compact_prompt_context(

11 - desktop_state.collect_state(include_screenshot=False),

11 + desktop_state.collect_state(include_screenshot=False, context_id=context_id),

12 )

13 except Exception as exc:

14 return (

plugins/_desktop/hooks.py

+7 -22

@@ -89,7 +89,7 @@ def cleanup_stale_runtime_state(force: bool = False) -> dict[str, Any]:

89 errors: list[str] = []

91 _migrate_retired_plugin_state(migrated, warnings, errors)

92 - _migrate_unscoped_screenshots(migrated, warnings, errors)

92 + _remove_persisted_screenshots(warnings, errors)

94 retired_packages = _installed_packages(RETIRED_RUNTIME_PACKAGES)

95 if retired_packages:

@@ -142,33 +142,18 @@ def _migrate_retired_plugin_state(

142 )

143

144

145 -def _migrate_unscoped_screenshots(

146 - migrated: list[str],

145 +def _remove_persisted_screenshots(

146 warnings: list[str],

147 errors: list[str],

148 ) -> None:

149 screenshots_dir = STATE_DIR / "screenshots"

150 if not screenshots_dir.exists():

151 return

153 - legacy_screenshots = [

154 - path

155 - for path in screenshots_dir.iterdir()

156 - if path.is_file() and path.suffix.lower() in {".png", ".jpg", ".jpeg", ".xwd"}

157 - ]

158 - if not legacy_screenshots:

159 - return

160 -

161 - context_dir = screenshots_dir / "default"

162 - context_dir.mkdir(parents=True, exist_ok=True)

163 - for screenshot in legacy_screenshots:

164 - state_migration.migrate_retired_state_tree(

165 - source=screenshot,

166 - destination=context_dir / screenshot.name,

167 - owner="Desktop screenshot",

168 - migrated=migrated,

169 - warnings=warnings,

170 - errors=errors,

171 - )

152 + try:

153 + shutil.rmtree(screenshots_dir)

154 + warnings.append(f"Removed retired persistent Desktop screenshots: {screenshots_dir}")

155 + except Exception as exc:

156 + errors.append(f"Failed to remove retired persistent Desktop screenshots at {screenshots_dir}: {exc}")

157

158

159 def _begin_runtime_preparation() -> None:

plugins/_desktop/skills/linux-desktop/SKILL.md

+2 -2

@@ -30,7 +30,7 @@ The Desktop is an observe-act-verify control surface. Use this decision hierarch

30 3. Prefer launcher commands, window focus, keyboard shortcuts, menus, paste, and save commands.

31 4. Use coordinate clicks only as a last resort, and only after a fresh Desktop observation.

32 5. After any GUI action, verify through Desktop state, active window titles, screenshots, saved file state, or exported output.

33 -6. For terminal or CLI-agent work, verify against a fresh final `observe --json --screenshot` captured after the command has finished or visibly returned to an input prompt. Do not report from an earlier screenshot path.

33 +6. For terminal or CLI-agent work, verify against a fresh final `observe --json --screenshot` captured after the command has finished or visibly returned to an input prompt. Agent-facing Desktop screenshots are ephemeral refs; `desktopctl` shell observations return temporary context paths. Do not report from an earlier screenshot path.

35 Keep these standing rules:

@@ -60,7 +60,7 @@ $DESKTOP key ctrl+s

61 The script targets the persistent `agent-zero-desktop` X display, sets `DISPLAY`, `XAUTHORITY`, and `HOME` to the XFCE profile, then uses `xdotool` for input. Startup normally prepares this session. If `check` fails during explicit Desktop work, report that the Desktop runtime is not ready instead of installing packages ad hoc.

63 -If `observe --json --screenshot` shows a reachable display, visible Desktop/window entries, and a fresh screenshot, the Desktop is usable even when `active_window` is `null`; a bare XFCE desktop can have no active application window. Treat missing screenshots, missing display, or unavailable `xdotool`/`xwd` as blockers and stop with the specific readiness message instead of repeating clicks or inventing a fallback.

63 +If `observe --json --screenshot` shows a reachable display, visible Desktop/window entries, and a fresh screenshot, the Desktop is usable even when `active_window` is `null`; a bare XFCE desktop can have no active application window. Treat missing screenshots, missing display, or unavailable `xdotool`/`xwd` as blockers and stop with the specific readiness message instead of repeating clicks or inventing a fallback. Use any returned shell screenshot path promptly; only the latest temporary context screenshot is retained.

65 For direct app launches without coordinates:

plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh

+1 -1

@@ -60,7 +60,7 @@ Commands:

60 observe --json [--screenshot] [--context-id ID]

61 Return structured state, optionally with a fresh screenshot.

62 screenshot [PATH] [--context-id ID]

63 - Capture the Desktop to PATH, or to the default screenshot directory.

63 + Capture the Desktop to PATH, or to the temporary context screenshot directory.

64 active-window Print the active window name.

65 geometry PATTERN Print the first matching visible window geometry.

66 wait-window PATTERN Wait for a visible matching window and print its id.

tests/test_office_desktop_state.py

+58 -1

@@ -185,11 +185,12 @@ def test_desktop_state_screenshot_capture_uses_xwd_and_pillow_when_available(tmp

185 assert screenshot["ok"] is True

186 assert screenshot["path"] == str(tmp_path / "shot.png")

187 assert screenshot["format"] == "png"

188 + assert screenshot["ephemeral"] is False

189 assert (tmp_path / "shot.png").read_bytes() == b"png"

190 assert not (tmp_path / "shot.xwd").exists()

191

192

192 -def test_desktop_state_default_screenshot_path_is_context_scoped(tmp_path, monkeypatch):

193 +def test_desktop_state_shell_screenshot_path_is_context_scoped(tmp_path, monkeypatch):

194 monkeypatch.setattr(desktop_state, "SCREENSHOT_DIR", tmp_path)

195 capabilities = {"xwd": "/usr/bin/xwd"}

196 env = {"DISPLAY": ":120"}

@@ -221,19 +222,75 @@ def test_desktop_state_default_screenshot_path_is_context_scoped(tmp_path, monke

222 monkeypatch.setattr(desktop_state, "run", fake_run)

223 monkeypatch.setitem(sys.modules, "PIL", pil_module)

224 monkeypatch.setitem(sys.modules, "PIL.Image", image_module)

225 + stale_path = tmp_path / "ctx_id" / "stale.png"

226 + stale_path.parent.mkdir(parents=True)

227 + stale_path.write_bytes(b"stale")

228

229 screenshot = desktop_state.capture_screenshot(

230 env,

231 capabilities,

232 errors=[],

233 context_id="ctx/id",

234 + transport="path",

235 )

236

237 path = Path(screenshot["path"])

238 assert screenshot["ok"] is True

239 + assert screenshot["ephemeral"] is True

240 + assert screenshot["context_id"] == "ctx_id"

241 assert path.parent == tmp_path / "ctx_id"

242 assert path.name.startswith("desktop-")

243 assert desktop_state.latest_screenshot(context_id="ctx/id")["path"] == str(path)

244 + assert not stale_path.exists()

245 +

246 +

247 +def test_desktop_state_default_screenshot_returns_ephemeral_ref(tmp_path, monkeypatch):

248 + monkeypatch.setattr(desktop_state, "SCREENSHOT_DIR", tmp_path)

249 + capabilities = {"xwd": "/usr/bin/xwd"}

250 + env = {"DISPLAY": ":120"}

251 +

252 + def fake_run(command, *, env, timeout):

253 + raw_path = Path(command[command.index("-out") + 1])

254 + raw_path.write_bytes(b"xwd")

255 + return _completed(command)

256 +

257 + image_module = types.ModuleType("PIL.Image")

258 +

259 + class FakeImage:

260 + width = 320

261 + height = 240

262 +

263 + def __enter__(self):

264 + return self

265 +

266 + def __exit__(self, *_args):

267 + return False

268 +

269 + def save(self, target):

270 + Path(target).write_bytes(b"png")

271 +

272 + image_module.open = lambda _path: FakeImage()

273 + pil_module = types.ModuleType("PIL")

274 + pil_module.Image = image_module

275 +

276 + monkeypatch.setattr(desktop_state, "run", fake_run)

277 + monkeypatch.setitem(sys.modules, "PIL", pil_module)

278 + monkeypatch.setitem(sys.modules, "PIL.Image", image_module)

279 +

280 + screenshot = desktop_state.capture_screenshot(

281 + env,

282 + capabilities,

283 + errors=[],

284 + context_id="ctx/id",

285 + )

286 +

287 + assert screenshot["ok"] is True

288 + assert screenshot["path"] == ""

289 + assert screenshot["ephemeral"] is True

290 + assert screenshot["ephemeral_ref"].startswith("a0-ephemeral-image://")

291 + assert screenshot["vision_load"]["tool_args"]["paths"] == [screenshot["ephemeral_ref"]]

292 + assert screenshot["context_id"] == "ctx_id"

293 + assert not (tmp_path / "ctx_id").exists()

294

295

296 def test_xwd_fallback_parser_handles_truecolor_pixels(tmp_path, monkeypatch):

tests/test_office_document_store.py

+2 -1

@@ -1631,7 +1631,8 @@ def test_desktop_cleanup_moves_retired_state_to_plugin_state(tmp_path, monkeypat

1631 assert result["ok"] is True

1632 assert (plugin_state / "profiles" / "agent-zero-desktop" / "profile.txt").read_text(encoding="utf-8") == "profile\n"

1633 assert (plugin_state / "sessions" / "agent-zero-desktop.json").read_text(encoding="utf-8") == "{}\n"

1634 - assert (plugin_state / "screenshots" / "default" / "desktop.png").read_bytes() == b"png"

1634 + assert not (plugin_state / "screenshots").exists()

1635 + assert any("Removed retired persistent Desktop screenshots" in warning for warning in result["warnings"])

1636 assert not retired_state.exists()

1637

1638