Harden internal Xpra desktop control
Move the internal Linux Desktop skill toward a structured-first workflow: state/window commands before screenshots, batched desktopctl sequences in one process, and final screenshots only when pixels matter. Normalize internal system Desktop resize requests so narrow portrait canvas sizes do not shrink the real X11 root display and distort screenshots. Keep the live Docker runtime in sync with the changed Desktop files.
Alessandro committed
May 30, 2026 at 22:22 UTC
ce4156a1109892a884ed65f7e591508020c9ea05
8 files changed
+121
-24
helpers/virtual_desktop.py
+33
@@ -23,6 +23,7 @@ MAX_WIDTH = 1920
23
MAX_HEIGHT = 1080
24
MIN_WIDTH = 360
25
MIN_HEIGHT = 240
26
+MIN_DESKTOP_ASPECT_RATIO = 4 / 3
27
SESSION_PATH = "/desktop/session"
28
XPRA_HTML_ROOT_CANDIDATES = (
29
Path("/usr/share/xpra/www"),
@@ -220,6 +221,38 @@ def normalize_size(
221
)
222
223
224
+def normalize_desktop_display_size(
225
+ width: int | float | str,
226
+ height: int | float | str,
227
+ *,
228
+ max_width: int = MAX_WIDTH,
229
+ max_height: int = MAX_HEIGHT,
230
+ min_width: int = MIN_WIDTH,
231
+ min_height: int = MIN_HEIGHT,
232
+ min_aspect_ratio: float = MIN_DESKTOP_ASPECT_RATIO,
233
+) -> tuple[int, int]:
234
+ normalized_width, normalized_height = normalize_size(
235
+ width,
236
+ height,
237
+ max_width=max_width,
238
+ max_height=max_height,
239
+ min_width=min_width,
240
+ min_height=min_height,
241
+ )
242
+ if normalized_height <= 0:
243
+ return normalized_width, normalized_height
244
+ if normalized_width / normalized_height >= min_aspect_ratio:
245
+ return normalized_width, normalized_height
246
+ return normalize_size(
247
+ DEFAULT_WIDTH,
248
+ DEFAULT_HEIGHT,
249
+ max_width=max_width,
250
+ max_height=max_height,
251
+ min_width=min_width,
252
+ min_height=min_height,
253
+ )
254
+
255
+
256
def resize_display(
257
*,
258
display: int,
plugins/_desktop/helpers/desktop_session.py
+2
@@ -399,6 +399,8 @@ class DesktopSessionManager:
399
if not session:
400
return {"ok": False, "error": "LibreOffice desktop session not found."}
401
is_system_desktop = session.session_id == SYSTEM_SESSION_ID and session.extension == "desktop"
402
+ if is_system_desktop:
403
+ width, height = virtual_desktop.normalize_desktop_display_size(width, height)
404
result = virtual_desktop.resize_display(
405
display=session.display,
406
width=width,
plugins/_desktop/helpers/desktop_state.py
+3
-2
@@ -721,9 +721,10 @@ def compact_prompt_context(state: dict[str, Any] | None = None) -> str:
721
lines.append(f"- screenshot_context={context_id}")
722
context_arg = f" --context-id {context_id}" if context_id else ""
723
lines.append(
724
- "- next=plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh observe --json --screenshot"
724
+ "- next=plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh state --json"
725
+ f"{context_arg} for structured checks; use observe --json --screenshot"
726
f"{context_arg} "
726
- "before any coordinate action; prefer focus/key/paste/save/app-native helpers first."
727
+ "before coordinate or visual-OCR actions; prefer sequence/focus/key/paste/save/app-native helpers first."
728
)
729
lines.append(
730
"- verify=for terminal/CLI-agent output, use the screenshot path from a fresh final "
plugins/_desktop/skills/linux-desktop/SKILL.md
+36
-16
@@ -31,14 +31,16 @@ Use the Desktop as a full Linux GUI when the user explicitly needs a visual work
31
32
## Operating Model
33
34
-The Desktop is an observe-act-verify control surface. Use this decision hierarchy:
34
+The Desktop is a structured-first X11 control surface. Use this decision hierarchy:
35
36
1. Prefer structured tools such as `office_artifact` for deterministic Office file creation, reads, and edits.
37
-2. Prefer app-native helpers for visible live edits, such as `desktopctl.sh calc-set-cell` for Calc/UNO spreadsheet changes.
38
-3. Prefer launcher commands, window focus, keyboard shortcuts, menus, paste, and save commands.
39
-4. Use coordinate clicks only as a last resort, and only after a fresh Desktop observation.
40
-5. After any GUI action, verify through Desktop state, active window titles, screenshots, saved file state, or exported output.
41
-6. For terminal or CLI-agent work, verify against a fresh final `observe --json --screenshot` captured after the command has finished or visibly returned to an input prompt. Agent-facing Desktop screenshots are ephemeral refs; `desktopctl` shell observations with `--context-id` return chat-scoped screenshot paths. Do not report from an earlier screenshot path.
37
+2. Prefer structured Desktop state and window commands: `check`, `state --json`, `windows`, `active-window`, `geometry`, and `wait-window`.
38
+3. Prefer app-native helpers for visible live edits, such as `desktopctl.sh calc-set-cell` for Calc/UNO spreadsheet changes.
39
+4. Prefer launcher commands, window focus, keyboard shortcuts, menus, paste, and save commands. Batch multi-step actions with `sequence`/`batch` so one helper process drives the flow.
40
+5. Use screenshots for visual inspection, OCR, coordinate work, and final evidence when the task depends on pixels. Do not take screenshots merely to learn window titles or readiness that structured state already reports.
41
+6. Use coordinate clicks only as a last resort, and only after a fresh screenshot observation.
42
+7. After any GUI action, verify through Desktop state, active window titles, screenshots when visually necessary, saved file state, or exported output.
43
+8. For terminal or CLI-agent work, prefer deterministic command output or saved transcripts. When exact visible terminal text matters, verify against a fresh final `observe --json --screenshot` captured after the command has finished or visibly returned to an input prompt. Agent-facing Desktop screenshots are ephemeral refs; `desktopctl` shell observations with `--context-id` return chat-scoped screenshot paths. Do not report from an earlier screenshot path.
44
45
Keep these standing rules:
46
@@ -58,7 +60,7 @@ Use the helper script when the Desktop is already open and you need reliable app
60
DESKTOP=/a0/plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh
61
$DESKTOP check
62
$DESKTOP state --json
61
-$DESKTOP observe --json --screenshot
63
+$DESKTOP observe --json
64
$DESKTOP launch calc
65
$DESKTOP wait-window LibreOffice
66
$DESKTOP windows LibreOffice
@@ -68,7 +70,7 @@ $DESKTOP key ctrl+s
70
71
The script targets the persistent `agent-zero-desktop` X display, sets `DISPLAY`, `XAUTHORITY`, and `HOME` to the XFCE profile, then uses `xdotool` for input. Startup normally prepares this session. If `check` fails during explicit Desktop work, report that the Desktop runtime is not ready instead of installing packages ad hoc.
72
71
-If `observe --json --screenshot` shows a reachable display, visible Desktop/window entries, and a fresh screenshot, the Desktop is usable even when `active_window` is `null`; a bare XFCE desktop can have no active application window. Treat missing screenshots, missing display, or unavailable `xdotool`/`xwd` as blockers and stop with the specific readiness message instead of repeating clicks or inventing a fallback. Shell screenshots captured with `--context-id` live in the owning chat's screenshot folder; screenshots without a chat context remain temporary.
73
+If `state --json` or `observe --json` shows a reachable display and visible Desktop/window entries, the Desktop is usable even when `active_window` is `null`; a bare XFCE desktop can have no active application window. Treat missing display or unavailable `xdotool` as blockers and stop with the specific readiness message instead of repeating clicks or inventing a fallback. Use `observe --json --screenshot` only when you need pixels, and treat unavailable `xwd` as a screenshot blocker rather than a general Desktop blocker. Shell screenshots captured with `--context-id` live in the owning chat's screenshot folder; screenshots without a chat context remain temporary.
74
75
For direct app launches without coordinates:
76
@@ -85,6 +87,20 @@ $DESKTOP paste-text "Text to insert"
87
$DESKTOP key ctrl+s
88
```
89
90
+For multi-step window or terminal actions, batch commands through one helper process:
91
+
92
+```bash
93
+DESKTOP=/a0/plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh
94
+$DESKTOP sequence - <<'EOF'
95
+focus Terminal
96
+paste-text echo ready
97
+key Return
98
+state --json
99
+EOF
100
+```
101
+
102
+Use one command per line. `paste-text` joins the remaining words on that line, so it is suitable for ordinary command text; for complex multiline payloads, write the payload to a file or invoke `paste-text` directly.
103
+
104
For live spreadsheet coworking, use the Calc helper instead of hand-written UNO snippets:
105
106
```bash
@@ -120,10 +136,10 @@ When browser automation is available, the higher-level QA flow is:
136
137
Terminal apps are visual state, not structured logs. When the task depends on exact terminal output, follow this stricter loop:
138
123
-1. Run `desktopctl.sh observe --json --screenshot` immediately before acting to record the starting window and screenshot path.
124
-2. Use `focus`, `paste-text` or `type`, and `key Return` to drive the terminal. Prefer CLI-native commands and keyboard input over clicks.
139
+1. Run `desktopctl.sh state --json` or `desktopctl.sh windows Terminal` before acting to confirm the target window. Add `observe --json --screenshot` only if the current prompt or screen contents must be read visually.
140
+2. Use `sequence`, `focus`, `paste-text` or `type`, and `key Return` to drive the terminal. Prefer CLI-native commands and keyboard input over clicks.
141
3. Wait until the CLI has visibly produced a response or returned to an input prompt.
126
-4. Run a new final `desktopctl.sh observe --json --screenshot`.
142
+4. If exact visible text matters and no deterministic transcript was saved, run a new final `desktopctl.sh observe --json --screenshot`.
143
5. Verify exact text only from the screenshot path returned by that final observation, or from a newer screenshot. Never use an earlier screenshot path as final evidence.
144
6. If the final screenshot is cropped, stale, or unreadable, capture another screenshot or report the result as unverified with that specific reason.
145
@@ -140,13 +156,17 @@ Example for a nested CLI-agent smoke test:
156
157
```bash
158
DESKTOP=/a0/plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh
143
-$DESKTOP focus "Terminal"
144
-$DESKTOP paste-text 'TARGET_CLI="example-cli-agent"; FALLBACK_CMD=""; if command -v "$TARGET_CLI" >/dev/null 2>&1; then "$TARGET_CLI"; elif [ -n "$FALLBACK_CMD" ]; then sh -lc "$FALLBACK_CMD"; else echo "CLI agent not found: $TARGET_CLI"; fi'
145
-$DESKTOP key Return
159
+$DESKTOP sequence - <<'EOF'
160
+focus Terminal
161
+paste-text TARGET_CLI="example-cli-agent"; FALLBACK_CMD=""; if command -v "$TARGET_CLI" >/dev/null 2>&1; then "$TARGET_CLI"; elif [ -n "$FALLBACK_CMD" ]; then sh -lc "$FALLBACK_CMD"; else echo "CLI agent not found: $TARGET_CLI"; fi
162
+key Return
163
+EOF
164
$DESKTOP observe --json --screenshot
165
# Verify the screenshot shows the target CLI prompt, not a shell prompt, before sending natural language:
148
-$DESKTOP paste-text 'Reply with exactly the requested smoke-test token.'
149
-$DESKTOP key Return
166
+$DESKTOP sequence - <<'EOF'
167
+paste-text Reply with exactly the requested smoke-test token.
168
+key Return
169
+EOF
170
$DESKTOP observe --json --screenshot
171
```
172
plugins/_desktop/skills/linux-desktop/scripts/desktopctl.sh
+13
-5
@@ -71,7 +71,8 @@ Commands:
71
drag X1 Y1 X2 Y2 Drag from X1,Y1 to X2,Y2 in Desktop coordinates.
72
right-click X Y Move and right-click at X,Y in Desktop coordinates.
73
paste-text TEXT Put TEXT on the Desktop clipboard and paste it with an app-native shortcut.
74
- sequence FILE|- Run a newline-delimited command sequence.
74
+ sequence FILE|- Run a newline-delimited command sequence in this process.
75
+ batch FILE|- Alias for sequence.
76
key KEY... Send one or more xdotool key names.
77
type TEXT Type text into the focused window.
78
click X Y Move and click at X,Y in Desktop coordinates.
@@ -228,7 +229,7 @@ run_sequence_line() {
229
\#*) return 0 ;;
230
esac
231
# shellcheck disable=SC2086
231
- "$0" $line
232
+ dispatch_command $line
233
}
234
235
run_sequence() {
@@ -279,7 +280,11 @@ launch_app() {
280
esac
281
}
282
282
-case "$command_name" in
283
+dispatch_command() {
284
+ local command_name="${1:-help}"
285
+ shift || true
286
+
287
+ case "$command_name" in
288
help|-h|--help)
289
usage
290
;;
@@ -387,7 +392,7 @@ case "$command_name" in
392
fi
393
paste_text "$@"
394
;;
390
- sequence)
395
+ sequence|batch)
396
source_file="${1:?sequence requires FILE or -}"
397
run_sequence "$source_file"
398
;;
@@ -448,4 +453,7 @@ case "$command_name" in
453
usage >&2
454
exit 2
455
;;
451
-esac
456
+ esac
457
+}
458
+
459
+dispatch_command "$command_name" "$@"
plugins/_desktop/webui/desktop-store.js
+1
-1
@@ -1818,7 +1818,7 @@ const model = {
1818
html, body, #screen {
1819
width: 100% !important;
1820
height: 100% !important;
1821
- overflow: hidden !important;
1821
+ overflow: auto !important;
1822
}
1823
#float_menu,
1824
.windowhead,
tests/test_office_canvas_setup.py
+5
@@ -272,7 +272,9 @@ def test_desktop_plugin_owns_routes_runtime_surface_and_state_paths():
272
assert "canvas.width = normalizedWidth" in desktop_store
273
assert "canvas.height = normalizedHeight" in desktop_store
274
assert "canvas?.clientWidth || canvas?.width" in desktop_store
275
+ assert "overflow: auto !important;" in desktop_store
276
assert "Installing Agent Zero Desktop runtime dependencies" in desktop_session
277
+ assert "normalize_desktop_display_size" in desktop_session
278
assert "__a0XpraOffsetWarnPatched" in desktop_store
279
assert "window does not fit in canvas, offsets" in desktop_store
280
assert "decode error packet" in desktop_store
@@ -529,6 +531,9 @@ def test_office_and_desktop_skills_are_rehomed_and_renamed():
531
assert "Open in Desktop action" in desktop_skill
532
assert "$BASE_DIR/usr/plugins/_desktop/profiles/$SESSION" in desktopctl
533
assert "$BASE_DIR/usr/plugins/_desktop/sessions/$SESSION.json" in desktopctl
534
+ assert "sequence|batch)" in desktopctl
535
+ assert "dispatch_command $line" in desktopctl
536
+ assert '"$0" $line' not in desktopctl
537
538
539
def test_skill_catalog_and_connector_boundaries_are_static_guarded():
tests/test_office_desktop_state.py
+28
@@ -10,6 +10,7 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
10
if str(PROJECT_ROOT) not in sys.path:
11
sys.path.insert(0, str(PROJECT_ROOT))
12
13
+from helpers import virtual_desktop
14
from plugins._desktop.helpers import desktop_state
15
16
@@ -296,6 +297,33 @@ def test_desktop_state_default_screenshot_returns_ephemeral_ref(tmp_path, monkey
297
assert not (tmp_path / "ctx_id").exists()
298
299
300
+def test_desktop_prompt_context_recommends_structured_state_before_screenshots():
301
+ context = desktop_state.compact_prompt_context(
302
+ {
303
+ "display": ":120",
304
+ "size": {"width": 1280, "height": 720},
305
+ "pointer": {"x": 10, "y": 20},
306
+ "active_window": {"title": "Terminal", "class": "Xfce4-terminal"},
307
+ "windows": [{"title": "Terminal", "class": "Xfce4-terminal"}],
308
+ "screenshot": {},
309
+ "context_id": "ctx_id",
310
+ "errors": [],
311
+ }
312
+ )
313
+
314
+ assert "state --json --context-id ctx_id for structured checks" in context
315
+ assert "observe --json --screenshot --context-id ctx_id before coordinate or visual-OCR actions" in context
316
+ assert "before any coordinate action" not in context
317
+
318
+
319
+def test_virtual_desktop_system_display_normalization_rejects_portrait_viewports():
320
+ assert virtual_desktop.normalize_desktop_display_size(395, 1080) == (
321
+ virtual_desktop.DEFAULT_WIDTH,
322
+ virtual_desktop.DEFAULT_HEIGHT,
323
+ )
324
+ assert virtual_desktop.normalize_desktop_display_size(1600, 900) == (1600, 900)
325
+
326
+
327
def test_xwd_fallback_parser_handles_truecolor_pixels(tmp_path, monkeypatch):
328
raw_path = tmp_path / "shot.xwd"
329
target = tmp_path / "shot.png"