@samitouri / QOSami_Agents_Zero-1 / commits / bb2a2ecf

Make the interactive Browser honor configured keyboard layouts

The private Browser Xvfb always started with the US XKB layout, so non-US keyboards could not type their printed characters in the interactive Browser window: German Mac users could not type "@" via Option+L, and umlauts, brackets, and AltGr/Option symbols landed on wrong keys or vanished. Add keyboard_layout / keyboard_variant Browser settings (validated as safe XKB tokens, empty keeps US behavior). Browser startup applies the layout to the private display with setxkbmap and pins it on the Xpra shadow server (--keyboard-sync=no, --keyboard-layout, --keyboard-variant) so connecting clients cannot override the server layout. Layout changes flow through browser_runtime_config and restart internal runtimes. The fallback canvas input path also stopped discarding Alt/AltGr combinations that produce printable characters, and the settings UI offers free-text layout/variant fields linking to the official xkeyboard-config list.

Jehu committed Aug 25, 2026 at 09:14 UTC bb2a2ecf16a6e651d31c99f29b214b02e5e36510
8 files changed +230 -2
plugins/_browser/AGENTS.md
+1
@@ -48,6 +48,7 @@
48 - Annotation voice input reuses Whisper STT's configured draft/send delivery mode and shared microphone state.
49 - Internal-browser proxy settings map directly to Playwright's persistent-context proxy option, never to Bring Your Own Browser, and changes must restart active internal runtimes.
50 - Run internal Chromium headful through Patchright on the private virtual display; do not add user-agent or header spoofing on top of the patched driver.
51 +- Browser keyboard layout settings (`keyboard_layout`/`keyboard_variant`, e.g. `de`/`mac`) apply the configured XKB layout to the private browser display with setxkbmap and pin it on the Xpra shadow server so non-US keyboards type their printed characters; layout changes flow through `browser_runtime_config` and restart internal runtimes.
52 - Browser startup and on-demand launch must converge on the Chromium revision declared by Patchright; let its installer select the host architecture rather than hardcoding x64 or ARM downloads.
53 - `hooks.prepare_playwright_cache()` owns reconciliation of the pinned Patchright package and Chromium binary so repository self-updates and fresh images use the same setup path.
54 - Browser startup must install the shared virtual-desktop route hook itself; do not make Browser depend on the Desktop plugin being enabled.
plugins/_browser/default_config.yaml
+6
@@ -49,3 +49,9 @@ host_browser_selection: ""
49 # Optional _model_config preset used by Browser-owned model helpers.
50 # Empty uses the effective Main Model.
51 model_preset: ""
52 +
53 +# XKB keyboard layout applied to the interactive Browser display so typed keys
54 +# match the physical keyboard. German Mac example: keyboard_layout: "de",
55 +# keyboard_variant: "mac" (Option+L then types "@"). Empty keeps US behavior.
56 +keyboard_layout: ""
57 +keyboard_variant: ""
plugins/_browser/helpers/config.py
+16
@@ -22,6 +22,8 @@ PROXY_SERVER_KEY = "proxy_server"
22 PROXY_BYPASS_KEY = "proxy_bypass"
23 PROXY_USERNAME_KEY = "proxy_username"
24 PROXY_PASSWORD_KEY = "proxy_password"
25 +KEYBOARD_LAYOUT_KEY = "keyboard_layout"
26 +KEYBOARD_VARIANT_KEY = "keyboard_variant"
27 RUNTIME_BACKENDS = {"container", "host_required"}
28 BROWSER_TAB_SCOPES = {"per_context", "shared"}
29 HOST_BROWSER_PRIVACY_POLICIES = {"enforce_local", "warn", "allow"}
@@ -100,6 +102,12 @@ def _normalize_int(value: Any, *, default: int, minimum: int, maximum: int) -> i
102 return max(minimum, min(maximum, number))
103
104
105 +def _normalize_xkb_token(value: Any) -> str:
106 + return "".join(
107 + ch for ch in str(value or "").strip().lower() if ch.isalnum() or ch in {"_", "-"}
108 + )[:32]
109 +
110 +
111 def _normalize_choice(value: Any, *, allowed: set[str], default: str) -> str:
112 normalized = str(value or "").strip().lower().replace("-", "_")
113 if normalized in allowed:
@@ -125,6 +133,10 @@ def _model_config_summary(config: dict[str, Any] | None) -> str:
133 def normalize_browser_config(settings: dict[str, Any] | None) -> dict[str, Any]:
134 raw = settings if isinstance(settings, dict) else {}
135 extension_paths = _normalize_extension_paths(raw.get("extension_paths", []))
136 + keyboard_layout = _normalize_xkb_token(raw.get(KEYBOARD_LAYOUT_KEY, ""))
137 + keyboard_variant = (
138 + _normalize_xkb_token(raw.get(KEYBOARD_VARIANT_KEY, "")) if keyboard_layout else ""
139 + )
140 return {
141 "extension_paths": extension_paths,
142 DEFAULT_HOMEPAGE_KEY: _normalize_default_homepage(
@@ -165,6 +177,8 @@ def normalize_browser_config(settings: dict[str, Any] | None) -> dict[str, Any]:
177 PROXY_BYPASS_KEY: str(raw.get(PROXY_BYPASS_KEY, "") or "").strip()[:4096],
178 PROXY_USERNAME_KEY: str(raw.get(PROXY_USERNAME_KEY, "") or "")[:1024],
179 PROXY_PASSWORD_KEY: str(raw.get(PROXY_PASSWORD_KEY, "") or "")[:4096],
180 + KEYBOARD_LAYOUT_KEY: keyboard_layout,
181 + KEYBOARD_VARIANT_KEY: keyboard_variant,
182 MODEL_PRESET_KEY: _normalize_model_preset(raw.get(MODEL_PRESET_KEY, "")),
183 }
184
@@ -177,6 +191,8 @@ def browser_runtime_config(settings: dict[str, Any] | None) -> dict[str, Any]:
191 PROXY_BYPASS_KEY: config[PROXY_BYPASS_KEY],
192 PROXY_USERNAME_KEY: config[PROXY_USERNAME_KEY],
193 PROXY_PASSWORD_KEY: config[PROXY_PASSWORD_KEY],
194 + KEYBOARD_LAYOUT_KEY: config[KEYBOARD_LAYOUT_KEY],
195 + KEYBOARD_VARIANT_KEY: config[KEYBOARD_VARIANT_KEY],
196 }
197
198
plugins/_browser/helpers/interactive_view.py
+54
@@ -20,6 +20,21 @@ DEFAULT_HEIGHT = 768
20 START_TIMEOUT_SECONDS = 15.0
21
22
23 +def keyboard_options() -> dict[str, str]:
24 + """Resolve the configured XKB keyboard layout for browser displays."""
25 + from plugins._browser.helpers.config import (
26 + KEYBOARD_LAYOUT_KEY,
27 + KEYBOARD_VARIANT_KEY,
28 + get_browser_config,
29 + )
30 +
31 + config = get_browser_config()
32 + return {
33 + "layout": str(config.get(KEYBOARD_LAYOUT_KEY, "") or "").strip(),
34 + "variant": str(config.get(KEYBOARD_VARIANT_KEY, "") or "").strip(),
35 + }
36 +
37 +
38 def collect_status() -> dict[str, Any]:
39 binaries = {
40 name: shutil.which(name) or ""
@@ -114,6 +129,7 @@ class BrowserInteractiveView:
129
130 self._xvfb = process
131 self.display = int(display_number)
132 + self._apply_keyboard_layout()
133 self.resize(self.width, self.height)
134 return self.display_name
135
@@ -190,6 +206,43 @@ class BrowserInteractiveView:
206 self._stop_locked()
207 shutil.rmtree(self.state_dir, ignore_errors=True)
208
209 + def _apply_keyboard_layout(self) -> None:
210 + """Apply the configured XKB layout to this private display."""
211 + if self.display is None:
212 + return
213 + options = keyboard_options()
214 + if not options["layout"]:
215 + return
216 + setxkbmap = shutil.which("setxkbmap")
217 + if not setxkbmap:
218 + return
219 + command = [setxkbmap, "-display", self.display_name, "-layout", options["layout"]]
220 + if options["variant"]:
221 + command.extend(["-variant", options["variant"]])
222 + try:
223 + subprocess.run(
224 + command,
225 + check=False,
226 + stdin=subprocess.DEVNULL,
227 + stdout=subprocess.DEVNULL,
228 + stderr=subprocess.DEVNULL,
229 + timeout=5,
230 + )
231 + except (OSError, subprocess.TimeoutExpired):
232 + pass
233 +
234 + def _keyboard_xpra_args(self) -> list[str]:
235 + options = keyboard_options()
236 + if not options["layout"]:
237 + return []
238 + args = [
239 + "--keyboard-sync=no",
240 + "--keyboard-layout", options["layout"],
241 + ]
242 + if options["variant"]:
243 + args.extend(["--keyboard-variant", options["variant"]])
244 + return args
245 +
246 def _start_xpra(self, xpra: str) -> None:
247 self.port = self._free_port()
248 runtime_dir = self.state_dir / "runtime"
@@ -227,6 +280,7 @@ class BrowserInteractiveView:
280 "--encoding=auto",
281 "--quality=90",
282 "--speed=90",
283 + *self._keyboard_xpra_args(),
284 f"--bind-tcp=127.0.0.1:{self.port}",
285 f"--socket-dir={socket_dir}",
286 f"--log-dir={self.state_dir}",
plugins/_browser/webui/browser-config-store.js
+10
@@ -41,6 +41,8 @@ function ensureConfig(config) {
41 config.proxy_bypass = String(config.proxy_bypass || "").trim();
42 config.proxy_username = String(config.proxy_username || "");
43 config.proxy_password = String(config.proxy_password || "");
44 + config.keyboard_layout = normalizeXkbToken(config.keyboard_layout);
45 + config.keyboard_variant = normalizeXkbToken(config.keyboard_variant);
46 config.host_browser_privacy_policy = normalizeChoice(
47 config.host_browser_privacy_policy,
48 HOST_PRIVACY_POLICIES,
@@ -68,6 +70,14 @@ function normalizeInt(value, fallback, minimum, maximum) {
70 return Math.max(minimum, Math.min(maximum, number));
71 }
72
73 +function normalizeXkbToken(value) {
74 + return String(value || "")
75 + .trim()
76 + .toLowerCase()
77 + .replace(/[^a-z0-9_-]/g, "")
78 + .slice(0, 32);
79 +}
80 +
81 function normalizeRuntimeBackend(value) {
82 const normalized = String(value || "").trim().toLowerCase().replace(/-/g, "_");
83 if (normalized === "host_when_available") return "host_required";
plugins/_browser/webui/browser-store.js
+3 -2
@@ -2908,10 +2908,11 @@ const model = {
2908 if (this.annotating) return;
2909 const contextId = this.normalizeContextId(this.activeBrowserContextId || this.contextId);
2910 if (!contextId || !this.activeBrowserId) return;
2911 - if (event.ctrlKey || event.metaKey || event.altKey) return;
2911 + const printable = event.key && event.key.length === 1;
2912 + const altGrText = printable && event.altKey && !event.metaKey;
2913 + if ((event.ctrlKey || event.metaKey || event.altKey) && !altGrText) return;
2914 if (isLocalEditableTarget(event?.target)) return;
2915 event.preventDefault();
2914 - const printable = event.key && event.key.length === 1;
2916 await websocket.emit("browser_viewer_input", {
2917 context_id: contextId,
2918 browser_id: this.activeBrowserId,
plugins/_browser/webui/config.html
+31
@@ -243,6 +243,37 @@
243 </span>
244 </label>
245
246 + <label class="browser-config-field">
247 + <span class="browser-config-field-label">Keyboard layout</span>
248 + <input
249 + type="text"
250 + x-model="$store.browserConfig.config.keyboard_layout"
251 + placeholder="de, fr, us, ..."
252 + autocomplete="off"
253 + />
254 + <span class="browser-config-field-help">
255 + XKB layout of the interactive Browser window, e.g. de for a German keyboard. Match your physical keyboard so characters like @, umlauts, and brackets land on the printed keys. Empty keeps US.
256 + <a href="https://man.archlinux.org/man/xkeyboard-config.7#LAYOUTS" target="_blank" rel="noreferrer">Full layout list</a>. Changing it restarts the internal browser on save.
257 + </span>
258 + </label>
259 +
260 + <label
261 + class="browser-config-field"
262 + x-show="$store.browserConfig.config.keyboard_layout"
263 + >
264 + <span class="browser-config-field-label">Keyboard variant</span>
265 + <input
266 + type="text"
267 + x-model="$store.browserConfig.config.keyboard_variant"
268 + placeholder="mac, nodeadkeys, colemak, ..."
269 + autocomplete="off"
270 + />
271 + <span class="browser-config-field-help">
272 + XKB variant for the layout above. German Mac keyboards use mac, so Option+L types @.
273 + <a href="https://man.archlinux.org/man/xkeyboard-config.7#LAYOUTS" target="_blank" rel="noreferrer">Variants per layout</a>. Changing it restarts the internal browser on save.
274 + </span>
275 + </label>
276 +
277 <label class="browser-config-switch-row">
278 <span class="browser-config-switch-copy">
279 <span class="browser-config-field-label">Autofocus active page</span>
tests/test_browser_agent_regressions.py
+109
@@ -209,10 +209,58 @@ def test_browser_config_normalizes_extension_paths(tmp_path):
209 "proxy_bypass": "",
210 "proxy_username": "",
211 "proxy_password": "",
212 + "keyboard_layout": "",
213 + "keyboard_variant": "",
214 "model_preset": "",
215 }
216
217
218 +def test_browser_config_normalizes_keyboard_layout():
219 + config = normalize_browser_config(
220 + {
221 + "keyboard_layout": " DE ",
222 + "keyboard_variant": " Mac ",
223 + }
224 + )
225 +
226 + assert config["keyboard_layout"] == "de"
227 + assert config["keyboard_variant"] == "mac"
228 +
229 + cleared = normalize_browser_config(
230 + {
231 + "keyboard_layout": "",
232 + "keyboard_variant": "mac",
233 + }
234 + )
235 +
236 + assert cleared["keyboard_layout"] == ""
237 + assert cleared["keyboard_variant"] == ""
238 +
239 + unsafe = normalize_browser_config(
240 + {
241 + "keyboard_layout": "de; rm -rf /",
242 + "keyboard_variant": "mac & echo pwned",
243 + }
244 + )
245 +
246 + for token in (unsafe["keyboard_layout"], unsafe["keyboard_variant"]):
247 + assert token == token.lower()
248 + for forbidden in (" ", ";", "&", "|", "$", "`", "'", '"'):
249 + assert forbidden not in token
250 +
251 +
252 +def test_browser_keyboard_layout_restarts_runtime():
253 + from plugins._browser.helpers.config import browser_runtime_config
254 +
255 + base = browser_runtime_config({"keyboard_layout": "de", "keyboard_variant": "mac"})
256 +
257 + assert base["keyboard_layout"] == "de"
258 + assert base["keyboard_variant"] == "mac"
259 +
260 + changed = browser_runtime_config({"keyboard_layout": "us", "keyboard_variant": "mac"})
261 + assert changed != base
262 +
263 +
264 def test_browser_config_normalizes_model_preset():
265 assert normalize_browser_config({"model_preset": " Research "})["model_preset"] == "Research"
266 assert "model" not in normalize_browser_config({"model": "main"})
@@ -2466,6 +2514,67 @@ def test_browser_startup_migration_prepares_current_playwright_binary():
2514 assert "PrintStyle.warning" in extension
2515
2516
2517 +
2518 +
2519 +def test_browser_interactive_view_keyboard_options(monkeypatch):
2520 + import plugins._browser.helpers.interactive_view as iv_module
2521 +
2522 + view = BrowserInteractiveView("ctx-kb")
2523 +
2524 + # no layout -> no xpra keyboard args
2525 + monkeypatch.setattr(
2526 + iv_module, "keyboard_options", lambda: {"layout": "", "variant": ""}
2527 + )
2528 + assert view._keyboard_xpra_args() == []
2529 +
2530 + # layout + variant -> pinned server layout, sync disabled
2531 + monkeypatch.setattr(
2532 + iv_module,
2533 + "keyboard_options",
2534 + lambda: {"layout": "de", "variant": "mac"},
2535 + )
2536 + assert view._keyboard_xpra_args() == [
2537 + "--keyboard-sync=no",
2538 + "--keyboard-layout", "de",
2539 + "--keyboard-variant", "mac",
2540 + ]
2541 +
2542 +
2543 +def test_browser_interactive_view_applies_keyboard_layout(monkeypatch):
2544 + commands = []
2545 +
2546 + def fake_run(command, **kwargs):
2547 + commands.append(list(command))
2548 + return SimpleNamespace(returncode=0)
2549 +
2550 + monkeypatch.setattr(browser_interactive_view_module.subprocess, "run", fake_run)
2551 + monkeypatch.setattr(
2552 + browser_interactive_view_module.shutil,
2553 + "which",
2554 + lambda name: "/usr/bin/setxkbmap" if name == "setxkbmap" else None,
2555 + )
2556 + monkeypatch.setattr(
2557 + browser_interactive_view_module,
2558 + "keyboard_options",
2559 + lambda: {"layout": "de", "variant": "mac"},
2560 + )
2561 +
2562 + view = BrowserInteractiveView("ctx-kb-apply")
2563 + view.display = 42
2564 + view._apply_keyboard_layout()
2565 +
2566 + assert commands == [["/usr/bin/setxkbmap", "-display", ":42", "-layout", "de", "-variant", "mac"]]
2567 +
2568 + # no configured layout -> no setxkbmap call
2569 + monkeypatch.setattr(
2570 + browser_interactive_view_module,
2571 + "keyboard_options",
2572 + lambda: {"layout": "", "variant": ""},
2573 + )
2574 + view._apply_keyboard_layout()
2575 + assert len(commands) == 1
2576 +
2577 +
2578 def test_browser_interactive_views_use_isolated_loopback_sessions(monkeypatch, tmp_path):
2579 class FakeProcess:
2580 def __init__(self):