@samitouri / QOSami_Agents_Zero-2 / commits / ee0acc72

Improve browser iframe DOM actions

Add an Agent Zero owned browser DOM helper that captures shadow DOM and iframe content with frame-chain/node references.\n\nInstall the DOM helper before page-content capture for both local and host-browser runtimes, and send DOM helper payloads to A0 CLI host browser sessions when needed.\n\nCover iframe content refs and host-browser payload delivery in focused regression tests.

Alessandro committed May 26, 2026 at 17:36 UTC ee0acc72f910433b6dd6d8dbc1f6db1a8db06e77
5 files changed +1502 -4
plugins/_browser/assets/browser-dom-helper.js new
+1343
@@ -0,0 +1,1343 @@
1 +(() => {
2 + const DOM_HELPER_CHANNEL = "a0.browser.dom_helper";
3 + const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";
4 + const VERSION = "1";
5 + const REQUEST_TIMEOUT_MS = 800;
6 +
7 + if (globalThis[DOM_HELPER_KEY]?.version === VERSION) {
8 + return;
9 + }
10 +
11 + const INTERACTIVE_ROLES = new Set([
12 + "button",
13 + "checkbox",
14 + "combobox",
15 + "link",
16 + "menuitem",
17 + "menuitemcheckbox",
18 + "menuitemradio",
19 + "option",
20 + "radio",
21 + "searchbox",
22 + "slider",
23 + "spinbutton",
24 + "switch",
25 + "tab",
26 + "textbox"
27 + ]);
28 + const STRUCTURAL_ROLES = new Set([
29 + "alertdialog",
30 + "article",
31 + "banner",
32 + "complementary",
33 + "contentinfo",
34 + "dialog",
35 + "document",
36 + "form",
37 + "group",
38 + "main",
39 + "navigation",
40 + "none",
41 + "presentation",
42 + "region"
43 + ]);
44 + const INTERACTIVE_EVENT_NAMES = new Set([
45 + "auxclick",
46 + "change",
47 + "click",
48 + "contextmenu",
49 + "dblclick",
50 + "input",
51 + "keydown",
52 + "keypress",
53 + "keyup",
54 + "mousedown",
55 + "mouseup",
56 + "pointerdown",
57 + "pointerup",
58 + "submit",
59 + "touchend",
60 + "touchstart"
61 + ]);
62 + const INTERACTIVE_EVENT_PROPERTIES = [...INTERACTIVE_EVENT_NAMES]
63 + .map((eventName) => `on${eventName}`);
64 + const SKIP_TAGS = new Set([
65 + "HEAD",
66 + "LINK",
67 + "META",
68 + "NOSCRIPT",
69 + "SCRIPT",
70 + "STYLE",
71 + "TEMPLATE"
72 + ]);
73 + const VOID_TAGS = new Set([
74 + "area",
75 + "base",
76 + "br",
77 + "col",
78 + "embed",
79 + "hr",
80 + "img",
81 + "input",
82 + "link",
83 + "meta",
84 + "param",
85 + "source",
86 + "track",
87 + "wbr"
88 + ]);
89 +
90 + const childFramesById = new Map();
91 + const elementsByNodeId = new Map();
92 + const nodeIdsByElement = new WeakMap();
93 + const pendingRequests = new Map();
94 + const helperFrameId = typeof globalThis.crypto?.randomUUID === "function"
95 + ? globalThis.crypto.randomUUID()
96 + : `a0-browser-frame-${Date.now()}-${Math.random().toString(16).slice(2)}`;
97 + let nextNodeId = 1;
98 + let nextRequestId = 1;
99 +
100 + function patchOpenShadowDom() {
101 + const original = globalThis.Element?.prototype?.attachShadow;
102 + if (!original || original.__a0BrowserDomHelperOpenShadowPatch) {
103 + return;
104 + }
105 + const patched = function attachShadow(options) {
106 + return original.call(this, { ...(options || {}), mode: "open" });
107 + };
108 + patched.__a0BrowserDomHelperOpenShadowPatch = true;
109 + globalThis.Element.prototype.attachShadow = patched;
110 + }
111 +
112 + patchOpenShadowDom();
113 +
114 + function createNamedError(name, message, details = {}) {
115 + const error = new Error(message);
116 + error.name = name;
117 + Object.assign(error, details);
118 + return error;
119 + }
120 +
121 + function normalizeText(value) {
122 + return String(value ?? "").replace(/\s+/gu, " ").trim();
123 + }
124 +
125 + function normalizeAttributeText(value) {
126 + return normalizeText(value).slice(0, 160);
127 + }
128 +
129 + function truncateText(value, maxLength = 120) {
130 + const normalizedValue = normalizeText(value);
131 + if (normalizedValue.length <= maxLength) {
132 + return normalizedValue;
133 + }
134 + return `${normalizedValue.slice(0, Math.max(0, maxLength - 1)).trimEnd()}...`;
135 + }
136 +
137 + function escapeHtmlText(value) {
138 + return String(value ?? "")
139 + .replace(/&/gu, "&amp;")
140 + .replace(/</gu, "&lt;")
141 + .replace(/>/gu, "&gt;");
142 + }
143 +
144 + function escapeHtmlAttribute(value) {
145 + return escapeHtmlText(value)
146 + .replace(/"/gu, "&quot;")
147 + .replace(/'/gu, "&#39;");
148 + }
149 +
150 + function isElementNode(value) {
151 + return Boolean(value && value.nodeType === 1);
152 + }
153 +
154 + function isTextNode(value) {
155 + return Boolean(value && value.nodeType === 3);
156 + }
157 +
158 + function getTagName(element) {
159 + return String(element?.tagName || "").toUpperCase();
160 + }
161 +
162 + function getAttributeNamesSafe(element) {
163 + try {
164 + if (typeof element?.getAttributeNames === "function") {
165 + return element.getAttributeNames();
166 + }
167 + return [...(element?.attributes || [])]
168 + .map((attribute) => String(attribute?.name || "").trim())
169 + .filter(Boolean);
170 + } catch {
171 + return [];
172 + }
173 + }
174 +
175 + function getComputedStyleSafe(element) {
176 + try {
177 + return globalThis.getComputedStyle?.(element) || null;
178 + } catch {
179 + return null;
180 + }
181 + }
182 +
183 + function isStyleDeclarationHidden(styleValue) {
184 + const normalizedStyleValue = String(styleValue || "")
185 + .toLowerCase()
186 + .replace(/\s+/gu, "");
187 + if (!normalizedStyleValue) {
188 + return false;
189 + }
190 + return /(?:^|;)display:none(?:;|$)/u.test(normalizedStyleValue)
191 + || /(?:^|;)visibility:hidden(?:;|$)/u.test(normalizedStyleValue)
192 + || /(?:^|;)visibility:collapse(?:;|$)/u.test(normalizedStyleValue)
193 + || /(?:^|;)content-visibility:hidden(?:;|$)/u.test(normalizedStyleValue)
194 + || /(?:^|;)opacity:0(?:\.0+)?(?:;|$)/u.test(normalizedStyleValue);
195 + }
196 +
197 + function isComputedStyleHidden(computedStyle) {
198 + if (!computedStyle) {
199 + return false;
200 + }
201 + const display = normalizeText(computedStyle.display).toLowerCase();
202 + const visibility = normalizeText(computedStyle.visibility).toLowerCase();
203 + const contentVisibility = normalizeText(computedStyle.contentVisibility).toLowerCase();
204 + const opacity = Number(computedStyle.opacity || 1);
205 + return display === "none"
206 + || visibility === "hidden"
207 + || visibility === "collapse"
208 + || contentVisibility === "hidden"
209 + || opacity <= 0;
210 + }
211 +
212 + function isEffectivelyHiddenByAncestor(element) {
213 + let current = element;
214 + while (isElementNode(current)) {
215 + if (current.hidden || current.getAttribute?.("aria-hidden") === "true") {
216 + return true;
217 + }
218 + if (isStyleDeclarationHidden(current.getAttribute?.("style"))) {
219 + return true;
220 + }
221 + if (isComputedStyleHidden(getComputedStyleSafe(current))) {
222 + return true;
223 + }
224 + current = current.parentElement;
225 + }
226 + return false;
227 + }
228 +
229 + function isHiddenElement(element) {
230 + if (!isElementNode(element)) {
231 + return true;
232 + }
233 + const tagName = getTagName(element);
234 + if (SKIP_TAGS.has(tagName)) {
235 + return true;
236 + }
237 + if (element.hidden || element.getAttribute?.("aria-hidden") === "true") {
238 + return true;
239 + }
240 + if (tagName === "INPUT" && String(element.getAttribute?.("type") || "").toLowerCase() === "hidden") {
241 + return true;
242 + }
243 + if (isStyleDeclarationHidden(element.getAttribute?.("style"))) {
244 + return true;
245 + }
246 + if (isComputedStyleHidden(getComputedStyleSafe(element))) {
247 + return true;
248 + }
249 + return isEffectivelyHiddenByAncestor(element.parentElement);
250 + }
251 +
252 + function normalizeInteractiveEventName(value) {
253 + return String(value || "")
254 + .trim()
255 + .toLowerCase()
256 + .split(/[.:]/u, 1)[0];
257 + }
258 +
259 + function isGlobalOrDelegatedEventBinding(value) {
260 + const parts = String(value || "")
261 + .trim()
262 + .toLowerCase()
263 + .split(/[.:]/u)
264 + .map((part) => part.trim())
265 + .filter(Boolean);
266 + return parts.includes("window")
267 + || parts.includes("document")
268 + || parts.includes("outside")
269 + || parts.includes("away");
270 + }
271 +
272 + function isInteractiveEventName(value) {
273 + return INTERACTIVE_EVENT_NAMES.has(normalizeInteractiveEventName(value));
274 + }
275 +
276 + function isInteractiveEventAttributeName(attributeName) {
277 + const normalizedName = String(attributeName || "").trim().toLowerCase();
278 + if (!normalizedName) {
279 + return false;
280 + }
281 + if (normalizedName.startsWith("@")) {
282 + return !isGlobalOrDelegatedEventBinding(normalizedName.slice(1))
283 + && isInteractiveEventName(normalizedName.slice(1));
284 + }
285 + if (normalizedName.startsWith("x-on:") || normalizedName.startsWith("v-on:")) {
286 + return !isGlobalOrDelegatedEventBinding(normalizedName.slice(5))
287 + && isInteractiveEventName(normalizedName.slice(5));
288 + }
289 + if (normalizedName.startsWith("ng-")) {
290 + return isInteractiveEventName(normalizedName.slice(3));
291 + }
292 + if (normalizedName.startsWith("on") && normalizedName.length > 2) {
293 + return isInteractiveEventName(normalizedName.slice(2));
294 + }
295 + return false;
296 + }
297 +
298 + function hasInteractiveEventHandlerAttribute(element) {
299 + return getAttributeNamesSafe(element).some((attributeName) => {
300 + return isInteractiveEventAttributeName(attributeName);
301 + });
302 + }
303 +
304 + function hasInteractiveEventHandlerProperty(element) {
305 + return INTERACTIVE_EVENT_PROPERTIES.some((propertyName) => {
306 + return typeof element?.[propertyName] === "function";
307 + });
308 + }
309 +
310 + function hasInteractiveEventHandler(element) {
311 + return hasInteractiveEventHandlerAttribute(element) || hasInteractiveEventHandlerProperty(element);
312 + }
313 +
314 + function isActionableElement(element) {
315 + if (!isElementNode(element) || isHiddenElement(element)) {
316 + return false;
317 + }
318 + const tagName = getTagName(element);
319 + if (tagName === "A" && element.hasAttribute?.("href")) {
320 + return true;
321 + }
322 + if (tagName === "IMG") {
323 + return true;
324 + }
325 + if (["BUTTON", "INPUT", "SELECT", "TEXTAREA", "SUMMARY"].includes(tagName)) {
326 + return true;
327 + }
328 + if (String(element.getAttribute?.("contenteditable") || "").toLowerCase() === "true") {
329 + return true;
330 + }
331 + const role = String(element.getAttribute?.("role") || "").trim().toLowerCase();
332 + if (INTERACTIVE_ROLES.has(role)) {
333 + return true;
334 + }
335 + if (STRUCTURAL_ROLES.has(role)) {
336 + return false;
337 + }
338 + if (hasInteractiveEventHandlerAttribute(element)) {
339 + return true;
340 + }
341 + return hasInteractiveEventHandlerProperty(element) && Boolean(normalizeText(element.textContent || ""));
342 + }
343 +
344 + function normalizeFrameChain(frameChain) {
345 + const rawFrameChain = Array.isArray(frameChain)
346 + ? frameChain
347 + : typeof frameChain === "string"
348 + ? frameChain.split(">")
349 + : [];
350 + return rawFrameChain
351 + .map((entry) => String(entry || "").trim())
352 + .filter(Boolean);
353 + }
354 +
355 + function encodeFrameChain(frameChain) {
356 + return normalizeFrameChain(frameChain).join(">");
357 + }
358 +
359 + function ensureNodeId(element) {
360 + if (nodeIdsByElement.has(element)) {
361 + return nodeIdsByElement.get(element);
362 + }
363 + const nodeId = String(nextNodeId++);
364 + nodeIdsByElement.set(element, nodeId);
365 + elementsByNodeId.set(nodeId, element);
366 + return nodeId;
367 + }
368 +
369 + function normalizeSnapshotMode(value) {
370 + return String(value || "").trim().toLowerCase() || "dom";
371 + }
372 +
373 + function isContentSnapshotMode(payload = {}) {
374 + return normalizeSnapshotMode(payload?.snapshotMode) === "content";
375 + }
376 +
377 + function normalizeSelectorList(payload = {}) {
378 + const rawSelectors = typeof payload === "string"
379 + ? [payload]
380 + : Array.isArray(payload?.selectors)
381 + ? payload.selectors
382 + : typeof payload?.selectors === "string"
383 + ? [payload.selectors]
384 + : Array.isArray(payload?.selector)
385 + ? payload.selector
386 + : typeof payload?.selector === "string"
387 + ? [payload.selector]
388 + : [];
389 + return rawSelectors.map((selector) => String(selector || "").trim()).filter(Boolean);
390 + }
391 +
392 + function getReferenceValueMetadata(element) {
393 + const tagName = getTagName(element);
394 + if (tagName === "INPUT") {
395 + const inputType = String(element.getAttribute?.("type") || element.type || "text").toLowerCase();
396 + if (inputType === "password") {
397 + return "";
398 + }
399 + if (["checkbox", "radio"].includes(inputType)) {
400 + return element.checked ? "checked" : "unchecked";
401 + }
402 + return truncateText(element.value || element.getAttribute?.("value") || "", 96);
403 + }
404 + if (tagName === "TEXTAREA") {
405 + return truncateText(element.value || "", 96);
406 + }
407 + if (tagName === "SELECT") {
408 + return [...(element.selectedOptions || [])]
409 + .map((option) => truncateText(option.textContent || option.label || option.value || "", 48))
410 + .filter(Boolean)
411 + .join(" | ");
412 + }
413 + if (String(element.getAttribute?.("contenteditable") || "").toLowerCase() === "true") {
414 + return truncateText(element.textContent || "", 96);
415 + }
416 + return "";
417 + }
418 +
419 + function collectElementStateMetadata(element) {
420 + if (!isElementNode(element)) {
421 + return {
422 + descriptorTags: [],
423 + semanticTags: [],
424 + stateTags: []
425 + };
426 + }
427 + const tagName = getTagName(element);
428 + const computedStyle = getComputedStyleSafe(element);
429 + const ariaDisabled = String(element.getAttribute?.("aria-disabled") || "").trim().toLowerCase() === "true";
430 + const ariaChecked = String(element.getAttribute?.("aria-checked") || "").trim().toLowerCase() === "true";
431 + const ariaExpanded = String(element.getAttribute?.("aria-expanded") || "").trim().toLowerCase() === "true";
432 + const ariaInvalid = String(element.getAttribute?.("aria-invalid") || "").trim().toLowerCase() === "true";
433 + const ariaPressed = String(element.getAttribute?.("aria-pressed") || "").trim().toLowerCase() === "true";
434 + const ariaReadonly = String(element.getAttribute?.("aria-readonly") || "").trim().toLowerCase() === "true";
435 + const ariaRequired = String(element.getAttribute?.("aria-required") || "").trim().toLowerCase() === "true";
436 + const ariaSelected = String(element.getAttribute?.("aria-selected") || "").trim().toLowerCase() === "true";
437 + const disabled = Boolean(element.disabled || ariaDisabled || element.closest?.("[inert]"));
438 + const checked = Boolean(element.checked || ariaChecked);
439 + const selected = tagName === "OPTION" ? Boolean(element.selected) : ariaSelected;
440 + const invalid = Boolean(ariaInvalid || element.matches?.(":invalid"));
441 + const readonly = Boolean(element.readOnly || ariaReadonly);
442 + const required = Boolean(element.required || ariaRequired);
443 + const blocked = !disabled && normalizeText(computedStyle?.pointerEvents || "").toLowerCase() === "none";
444 + const stateTags = [
445 + disabled ? "disabled" : "",
446 + blocked ? "blocked" : "",
447 + checked ? "checked" : "",
448 + selected && tagName !== "SELECT" ? "selected" : "",
449 + invalid ? "invalid" : "",
450 + ariaExpanded ? "expanded" : "",
451 + ariaPressed ? "pressed" : ""
452 + ].filter(Boolean);
453 + return {
454 + blocked,
455 + checked,
456 + descriptorTags: stateTags.slice(),
457 + disabled,
458 + expanded: ariaExpanded,
459 + invalid,
460 + pointerEventsNone: blocked,
461 + pressed: ariaPressed,
462 + readonly,
463 + required,
464 + selected,
465 + semanticTags: [],
466 + stateTags,
467 + visible: !isHiddenElement(element)
468 + };
469 + }
470 +
471 + function serializeAttributes(element, frameChain) {
472 + const serializedAttributes = [];
473 + const helperManagedAttributes = new Set([
474 + "data-space-browser-node-id",
475 + "data-space-browser-frame-id",
476 + "data-space-browser-frame-chain",
477 + "data-space-browser-state-tags",
478 + "data-space-browser-semantic-tags",
479 + "data-space-browser-descriptor-tags",
480 + "data-space-browser-live-value",
481 + "data-space-browser-selected-text"
482 + ]);
483 + try {
484 + [...(element?.attributes || [])].forEach((attribute) => {
485 + const name = String(attribute?.name || "").trim();
486 + if (!name || helperManagedAttributes.has(name)) {
487 + return;
488 + }
489 + serializedAttributes.push(` ${name}="${escapeHtmlAttribute(attribute?.value || "")}"`);
490 + });
491 + } catch {
492 + // Attribute reads are best effort across unusual DOM nodes.
493 + }
494 +
495 + if (isActionableElement(element)) {
496 + const stateMetadata = collectElementStateMetadata(element);
497 + const liveValue = getReferenceValueMetadata(element);
498 + serializedAttributes.push(` data-space-browser-node-id="${escapeHtmlAttribute(ensureNodeId(element))}"`);
499 + serializedAttributes.push(` data-space-browser-frame-id="${escapeHtmlAttribute(helperFrameId)}"`);
500 + serializedAttributes.push(` data-space-browser-frame-chain="${escapeHtmlAttribute(encodeFrameChain(frameChain))}"`);
501 + if (stateMetadata.stateTags.length) {
502 + serializedAttributes.push(` data-space-browser-state-tags="${escapeHtmlAttribute(stateMetadata.stateTags.join(" "))}"`);
503 + }
504 + if (stateMetadata.semanticTags.length) {
505 + serializedAttributes.push(` data-space-browser-semantic-tags="${escapeHtmlAttribute(stateMetadata.semanticTags.join(" "))}"`);
506 + }
507 + if (stateMetadata.descriptorTags.length) {
508 + serializedAttributes.push(` data-space-browser-descriptor-tags="${escapeHtmlAttribute(stateMetadata.descriptorTags.join(" "))}"`);
509 + }
510 + if (liveValue) {
511 + serializedAttributes.push(` data-space-browser-live-value="${escapeHtmlAttribute(liveValue)}"`);
512 + if (getTagName(element) === "SELECT") {
513 + serializedAttributes.push(` data-space-browser-selected-text="${escapeHtmlAttribute(liveValue)}"`);
514 + }
515 + }
516 + }
517 + return serializedAttributes.join("");
518 + }
519 +
520 + function isFrameLikeElement(element) {
521 + return ["IFRAME", "FRAME", "OBJECT", "EMBED"].includes(getTagName(element));
522 + }
523 +
524 + function createRequestId() {
525 + nextRequestId += 1;
526 + return `a0-browser-dom-${Date.now()}-${nextRequestId}-${Math.random().toString(16).slice(2)}`;
527 + }
528 +
529 + function resolveElementWindow(element) {
530 + try {
531 + return element?.contentWindow || null;
532 + } catch {
533 + return null;
534 + }
535 + }
536 +
537 + function requestChildFrameOperation(targetWindow, type, payload = {}, frameElement = null) {
538 + if (!targetWindow || typeof targetWindow.postMessage !== "function") {
539 + throw createNamedError(
540 + "BrowserDomHelperFrameUnavailableError",
541 + "Embedded frame window is unavailable.",
542 + {
543 + code: "browser_dom_helper_frame_window_unavailable",
544 + details: { frameElementTag: getTagName(frameElement).toLowerCase() }
545 + }
546 + );
547 + }
548 + return new Promise((resolve, reject) => {
549 + const requestId = createRequestId();
550 + const timer = globalThis.setTimeout(() => {
551 + pendingRequests.delete(requestId);
552 + reject(createNamedError(
553 + "BrowserDomHelperFrameTimeoutError",
554 + `Embedded frame request "${type}" timed out.`,
555 + { code: "browser_dom_helper_frame_timeout", details: { type } }
556 + ));
557 + }, REQUEST_TIMEOUT_MS);
558 + pendingRequests.set(requestId, { reject, resolve, timer, type });
559 + try {
560 + targetWindow.postMessage({
561 + channel: DOM_HELPER_CHANNEL,
562 + payload,
563 + requestId,
564 + type
565 + }, "*");
566 + } catch (error) {
567 + globalThis.clearTimeout(timer);
568 + pendingRequests.delete(requestId);
569 + reject(createNamedError(
570 + "BrowserDomHelperFrameRequestError",
571 + `Embedded frame request "${type}" could not be posted.`,
572 + { cause: error, code: "browser_dom_helper_frame_postmessage_failed", details: { type } }
573 + ));
574 + }
575 + });
576 + }
577 +
578 + function registerChildFrame(frameId, targetWindow) {
579 + const normalizedFrameId = String(frameId || "").trim();
580 + if (normalizedFrameId && targetWindow && typeof targetWindow.postMessage === "function") {
581 + childFramesById.set(normalizedFrameId, targetWindow);
582 + }
583 + }
584 +
585 + function extractDocumentBodyHtml(html) {
586 + const normalizedHtml = String(html || "").trim();
587 + if (!normalizedHtml) {
588 + return "";
589 + }
590 + try {
591 + if (typeof DOMParser === "function") {
592 + const parsedDocument = new DOMParser().parseFromString(normalizedHtml, "text/html");
593 + return String(parsedDocument.body?.innerHTML || normalizedHtml).trim();
594 + }
595 + } catch {
596 + // Fall through to the raw snapshot.
597 + }
598 + return normalizedHtml.replace(/<!doctype[\s\S]*?>/iu, "").trim();
599 + }
600 +
601 + async function captureFrameElement(frameElement, frameChain, payload = {}) {
602 + const childWindow = resolveElementWindow(frameElement);
603 + const childPayload = {
604 + snapshotMode: normalizeSnapshotMode(payload?.snapshotMode),
605 + parentFrameChain: frameChain
606 + };
607 + if (!childWindow) {
608 + return {
609 + frameChain: [],
610 + frameId: "",
611 + html: escapeHtmlText("Embedded frame snapshot unavailable."),
612 + message: "Embedded frame snapshot unavailable.",
613 + ok: false,
614 + status: "window_unavailable",
615 + title: "",
616 + url: String(frameElement?.getAttribute?.("src") || "").trim()
617 + };
618 + }
619 +
620 + try {
621 + const snapshot = await requestChildFrameOperation(childWindow, "capture_document", childPayload, frameElement);
622 + registerChildFrame(snapshot?.frameId, childWindow);
623 + return snapshot;
624 + } catch (error) {
625 + try {
626 + const frameDocument = frameElement?.contentDocument;
627 + if (frameDocument) {
628 + const currentFrameChain = normalizeFrameChain(frameChain);
629 + return {
630 + frameChain: currentFrameChain,
631 + frameId: helperFrameId,
632 + html: await serializeDocumentNode(frameDocument, currentFrameChain, childPayload),
633 + ok: true,
634 + title: String(frameDocument.title || "").trim(),
635 + url: String(childWindow.location?.href || frameElement?.src || "").trim()
636 + };
637 + }
638 + } catch {
639 + // Cross-origin frames stay best effort and report the postMessage failure below.
640 + }
641 + return {
642 + frameChain: normalizeFrameChain(frameChain),
643 + frameId: "",
644 + html: escapeHtmlText(String(error?.message || "Embedded frame snapshot unavailable.")),
645 + message: String(error?.message || "Embedded frame snapshot unavailable."),
646 + ok: false,
647 + status: String(error?.code || "capture_failed"),
648 + title: String(frameElement?.getAttribute?.("title") || "").trim(),
649 + url: String(frameElement?.getAttribute?.("src") || frameElement?.src || "").trim()
650 + };
651 + }
652 + }
653 +
654 + function renderFrameDocument(snapshot, frameElement, payload = {}) {
655 + const normalizedSnapshot = snapshot && typeof snapshot === "object" ? snapshot : {};
656 + const content = String(
657 + normalizedSnapshot.html
658 + || escapeHtmlText(normalizedSnapshot.message || "Embedded frame snapshot unavailable.")
659 + );
660 + if (isContentSnapshotMode(payload)) {
661 + return normalizedSnapshot.ok === false ? "" : extractDocumentBodyHtml(content);
662 + }
663 + return `<space-browser-frame-document`
664 + + ` data-space-browser-frame-id="${escapeHtmlAttribute(normalizedSnapshot.frameId || "")}"`
665 + + ` data-space-browser-frame-chain="${escapeHtmlAttribute(encodeFrameChain(normalizedSnapshot.frameChain))}"`
666 + + ` data-space-browser-status="${escapeHtmlAttribute(normalizedSnapshot.ok === false ? normalizedSnapshot.status || "error" : "ok")}"`
667 + + ` data-space-browser-frame-url="${escapeHtmlAttribute(normalizedSnapshot.url || frameElement?.src || "")}"`
668 + + ` data-space-browser-frame-title="${escapeHtmlAttribute(normalizedSnapshot.title || frameElement?.getAttribute?.("title") || "")}">`
669 + + content
670 + + `</space-browser-frame-document>`;
671 + }
672 +
673 + async function serializeChildNodes(parentNode, frameChain, payload = {}) {
674 + const parts = [];
675 + for (const childNode of Array.from(parentNode?.childNodes || [])) {
676 + parts.push(await serializeNode(childNode, frameChain, payload));
677 + }
678 + return parts.join("");
679 + }
680 +
681 + async function serializeElementNode(element, frameChain, payload = {}) {
682 + const tagName = String(element?.tagName || "").toLowerCase();
683 + if (!tagName) {
684 + return "";
685 + }
686 + if (isContentSnapshotMode(payload) && isHiddenElement(element)) {
687 + return "";
688 + }
689 + const openTag = `<${tagName}${serializeAttributes(element, frameChain)}>`;
690 + const lightDom = await serializeChildNodes(element, frameChain, payload);
691 + let shadowDom = "";
692 + try {
693 + const shadowRoot = element?.shadowRoot;
694 + if (shadowRoot) {
695 + const shadowInnerHtml = await serializeChildNodes(shadowRoot, frameChain, payload);
696 + shadowDom = isContentSnapshotMode(payload)
697 + ? shadowInnerHtml
698 + : `<space-browser-shadow-root>${shadowInnerHtml}</space-browser-shadow-root>`;
699 + }
700 + } catch {
701 + shadowDom = "";
702 + }
703 + let frameDom = "";
704 + if (isFrameLikeElement(element)) {
705 + frameDom = renderFrameDocument(
706 + await captureFrameElement(element, frameChain, payload),
707 + element,
708 + payload
709 + );
710 + if (isContentSnapshotMode(payload)) {
711 + return frameDom;
712 + }
713 + }
714 + if (VOID_TAGS.has(tagName)) {
715 + return `${openTag}${shadowDom}${frameDom}`;
716 + }
717 + return `${openTag}${lightDom}${shadowDom}${frameDom}</${tagName}>`;
718 + }
719 +
720 + async function serializeNode(node, frameChain, payload = {}) {
721 + if (!node || typeof node.nodeType !== "number") {
722 + return "";
723 + }
724 + if (node.nodeType === 9) {
725 + return serializeDocumentNode(node, frameChain, payload);
726 + }
727 + if (node.nodeType === 11) {
728 + return serializeChildNodes(node, frameChain, payload);
729 + }
730 + if (node.nodeType === 1) {
731 + return serializeElementNode(node, frameChain, payload);
732 + }
733 + if (node.nodeType === 3) {
734 + return escapeHtmlText(node.textContent || "");
735 + }
736 + if (node.nodeType === 8 && !isContentSnapshotMode(payload)) {
737 + return `<!--${escapeHtmlText(node.data || "")}-->`;
738 + }
739 + return "";
740 + }
741 +
742 + async function serializeDocumentNode(doc, frameChain, payload = {}) {
743 + return serializeChildNodes(doc, frameChain, payload);
744 + }
745 +
746 + async function serializeSelectorTargets(doc, selectors, frameChain, payload = {}) {
747 + const targets = {};
748 + for (const selector of selectors) {
749 + let elements = [];
750 + try {
751 + elements = [...(doc?.querySelectorAll?.(selector) || [])];
752 + } catch (error) {
753 + throw createNamedError(
754 + "BrowserDomHelperSelectorError",
755 + `Browser DOM helper could not resolve selector "${selector}".`,
756 + { cause: error, code: "browser_dom_helper_selector_error", details: { selector } }
757 + );
758 + }
759 + const parts = [];
760 + for (const element of elements) {
761 + parts.push(await serializeNode(element, frameChain, payload));
762 + }
763 + targets[selector] = parts.join("");
764 + }
765 + return targets;
766 + }
767 +
768 + async function captureDocument(payload = {}) {
769 + childFramesById.clear();
770 + const currentFrameChain = normalizeFrameChain(payload?.parentFrameChain).concat(helperFrameId);
771 + const selectors = normalizeSelectorList(payload);
772 + const documentSnapshot = {
773 + frameChain: currentFrameChain,
774 + frameId: helperFrameId,
775 + ok: true,
776 + title: String(globalThis.document?.title || "").trim(),
777 + url: String(globalThis.location?.href || "").trim()
778 + };
779 + if (selectors.length) {
780 + return {
781 + ...documentSnapshot,
782 + targets: await serializeSelectorTargets(globalThis.document, selectors, currentFrameChain, payload)
783 + };
784 + }
785 + return {
786 + ...documentSnapshot,
787 + html: await serializeDocumentNode(globalThis.document, currentFrameChain, payload)
788 + };
789 + }
790 +
791 + function getElementByNodeId(nodeId, actionLabel) {
792 + const normalizedNodeId = String(nodeId || "").trim();
793 + if (!normalizedNodeId) {
794 + throw createNamedError(
795 + "BrowserDomHelperReferenceError",
796 + `Browser DOM helper ${actionLabel} requires a node id.`,
797 + { code: "browser_dom_helper_node_required", details: { action: actionLabel } }
798 + );
799 + }
800 + const element = elementsByNodeId.get(normalizedNodeId);
801 + if (!element) {
802 + throw createNamedError(
803 + "BrowserDomHelperReferenceError",
804 + `Browser DOM helper could not find node "${normalizedNodeId}".`,
805 + { code: "browser_dom_helper_node_not_found", details: { action: actionLabel, nodeId: normalizedNodeId } }
806 + );
807 + }
808 + if (element.isConnected === false) {
809 + throw createNamedError(
810 + "BrowserDomHelperReferenceError",
811 + `Browser DOM helper node "${normalizedNodeId}" is no longer connected.`,
812 + { code: "browser_dom_helper_node_disconnected", details: { action: actionLabel, nodeId: normalizedNodeId } }
813 + );
814 + }
815 + return element;
816 + }
817 +
818 + function serializeElementSnapshot(element) {
819 + if (!isElementNode(element)) {
820 + return "";
821 + }
822 + try {
823 + if (typeof element.outerHTML === "string" && element.outerHTML) {
824 + return element.outerHTML;
825 + }
826 + } catch {
827 + // Fall through.
828 + }
829 + return "";
830 + }
831 +
832 + function scrollElementIntoView(element) {
833 + try {
834 + element.scrollIntoView?.({ behavior: "auto", block: "center", inline: "center" });
835 + return true;
836 + } catch {
837 + return false;
838 + }
839 + }
840 +
841 + function focusElement(element) {
842 + try {
843 + element.focus?.({ preventScroll: true });
844 + return true;
845 + } catch {
846 + try {
847 + element.focus?.();
848 + return true;
849 + } catch {
850 + return false;
851 + }
852 + }
853 + }
854 +
855 + function dispatchDomEvent(target, eventName, EventType = "Event", options = {}) {
856 + const EventConstructor = typeof globalThis[EventType] === "function"
857 + ? globalThis[EventType]
858 + : globalThis.Event;
859 + const event = new EventConstructor(eventName, {
860 + bubbles: true,
861 + cancelable: true,
862 + composed: true,
863 + ...options
864 + });
865 + target.dispatchEvent(event);
866 + return event;
867 + }
868 +
869 + function dispatchKeyboardEvent(target, eventName, options = {}) {
870 + const KeyboardEventConstructor = typeof globalThis.KeyboardEvent === "function"
871 + ? globalThis.KeyboardEvent
872 + : globalThis.Event;
873 + const event = new KeyboardEventConstructor(eventName, {
874 + bubbles: true,
875 + cancelable: true,
876 + composed: true,
877 + code: "Enter",
878 + key: "Enter",
879 + keyCode: 13,
880 + which: 13,
881 + ...options
882 + });
883 + target.dispatchEvent(event);
884 + return event;
885 + }
886 +
887 + function setNativeValue(element, nextValue) {
888 + const tagName = getTagName(element);
889 + const normalizedValue = String(nextValue ?? "");
890 + if (tagName === "INPUT") {
891 + const descriptor = Object.getOwnPropertyDescriptor(globalThis.HTMLInputElement?.prototype || {}, "value");
892 + if (typeof descriptor?.set === "function") {
893 + descriptor.set.call(element, normalizedValue);
894 + } else {
895 + element.value = normalizedValue;
896 + }
897 + return normalizedValue;
898 + }
899 + if (tagName === "TEXTAREA") {
900 + const descriptor = Object.getOwnPropertyDescriptor(globalThis.HTMLTextAreaElement?.prototype || {}, "value");
901 + if (typeof descriptor?.set === "function") {
902 + descriptor.set.call(element, normalizedValue);
903 + } else {
904 + element.value = normalizedValue;
905 + }
906 + return normalizedValue;
907 + }
908 + if (tagName === "SELECT") {
909 + element.value = normalizedValue;
910 + return element.value;
911 + }
912 + if (String(element.getAttribute?.("contenteditable") || "").toLowerCase() === "true") {
913 + element.textContent = normalizedValue;
914 + return normalizedValue;
915 + }
916 + throw createNamedError(
917 + "BrowserDomHelperActionError",
918 + `Browser DOM helper cannot type into <${getTagName(element).toLowerCase()}>.`,
919 + { code: "browser_dom_helper_type_unsupported" }
920 + );
921 + }
922 +
923 + function delayMs(timeoutMs) {
924 + return new Promise((resolve) => {
925 + globalThis.setTimeout(resolve, Math.max(0, Number(timeoutMs) || 0));
926 + });
927 + }
928 +
929 + function describeActiveElement(element) {
930 + if (!isElementNode(element)) {
931 + return "";
932 + }
933 + return [
934 + getTagName(element).toLowerCase(),
935 + normalizeAttributeText(element.getAttribute?.("id")) ? `#${normalizeAttributeText(element.getAttribute?.("id"))}` : "",
936 + normalizeAttributeText(element.getAttribute?.("name")) ? `name=${normalizeAttributeText(element.getAttribute?.("name"))}` : ""
937 + ].filter(Boolean).join(" ");
938 + }
939 +
940 + function getActionObservationRoot(element) {
941 + return element?.closest?.("form, fieldset, dialog, [role='dialog'], [role='alert'], [role='status'], [aria-live], article, section, main, li, tr, td, th")
942 + || element?.parentElement
943 + || element
944 + || globalThis.document?.body
945 + || globalThis.document?.documentElement
946 + || null;
947 + }
948 +
949 + function captureActionEffectSnapshot(element) {
950 + const observationRoot = getActionObservationRoot(element);
951 + return {
952 + activeElement: describeActiveElement(globalThis.document?.activeElement),
953 + observationRoot,
954 + observationText: truncateText(normalizeText(observationRoot?.textContent || ""), 2000),
955 + targetDom: truncateText(serializeElementSnapshot(element), 2000),
956 + targetState: collectElementStateMetadata(element),
957 + value: getReferenceValueMetadata(element)
958 + };
959 + }
960 +
961 + async function withObservedActionWindow(observationRoot, action, { quietMs = 40, timeoutMs = 180 } = {}) {
962 + const target = observationRoot?.ownerDocument?.body
963 + || observationRoot?.ownerDocument?.documentElement
964 + || globalThis.document?.body
965 + || globalThis.document?.documentElement;
966 + if (!target || typeof globalThis.MutationObserver !== "function") {
967 + const result = await action();
968 + await delayMs(timeoutMs);
969 + return { observedMutations: { attributeNames: [], mutationCount: 0 }, result };
970 + }
971 + const attributeNames = new Set();
972 + let lastMutationAt = 0;
973 + let mutationCount = 0;
974 + const observer = new globalThis.MutationObserver((mutations) => {
975 + mutationCount += mutations.length;
976 + lastMutationAt = Date.now();
977 + mutations.forEach((mutation) => {
978 + if (mutation.type === "attributes" && mutation.attributeName) {
979 + attributeNames.add(String(mutation.attributeName));
980 + }
981 + });
982 + });
983 + try {
984 + observer.observe(target, {
985 + attributes: true,
986 + characterData: true,
987 + childList: true,
988 + subtree: true
989 + });
990 + const result = await action();
991 + const startedAt = Date.now();
992 + while (Date.now() - startedAt < timeoutMs) {
993 + await delayMs(20);
994 + if (mutationCount > 0 && Date.now() - lastMutationAt >= quietMs) {
995 + break;
996 + }
997 + }
998 + return { observedMutations: { attributeNames: [...attributeNames], mutationCount }, result };
999 + } finally {
1000 + observer.disconnect();
1001 + }
1002 + }
1003 +
1004 + function buildActionEffectResult(beforeSnapshot, afterSnapshot, observedMutations, extra = {}) {
1005 + const focusChanged = beforeSnapshot.activeElement !== afterSnapshot.activeElement;
1006 + const nearbyTextChanged = beforeSnapshot.observationText !== afterSnapshot.observationText;
1007 + const valueChanged = beforeSnapshot.value !== afterSnapshot.value;
1008 + const checkedChanged = beforeSnapshot.targetState.checked !== afterSnapshot.targetState.checked;
1009 + const selectedChanged = beforeSnapshot.targetState.selected !== afterSnapshot.targetState.selected;
1010 + const expandedChanged = beforeSnapshot.targetState.expanded !== afterSnapshot.targetState.expanded;
1011 + const pressedChanged = beforeSnapshot.targetState.pressed !== afterSnapshot.targetState.pressed;
1012 + const targetDomChanged = beforeSnapshot.targetDom !== afterSnapshot.targetDom;
1013 + const domChanged = Boolean(observedMutations.mutationCount) || targetDomChanged || nearbyTextChanged;
1014 + const status = {
1015 + checkedChanged,
1016 + domChanged,
1017 + expandedChanged,
1018 + focusChanged,
1019 + nearbyTextChanged,
1020 + pressedChanged,
1021 + reacted: false,
1022 + selectedChanged,
1023 + targetChanged: targetDomChanged || valueChanged || checkedChanged || selectedChanged || expandedChanged || pressedChanged,
1024 + targetDomChanged,
1025 + valueChanged
1026 + };
1027 + status.reacted = Object.entries(status).some(([key, value]) => key !== "reacted" && value === true);
1028 + status.noObservedEffect = !status.reacted;
1029 + return {
1030 + ...extra,
1031 + effect: {
1032 + mutationAttributes: observedMutations.attributeNames.slice(0, 8),
1033 + mutationCount: observedMutations.mutationCount
1034 + },
1035 + status
1036 + };
1037 + }
1038 +
1039 + function collectActionResult(element) {
1040 + const state = collectElementStateMetadata(element);
1041 + return {
1042 + connected: element.isConnected !== false,
1043 + descriptorTags: state.descriptorTags.slice(),
1044 + dom: serializeElementSnapshot(element),
1045 + frameId: helperFrameId,
1046 + nodeId: ensureNodeId(element),
1047 + semanticTags: state.semanticTags.slice(),
1048 + state,
1049 + tagName: getTagName(element)
1050 + };
1051 + }
1052 +
1053 + function detailLocalNode(payload = {}) {
1054 + return collectActionResult(getElementByNodeId(payload?.nodeId, "detail"));
1055 + }
1056 +
1057 + async function clickLocalNode(payload = {}) {
1058 + const element = getElementByNodeId(payload?.nodeId, "click");
1059 + const beforeSnapshot = captureActionEffectSnapshot(element);
1060 + scrollElementIntoView(element);
1061 + focusElement(element);
1062 + if (beforeSnapshot.targetState.disabled) {
1063 + throw createNamedError(
1064 + "BrowserDomHelperActionError",
1065 + `Browser DOM helper node "${payload?.nodeId}" is disabled.`,
1066 + { code: "browser_dom_helper_click_disabled" }
1067 + );
1068 + }
1069 + const { observedMutations } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {
1070 + if (typeof element.click === "function") {
1071 + element.click();
1072 + } else {
1073 + dispatchDomEvent(element, "click", "MouseEvent", { button: 0 });
1074 + }
1075 + });
1076 + return {
1077 + ...collectActionResult(element),
1078 + ...buildActionEffectResult(beforeSnapshot, captureActionEffectSnapshot(element), observedMutations)
1079 + };
1080 + }
1081 +
1082 + async function typeLocalNode(payload = {}) {
1083 + const element = getElementByNodeId(payload?.nodeId, "type");
1084 + const beforeSnapshot = captureActionEffectSnapshot(element);
1085 + const { observedMutations, result } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {
1086 + scrollElementIntoView(element);
1087 + focusElement(element);
1088 + const appliedValue = setNativeValue(element, payload?.value ?? "");
1089 + dispatchDomEvent(element, "beforeinput", "InputEvent", {
1090 + data: String(payload?.value ?? ""),
1091 + inputType: "insertText"
1092 + });
1093 + dispatchDomEvent(element, "input", "InputEvent", {
1094 + data: String(payload?.value ?? ""),
1095 + inputType: "insertText"
1096 + });
1097 + dispatchDomEvent(element, "change");
1098 + return appliedValue;
1099 + });
1100 + return {
1101 + ...collectActionResult(element),
1102 + ...buildActionEffectResult(beforeSnapshot, captureActionEffectSnapshot(element), observedMutations),
1103 + value: result
1104 + };
1105 + }
1106 +
1107 + async function submitLocalNode(payload = {}) {
1108 + const element = getElementByNodeId(payload?.nodeId, "submit");
1109 + const tagName = getTagName(element);
1110 + const beforeSnapshot = captureActionEffectSnapshot(element);
1111 + const { observedMutations } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {
1112 + scrollElementIntoView(element);
1113 + focusElement(element);
1114 + if (tagName === "FORM" && typeof element.requestSubmit === "function") {
1115 + element.requestSubmit();
1116 + } else if (element.form && typeof element.form.requestSubmit === "function") {
1117 + element.form.requestSubmit(["BUTTON", "INPUT"].includes(tagName) ? element : undefined);
1118 + } else if (element.form) {
1119 + const submitEvent = dispatchDomEvent(element.form, "submit");
1120 + if (!submitEvent.defaultPrevented) {
1121 + element.form.submit?.();
1122 + }
1123 + } else if (typeof element.click === "function") {
1124 + element.click();
1125 + } else {
1126 + throw createNamedError(
1127 + "BrowserDomHelperActionError",
1128 + `Browser DOM helper cannot submit node "${payload?.nodeId}".`,
1129 + { code: "browser_dom_helper_submit_unsupported" }
1130 + );
1131 + }
1132 + });
1133 + return {
1134 + ...collectActionResult(element),
1135 + ...buildActionEffectResult(beforeSnapshot, captureActionEffectSnapshot(element), observedMutations)
1136 + };
1137 + }
1138 +
1139 + async function pressEnterLocalNode(payload = {}) {
1140 + const element = getElementByNodeId(payload?.nodeId, "type_submit");
1141 + const beforeSnapshot = captureActionEffectSnapshot(element);
1142 + const { observedMutations } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {
1143 + scrollElementIntoView(element);
1144 + focusElement(element);
1145 + dispatchKeyboardEvent(element, "keydown");
1146 + dispatchKeyboardEvent(element, "keypress");
1147 + dispatchKeyboardEvent(element, "keyup");
1148 + if (getTagName(element) === "INPUT" && element.form && typeof element.form.requestSubmit === "function") {
1149 + element.form.requestSubmit();
1150 + }
1151 + });
1152 + return {
1153 + ...collectActionResult(element),
1154 + ...buildActionEffectResult(beforeSnapshot, captureActionEffectSnapshot(element), observedMutations)
1155 + };
1156 + }
1157 +
1158 + async function typeSubmitLocalNode(payload = {}) {
1159 + const typed = await typeLocalNode(payload);
1160 + const submitted = await pressEnterLocalNode(payload);
1161 + return {
1162 + ...submitted,
1163 + effect: {
1164 + mutationAttributes: [...new Set([...(typed?.effect?.mutationAttributes || []), ...(submitted?.effect?.mutationAttributes || [])])],
1165 + mutationCount: Number(typed?.effect?.mutationCount || 0) + Number(submitted?.effect?.mutationCount || 0)
1166 + },
1167 + status: {
1168 + ...(typed?.status || {}),
1169 + ...(submitted?.status || {}),
1170 + reacted: Boolean(typed?.status?.reacted || submitted?.status?.reacted),
1171 + noObservedEffect: !Boolean(typed?.status?.reacted || submitted?.status?.reacted)
1172 + },
1173 + value: typed.value
1174 + };
1175 + }
1176 +
1177 + function scrollLocalNode(payload = {}) {
1178 + const element = getElementByNodeId(payload?.nodeId, "scroll");
1179 + const beforeSnapshot = captureActionEffectSnapshot(element);
1180 + scrollElementIntoView(element);
1181 + focusElement(element);
1182 + const scrollEffect = buildActionEffectResult(beforeSnapshot, captureActionEffectSnapshot(element), {
1183 + attributeNames: [],
1184 + mutationCount: 0
1185 + });
1186 + return {
1187 + ...collectActionResult(element),
1188 + ...scrollEffect,
1189 + status: {
1190 + ...scrollEffect.status,
1191 + reacted: true,
1192 + noObservedEffect: false
1193 + }
1194 + };
1195 + }
1196 +
1197 + async function invokeLocalOperation(type, payload = {}) {
1198 + if (type === "capture_document") {
1199 + return captureDocument(payload);
1200 + }
1201 + if (type === "detail_node") {
1202 + return detailLocalNode(payload);
1203 + }
1204 + if (type === "click_node") {
1205 + return clickLocalNode(payload);
1206 + }
1207 + if (type === "type_node") {
1208 + return typeLocalNode(payload);
1209 + }
1210 + if (type === "submit_node") {
1211 + return submitLocalNode(payload);
1212 + }
1213 + if (type === "type_submit_node") {
1214 + return typeSubmitLocalNode(payload);
1215 + }
1216 + if (type === "scroll_node") {
1217 + return scrollLocalNode(payload);
1218 + }
1219 + throw createNamedError(
1220 + "BrowserDomHelperActionError",
1221 + `Browser DOM helper does not support "${type}".`,
1222 + { code: "browser_dom_helper_action_unsupported", details: { type } }
1223 + );
1224 + }
1225 +
1226 + async function routeOperation(type, payload = {}) {
1227 + const frameChain = normalizeFrameChain(payload?.frameChain);
1228 + if (!frameChain.length) {
1229 + return invokeLocalOperation(type, payload);
1230 + }
1231 + if (frameChain[0] !== helperFrameId) {
1232 + throw createNamedError(
1233 + "BrowserDomHelperFrameRouteError",
1234 + `Browser DOM helper cannot route frame chain "${encodeFrameChain(frameChain)}" from "${helperFrameId}".`,
1235 + { code: "browser_dom_helper_frame_chain_mismatch", details: { frameChain } }
1236 + );
1237 + }
1238 + if (frameChain.length === 1) {
1239 + return invokeLocalOperation(type, payload);
1240 + }
1241 + const nextFrameId = frameChain[1];
1242 + const childWindow = childFramesById.get(nextFrameId);
1243 + if (!childWindow) {
1244 + throw createNamedError(
1245 + "BrowserDomHelperFrameRouteError",
1246 + `Browser DOM helper does not know child frame "${nextFrameId}".`,
1247 + { code: "browser_dom_helper_child_frame_missing", details: { frameChain } }
1248 + );
1249 + }
1250 + return requestChildFrameOperation(childWindow, type, {
1251 + ...payload,
1252 + frameChain: frameChain.slice(1)
1253 + });
1254 + }
1255 +
1256 + globalThis.addEventListener("message", (event) => {
1257 + const rawMessage = event?.data;
1258 + if (!rawMessage || rawMessage.channel !== DOM_HELPER_CHANNEL || typeof rawMessage.type !== "string") {
1259 + return;
1260 + }
1261 + const requestId = typeof rawMessage.requestId === "string" ? rawMessage.requestId : "";
1262 + if (!requestId) {
1263 + return;
1264 + }
1265 + if (rawMessage.type.endsWith("_result")) {
1266 + const pendingRequest = pendingRequests.get(requestId);
1267 + if (!pendingRequest) {
1268 + return;
1269 + }
1270 + pendingRequests.delete(requestId);
1271 + if (pendingRequest.timer != null) {
1272 + globalThis.clearTimeout(pendingRequest.timer);
1273 + }
1274 + if (rawMessage.ok === false) {
1275 + pendingRequest.reject(createNamedError(
1276 + "BrowserDomHelperRemoteError",
1277 + String(rawMessage?.payload?.message || `Embedded frame request "${pendingRequest.type}" failed.`),
1278 + {
1279 + code: rawMessage?.payload?.code ?? "browser_dom_helper_remote_error",
1280 + details: rawMessage?.payload?.details || {},
1281 + payload: rawMessage.payload
1282 + }
1283 + ));
1284 + return;
1285 + }
1286 + pendingRequest.resolve(rawMessage.payload);
1287 + return;
1288 + }
1289 + if (typeof event?.source?.postMessage !== "function") {
1290 + return;
1291 + }
1292 + Promise.resolve(routeOperation(rawMessage.type, rawMessage.payload || {}))
1293 + .then((payload) => {
1294 + event.source.postMessage({
1295 + channel: DOM_HELPER_CHANNEL,
1296 + ok: true,
1297 + payload,
1298 + requestId,
1299 + type: `${rawMessage.type}_result`
1300 + }, "*");
1301 + })
1302 + .catch((error) => {
1303 + console.error(`[a0-browser/dom-helper] Request "${rawMessage.type}" failed.`, error);
1304 + event.source.postMessage({
1305 + channel: DOM_HELPER_CHANNEL,
1306 + ok: false,
1307 + payload: {
1308 + code: error?.code ?? "browser_dom_helper_error",
1309 + details: error?.details || {},
1310 + message: String(error?.message || `Embedded frame request "${rawMessage.type}" failed.`)
1311 + },
1312 + requestId,
1313 + type: `${rawMessage.type}_result`
1314 + }, "*");
1315 + });
1316 + });
1317 +
1318 + globalThis[DOM_HELPER_KEY] = {
1319 + captureDocument(payload) {
1320 + return captureDocument(payload);
1321 + },
1322 + clickNode(frameChain, nodeId) {
1323 + return routeOperation("click_node", { frameChain, nodeId });
1324 + },
1325 + detailNode(frameChain, nodeId) {
1326 + return routeOperation("detail_node", { frameChain, nodeId });
1327 + },
1328 + frameId: helperFrameId,
1329 + scrollNode(frameChain, nodeId) {
1330 + return routeOperation("scroll_node", { frameChain, nodeId });
1331 + },
1332 + submitNode(frameChain, nodeId) {
1333 + return routeOperation("submit_node", { frameChain, nodeId });
1334 + },
1335 + typeNode(frameChain, nodeId, value) {
1336 + return routeOperation("type_node", { frameChain, nodeId, value });
1337 + },
1338 + typeSubmitNode(frameChain, nodeId, value) {
1339 + return routeOperation("type_submit_node", { frameChain, nodeId, value });
1340 + },
1341 + version: VERSION
1342 + };
1343 +})();
plugins/_browser/helpers/connector_runtime.py
+45 -4
@@ -39,6 +39,7 @@ from plugins._browser.helpers.url import normalize_url
39
40 BROWSER_OP_EVENT = "connector_browser_op"
41 BROWSER_OP_TIMEOUT = 120.0
42 +DOM_HELPER_PATH = Path(__file__).resolve().parents[1] / "assets" / "browser-dom-helper.js"
43 CONTENT_HELPER_PATH = Path(__file__).resolve().parents[1] / "assets" / "browser-page-content.js"
44 MAX_ARTIFACT_SIZE_BYTES = 25 * 1024 * 1024
45 HOST_BROWSER_PRIVACY_POLICY_KEY = getattr(
@@ -287,7 +288,7 @@ class ConnectorBrowserRuntime:
288 )
289 sid = self._select_sid() or sid
290
290 - return await self._send_browser_op(sid, self._with_content_helper(sid, payload))
291 + return await self._send_browser_op(sid, self._with_browser_helpers(sid, payload))
292
293 def _host_browser_profile_mode(self) -> str:
294 config = get_browser_config(self.agent)
@@ -295,11 +296,25 @@ class ConnectorBrowserRuntime:
296 return "agent" if mode == "agent" else "existing"
297
298 def _with_content_helper(self, sid: str, payload: dict[str, Any]) -> dict[str, Any]:
299 + return self._with_browser_helpers(sid, payload)
300 +
301 + def _with_browser_helpers(self, sid: str, payload: dict[str, Any]) -> dict[str, Any]:
302 metadata = host_browser_metadata_for_sid(sid) or {}
299 - if str(metadata.get("content_helper_sha256") or "").strip().lower() == _content_helper_sha256():
303 + content_helper_current = (
304 + str(metadata.get("content_helper_sha256") or "").strip().lower()
305 + == _content_helper_sha256()
306 + )
307 + dom_helper_current = (
308 + str(metadata.get("dom_helper_sha256") or "").strip().lower()
309 + == _dom_helper_sha256()
310 + )
311 + if content_helper_current and dom_helper_current:
312 return payload
313 payload = dict(payload)
302 - payload["content_helper"] = _content_helper_payload()
314 + if not dom_helper_current:
315 + payload["dom_helper"] = _dom_helper_payload()
316 + if not content_helper_current:
317 + payload["content_helper"] = _content_helper_payload()
318 return payload
319
320 async def _send_browser_op(self, sid: str, payload: dict[str, Any]) -> Any:
@@ -521,6 +536,33 @@ def _content_helper_sha256() -> str:
536 return str(_content_helper_payload()["sha256"])
537
538
539 +@lru_cache(maxsize=1)
540 +def _dom_helper_payload() -> dict[str, Any]:
541 + try:
542 + source = DOM_HELPER_PATH.read_text(encoding="utf-8")
543 + except OSError as exc:
544 + raise RuntimeError(
545 + f"Host-browser DOM helper could not be read from {DOM_HELPER_PATH}: {exc}"
546 + ) from exc
547 + return {
548 + "required_apis": [
549 + "captureDocument",
550 + "clickNode",
551 + "detailNode",
552 + "scrollNode",
553 + "submitNode",
554 + "typeNode",
555 + "typeSubmitNode",
556 + ],
557 + "source": source,
558 + "sha256": hashlib.sha256(source.encode("utf-8")).hexdigest(),
559 + }
560 +
561 +
562 +def _dom_helper_sha256() -> str:
563 + return str(_dom_helper_payload()["sha256"])
564 +
565 +
566 def _content_helper_required_apis(source: str) -> list[str]:
567 match = _REQUIRED_API_NAMES_RE.search(source)
568 if not match:
@@ -561,4 +603,3 @@ def _api_base_is_local(api_base: str) -> bool:
603 parsed = urlparse(api_base if "://" in api_base else f"http://{api_base}")
604 hostname = (parsed.hostname or "").strip().lower()
605 return hostname in _LOCAL_HOSTS
564 -
plugins/_browser/helpers/runtime.py
+28
@@ -32,6 +32,7 @@ from plugins._browser.helpers.url import normalize_url
32
33
34 PLUGIN_DIR = Path(__file__).resolve().parents[1]
35 +DOM_HELPER_PATH = PLUGIN_DIR / "assets" / "browser-dom-helper.js"
36 CONTENT_HELPER_PATH = PLUGIN_DIR / "assets" / "browser-page-content.js"
37 RUNTIME_DATA_KEY = "_browser_runtime"
38 DEFAULT_VIEWPORT = {"width": 1024, "height": 768}
@@ -558,6 +559,7 @@ class _BrowserRuntimeCore:
559 self.screencasts: dict[str, _BrowserScreencast] = {}
560 self.next_browser_id = 1
561 self.last_interacted_browser_id: int | None = None
562 + self._dom_helper_source: str | None = None
563 self._content_helper_source: str | None = None
564 self._start_lock: asyncio.Lock | None = None
565 self._registry_lock: asyncio.Lock | None = None
@@ -811,6 +813,7 @@ class _BrowserRuntimeCore:
813 self.context.set_default_navigation_timeout(30000)
814 self.context.on("close", self._on_context_closed)
815 self.context.on("page", self._on_new_page_sync)
816 + await self.context.add_init_script(path=str(DOM_HELPER_PATH))
817 await self.context.add_init_script(path=str(CONTENT_HELPER_PATH))
818
819 for page in list(self.context.pages):
@@ -2325,6 +2328,7 @@ class _BrowserRuntimeCore:
2328 await self.stop_screencast(stream_id)
2329
2330 async def _ensure_content_helper(self, page: Any) -> None:
2331 + await self._ensure_dom_helper(page)
2332 has_helper = await page.evaluate(
2333 "() => Boolean(globalThis.__spaceBrowserPageContent__?.ready?.())"
2334 )
@@ -2334,6 +2338,30 @@ class _BrowserRuntimeCore:
2338 self._content_helper_source = CONTENT_HELPER_PATH.read_text(encoding="utf-8")
2339 await page.evaluate(self._content_helper_source)
2340
2341 + async def _ensure_dom_helper(self, page: Any) -> None:
2342 + if self._dom_helper_source is None:
2343 + self._dom_helper_source = DOM_HELPER_PATH.read_text(encoding="utf-8")
2344 + await self._ensure_helper_source(
2345 + page,
2346 + self._dom_helper_source,
2347 + "() => Boolean(globalThis.__spaceBrowserDomHelper__?.captureDocument)",
2348 + )
2349 +
2350 + async def _ensure_helper_source(self, page: Any, source: str, ready_script: str) -> None:
2351 + targets = [page]
2352 + frames = getattr(page, "frames", None)
2353 + if isinstance(frames, list) and frames:
2354 + targets = frames
2355 + for target in targets:
2356 + try:
2357 + has_helper = await target.evaluate(ready_script)
2358 + except Exception:
2359 + continue
2360 + if has_helper:
2361 + continue
2362 + with contextlib.suppress(Exception):
2363 + await target.evaluate(source)
2364 +
2365 _runtimes: dict[str, BrowserRuntime] = {}
2366 _runtime_lock = threading.RLock()
2367
tests/test_browser_agent_regressions.py
+83
@@ -1,5 +1,6 @@
1 import asyncio
2 import json
3 +import re
4 import sys
5 import threading
6 from pathlib import Path
@@ -1226,7 +1227,13 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():
1227 helper = (
1228 PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"
1229 ).read_text(encoding="utf-8")
1230 + dom_helper = (
1231 + PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-dom-helper.js"
1232 + ).read_text(encoding="utf-8")
1233
1234 + assert "DOM_HELPER_PATH" in runtime
1235 + assert "browser-dom-helper.js" in runtime
1236 + assert "globalThis.__spaceBrowserDomHelper__?.captureDocument" in runtime
1237 assert "async def annotation_target" in runtime
1238 assert "globalThis.__spaceBrowserPageContent__.annotate(payload || null)" in runtime
1239 assert "function annotate(payload = null)" in helper
@@ -1238,6 +1245,11 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():
1245 assert "fileInputFor," in helper
1246 assert "sanitizeAnnotationDom" in helper
1247 assert "password" in helper
1248 + assert "__spaceBrowserDomHelper__" in dom_helper
1249 + assert "captureDocument(payload)" in dom_helper
1250 + assert "clickNode(frameChain, nodeId)" in dom_helper
1251 + assert "requestChildFrameOperation" in dom_helper
1252 + assert "data-space-browser-frame-chain" in dom_helper
1253
1254
1255 def test_browser_content_helper_keeps_label_wrapped_controls_referenceable():
@@ -1279,6 +1291,77 @@ def test_browser_runtime_requires_current_content_helper_for_modifier_clicks():
1291 assert "__spaceBrowserPageContent__?.ready?.()" in runtime
1292
1293
1294 +@pytest.mark.anyio
1295 +async def test_browser_dom_helper_clicks_content_ref_inside_iframe():
1296 + pytest.importorskip("playwright.async_api")
1297 + from playwright.async_api import async_playwright
1298 +
1299 + browser_binary = get_playwright_binary()
1300 + if not browser_binary:
1301 + pytest.skip("Playwright Chromium binary is not installed")
1302 +
1303 + dom_helper = (
1304 + PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-dom-helper.js"
1305 + ).read_text(encoding="utf-8")
1306 + content_helper = (
1307 + PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"
1308 + ).read_text(encoding="utf-8")
1309 +
1310 + async with async_playwright() as playwright:
1311 + try:
1312 + browser = await playwright.chromium.launch(
1313 + executable_path=str(browser_binary),
1314 + headless=True,
1315 + args=["--no-sandbox"],
1316 + )
1317 + except Exception as exc:
1318 + pytest.skip(f"Playwright Chromium could not launch: {exc}")
1319 +
1320 + try:
1321 + context = await browser.new_context()
1322 + await context.add_init_script(dom_helper)
1323 + await context.add_init_script(content_helper)
1324 + page = await context.new_page()
1325 + await page.set_content(
1326 + """
1327 + <html>
1328 + <body>
1329 + <iframe srcdoc="
1330 + <html>
1331 + <body>
1332 + <button id='inside' onclick=&quot;document.body.dataset.clicked = 'yes'; this.textContent = 'Clicked inside frame'&quot;>
1333 + Frame Launch
1334 + </button>
1335 + </body>
1336 + </html>
1337 + "></iframe>
1338 + </body>
1339 + </html>
1340 + """
1341 + )
1342 + await page.wait_for_function(
1343 + "() => Boolean(document.querySelector('iframe')?.contentWindow?.__spaceBrowserDomHelper__)"
1344 + )
1345 +
1346 + captured = await page.evaluate(
1347 + "(payload) => globalThis.__spaceBrowserPageContent__.capture(payload || null)",
1348 + None,
1349 + )
1350 + document_content = str(captured.get("document") or "")
1351 + match = re.search(r"\[button (\d+)\]\s*Frame Launch", document_content)
1352 + assert match, document_content
1353 +
1354 + action = await page.evaluate(
1355 + "(ref) => globalThis.__spaceBrowserPageContent__.click(ref)",
1356 + match.group(1),
1357 + )
1358 + frame = next(frame for frame in page.frames if frame != page.main_frame)
1359 + assert await frame.evaluate("() => document.body.dataset.clicked") == "yes"
1360 + assert action["status"]["reacted"] is True
1361 + finally:
1362 + await browser.close()
1363 +
1364 +
1365 @pytest.mark.anyio
1366 async def test_browser_screencast_acknowledges_and_drops_stale_frames():
1367 first_image = SMALL_JPEG_10X10
tests/test_host_browser_connector.py
+3
@@ -448,6 +448,9 @@ def test_connector_runtime_ensures_preparable_host_browser_before_action(monkeyp
448 assert result == {"id": 1, "state": {"runtime": "host"}}
449 assert [payload["action"] for payload in emitted] == ["ensure", "open"]
450 assert [payload["profile_mode"] for payload in emitted] == ["existing", "existing"]
451 + assert "__spaceBrowserDomHelper__" in emitted[0]["dom_helper"]["source"]
452 + assert "captureDocument" in emitted[0]["dom_helper"]["required_apis"]
453 + assert emitted[0]["dom_helper"]["sha256"]
454 assert "__spaceBrowserPageContent__" in emitted[0]["content_helper"]["source"]
455 assert "capture" in emitted[0]["content_helper"]["required_apis"]
456 assert emitted[0]["content_helper"]["sha256"]