@samitouri / QOSami_Agents_Zero-2 / commits / d3c249cb

Add Browser v1 explicit screenshot and form actions

Adds the explicit browser:screenshot action that writes JPEG/PNG files for vision_load, extends agent-callable Browser input actions, and documents the explicit vision workflow. Adds the browser-forms on-demand skill and regression coverage for dispatch, runtime screenshot files, ref point resolution, upload path normalization, prompt discoverability, and label-wrapped form controls surfaced by the chat-driven E2E.

Alessandro committed May 5, 2026 at 15:54 UTC d3c249cbdd2a9f9aa80f19582b98efc6ca10249c
6 files changed +1619 -20
plugins/_browser/assets/browser-page-content.js
+560 -5
@@ -1,7 +1,7 @@
1 (() => {
2 const GLOBAL_KEY = "__spaceBrowserPageContent__";
3 const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";
4 - const VERSION = "9";
4 + const VERSION = "11";
5 const BLOCK_TAGS = new Set([
6 "ADDRESS",
7 "ARTICLE",
@@ -779,6 +779,44 @@
779 return INTERACTIVE_ROLES.has(role) || hasInteractiveEventHandler(element);
780 }
781
782 + function isFileInputElement(element) {
783 + return getTagName(element) === "INPUT"
784 + && String(element.getAttribute?.("type") || element.type || "").toLowerCase() === "file";
785 + }
786 +
787 + function getAssociatedLabelFileInput(labelElement) {
788 + if (getTagName(labelElement) !== "LABEL") {
789 + return null;
790 + }
791 +
792 + if (isFileInputElement(labelElement.control)) {
793 + return labelElement.control;
794 + }
795 +
796 + const descendantInput = labelElement.querySelector?.("input[type='file']");
797 + if (isFileInputElement(descendantInput)) {
798 + return descendantInput;
799 + }
800 +
801 + const forId = normalizeAttributeText(labelElement.getAttribute?.("for"));
802 + if (!forId) {
803 + return null;
804 + }
805 +
806 + return isFileInputElement(labelElement.ownerDocument?.getElementById?.(forId))
807 + ? labelElement.ownerDocument.getElementById(forId)
808 + : null;
809 + }
810 +
811 + function isFileInputLabel(element) {
812 + if (getTagName(element) !== "LABEL" || isHiddenElement(element)) {
813 + return false;
814 + }
815 +
816 + const input = getAssociatedLabelFileInput(element);
817 + return Boolean(input && isHiddenElement(input));
818 + }
819 +
820 function getComputedStyleSafe(element) {
821 try {
822 return globalThis.getComputedStyle?.(element) || null;
@@ -1048,6 +1086,33 @@
1086 return readableText || normalizeText(element?.textContent || "");
1087 }
1088
1089 + function isLabelableControlForText(element) {
1090 + return ["BUTTON", "INPUT", "METER", "OUTPUT", "PROGRESS", "SELECT", "TEXTAREA"].includes(getTagName(element));
1091 + }
1092 +
1093 + function getLabelElementText(labelElement, controlElement = null) {
1094 + const collect = (node) => {
1095 + if (isTextNode(node)) {
1096 + return node.textContent || "";
1097 + }
1098 +
1099 + if (!isElementNode(node) || isHiddenElement(node)) {
1100 + return "";
1101 + }
1102 +
1103 + if (node !== labelElement && (node === controlElement || isLabelableControlForText(node))) {
1104 + return "";
1105 + }
1106 +
1107 + return getReadableChildNodes(node)
1108 + .map((childNode) => collect(childNode))
1109 + .filter(Boolean)
1110 + .join(" ");
1111 + };
1112 +
1113 + return normalizeText(collect(labelElement)) || getElementText(labelElement);
1114 + }
1115 +
1116 function collectLabelCandidates(element, options = {}) {
1117 const includeAlt = options.includeAlt !== false;
1118 const includeDescendantImageAlt = options.includeDescendantImageAlt !== false;
@@ -1058,7 +1123,7 @@
1123 try {
1124 if (Array.isArray(element?.labels) || typeof element?.labels?.forEach === "function") {
1125 element.labels.forEach((labelElement) => {
1061 - const text = getElementText(labelElement);
1126 + const text = getLabelElementText(labelElement, element);
1127 if (text) {
1128 collectedLabels.push(text);
1129 }
@@ -1193,6 +1258,10 @@
1258 return `input ${inputType || "text"}`;
1259 }
1260
1261 + if (tagName === "LABEL" && isFileInputLabel(element)) {
1262 + return "file input label";
1263 + }
1264 +
1265 if (String(element.getAttribute?.("contenteditable") || "").toLowerCase() === "true") {
1266 return "editable";
1267 }
@@ -1493,7 +1562,47 @@
1562 }
1563
1564 function isReferenceableElement(element) {
1496 - return isInteractiveElement(element) || getTagName(element) === "IMG";
1565 + return isInteractiveElement(element) || getTagName(element) === "IMG" || isFileInputLabel(element);
1566 + }
1567 +
1568 + function collectLabelControlElements(labelElement) {
1569 + const controls = [];
1570 + const seen = new Set();
1571 + const addControl = (element) => {
1572 + if (!isElementNode(element) || seen.has(element) || !isReferenceableElement(element)) {
1573 + return;
1574 + }
1575 +
1576 + seen.add(element);
1577 + controls.push(element);
1578 + };
1579 +
1580 + [
1581 + "input",
1582 + "textarea",
1583 + "select",
1584 + "button",
1585 + "summary",
1586 + "a[href]",
1587 + "[role]",
1588 + "[contenteditable='true']",
1589 + "[contenteditable='']"
1590 + ].forEach((selector) => {
1591 + try {
1592 + [...(labelElement.querySelectorAll?.(selector) || [])].forEach(addControl);
1593 + } catch {
1594 + // Ignore unsupported selectors in unusual DOMs.
1595 + }
1596 + });
1597 +
1598 + return controls;
1599 + }
1600 +
1601 + function renderControlLabelReferences(labelElement, context) {
1602 + return collectLabelControlElements(labelElement)
1603 + .map((controlElement) => renderReference(controlElement, context))
1604 + .filter(Boolean)
1605 + .join("\n");
1606 }
1607
1608 function renderInlineNode(node, context) {
@@ -1517,7 +1626,7 @@
1626 const tagName = getTagName(node);
1627
1628 if (tagName === "LABEL" && (node.getAttribute?.("for") || node.querySelector?.("input, textarea, select, button"))) {
1520 - return "";
1629 + return renderControlLabelReferences(node, context);
1630 }
1631
1632 if (tagName === "BR") {
@@ -1693,7 +1802,7 @@
1802 const tagName = getTagName(element);
1803
1804 if (tagName === "LABEL" && (element.getAttribute?.("for") || element.querySelector?.("input, textarea, select, button"))) {
1696 - return "";
1805 + return renderControlLabelReferences(element, context);
1806 }
1807
1808 if (/^H[1-6]$/u.test(tagName)) {
@@ -2175,6 +2284,443 @@
2284 };
2285 }
2286
2287 + function pointFor(referenceId, offsets = {}) {
2288 + const entry = requireReferenceEntry(referenceId, {
2289 + actionLabel: "point",
2290 + requireConnected: true
2291 + });
2292 + if (entry.helperBacked || !entry.element) {
2293 + throw createNamedError(
2294 + "BrowserPageContentActionError",
2295 + `Browser page content cannot resolve point for helper-backed reference "${entry.referenceId}".`,
2296 + {
2297 + code: "browser_page_content_point_helper_backed"
2298 + }
2299 + );
2300 + }
2301 +
2302 + const element = entry.element;
2303 + scrollElementIntoView(element);
2304 + const rect = getElementRectSafe(element);
2305 + if (!rect || rect.width <= 0 || rect.height <= 0) {
2306 + throw createNamedError(
2307 + "BrowserPageContentActionError",
2308 + `Browser page content reference "${entry.referenceId}" has no visible viewport box.`,
2309 + {
2310 + code: "browser_page_content_point_no_box"
2311 + }
2312 + );
2313 + }
2314 +
2315 + const offsetX = Number(offsets?.offset_x ?? offsets?.offsetX ?? 0) || 0;
2316 + const offsetY = Number(offsets?.offset_y ?? offsets?.offsetY ?? 0) || 0;
2317 + const useOffsets = offsets?.useOffsets === true || offsetX !== 0 || offsetY !== 0;
2318 + return {
2319 + rect,
2320 + selector: computeStableSelector(element),
2321 + x: rect.x + (useOffsets ? offsetX : rect.width / 2),
2322 + y: rect.y + (useOffsets ? offsetY : rect.height / 2)
2323 + };
2324 + }
2325 +
2326 + function normalizeActionValues(valueOrValues) {
2327 + if (Array.isArray(valueOrValues)) {
2328 + return valueOrValues.map((value) => String(value ?? ""));
2329 + }
2330 +
2331 + if (valueOrValues === null || valueOrValues === undefined) {
2332 + return [];
2333 + }
2334 +
2335 + return [String(valueOrValues)];
2336 + }
2337 +
2338 + function optionMatchesValue(option, value) {
2339 + const normalizedValue = normalizeText(value);
2340 + const candidates = [
2341 + option?.value,
2342 + option?.label,
2343 + option?.textContent,
2344 + option?.getAttribute?.("aria-label"),
2345 + option?.getAttribute?.("data-value"),
2346 + option?.getAttribute?.("id")
2347 + ].map((candidate) => normalizeText(candidate));
2348 + return candidates.some((candidate) => candidate === normalizedValue);
2349 + }
2350 +
2351 + function findNativeSelectOption(selectElement, value) {
2352 + const options = [...(selectElement.options || [])];
2353 + return options.find((option) => optionMatchesValue(option, value)) || null;
2354 + }
2355 +
2356 + function setNativeChecked(element, checked) {
2357 + const descriptor = Object.getOwnPropertyDescriptor(globalThis.HTMLInputElement?.prototype || {}, "checked");
2358 + if (typeof descriptor?.set === "function") {
2359 + descriptor.set.call(element, Boolean(checked));
2360 + } else {
2361 + element.checked = Boolean(checked);
2362 + }
2363 + }
2364 +
2365 + async function selectNativeElement(entry, values) {
2366 + const element = entry.element;
2367 + const beforeSnapshot = captureActionEffectSnapshot(element);
2368 + const requestedValues = values.length ? values : [""];
2369 + const appliedValues = [];
2370 +
2371 + const {
2372 + observedMutations
2373 + } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {
2374 + scrollElementIntoView(element);
2375 + focusElement(element);
2376 +
2377 + if (element.multiple) {
2378 + const matchedOptions = requestedValues.map((requestedValue) => {
2379 + const option = findNativeSelectOption(element, requestedValue);
2380 + if (!option) {
2381 + throw createNamedError(
2382 + "BrowserPageContentActionError",
2383 + `Browser page content could not find select option "${requestedValue}".`,
2384 + {
2385 + code: "browser_page_content_select_option_not_found"
2386 + }
2387 + );
2388 + }
2389 + return option;
2390 + });
2391 + const matchedSet = new Set(matchedOptions);
2392 + [...(element.options || [])].forEach((option) => {
2393 + option.selected = matchedSet.has(option);
2394 + });
2395 + matchedOptions.forEach((option) => appliedValues.push(option.value));
2396 + } else {
2397 + const option = findNativeSelectOption(element, requestedValues[0]);
2398 + if (!option) {
2399 + throw createNamedError(
2400 + "BrowserPageContentActionError",
2401 + `Browser page content could not find select option "${requestedValues[0]}".`,
2402 + {
2403 + code: "browser_page_content_select_option_not_found"
2404 + }
2405 + );
2406 + }
2407 + appliedValues.push(setNativeValue(element, option.value));
2408 + }
2409 +
2410 + dispatchDomEvent(element, "input", "InputEvent", {
2411 + inputType: "insertReplacementText"
2412 + });
2413 + dispatchDomEvent(element, "change");
2414 + return appliedValues.slice();
2415 + });
2416 +
2417 + refreshReferenceEntry(entry);
2418 + return buildActionResult(entry, {
2419 + ...buildActionEffectResult(entry, beforeSnapshot, captureActionEffectSnapshot(element), observedMutations),
2420 + values: appliedValues.slice()
2421 + });
2422 + }
2423 +
2424 + function ariaOptionMatchesValue(option, value) {
2425 + const normalizedValue = normalizeText(value);
2426 + const candidates = [
2427 + option?.getAttribute?.("aria-label"),
2428 + option?.getAttribute?.("data-value"),
2429 + option?.getAttribute?.("value"),
2430 + option?.getAttribute?.("id"),
2431 + getElementText(option)
2432 + ].map((candidate) => normalizeText(candidate));
2433 + return candidates.some((candidate) => candidate === normalizedValue);
2434 + }
2435 +
2436 + function visibleAriaOptions(root) {
2437 + const scope = isElementNode(root) && String(root.getAttribute?.("role") || "").trim().toLowerCase() === "listbox"
2438 + ? root
2439 + : globalThis.document;
2440 + try {
2441 + return [...(scope.querySelectorAll?.("[role='option']") || [])]
2442 + .filter((option) => isElementNode(option) && !isHiddenElement(option));
2443 + } catch {
2444 + return [];
2445 + }
2446 + }
2447 +
2448 + function findAriaOption(root, value) {
2449 + const matches = visibleAriaOptions(root).filter((option) => ariaOptionMatchesValue(option, value));
2450 + return matches.length === 1 ? matches[0] : null;
2451 + }
2452 +
2453 + async function selectAriaElement(entry, values) {
2454 + const element = entry.element;
2455 + const role = String(element.getAttribute?.("role") || "").trim().toLowerCase();
2456 + if (!["combobox", "listbox"].includes(role)) {
2457 + throw createNamedError(
2458 + "BrowserPageContentActionError",
2459 + `Browser page content cannot select options on <${getTagName(element).toLowerCase()}>.`,
2460 + {
2461 + code: "browser_page_content_select_unsupported"
2462 + }
2463 + );
2464 + }
2465 +
2466 + const beforeSnapshot = captureActionEffectSnapshot(element);
2467 + const requestedValues = values.length ? values : [""];
2468 + const appliedValues = [];
2469 + const {
2470 + observedMutations
2471 + } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {
2472 + scrollElementIntoView(element);
2473 + focusElement(element);
2474 + if (role === "combobox") {
2475 + dispatchDomEvent(element, "mousedown", "MouseEvent", { button: 0 });
2476 + if (typeof element.click === "function") {
2477 + element.click();
2478 + } else {
2479 + dispatchDomEvent(element, "click", "MouseEvent", { button: 0 });
2480 + }
2481 + await delayMs(80);
2482 + }
2483 +
2484 + for (const requestedValue of requestedValues) {
2485 + const option = findAriaOption(element, requestedValue);
2486 + if (!option) {
2487 + throw createNamedError(
2488 + "BrowserPageContentActionError",
2489 + `Browser page content could not safely find one ARIA option "${requestedValue}".`,
2490 + {
2491 + code: "browser_page_content_aria_option_not_found"
2492 + }
2493 + );
2494 + }
2495 + scrollElementIntoView(option);
2496 + dispatchDomEvent(option, "mousedown", "MouseEvent", { button: 0 });
2497 + if (typeof option.click === "function") {
2498 + option.click();
2499 + } else {
2500 + dispatchDomEvent(option, "click", "MouseEvent", { button: 0 });
2501 + }
2502 + appliedValues.push(requestedValue);
2503 + await delayMs(40);
2504 + }
2505 + });
2506 +
2507 + refreshReferenceEntry(entry);
2508 + return buildActionResult(entry, {
2509 + ...buildActionEffectResult(entry, beforeSnapshot, captureActionEffectSnapshot(element), observedMutations),
2510 + values: appliedValues.slice()
2511 + });
2512 + }
2513 +
2514 + async function selectReference(referenceId, valueOrValues) {
2515 + const entry = requireReferenceEntry(referenceId, {
2516 + actionLabel: "select"
2517 + });
2518 + if (entry.helperBacked) {
2519 + throw createNamedError(
2520 + "BrowserPageContentActionError",
2521 + `Browser page content cannot select helper-backed reference "${entry.referenceId}".`,
2522 + {
2523 + code: "browser_page_content_select_helper_backed"
2524 + }
2525 + );
2526 + }
2527 +
2528 + const element = entry.element;
2529 + const values = normalizeActionValues(valueOrValues);
2530 + if (getTagName(element) === "SELECT") {
2531 + return selectNativeElement(entry, values);
2532 + }
2533 +
2534 + return selectAriaElement(entry, values);
2535 + }
2536 +
2537 + function checkedStateForElement(element) {
2538 + const tagName = getTagName(element);
2539 + const role = String(element.getAttribute?.("role") || "").trim().toLowerCase();
2540 + if (tagName === "INPUT") {
2541 + const inputType = String(element.getAttribute?.("type") || element.type || "").toLowerCase();
2542 + if (["checkbox", "radio"].includes(inputType)) {
2543 + return Boolean(element.checked);
2544 + }
2545 + }
2546 + if (["checkbox", "radio", "switch", "menuitemcheckbox", "menuitemradio"].includes(role)) {
2547 + return String(element.getAttribute?.("aria-checked") || "").trim().toLowerCase() === "true";
2548 + }
2549 + if (role === "button" && element.hasAttribute?.("aria-pressed")) {
2550 + return String(element.getAttribute?.("aria-pressed") || "").trim().toLowerCase() === "true";
2551 + }
2552 + return null;
2553 + }
2554 +
2555 + async function setCheckedReference(referenceId, checked = true) {
2556 + const entry = requireReferenceEntry(referenceId, {
2557 + actionLabel: "setChecked"
2558 + });
2559 + if (entry.helperBacked) {
2560 + throw createNamedError(
2561 + "BrowserPageContentActionError",
2562 + `Browser page content cannot set helper-backed reference "${entry.referenceId}".`,
2563 + {
2564 + code: "browser_page_content_checked_helper_backed"
2565 + }
2566 + );
2567 + }
2568 +
2569 + const element = entry.element;
2570 + const beforeSnapshot = captureActionEffectSnapshot(element);
2571 + const desiredChecked = Boolean(checked);
2572 + const tagName = getTagName(element);
2573 + const role = String(element.getAttribute?.("role") || "").trim().toLowerCase();
2574 + const currentChecked = checkedStateForElement(element);
2575 + if (currentChecked === null) {
2576 + throw createNamedError(
2577 + "BrowserPageContentActionError",
2578 + `Browser page content cannot set checked state on <${tagName.toLowerCase()}>.`,
2579 + {
2580 + code: "browser_page_content_checked_unsupported"
2581 + }
2582 + );
2583 + }
2584 +
2585 + const {
2586 + observedMutations
2587 + } = await withObservedActionWindow(beforeSnapshot.observationRoot, async () => {
2588 + scrollElementIntoView(element);
2589 + focusElement(element);
2590 +
2591 + if (tagName === "INPUT") {
2592 + setNativeChecked(element, desiredChecked);
2593 + dispatchDomEvent(element, "input", "InputEvent", {
2594 + inputType: "insertReplacementText"
2595 + });
2596 + dispatchDomEvent(element, "change");
2597 + } else if (["checkbox", "radio", "switch", "menuitemcheckbox", "menuitemradio"].includes(role)) {
2598 + if (currentChecked !== desiredChecked) {
2599 + if (typeof element.click === "function") {
2600 + element.click();
2601 + } else {
2602 + dispatchDomEvent(element, "click", "MouseEvent", {
2603 + button: 0
2604 + });
2605 + }
2606 + await delayMs(40);
2607 + }
2608 + if (checkedStateForElement(element) !== desiredChecked) {
2609 + element.setAttribute("aria-checked", desiredChecked ? "true" : "false");
2610 + dispatchDomEvent(element, "input", "InputEvent", {
2611 + inputType: "insertReplacementText"
2612 + });
2613 + dispatchDomEvent(element, "change");
2614 + }
2615 + } else if (role === "button" && element.hasAttribute?.("aria-pressed")) {
2616 + if (currentChecked !== desiredChecked) {
2617 + if (typeof element.click === "function") {
2618 + element.click();
2619 + } else {
2620 + dispatchDomEvent(element, "click", "MouseEvent", {
2621 + button: 0
2622 + });
2623 + }
2624 + await delayMs(40);
2625 + }
2626 + if (checkedStateForElement(element) !== desiredChecked) {
2627 + element.setAttribute("aria-pressed", desiredChecked ? "true" : "false");
2628 + }
2629 + }
2630 + });
2631 +
2632 + refreshReferenceEntry(entry);
2633 + return buildActionResult(entry, {
2634 + ...buildActionEffectResult(entry, beforeSnapshot, captureActionEffectSnapshot(element), observedMutations),
2635 + checked: desiredChecked
2636 + });
2637 + }
2638 +
2639 + function resolveFileInputElement(referenceId) {
2640 + const entry = requireReferenceEntry(referenceId, {
2641 + actionLabel: "fileInput"
2642 + });
2643 + if (entry.helperBacked) {
2644 + throw createNamedError(
2645 + "BrowserPageContentActionError",
2646 + `Browser page content cannot upload files through helper-backed reference "${entry.referenceId}".`,
2647 + {
2648 + code: "browser_page_content_file_input_helper_backed"
2649 + }
2650 + );
2651 + }
2652 +
2653 + const element = entry.element;
2654 + const isFileInput = (candidate) => {
2655 + return getTagName(candidate) === "INPUT"
2656 + && String(candidate.getAttribute?.("type") || candidate.type || "").toLowerCase() === "file";
2657 + };
2658 +
2659 + if (isFileInput(element)) {
2660 + return element;
2661 + }
2662 +
2663 + if (getTagName(element) === "LABEL") {
2664 + if (isFileInput(element.control)) {
2665 + return element.control;
2666 + }
2667 + const labelledInput = element.querySelector?.("input[type='file']");
2668 + if (isFileInput(labelledInput)) {
2669 + return labelledInput;
2670 + }
2671 + const forId = normalizeAttributeText(element.getAttribute?.("for"));
2672 + if (forId) {
2673 + const byId = element.ownerDocument?.getElementById?.(forId);
2674 + if (isFileInput(byId)) {
2675 + return byId;
2676 + }
2677 + }
2678 + }
2679 +
2680 + const descendantInput = element.querySelector?.("input[type='file']");
2681 + if (isFileInput(descendantInput)) {
2682 + return descendantInput;
2683 + }
2684 +
2685 + const closestLabel = element.closest?.("label");
2686 + if (closestLabel) {
2687 + if (isFileInput(closestLabel.control)) {
2688 + return closestLabel.control;
2689 + }
2690 + const labelledInput = closestLabel.querySelector?.("input[type='file']");
2691 + if (isFileInput(labelledInput)) {
2692 + return labelledInput;
2693 + }
2694 + }
2695 +
2696 + return null;
2697 + }
2698 +
2699 + function fileInputFor(referenceId) {
2700 + const input = resolveFileInputElement(referenceId);
2701 + if (!input) {
2702 + throw createNamedError(
2703 + "BrowserPageContentActionError",
2704 + `Browser page content reference "${normalizeReferenceId(referenceId)}" is not a file input or associated label.`,
2705 + {
2706 + code: "browser_page_content_file_input_not_found"
2707 + }
2708 + );
2709 + }
2710 + return {
2711 + accept: normalizeAttributeText(input.getAttribute?.("accept")),
2712 + multiple: Boolean(input.multiple),
2713 + name: normalizeAttributeText(input.getAttribute?.("name")),
2714 + selector: computeStableSelector(input),
2715 + tagName: getTagName(input),
2716 + type: String(input.getAttribute?.("type") || input.type || "").toLowerCase()
2717 + };
2718 + }
2719 +
2720 + function fileInputElementFor(referenceId) {
2721 + return resolveFileInputElement(referenceId);
2722 + }
2723 +
2724 function refreshReferenceEntry(entry) {
2725 if (!entry || entry.helperBacked || !entry.element) {
2726 return entry;
@@ -3403,6 +3949,15 @@
3949 return typeAndSubmit(referenceId, value);
3950 },
3951 boundingBoxFor,
3952 + fileInputElementFor,
3953 + fileInputFor,
3954 + pointFor,
3955 + select(referenceId, valueOrValues) {
3956 + return selectReference(referenceId, valueOrValues);
3957 + },
3958 + setChecked(referenceId, checked) {
3959 + return setCheckedReference(referenceId, checked);
3960 + },
3961 version: VERSION
3962 };
3963 })();
plugins/_browser/helpers/runtime.py
+514 -1
@@ -636,6 +636,54 @@ class _BrowserRuntimeCore:
636 )
637 return normalized
638
639 + @staticmethod
640 + def _has_reference(reference_id: int | str | None) -> bool:
641 + return reference_id is not None and str(reference_id).strip() != ""
642 +
643 + def _screenshot_output_path(self, browser_id: int, path: str = "") -> tuple[Path, str, str]:
644 + raw_path = str(path or "").strip()
645 + if raw_path:
646 + output_path = Path(files.fix_dev_path(raw_path) if raw_path.startswith("/a0/") else raw_path)
647 + if not output_path.is_absolute():
648 + output_path = Path(files.get_abs_path(str(output_path)))
649 + suffix = output_path.suffix.lower()
650 + if suffix == ".png":
651 + return output_path, "png", "image/png"
652 + if suffix not in {".jpg", ".jpeg"}:
653 + output_path = output_path.with_suffix(".jpg") if not suffix else output_path.with_name(f"{output_path.name}.jpg")
654 + return output_path, "jpeg", "image/jpeg"
655 +
656 + timestamp = time.strftime("%Y%m%d-%H%M%S")
657 + millis = int((time.time() % 1) * 1000)
658 + output_path = self.screenshots_dir / f"browser-{int(browser_id)}-{timestamp}-{millis:03d}.jpg"
659 + return output_path, "jpeg", "image/jpeg"
660 +
661 + @staticmethod
662 + def _normalize_upload_paths(path: str = "", paths: list[str] | None = None) -> list[str]:
663 + raw_paths: list[str] = []
664 + if paths:
665 + if not isinstance(paths, list):
666 + raise ValueError("paths must be a list of file paths")
667 + raw_paths.extend(str(item or "").strip() for item in paths)
668 + if str(path or "").strip():
669 + raw_paths.append(str(path or "").strip())
670 +
671 + normalized_paths: list[str] = []
672 + for raw_path in raw_paths:
673 + if not raw_path:
674 + continue
675 + candidate = Path(files.fix_dev_path(raw_path) if raw_path.startswith("/a0/") else raw_path)
676 + if not candidate.is_absolute():
677 + candidate = Path(files.get_abs_path(str(candidate)))
678 + candidate = candidate.expanduser().resolve()
679 + if not candidate.is_file():
680 + raise FileNotFoundError(f"Upload file does not exist: {candidate}")
681 + normalized_paths.append(str(candidate))
682 +
683 + if not normalized_paths:
684 + raise ValueError("upload_file requires path or non-empty paths")
685 + return normalized_paths
686 +
687 @staticmethod
688 def _multi_group_key(call: dict[str, Any]) -> Any:
689 value = call.get("browser_id")
@@ -657,6 +705,10 @@ class _BrowserRuntimeCore:
705 def downloads_dir(self) -> Path:
706 return Path(files.get_abs_path("usr/downloads/browser"))
707
708 + @property
709 + def screenshots_dir(self) -> Path:
710 + return Path(files.get_abs_path("tmp/browser/screenshots", self.safe_context_id))
711 +
712 async def ensure_started(self) -> None:
713 if self._context_is_alive():
714 return
@@ -915,6 +967,13 @@ class _BrowserRuntimeCore:
967 bid = call.get("browser_id")
968 if action == "open":
969 return await self.open(call.get("url") or "")
970 + if action == "screenshot":
971 + return await self.screenshot_file(
972 + bid,
973 + quality=int(call.get("quality") or 80),
974 + full_page=bool(call.get("full_page")),
975 + path=call.get("path") or "",
976 + )
977 if action == "list":
978 return await self.list(include_content=bool(call.get("include_content")))
979 if action == "state":
@@ -986,6 +1045,118 @@ class _BrowserRuntimeCore:
1045 button=call.get("button") or "left",
1046 modifiers=self._normalize_modifiers(call.get("modifiers")),
1047 )
1048 + if action == "hover":
1049 + return await self.hover(
1050 + bid,
1051 + ref=call.get("ref"),
1052 + x=float(call.get("x") or 0),
1053 + y=float(call.get("y") or 0),
1054 + offset_x=float(call.get("offset_x") or 0),
1055 + offset_y=float(call.get("offset_y") or 0),
1056 + )
1057 + if action == "double_click":
1058 + return await self.double_click(
1059 + bid,
1060 + ref=call.get("ref"),
1061 + x=float(call.get("x") or 0),
1062 + y=float(call.get("y") or 0),
1063 + button=call.get("button") or "left",
1064 + modifiers=self._normalize_modifiers(call.get("modifiers")),
1065 + offset_x=float(call.get("offset_x") or 0),
1066 + offset_y=float(call.get("offset_y") or 0),
1067 + )
1068 + if action == "right_click":
1069 + return await self.right_click(
1070 + bid,
1071 + ref=call.get("ref"),
1072 + x=float(call.get("x") or 0),
1073 + y=float(call.get("y") or 0),
1074 + modifiers=self._normalize_modifiers(call.get("modifiers")),
1075 + offset_x=float(call.get("offset_x") or 0),
1076 + offset_y=float(call.get("offset_y") or 0),
1077 + )
1078 + if action == "drag":
1079 + return await self.drag(
1080 + bid,
1081 + ref=call.get("ref"),
1082 + target_ref=call.get("target_ref"),
1083 + x=float(call.get("x") or 0),
1084 + y=float(call.get("y") or 0),
1085 + to_x=float(call.get("to_x") or 0),
1086 + to_y=float(call.get("to_y") or 0),
1087 + offset_x=float(call.get("offset_x") or 0),
1088 + offset_y=float(call.get("offset_y") or 0),
1089 + target_offset_x=float(call.get("target_offset_x") or 0),
1090 + target_offset_y=float(call.get("target_offset_y") or 0),
1091 + )
1092 + if action == "wheel":
1093 + return await self.wheel(
1094 + bid,
1095 + float(call.get("x") or 0),
1096 + float(call.get("y") or 0),
1097 + float(call.get("delta_x") or 0),
1098 + float(call.get("delta_y") or 0),
1099 + )
1100 + if action == "keyboard":
1101 + return await self.keyboard(
1102 + bid,
1103 + key=str(call.get("key") or ""),
1104 + text=str(call.get("text") or ""),
1105 + )
1106 + if action == "clipboard":
1107 + clipboard_action = str(
1108 + call.get("clipboard_action")
1109 + or call.get("operation")
1110 + or call.get("event_type")
1111 + or ""
1112 + ).strip().lower()
1113 + return await self.clipboard(
1114 + bid,
1115 + action=clipboard_action,
1116 + text=str(call.get("text") or ""),
1117 + )
1118 + if action in {"copy", "cut", "paste"}:
1119 + return await self.clipboard(
1120 + bid,
1121 + action=action,
1122 + text=str(call.get("text") or ""),
1123 + )
1124 + if action == "set_viewport":
1125 + return await self.set_viewport(
1126 + bid,
1127 + int(call.get("width") or 0),
1128 + int(call.get("height") or 0),
1129 + )
1130 + if action == "select_option":
1131 + ref = call.get("ref")
1132 + if ref is None:
1133 + raise ValueError("select_option requires ref")
1134 + return await self.select_option(
1135 + bid,
1136 + ref,
1137 + value=str(call.get("value") or ""),
1138 + values=call.get("values"),
1139 + )
1140 + if action == "set_checked":
1141 + ref = call.get("ref")
1142 + if ref is None:
1143 + raise ValueError("set_checked requires ref")
1144 + checked = call.get("checked")
1145 + return await self.set_checked(
1146 + bid,
1147 + ref,
1148 + checked=True if checked is None else bool(checked),
1149 + )
1150 + if action == "upload_file":
1151 + ref = call.get("ref")
1152 + if ref is None:
1153 + raise ValueError("upload_file requires ref")
1154 + return await self.upload_file(
1155 + bid,
1156 + ref,
1157 + path=call.get("path") or "",
1158 + paths=call.get("paths"),
1159 + )
1160 if action == "close":
1161 return await self.close_browser(bid)
1162 if action == "close_all":
@@ -1342,6 +1513,43 @@ class _BrowserRuntimeCore:
1513 "state": await self._state(resolved_id),
1514 }
1515
1516 + async def screenshot_file(
1517 + self,
1518 + browser_id: int | str | None = None,
1519 + *,
1520 + quality: int = 80,
1521 + full_page: bool = False,
1522 + path: str = "",
1523 + ) -> dict[str, Any]:
1524 + await self.ensure_started()
1525 + resolved_id = self._resolve_browser_id(browser_id)
1526 + page = self._page(resolved_id)
1527 + output_path, image_type, mime = self._screenshot_output_path(resolved_id, path)
1528 + output_path.parent.mkdir(parents=True, exist_ok=True)
1529 + clamped_quality = max(20, min(95, int(quality)))
1530 + screenshot_kwargs: dict[str, Any] = {
1531 + "path": str(output_path),
1532 + "type": image_type,
1533 + "full_page": bool(full_page),
1534 + }
1535 + if image_type == "jpeg":
1536 + screenshot_kwargs["quality"] = clamped_quality
1537 + await page.screenshot(**screenshot_kwargs)
1538 + local_path = str(output_path)
1539 + return {
1540 + "browser_id": resolved_id,
1541 + "path": local_path,
1542 + "a0_path": files.normalize_a0_path(local_path),
1543 + "mime": mime,
1544 + "state": await self._state(resolved_id),
1545 + "vision_load": {
1546 + "tool_name": "vision_load",
1547 + "tool_args": {
1548 + "paths": [local_path],
1549 + },
1550 + },
1551 + }
1552 +
1553 async def start_screencast(
1554 self,
1555 browser_id: int | str | None = None,
@@ -1448,6 +1656,311 @@ class _BrowserRuntimeCore:
1656 def _nudged_viewport(viewport: dict[str, int]) -> dict[str, int]:
1657 return _nudged_viewport(viewport)
1658
1659 + async def _point_for(
1660 + self,
1661 + page: Any,
1662 + reference_id: int | str,
1663 + *,
1664 + offset_x: float = 0,
1665 + offset_y: float = 0,
1666 + ) -> dict[str, Any]:
1667 + await self._ensure_content_helper(page)
1668 + point = await page.evaluate(
1669 + "(args) => globalThis.__spaceBrowserPageContent__.pointFor(args.ref, args.offsets)",
1670 + {
1671 + "ref": reference_id,
1672 + "offsets": {
1673 + "offset_x": float(offset_x),
1674 + "offset_y": float(offset_y),
1675 + "useOffsets": bool(offset_x or offset_y),
1676 + },
1677 + },
1678 + )
1679 + if not point or not isinstance(point, dict):
1680 + raise ValueError(f"Could not resolve Browser ref {reference_id!r} to a viewport point")
1681 + return point
1682 +
1683 + async def _input_point(
1684 + self,
1685 + page: Any,
1686 + reference_id: int | str | None,
1687 + *,
1688 + x: float = 0,
1689 + y: float = 0,
1690 + offset_x: float = 0,
1691 + offset_y: float = 0,
1692 + ) -> dict[str, Any]:
1693 + if self._has_reference(reference_id):
1694 + return await self._point_for(
1695 + page,
1696 + reference_id,
1697 + offset_x=offset_x,
1698 + offset_y=offset_y,
1699 + )
1700 + return {
1701 + "x": float(x),
1702 + "y": float(y),
1703 + "rect": None,
1704 + "selector": None,
1705 + }
1706 +
1707 + async def hover(
1708 + self,
1709 + browser_id: int | str | None,
1710 + ref: int | str | None = None,
1711 + x: float = 0,
1712 + y: float = 0,
1713 + offset_x: float = 0,
1714 + offset_y: float = 0,
1715 + ) -> dict[str, Any]:
1716 + await self.ensure_started()
1717 + resolved_id = self._resolve_browser_id(browser_id)
1718 + page = self._page(resolved_id)
1719 + point = await self._input_point(
1720 + page,
1721 + ref,
1722 + x=x,
1723 + y=y,
1724 + offset_x=offset_x,
1725 + offset_y=offset_y,
1726 + )
1727 + await page.mouse.move(float(point["x"]), float(point["y"]))
1728 + self._maybe_promote(resolved_id)
1729 + return {
1730 + "action": {
1731 + "point": point,
1732 + "ref": ref if self._has_reference(ref) else None,
1733 + },
1734 + "state": await self._state(resolved_id),
1735 + }
1736 +
1737 + async def double_click(
1738 + self,
1739 + browser_id: int | str | None,
1740 + ref: int | str | None = None,
1741 + x: float = 0,
1742 + y: float = 0,
1743 + button: str = "left",
1744 + modifiers: list[str] | str | None = None,
1745 + offset_x: float = 0,
1746 + offset_y: float = 0,
1747 + ) -> dict[str, Any]:
1748 + modifiers = self._normalize_modifiers(modifiers)
1749 + await self.ensure_started()
1750 + resolved_id = self._resolve_browser_id(browser_id)
1751 + page = self._page(resolved_id)
1752 + point = await self._input_point(
1753 + page,
1754 + ref,
1755 + x=x,
1756 + y=y,
1757 + offset_x=offset_x,
1758 + offset_y=offset_y,
1759 + )
1760 + pressed: list[str] = []
1761 + try:
1762 + if modifiers:
1763 + for mod in modifiers:
1764 + await page.keyboard.down(mod)
1765 + pressed.append(mod)
1766 + await page.mouse.dblclick(float(point["x"]), float(point["y"]), button=button or "left")
1767 + finally:
1768 + for mod in reversed(pressed):
1769 + with contextlib.suppress(Exception):
1770 + await page.keyboard.up(mod)
1771 + await self._settle(page, short=True)
1772 + self._maybe_promote(resolved_id)
1773 + return {
1774 + "action": {
1775 + "button": button or "left",
1776 + "modifiers": modifiers or [],
1777 + "point": point,
1778 + "ref": ref if self._has_reference(ref) else None,
1779 + },
1780 + "state": await self._state(resolved_id),
1781 + }
1782 +
1783 + async def right_click(
1784 + self,
1785 + browser_id: int | str | None,
1786 + ref: int | str | None = None,
1787 + x: float = 0,
1788 + y: float = 0,
1789 + modifiers: list[str] | str | None = None,
1790 + offset_x: float = 0,
1791 + offset_y: float = 0,
1792 + ) -> dict[str, Any]:
1793 + modifiers = self._normalize_modifiers(modifiers)
1794 + await self.ensure_started()
1795 + resolved_id = self._resolve_browser_id(browser_id)
1796 + page = self._page(resolved_id)
1797 + point = await self._input_point(
1798 + page,
1799 + ref,
1800 + x=x,
1801 + y=y,
1802 + offset_x=offset_x,
1803 + offset_y=offset_y,
1804 + )
1805 + pressed: list[str] = []
1806 + try:
1807 + if modifiers:
1808 + for mod in modifiers:
1809 + await page.keyboard.down(mod)
1810 + pressed.append(mod)
1811 + await page.mouse.click(float(point["x"]), float(point["y"]), button="right")
1812 + finally:
1813 + for mod in reversed(pressed):
1814 + with contextlib.suppress(Exception):
1815 + await page.keyboard.up(mod)
1816 + await self._settle(page, short=True)
1817 + self._maybe_promote(resolved_id)
1818 + return {
1819 + "action": {
1820 + "button": "right",
1821 + "modifiers": modifiers or [],
1822 + "point": point,
1823 + "ref": ref if self._has_reference(ref) else None,
1824 + },
1825 + "state": await self._state(resolved_id),
1826 + }
1827 +
1828 + async def drag(
1829 + self,
1830 + browser_id: int | str | None,
1831 + ref: int | str | None = None,
1832 + target_ref: int | str | None = None,
1833 + x: float = 0,
1834 + y: float = 0,
1835 + to_x: float = 0,
1836 + to_y: float = 0,
1837 + offset_x: float = 0,
1838 + offset_y: float = 0,
1839 + target_offset_x: float = 0,
1840 + target_offset_y: float = 0,
1841 + ) -> dict[str, Any]:
1842 + await self.ensure_started()
1843 + resolved_id = self._resolve_browser_id(browser_id)
1844 + page = self._page(resolved_id)
1845 + start_point = await self._input_point(
1846 + page,
1847 + ref,
1848 + x=x,
1849 + y=y,
1850 + offset_x=offset_x,
1851 + offset_y=offset_y,
1852 + )
1853 + end_point = await self._input_point(
1854 + page,
1855 + target_ref,
1856 + x=to_x,
1857 + y=to_y,
1858 + offset_x=target_offset_x,
1859 + offset_y=target_offset_y,
1860 + )
1861 + await page.mouse.move(float(start_point["x"]), float(start_point["y"]))
1862 + await page.mouse.down()
1863 + await page.mouse.move(float(end_point["x"]), float(end_point["y"]), steps=12)
1864 + await page.mouse.up()
1865 + await self._settle(page, short=True)
1866 + self._maybe_promote(resolved_id)
1867 + return {
1868 + "action": {
1869 + "from": start_point,
1870 + "ref": ref if self._has_reference(ref) else None,
1871 + "target_ref": target_ref if self._has_reference(target_ref) else None,
1872 + "to": end_point,
1873 + },
1874 + "state": await self._state(resolved_id),
1875 + }
1876 +
1877 + async def select_option(
1878 + self,
1879 + browser_id: int | str | None,
1880 + ref: int | str,
1881 + value: str = "",
1882 + values: list[str] | None = None,
1883 + ) -> dict[str, Any]:
1884 + await self.ensure_started()
1885 + resolved_id = self._resolve_browser_id(browser_id)
1886 + page = self._page(resolved_id)
1887 + await self._ensure_content_helper(page)
1888 + action = await page.evaluate(
1889 + "(args) => globalThis.__spaceBrowserPageContent__.select(args.ref, args.values)",
1890 + {
1891 + "ref": ref,
1892 + "values": values if values is not None else value,
1893 + },
1894 + )
1895 + await self._settle(page, short=True)
1896 + self._maybe_promote(resolved_id)
1897 + return {"action": action or {}, "state": await self._state(resolved_id)}
1898 +
1899 + async def set_checked(
1900 + self,
1901 + browser_id: int | str | None,
1902 + ref: int | str,
1903 + checked: bool = True,
1904 + ) -> dict[str, Any]:
1905 + await self.ensure_started()
1906 + resolved_id = self._resolve_browser_id(browser_id)
1907 + page = self._page(resolved_id)
1908 + await self._ensure_content_helper(page)
1909 + action = await page.evaluate(
1910 + "(args) => globalThis.__spaceBrowserPageContent__.setChecked(args.ref, args.checked)",
1911 + {
1912 + "ref": ref,
1913 + "checked": bool(checked),
1914 + },
1915 + )
1916 + await self._settle(page, short=True)
1917 + self._maybe_promote(resolved_id)
1918 + return {"action": action or {}, "state": await self._state(resolved_id)}
1919 +
1920 + async def upload_file(
1921 + self,
1922 + browser_id: int | str | None,
1923 + ref: int | str,
1924 + path: str = "",
1925 + paths: list[str] | None = None,
1926 + ) -> dict[str, Any]:
1927 + upload_paths = self._normalize_upload_paths(path=path, paths=paths)
1928 + await self.ensure_started()
1929 + resolved_id = self._resolve_browser_id(browser_id)
1930 + page = self._page(resolved_id)
1931 + await self._ensure_content_helper(page)
1932 + metadata = await page.evaluate(
1933 + "(ref) => globalThis.__spaceBrowserPageContent__.fileInputFor(ref)",
1934 + ref,
1935 + )
1936 + handle = None
1937 + try:
1938 + handle = await page.evaluate_handle(
1939 + "(ref) => globalThis.__spaceBrowserPageContent__.fileInputElementFor(ref)",
1940 + ref,
1941 + )
1942 + element = handle.as_element() if handle else None
1943 + if element:
1944 + await element.set_input_files(upload_paths)
1945 + elif metadata and metadata.get("selector"):
1946 + await page.set_input_files(metadata["selector"], upload_paths)
1947 + else:
1948 + raise ValueError(f"Browser ref {ref!r} does not resolve to a file input")
1949 + finally:
1950 + if handle:
1951 + with contextlib.suppress(Exception):
1952 + await handle.dispose()
1953 + await self._settle(page, short=True)
1954 + self._maybe_promote(resolved_id)
1955 + return {
1956 + "action": {
1957 + "files": upload_paths,
1958 + "input": metadata or {},
1959 + "ref": ref,
1960 + },
1961 + "state": await self._state(resolved_id),
1962 + }
1963 +
1964 async def mouse(
1965 self,
1966 browser_id: int | str | None,
@@ -1747,7 +2260,7 @@ class _BrowserRuntimeCore:
2260
2261 async def _ensure_content_helper(self, page: Any) -> None:
2262 has_helper = await page.evaluate(
1750 - "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate && globalThis.__spaceBrowserPageContent__?.boundingBoxFor)"
2263 + "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate && globalThis.__spaceBrowserPageContent__?.boundingBoxFor && globalThis.__spaceBrowserPageContent__?.pointFor && globalThis.__spaceBrowserPageContent__?.select && globalThis.__spaceBrowserPageContent__?.setChecked && globalThis.__spaceBrowserPageContent__?.fileInputFor)"
2264 )
2265 if has_helper:
2266 return
plugins/_browser/prompts/agent.system.tool.browser.md
+56 -13
@@ -6,8 +6,10 @@ refs come from content as typed markers: [link 3], [button 6], [image 1], [input
6
7 Browser tool actions must not open the right canvas automatically. Use the tool headlessly unless the user opens the Browser canvas or explicitly asks for a visible browser view; if the Browser canvas is already open, it may reflect the active page.
8
9 -actions: open list state set_active navigate back forward reload content detail click type submit type_submit scroll evaluate key_chord mouse multi close close_all
10 -common args: action browser_id url ref text selector selectors script modifiers keys include_content focus_popup event_type x y button calls
9 +Browser does not automatically load screenshots or canvas images into model context. Screenshots are explicit only.
10 +
11 +actions: open list state set_active navigate back forward reload content detail screenshot click hover double_click right_click drag type submit type_submit scroll evaluate key_chord mouse wheel keyboard clipboard set_viewport select_option set_checked upload_file multi close close_all
12 +common args: action browser_id url ref target_ref text selector selectors script modifiers keys key include_content focus_popup event_type x y to_x to_y offset_x offset_y target_offset_x target_offset_y delta_x delta_y button quality full_page path paths value values checked width height calls
13
14 workflow:
15 - open creates a new browser and returns id/state
@@ -17,6 +19,34 @@ workflow:
19 - navigate/back/forward/reload return fresh state
20 - list shows open browsers; pass include_content: true for one-call bulk read
21
22 +explicit vision workflow:
23 +1. call browser with action: "screenshot"
24 +2. call vision_load with the returned path
25 +3. reason from the latest loaded screenshot, not an older screenshot
26 +
27 +screenshot:
28 +- saves a JPEG by default and returns path, a0_path, mime, state, and a ready vision_load tool_args object
29 +- pass quality 20..95, full_page true/false, or path
30 +- PNG is used only when path ends in .png
31 +- no base64 image data is returned in the tool message
32 +
33 +pointer and raw input:
34 +- hover moves to a ref center or x/y viewport CSS pixels
35 +- double_click and right_click accept ref or x/y; double_click accepts button and modifiers
36 +- drag moves from ref or x/y to target_ref or to_x/to_y
37 +- wheel scrolls at x/y with delta_x and delta_y
38 +- keyboard presses key or types text into the active page
39 +- clipboard is copy, cut, or paste; for browser:clipboard pass action: "paste" and optional text
40 +- set_viewport resizes the page viewport with width and height
41 +- coordinates are Chromium viewport CSS pixels and match screenshots/Browser canvas
42 +- ref offsets are relative to the target element top-left; refs default to element center
43 +
44 +forms:
45 +- use select_option for native select and safely detectable ARIA listbox/combobox controls
46 +- use set_checked for checkbox, radio, switch, and toggle-like refs
47 +- use upload_file for file input refs or associated labels; verify file paths exist before upload
48 +- for complex forms, load browser-forms first with skills_tool:load
49 +
50 modifier clicks:
51 - click accepts modifiers like ["Control"], ["Shift"], ["Alt"], ["Meta"]
52 - ctrl/meta-click opens link in new tab in background (Chrome rule)
@@ -33,7 +63,7 @@ background work (do not steal focus):
63 - open (new tab created)
64 - explicit set_active action
65 - action on the already-active tab
36 - - chrome popup-focus rule (plain click on target=_blank → follow; ctrl-click → stay)
66 + - chrome popup-focus rule (plain click on target=_blank -> follow; ctrl-click -> stay)
67 - to switch focus deliberately: {"action":"set_active","browser_id":N}
68
69 key_chord:
@@ -46,6 +76,7 @@ multi (parallel batch):
76 - different browser_ids run in parallel; same browser_id runs in submit order
77 - returns array of {"ok":true,"result":...} or {"ok":false,"error":"..."} matching input order
78 - ideal for: scrape N tabs at once, fan-out reads, parallel evaluate
79 +- new v1 actions such as screenshot, hover, wheel, keyboard, select_option, set_checked, and upload_file are accepted
80 - avoid mutating same tab twice in one batch unless serial order is intended
81
82 examples:
@@ -73,9 +104,18 @@ examples:
104 {
105 "tool_name": "browser",
106 "tool_args": {
76 - "action": "click",
107 + "action": "screenshot",
108 "browser_id": 1,
78 - "ref": 3
109 + "quality": 80
110 + }
111 +}
112 +~~~
113 +
114 +~~~json
115 +{
116 + "tool_name": "vision_load",
117 + "tool_args": {
118 + "paths": ["/absolute/local/path.jpg"]
119 }
120 }
121 ~~~
@@ -84,10 +124,10 @@ examples:
124 {
125 "tool_name": "browser",
126 "tool_args": {
87 - "action": "click",
127 + "action": "select_option",
128 "browser_id": 1,
89 - "ref": 3,
90 - "modifiers": ["Control"]
129 + "ref": 8,
130 + "value": "Canada"
131 }
132 }
133 ~~~
@@ -96,9 +136,10 @@ examples:
136 {
137 "tool_name": "browser",
138 "tool_args": {
99 - "action": "key_chord",
139 + "action": "set_checked",
140 "browser_id": 1,
101 - "keys": ["Control", "a"]
141 + "ref": 9,
142 + "checked": true
143 }
144 }
145 ~~~
@@ -107,8 +148,10 @@ examples:
148 {
149 "tool_name": "browser",
150 "tool_args": {
110 - "action": "list",
111 - "include_content": true
151 + "action": "upload_file",
152 + "browser_id": 1,
153 + "ref": 10,
154 + "path": "/a0/usr/workdir/resume.pdf"
155 }
156 }
157 ~~~
@@ -120,7 +163,7 @@ examples:
163 "action": "multi",
164 "calls": [
165 {"action": "content", "browser_id": 1},
123 - {"action": "content", "browser_id": 2},
166 + {"action": "screenshot", "browser_id": 2},
167 {"action": "evaluate", "browser_id": 3, "script": "document.title"}
168 ]
169 }
plugins/_browser/skills/browser-forms/SKILL.md new
+18
@@ -0,0 +1,18 @@
1 +---
2 +name: browser-forms
3 +description: Use for complex Agent Zero Browser form workflows involving selects, checkboxes, radios, file uploads, contenteditable fields, multi-step validation, or visually verified submission.
4 +---
5 +
6 +# Browser Forms
7 +
8 +Use this skill for complex Browser form workflows where the page state may depend on selects, checkboxes, radios, file uploads, contenteditable fields, validation, or visual confirmation.
9 +
10 +Start with `browser:content` to capture current refs, then use `browser:detail` on ambiguous fields before acting. Prefer ref-based form actions before coordinates.
11 +
12 +Use `select_option`, `set_checked`, `upload_file`, `type`, `type_submit`, and `submit` for form interaction. Use coordinates only when no stable ref exists or the UI is intentionally canvas-like.
13 +
14 +Use `browser:screenshot` plus `vision_load` when layout, visual validation, captcha-like UI, canvas content, or hidden state matters. Browser screenshots are not automatically loaded into model-visible history.
15 +
16 +Verify after submission with `browser:content`, `browser:state`, or another explicit `browser:screenshot` plus `vision_load`.
17 +
18 +Do not guess file paths for upload. Verify that every path exists before calling `upload_file`.
plugins/_browser/tools/browser.py
+144 -1
@@ -14,22 +14,48 @@ class Browser(Tool):
14 browser_id: int | str | None = None,
15 url: str = "",
16 ref: int | str | None = None,
17 + target_ref: int | str | None = None,
18 text: str = "",
19 selector: str = "",
20 selectors: list[str] | None = None,
21 script: str = "",
22 modifiers: list[str] | str | None = None,
23 keys: list[str] | None = None,
24 + key: str = "",
25 include_content: bool = False,
26 focus_popup: bool | None = None,
27 event_type: str = "",
28 x: float = 0.0,
29 y: float = 0.0,
30 + to_x: float = 0.0,
31 + to_y: float = 0.0,
32 + offset_x: float = 0.0,
33 + offset_y: float = 0.0,
34 + target_offset_x: float = 0.0,
35 + target_offset_y: float = 0.0,
36 + delta_x: float = 0.0,
37 + delta_y: float = 0.0,
38 button: str = "left",
39 + quality: int = 80,
40 + full_page: bool = False,
41 + path: str = "",
42 + paths: list[str] | None = None,
43 + value: str = "",
44 + values: list[str] | None = None,
45 + checked: bool | None = None,
46 + width: int = 0,
47 + height: int = 0,
48 calls: list[dict[str, Any]] | None = None,
49 **kwargs: Any,
50 ) -> Response:
32 - action = str(action or self.method or "state").strip().lower().replace("-", "_")
51 + method_action = str(self.method or "").strip().lower().replace("-", "_")
52 + requested_action = str(action or "").strip().lower().replace("-", "_")
53 + clipboard_action = ""
54 + if method_action == "clipboard" and requested_action in {"copy", "cut", "paste"}:
55 + clipboard_action = requested_action
56 + action = "clipboard"
57 + else:
58 + action = str(action or self.method or "state").strip().lower().replace("-", "_")
59 runtime = await get_runtime(self.agent.context.id)
60
61 if isinstance(modifiers, str):
@@ -40,6 +66,14 @@ class Browser(Tool):
66 try:
67 if action == "open":
68 result = await runtime.call("open", url or "")
69 + elif action == "screenshot":
70 + result = await runtime.call(
71 + "screenshot_file",
72 + browser_id,
73 + quality=quality,
74 + full_page=full_page,
75 + path=path,
76 + )
77 elif action == "list":
78 result = await runtime.call("list", include_content=bool(include_content))
79 elif action == "state":
@@ -86,6 +120,115 @@ class Browser(Tool):
120 if not keys:
121 raise ValueError("key_chord requires non-empty 'keys' list")
122 result = await runtime.call("key_chord", browser_id, list(keys))
123 + elif action == "hover":
124 + result = await runtime.call(
125 + "hover",
126 + browser_id,
127 + ref=ref,
128 + x=x,
129 + y=y,
130 + offset_x=offset_x,
131 + offset_y=offset_y,
132 + )
133 + elif action == "double_click":
134 + result = await runtime.call(
135 + "double_click",
136 + browser_id,
137 + ref=ref,
138 + x=x,
139 + y=y,
140 + button=button or "left",
141 + modifiers=modifiers,
142 + offset_x=offset_x,
143 + offset_y=offset_y,
144 + )
145 + elif action == "right_click":
146 + result = await runtime.call(
147 + "right_click",
148 + browser_id,
149 + ref=ref,
150 + x=x,
151 + y=y,
152 + modifiers=modifiers,
153 + offset_x=offset_x,
154 + offset_y=offset_y,
155 + )
156 + elif action == "drag":
157 + result = await runtime.call(
158 + "drag",
159 + browser_id,
160 + ref=ref,
161 + target_ref=target_ref,
162 + x=x,
163 + y=y,
164 + to_x=to_x,
165 + to_y=to_y,
166 + offset_x=offset_x,
167 + offset_y=offset_y,
168 + target_offset_x=target_offset_x,
169 + target_offset_y=target_offset_y,
170 + )
171 + elif action == "wheel":
172 + result = await runtime.call(
173 + "wheel",
174 + browser_id,
175 + x,
176 + y,
177 + delta_x,
178 + delta_y,
179 + )
180 + elif action == "keyboard":
181 + result = await runtime.call(
182 + "keyboard",
183 + browser_id,
184 + key=key,
185 + text=text,
186 + )
187 + elif action == "clipboard":
188 + normalized_clipboard_action = clipboard_action or str(
189 + kwargs.get("clipboard_action")
190 + or kwargs.get("operation")
191 + or event_type
192 + or ""
193 + ).strip().lower()
194 + result = await runtime.call(
195 + "clipboard",
196 + browser_id,
197 + action=normalized_clipboard_action,
198 + text=text,
199 + )
200 + elif action in {"copy", "cut", "paste"}:
201 + result = await runtime.call(
202 + "clipboard",
203 + browser_id,
204 + action=action,
205 + text=text,
206 + )
207 + elif action == "set_viewport":
208 + result = await runtime.call("set_viewport", browser_id, width, height)
209 + elif action == "select_option":
210 + result = await runtime.call(
211 + "select_option",
212 + browser_id,
213 + self._require_ref(ref),
214 + value=value,
215 + values=values,
216 + )
217 + elif action == "set_checked":
218 + result = await runtime.call(
219 + "set_checked",
220 + browser_id,
221 + self._require_ref(ref),
222 + checked=True if checked is None else bool(checked),
223 + )
224 + elif action == "upload_file":
225 + result = await runtime.call(
226 + "upload_file",
227 + browser_id,
228 + self._require_ref(ref),
229 + path=path,
230 + paths=paths,
231 + )
232 elif action == "mouse":
233 result = await runtime.call(
234 "mouse", browser_id, event_type or "click", x, y,
tests/test_browser_agent_regressions.py
+327
@@ -733,10 +733,32 @@ def test_browser_tool_does_not_auto_open_canvas_policy_is_documented():
733 assert "must not open the right canvas automatically" in prompt
734 assert "Use the tool headlessly unless the user opens the Browser canvas" in prompt
735 assert "optional visible WebUI viewer" in prompt
736 + assert "screenshot" in prompt
737 + assert "vision_load" in prompt
738 + assert "select_option" in prompt
739 + assert "set_checked" in prompt
740 + assert "upload_file" in prompt
741 + assert "browser-forms" in prompt
742 + assert "does not automatically load screenshots" in prompt
743 assert "already open" in config
744 assert "already-open Browser canvas" in config_html
745
746
747 +def test_browser_forms_skill_is_plugin_owned_and_discoverable():
748 + skill_path = PROJECT_ROOT / "plugins" / "_browser" / "skills" / "browser-forms" / "SKILL.md"
749 + assert skill_path.exists()
750 + skill = skill_path.read_text(encoding="utf-8")
751 + assert skill.startswith("---\n")
752 + frontmatter = skill.split("---", 2)[1]
753 + assert "name: browser-forms" in frontmatter
754 + assert "description:" in frontmatter
755 + assert "select_option" in skill
756 + assert "set_checked" in skill
757 + assert "upload_file" in skill
758 + assert "browser:screenshot" in skill
759 + assert "vision_load" in skill
760 +
761 +
762 def test_browser_canvas_uses_plain_panel_without_debug_probe():
763 panel_html = (
764 PROJECT_ROOT
@@ -1014,10 +1036,26 @@ def test_browser_runtime_and_content_helper_expose_annotation_target():
1036 assert "function annotate(payload = null)" in helper
1037 assert "annotate," in helper
1038 assert "boundingBoxFor," in helper
1039 + assert "pointFor," in helper
1040 + assert "select(referenceId, valueOrValues)" in helper
1041 + assert "setChecked(referenceId, checked)" in helper
1042 + assert "fileInputFor," in helper
1043 assert "sanitizeAnnotationDom" in helper
1044 assert "password" in helper
1045
1046
1047 +def test_browser_content_helper_keeps_label_wrapped_controls_referenceable():
1048 + helper = (
1049 + PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"
1050 + ).read_text(encoding="utf-8")
1051 +
1052 + assert 'const VERSION = "11"' in helper
1053 + assert "function renderControlLabelReferences" in helper
1054 + assert "getLabelElementText(labelElement, element)" in helper
1055 + assert "return renderControlLabelReferences(node, context);" in helper
1056 + assert "return renderControlLabelReferences(element, context);" in helper
1057 +
1058 +
1059 def test_browser_runtime_requires_current_content_helper_for_modifier_clicks():
1060 runtime = (
1061 PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
@@ -1345,6 +1383,164 @@ async def test_browser_tool_dispatches_direct_actions(monkeypatch):
1383 assert calls == [("content", (1, None))]
1384
1385
1386 +@pytest.mark.anyio
1387 +async def test_browser_tool_dispatches_v1_agent_actions(monkeypatch):
1388 + calls = []
1389 +
1390 + class FakeRuntime:
1391 + async def call(self, method, *args, **kwargs):
1392 + calls.append((method, args, kwargs))
1393 + return {"ok": True, "method": method, "args": args, "kwargs": kwargs}
1394 +
1395 + async def fake_get_runtime(context_id, create=True):
1396 + assert context_id == "ctx"
1397 + return FakeRuntime()
1398 +
1399 + monkeypatch.setattr(browser_tool_module, "get_runtime", fake_get_runtime)
1400 + agent = SimpleNamespace(context=SimpleNamespace(id="ctx"))
1401 +
1402 + async def execute(**kwargs):
1403 + tool = browser_tool_module.Browser(
1404 + agent=agent,
1405 + name="browser",
1406 + method=kwargs.pop("_method", None),
1407 + args={},
1408 + message="",
1409 + loop_data=None,
1410 + )
1411 + response = await tool.execute(**kwargs)
1412 + assert response.break_loop is False
1413 +
1414 + await execute(action="screenshot", browser_id=1, quality=91, full_page=True, path="/tmp/a.jpg")
1415 + await execute(action="hover", browser_id=1, ref=2, offset_x=3, offset_y=4)
1416 + await execute(action="double_click", browser_id=1, x=10, y=20, button="left", modifiers=["Shift"])
1417 + await execute(action="right_click", browser_id=1, ref=3, modifiers="Control")
1418 + await execute(action="drag", browser_id=1, ref=4, target_ref=5, target_offset_x=6, target_offset_y=7)
1419 + await execute(action="wheel", browser_id=1, x=8, y=9, delta_x=1, delta_y=2)
1420 + await execute(action="keyboard", browser_id=1, key="Enter")
1421 + await execute(_method="clipboard", action="paste", browser_id=1, text="hello")
1422 + await execute(action="copy", browser_id=1)
1423 + await execute(action="set_viewport", browser_id=1, width=1280, height=720)
1424 + await execute(action="select_option", browser_id=1, ref=6, value="CA")
1425 + await execute(action="set_checked", browser_id=1, ref=7, checked=False)
1426 + await execute(action="upload_file", browser_id=1, ref=8, paths=["/tmp/a.txt"])
1427 +
1428 + assert calls == [
1429 + ("screenshot_file", (1,), {"quality": 91, "full_page": True, "path": "/tmp/a.jpg"}),
1430 + ("hover", (1,), {"ref": 2, "x": 0.0, "y": 0.0, "offset_x": 3, "offset_y": 4}),
1431 + (
1432 + "double_click",
1433 + (1,),
1434 + {
1435 + "ref": None,
1436 + "x": 10,
1437 + "y": 20,
1438 + "button": "left",
1439 + "modifiers": ["Shift"],
1440 + "offset_x": 0.0,
1441 + "offset_y": 0.0,
1442 + },
1443 + ),
1444 + (
1445 + "right_click",
1446 + (1,),
1447 + {
1448 + "ref": 3,
1449 + "x": 0.0,
1450 + "y": 0.0,
1451 + "modifiers": ["Control"],
1452 + "offset_x": 0.0,
1453 + "offset_y": 0.0,
1454 + },
1455 + ),
1456 + (
1457 + "drag",
1458 + (1,),
1459 + {
1460 + "ref": 4,
1461 + "target_ref": 5,
1462 + "x": 0.0,
1463 + "y": 0.0,
1464 + "to_x": 0.0,
1465 + "to_y": 0.0,
1466 + "offset_x": 0.0,
1467 + "offset_y": 0.0,
1468 + "target_offset_x": 6,
1469 + "target_offset_y": 7,
1470 + },
1471 + ),
1472 + ("wheel", (1, 8, 9, 1, 2), {}),
1473 + ("keyboard", (1,), {"key": "Enter", "text": ""}),
1474 + ("clipboard", (1,), {"action": "paste", "text": "hello"}),
1475 + ("clipboard", (1,), {"action": "copy", "text": ""}),
1476 + ("set_viewport", (1, 1280, 720), {}),
1477 + ("select_option", (1, 6), {"value": "CA", "values": None}),
1478 + ("set_checked", (1, 7), {"checked": False}),
1479 + ("upload_file", (1, 8), {"path": "", "paths": ["/tmp/a.txt"]}),
1480 + ]
1481 +
1482 +
1483 +@pytest.mark.anyio
1484 +async def test_browser_multi_dispatch_accepts_v1_actions():
1485 + calls = []
1486 + core = _BrowserRuntimeCore("ctx")
1487 +
1488 + async def record(method):
1489 + async def inner(*args, **kwargs):
1490 + calls.append((method, args, kwargs))
1491 + return {"method": method}
1492 + return inner
1493 +
1494 + for method in (
1495 + "screenshot_file",
1496 + "hover",
1497 + "double_click",
1498 + "right_click",
1499 + "drag",
1500 + "wheel",
1501 + "keyboard",
1502 + "clipboard",
1503 + "set_viewport",
1504 + "select_option",
1505 + "set_checked",
1506 + "upload_file",
1507 + ):
1508 + setattr(core, method, await record(method))
1509 +
1510 + results = await core.multi(
1511 + [
1512 + {"action": "screenshot", "browser_id": 1, "quality": 5, "full_page": True},
1513 + {"action": "hover", "browser_id": 1, "ref": 2},
1514 + {"action": "double_click", "browser_id": 1, "x": 1, "y": 2},
1515 + {"action": "right_click", "browser_id": 1, "ref": 3},
1516 + {"action": "drag", "browser_id": 1, "ref": 4, "target_ref": 5},
1517 + {"action": "wheel", "browser_id": 1, "delta_y": 100},
1518 + {"action": "keyboard", "browser_id": 1, "key": "Enter"},
1519 + {"action": "paste", "browser_id": 1, "text": "x"},
1520 + {"action": "set_viewport", "browser_id": 1, "width": 640, "height": 480},
1521 + {"action": "select_option", "browser_id": 1, "ref": 6, "values": ["a", "b"]},
1522 + {"action": "set_checked", "browser_id": 1, "ref": 7, "checked": False},
1523 + {"action": "upload_file", "browser_id": 1, "ref": 8, "path": "/tmp/file.txt"},
1524 + ]
1525 + )
1526 +
1527 + assert all(result["ok"] for result in results)
1528 + assert [call[0] for call in calls] == [
1529 + "screenshot_file",
1530 + "hover",
1531 + "double_click",
1532 + "right_click",
1533 + "drag",
1534 + "wheel",
1535 + "keyboard",
1536 + "clipboard",
1537 + "set_viewport",
1538 + "select_option",
1539 + "set_checked",
1540 + "upload_file",
1541 + ]
1542 +
1543 +
1544 @pytest.mark.anyio
1545 async def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):
1546 class FakeRuntime:
@@ -1641,6 +1837,137 @@ async def test_browser_runtime_remounts_initial_changed_viewport():
1837 assert settled == [True]
1838
1839
1840 +@pytest.mark.anyio
1841 +async def test_browser_runtime_screenshot_file_writes_without_base64(monkeypatch, tmp_path):
1842 + screenshot_calls = []
1843 +
1844 + def fake_get_abs_path(*parts):
1845 + return str(tmp_path.joinpath(*parts))
1846 +
1847 + def fake_normalize_a0_path(path):
1848 + return "/a0/" + str(Path(path).relative_to(tmp_path)).replace("\\", "/")
1849 +
1850 + monkeypatch.setattr(browser_runtime_module.files, "get_abs_path", fake_get_abs_path)
1851 + monkeypatch.setattr(browser_runtime_module.files, "normalize_a0_path", fake_normalize_a0_path)
1852 +
1853 + class FakePage:
1854 + url = "about:blank"
1855 + viewport_size = {"width": 1024, "height": 768}
1856 +
1857 + async def screenshot(self, **kwargs):
1858 + screenshot_calls.append(kwargs)
1859 + Path(kwargs["path"]).parent.mkdir(parents=True, exist_ok=True)
1860 + Path(kwargs["path"]).write_bytes(b"image-bytes")
1861 + return b"image-bytes"
1862 +
1863 + async def title(self):
1864 + return "Blank"
1865 +
1866 + async def evaluate(self, script, payload=None):
1867 + return 1
1868 +
1869 + core = _BrowserRuntimeCore("ctx/id")
1870 + core.context = object()
1871 + core.pages[5] = browser_runtime_module.BrowserPage(id=5, page=FakePage())
1872 +
1873 + result = await core.screenshot_file(5, quality=500)
1874 +
1875 + path = Path(result["path"])
1876 + assert path.exists()
1877 + assert path.parent == tmp_path / "tmp" / "browser" / "screenshots" / "ctx_id"
1878 + assert path.name.startswith("browser-5-")
1879 + assert path.suffix == ".jpg"
1880 + assert result["a0_path"].startswith("/a0/tmp/browser/screenshots/ctx_id/browser-5-")
1881 + assert result["mime"] == "image/jpeg"
1882 + assert result["vision_load"] == {
1883 + "tool_name": "vision_load",
1884 + "tool_args": {"paths": [result["path"]]},
1885 + }
1886 + assert "image" not in result
1887 + assert screenshot_calls[-1]["type"] == "jpeg"
1888 + assert screenshot_calls[-1]["quality"] == 95
1889 + assert screenshot_calls[-1]["full_page"] is False
1890 +
1891 + png_path = tmp_path / "custom.png"
1892 + png_result = await core.screenshot_file(5, quality=1, full_page=True, path=str(png_path))
1893 +
1894 + assert png_result["path"] == str(png_path)
1895 + assert png_result["mime"] == "image/png"
1896 + assert screenshot_calls[-1] == {
1897 + "path": str(png_path),
1898 + "type": "png",
1899 + "full_page": True,
1900 + }
1901 +
1902 +
1903 +@pytest.mark.anyio
1904 +async def test_browser_runtime_ref_point_resolution_applies_offsets():
1905 + eval_payloads = []
1906 + moves = []
1907 +
1908 + class FakeMouse:
1909 + async def move(self, x, y, **kwargs):
1910 + moves.append((x, y, kwargs))
1911 +
1912 + class FakePage:
1913 + url = "about:blank"
1914 +
1915 + def __init__(self):
1916 + self.mouse = FakeMouse()
1917 +
1918 + async def evaluate(self, script, payload=None):
1919 + eval_payloads.append((script, payload))
1920 + if payload and "offsets" in payload:
1921 + return {
1922 + "x": 10 + payload["offsets"]["offset_x"],
1923 + "y": 20 + payload["offsets"]["offset_y"],
1924 + "rect": {"x": 10, "y": 20, "width": 100, "height": 40},
1925 + "selector": "#target",
1926 + }
1927 + return 1
1928 +
1929 + async def title(self):
1930 + return "Blank"
1931 +
1932 + core = _BrowserRuntimeCore("ctx")
1933 + core.context = object()
1934 + core.pages[7] = browser_runtime_module.BrowserPage(id=7, page=FakePage())
1935 + core._ensure_content_helper = lambda _page: asyncio.sleep(0)
1936 +
1937 + result = await core.hover(7, ref=4, offset_x=3, offset_y=5)
1938 +
1939 + assert moves == [(13.0, 25.0, {})]
1940 + assert result["action"]["point"]["selector"] == "#target"
1941 + assert eval_payloads[0][1] == {
1942 + "ref": 4,
1943 + "offsets": {
1944 + "offset_x": 3.0,
1945 + "offset_y": 5.0,
1946 + "useOffsets": True,
1947 + },
1948 + }
1949 +
1950 +
1951 +def test_browser_runtime_upload_path_normalization(monkeypatch, tmp_path):
1952 + first = tmp_path / "first.txt"
1953 + second = tmp_path / "second.txt"
1954 + first.write_text("one", encoding="utf-8")
1955 + second.write_text("two", encoding="utf-8")
1956 +
1957 + monkeypatch.setattr(
1958 + browser_runtime_module.files,
1959 + "get_abs_path",
1960 + lambda *parts: str(tmp_path.joinpath(*parts)),
1961 + )
1962 +
1963 + assert _BrowserRuntimeCore._normalize_upload_paths(path=str(first)) == [str(first)]
1964 + assert _BrowserRuntimeCore._normalize_upload_paths(paths=["second.txt"]) == [str(second)]
1965 + assert _BrowserRuntimeCore._normalize_upload_paths(path=str(first), paths=["second.txt"]) == [
1966 + str(second),
1967 + str(first),
1968 + ]
1969 +
1970 +
1971 @pytest.mark.anyio
1972 async def test_browser_runtime_clipboard_paste_uses_dom_bridge():
1973 eval_payloads = []