@samitouri / QOSami_Agents_Zero-1 / commits / 4ff3244c

Add browser annotate mode

Add Codex-inspired annotation UI to the built-in Browser surfaces, including the Annotate toggle, Cmd/Ctrl+. shortcut, selection overlay, inline comments, and batch Draft to chat / Send now actions. Wire browser_viewer_annotation through the WebSocket and runtime layers, and expose safe DOM metadata extraction for clicked elements and selected areas without leaking password/value data. Expand regression coverage for the Browser UI, annotation dispatch, runtime helper exposure, prompt formatting, and WebUI extension surface harness behavior.

Alessandro committed Apr 26, 2026 at 23:57 UTC 4ff3244ce63ee336071f875de99cf6922db93daa
7 files changed +1382 -35
plugins/_browser/api/ws_browser.py
+25
@@ -43,6 +43,8 @@ class WsBrowser(WsHandler):
43 return await self._command(data, sid)
44 if event == "browser_viewer_input":
45 return await self._input(data, sid)
46 + if event == "browser_viewer_annotation":
47 + return await self._annotation(data, sid)
48
49 return WsResult.error(
50 code="UNKNOWN_BROWSER_EVENT",
@@ -215,6 +217,29 @@ class WsBrowser(WsHandler):
217 else None,
218 }
219
220 + async def _annotation(self, data: dict[str, Any], sid: str) -> dict[str, Any] | WsResult:
221 + context_id = self._context_id(data)
222 + if not context_id:
223 + return self._error("MISSING_CONTEXT", "context_id is required", data)
224 + runtime = await get_runtime(context_id, create=False)
225 + if not runtime:
226 + return self._error("NO_BROWSER_RUNTIME", "No browser runtime exists for this context", data)
227 +
228 + browser_id = data.get("browser_id")
229 + viewer_id = str(data.get("viewer_id") or "")
230 + payload = data.get("payload") if isinstance(data.get("payload"), dict) else {}
231 + try:
232 + annotation = await runtime.call("annotation_target", browser_id, payload)
233 + except Exception as exc:
234 + return self._error("ANNOTATION_FAILED", str(exc), data)
235 +
236 + return {
237 + "annotation": annotation,
238 + "context_id": context_id,
239 + "browser_id": browser_id,
240 + "viewer_id": viewer_id,
241 + }
242 +
243 async def _snapshot_for_result(
244 self,
245 runtime: Any,
plugins/_browser/assets/browser-page-content.js
+368 -1
@@ -1,7 +1,7 @@
1 (() => {
2 const GLOBAL_KEY = "__spaceBrowserPageContent__";
3 const DOM_HELPER_KEY = "__spaceBrowserDomHelper__";
4 - const VERSION = "6";
4 + const VERSION = "7";
5 const BLOCK_TAGS = new Set([
6 "ADDRESS",
7 "ARTICLE",
@@ -2842,10 +2842,377 @@
2842 });
2843 }
2844
2845 + function cssEscape(value) {
2846 + const rawValue = String(value || "");
2847 + if (!rawValue) {
2848 + return "";
2849 + }
2850 +
2851 + if (typeof globalThis.CSS?.escape === "function") {
2852 + return globalThis.CSS.escape(rawValue);
2853 + }
2854 +
2855 + return rawValue.replace(/[^a-zA-Z0-9_-]/gu, (character) => `\\${character}`);
2856 + }
2857 +
2858 + function getClassSummary(element) {
2859 + try {
2860 + return [...(element?.classList || [])]
2861 + .map((className) => normalizeAttributeText(className))
2862 + .filter(Boolean)
2863 + .slice(0, 4)
2864 + .join(" ");
2865 + } catch {
2866 + return "";
2867 + }
2868 + }
2869 +
2870 + function buildCssSelector(element) {
2871 + if (!isElementNode(element)) {
2872 + return "";
2873 + }
2874 +
2875 + const id = normalizeAttributeText(element.getAttribute?.("id"));
2876 + if (id) {
2877 + return `#${cssEscape(id)}`;
2878 + }
2879 +
2880 + const parts = [];
2881 + let current = element;
2882 + while (isElementNode(current) && current !== globalThis.document?.documentElement && parts.length < 6) {
2883 + const tagName = getTagName(current).toLowerCase();
2884 + if (!tagName) {
2885 + break;
2886 + }
2887 +
2888 + let part = tagName;
2889 + const classes = getClassSummary(current)
2890 + .split(/\s+/u)
2891 + .filter(Boolean)
2892 + .slice(0, 2);
2893 + if (classes.length && !["body", "html"].includes(tagName)) {
2894 + part += classes.map((className) => `.${cssEscape(className)}`).join("");
2895 + }
2896 +
2897 + const parent = current.parentElement;
2898 + if (parent) {
2899 + const siblings = [...parent.children].filter((sibling) => getTagName(sibling) === getTagName(current));
2900 + if (siblings.length > 1) {
2901 + part += `:nth-of-type(${siblings.indexOf(current) + 1})`;
2902 + }
2903 + }
2904 +
2905 + parts.unshift(part);
2906 + if (tagName === "body") {
2907 + break;
2908 + }
2909 + current = parent;
2910 + }
2911 +
2912 + return parts.join(" > ");
2913 + }
2914 +
2915 + function sanitizeAnnotationDom(value) {
2916 + return truncateText(
2917 + String(value || "")
2918 + .replace(/(<input\b(?=[^>]*\btype\s*=\s*(["'])?password\2?)[^>]*?)\s+value\s*=\s*(["'])[\s\S]*?\3/giu, "$1 value=\"[redacted]\"")
2919 + .replace(/\svalue\s*=\s*(["'])[\s\S]{0,600}?\1/giu, " value=\"[redacted]\"")
2920 + .replace(/\sdata-space-browser-live-value\s*=\s*(["'])[\s\S]{0,600}?\1/giu, "")
2921 + .replace(/\sdata-space-browser-selected-text\s*=\s*(["'])[\s\S]{0,600}?\1/giu, ""),
2922 + 1200
2923 + );
2924 + }
2925 +
2926 + function summarizeAnnotationElement(element) {
2927 + if (!isElementNode(element)) {
2928 + return null;
2929 + }
2930 +
2931 + const summaryData = collectReferenceSummaryData(element, {
2932 + includeLabelQuotes: false,
2933 + includeLinkUrls: true,
2934 + includeSemanticTags: true,
2935 + includeStateTags: true
2936 + });
2937 + const rawDom = serializeElementSnapshot(element);
2938 + return {
2939 + classes: getClassSummary(element),
2940 + dom: sanitizeAnnotationDom(rawDom),
2941 + id: normalizeAttributeText(element.getAttribute?.("id")),
2942 + kind: summaryData.kind,
2943 + name: normalizeAttributeText(element.getAttribute?.("name")),
2944 + rect: getElementRectSafe(element),
2945 + role: normalizeAttributeText(element.getAttribute?.("role")).toLowerCase(),
2946 + selector: buildCssSelector(element),
2947 + semanticTags: Array.isArray(summaryData.semanticTags) ? summaryData.semanticTags.slice(0, 4) : [],
2948 + stateTags: Array.isArray(summaryData.state?.stateTags) ? summaryData.state.stateTags.slice(0, 8) : [],
2949 + summary: truncateText(summaryData.summary || getLabelText(element, {
2950 + includeAlt: true,
2951 + includeDescendantImageAlt: true,
2952 + includePlaceholder: true,
2953 + includeText: true
2954 + }), 240),
2955 + tagName: getTagName(element)
2956 + };
2957 + }
2958 +
2959 + function annotationViewport() {
2960 + return {
2961 + height: Math.max(0, Number(globalThis.innerHeight || globalThis.document?.documentElement?.clientHeight || 0)),
2962 + scrollX: Number(globalThis.scrollX || globalThis.pageXOffset || 0),
2963 + scrollY: Number(globalThis.scrollY || globalThis.pageYOffset || 0),
2964 + width: Math.max(0, Number(globalThis.innerWidth || globalThis.document?.documentElement?.clientWidth || 0))
2965 + };
2966 + }
2967 +
2968 + function normalizeAnnotationPoint(payload = {}, viewport = annotationViewport()) {
2969 + const source = payload?.point && typeof payload.point === "object" ? payload.point : payload;
2970 + const width = Math.max(1, Number(viewport.width || 1));
2971 + const height = Math.max(1, Number(viewport.height || 1));
2972 + return {
2973 + x: Math.max(0, Math.min(width, Number(source?.x || 0))),
2974 + y: Math.max(0, Math.min(height, Number(source?.y || 0)))
2975 + };
2976 + }
2977 +
2978 + function normalizeAnnotationRectPayload(payload = {}, viewport = annotationViewport()) {
2979 + const source = payload?.rect && typeof payload.rect === "object" ? payload.rect : payload;
2980 + const width = Math.max(1, Number(viewport.width || 1));
2981 + const height = Math.max(1, Number(viewport.height || 1));
2982 + const x = Math.max(0, Math.min(width, Number(source?.x || 0)));
2983 + const y = Math.max(0, Math.min(height, Number(source?.y || 0)));
2984 + return {
2985 + height: Math.max(1, Math.min(height - y, Number(source?.height || source?.h || 1))),
2986 + width: Math.max(1, Math.min(width - x, Number(source?.width || source?.w || 1))),
2987 + x,
2988 + y
2989 + };
2990 + }
2991 +
2992 + function intersectRects(leftRect, rightRect) {
2993 + if (!leftRect || !rightRect) {
2994 + return null;
2995 + }
2996 +
2997 + const x = Math.max(Number(leftRect.x || 0), Number(rightRect.x || 0));
2998 + const y = Math.max(Number(leftRect.y || 0), Number(rightRect.y || 0));
2999 + const right = Math.min(
3000 + Number(leftRect.x || 0) + Number(leftRect.width || 0),
3001 + Number(rightRect.x || 0) + Number(rightRect.width || 0)
3002 + );
3003 + const bottom = Math.min(
3004 + Number(leftRect.y || 0) + Number(leftRect.height || 0),
3005 + Number(rightRect.y || 0) + Number(rightRect.height || 0)
3006 + );
3007 + const width = right - x;
3008 + const height = bottom - y;
3009 + if (width <= 0 || height <= 0) {
3010 + return null;
3011 + }
3012 + return {
3013 + area: width * height,
3014 + height,
3015 + width,
3016 + x,
3017 + y
3018 + };
3019 + }
3020 +
3021 + function deepElementFromPoint(x, y) {
3022 + let element = null;
3023 + try {
3024 + element = globalThis.document?.elementFromPoint?.(x, y) || null;
3025 + } catch {
3026 + return null;
3027 + }
3028 +
3029 + let guard = 0;
3030 + while (isElementNode(element) && element.shadowRoot && guard < 8) {
3031 + guard += 1;
3032 + try {
3033 + const nestedElement = element.shadowRoot.elementFromPoint?.(x, y);
3034 + if (!nestedElement || nestedElement === element) {
3035 + break;
3036 + }
3037 + element = nestedElement;
3038 + } catch {
3039 + break;
3040 + }
3041 + }
3042 +
3043 + return element;
3044 + }
3045 +
3046 + function findAnnotationTarget(element) {
3047 + if (!isElementNode(element)) {
3048 + return null;
3049 + }
3050 +
3051 + const selector = [
3052 + "a[href]",
3053 + "button",
3054 + "input",
3055 + "textarea",
3056 + "select",
3057 + "summary",
3058 + "[role]",
3059 + "img",
3060 + "label",
3061 + "form",
3062 + "h1",
3063 + "h2",
3064 + "h3",
3065 + "h4",
3066 + "h5",
3067 + "h6",
3068 + "p",
3069 + "li",
3070 + "td",
3071 + "th",
3072 + "article",
3073 + "section",
3074 + "nav",
3075 + "header",
3076 + "main",
3077 + "footer"
3078 + ].join(",");
3079 + const target = element.closest?.(selector) || element;
3080 + return isElementNode(target) && !isHiddenElement(target) ? target : element;
3081 + }
3082 +
3083 + function isMeaningfulAnnotationElement(element) {
3084 + if (!isElementNode(element) || isHiddenElement(element)) {
3085 + return false;
3086 + }
3087 +
3088 + if (isInteractiveElement(element) || getTagName(element) === "IMG") {
3089 + return true;
3090 + }
3091 +
3092 + const tagName = getTagName(element);
3093 + const role = normalizeAttributeText(element.getAttribute?.("role")).toLowerCase();
3094 + return Boolean(
3095 + role
3096 + || /^H[1-6]$/u.test(tagName)
3097 + || ["ARTICLE", "SECTION", "MAIN", "NAV", "HEADER", "FOOTER", "FORM", "LABEL", "P", "LI", "TD", "TH"].includes(tagName)
3098 + );
3099 + }
3100 +
3101 + function collectIntersectingAnnotationElements(rect) {
3102 + const selector = [
3103 + "a[href]",
3104 + "button",
3105 + "input",
3106 + "textarea",
3107 + "select",
3108 + "summary",
3109 + "[role]",
3110 + "img",
3111 + "label",
3112 + "form",
3113 + "h1",
3114 + "h2",
3115 + "h3",
3116 + "h4",
3117 + "h5",
3118 + "h6",
3119 + "p",
3120 + "li",
3121 + "td",
3122 + "th",
3123 + "article",
3124 + "section",
3125 + "main",
3126 + "nav",
3127 + "header",
3128 + "footer"
3129 + ].join(",");
3130 + let candidates = [];
3131 + try {
3132 + candidates = [...(globalThis.document?.querySelectorAll?.(selector) || [])];
3133 + } catch {
3134 + candidates = [];
3135 + }
3136 +
3137 + const seen = new Set();
3138 + return candidates
3139 + .map((element) => {
3140 + if (!isMeaningfulAnnotationElement(element) || seen.has(element)) {
3141 + return null;
3142 + }
3143 + seen.add(element);
3144 + const elementRect = getElementRectSafe(element);
3145 + const intersection = intersectRects(rect, elementRect);
3146 + if (!intersection || intersection.area < 48) {
3147 + return null;
3148 + }
3149 + return {
3150 + element,
3151 + elementArea: Math.max(1, Number(elementRect.width || 0) * Number(elementRect.height || 0)),
3152 + intersection
3153 + };
3154 + })
3155 + .filter(Boolean)
3156 + .sort((left, right) => {
3157 + if (right.intersection.area !== left.intersection.area) {
3158 + return right.intersection.area - left.intersection.area;
3159 + }
3160 + return left.elementArea - right.elementArea;
3161 + })
3162 + .slice(0, 12)
3163 + .map((entry) => summarizeAnnotationElement(entry.element))
3164 + .filter(Boolean);
3165 + }
3166 +
3167 + function annotate(payload = null) {
3168 + const request = payload && typeof payload === "object" ? payload : {};
3169 + const viewport = annotationViewport();
3170 + const kind = request.kind === "area" || request.rect ? "area" : "element";
3171 +
3172 + if (kind === "area") {
3173 + const rect = normalizeAnnotationRectPayload(request, viewport);
3174 + const point = {
3175 + x: rect.x + rect.width / 2,
3176 + y: rect.y + rect.height / 2
3177 + };
3178 + const elements = collectIntersectingAnnotationElements(rect);
3179 + const fallbackElement = findAnnotationTarget(deepElementFromPoint(point.x, point.y));
3180 + const fallbackTarget = fallbackElement ? summarizeAnnotationElement(fallbackElement) : null;
3181 + return {
3182 + elements,
3183 + kind,
3184 + point,
3185 + rect,
3186 + status: elements.length || fallbackTarget ? "ok" : "empty",
3187 + target: elements[0] || fallbackTarget,
3188 + viewport
3189 + };
3190 + }
3191 +
3192 + const point = normalizeAnnotationPoint(request, viewport);
3193 + const rawElement = deepElementFromPoint(point.x, point.y);
3194 + const targetElement = findAnnotationTarget(rawElement);
3195 + const target = targetElement ? summarizeAnnotationElement(targetElement) : null;
3196 + return {
3197 + kind,
3198 + point,
3199 + rect: target?.rect || {
3200 + height: 1,
3201 + width: 1,
3202 + x: point.x,
3203 + y: point.y
3204 + },
3205 + status: target ? "ok" : "empty",
3206 + target,
3207 + viewport
3208 + };
3209 + }
3210 +
3211 globalThis[GLOBAL_KEY] = {
3212 click(referenceId) {
3213 return activateElement(referenceId);
3214 },
3215 + annotate,
3216 capture,
3217 clear() {
3218 state.captureId = 0;
plugins/_browser/helpers/runtime.py
+17 -1
@@ -557,6 +557,22 @@ class _BrowserRuntimeCore:
557 self.last_interacted_browser_id = resolved_id
558 return result or {}
559
560 + async def annotation_target(
561 + self,
562 + browser_id: int | str | None,
563 + payload: dict[str, Any] | None = None,
564 + ) -> dict[str, Any]:
565 + await self.ensure_started()
566 + resolved_id = self._resolve_browser_id(browser_id)
567 + page = self._page(resolved_id)
568 + await self._ensure_content_helper(page)
569 + result = await page.evaluate(
570 + "(payload) => globalThis.__spaceBrowserPageContent__.annotate(payload || null)",
571 + payload or None,
572 + )
573 + self.last_interacted_browser_id = resolved_id
574 + return result or {}
575 +
576 async def evaluate(self, browser_id: int | str | None, script: str) -> dict[str, Any]:
577 await self.ensure_started()
578 resolved_id = self._resolve_browser_id(browser_id)
@@ -918,7 +934,7 @@ class _BrowserRuntimeCore:
934
935 async def _ensure_content_helper(self, page: Any) -> None:
936 has_helper = await page.evaluate(
921 - "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture)"
937 + "() => Boolean(globalThis.__spaceBrowserPageContent__?.capture && globalThis.__spaceBrowserPageContent__?.annotate)"
938 )
939 if has_helper:
940 return
plugins/_browser/webui/browser-panel.html
+311 -2
@@ -11,7 +11,7 @@
11 <div x-data>
12 <template x-if="$store.browserPage">
13 <div class="browser-panel" x-create="$store.browserPage.onOpen($el, xAttrs($el) || {})" x-destroy="$store.browserPage.cleanup()"
14 - @keydown.window="$store.browserPage.sendKey($event)">
14 + @keydown.window="$store.browserPage.handleKeydown($event)">
15 <div class="browser-meta">
16 <div class="browser-meta-top">
17 <div class="browser-session-tabs" role="tablist" aria-label="Browser sessions">
@@ -41,6 +41,15 @@
41 </div>
42
43 <div class="browser-session-controls">
44 + <button type="button" class="btn browser-annotate-toggle" title="Annotate" aria-label="Annotate"
45 + :aria-pressed="$store.browserPage.annotating.toString()"
46 + :class="{ 'is-active': $store.browserPage.annotating }"
47 + :disabled="!$store.browserPage.canAnnotate() && !$store.browserPage.annotating"
48 + @click="$store.browserPage.toggleAnnotationMode()">
49 + <span class="material-symbols-outlined" aria-hidden="true"
50 + x-text="$store.browserPage.annotating ? 'rate_review' : 'edit_note'"></span>
51 + <span x-text="$store.browserPage.annotating ? 'Annotating' : 'Annotate'"></span>
52 + </button>
53 <div class="browser-extension-menu" @click.outside="$store.browserPage.closeExtensionsMenu()"
54 @keydown.escape.window="$store.browserPage.closeExtensionsMenu()">
55 <button type="button" class="btn btn-icon-action browser-extensions" title="Browser settings"
@@ -161,12 +170,82 @@
170 </div>
171
172 <div class="browser-stage" tabindex="0" @click="$el.focus()"
164 - @wheel.prevent="$store.browserPage.sendWheel($event)">
173 + :class="{ 'is-annotating': $store.browserPage.annotating }"
174 + @wheel.prevent="$store.browserPage.handleStageWheel($event)">
175 <template x-if="$store.browserPage.frameSrc">
176 <img class="browser-frame" :src="$store.browserPage.frameSrc"
177 @click="$store.browserPage.sendMouse('click', $event)"
178 @mousemove.throttle.250ms="$store.browserPage.sendMouse('move', $event)" draggable="false" />
179 </template>
180 + <template x-if="$store.browserPage.annotating && $store.browserPage.frameSrc">
181 + <div class="browser-annotation-layer"
182 + :class="{ 'is-busy': $store.browserPage.annotationBusy }"
183 + @pointerdown.stop.prevent="$store.browserPage.startAnnotationSelection($event)"
184 + @pointermove.stop.prevent="$store.browserPage.moveAnnotationSelection($event)"
185 + @pointerup.stop.prevent="$store.browserPage.finishAnnotationSelection($event)"
186 + @pointercancel.stop.prevent="$store.browserPage.cancelAnnotationSelection($event)">
187 + <template x-for="annotation in $store.browserPage.visibleAnnotations()" :key="annotation.id">
188 + <div class="browser-annotation-box is-saved"
189 + :style="$store.browserPage.annotationBoxStyle(annotation.rect)">
190 + <span class="browser-annotation-number" x-text="annotation.index"></span>
191 + </div>
192 + </template>
193 + <template x-if="$store.browserPage.annotationDragRect">
194 + <div class="browser-annotation-box is-draft"
195 + :style="$store.browserPage.annotationBoxStyle($store.browserPage.annotationDragRect)">
196 + </div>
197 + </template>
198 + </div>
199 + </template>
200 + <template x-if="$store.browserPage.annotationDraft">
201 + <div class="browser-annotation-popover"
202 + :style="$store.browserPage.annotationPopoverStyle()"
203 + @click.stop @pointerdown.stop @keydown.stop>
204 + <div class="browser-annotation-popover-title">
205 + <span class="browser-annotation-number" x-text="$store.browserPage.nextAnnotationIndex()"></span>
206 + <span x-text="$store.browserPage.annotationDraftTitle()"></span>
207 + </div>
208 + <textarea x-model="$store.browserPage.annotationDraftText" placeholder="Comment"
209 + maxlength="1200"></textarea>
210 + <div class="browser-annotation-actions">
211 + <button type="button" class="btn btn-field"
212 + @click="$store.browserPage.cancelAnnotationDraft()">Cancel</button>
213 + <button type="button" class="btn btn-ok"
214 + :disabled="!String($store.browserPage.annotationDraftText || '').trim()"
215 + @click="$store.browserPage.addAnnotationComment()">Add</button>
216 + </div>
217 + </div>
218 + </template>
219 + <div class="browser-annotation-tray"
220 + x-show="$store.browserPage.visibleAnnotations().length"
221 + x-transition style="display: none;"
222 + @click.stop @pointerdown.stop @keydown.stop>
223 + <div class="browser-annotation-tray-header">
224 + <span>Annotations</span>
225 + <button type="button" class="browser-annotation-clear" title="Clear annotations"
226 + aria-label="Clear annotations" @click="$store.browserPage.clearVisibleAnnotations()">
227 + <span class="material-symbols-outlined">delete</span>
228 + </button>
229 + </div>
230 + <div class="browser-annotation-chips">
231 + <template x-for="annotation in $store.browserPage.visibleAnnotations()" :key="annotation.id">
232 + <div class="browser-annotation-chip">
233 + <span class="browser-annotation-number" x-text="annotation.index"></span>
234 + <span class="browser-annotation-chip-text" x-text="annotation.comment"></span>
235 + <button type="button" title="Remove annotation" aria-label="Remove annotation"
236 + @click="$store.browserPage.removeAnnotationComment(annotation.id)">
237 + <span class="material-symbols-outlined">close</span>
238 + </button>
239 + </div>
240 + </template>
241 + </div>
242 + <div class="browser-annotation-tray-actions">
243 + <button type="button" class="btn btn-field"
244 + @click="$store.browserPage.draftAnnotationsToChat()">Draft to chat</button>
245 + <button type="button" class="btn btn-ok"
246 + @click="$store.browserPage.sendAnnotationsToChat()">Send now</button>
247 + </div>
248 + </div>
249 <template x-if="!$store.browserPage.frameSrc && !$store.browserPage.isBusy()">
250 <div class="browser-empty">
251 <span class="material-symbols-outlined">captive_portal</span>
@@ -549,6 +628,34 @@
628 padding-bottom: 1px;
629 }
630
631 + .browser-annotate-toggle {
632 + display: inline-flex;
633 + align-items: center;
634 + justify-content: center;
635 + gap: 6px;
636 + min-width: 0;
637 + min-height: var(--browser-control-size);
638 + padding: 0 10px;
639 + border: 1px solid var(--browser-chrome-border);
640 + border-radius: var(--browser-control-radius);
641 + background: color-mix(in srgb, var(--color-background) 26%, transparent);
642 + color: color-mix(in srgb, var(--color-text) 74%, var(--color-primary) 26%);
643 + font-size: 0.82rem;
644 + font-weight: 650;
645 + white-space: nowrap;
646 + }
647 +
648 + .browser-annotate-toggle:hover:not(:disabled),
649 + .browser-annotate-toggle.is-active {
650 + border-color: color-mix(in srgb, var(--color-primary) 48%, var(--browser-chrome-border));
651 + background: color-mix(in srgb, var(--color-primary) 16%, var(--color-background));
652 + color: var(--color-text);
653 + }
654 +
655 + .browser-annotate-toggle .material-symbols-outlined {
656 + font-size: 1.05rem;
657 + }
658 +
659 .browser-session-controls .browser-extensions.is-active {
660 color: #2e7d32;
661 }
@@ -813,6 +920,10 @@
920 outline: none;
921 }
922
923 + .browser-stage.is-annotating {
924 + cursor: crosshair;
925 + }
926 +
927 .browser-frame {
928 display: block;
929 position: absolute;
@@ -827,6 +938,194 @@
938 background: #fff;
939 }
940
941 + .browser-annotation-layer {
942 + position: absolute;
943 + inset: 0;
944 + z-index: 12;
945 + cursor: crosshair;
946 + touch-action: none;
947 + background: rgba(37, 99, 235, 0.035);
948 + }
949 +
950 + .browser-annotation-layer.is-busy {
951 + cursor: progress;
952 + }
953 +
954 + .browser-annotation-box {
955 + position: absolute;
956 + box-sizing: border-box;
957 + min-width: 8px;
958 + min-height: 8px;
959 + border: 2px solid #3399ff;
960 + background: rgba(51, 153, 255, 0.16);
961 + box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.52), 0 8px 22px rgba(0, 0, 0, 0.18);
962 + pointer-events: none;
963 + }
964 +
965 + .browser-annotation-box.is-draft {
966 + border-style: dashed;
967 + background: rgba(51, 153, 255, 0.1);
968 + }
969 +
970 + .browser-annotation-number {
971 + display: inline-flex;
972 + align-items: center;
973 + justify-content: center;
974 + width: 22px;
975 + min-width: 22px;
976 + height: 22px;
977 + min-height: 22px;
978 + border-radius: 50%;
979 + background: #3399ff;
980 + color: #fff;
981 + font-size: 0.74rem;
982 + font-weight: 800;
983 + line-height: 1;
984 + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.28);
985 + }
986 +
987 + .browser-annotation-box .browser-annotation-number {
988 + position: absolute;
989 + top: -12px;
990 + left: -12px;
991 + }
992 +
993 + .browser-annotation-popover,
994 + .browser-annotation-tray {
995 + position: absolute;
996 + z-index: 18;
997 + border: 1px solid color-mix(in srgb, var(--color-border) 76%, transparent);
998 + border-radius: 7px;
999 + background: color-mix(in srgb, var(--color-background) 96%, #000 4%);
1000 + color: var(--color-text);
1001 + box-shadow: 0 16px 38px rgba(0, 0, 0, 0.28);
1002 + }
1003 +
1004 + .browser-annotation-popover {
1005 + display: flex;
1006 + flex-direction: column;
1007 + gap: 8px;
1008 + padding: 10px;
1009 + }
1010 +
1011 + .browser-annotation-popover-title,
1012 + .browser-annotation-tray-header {
1013 + display: flex;
1014 + align-items: center;
1015 + gap: 8px;
1016 + min-width: 0;
1017 + font-size: 0.82rem;
1018 + font-weight: 750;
1019 + }
1020 +
1021 + .browser-annotation-popover textarea {
1022 + width: 100%;
1023 + min-height: 82px;
1024 + max-height: 160px;
1025 + resize: vertical;
1026 + padding: 8px;
1027 + border: 1px solid color-mix(in srgb, var(--color-border) 74%, transparent);
1028 + border-radius: 6px;
1029 + background: var(--color-input);
1030 + color: var(--color-text);
1031 + font: inherit;
1032 + line-height: 1.35;
1033 + }
1034 +
1035 + .browser-annotation-actions,
1036 + .browser-annotation-tray-actions {
1037 + display: flex;
1038 + justify-content: flex-end;
1039 + gap: 7px;
1040 + }
1041 +
1042 + .browser-annotation-actions .btn,
1043 + .browser-annotation-tray-actions .btn {
1044 + min-height: 30px;
1045 + padding: 0 10px;
1046 + white-space: nowrap;
1047 + }
1048 +
1049 + .browser-annotation-tray {
1050 + right: 10px;
1051 + bottom: 10px;
1052 + display: flex;
1053 + flex-direction: column;
1054 + gap: 9px;
1055 + width: min(360px, calc(100% - 20px));
1056 + max-height: min(48%, 310px);
1057 + padding: 10px;
1058 + }
1059 +
1060 + .browser-annotation-tray-header {
1061 + justify-content: space-between;
1062 + }
1063 +
1064 + .browser-annotation-clear,
1065 + .browser-annotation-chip button {
1066 + display: inline-flex;
1067 + align-items: center;
1068 + justify-content: center;
1069 + width: 24px;
1070 + height: 24px;
1071 + min-width: 24px;
1072 + min-height: 24px;
1073 + padding: 0;
1074 + border: 0;
1075 + border-radius: 6px;
1076 + background: transparent;
1077 + color: color-mix(in srgb, var(--color-text) 64%, transparent);
1078 + cursor: pointer;
1079 + }
1080 +
1081 + .browser-annotation-clear:hover,
1082 + .browser-annotation-chip button:hover {
1083 + background: color-mix(in srgb, var(--color-panel) 82%, transparent);
1084 + color: var(--color-text);
1085 + }
1086 +
1087 + .browser-annotation-clear .material-symbols-outlined,
1088 + .browser-annotation-chip button .material-symbols-outlined {
1089 + font-size: 16px;
1090 + }
1091 +
1092 + .browser-annotation-chips {
1093 + display: flex;
1094 + flex-direction: column;
1095 + gap: 6px;
1096 + min-height: 0;
1097 + overflow: auto;
1098 + }
1099 +
1100 + .browser-annotation-chip {
1101 + display: grid;
1102 + grid-template-columns: auto minmax(0, 1fr) auto;
1103 + align-items: center;
1104 + gap: 7px;
1105 + min-height: 32px;
1106 + padding: 5px 6px;
1107 + border: 1px solid color-mix(in srgb, var(--color-border) 54%, transparent);
1108 + border-radius: 7px;
1109 + background: color-mix(in srgb, var(--color-panel) 74%, transparent);
1110 + }
1111 +
1112 + .browser-annotation-chip .browser-annotation-number {
1113 + width: 20px;
1114 + min-width: 20px;
1115 + height: 20px;
1116 + min-height: 20px;
1117 + font-size: 0.68rem;
1118 + }
1119 +
1120 + .browser-annotation-chip-text {
1121 + min-width: 0;
1122 + overflow: hidden;
1123 + text-overflow: ellipsis;
1124 + white-space: nowrap;
1125 + font-size: 0.78rem;
1126 + line-height: 1.25;
1127 + }
1128 +
1129 .browser-empty {
1130 display: flex;
1131 align-items: center;
@@ -928,6 +1227,16 @@
1227 height: var(--browser-control-size);
1228 }
1229
1230 + .browser-annotate-toggle {
1231 + width: var(--browser-control-size);
1232 + min-width: var(--browser-control-size);
1233 + padding: 0;
1234 + }
1235 +
1236 + .browser-annotate-toggle span:not(.material-symbols-outlined) {
1237 + display: none;
1238 + }
1239 +
1240 .browser-extension-dropdown {
1241 right: 0;
1242 left: auto;
plugins/_browser/webui/browser-store.js
+457 -14
@@ -13,6 +13,9 @@ const BROWSER_FIRST_INSTALL_TIMEOUT_MS = 300000;
13 const BROWSER_CONFIG_REFRESH_MS = 15000;
14 const VIEWPORT_SYNC_DEBOUNCE_MS = 220;
15 const VIEWPORT_SYNC_SIZE_TOLERANCE = 4;
16 +const ANNOTATION_DRAG_THRESHOLD = 6;
17 +const ANNOTATION_MAX_COMMENTS = 24;
18 +const ANNOTATION_DOM_LIMIT = 1200;
19
20 function makeViewerToken() {
21 return globalThis.crypto?.randomUUID?.()
@@ -60,6 +63,13 @@ const model = {
63 address: "",
64 frameSrc: "",
65 frameState: null,
66 + annotating: false,
67 + annotationComments: [],
68 + annotationDraft: null,
69 + annotationDraftText: "",
70 + annotationDragRect: null,
71 + annotationBusy: false,
72 + annotationError: "",
73 connected: false,
74 switchingBrowserId: null,
75 commandInFlight: false,
@@ -76,6 +86,8 @@ const model = {
86 _viewportSyncTimer: null,
87 _lastViewportKey: "",
88 _lastViewport: null,
89 + _annotationPointer: null,
90 + _annotationSequence: 0,
91 _mode: "",
92 _surfaceMounted: false,
93 _surfaceSwitching: false,
@@ -613,6 +625,7 @@ const model = {
625
626 async command(command, extra = {}) {
627 this.error = "";
628 + this.annotationError = "";
629 this.commandInFlight = true;
630 const previousActiveBrowserId = this.activeBrowserId;
631 try {
@@ -642,13 +655,17 @@ const model = {
655 this.frameState = null;
656 this.frameSrc = "";
657 }
645 - if (result.state?.currentUrl || result.currentUrl) {
646 - this.address = result.state?.currentUrl || result.currentUrl;
647 - }
648 - this.applySnapshot(data.snapshot);
649 - const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;
650 - if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
651 - await this.connectViewer({ browserId: this.activeBrowserId });
658 + if (result.state?.currentUrl || result.currentUrl) {
659 + this.address = result.state?.currentUrl || result.currentUrl;
660 + }
661 + this.applySnapshot(data.snapshot);
662 + if (["navigate", "back", "forward", "reload", "close"].includes(String(command || "").toLowerCase())) {
663 + this.clearAnnotationsForBrowser(previousActiveBrowserId);
664 + this.cancelAnnotationDraft();
665 + }
666 + const activeChanged = this.activeBrowserId && this.activeBrowserId !== previousActiveBrowserId;
667 + if ((command === "open" || command === "close" || activeChanged) && this.contextId && this.activeBrowserId) {
668 + await this.connectViewer({ browserId: this.activeBrowserId });
669 }
670 } catch (error) {
671 this.error = error instanceof Error ? error.message : String(error);
@@ -761,17 +778,22 @@ const model = {
778 return null;
779 },
780
764 - applyActiveFrameState(nextState = null) {
765 - if (!nextState) return;
766 - const stateId = this.normalizeBrowserId(nextState.id);
767 - if (stateId && this.activeBrowserId && !this.sameBrowserId(stateId, this.activeBrowserId)) {
768 - return;
781 + applyActiveFrameState(nextState = null) {
782 + if (!nextState) return;
783 + const stateId = this.normalizeBrowserId(nextState.id);
784 + if (stateId && this.activeBrowserId && !this.sameBrowserId(stateId, this.activeBrowserId)) {
785 + return;
786 }
787 + const previousUrl = String(this.frameState?.currentUrl || "");
788 + const nextUrl = String(nextState.currentUrl || "");
789 this.frameState = nextState;
790 + if (previousUrl && nextUrl && previousUrl !== nextUrl) {
791 + this.cancelAnnotationDraft();
792 + }
793 if (!this.addressFocused && nextState.currentUrl) {
794 this.address = nextState.currentUrl;
773 - }
774 - },
795 + }
796 + },
797
798 applySnapshot(snapshot = null) {
799 if (!snapshot?.image) return;
@@ -805,6 +827,7 @@ const model = {
827 if (this.activeBrowserId !== previous) {
828 this._lastViewportKey = "";
829 this._lastViewport = null;
830 + this.cancelAnnotationDraft();
831 }
832 },
833
@@ -849,6 +872,419 @@ const model = {
872 };
873 },
874
875 + handleKeydown(event) {
876 + const annotateShortcut = event?.key === "." && (event.metaKey || event.ctrlKey) && !event.altKey;
877 + if (annotateShortcut && this._surfaceMounted) {
878 + event.preventDefault();
879 + event.stopPropagation?.();
880 + this.toggleAnnotationMode();
881 + return;
882 + }
883 +
884 + if (this.annotating) {
885 + if (event?.key === "Escape") {
886 + event.preventDefault();
887 + if (this.annotationDraft || this.annotationDragRect) {
888 + this.cancelAnnotationDraft();
889 + } else {
890 + this.toggleAnnotationMode(false);
891 + }
892 + }
893 + return;
894 + }
895 +
896 + void this.sendKey(event);
897 + },
898 +
899 + handleStageWheel(event) {
900 + if (this.annotating) return;
901 + void this.sendWheel(event);
902 + },
903 +
904 + toggleAnnotationMode(force = null) {
905 + const nextValue = force === null ? !this.annotating : Boolean(force);
906 + if (nextValue && !this.canAnnotate()) return;
907 +
908 + this.annotating = nextValue;
909 + this.annotationError = "";
910 + this.closeExtensionsMenu();
911 + if (!nextValue) {
912 + this.cancelAnnotationDraft();
913 + this.annotationDragRect = null;
914 + this._annotationPointer = null;
915 + } else {
916 + this._stageElement?.focus?.({ preventScroll: true });
917 + }
918 + },
919 +
920 + canAnnotate() {
921 + return Boolean(this.activeBrowserId && this.frameSrc && !this.isBusy());
922 + },
923 +
924 + activeAnnotationUrl() {
925 + return String(this.frameState?.currentUrl || this.address || "about:blank");
926 + },
927 +
928 + visibleAnnotations() {
929 + const browserId = this.normalizeBrowserId(this.activeBrowserId);
930 + const url = this.activeAnnotationUrl();
931 + return this.annotationComments.filter((annotation) => (
932 + this.sameBrowserId(annotation.browserId, browserId)
933 + && String(annotation.url || "") === url
934 + ));
935 + },
936 +
937 + nextAnnotationIndex() {
938 + return this.visibleAnnotations().length + 1;
939 + },
940 +
941 + clearVisibleAnnotations() {
942 + this.clearAnnotationsForBrowser(this.activeBrowserId, this.activeAnnotationUrl());
943 + },
944 +
945 + clearAnnotationsForBrowser(browserId, url = null) {
946 + const numericBrowserId = this.normalizeBrowserId(browserId);
947 + if (!numericBrowserId) return;
948 + this.annotationComments = this.annotationComments.filter((annotation) => {
949 + if (!this.sameBrowserId(annotation.browserId, numericBrowserId)) return true;
950 + return url ? String(annotation.url || "") !== String(url) : false;
951 + });
952 + },
953 +
954 + annotationBoxStyle(rect = {}) {
955 + const viewport = this.currentViewportSize() || this._lastViewport || {};
956 + const width = Math.max(1, Number(viewport.width || rect.width || 1));
957 + const height = Math.max(1, Number(viewport.height || rect.height || 1));
958 + const normalized = this.clampAnnotationRect(rect);
959 + return [
960 + `left: ${(normalized.x / width) * 100}%`,
961 + `top: ${(normalized.y / height) * 100}%`,
962 + `width: ${(Math.max(1, normalized.width) / width) * 100}%`,
963 + `height: ${(Math.max(1, normalized.height) / height) * 100}%`,
964 + ].join("; ");
965 + },
966 +
967 + annotationPopoverStyle() {
968 + const rect = this.annotationDraft?.rect || this.annotationDragRect || {};
969 + const viewport = this.currentViewportSize() || this._lastViewport || {};
970 + const width = Math.max(1, Number(viewport.width || 1));
971 + const height = Math.max(1, Number(viewport.height || 1));
972 + const popoverWidth = Math.min(320, Math.max(240, width - 20));
973 + const popoverHeight = 190;
974 + const nextLeft = Math.min(
975 + Math.max(10, Number(rect.x || 0) + Number(rect.width || 0) + 10),
976 + Math.max(10, width - popoverWidth - 10),
977 + );
978 + const nextTop = Math.min(
979 + Math.max(10, Number(rect.y || 0) + Number(rect.height || 0) + 10),
980 + Math.max(10, height - popoverHeight - 10),
981 + );
982 + return [
983 + `left: ${(nextLeft / width) * 100}%`,
984 + `top: ${(nextTop / height) * 100}%`,
985 + `width: min(${popoverWidth}px, calc(100% - 20px))`,
986 + ].join("; ");
987 + },
988 +
989 + annotationDraftTitle() {
990 + if (!this.annotationDraft) return "Annotation";
991 + return this.annotationDraft.kind === "area" ? "Area annotation" : "Element annotation";
992 + },
993 +
994 + stagePointForEvent(event) {
995 + const image = this._stageElement?.querySelector?.(".browser-frame") || null;
996 + return this.pointerCoordinatesFor(event, image);
997 + },
998 +
999 + normalizeAnnotationRect(start = {}, end = {}) {
1000 + const x1 = Number(start.x || 0);
1001 + const y1 = Number(start.y || 0);
1002 + const x2 = Number(end.x || x1);
1003 + const y2 = Number(end.y || y1);
1004 + return this.clampAnnotationRect({
1005 + x: Math.min(x1, x2),
1006 + y: Math.min(y1, y2),
1007 + width: Math.abs(x2 - x1),
1008 + height: Math.abs(y2 - y1),
1009 + });
1010 + },
1011 +
1012 + clampAnnotationRect(rect = {}) {
1013 + const viewport = this.currentViewportSize() || this._lastViewport || {};
1014 + const viewportWidth = Math.max(1, Number(viewport.width || rect.x + rect.width || 1));
1015 + const viewportHeight = Math.max(1, Number(viewport.height || rect.y + rect.height || 1));
1016 + const x = Math.max(0, Math.min(viewportWidth, Number(rect.x || 0)));
1017 + const y = Math.max(0, Math.min(viewportHeight, Number(rect.y || 0)));
1018 + const width = Math.max(1, Math.min(viewportWidth - x, Number(rect.width || 1)));
1019 + const height = Math.max(1, Math.min(viewportHeight - y, Number(rect.height || 1)));
1020 + return {
1021 + x: Math.round(x),
1022 + y: Math.round(y),
1023 + width: Math.round(width),
1024 + height: Math.round(height),
1025 + };
1026 + },
1027 +
1028 + startAnnotationSelection(event) {
1029 + if (!this.annotating || this.annotationBusy || !this.canAnnotate()) return;
1030 + const point = this.stagePointForEvent(event);
1031 + if (!point) return;
1032 + this.cancelAnnotationDraft();
1033 + this.annotationError = "";
1034 + this._annotationPointer = {
1035 + id: event.pointerId,
1036 + start: point,
1037 + last: point,
1038 + };
1039 + this.annotationDragRect = this.clampAnnotationRect({
1040 + x: point.x,
1041 + y: point.y,
1042 + width: 1,
1043 + height: 1,
1044 + });
1045 + event.currentTarget?.setPointerCapture?.(event.pointerId);
1046 + },
1047 +
1048 + moveAnnotationSelection(event) {
1049 + if (!this.annotating || !this._annotationPointer) return;
1050 + if (event.pointerId !== this._annotationPointer.id) return;
1051 + const point = this.stagePointForEvent(event);
1052 + if (!point) return;
1053 + this._annotationPointer.last = point;
1054 + this.annotationDragRect = this.normalizeAnnotationRect(this._annotationPointer.start, point);
1055 + },
1056 +
1057 + async finishAnnotationSelection(event) {
1058 + if (!this.annotating || !this._annotationPointer) return;
1059 + if (event.pointerId !== this._annotationPointer.id) return;
1060 + const pointer = this._annotationPointer;
1061 + this._annotationPointer = null;
1062 + event.currentTarget?.releasePointerCapture?.(event.pointerId);
1063 + const endPoint = this.stagePointForEvent(event) || pointer.last || pointer.start;
1064 + const rect = this.normalizeAnnotationRect(pointer.start, endPoint);
1065 + this.annotationDragRect = null;
1066 + const isDrag = rect.width >= ANNOTATION_DRAG_THRESHOLD || rect.height >= ANNOTATION_DRAG_THRESHOLD;
1067 + const point = {
1068 + x: Math.round(endPoint.x),
1069 + y: Math.round(endPoint.y),
1070 + };
1071 + const payload = {
1072 + kind: isDrag ? "area" : "element",
1073 + point,
1074 + rect: isDrag ? rect : null,
1075 + viewport: this.currentViewportSize(),
1076 + url: this.activeAnnotationUrl(),
1077 + title: this.activeTitle,
1078 + };
1079 + await this.createAnnotationDraft(payload, isDrag ? rect : {
1080 + x: point.x - 10,
1081 + y: point.y - 10,
1082 + width: 20,
1083 + height: 20,
1084 + });
1085 + },
1086 +
1087 + cancelAnnotationSelection(event = null) {
1088 + if (event && this._annotationPointer?.id === event.pointerId) {
1089 + event.currentTarget?.releasePointerCapture?.(event.pointerId);
1090 + }
1091 + this._annotationPointer = null;
1092 + this.annotationDragRect = null;
1093 + },
1094 +
1095 + cancelAnnotationDraft() {
1096 + this.annotationDraft = null;
1097 + this.annotationDraftText = "";
1098 + this.annotationDragRect = null;
1099 + },
1100 +
1101 + async createAnnotationDraft(payload, fallbackRect) {
1102 + if (!this.activeBrowserId || !this.contextId) return;
1103 + const sequence = this._annotationSequence + 1;
1104 + const browserId = this.activeBrowserId;
1105 + const url = this.activeAnnotationUrl();
1106 + const title = this.activeTitle;
1107 + this._annotationSequence = sequence;
1108 + this.annotationBusy = true;
1109 + this.annotationError = "";
1110 + try {
1111 + const response = await websocket.request(
1112 + "browser_viewer_annotation",
1113 + {
1114 + context_id: this.contextId,
1115 + browser_id: browserId,
1116 + viewer_id: this._viewerToken,
1117 + payload,
1118 + },
1119 + { timeoutMs: 10000 },
1120 + );
1121 + if (sequence !== this._annotationSequence) return;
1122 + const data = firstOk(response);
1123 + const metadata = data.annotation || {};
1124 + this.annotationDraft = {
1125 + id: makeViewerToken(),
1126 + browserId,
1127 + url,
1128 + title,
1129 + kind: metadata.kind || payload.kind,
1130 + rect: this.annotationRectFromMetadata(metadata, fallbackRect),
1131 + metadata,
1132 + createdAt: Date.now(),
1133 + };
1134 + this.annotationDraftText = "";
1135 + } catch (error) {
1136 + this.annotationError = error instanceof Error ? error.message : String(error);
1137 + this.error = this.annotationError;
1138 + } finally {
1139 + if (sequence === this._annotationSequence) {
1140 + this.annotationBusy = false;
1141 + }
1142 + }
1143 + },
1144 +
1145 + annotationRectFromMetadata(metadata = {}, fallbackRect = {}) {
1146 + const targetRect = metadata?.target?.rect || metadata?.rect || null;
1147 + return this.clampAnnotationRect(targetRect || fallbackRect);
1148 + },
1149 +
1150 + addAnnotationComment() {
1151 + const comment = String(this.annotationDraftText || "").trim();
1152 + if (!this.annotationDraft || !comment) return;
1153 + if (this.visibleAnnotations().length >= ANNOTATION_MAX_COMMENTS) {
1154 + this.annotationError = `Keep each batch to ${ANNOTATION_MAX_COMMENTS} annotations or fewer.`;
1155 + this.error = this.annotationError;
1156 + return;
1157 + }
1158 + this.annotationComments = [
1159 + ...this.annotationComments,
1160 + {
1161 + ...this.annotationDraft,
1162 + comment,
1163 + index: this.nextAnnotationIndex(),
1164 + },
1165 + ];
1166 + this.cancelAnnotationDraft();
1167 + },
1168 +
1169 + removeAnnotationComment(annotationId) {
1170 + this.annotationComments = this.annotationComments.filter((annotation) => annotation.id !== annotationId);
1171 + },
1172 +
1173 + annotationChipLabel(annotation) {
1174 + const prefix = annotation?.kind === "area" ? "Area" : "Element";
1175 + return `${prefix} ${annotation?.index || ""}`.trim();
1176 + },
1177 +
1178 + formatAnnotationRect(rect = {}) {
1179 + const normalized = this.clampAnnotationRect(rect);
1180 + return `x=${normalized.x}, y=${normalized.y}, width=${normalized.width}, height=${normalized.height}`;
1181 + },
1182 +
1183 + redactAnnotationText(value) {
1184 + return String(value || "")
1185 + .replace(/(<input\b(?=[^>]*\btype=(["'])?password\2?)[^>]*?)\svalue=(["'])[\s\S]*?\3/giu, "$1 value=\"[redacted]\"")
1186 + .replace(/\b(password|passcode|token|secret|value)=((["'])[\s\S]{1,240}?\3)/giu, "$1=\"[redacted]\"");
1187 + },
1188 +
1189 + formatAnnotationMetadata(metadata = {}) {
1190 + const lines = [];
1191 + const target = metadata.target || {};
1192 + const selector = target.selector || metadata.selector || "";
1193 + const summary = target.summary || metadata.summary || "";
1194 + const dom = this.redactAnnotationText(target.dom || metadata.dom || "").slice(0, ANNOTATION_DOM_LIMIT);
1195 +
1196 + if (selector) {
1197 + lines.push(`Selector: ${selector}`);
1198 + }
1199 + if (target.tagName || target.role || target.id || target.name || target.classes) {
1200 + lines.push([
1201 + "Element:",
1202 + target.tagName ? `<${String(target.tagName).toLowerCase()}>` : "",
1203 + target.role ? `role=${target.role}` : "",
1204 + target.id ? `id=${target.id}` : "",
1205 + target.name ? `name=${target.name}` : "",
1206 + target.classes ? `class=${target.classes}` : "",
1207 + ].filter(Boolean).join(" "));
1208 + }
1209 + if (summary) {
1210 + lines.push(`Summary: ${summary}`);
1211 + }
1212 + if (Array.isArray(metadata.elements) && metadata.elements.length) {
1213 + lines.push("Intersecting elements:");
1214 + metadata.elements.slice(0, 8).forEach((element, index) => {
1215 + const elementLabel = [
1216 + `${index + 1}.`,
1217 + element.tagName ? `<${String(element.tagName).toLowerCase()}>` : "",
1218 + element.selector || "",
1219 + element.summary || "",
1220 + ].filter(Boolean).join(" ");
1221 + lines.push(elementLabel);
1222 + });
1223 + }
1224 + if (dom) {
1225 + lines.push(`DOM: ${dom}`);
1226 + }
1227 + return lines.join("\n");
1228 + },
1229 +
1230 + buildAnnotationsPrompt() {
1231 + const annotations = this.visibleAnnotations();
1232 + if (!annotations.length) return "";
1233 + const lines = [
1234 + "Browser annotations",
1235 + `Page title: ${this.activeTitle}`,
1236 + `Page URL: ${this.activeAnnotationUrl()}`,
1237 + `Browser id: ${this.activeBrowserId}`,
1238 + "",
1239 + ];
1240 + annotations.forEach((annotation, index) => {
1241 + lines.push(
1242 + `Annotation ${index + 1}`,
1243 + `Comment: ${annotation.comment}`,
1244 + `Selection kind: ${annotation.kind}`,
1245 + `Coordinates: ${this.formatAnnotationRect(annotation.rect)}`,
1246 + );
1247 + const metadata = this.formatAnnotationMetadata(annotation.metadata);
1248 + if (metadata) {
1249 + lines.push(metadata);
1250 + }
1251 + lines.push("");
1252 + });
1253 + return lines.join("\n").trim();
1254 + },
1255 +
1256 + draftAnnotationsToChat() {
1257 + const prompt = this.buildAnnotationsPrompt();
1258 + if (!prompt) return;
1259 + const existingMessage = String(chatInputStore.message || "").trim();
1260 + chatInputStore.message = existingMessage ? `${existingMessage}\n\n${prompt}` : prompt;
1261 + chatInputStore.adjustTextareaHeight?.();
1262 + chatInputStore.focus?.();
1263 + this.clearVisibleAnnotations();
1264 + this.toggleAnnotationMode(false);
1265 + },
1266 +
1267 + async sendAnnotationsToChat() {
1268 + const prompt = this.buildAnnotationsPrompt();
1269 + if (!prompt) return;
1270 + chatInputStore.message = prompt;
1271 + chatInputStore.adjustTextareaHeight?.();
1272 + try {
1273 + if (typeof chatInputStore.sendMessage === "function") {
1274 + await chatInputStore.sendMessage();
1275 + } else if (typeof globalThis.sendMessage === "function") {
1276 + await globalThis.sendMessage();
1277 + } else {
1278 + chatInputStore.focus?.();
1279 + return;
1280 + }
1281 + this.clearVisibleAnnotations();
1282 + this.toggleAnnotationMode(false);
1283 + } catch (error) {
1284 + this.error = error instanceof Error ? error.message : String(error);
1285 + }
1286 + },
1287 +
1288 currentViewportSize() {
1289 const stage = this._stageElement;
1290 if (!stage) return null;
@@ -910,6 +1346,7 @@ const model = {
1346 },
1347
1348 async sendMouse(eventType, event) {
1349 + if (this.annotating) return;
1350 if (!this.activeBrowserId || !event?.currentTarget) return;
1351 const pointer = this.pointerCoordinatesFor(event);
1352 if (!pointer) return;
@@ -960,6 +1397,7 @@ const model = {
1397 },
1398
1399 async sendKey(event) {
1400 + if (this.annotating) return;
1401 if (!this.activeBrowserId) return;
1402 if (event.ctrlKey || event.metaKey || event.altKey) return;
1403 const editable = ["INPUT", "TEXTAREA", "SELECT"].includes(event.target?.tagName);
@@ -982,6 +1420,11 @@ const model = {
1420 this._surfaceMounted = false;
1421 this._surfaceSwitching = false;
1422 this.commandInFlight = false;
1423 + this.annotating = false;
1424 + this.annotationBusy = false;
1425 + this.annotationError = "";
1426 + this.cancelAnnotationDraft();
1427 + this.cancelAnnotationSelection();
1428 if (this.contextId) {
1429 try {
1430 await websocket.emit("browser_viewer_unsubscribe", { context_id: this.contextId });
tests/test_browser_agent_regressions.py
+188 -16
@@ -2,7 +2,7 @@ import asyncio
2 import sys
3 import threading
4 from pathlib import Path
5 -from types import SimpleNamespace
5 +from types import ModuleType, SimpleNamespace
6
7 import pytest
8
@@ -11,6 +11,73 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
11 if str(PROJECT_ROOT) not in sys.path:
12 sys.path.insert(0, str(PROJECT_ROOT))
13
14 +
15 +class _TestAgentContext:
16 + @staticmethod
17 + def get(context_id):
18 + return None
19 +
20 +
21 +class _TestResponse(SimpleNamespace):
22 + def __init__(self, message="", break_loop=False, **kwargs):
23 + super().__init__(message=message, break_loop=break_loop, **kwargs)
24 +
25 +
26 +class _TestTool:
27 + def __init__(
28 + self,
29 + agent=None,
30 + name="",
31 + method=None,
32 + args=None,
33 + message="",
34 + loop_data=None,
35 + **kwargs,
36 + ):
37 + self.agent = agent
38 + self.name = name
39 + self.method = method
40 + self.args = args or {}
41 + self.message = message
42 + self.loop_data = loop_data
43 +
44 +
45 +class _TestWsHandler:
46 + def __init__(self, *args, **kwargs):
47 + self.emitted = []
48 +
49 + async def emit_to(self, sid, event, data, correlation_id=None):
50 + self.emitted.append((sid, event, data, correlation_id))
51 +
52 +
53 +class _TestWsResult(dict):
54 + @staticmethod
55 + def error(code="", message="", correlation_id=None):
56 + return _TestWsResult(
57 + {
58 + "ok": False,
59 + "code": code,
60 + "error": message,
61 + "correlation_id": correlation_id,
62 + }
63 + )
64 +
65 +
66 +sys.modules.setdefault("agent", SimpleNamespace(AgentContext=_TestAgentContext))
67 +sys.modules.setdefault("helpers.tool", SimpleNamespace(Response=_TestResponse, Tool=_TestTool))
68 +sys.modules.setdefault("helpers.ws", SimpleNamespace(WsHandler=_TestWsHandler))
69 +sys.modules.setdefault("helpers.ws_manager", SimpleNamespace(WsResult=_TestWsResult))
70 +_model_config_stub = ModuleType("plugins._model_config.helpers.model_config")
71 +_model_config_stub.get_presets = lambda: []
72 +_model_config_stub.get_preset_by_name = lambda name: None
73 +_model_config_stub.get_chat_model_config = lambda agent=None: {}
74 +sys.modules.setdefault("plugins._model_config.helpers.model_config", _model_config_stub)
75 +
76 +
77 +@pytest.fixture
78 +def anyio_backend():
79 + return "asyncio"
80 +
81 from plugins._browser.helpers.config import (
82 build_browser_launch_config,
83 get_browser_main_model_summary,
@@ -63,6 +130,8 @@ def test_browser_config_normalizes_extension_paths(tmp_path):
130
131 assert config == {
132 "extension_paths": [str(extension_dir)],
133 + "default_homepage": "about:blank",
134 + "autofocus_active_page": True,
135 "model_preset": "",
136 }
137
@@ -246,7 +315,7 @@ def test_browser_extension_manager_uses_modern_chrome_prodversion(monkeypatch):
315
316
317 def test_browser_extension_menu_exposes_agent_and_url_paths():
249 - html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
318 + html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
319 encoding="utf-8"
320 )
321 skill = PROJECT_ROOT / "skills" / "a0-browser-ext" / "SKILL.md"
@@ -279,7 +348,7 @@ def test_browser_viewer_allows_slow_extension_startup():
348
349
350 def test_browser_ui_spinners_have_browser_local_animation():
282 - main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
351 + main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
352 encoding="utf-8"
353 )
354 config_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "config.html").read_text(
@@ -305,7 +374,7 @@ def test_browser_extension_settings_stay_user_facing():
374
375
376 def test_browser_viewer_uses_tabs_for_session_switching():
308 - main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
377 + main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
378 encoding="utf-8"
379 )
380 browser_store = (
@@ -329,7 +398,7 @@ def test_browser_viewer_uses_cdp_screencast_transport():
398 ws_browser = (PROJECT_ROOT / "plugins" / "_browser" / "api" / "ws_browser.py").read_text(
399 encoding="utf-8"
400 )
332 - main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "main.html").read_text(
401 + main_html = (PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html").read_text(
402 encoding="utf-8"
403 )
404 runtime = (
@@ -339,12 +408,12 @@ def test_browser_viewer_uses_cdp_screencast_transport():
408 PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
409 ).read_text(encoding="utf-8")
410
342 - assert 'runtime.call("screenshot"' not in ws_browser
411 + assert 'runtime.call("screenshot"' in ws_browser
412 assert "SCREENCAST_QUALITY = 92" in ws_browser
413 assert "initial_viewport = self._viewport_from_data(data)" in ws_browser
414 assert '"set_viewport"' in ws_browser
415 assert "start_screencast" in ws_browser
347 - assert "read_screencast_frame" in ws_browser
416 + assert "pop_screencast_frame" in ws_browser
417 assert "stop_screencast" in ws_browser
418 assert '"Page.startScreencast"' in runtime
419 assert '"Page.screencastFrame"' in runtime
@@ -360,12 +429,54 @@ def test_browser_viewer_uses_cdp_screencast_transport():
429 assert "viewport_height: initialViewport?.height" in browser_store
430 assert "this.frameState = data.state || null" not in browser_store
431 assert "overflow: hidden;" in main_html
363 - assert "object-fit: fill;" not in main_html
364 - assert "height: auto;" in main_html
432 + assert "object-fit: fill;" in main_html
433 assert "image-rendering: auto;" in main_html
434
435
368 -@pytest.mark.asyncio
436 +def test_browser_annotate_mode_ui_and_prompt_hooks():
437 + panel_html = (
438 + PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-panel.html"
439 + ).read_text(encoding="utf-8")
440 + browser_store = (
441 + PROJECT_ROOT / "plugins" / "_browser" / "webui" / "browser-store.js"
442 + ).read_text(encoding="utf-8")
443 +
444 + assert "Annotate" in panel_html
445 + assert "Annotating" in panel_html
446 + assert "browser-annotation-layer" in panel_html
447 + assert "browser-annotation-tray" in panel_html
448 + assert "Draft to chat" in panel_html
449 + assert "Send now" in panel_html
450 + assert "@pointerdown.stop.prevent=\"$store.browserPage.startAnnotationSelection($event)\"" in panel_html
451 + assert "@keydown.window=\"$store.browserPage.handleKeydown($event)\"" in panel_html
452 + assert "annotationComments: []" in browser_store
453 + assert '"browser_viewer_annotation"' in browser_store
454 + assert 'event?.key === "." && (event.metaKey || event.ctrlKey)' in browser_store
455 + assert "Browser annotations" in browser_store
456 + assert "Comment:" in browser_store
457 + assert "Coordinates:" in browser_store
458 + assert "Selector:" in browser_store
459 + assert "DOM:" in browser_store
460 + assert "value=\\\"[redacted]\\\"" in browser_store
461 +
462 +
463 +def test_browser_runtime_and_content_helper_expose_annotation_target():
464 + runtime = (
465 + PROJECT_ROOT / "plugins" / "_browser" / "helpers" / "runtime.py"
466 + ).read_text(encoding="utf-8")
467 + helper = (
468 + PROJECT_ROOT / "plugins" / "_browser" / "assets" / "browser-page-content.js"
469 + ).read_text(encoding="utf-8")
470 +
471 + assert "async def annotation_target" in runtime
472 + assert "globalThis.__spaceBrowserPageContent__.annotate(payload || null)" in runtime
473 + assert "function annotate(payload = null)" in helper
474 + assert "annotate," in helper
475 + assert "sanitizeAnnotationDom" in helper
476 + assert "password" in helper
477 +
478 +
479 +@pytest.mark.anyio
480 async def test_browser_screencast_acknowledges_and_drops_stale_frames():
481 first_image = (
482 "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsL"
@@ -526,7 +637,7 @@ def test_browser_save_plugin_config_does_not_restart_runtimes_for_preset_only(mo
637 assert restarted == []
638
639
529 -@pytest.mark.asyncio
640 +@pytest.mark.anyio
641 async def test_browser_tool_dispatches_direct_actions(monkeypatch):
642 calls = []
643
@@ -558,7 +669,7 @@ async def test_browser_tool_dispatches_direct_actions(monkeypatch):
669 assert calls == [("content", (1, None))]
670
671
561 -@pytest.mark.asyncio
672 +@pytest.mark.anyio
673 async def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):
674 class FakeRuntime:
675 def __init__(self) -> None:
@@ -608,7 +719,7 @@ async def test_browser_viewer_subscribe_unregisters_stream(monkeypatch):
719 assert ("sid-1", "ctx") not in ws_browser_module.WsBrowser._streams
720
721
611 -@pytest.mark.asyncio
722 +@pytest.mark.anyio
723 async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
724 calls = []
725
@@ -642,11 +753,14 @@ async def test_browser_viewer_viewport_input_dispatches_resize(monkeypatch):
753 "sid-1",
754 )
755
645 - assert result == {"state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)}}
756 + assert result == {
757 + "state": {"ok": True, "method": "set_viewport", "args": (7, 1280, 720)},
758 + "snapshot": None,
759 + }
760 assert calls == [("set_viewport", (7, 1280, 720), {})]
761
762
649 -@pytest.mark.asyncio
763 +@pytest.mark.anyio
764 async def test_browser_viewer_wheel_input_dispatches_scroll(monkeypatch):
765 calls = []
766
@@ -682,10 +796,68 @@ async def test_browser_viewer_wheel_input_dispatches_scroll(monkeypatch):
796 "sid-1",
797 )
798
685 - assert result == {"state": {"ok": True, "method": "wheel", "args": (3, 320.0, 480.0, 0.0, 640.0)}}
799 + assert result == {
800 + "state": {"ok": True, "method": "wheel", "args": (3, 320.0, 480.0, 0.0, 640.0)},
801 + "snapshot": None,
802 + }
803 assert calls == [("wheel", (3, 320.0, 480.0, 0.0, 640.0), {})]
804
805
806 +@pytest.mark.anyio
807 +async def test_browser_viewer_annotation_dispatches_runtime(monkeypatch):
808 + calls = []
809 +
810 + class FakeRuntime:
811 + async def call(self, method, *args, **kwargs):
812 + calls.append((method, args, kwargs))
813 + return {
814 + "kind": "element",
815 + "point": {"x": 320, "y": 180},
816 + "target": {"tagName": "BUTTON", "selector": "#save"},
817 + }
818 +
819 + async def fake_get_runtime(context_id, create=True):
820 + assert context_id == "ctx"
821 + assert create is False
822 + return FakeRuntime()
823 +
824 + monkeypatch.setattr(ws_browser_module, "get_runtime", fake_get_runtime)
825 +
826 + handler = ws_browser_module.WsBrowser(
827 + SimpleNamespace(),
828 + threading.RLock(),
829 + manager=None,
830 + )
831 +
832 + payload = {
833 + "kind": "element",
834 + "point": {"x": 320, "y": 180},
835 + "viewport": {"width": 1280, "height": 720},
836 + }
837 + result = await handler.process(
838 + "browser_viewer_annotation",
839 + {
840 + "context_id": "ctx",
841 + "browser_id": 4,
842 + "viewer_id": "viewer-1",
843 + "payload": payload,
844 + },
845 + "sid-1",
846 + )
847 +
848 + assert result == {
849 + "annotation": {
850 + "kind": "element",
851 + "point": {"x": 320, "y": 180},
852 + "target": {"tagName": "BUTTON", "selector": "#save"},
853 + },
854 + "context_id": "ctx",
855 + "browser_id": 4,
856 + "viewer_id": "viewer-1",
857 + }
858 + assert calls == [("annotation_target", (4, payload), {})]
859 +
860 +
861 def test_browser_cleanup_extensions_follow_extensible_path_layout():
862 extension = __import__("helpers.extension", fromlist=["_get_extension_classes"])
863 remove_classes = extension._get_extension_classes( # type: ignore[attr-defined]
tests/test_webui_extension_surfaces.py
+16 -1
@@ -5,6 +5,7 @@ import tempfile
5 import threading
6 from contextlib import contextmanager
7 from pathlib import Path
8 +from types import SimpleNamespace
9 from typing import Iterator
10
11 import pytest
@@ -14,6 +15,15 @@ PROJECT_ROOT = Path(__file__).resolve().parents[1]
15 if str(PROJECT_ROOT) not in sys.path:
16 sys.path.insert(0, str(PROJECT_ROOT))
17
18 +
19 +class _TestAgentContext:
20 + @staticmethod
21 + def get(context_id):
22 + return None
23 +
24 +
25 +sys.modules.setdefault("agent", SimpleNamespace(AgentContext=_TestAgentContext))
26 +
27 from api.load_webui_extensions import LoadWebuiExtensions
28
29
@@ -69,6 +79,11 @@ def _new_handler() -> LoadWebuiExtensions:
79 return LoadWebuiExtensions(app, threading.RLock())
80
81
82 +@pytest.fixture
83 +def anyio_backend():
84 + return "asyncio"
85 +
86 +
87 def _assert_surface_anchor_in_template(surface: str, template_rel_path: str) -> None:
88 template_path = PROJECT_ROOT / template_rel_path
89 template_html = template_path.read_text(encoding="utf-8")
@@ -118,7 +133,7 @@ def _temporary_probe_plugin(surface: str) -> Iterator[tuple[str, str]]:
133 cache.clear("*(plugins)*")
134
135
121 -@pytest.mark.asyncio
136 +@pytest.mark.anyio
137 @pytest.mark.parametrize(
138 ("surface", "template_rel_path"),
139 SURFACE_SCENARIOS,