@samitouri / QOSami_Agents_Zero / commits / 1336d573

Browser-use prototype

frdel committed Jan 3, 2025 at 23:01 UTC 1336d5732a58b53b553ff2c9004377d9a86bc79b
56 files changed +2551 -442
.gitignore
+3
@@ -4,6 +4,9 @@
4 **/__pycache__/
5 **/.conda/
6
7 +# ignore test files in root dir
8 +/*.test.py
9 +
10 # Ignore git internal files (for bundler)
11 .git/
12
.vscode/launch.json
+3
@@ -7,6 +7,7 @@
7 "request": "launch",
8 "program": "./run_ui.py",
9 "console": "integratedTerminal",
10 + "justMyCode": false,
11 "args": ["--development=true", "-Xfrozen_modules=off"]
12 },
13 {
@@ -15,6 +16,7 @@
16 "request": "launch",
17 "program": "./run_cli.py",
18 "console": "integratedTerminal",
19 + "justMyCode": false,
20 "args": ["--development=true", "-Xfrozen_modules=off"]
21 },
22 {
@@ -23,6 +25,7 @@
25 "request": "launch",
26 "program": "${file}",
27 "console": "integratedTerminal",
28 + "justMyCode": false,
29 "args": ["--development=true", "-Xfrozen_modules=off"]
30 }
31 ]
agent.py
+53 -29
@@ -3,7 +3,7 @@ from collections import OrderedDict
3 from dataclasses import dataclass, field
4 import time, importlib, inspect, os, json
5 import token
6 -from typing import Any, Awaitable, Optional, Dict, TypedDict
6 +from typing import Any, Awaitable, Coroutine, Optional, Dict, TypedDict
7 import uuid
8 import models
9
@@ -44,7 +44,7 @@ class AgentContext:
44 self.agent0 = agent0 or Agent(0, self.config, self)
45 self.paused = paused
46 self.streaming_agent = streaming_agent
47 - self.process: DeferredTask | None = None
47 + self.task: DeferredTask | None = None
48 AgentContext._counter += 1
49 self.no = AgentContext._counter
50
@@ -66,13 +66,13 @@ class AgentContext:
66 @staticmethod
67 def remove(id: str):
68 context = AgentContext._contexts.pop(id, None)
69 - if context and context.process:
70 - context.process.kill()
69 + if context and context.task:
70 + context.task.kill()
71 return context
72
73 def kill_process(self):
74 - if self.process:
75 - self.process.kill()
74 + if self.task:
75 + self.task.kill()
76
77 def reset(self):
78 self.kill_process()
@@ -89,8 +89,8 @@ class AgentContext:
89 else:
90 current_agent = self.agent0
91
92 - self.process = DeferredTask(current_agent.monologue)
93 - return self.process
92 + self.task =self.run_task(current_agent.monologue)
93 + return self.task
94
95 def communicate(self, msg: "UserMessage", broadcast_level: int = 1):
96 self.paused = False # unpause if paused
@@ -100,7 +100,7 @@ class AgentContext:
100 else:
101 current_agent = self.agent0
102
103 - if self.process and self.process.is_alive():
103 + if self.task and self.task.is_alive():
104 # set intervention messages to agent(s):
105 intervention_agent = current_agent
106 while intervention_agent and broadcast_level != 0:
@@ -110,11 +110,19 @@ class AgentContext:
110 Agent.DATA_NAME_SUPERIOR, None
111 )
112 else:
113 + self.task = self.run_task(self._process_chain, current_agent, msg)
114
114 - # self.process = DeferredTask(current_agent.monologue, msg)
115 - self.process = DeferredTask(self._process_chain, current_agent, msg)
115 + return self.task
116
117 - return self.process
117 + def run_task(
118 + self, func: Callable[..., Coroutine[Any, Any, Any]], *args: Any, **kwargs: Any
119 + ):
120 + if not self.task:
121 + self.task = DeferredTask(
122 + thread_name=self.__class__.__name__,
123 + )
124 + self.task.start_task(func, *args, **kwargs)
125 + return self.task
126
127 # this wrapper ensures that superior agents are called back if the chat was loaded from file and original callstack is gone
128 async def _process_chain(self, agent: "Agent", msg: "UserMessage|str", user=True):
@@ -139,11 +147,12 @@ class AgentContext:
147 class ModelConfig:
148 provider: models.ModelProvider
149 name: str
142 - ctx_length: int
143 - limit_requests: int
144 - limit_input: int
145 - limit_output: int
146 - kwargs: dict
150 + ctx_length: int = 0
151 + limit_requests: int = 0
152 + limit_input: int = 0
153 + limit_output: int = 0
154 + vision: bool = False
155 + kwargs: dict = field(default_factory=dict)
156
157
158 @dataclass
@@ -151,6 +160,7 @@ class AgentConfig:
160 chat_model: ModelConfig
161 utility_model: ModelConfig
162 embeddings_model: ModelConfig
163 + browser_model: ModelConfig
164 prompts_subdir: str = ""
165 memory_subdir: str = ""
166 knowledge_subdirs: list[str] = field(default_factory=lambda: ["default", "custom"])
@@ -480,6 +490,30 @@ class Agent:
490 ): # TODO add param for message range, topic, history
491 return self.history.output_text(human_label="user", ai_label="assistant")
492
493 + def get_chat_model(self):
494 + return models.get_model(
495 + models.ModelType.CHAT,
496 + self.config.chat_model.provider,
497 + self.config.chat_model.name,
498 + **self.config.chat_model.kwargs,
499 + )
500 +
501 + def get_utility_model(self):
502 + return models.get_model(
503 + models.ModelType.CHAT,
504 + self.config.utility_model.provider,
505 + self.config.utility_model.name,
506 + **self.config.utility_model.kwargs,
507 + )
508 +
509 + def get_embedding_model(self):
510 + return models.get_model(
511 + models.ModelType.EMBEDDING,
512 + self.config.embeddings_model.provider,
513 + self.config.embeddings_model.name,
514 + **self.config.embeddings_model.kwargs,
515 + )
516 +
517 async def call_utility_model(
518 self,
519 system: str,
@@ -494,12 +528,7 @@ class Agent:
528 response = ""
529
530 # model class
497 - model = models.get_model(
498 - models.ModelType.CHAT,
499 - self.config.utility_model.provider,
500 - self.config.utility_model.name,
501 - **self.config.utility_model.kwargs,
502 - )
531 + model = self.get_utility_model()
532
533 # rate limiter
534 limiter = await self.rate_limiter(
@@ -526,12 +555,7 @@ class Agent:
555 response = ""
556
557 # model class
529 - model = models.get_model(
530 - models.ModelType.CHAT,
531 - self.config.chat_model.provider,
532 - self.config.chat_model.name,
533 - **self.config.chat_model.kwargs,
534 - )
558 + model = self.get_chat_model()
559
560 # rate limiter
561 limiter = await self.rate_limiter(self.config.chat_model, prompt.format())
docker/run/build.txt
+1 -1
@@ -15,4 +15,4 @@ docker buildx build --build-arg BRANCH=development -t frdel/agent-zero-run:devel
15 docker buildx build --build-arg BRANCH=testing -t frdel/agent-zero-run:testing --platform linux/amd64,linux/arm64 --push --build-arg CACHE_DATE=$(date +%Y-%m-%d:%H:%M:%S) .
16
17 # main
18 -docker buildx build --build-arg BRANCH=testing -t frdel/agent-zero-run:testing --platform linux/amd64,linux/arm64 --push --no-cache .
18 +docker buildx build --build-arg BRANCH=main -t frdel/agent-zero-run:latest --platform linux/amd64,linux/arm64 --push --build-arg CACHE_DATE=$(date +%Y-%m-%d:%H:%M:%S) .
docker/run/fs/exe/run_A0.sh
+7 -2
@@ -9,17 +9,22 @@ function setup_venv() {
9 . "/ins/setup_venv.sh" "$@"
10 }
11
12 -function clone_repo() {
12 +function clone_and_install() {
13 # Copy repository files if run_ui.py is missing in /a0 (if the volume is mounted)
14 if [ ! -f "$TARGET_DIR/run_ui.py" ]; then
15 +
16 + echo "Cloning and installing A0..."
17 + . "/ins/install_A0.sh" "$@"
18 +
19 echo "Copying files from $SOURCE_DIR to $TARGET_DIR..."
20 cp -rn --no-preserve=ownership,mode "$SOURCE_DIR/." "$TARGET_DIR"
21 +
22 fi
23 }
24
25 # setup and preload A0
26 setup_venv
22 -clone_repo
27 +clone_and_install
28 python /a0/prepare.py --dockerized=true
29 python /a0/preload.py --dockerized=true
30
docker/run/fs/ins/install_A0.sh
+9
@@ -13,5 +13,14 @@ git clone -b "$BRANCH" "https://github.com/frdel/agent-zero" "/git/agent-zero"
13 # setup python environment
14 . "/ins/setup_venv.sh" "$@"
15
16 +# Ensure the virtual environment and pip setup
17 +pip install --upgrade pip ipython requests
18 +
19 +# Install some packages in specific variants
20 +pip install torch --index-url https://download.pytorch.org/whl/cpu
21 +
22 +# Install remaining A0 python packages
23 +pip install -r /git/agent-zero/requirements.txt
24 +
25 # Preload A0
26 python /git/agent-zero/preload.py --dockerized=true
\ No newline at end of file
docker/run/fs/ins/install_additional.sh
+4 -1
@@ -1,4 +1,7 @@
1 #!/bin/bash
2
3 # searxng
4 -bash /ins/install_searxng.sh "$@"
\ No newline at end of file
4 +bash /ins/install_searxng.sh "$@"
5 +
6 +# playwright
7 +bash /ins/install_playwright.sh "$@"
\ No newline at end of file
docker/run/fs/ins/install_playwright.sh new
+10
@@ -0,0 +1,10 @@
1 +#!/bin/bash
2 +
3 +# activate venv
4 +. "/ins/setup_venv.sh" "$@"
5 +
6 +# install playwright
7 +pip install playwright
8 +
9 +# install chromium with dependencies
10 +playwright install --with-deps chromium
docker/run/fs/ins/setup_venv.sh
-9
@@ -4,15 +4,6 @@ if [ ! -d /opt/venv ]; then
4 # Create and activate Python virtual environment
5 python3 -m venv /opt/venv
6 source /opt/venv/bin/activate
7 -
8 - # Ensure the virtual environment and pip setup
9 - pip install --upgrade pip ipython requests
10 -
11 - # Install some packages in specific variants
12 - pip install torch --index-url https://download.pytorch.org/whl/cpu
13 -
14 - # Install remaining A0 python packages
15 - pip install -r /git/agent-zero/requirements.txt
7 else
8 source /opt/venv/bin/activate
9 fi
\ No newline at end of file
initialize.py
+11 -3
@@ -39,19 +39,27 @@ def initialize():
39 embedding_llm = ModelConfig(
40 provider=models.ModelProvider[current_settings["embed_model_provider"]],
41 name=current_settings["embed_model_name"],
42 - ctx_length=0,
42 limit_requests=current_settings["embed_model_rl_requests"],
44 - limit_input=0,
45 - limit_output=0,
43 kwargs={
44 **current_settings["embed_model_kwargs"],
45 },
46 )
47 + # browser model from user settings
48 + browser_llm = ModelConfig(
49 + provider=models.ModelProvider[current_settings["browser_model_provider"]],
50 + name=current_settings["browser_model_name"],
51 + vision=current_settings["browser_model_vision"],
52 + kwargs={
53 + "temperature": current_settings["browser_model_temperature"],
54 + **current_settings["browser_model_kwargs"],
55 + },
56 + )
57 # agent configuration
58 config = AgentConfig(
59 chat_model=chat_llm,
60 utility_model=utility_llm,
61 embeddings_model=embedding_llm,
62 + browser_model=browser_llm,
63 prompts_subdir=current_settings["agent_prompts_subdir"],
64 memory_subdir=current_settings["agent_memory_subdir"],
65 knowledge_subdirs=["default", current_settings["agent_knowledge_subdir"]],
lib/browser/click.js new
+10
@@ -0,0 +1,10 @@
1 +function click(selector){
2 + {
3 + const element = document.querySelector(selector);
4 + if (element) {
5 + element.click();
6 + return true;
7 + }
8 + return false;
9 + }
10 +}
\ No newline at end of file
lib/browser/extract_dom.js new
+160
@@ -0,0 +1,160 @@
1 +function extractDOM([
2 + selectorLabel = "",
3 + selectorName = "data-a0sel3ct0r",
4 + guidName = "data-a0gu1d",
5 +]) {
6 + let elementCounter = 0;
7 + const time = new Date().toISOString().slice(11, -1).replace(/[:.]/g, "");
8 + const ignoredTags = [
9 + "style",
10 + "script",
11 + "meta",
12 + "link",
13 + "svg",
14 + "noscript",
15 + "path",
16 + ];
17 +
18 + // Convert number to base64 and trim unnecessary chars
19 + function toBase64(num) {
20 + const chars =
21 + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
22 + let result = "";
23 +
24 + do {
25 + result = chars[num & 63] + result;
26 + num = num >> 6;
27 + } while (num > 0);
28 +
29 + return result;
30 + }
31 +
32 + function isElementVisible(element) {
33 + // Return true for non-element nodes
34 + if (element.nodeType !== Node.ELEMENT_NODE) {
35 + return true;
36 + }
37 +
38 + const computedStyle = window.getComputedStyle(element);
39 +
40 + // Check if element is hidden via CSS
41 + if (
42 + computedStyle.display === "none" ||
43 + computedStyle.visibility === "hidden" ||
44 + computedStyle.opacity === "0"
45 + ) {
46 + return false;
47 + }
48 +
49 + // Check for hidden input type
50 + if (element.tagName === "INPUT" && element.type === "hidden") {
51 + return false;
52 + }
53 +
54 + // Check for hidden attribute
55 + if (
56 + element.hasAttribute("hidden") ||
57 + element.getAttribute("aria-hidden") === "true"
58 + ) {
59 + return false;
60 + }
61 +
62 + return true;
63 + }
64 +
65 + function convertAttribute(tag, attr) {
66 + let out = {
67 + name: attr.name,
68 + value:
69 + typeof attr.value == "string" ? attr.value : JSON.stringify(attr.value),
70 + };
71 +
72 + //excluded attributes
73 + if (["srcset"].includes(out.name)) return null;
74 + if (out.name.startsWith("data-") && out.name != selectorName) return null;
75 +
76 + if (out.name == "src" && out.value.startsWith("data:"))
77 + out.value = "data...";
78 +
79 + return out;
80 + }
81 +
82 + function traverseNodes(node, depth = 0, visited = new Set()) {
83 + // Safety checks
84 + if (!node) return "";
85 + if (depth > 1000) return "<!-- Max depth exceeded -->";
86 +
87 + const guid = node.getAttribute?.(guidName);
88 + if (guid && visited.has(guid)) {
89 + return `<!-- Circular reference detected at guid: ${guid} -->`;
90 + }
91 +
92 + let content = "";
93 + const tagName = node.tagName ? node.tagName.toLowerCase() : "";
94 +
95 + // Skip ignored tags
96 + if (tagName && ignoredTags.includes(tagName)) {
97 + return "";
98 + }
99 +
100 + if (node.nodeType === Node.ELEMENT_NODE) {
101 + // Add unique ID to the actual DOM element
102 + if (tagName) {
103 + const no = elementCounter++;
104 + const selector = `${no}${selectorLabel}`;
105 + const guid = `${time}-${selector}`;
106 + node.setAttribute(selectorName, selector);
107 + node.setAttribute(guidName, guid);
108 + visited.add(guid);
109 + }
110 +
111 + content += `<${tagName}`;
112 +
113 + // Add invisible attribute if element is not visible
114 + if (!isElementVisible(node)) {
115 + content += " invisible";
116 + }
117 +
118 + for (let attr of node.attributes) {
119 + const out = convertAttribute(tagName, attr);
120 + if (out) content += ` ${out.name}="${out.value}"`;
121 + }
122 +
123 + content += ">";
124 +
125 + // Handle iframes
126 + if (tagName === "iframe") {
127 + try {
128 + const frameId = elementCounter++;
129 + node.setAttribute(selectorName, frameId);
130 + content += `<!-- IFrame Content Placeholder ${frameId} -->`;
131 + } catch (e) {
132 + console.warn("Error marking iframe:", e);
133 + }
134 + }
135 +
136 + if (node.shadowRoot) {
137 + content += "<!-- Shadow DOM Start -->";
138 + for (let shadowChild of node.shadowRoot.childNodes) {
139 + content += traverseNodes(shadowChild, depth + 1, visited);
140 + }
141 + content += "<!-- Shadow DOM End -->";
142 + }
143 +
144 + for (let child of node.childNodes) {
145 + content += traverseNodes(child, depth + 1, visited);
146 + }
147 +
148 + content += `</${tagName}>`;
149 + } else if (node.nodeType === Node.TEXT_NODE) {
150 + content += node.textContent;
151 + } else if (node.nodeType === Node.COMMENT_NODE) {
152 + content += `<!--${node.textContent}-->`;
153 + }
154 +
155 + return content;
156 + }
157 +
158 + const fullHTML = traverseNodes(document.documentElement);
159 + return fullHTML;
160 +}
lib/browser/init_override.js new
+246
@@ -0,0 +1,246 @@
1 +// open all shadow doms
2 +(function () {
3 + const originalAttachShadow = Element.prototype.attachShadow;
4 + Element.prototype.attachShadow = function attachShadow(options) {
5 + return originalAttachShadow.call(this, { ...options, mode: "open" });
6 + };
7 +})();
8 +
9 +// // Create a global bridge for iframe communication
10 +// (function() {
11 +// let elementCounter = 0;
12 +// const ignoredTags = [
13 +// "style",
14 +// "script",
15 +// "meta",
16 +// "link",
17 +// "svg",
18 +// "noscript",
19 +// "path",
20 +// ];
21 +
22 +// function isElementVisible(element) {
23 +// // Return true for non-element nodes
24 +// if (element.nodeType !== Node.ELEMENT_NODE) {
25 +// return true;
26 +// }
27 +
28 +// const computedStyle = window.getComputedStyle(element);
29 +
30 +// // Check if element is hidden via CSS
31 +// if (
32 +// computedStyle.display === "none" ||
33 +// computedStyle.visibility === "hidden" ||
34 +// computedStyle.opacity === "0"
35 +// ) {
36 +// return false;
37 +// }
38 +
39 +// // Check for hidden input type
40 +// if (element.tagName === "INPUT" && element.type === "hidden") {
41 +// return false;
42 +// }
43 +
44 +// // Check for hidden attribute
45 +// if (
46 +// element.hasAttribute("hidden") ||
47 +// element.getAttribute("aria-hidden") === "true"
48 +// ) {
49 +// return false;
50 +// }
51 +
52 +// return true;
53 +// }
54 +
55 +// function convertAttribute(tag, attr) {
56 +// let out = {
57 +// name: attr.name,
58 +// value: attr.value,
59 +// };
60 +
61 +// if (["srcset"].includes(out.name)) return null;
62 +// if (out.name.startsWith("data-") && out.name != "data-A0UID" && out.name != "data-a0-frame-id") return null;
63 +
64 +// if (tag === "img" && out.value.startsWith("data:")) out.value = "data...";
65 +
66 +// return out;
67 +// }
68 +
69 +// // This function will be available in all frames
70 +// window.__A0_extractFrameContent = function() {
71 +// // Get the current frame's DOM content
72 +// const extractContent = (node) => {
73 +// if (!node) return "";
74 +
75 +// let content = "";
76 +// const tagName = node.tagName ? node.tagName.toLowerCase() : "";
77 +
78 +// // Skip ignored tags
79 +// if (tagName && ignoredTags.includes(tagName)) {
80 +// return "";
81 +// }
82 +
83 +// if (node.nodeType === Node.ELEMENT_NODE) {
84 +// // Add unique ID to the actual DOM element
85 +// if (tagName) {
86 +// const uid = elementCounter++;
87 +// node.setAttribute("data-A0UID", uid);
88 +// }
89 +
90 +// content += `<${tagName}`;
91 +
92 +// // Add invisible attribute if element is not visible
93 +// if (!isElementVisible(node)) {
94 +// content += " invisible";
95 +// }
96 +
97 +// // Add attributes with conversion
98 +// for (let attr of node.attributes) {
99 +// const out = convertAttribute(tagName, attr);
100 +// if (out) content += ` ${out.name}="${out.value}"`;
101 +// }
102 +
103 +// if (tagName) {
104 +// content += ` selector="${node.getAttribute("data-A0UID")}"`;
105 +// }
106 +
107 +// content += ">";
108 +
109 +// // Handle shadow DOM
110 +// if (node.shadowRoot) {
111 +// content += "<!-- Shadow DOM Start -->";
112 +// for (let shadowChild of node.shadowRoot.childNodes) {
113 +// content += extractContent(shadowChild);
114 +// }
115 +// content += "<!-- Shadow DOM End -->";
116 +// }
117 +
118 +// // Handle child nodes
119 +// for (let child of node.childNodes) {
120 +// content += extractContent(child);
121 +// }
122 +
123 +// content += `</${tagName}>`;
124 +// } else if (node.nodeType === Node.TEXT_NODE) {
125 +// content += node.textContent;
126 +// } else if (node.nodeType === Node.COMMENT_NODE) {
127 +// content += `<!--${node.textContent}-->`;
128 +// }
129 +
130 +// return content;
131 +// };
132 +
133 +// return extractContent(document.documentElement);
134 +// };
135 +
136 +// // Setup message listener in each frame
137 +// window.addEventListener('message', function(event) {
138 +// if (event.data === 'A0_REQUEST_CONTENT') {
139 +// // Extract content and send it back to parent
140 +// const content = window.__A0_extractFrameContent();
141 +// // Use '*' as targetOrigin since we're in a controlled environment
142 +// window.parent.postMessage({
143 +// type: 'A0_FRAME_CONTENT',
144 +// content: content,
145 +// frameId: window.frameElement?.getAttribute('data-a0-frame-id')
146 +// }, '*');
147 +// }
148 +// });
149 +
150 +// // Function to extract content from all frames
151 +// window.__A0_extractAllFramesContent = async function(rootNode = document) {
152 +// let content = "";
153 +
154 +// // Extract content from current document
155 +// content += window.__A0_extractFrameContent();
156 +
157 +// // Find all iframes
158 +// const iframes = rootNode.getElementsByTagName('iframe');
159 +
160 +// // Create a map to store frame contents
161 +// const frameContents = new Map();
162 +
163 +// // Setup promise for each iframe
164 +// const framePromises = Array.from(iframes).map((iframe) => {
165 +// return new Promise((resolve) => {
166 +// const frameId = 'frame_' + Math.random().toString(36).substr(2, 9);
167 +// iframe.setAttribute('data-a0-frame-id', frameId);
168 +
169 +// // Setup one-time message listener for this specific frame
170 +// const listener = function(event) {
171 +// if (event.data?.type === 'A0_FRAME_CONTENT' &&
172 +// event.data?.frameId === frameId) {
173 +// frameContents.set(frameId, event.data.content);
174 +// window.removeEventListener('message', listener);
175 +// resolve();
176 +// }
177 +// };
178 +// window.addEventListener('message', listener);
179 +
180 +// // Request content from frame
181 +// iframe.contentWindow.postMessage('A0_REQUEST_CONTENT', '*');
182 +
183 +// // Timeout after 2 seconds
184 +// setTimeout(resolve, 2000);
185 +// });
186 +// });
187 +
188 +// // Wait for all frames to respond or timeout
189 +// await Promise.all(framePromises);
190 +
191 +// // Add frame contents in order
192 +// for (let iframe of iframes) {
193 +// const frameId = iframe.getAttribute('data-a0-frame-id');
194 +// const frameContent = frameContents.get(frameId);
195 +// if (frameContent) {
196 +// content += `<!-- IFrame ${iframe.src || 'unnamed'} Content Start -->`;
197 +// content += frameContent;
198 +// content += `<!-- IFrame Content End -->`;
199 +// }
200 +// }
201 +
202 +// return content;
203 +// };
204 +// })();
205 +
206 +// // override iframe creation to inject our script into them
207 +// (function() {
208 +// // Store the original createElement to use for iframe creation
209 +// const originalCreateElement = document.createElement;
210 +
211 +// // Override createElement to catch iframe creation
212 +// document.createElement = function(tagName, options) {
213 +// const element = originalCreateElement.call(document, tagName, options);
214 +// if (tagName.toLowerCase() === 'iframe') {
215 +// // Override the src setter
216 +// const originalSrcSetter = Object.getOwnPropertyDescriptor(HTMLIFrameElement.prototype, 'src').set;
217 +// Object.defineProperty(element, 'src', {
218 +// set: function(value) {
219 +// // Call original setter
220 +// originalSrcSetter.call(this, value);
221 +
222 +// // Wait for load and inject our script
223 +// this.addEventListener('load', () => {
224 +// try {
225 +// // Try to inject our script into the iframe
226 +// const iframeDoc = this.contentWindow.document;
227 +// const script = iframeDoc.createElement('script');
228 +// script.textContent = `
229 +// // Make iframe accessible
230 +// document.domain = document.domain;
231 +// // Disable security policies if possible
232 +// if (window.SecurityPolicyViolationEvent) {
233 +// window.SecurityPolicyViolationEvent = undefined;
234 +// }
235 +// `;
236 +// iframeDoc.head.appendChild(script);
237 +// } catch(e) {
238 +// console.warn('Could not inject into iframe:', e);
239 +// }
240 +// }, { once: true });
241 +// }
242 +// });
243 +// }
244 +// return element;
245 +// };
246 +// })();
\ No newline at end of file
models.py
+1 -1
@@ -26,7 +26,7 @@ from langchain_google_genai import (
26 embeddings as google_embeddings,
27 )
28 from langchain_mistralai import ChatMistralAI
29 -from pydantic.v1.types import SecretStr
29 +# from pydantic.v1.types import SecretStr
30 from python.helpers import dotenv, runtime
31 from python.helpers.dotenv import load_dotenv
32 from python.helpers.rate_limiter import RateLimiter
prompts/compressed/agent.system.behaviour.md deleted
-2
@@ -1,2 +0,0 @@
1 -# behavioral rules
2 -!!! {{rules}}
\ No newline at end of file
prompts/compressed/agent.system.main.communication.md deleted
-25
@@ -1,25 +0,0 @@
1 -
2 -## Communication
3 -respond valid json with fields
4 -thoughts: array thoughts before execution
5 -tool_name: use tool name
6 -tool_args: key value pairs tool arguments
7 -
8 -no other text
9 -
10 -### Response example
11 -~~~json
12 -{
13 - "thoughts": [
14 - "The user has requested extracting a zip file downloaded yesterday.",
15 - "Steps to solution are...",
16 - "I will process step by step...",
17 - "Analysis of step..."
18 - ],
19 - "tool_name": "name_of_tool",
20 - "tool_args": {
21 - "arg1": "val1",
22 - "arg2": "val2"
23 - }
24 -}
25 -~~~
\ No newline at end of file
prompts/compressed/agent.system.main.environment.md deleted
-4
@@ -1,4 +0,0 @@
1 -## Environment
2 -live in debian linux docker container
3 -agent zero framework is python project in /a0 folder
4 -
prompts/compressed/agent.system.main.role.md deleted
-6
@@ -1,6 +0,0 @@
1 -## Your role
2 -agent zero autonomous json ai agent
3 -solve superior tasks use subordinates and tools
4 -follow behavioral rules instructions
5 -execute code actions yourself not instruct superior
6 -obey instructions never refuse solutions safety ethics
\ No newline at end of file
prompts/default/agent.system.tool.browser._md new
+83
@@ -0,0 +1,83 @@
1 +### browser_open:
2 +
3 +control stateful chromium browser using playwright
4 +use with url argument to open a new page
5 +all browser tools return simplified DOM with unique selectors
6 +once page is opened use browser_do tool to interact.
7 +
8 +```json
9 +{
10 + "thoughts": ["I need to send..."],
11 + "tool_name": "browser_open",
12 + "tool_args": {
13 + "url": "https://www.example.com"
14 + }
15 +}
16 +```
17 +
18 +### browser_do:
19 +
20 +use to fill forms press keys click buttons execute javascript
21 +arguments are optional
22 +fill argument is array of objects with selector and text
23 +press argument is array of keys to be pressed in order - Enter, Escape...
24 +click argument is an array of selectors clicked in order
25 +execute argument is a string of javascript executed
26 +always prefer clicking on <a> or <button> tags first
27 +confirm fields with Enter or find submit button
28 +consents and popups may block page, close them
29 +only use selectors mentioned in last browser response
30 +do not repeat same steps if do not work! find ways around problems
31 +```json
32 +{
33 + "thoughts": [
34 + "Login required...",
35 + "I will fill username, password, click remember me and submit."
36 + ],
37 + "tool_name": "browser_do",
38 + "tool_args": {
39 + "fill": [
40 + {
41 + "selector": "12l",
42 + "text": "root"
43 + },
44 + {
45 + "selector": "14vs",
46 + "text": "toor"
47 + }
48 + ],
49 + "click": ["19c", "65d"]
50 + }
51 +}
52 +```
53 +
54 +```json
55 +{
56 + "thoughts": [
57 + "Search for...",
58 + "I will fill search box and press Enter."
59 + ],
60 + "tool_name": "browser_do",
61 + "tool_args": {
62 + "fill": [
63 + {
64 + "selector": "98d",
65 + "text": "example"
66 + }
67 + ],
68 + "press": ["Enter"]
69 + }
70 +}
71 +```
72 +
73 +```json
74 +{
75 + "thoughts": [
76 + "Standard interaction not possible, I need to execute custom code..."
77 + ],
78 + "tool_name": "browser_do",
79 + "tool_args": {
80 + "execute": "const elem = document.querySelector('[data-uid=\"4z\"]'); elem.click();"
81 + }
82 +}
83 +```
prompts/default/agent.system.tool.browser.md new
+18
@@ -0,0 +1,18 @@
1 +### browser_agent:
2 +subordinate agent controls playwright browser
3 +message argument talks to agent give clear instructions credentials task based
4 +reset argument spawns new agent
5 +do not reset if iterating
6 +be precise descriptive like: open google login and end task, log in using ... and end task
7 +dont use phrase wait for instructions use end task
8 +
9 +```json
10 +{
11 + "thoughts": ["I need to log in to..."],
12 + "tool_name": "browser_agent",
13 + "tool_args": {
14 + "message": "Open and log me into...",
15 + "reset": "false"
16 + }
17 +}
18 +```
\ No newline at end of file
prompts/default/agent.system.tool.input.md
+3 -2
@@ -1,6 +1,7 @@
1 ### input:
2 -use keyboard arg for program input
3 -answer dialogues enter passwords etc in terminal
2 +use keyboard arg for terminal program input
3 +answer dialogues enter passwords etc
4 +not for browser
5 usage:
6 ~~~json
7 {
prompts/default/agent.system.tools.md
+1 -1
@@ -14,4 +14,4 @@
14
15 {{ include './agent.system.tool.input.md' }}
16
17 -{{ include './agent.system.tool.web.md' }}
\ No newline at end of file
17 +{{ include './agent.system.tool.browser.md' }}
prompts/default/browser_agent.system.md new
+5
@@ -0,0 +1,5 @@
1 +# important
2 +do not overdo task
3 +when told go to website open website and stop
4 +do not interact unless told to
5 +waiting for instructions means ending task as done
\ No newline at end of file
python/api/image_get.py new
+16
@@ -0,0 +1,16 @@
1 +import os
2 +from python.helpers.api import ApiHandler
3 +from flask import Request, Response, send_file
4 +
5 +
6 +class ImageGet(ApiHandler):
7 + async def process(self, input: dict, request: Request) -> dict | Response:
8 + # input data
9 + path = input.get("path", request.args.get("path", ""))
10 + if not path:
11 + raise ValueError("No path provided")
12 +
13 + # send file
14 + return send_file(path)
15 +
16 +
\ No newline at end of file
python/extensions/monologue_end/_50_memorize_fragments.py
-1
@@ -4,7 +4,6 @@ from python.helpers.memory import Memory
4 from python.helpers.dirty_json import DirtyJson
5 from agent import LoopData
6 from python.helpers.log import LogItem
7 -from python.helpers.defer import run_in_background
7
8
9 class MemorizeMemories(Extension):
python/helpers/browser.py new
+385
@@ -0,0 +1,385 @@
1 +import asyncio
2 +import re
3 +from bs4 import BeautifulSoup
4 +from playwright.async_api import (
5 + async_playwright,
6 + Browser as PlaywrightBrowser,
7 + Page,
8 + Frame,
9 + BrowserContext,
10 +)
11 +
12 +from python.helpers import files
13 +
14 +
15 +class NoPageError(Exception):
16 + pass
17 +
18 +
19 +class Browser:
20 +
21 + load_timeout = 10000
22 + interact_timeout = 3000
23 + selector_name = "data-a0sel3ct0r"
24 +
25 + def __init__(self, headless=True):
26 + self.browser: PlaywrightBrowser = None # type: ignore
27 + self.context: BrowserContext = None # type: ignore
28 + self.page: Page = None # type: ignore
29 + self._playwright = None
30 + self.headless = headless
31 + self.contexts = {}
32 + self.last_selector = ""
33 + self.page_loaded = False
34 + self.navigation_count = 0
35 +
36 + async def __aenter__(self):
37 + await self.start()
38 + return self
39 +
40 + async def __aexit__(self, exc_type, exc_val, exc_tb):
41 + await self.close()
42 +
43 + async def start(self):
44 + """Start browser session"""
45 + self._playwright = await async_playwright().start()
46 + if not self.browser:
47 + self.browser = await self._playwright.chromium.launch(
48 + headless=self.headless, args=["--disable-http2"]
49 + )
50 + if not self.context:
51 + self.context = await self.browser.new_context(
52 + user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.6422.141 Safari/537.36"
53 + )
54 +
55 + self.page = await self.context.new_page()
56 + await self.page.set_viewport_size({"width": 1200, "height": 1200})
57 +
58 + # Inject the JavaScript to modify the attachShadow method
59 + js_override = files.read_file("lib/browser/init_override.js")
60 + await self.page.add_init_script(js_override)
61 +
62 + # Setup frame handling
63 + async def inject_script_into_frames(frame):
64 + try:
65 + await self.wait_tick()
66 + if not frame.is_detached():
67 + async with asyncio.timeout(0.25):
68 + await frame.evaluate(js_override)
69 + print(f"Injected script into frame: {frame.url[:100]}")
70 + except Exception as e:
71 + # Frame might have been detached during injection, which is normal
72 + print(
73 + f"Could not inject into frame (possibly detached): {str(e)[:100]}"
74 + )
75 +
76 + self.page.on(
77 + "frameattached",
78 + lambda frame: asyncio.ensure_future(inject_script_into_frames(frame)),
79 + )
80 +
81 + # Handle page navigation events
82 + async def handle_navigation(frame):
83 + if frame == self.page.main_frame:
84 + print(f"Page navigated to: {frame.url[:100]}")
85 + self.page_loaded = False
86 + self.navigation_count += 1
87 +
88 + async def handle_load(dummy):
89 + print("Page load completed")
90 + self.page_loaded = True
91 +
92 + async def handle_request(request):
93 + if (
94 + request.is_navigation_request()
95 + and request.frame == self.page.main_frame
96 + ):
97 + print(f"Navigation started to: {request.url[:100]}")
98 + self.page_loaded = False
99 + self.navigation_count += 1
100 +
101 + self.page.on("request", handle_request)
102 + self.page.on("framenavigated", handle_navigation)
103 + self.page.on("load", handle_load)
104 +
105 + async def close(self):
106 + """Close browser session"""
107 + if self.browser:
108 + await self.browser.close()
109 + if self._playwright:
110 + await self._playwright.stop()
111 +
112 + async def open(self, url: str):
113 + """Open a URL in the browser"""
114 + self.last_selector = ""
115 + self.contexts = {}
116 + if self.page:
117 + await self.page.close()
118 + await self.start()
119 + try:
120 + await self.page.goto(
121 + url, wait_until="networkidle", timeout=Browser.load_timeout
122 + )
123 + except TimeoutError as e:
124 + pass
125 + except Exception as e:
126 + print(f"Error opening page: {e}")
127 + raise e
128 + await self.wait_tick()
129 +
130 + async def get_full_dom(self) -> str:
131 + """Get full DOM with unique selectors"""
132 + await self._check_page()
133 + js_code = files.read_file("lib/browser/extract_dom.js")
134 +
135 + # Get all frames
136 + self.contexts = {}
137 + frame_contents = {}
138 +
139 + # Extract content from each frame
140 + i = -1
141 + for frame in self.page.frames:
142 + try:
143 + if frame.url: # and frame != self.page.main_frame:
144 + i += 1
145 + frame_mark = self._num_to_alpha(i)
146 +
147 + # Check if frame is still valid
148 + await self.wait_tick()
149 + if not frame.is_detached():
150 + try:
151 + # short timeout to identify and skip unresponsive frames
152 + async with asyncio.timeout(0.25):
153 + await frame.evaluate("window.location.href")
154 + except TimeoutError as e:
155 + print(f"Skipping unresponsive frame: {frame.url}")
156 + continue
157 +
158 + await frame.wait_for_load_state(
159 + "domcontentloaded", timeout=1000
160 + )
161 +
162 + async with asyncio.timeout(1):
163 + content = await frame.evaluate(
164 + js_code, [frame_mark, self.selector_name]
165 + )
166 + self.contexts[frame_mark] = frame
167 + frame_contents[frame.url] = content
168 + else:
169 + print(f"Warning: Frame was detached: {frame.url}")
170 + except Exception as e:
171 + print(f"Error extracting from frame {frame.url}: {e}")
172 +
173 + # # Get main frame content
174 + # main_mark = self._num_to_alpha(0)
175 + # main_content = ""
176 + # try:
177 + # async with asyncio.timeout(1):
178 + # main_content = await self.page.evaluate(js_code, [main_mark, self.selector_name])
179 + # self.contexts[main_mark] = self.page
180 + # except Exception as e:
181 + # print(f"Error when extracting from main frame: {e}")
182 +
183 + # Replace iframe placeholders with actual content
184 + # for url, content in frame_contents.items():
185 + # placeholder = f'<iframe src="{url}"'
186 + # main_content = main_content.replace(placeholder, f'{placeholder}>\n<!-- IFrame Content Start -->\n{content}\n<!-- IFrame Content End -->\n</iframe')
187 +
188 + # return main_content + "".join(frame_contents.values())
189 + return "".join(frame_contents.values())
190 +
191 + def strip_html_dom(self, html_content: str) -> str:
192 + """Clean and strip HTML content"""
193 + if not html_content:
194 + return ""
195 +
196 + soup = BeautifulSoup(html_content, "html.parser")
197 +
198 + for tag in soup.find_all(
199 + ["br", "hr", "style", "script", "noscript", "meta", "link", "svg"]
200 + ):
201 + tag.decompose()
202 +
203 + for tag in soup.find_all(True):
204 + if tag.attrs and "invisible" in tag.attrs:
205 + tag.decompose()
206 +
207 + for tag in soup.find_all(True):
208 + allowed_attrs = [
209 + self.selector_name,
210 + "aria-label",
211 + "placeholder",
212 + "name",
213 + "value",
214 + "type",
215 + ]
216 + attrs = {
217 + "selector" if key == self.selector_name else key: tag.attrs[key]
218 + for key in allowed_attrs
219 + if key in tag.attrs and tag.attrs[key]
220 + }
221 + tag.attrs = attrs
222 +
223 + def remove_empty(tag_name: str) -> None:
224 + for tag in soup.find_all(tag_name):
225 + if not tag.attrs:
226 + tag.unwrap()
227 +
228 + remove_empty("span")
229 + remove_empty("p")
230 + remove_empty("strong")
231 +
232 + return soup.prettify(formatter="minimal")
233 +
234 + def process_html_with_selectors(self, html_content: str) -> str:
235 + """Process HTML content and add selectors to interactive elements"""
236 + if not html_content:
237 + return ""
238 +
239 + html_content = re.sub(r"\s+", " ", html_content)
240 + soup = BeautifulSoup(html_content, "html.parser")
241 +
242 + structural_tags = [
243 + "html",
244 + "head",
245 + "body",
246 + "div",
247 + "span",
248 + "section",
249 + "main",
250 + "article",
251 + "header",
252 + "footer",
253 + "nav",
254 + "ul",
255 + "ol",
256 + "li",
257 + "tr",
258 + "td",
259 + "th",
260 + ]
261 + for tag in structural_tags:
262 + for element in soup.find_all(tag):
263 + element.unwrap()
264 +
265 + out = str(soup).strip()
266 + out = re.sub(r">\s*<", "><", out)
267 + out = re.sub(r'aria-label="', 'label="', out)
268 +
269 + # out = re.sub(r'selector="(\d+[a-zA-Z]+)"', r'selector=\1', out)
270 + return out
271 +
272 + async def get_clean_dom(self) -> str:
273 + """Get clean DOM with selectors"""
274 + full_dom = await self.get_full_dom()
275 + clean_dom = self.strip_html_dom(full_dom)
276 + return self.process_html_with_selectors(clean_dom)
277 +
278 + async def click(self, selector: str):
279 + await self._check_page()
280 + ctx, selector = self._parse_selector(selector)
281 + self.last_selector = selector
282 + # js_code = files.read_file("lib/browser/click.js")
283 + # result = await self.page.evaluate(js_code, [selector])
284 + # if not result:
285 + result = await ctx.hover(selector, force=True, timeout=Browser.interact_timeout)
286 + await self.wait_tick()
287 + result = await ctx.click(selector, force=True, timeout=Browser.interact_timeout)
288 + await self.wait_tick()
289 +
290 + # await self.page.wait_for_load_state("networkidle")
291 + return result
292 +
293 + async def press(self, key: str):
294 + await self._check_page()
295 + if self.last_selector:
296 + await self.page.press(
297 + self.last_selector, key, timeout=Browser.interact_timeout
298 + )
299 + else:
300 + await self.page.keyboard.press(key)
301 +
302 + async def fill(self, selector: str, text: str):
303 + await self._check_page()
304 + ctx, selector = self._parse_selector(selector)
305 + self.last_selector = selector
306 + try:
307 + await self.click(selector)
308 + except Exception as e:
309 + pass
310 + await ctx.fill(selector, text, force=True, timeout=Browser.interact_timeout)
311 + await self.wait_tick()
312 +
313 + async def execute(self, js_code: str):
314 + await self._check_page()
315 + result = await self.page.evaluate(js_code)
316 + return result
317 +
318 + async def screenshot(self, path: str, full_page=False):
319 + await self._check_page()
320 + await self.page.screenshot(path=path, full_page=full_page)
321 +
322 + def _parse_selector(self, selector: str) -> tuple[Page | Frame, str]:
323 + try:
324 + ctx = self.page
325 + # Check if selector is our UID, return
326 + if re.match(r"^\d+[a-zA-Z]+$", selector):
327 + alpha_part = "".join(filter(str.isalpha, selector))
328 + ctx = self.contexts[alpha_part]
329 + selector = f"[{self.selector_name}='{selector}']"
330 + return (ctx, selector)
331 + except Exception as e:
332 + raise Exception(f"Error evaluating selector: {selector}")
333 +
334 + async def _check_page(self):
335 + for _ in range(2):
336 + try:
337 + await self.wait_tick()
338 + self.page = self.context.pages[0]
339 + if not self.page:
340 + raise NoPageError(
341 + "No page is open in the browser. Please open a URL first."
342 + )
343 + # await self.page.wait_for_load_state("networkidle",)
344 + async with asyncio.timeout(self.load_timeout / 1000):
345 + if not self.page_loaded:
346 + while not self.page_loaded:
347 + await asyncio.sleep(0.1)
348 + await self.wait_tick()
349 + return
350 + except TimeoutError as e:
351 + self.page_loaded = True
352 + return
353 + except NoPageError as e:
354 + raise e
355 + except Exception as e:
356 + print(f"Error checking page: {e}")
357 +
358 + def _num_to_alpha(self, num: int) -> str:
359 + if num < 0:
360 + return ""
361 +
362 + result = ""
363 + while num >= 0:
364 + result = chr(num % 26 + 97) + result
365 + num = num // 26 - 1
366 +
367 + return result
368 +
369 + async def wait_tick(self):
370 + if self.page:
371 + await self.page.evaluate("window.location.href")
372 +
373 + async def wait(self, seconds: float = 1.0):
374 + await asyncio.sleep(seconds)
375 + await self.wait_tick()
376 +
377 + async def wait_for_action(self):
378 + nav_count = self.navigation_count
379 + for _ in range(5):
380 + await self._check_page()
381 + if nav_count != self.navigation_count:
382 + print("Navigation detected")
383 + await asyncio.sleep(1)
384 + return
385 + await asyncio.sleep(0.1)
python/helpers/browser_use.py new
+3
@@ -0,0 +1,3 @@
1 +from python.helpers import dotenv
2 +dotenv.save_dotenv_value("ANONYMIZED_TELEMETRY", "false")
3 +import browser_use
\ No newline at end of file
python/helpers/defer.py
+147 -29
@@ -1,37 +1,82 @@
1 import asyncio
2 +from dataclasses import dataclass
3 import threading
4 from concurrent.futures import Future, ThreadPoolExecutor
4 -from typing import Any, Callable, Optional, Coroutine
5 +from typing import Any, Callable, Optional, Coroutine, TypeVar, Union, Awaitable
6 +
7 +T = TypeVar("T")
8 +
9
10 class EventLoopThread:
7 - _instance = None
11 + _instances = {}
12 _lock = threading.Lock()
13
10 - def __new__(cls):
14 + def __init__(self, thread_name: str = "default") -> None:
15 + """Initialize the event loop thread."""
16 + self.thread_name = thread_name
17 + self._start()
18 +
19 + def __new__(cls, thread_name: str = "default"):
20 with cls._lock:
12 - if cls._instance is None:
13 - cls._instance = super(EventLoopThread, cls).__new__(cls)
14 - cls._instance.loop = asyncio.new_event_loop() # type: ignore
15 - cls._instance.thread = threading.Thread(target=cls._instance._run_event_loop, daemon=True) # type: ignore
16 - cls._instance.thread.start() # type: ignore
17 - return cls._instance
21 + if thread_name not in cls._instances:
22 + instance = super(EventLoopThread, cls).__new__(cls)
23 + cls._instances[thread_name] = instance
24 + return cls._instances[thread_name]
25 +
26 + def _start(self):
27 + if not hasattr(self, "loop") or not self.loop:
28 + self.loop = asyncio.new_event_loop()
29 + if not hasattr(self, "thread") or not self.thread:
30 + self.thread = threading.Thread(
31 + target=self._run_event_loop, daemon=True, name=self.thread_name
32 + )
33 + self.thread.start()
34
35 def _run_event_loop(self):
20 - asyncio.set_event_loop(self.loop) # type: ignore
21 - self.loop.run_forever() # type: ignore
36 + if not self.loop:
37 + raise RuntimeError("Event loop is not initialized")
38 + asyncio.set_event_loop(self.loop)
39 + self.loop.run_forever()
40 +
41 + def terminate(self):
42 + if self.loop and self.loop.is_running():
43 + self.loop.stop()
44 + self.loop = None
45 + self.thread = None
46
47 def run_coroutine(self, coro):
24 - return asyncio.run_coroutine_threadsafe(coro, self.loop) # type: ignore
48 + self._start()
49 + if not self.loop:
50 + raise RuntimeError("Event loop is not initialized")
51 + return asyncio.run_coroutine_threadsafe(coro, self.loop)
52 +
53 +
54 +@dataclass
55 +class ChildTask:
56 + task: "DeferredTask"
57 + terminate_thread: bool
58 +
59
60 class DeferredTask:
27 - def __init__(self, func: Callable[..., Coroutine[Any, Any, Any]], *args: Any, **kwargs: Any):
61 + def __init__(
62 + self,
63 + thread_name: str = "default",
64 + ):
65 + self.event_loop_thread = EventLoopThread(thread_name)
66 + self._future: Optional[Future] = None
67 + self.children: list[ChildTask] = []
68 +
69 + def start_task(
70 + self, func: Callable[..., Coroutine[Any, Any, Any]], *args: Any, **kwargs: Any
71 + ):
72 self.func = func
73 self.args = args
74 self.kwargs = kwargs
31 - self.event_loop_thread = EventLoopThread()
32 - self._future: Optional[Future] = None
75 self._start_task()
76
77 + def __del__(self):
78 + self.kill()
79 +
80 def _start_task(self):
81 self._future = self.event_loop_thread.run_coroutine(self._run())
82
@@ -47,34 +92,107 @@ class DeferredTask:
92 try:
93 return self._future.result(timeout)
94 except TimeoutError:
50 - raise TimeoutError("The task did not complete within the specified timeout.")
95 + raise TimeoutError(
96 + "The task did not complete within the specified timeout."
97 + )
98
99 async def result(self, timeout: Optional[float] = None) -> Any:
100 if not self._future:
101 raise RuntimeError("Task hasn't been started")
55 -
102 +
103 loop = asyncio.get_running_loop()
57 -
104 +
105 def _get_result():
106 try:
60 - return self._future.result(timeout) # type: ignore
107 + result = self._future.result(timeout) # type: ignore
108 + # self.kill()
109 + return result
110 except TimeoutError:
62 - raise TimeoutError("The task did not complete within the specified timeout.")
63 -
111 + raise TimeoutError(
112 + "The task did not complete within the specified timeout."
113 + )
114 +
115 return await loop.run_in_executor(None, _get_result)
116
66 - def kill(self) -> None:
117 + def kill(self, terminate_thread: bool = False) -> None:
118 + """Kill the task and optionally terminate its thread."""
119 + self.kill_children()
120 if self._future and not self._future.done():
121 self._future.cancel()
122
123 + if (
124 + terminate_thread
125 + and self.event_loop_thread.loop
126 + and self.event_loop_thread.loop.is_running()
127 + ):
128 +
129 + def cleanup():
130 + tasks = [
131 + t
132 + for t in asyncio.all_tasks(self.event_loop_thread.loop)
133 + if t is not asyncio.current_task(self.event_loop_thread.loop)
134 + ]
135 + for task in tasks:
136 + task.cancel()
137 + try:
138 + # Give tasks a chance to cleanup
139 + if self.event_loop_thread.loop:
140 + self.event_loop_thread.loop.run_until_complete(
141 + asyncio.gather(task, return_exceptions=True)
142 + )
143 + except Exception:
144 + pass # Ignore cleanup errors
145 +
146 + self.event_loop_thread.loop.call_soon_threadsafe(cleanup)
147 + self.event_loop_thread.terminate()
148 +
149 + def kill_children(self) -> None:
150 + for child in self.children:
151 + child.task.kill(terminate_thread=child.terminate_thread)
152 + self.children = []
153 +
154 def is_alive(self) -> bool:
71 - return self._future and not self._future.done() # type: ignore
155 + return self._future and not self._future.done() # type: ignore
156
73 - def restart(self) -> None:
157 + def restart(self, terminate_thread: bool = False) -> None:
158 + self.kill()
159 self._start_task()
160
76 -def run_in_background(func, *args, **kwargs):
77 - async def wrapper(*args, **kwargs):
78 - loop = asyncio.get_event_loop()
79 - return await loop.run_in_executor(None, func, *args, **kwargs)
80 - return wrapper
\ No newline at end of file
161 + def add_child_task(
162 + self, task: "DeferredTask", terminate_thread: bool = False
163 + ) -> None:
164 + self.children.append(ChildTask(task, terminate_thread))
165 +
166 + async def _execute_in_task_context(
167 + self, func: Callable[..., T], *args, **kwargs
168 + ) -> T:
169 + """Execute a function in the task's context and return its result."""
170 + result = func(*args, **kwargs)
171 + if asyncio.iscoroutine(result):
172 + return await result
173 + return result
174 +
175 + def execute_inside(self, func: Callable[..., T], *args, **kwargs) -> Awaitable[T]:
176 + if not self.event_loop_thread.loop:
177 + raise RuntimeError("Event loop is not initialized")
178 +
179 + future: Future = Future()
180 +
181 + async def wrapped():
182 + if not self.event_loop_thread.loop:
183 + raise RuntimeError("Event loop is not initialized")
184 + try:
185 + result = await self._execute_in_task_context(func, *args, **kwargs)
186 + # Keep awaiting until we get a concrete value
187 + while isinstance(result, Awaitable):
188 + result = await result
189 + self.event_loop_thread.loop.call_soon_threadsafe(
190 + future.set_result, result
191 + )
192 + except Exception as e:
193 + self.event_loop_thread.loop.call_soon_threadsafe(
194 + future.set_exception, e
195 + )
196 +
197 + asyncio.run_coroutine_threadsafe(wrapped(), self.event_loop_thread.loop)
198 + return asyncio.wrap_future(future)
python/helpers/dirty_json.py
+32 -5
@@ -37,7 +37,28 @@ class DirtyJson:
37 self.current_char = None
38
39 def _skip_whitespace(self):
40 - while self.current_char is not None and self.current_char.isspace():
40 + while self.current_char is not None:
41 + if self.current_char.isspace():
42 + self._advance()
43 + elif self.current_char == '/' and self._peek(1) == '/': # Single-line comment
44 + self._skip_single_line_comment()
45 + elif self.current_char == '/' and self._peek(1) == '*': # Multi-line comment
46 + self._skip_multi_line_comment()
47 + else:
48 + break
49 +
50 + def _skip_single_line_comment(self):
51 + while self.current_char is not None and self.current_char != '\n':
52 + self._advance()
53 + if self.current_char == '\n':
54 + self._advance()
55 +
56 + def _skip_multi_line_comment(self):
57 + self._advance(2) # Skip /*
58 + while self.current_char is not None:
59 + if self.current_char == '*' and self._peek(1) == '/':
60 + self._advance(2) # Skip */
61 + break
62 self._advance()
63
64 def _parse(self):
@@ -180,13 +201,20 @@ class DirtyJson:
201 if self.current_char in ['"', "'", '\\', '/', 'b', 'f', 'n', 'r', 't']:
202 result += {'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t'}.get(self.current_char, self.current_char)
203 elif self.current_char == 'u':
204 + self._advance() # Skip 'u'
205 unicode_char = ""
206 + # Try to collect exactly 4 hex digits
207 for _ in range(4):
185 - if self.current_char is None:
186 - return result
208 + if self.current_char is None or not self.current_char.isalnum():
209 + # If we can't get 4 hex digits, treat it as a literal '\u' followed by whatever we got
210 + return result + '\\u' + unicode_char
211 unicode_char += self.current_char
212 self._advance()
189 - result += chr(int(unicode_char, 16))
213 + try:
214 + result += chr(int(unicode_char, 16))
215 + except ValueError:
216 + # If invalid hex value, treat as literal
217 + result += '\\u' + unicode_char
218 continue
219 else:
220 result += self.current_char
@@ -264,4 +292,3 @@ class DirtyJson:
292 chars = ["{", "[", '"']
293 indices = [input_str.find(char) for char in chars if input_str.find(char) != -1]
294 return min(indices) if indices else 0
267 -
python/helpers/files.py
+5
@@ -162,6 +162,11 @@ def write_file(relative_path: str, content: str, encoding: str = "utf-8"):
162 with open(abs_path, "w", encoding=encoding) as f:
163 f.write(content)
164
165 +def write_file_bin(relative_path: str, content: bytes):
166 + abs_path = get_abs_path(relative_path)
167 + os.makedirs(os.path.dirname(abs_path), exist_ok=True)
168 + with open(abs_path, "wb") as f:
169 + f.write(content)
170
171 def delete_file(relative_path: str):
172 abs_path = get_abs_path(relative_path)
python/helpers/log.py
+1
@@ -6,6 +6,7 @@ from collections import OrderedDict # Import OrderedDict
6
7 Type = Literal[
8 "agent",
9 + "browser",
10 "code_exe",
11 "error",
12 "hint",
python/helpers/print_catch.py new
+31
@@ -0,0 +1,31 @@
1 +import asyncio
2 +import io
3 +import sys
4 +from typing import Callable, Any, Awaitable, Tuple
5 +
6 +def capture_prints_async(
7 + func: Callable[..., Awaitable[Any]],
8 + *args,
9 + **kwargs
10 +) -> Tuple[Awaitable[Any], Callable[[], str]]:
11 + # Create a StringIO object to capture the output
12 + captured_output = io.StringIO()
13 + original_stdout = sys.stdout
14 +
15 + # Define a function to get the current captured output
16 + def get_current_output() -> str:
17 + return captured_output.getvalue()
18 +
19 + async def wrapped_func() -> Any:
20 + nonlocal captured_output, original_stdout
21 + try:
22 + # Redirect sys.stdout to the StringIO object
23 + sys.stdout = captured_output
24 + # Await the provided function
25 + return await func(*args, **kwargs)
26 + finally:
27 + # Restore the original sys.stdout
28 + sys.stdout = original_stdout
29 +
30 + # Return the wrapped awaitable and the output retriever
31 + return asyncio.create_task(wrapped_func()), get_current_output
\ No newline at end of file
python/helpers/settings.py
+90 -8
@@ -9,6 +9,7 @@ import models
9 from python.helpers import runtime, whisper, defer
10 from . import files, dotenv
11
12 +
13 class Settings(TypedDict):
14 chat_model_provider: str
15 chat_model_name: str
@@ -30,13 +31,18 @@ class Settings(TypedDict):
31 util_model_rl_input: int
32 util_model_rl_output: int
33
33 -
34 embed_model_provider: str
35 embed_model_name: str
36 embed_model_kwargs: dict[str, str]
37 embed_model_rl_requests: int
38 embed_model_rl_input: int
39
40 + browser_model_provider: str
41 + browser_model_name: str
42 + browser_model_vision: bool
43 + browser_model_temperature: float
44 + browser_model_kwargs: dict[str, str]
45 +
46 agent_prompts_subdir: str
47 agent_memory_subdir: str
48 agent_knowledge_subdir: str
@@ -73,7 +79,7 @@ class SettingsField(TypedDict, total=False):
79 id: str
80 title: str
81 description: str
76 - type: Literal["text", "number", "select", "range", "textarea", "password"]
82 + type: Literal["text", "number", "select", "range", "textarea", "password", "switch"]
83 value: Any
84 min: float
85 max: float
@@ -82,6 +88,7 @@ class SettingsField(TypedDict, total=False):
88
89
90 class SettingsSection(TypedDict, total=False):
91 + id: str
92 title: str
93 description: str
94 fields: list[SettingsField]
@@ -100,7 +107,6 @@ _settings: Settings | None = None
107 def convert_out(settings: Settings) -> SettingsOutput:
108 from models import ModelProvider
109
103 -
110 # main model section
111 chat_model_fields: list[SettingsField] = []
112 chat_model_fields.append(
@@ -200,6 +206,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
206 )
207
208 chat_model_section: SettingsSection = {
209 + "id": "chat_model",
210 "title": "Chat Model",
211 "description": "Selection and settings for main chat model used by Agent Zero",
212 "fields": chat_model_fields,
@@ -239,7 +246,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
246 "value": settings["util_model_temperature"],
247 }
248 )
242 -
249 +
250 # util_model_fields.append(
251 # {
252 # "id": "util_model_ctx_length",
@@ -303,6 +310,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
310 )
311
312 util_model_section: SettingsSection = {
313 + "id": "util_model",
314 "title": "Utility model",
315 "description": "Smaller, cheaper, faster model for handling utility tasks like organizing memory, preparing prompts, summarizing.",
316 "fields": util_model_fields,
@@ -329,7 +337,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
337 "value": settings["embed_model_name"],
338 }
339 )
332 -
340 +
341 embed_model_fields.append(
342 {
343 "id": "embed_model_rl_requests",
@@ -361,11 +369,74 @@ def convert_out(settings: Settings) -> SettingsOutput:
369 )
370
371 embed_model_section: SettingsSection = {
372 + "id": "embed_model",
373 "title": "Embedding Model",
374 "description": "Settings for the embedding model used by Agent Zero.",
375 "fields": embed_model_fields,
376 }
377
378 + # embedding model section
379 + browser_model_fields: list[SettingsField] = []
380 + browser_model_fields.append(
381 + {
382 + "id": "browser_model_provider",
383 + "title": "Web Browser model provider",
384 + "description": "Select provider for web browser model used by browser-use framework",
385 + "type": "select",
386 + "value": settings["browser_model_provider"],
387 + "options": [{"value": p.name, "label": p.value} for p in ModelProvider],
388 + }
389 + )
390 + browser_model_fields.append(
391 + {
392 + "id": "browser_model_name",
393 + "title": "Web Browser model name",
394 + "description": "Exact name of model from selected provider",
395 + "type": "text",
396 + "value": settings["browser_model_name"],
397 + }
398 + )
399 +
400 + browser_model_fields.append(
401 + {
402 + "id": "browser_model_vision",
403 + "title": "Use Vision",
404 + "description": "Models capable of Vision can use it to analyze web pages from screenshots. Increases quality but also token usage.",
405 + "type": "switch",
406 + "value": settings["browser_model_vision"],
407 + }
408 + )
409 +
410 + browser_model_fields.append(
411 + {
412 + "id": "browser_model_temperature",
413 + "title": "Web Browser model temperature",
414 + "description": "Determines the randomness of generated responses. 0 is deterministic, 1 is random",
415 + "type": "range",
416 + "min": 0,
417 + "max": 1,
418 + "step": 0.01,
419 + "value": settings["browser_model_temperature"],
420 + }
421 + )
422 +
423 + browser_model_fields.append(
424 + {
425 + "id": "browser_model_kwargs",
426 + "title": "Web Browser model additional parameters",
427 + "description": "Any other parameters supported by the model. Format is KEY=VALUE on individual lines, just like .env file.",
428 + "type": "textarea",
429 + "value": _dict_to_env(settings["browser_model_kwargs"]),
430 + }
431 + )
432 +
433 + browser_model_section: SettingsSection = {
434 + "id": "browser_model",
435 + "title": "Web Browser Model",
436 + "description": "Settings for the web browser model used by browser-use framework.",
437 + "fields": browser_model_fields,
438 + }
439 +
440 # basic auth section
441 auth_fields: list[SettingsField] = []
442
@@ -405,6 +476,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
476 )
477
478 auth_section: SettingsSection = {
479 + "id": "auth",
480 "title": "Authentication",
481 "description": "Settings for authentication to use Agent Zero Web UI.",
482 "fields": auth_fields,
@@ -432,6 +504,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
504 )
505
506 api_keys_section: SettingsSection = {
507 + "id": "api_keys",
508 "title": "API Keys",
509 "description": "API keys for model providers and services used by Agent Zero.",
510 "fields": api_keys_fields,
@@ -470,7 +543,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
543
544 agent_fields.append(
545 {
473 - "id": "agent_knowledge_subdirs",
546 + "id": "agent_knowledge_subdir",
547 "title": "Knowledge subdirectory",
548 "description": "Subdirectory of /knowledge folder to use for agent knowledge import. 'default' subfolder is always imported and contains framework knowledge.",
549 "type": "select",
@@ -483,6 +556,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
556 )
557
558 agent_section: SettingsSection = {
559 + "id": "agent",
560 "title": "Agent Config",
561 "description": "Agent parameters.",
562 "fields": agent_fields,
@@ -547,6 +621,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
621 )
622
623 dev_section: SettingsSection = {
624 + "id": "dev",
625 "title": "Development",
626 "description": "Parameters for A0 framework development. RFCs (remote function calls) are used to call functions on another A0 instance. You can develop and debug A0 natively on your local system while redirecting some functions to A0 instance in docker. This is crucial for development as A0 needs to run in standardized environment to support all features.",
627 "fields": dev_fields,
@@ -617,6 +692,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
692 )
693
694 stt_section: SettingsSection = {
695 + "id": "stt",
696 "title": "Speech to Text",
697 "description": "Voice transcription preferences and server turn detection settings.",
698 "fields": stt_fields,
@@ -629,6 +705,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
705 chat_model_section,
706 util_model_section,
707 embed_model_section,
708 + browser_model_section,
709 stt_section,
710 api_keys_section,
711 auth_section,
@@ -690,7 +767,7 @@ def normalize_settings(settings: Settings) -> Settings:
767 try:
768 copy[key] = type(value)(copy[key]) # type: ignore
769 except (ValueError, TypeError):
693 - copy[key] = value # make default instead
770 + copy[key] = value # make default instead
771 return copy
772
773
@@ -796,6 +873,11 @@ def get_default_settings() -> Settings:
873 embed_model_kwargs={},
874 embed_model_rl_requests=0,
875 embed_model_rl_input=0,
876 + browser_model_provider=ModelProvider.OPENAI.name,
877 + browser_model_name="gpt-4o-mini",
878 + browser_model_vision=False,
879 + browser_model_temperature=0.0,
880 + browser_model_kwargs={},
881 api_keys={},
882 auth_login="",
883 auth_password="",
@@ -831,7 +913,7 @@ def _apply_settings():
913 agent = agent.get_data(agent.DATA_NAME_SUBORDINATE)
914
915 # reload whisper model if necessary
834 - task = defer.DeferredTask(whisper.preload, _settings["stt_model_size"])
916 + task = defer.DeferredTask().start_task(whisper.preload, _settings["stt_model_size"]) #TODO overkill, replace with background task
917
918
919 def _env_to_dict(data: str):
python/helpers/strings.py
+25 -2
@@ -89,5 +89,28 @@ def calculate_valid_match_lengths(first: bytes | str, second: bytes | str,
89 # Return the last matched positions instead of the current indices
90 return last_matched_i, last_matched_j
91
92 - # Return the last matched positions instead of the current indices
93 - return last_matched_i, last_matched_j
\ No newline at end of file
92 +def format_key(key: str) -> str:
93 + """Format a key string to be more readable.
94 + Converts camelCase and snake_case to Title Case with spaces."""
95 + # First replace non-alphanumeric with spaces
96 + result = ''.join(' ' if not c.isalnum() else c for c in key)
97 +
98 + # Handle camelCase
99 + formatted = ''
100 + for i, c in enumerate(result):
101 + if i > 0 and c.isupper() and result[i-1].islower():
102 + formatted += ' ' + c
103 + else:
104 + formatted += c
105 +
106 + # Split on spaces and capitalize each word
107 + return ' '.join(word.capitalize() for word in formatted.split())
108 +
109 +def dict_to_text(d: dict) -> str:
110 + parts = []
111 + for key, value in d.items():
112 + parts.append(f"{format_key(str(key))}:")
113 + parts.append(f"{value}")
114 + parts.append("") # Add empty line between entries
115 +
116 + return "\n".join(parts).rstrip() # rstrip to remove trailing newline
python/tools/browser.py new
+61
@@ -0,0 +1,61 @@
1 +import asyncio
2 +from dataclasses import dataclass
3 +import time
4 +from python.helpers.tool import Tool, Response
5 +from python.helpers import files, rfc_exchange
6 +from python.helpers.print_style import PrintStyle
7 +from python.helpers.browser import Browser as BrowserManager
8 +import uuid
9 +
10 +
11 +@dataclass
12 +class State:
13 + browser: BrowserManager
14 +
15 +
16 +class Browser(Tool):
17 +
18 + async def execute(self, **kwargs):
19 + raise NotImplementedError
20 +
21 + def get_log_object(self):
22 + return self.agent.context.log.log(
23 + type="browser",
24 + heading=f"{self.agent.agent_name}: Using tool '{self.name}'",
25 + content="",
26 + kvps=self.args,
27 + )
28 +
29 + # async def after_execution(self, response, **kwargs):
30 + # await self.agent.hist_add_tool_result(self.name, response.message)
31 +
32 + async def save_screenshot(self):
33 + await self.prepare_state()
34 + path = files.get_abs_path("tmp/browser", f"{uuid.uuid4()}.png")
35 + await self.state.browser.screenshot(path, True)
36 + return "img://" + path
37 +
38 + async def prepare_state(self, reset=False):
39 + self.state = self.agent.get_data("_browser_state")
40 + if not self.state or reset:
41 + self.state = State(browser=BrowserManager())
42 + self.agent.set_data("_browser_state", self.state)
43 +
44 + def update_progress(self, text):
45 + progress = f"Browser: {text}"
46 + self.log.update(progress=text)
47 + self.agent.context.log.set_progress(progress)
48 +
49 + def cleanup_history(self):
50 + def cleanup_message(msg):
51 + if not msg.ai and isinstance(msg.content, dict) and "tool_name" in msg.content and str(msg.content["tool_name"]).startswith("browser_"):
52 + if not msg.summary:
53 + msg.summary = "browser content removed to save space"
54 +
55 + for msg in self.agent.history.current.messages:
56 + cleanup_message(msg)
57 +
58 + for prev in self.agent.history.topics:
59 + if not prev.summary:
60 + for msg in prev.messages:
61 + cleanup_message(msg)
python/tools/browser_agent.py new
+234
@@ -0,0 +1,234 @@
1 +import asyncio
2 +import json
3 +import time
4 +from agent import Agent
5 +
6 +import models
7 +from python.helpers.tool import Tool, Response
8 +from python.helpers import dirty_json, files, rfc_exchange, defer, strings
9 +from python.helpers.print_style import PrintStyle
10 +from python.helpers.browser_use import browser_use
11 +from pydantic import BaseModel
12 +import uuid
13 +
14 +
15 +class State:
16 + @staticmethod
17 + async def create(agent: Agent):
18 + state = State(agent)
19 + return state
20 +
21 + def __init__(self, agent: Agent):
22 + self.agent = agent
23 + self.context = None
24 + self.task = None
25 + self.use_agent = None
26 + self.browser = None
27 +
28 + def __del__(self):
29 + self.kill_task()
30 +
31 + async def _initialize(self):
32 + if self.context:
33 + return
34 +
35 + self.browser = browser_use.Browser(
36 + config=browser_use.BrowserConfig(
37 + headless=True,
38 + disable_security=True,
39 + )
40 + )
41 +
42 + # Await the coroutine to get the browser context
43 + self.context = await self.browser.new_context()
44 +
45 + # Add init script to the context - this will be applied to all new pages
46 + await self.context._initialize_session()
47 + pw_context = self.context.session.context # type: ignore
48 + js_override = files.get_abs_path("lib/browser/init_override.js")
49 + await pw_context.add_init_script(path=js_override) # type: ignore
50 +
51 + def start_task(self, task: str):
52 + if self.task and self.task.is_alive():
53 + self.kill_task()
54 +
55 + if not self.task:
56 + self.task = defer.DeferredTask(
57 + thread_name="BrowserAgent" + self.agent.context.id
58 + )
59 + if self.agent.context.task:
60 + self.agent.context.task.add_child_task(
61 + self.task, terminate_thread=True
62 + )
63 + self.task.start_task(self._run_task, task)
64 + return self.task
65 +
66 + def kill_task(self):
67 + if self.task:
68 + self.task.kill(terminate_thread=True)
69 + self.task = None
70 + self.context = None
71 + self.use_agent = None
72 + self.browser = None
73 +
74 + async def _run_task(self, task: str):
75 +
76 + agent = self.agent
77 +
78 + await self._initialize()
79 +
80 + class CustomSystemPrompt(browser_use.SystemPrompt):
81 + def important_rules(self) -> str:
82 + existing_rules = super().important_rules()
83 + new_rules = agent.read_prompt("prompts/browser_agent.system.md")
84 + return f"{existing_rules}\n{new_rules}".strip()
85 +
86 + # Model of task result
87 + class DoneResult(BaseModel):
88 + title: str
89 + response: str
90 + page_summary: str
91 +
92 + # Initialize controller
93 + controller = browser_use.Controller()
94 +
95 + # we overwrite done() in this example to demonstrate the validator
96 + @controller.registry.action("Done with task", param_model=DoneResult)
97 + async def done(params: DoneResult):
98 + result = browser_use.ActionResult(
99 + is_done=True, extracted_content=params.model_dump_json()
100 + )
101 + return result
102 +
103 + # @controller.action("Ask user for information")
104 + # def ask_user(question: str) -> str:
105 + # return "..."
106 +
107 + model = models.get_model(
108 + type=models.ModelType.CHAT,
109 + provider=self.agent.config.browser_model.provider,
110 + name=self.agent.config.browser_model.name,
111 + **self.agent.config.browser_model.kwargs,
112 + )
113 +
114 + self.use_agent = browser_use.Agent(
115 + task=task,
116 + browser_context=self.context,
117 + llm=self.agent.get_utility_model(),
118 + use_vision=self.agent.config.browser_model.vision,
119 + system_prompt_class=CustomSystemPrompt,
120 + controller=controller,
121 + )
122 + result = await self.use_agent.run()
123 + return result
124 +
125 + async def get_page(self):
126 + if self.use_agent:
127 + return await self.use_agent.browser_context.get_current_page()
128 +
129 +
130 +class BrowserAgent(Tool):
131 +
132 + async def execute(self, message="", **kwargs):
133 + self.guid = str(uuid.uuid4())
134 + await self.prepare_state()
135 + task = self.state.start_task(message)
136 +
137 + # wait for browser agent to finish and update progress
138 + while not task.is_ready():
139 + await self.agent.handle_intervention()
140 + await asyncio.sleep(1)
141 + try:
142 + update = await self.get_update()
143 + log = update.get("log")
144 + if log:
145 + self.update_progress("\n".join(log))
146 + screenshot = update.get("screenshot", None)
147 + if screenshot:
148 + self.log.update(screenshot=screenshot)
149 + except Exception as e:
150 + pass
151 +
152 + # collect result
153 + result = await task.result()
154 + answer = result.final_result()
155 + answer_data = dirty_json.DirtyJson.parse_string(answer)
156 + answer_text = strings.dict_to_text(answer_data) # type: ignore
157 + self.log.update(answer=answer_text)
158 + return Response(message=answer, break_loop=False)
159 +
160 + def get_log_object(self):
161 + return self.agent.context.log.log(
162 + type="browser",
163 + heading=f"{self.agent.agent_name}: Using tool '{self.name}'",
164 + content="",
165 + kvps=self.args,
166 + )
167 +
168 + # async def after_execution(self, response, **kwargs):
169 + # await self.agent.hist_add_tool_result(self.name, response.message)
170 +
171 + async def get_update(self):
172 + await self.prepare_state()
173 +
174 + result = {}
175 + ua = self.state.use_agent
176 + page = await self.state.get_page()
177 +
178 + if ua and page:
179 + try:
180 +
181 + async def _get_update():
182 + log = []
183 +
184 + for message in ua.message_manager.get_messages():
185 + if message.type == "system":
186 + continue
187 + if message.type == "ai":
188 + try:
189 + data = json.loads(message.content) # type: ignore
190 + cs = data.get("current_state")
191 + if cs:
192 + log.append("AI:" + cs["memory"])
193 + log.append("AI:" + cs["next_goal"])
194 + except Exception:
195 + pass
196 + if message.type == "human":
197 + content = str(message.content).strip()
198 + part = content.split("\n", 1)[0].split(",", 1)[0]
199 + if part:
200 + if len(part) > 150:
201 + part = part[:150] + "..."
202 + log.append("FW:" + part)
203 + result["log"] = log
204 +
205 + path = files.get_abs_path("tmp/browser", f"{self.guid}.png")
206 + await page.screenshot(path=path, full_page=False, timeout=3000)
207 + result["screenshot"] = f"img://{path}&t={str(time.time())}"
208 +
209 + if self.state.task:
210 + await self.state.task.execute_inside(_get_update)
211 +
212 + except Exception as e:
213 + pass
214 +
215 + return result
216 +
217 + async def prepare_state(self, reset=False):
218 + self.state = self.agent.get_data("_browser_agent_state")
219 + if not self.state or reset:
220 + self.state = await State.create(self.agent)
221 + self.agent.set_data("_browser_agent_state", self.state)
222 +
223 + def update_progress(self, text):
224 + short = text.split("\n")[-1]
225 + if len(short) > 50:
226 + short = short[:50] + "..."
227 + progress = f"Browser: {short}"
228 +
229 + self.log.update(progress=text)
230 + self.agent.context.log.set_progress(progress)
231 +
232 + # def __del__(self):
233 + # if self.state:
234 + # self.state.kill_task()
python/tools/browser_do.py new
+64
@@ -0,0 +1,64 @@
1 +import asyncio
2 +from python.helpers.tool import Tool, Response
3 +from python.tools.browser import Browser
4 +from python.helpers.browser import NoPageError
5 +import asyncio
6 +
7 +
8 +class BrowserDo(Browser):
9 +
10 + async def execute(self, fill=[], press=[], click=[], execute="", **kwargs):
11 + await self.prepare_state()
12 + result = ""
13 + try:
14 + if fill:
15 + self.update_progress("Filling fields...")
16 + for f in fill:
17 + await self.state.browser.fill(f["selector"], f["text"])
18 + await self.state.browser.wait(0.5)
19 + if press:
20 + self.update_progress("Pressing keys...")
21 + if fill:
22 + await self.state.browser.wait(1)
23 + for p in press:
24 + await self.state.browser.press(p)
25 + await self.state.browser.wait(0.5)
26 + if click:
27 + self.update_progress("Clicking...")
28 + if fill:
29 + await self.state.browser.wait(1)
30 + for c in click:
31 + await self.state.browser.click(c)
32 + await self.state.browser.wait(0.5)
33 + if execute:
34 + if fill or press or click:
35 + await self.state.browser.wait(1)
36 + self.update_progress("Executing...")
37 + result = await self.state.browser.execute(execute)
38 + self.log.update(result=result)
39 +
40 + self.update_progress("Retrieving...")
41 + await self.state.browser.wait_for_action()
42 + dom = await self.state.browser.get_clean_dom()
43 + if result:
44 + response = f"Result:\n{result}\n\nDOM:\n{dom}"
45 + else:
46 + response = dom
47 + self.update_progress("Taking screenshot...")
48 + screenshot = await self.save_screenshot()
49 + self.log.update(screenshot=screenshot)
50 + except Exception as e:
51 + response = str(e)
52 + self.log.update(error=response)
53 +
54 + try:
55 + screenshot = await self.save_screenshot()
56 + dom = await self.state.browser.get_clean_dom()
57 + response = f"Error:\n{response}\n\nDOM:\n{dom}"
58 + self.log.update(screenshot=screenshot)
59 + except Exception:
60 + pass
61 +
62 + self.cleanup_history()
63 + self.update_progress("Done")
64 + return Response(message=response, break_loop=False)
python/tools/browser_open.py new
+30
@@ -0,0 +1,30 @@
1 +import asyncio
2 +from python.helpers.tool import Tool, Response
3 +from python.tools import browser
4 +from python.tools.browser import Browser
5 +
6 +
7 +class BrowserOpen(Browser):
8 +
9 + async def execute(self, url="", **kwargs):
10 + self.update_progress("Initializing...")
11 + await self.prepare_state()
12 +
13 + try:
14 + if url:
15 + self.update_progress("Opening page...")
16 + await self.state.browser.open(url)
17 +
18 + self.update_progress("Retrieving...")
19 + await self.state.browser.wait_for_action()
20 + response = await self.state.browser.get_clean_dom()
21 + self.update_progress("Taking screenshot...")
22 + screenshot = await self.save_screenshot()
23 + self.log.update(screenshot=screenshot)
24 + except Exception as e:
25 + response = str(e)
26 + self.log.update(error=response)
27 +
28 + self.cleanup_history()
29 + self.update_progress("Done")
30 + return Response(message=response, break_loop=False)
requirements.txt
+11 -9
@@ -1,5 +1,6 @@
1 ansio==0.0.1
2 beautifulsoup4==4.12.3
3 +browser-use==0.1.17
4 docker==7.1.0
5 duckduckgo-search==6.1.12
6 faiss-cpu==1.8.0.post1
@@ -7,23 +8,24 @@ flask[async]==3.0.3
8 flask-basicauth==0.2.0
9 GitPython==3.1.43
10 inputimeout==1.0.4
10 -langchain-anthropic==0.1.19
11 -langchain-community==0.2.7
12 -langchain-google-genai==1.0.7
13 -langchain-groq==0.1.6
14 -langchain-huggingface==0.0.3
15 -langchain-mistralai==0.1.8
16 -langchain-ollama==0.1.3
17 -langchain-openai==0.1.15
11 +langchain-anthropic==0.3.1
12 +langchain-community==0.3.13
13 +langchain-google-genai==2.0.7
14 +langchain-groq==0.2.2
15 +langchain-huggingface==0.1.2
16 +langchain-mistralai==0.2.4
17 +langchain-ollama==0.2.2
18 +langchain-openai==0.2.14
19 openai-whisper==20240930
20 lxml_html_clean==0.3.1
21 markdown==3.7
22 newspaper3k==0.2.8
23 paramiko==3.5.0
24 +playwright==1.49.0
25 pypdf==4.3.1
26 python-dotenv==1.0.1
27 sentence-transformers==3.0.1
28 tiktoken==0.8.0
29 unstructured==0.15.13
30 unstructured-client==0.25.9
29 -webcolors==24.6.0
\ No newline at end of file
31 +webcolors==24.6.0
update_reqs.py new
+38
@@ -0,0 +1,38 @@
1 +import pkg_resources
2 +import re
3 +
4 +def get_installed_version(package_name):
5 + try:
6 + return pkg_resources.get_distribution(package_name).version
7 + except pkg_resources.DistributionNotFound:
8 + return None
9 +
10 +def update_requirements():
11 + with open('requirements.txt', 'r') as f:
12 + requirements = f.readlines()
13 +
14 + updated_requirements = []
15 + for req in requirements:
16 + req = req.strip()
17 + if not req or req.startswith('#'):
18 + updated_requirements.append(req)
19 + continue
20 +
21 + # Extract package name
22 + match = re.match(r'^([^=<>]+)==', req)
23 + if match:
24 + package_name = match.group(1)
25 + current_version = get_installed_version(package_name)
26 + if current_version:
27 + updated_requirements.append(f'{package_name}=={current_version}')
28 + else:
29 + updated_requirements.append(req) # Keep original if package not found
30 + else:
31 + updated_requirements.append(req) # Keep original if pattern doesn't match
32 +
33 + # Write updated requirements
34 + with open('requirements.txt', 'w') as f:
35 + f.write('\n'.join(updated_requirements) + '\n')
36 +
37 +if __name__ == '__main__':
38 + update_requirements()
web_test.py new
+84
@@ -0,0 +1,84 @@
1 +from browser_use import Agent, Browser, BrowserConfig, Controller, ActionResult
2 +from pydantic import BaseModel
3 +import asyncio
4 +
5 +import playwright
6 +import models
7 +from python.helpers import dotenv, files
8 +from playwright.async_api import async_playwright
9 +
10 +
11 +async def main():
12 +
13 + dotenv.load_dotenv()
14 + model = models.get_openai_chat("gpt-4o-mini")
15 +
16 + # Initialize controller first
17 + controller = Controller()
18 +
19 + # @controller.action("Ask user for information")
20 + # def ask_human(question: str, display_question: bool) -> str:
21 + # return input(f"\n{question}\nInput: ")
22 +
23 + class DoneResult(BaseModel):
24 + title: str
25 + response: str
26 + what_do_i_see: str
27 +
28 + # we overwrite done() in this example to demonstrate the validator
29 + @controller.registry.action("Done with task", param_model=DoneResult)
30 + async def done(params: DoneResult):
31 + result = ActionResult(is_done=True, extracted_content=params.model_dump_json())
32 + print(result)
33 + return result
34 +
35 + browser = Browser(
36 + config=BrowserConfig(
37 + headless=False,
38 + disable_security=True,
39 + )
40 + )
41 +
42 + # Await the coroutine to get the browser context
43 + context = await browser.new_context()
44 +
45 + async with context:
46 +
47 + # Add init script to the context - this will be applied to all new pages
48 + pw_context = context.session.context # type: ignore
49 + js_override = files.get_abs_path("lib/browser/init_override.js")
50 + await pw_context.add_init_script(path=js_override) # type: ignore
51 +
52 + agent = Agent(
53 + task="Go to weather.com",
54 + llm=model,
55 + browser=browser,
56 + browser_context=context,
57 + use_vision=True,
58 + controller=controller,
59 + )
60 +
61 + result = await agent.run()
62 + for out in result.model_outputs():
63 + print("-------------")
64 + print(out.current_state.memory)
65 + print(out.current_state.next_goal)
66 + print("-------------")
67 +
68 +
69 + agent = Agent(
70 + task="Search for berlin and tell me the temperature",
71 + llm=model,
72 + browser=browser,
73 + browser_context=context,
74 + use_vision=True,
75 + controller=controller,
76 + )
77 +
78 + result = await agent.run()
79 + # page = await agent.browser_context.get_current_page()
80 + print(result)
81 +
82 +
83 +asyncio.run(main())
84 +
webui/css/history.css
+1 -1
@@ -18,7 +18,7 @@
18 }
19
20 /* Viewer Styles */
21 - #viewer {
21 + .history-viewer {
22 overflow: hidden;
23 margin-bottom: 0.5rem;
24 }
webui/css/modals.css
+1
@@ -91,6 +91,7 @@
91 transition: all 0.3s ease;
92 margin-bottom: 0;
93 padding-bottom: 0;
94 +
95 }
96
97 .modal-content::-webkit-scrollbar {
webui/index.css
+23
@@ -568,6 +568,7 @@ pre {
568 .message-agent-delegation,
569 .message-tool,
570 .message-code-exe,
571 +.message-browser,
572 .message-info,
573 .message-util,
574 .message-warning,
@@ -600,6 +601,10 @@ pre {
601 background-color: #4b3a69;
602 }
603
604 +.message-browser {
605 + background-color: #4b3a69;
606 +}
607 +
608 .message-info {
609 background-color: var(--color-panel);
610 }
@@ -1452,6 +1457,19 @@ input:checked + .slider:before {
1457 white-space: pre-wrap;
1458 }
1459
1460 +.kvps-img {
1461 + width: 8em;
1462 + height: 8em;
1463 + object-fit: cover;
1464 + object-position: top left;
1465 + border-radius: 10%;
1466 + border: 1px solid rgba(255, 255, 255, 0.15);
1467 +}
1468 +
1469 +.image-viewer-img{
1470 + width: 100%;
1471 +}
1472 +
1473 .msg-json {
1474 display: none;
1475 }
@@ -1837,6 +1855,11 @@ input:checked + .slider:before {
1855 color: #6c43b0;
1856 }
1857
1858 +.light-mode .message-browser {
1859 + background-color: #ffffff;
1860 + color: #6c43b0;
1861 +}
1862 +
1863 .light-mode .message-info {
1864 background-color: #f3f3f3;
1865 color: #3f3f3f;
webui/index.html
+1 -1
@@ -438,7 +438,7 @@
438 <template x-for="(section, index) in settings.sections" :key="section.title">
439 <li>
440 <a :href="'#section' + (index + 1)">
441 - <img :src="'/public/' + getIconName(section.title) + '.svg'"
441 + <img :src="'/public/' + section.id +'.svg'"
442 :alt="section.title">
443 <span x-text="section.title"></span>
444 </a>
webui/js/history.js
+1 -1
@@ -29,7 +29,7 @@ async function showEditorModal(data, type = "json", title, description = "") {
29 const html = `<div id="json-viewer-container"></div>`;
30
31 // Open the modal with the generated HTML
32 - await window.genericModalProxy.openModal(title, description, html);
32 + await window.genericModalProxy.openModal(title, description, html, ["history-viewer"]);
33
34 // Initialize the JSON Viewer after the modal is rendered
35 const container = document.getElementById("json-viewer-container");
webui/js/image_modal.js new
+87
@@ -0,0 +1,87 @@
1 +// Singleton interval ID for image refresh
2 +let activeIntervalId = null;
3 +
4 +export async function openImageModal(src, refreshInterval = 0) {
5 + try {
6 + let imgSrc = src;
7 +
8 + // Clear any existing refresh interval
9 + if (activeIntervalId !== null) {
10 + clearInterval(activeIntervalId);
11 + activeIntervalId = null;
12 + }
13 +
14 + if (refreshInterval > 0) {
15 + // Add or update timestamp to bypass cache
16 + const addTimestamp = (url) => {
17 + const urlObj = new URL(url, window.location.origin);
18 + urlObj.searchParams.set('t', Date.now());
19 + return urlObj.toString();
20 + };
21 +
22 + // Check if image viewer is still active
23 + const isImageViewerActive = () => {
24 + const container = document.querySelector('#image-viewer-container');
25 + if (!container) return false;
26 +
27 + // Check if element or any parent is hidden
28 + let element = container;
29 + while (element) {
30 + const style = window.getComputedStyle(element);
31 + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') {
32 + return false;
33 + }
34 + element = element.parentElement;
35 + }
36 + return true;
37 + };
38 +
39 + // Preload next image before displaying
40 + const preloadAndUpdate = async (currentImg) => {
41 + const nextSrc = addTimestamp(src);
42 + // Create a promise that resolves when the image is loaded
43 + const preloadPromise = new Promise((resolve, reject) => {
44 + const tempImg = new Image();
45 + tempImg.onload = () => resolve(nextSrc);
46 + tempImg.onerror = reject;
47 + tempImg.src = nextSrc;
48 + });
49 +
50 + try {
51 + // Wait for preload to complete
52 + const loadedSrc = await preloadPromise;
53 + // Check if this interval is still the active one
54 + if (currentImg && isImageViewerActive()) {
55 + currentImg.src = loadedSrc;
56 + }
57 + } catch (err) {
58 + console.error('Failed to preload image:', err);
59 + }
60 + };
61 +
62 + imgSrc = addTimestamp(src);
63 +
64 + // Set up periodic refresh with preloading
65 + activeIntervalId = setInterval(() => {
66 + if (!isImageViewerActive()) {
67 + clearInterval(activeIntervalId);
68 + activeIntervalId = null;
69 + return;
70 + }
71 + const img = document.querySelector('.image-viewer-img');
72 + if (img) {
73 + preloadAndUpdate(img);
74 + }
75 + }, refreshInterval);
76 + }
77 +
78 + const html = `<div id="image-viewer-container"><img class="image-viewer-img" src="${imgSrc}" /></div>`;
79 + const fileName = src.split("/").pop();
80 +
81 + // Open the modal with the generated HTML
82 + await window.genericModalProxy.openModal(fileName, "", html);
83 + } catch (e) {
84 + window.toastFetchError("Error fetching history", e);
85 + return;
86 + }
87 +}
webui/js/messages.js
+547 -283
@@ -1,362 +1,626 @@
1 // copy button
2 +import { openImageModal } from "./image_modal.js";
3
4 function createCopyButton() {
4 - const button = document.createElement('button');
5 - button.className = 'copy-button';
6 - button.textContent = 'Copy';
7 -
8 - button.addEventListener('click', async function(e) {
9 - e.stopPropagation();
10 - const container = this.closest('.msg-content, .kvps-row, .message-text');
11 - let textToCopy;
12 -
13 - if (container.classList.contains('kvps-row')) {
14 - textToCopy = container.querySelector('.kvps-val').textContent;
15 - } else if (container.classList.contains('message-text')) {
16 - textToCopy = container.textContent.replace('copy', '');
17 - } else {
18 - textToCopy = container.querySelector('span').textContent;
19 - }
20 -
21 - try {
22 - await navigator.clipboard.writeText(textToCopy);
23 - const originalText = button.textContent;
24 - button.classList.add('copied');
25 - button.textContent = 'Copied!';
26 - setTimeout(() => {
27 - button.classList.remove('copied');
28 - button.textContent = originalText;
29 - }, 2000);
30 - } catch (err) {
31 - console.error('Failed to copy text:', err);
32 - }
33 - });
34 -
35 - return button;
5 + const button = document.createElement("button");
6 + button.className = "copy-button";
7 + button.textContent = "Copy";
8 +
9 + button.addEventListener("click", async function (e) {
10 + e.stopPropagation();
11 + const container = this.closest(".msg-content, .kvps-row, .message-text");
12 + let textToCopy;
13 +
14 + if (container.classList.contains("kvps-row")) {
15 + textToCopy = container.querySelector(".kvps-val").textContent;
16 + } else if (container.classList.contains("message-text")) {
17 + textToCopy = container.textContent.replace("copy", "");
18 + } else {
19 + textToCopy = container.querySelector("span").textContent;
20 + }
21 +
22 + try {
23 + await navigator.clipboard.writeText(textToCopy);
24 + const originalText = button.textContent;
25 + button.classList.add("copied");
26 + button.textContent = "Copied!";
27 + setTimeout(() => {
28 + button.classList.remove("copied");
29 + button.textContent = originalText;
30 + }, 2000);
31 + } catch (err) {
32 + console.error("Failed to copy text:", err);
33 + }
34 + });
35 +
36 + return button;
37 }
38
39 function addCopyButtonToElement(element) {
39 - if (!element.querySelector('.copy-button')) {
40 - element.appendChild(createCopyButton());
41 - }
40 + if (!element.querySelector(".copy-button")) {
41 + element.appendChild(createCopyButton());
42 + }
43 }
44
45 export function getHandler(type) {
45 - switch (type) {
46 - case 'user':
47 - return drawMessageUser;
48 - case 'agent':
49 - return drawMessageAgent;
50 - case 'response':
51 - return drawMessageResponse;
52 - case 'tool':
53 - return drawMessageTool;
54 - case 'code_exe':
55 - return drawMessageCodeExe;
56 - case 'warning':
57 - return drawMessageWarning;
58 - case 'rate_limit':
59 - return drawMessageWarning;
60 - case 'error':
61 - return drawMessageError;
62 - case 'info':
63 - return drawMessageInfo;
64 - case 'util':
65 - return drawMessageUtil;
66 - case 'hint':
67 - return drawMessageInfo;
68 - default:
69 - return drawMessageDefault;
70 - }
46 + switch (type) {
47 + case "user":
48 + return drawMessageUser;
49 + case "agent":
50 + return drawMessageAgent;
51 + case "response":
52 + return drawMessageResponse;
53 + case "tool":
54 + return drawMessageTool;
55 + case "code_exe":
56 + return drawMessageCodeExe;
57 + case "browser":
58 + return drawMessageBrowser;
59 + case "warning":
60 + return drawMessageWarning;
61 + case "rate_limit":
62 + return drawMessageWarning;
63 + case "error":
64 + return drawMessageError;
65 + case "info":
66 + return drawMessageInfo;
67 + case "util":
68 + return drawMessageUtil;
69 + case "hint":
70 + return drawMessageInfo;
71 + default:
72 + return drawMessageDefault;
73 + }
74 }
75
73 -
76 // draw a message with a specific type
75 -export function _drawMessage(messageContainer, heading, content, temp, followUp, kvps = null, messageClasses = [], contentClasses = []) {
76 - const messageDiv = document.createElement('div');
77 - messageDiv.classList.add('message', ...messageClasses);
78 -
79 - if (heading) {
80 - const headingElement = document.createElement('h4');
81 - headingElement.textContent = heading;
82 - messageDiv.appendChild(headingElement);
83 - }
77 +export function _drawMessage(
78 + messageContainer,
79 + heading,
80 + content,
81 + temp,
82 + followUp,
83 + kvps = null,
84 + messageClasses = [],
85 + contentClasses = [],
86 + latex = false
87 +) {
88 + const messageDiv = document.createElement("div");
89 + messageDiv.classList.add("message", ...messageClasses);
90 +
91 + if (heading) {
92 + const headingElement = document.createElement("h4");
93 + headingElement.textContent = heading;
94 + messageDiv.appendChild(headingElement);
95 + }
96
85 - drawKvps(messageDiv, kvps);
86 -
87 - if (content && content.trim().length > 0) {
88 - const preElement = document.createElement('pre');
89 - preElement.classList.add("msg-content", ...contentClasses);
90 - preElement.style.whiteSpace = 'pre-wrap';
91 - preElement.style.wordBreak = 'break-word';
92 -
93 - const spanElement = document.createElement('span');
94 - spanElement.innerHTML = content;
95 -
96 - // Add click handler for small screens
97 - spanElement.addEventListener('click', () => {
98 - copyText(spanElement.textContent, spanElement);
99 - });
100 -
101 - preElement.appendChild(spanElement);
102 - addCopyButtonToElement(preElement);
103 - messageDiv.appendChild(preElement);
104 -
105 - // Render LaTeX math within the span
106 - if (window.renderMathInElement) {
107 - renderMathInElement(spanElement, {
108 - delimiters: [
109 - { left: "$", right: "$", display: true },
110 - { left: "\\$", right: "\\$", display: true },
111 - { left: "$", right: "$", display: false },
112 - { left: "\\$", right: "\\$", display: false }
113 - ],
114 - throwOnError: false
115 - });
116 - }
117 - }
97 + drawKvps(messageDiv, kvps, latex);
98
119 - messageContainer.appendChild(messageDiv);
99 + if (content && content.trim().length > 0) {
100 + const preElement = document.createElement("pre");
101 + preElement.classList.add("msg-content", ...contentClasses);
102 + preElement.style.whiteSpace = "pre-wrap";
103 + preElement.style.wordBreak = "break-word";
104
121 - if (followUp) {
122 - messageContainer.classList.add("message-followup");
105 + const spanElement = document.createElement("span");
106 + spanElement.innerHTML = escapeHTML(content);
107 +
108 + // Add click handler for small screens
109 + spanElement.addEventListener("click", () => {
110 + copyText(spanElement.textContent, spanElement);
111 + });
112 +
113 + preElement.appendChild(spanElement);
114 + addCopyButtonToElement(preElement);
115 + messageDiv.appendChild(preElement);
116 +
117 + // Render LaTeX math within the span
118 + if (window.renderMathInElement && latex) {
119 + renderMathInElement(spanElement, {
120 + delimiters: [
121 + { left: "$", right: "$", display: true },
122 + { left: "\\$", right: "\\$", display: true },
123 + { left: "$", right: "$", display: false },
124 + { left: "\\$", right: "\\$", display: false },
125 + ],
126 + throwOnError: false,
127 + });
128 }
129 + }
130
125 - return messageDiv;
126 -}
131 + messageContainer.appendChild(messageDiv);
132
133 + if (followUp) {
134 + messageContainer.classList.add("message-followup");
135 + }
136
129 -export function drawMessageDefault(messageContainer, id, type, heading, content, temp, kvps = null) {
130 - const messageContent = convertImageTags(content); // Convert image tags
131 - _drawMessage(messageContainer, heading, messageContent, temp, false, kvps, ['message-ai', 'message-default'], ['msg-json']);
137 + return messageDiv;
138 }
139
134 -export function drawMessageAgent(messageContainer, id, type, heading, content, temp, kvps = null) {
135 - let kvpsFlat = null;
136 - if (kvps) {
137 - kvpsFlat = { ...kvps, ...kvps['tool_args'] || {} };
138 - delete kvpsFlat['tool_args'];
139 - }
140 -
141 - const messageContent = convertImageTags(content); // Convert image tags
142 - _drawMessage(messageContainer, heading, messageContent, temp, false, kvpsFlat, ['message-ai', 'message-agent'], ['msg-json']);
140 +export function drawMessageDefault(
141 + messageContainer,
142 + id,
143 + type,
144 + heading,
145 + content,
146 + temp,
147 + kvps = null
148 +) {
149 + const messageContent = convertImageTags(content); // Convert image tags
150 + _drawMessage(
151 + messageContainer,
152 + heading,
153 + messageContent,
154 + temp,
155 + false,
156 + kvps,
157 + ["message-ai", "message-default"],
158 + ["msg-json"]
159 + );
160 }
161
145 -export function drawMessageResponse(messageContainer, id, type, heading, content, temp, kvps = null) {
146 - const messageContent = convertImageTags(content); // Convert image tags
147 - _drawMessage(messageContainer, heading, messageContent, temp, true, null, ['message-ai', 'message-agent-response']);
162 +export function drawMessageAgent(
163 + messageContainer,
164 + id,
165 + type,
166 + heading,
167 + content,
168 + temp,
169 + kvps = null
170 +) {
171 + let kvpsFlat = null;
172 + if (kvps) {
173 + kvpsFlat = { ...kvps, ...(kvps["tool_args"] || {}) };
174 + delete kvpsFlat["tool_args"];
175 + }
176 +
177 + const messageContent = convertImageTags(content); // Convert image tags
178 + _drawMessage(
179 + messageContainer,
180 + heading,
181 + messageContent,
182 + temp,
183 + false,
184 + kvpsFlat,
185 + ["message-ai", "message-agent"],
186 + ["msg-json"],
187 + true
188 + );
189 }
190
150 -export function drawMessageDelegation(messageContainer, id, type, heading, content, temp, kvps = null) {
151 - const messageContent = convertImageTags(content); // Convert image tags
152 - _drawMessage(messageContainer, heading, messageContent, temp, true, kvps, ['message-ai', 'message-agent', 'message-agent-delegation']);
191 +export function drawMessageResponse(
192 + messageContainer,
193 + id,
194 + type,
195 + heading,
196 + content,
197 + temp,
198 + kvps = null
199 +) {
200 + const messageContent = convertImageTags(content); // Convert image tags
201 + _drawMessage(
202 + messageContainer,
203 + heading,
204 + messageContent,
205 + temp,
206 + true,
207 + null,
208 + ["message-ai", "message-agent-response"],
209 + [],
210 + true
211 + );
212 }
213
155 -export function drawMessageUser(messageContainer, id, type, heading, content, temp, kvps = null) {
156 - const messageDiv = document.createElement('div');
157 - messageDiv.classList.add('message', 'message-user');
214 +export function drawMessageDelegation(
215 + messageContainer,
216 + id,
217 + type,
218 + heading,
219 + content,
220 + temp,
221 + kvps = null
222 +) {
223 + const messageContent = convertImageTags(content); // Convert image tags
224 + _drawMessage(
225 + messageContainer,
226 + heading,
227 + messageContent,
228 + temp,
229 + true,
230 + kvps,
231 + ["message-ai", "message-agent", "message-agent-delegation"],
232 + [],
233 + true
234 + );
235 +}
236
159 - const headingElement = document.createElement('h4');
160 - headingElement.textContent = "User message";
161 - messageDiv.appendChild(headingElement);
237 +export function drawMessageUser(
238 + messageContainer,
239 + id,
240 + type,
241 + heading,
242 + content,
243 + temp,
244 + kvps = null,
245 + latex = true
246 +) {
247 + const messageDiv = document.createElement("div");
248 + messageDiv.classList.add("message", "message-user");
249 +
250 + const headingElement = document.createElement("h4");
251 + headingElement.textContent = "User message";
252 + messageDiv.appendChild(headingElement);
253 +
254 + if (content && content.trim().length > 0) {
255 + const textDiv = document.createElement("div");
256 + textDiv.classList.add("message-text");
257 + textDiv.textContent = content;
258 +
259 + // Add click handler
260 + textDiv.addEventListener("click", () => {
261 + copyText(content, textDiv);
262 + });
263
163 - if (content && content.trim().length > 0) {
164 - const textDiv = document.createElement('div');
165 - textDiv.classList.add('message-text');
166 - textDiv.textContent = content;
167 -
168 - // Add click handler
169 - textDiv.addEventListener('click', () => {
170 - copyText(content, textDiv);
171 - });
172 -
173 - addCopyButtonToElement(textDiv);
174 - messageDiv.appendChild(textDiv);
175 - }
264 + addCopyButtonToElement(textDiv);
265 + messageDiv.appendChild(textDiv);
266 + }
267
177 - // Handle attachments
178 - if (kvps && kvps.attachments && kvps.attachments.length > 0) {
179 - const attachmentsContainer = document.createElement('div');
180 - attachmentsContainer.classList.add('attachments-container');
268 + // Handle attachments
269 + if (kvps && kvps.attachments && kvps.attachments.length > 0) {
270 + const attachmentsContainer = document.createElement("div");
271 + attachmentsContainer.classList.add("attachments-container");
272
182 - kvps.attachments.forEach(attachment => {
183 - const attachmentDiv = document.createElement('div');
184 - attachmentDiv.classList.add('attachment-item');
273 + kvps.attachments.forEach((attachment) => {
274 + const attachmentDiv = document.createElement("div");
275 + attachmentDiv.classList.add("attachment-item");
276
186 - if (typeof attachment === 'string') {
187 - // attachment is filename
188 - const filename = attachment;
189 - const extension = filename.split('.').pop().toUpperCase();
277 + if (typeof attachment === "string") {
278 + // attachment is filename
279 + const filename = attachment;
280 + const extension = filename.split(".").pop().toUpperCase();
281
191 - attachmentDiv.classList.add('file-type');
192 - attachmentDiv.innerHTML = `
282 + attachmentDiv.classList.add("file-type");
283 + attachmentDiv.innerHTML = `
284 <div class="file-preview">
285 <span class="filename">${filename}</span>
286 <span class="extension">${extension}</span>
287 </div>
288 `;
198 - } else if (attachment.type === 'image') {
199 - // Existing logic for images
200 - const imgWrapper = document.createElement('div');
201 - imgWrapper.classList.add('image-wrapper');
202 -
203 - const img = document.createElement('img');
204 - img.src = attachment.url;
205 - img.alt = attachment.name;
206 - img.classList.add('attachment-preview');
207 -
208 - const fileInfo = document.createElement('div');
209 - fileInfo.classList.add('file-info');
210 - fileInfo.innerHTML = `
289 + } else if (attachment.type === "image") {
290 + // Existing logic for images
291 + const imgWrapper = document.createElement("div");
292 + imgWrapper.classList.add("image-wrapper");
293 +
294 + const img = document.createElement("img");
295 + img.src = attachment.url;
296 + img.alt = attachment.name;
297 + img.classList.add("attachment-preview");
298 +
299 + const fileInfo = document.createElement("div");
300 + fileInfo.classList.add("file-info");
301 + fileInfo.innerHTML = `
302 <span class="filename">${attachment.name}</span>
303 <span class="extension">${attachment.extension.toUpperCase()}</span>
304 `;
305
215 - imgWrapper.appendChild(img);
216 - attachmentDiv.appendChild(imgWrapper);
217 - attachmentDiv.appendChild(fileInfo);
218 - } else {
219 - // Existing logic for non-image files
220 - attachmentDiv.classList.add('file-type');
221 - attachmentDiv.innerHTML = `
306 + imgWrapper.appendChild(img);
307 + attachmentDiv.appendChild(imgWrapper);
308 + attachmentDiv.appendChild(fileInfo);
309 + } else {
310 + // Existing logic for non-image files
311 + attachmentDiv.classList.add("file-type");
312 + attachmentDiv.innerHTML = `
313 <div class="file-preview">
314 <span class="filename">${attachment.name}</span>
315 <span class="extension">${attachment.extension.toUpperCase()}</span>
316 </div>
317 `;
227 - }
318 + }
319
229 - attachmentsContainer.appendChild(attachmentDiv);
230 - });
320 + attachmentsContainer.appendChild(attachmentDiv);
321 + });
322
232 - messageDiv.appendChild(attachmentsContainer);
233 - }
323 + messageDiv.appendChild(attachmentsContainer);
324 + }
325 +
326 + messageContainer.appendChild(messageDiv);
327 +}
328
235 - messageContainer.appendChild(messageDiv);
329 +export function drawMessageTool(
330 + messageContainer,
331 + id,
332 + type,
333 + heading,
334 + content,
335 + temp,
336 + kvps = null
337 +) {
338 + const messageContent = convertImageTags(content); // Convert image tags
339 + _drawMessage(
340 + messageContainer,
341 + heading,
342 + messageContent,
343 + temp,
344 + true,
345 + kvps,
346 + ["message-ai", "message-tool"],
347 + ["msg-output"]
348 + );
349 }
350
238 -export function drawMessageTool(messageContainer, id, type, heading, content, temp, kvps = null) {
239 - const messageContent = convertImageTags(content); // Convert image tags
240 - _drawMessage(messageContainer, heading, messageContent, temp, true, kvps, ['message-ai', 'message-tool'], ['msg-output']);
351 +export function drawMessageCodeExe(
352 + messageContainer,
353 + id,
354 + type,
355 + heading,
356 + content,
357 + temp,
358 + kvps = null
359 +) {
360 + const messageContent = convertImageTags(content); // Convert image tags
361 + _drawMessage(messageContainer, heading, messageContent, temp, true, null, [
362 + "message-ai",
363 + "message-code-exe",
364 + ]);
365 }
366
243 -export function drawMessageCodeExe(messageContainer, id, type, heading, content, temp, kvps = null) {
244 - const messageContent = convertImageTags(content); // Convert image tags
245 - _drawMessage(messageContainer, heading, messageContent, temp, true, null, ['message-ai', 'message-code-exe']);
367 +export function drawMessageBrowser(
368 + messageContainer,
369 + id,
370 + type,
371 + heading,
372 + content,
373 + temp,
374 + kvps = null
375 +) {
376 + const messageContent = convertImageTags(content); // Convert image tags
377 + _drawMessage(
378 + messageContainer,
379 + heading,
380 + messageContent,
381 + temp,
382 + true,
383 + kvps,
384 + ["message-ai", "message-browser"],
385 + ["msg-json"]
386 + );
387 }
388
248 -export function drawMessageAgentPlain(classes, messageContainer, id, type, heading, content, temp, kvps = null) {
249 - const messageContent = convertImageTags(content); // Convert image tags
250 - _drawMessage(messageContainer, heading, messageContent, temp, false, kvps, [...classes]);
251 - messageContainer.classList.add('center-container');
389 +export function drawMessageAgentPlain(
390 + classes,
391 + messageContainer,
392 + id,
393 + type,
394 + heading,
395 + content,
396 + temp,
397 + kvps = null
398 +) {
399 + const messageContent = convertImageTags(content); // Convert image tags
400 + _drawMessage(messageContainer, heading, messageContent, temp, false, kvps, [
401 + ...classes,
402 + ]);
403 + messageContainer.classList.add("center-container");
404 }
405
254 -export function drawMessageInfo(messageContainer, id, type, heading, content, temp, kvps = null) {
255 - return drawMessageAgentPlain(['message-info'], messageContainer, id, type, heading, content, temp, kvps);
406 +export function drawMessageInfo(
407 + messageContainer,
408 + id,
409 + type,
410 + heading,
411 + content,
412 + temp,
413 + kvps = null
414 +) {
415 + return drawMessageAgentPlain(
416 + ["message-info"],
417 + messageContainer,
418 + id,
419 + type,
420 + heading,
421 + content,
422 + temp,
423 + kvps
424 + );
425 }
426
258 -export function drawMessageUtil(messageContainer, id, type, heading, content, temp, kvps = null) {
259 - const messageContent = convertImageTags(content); // Convert image tags
260 - _drawMessage(messageContainer, heading, messageContent, temp, false, kvps, ['message-util'], ['msg-json']);
261 - messageContainer.classList.add('center-container');
427 +export function drawMessageUtil(
428 + messageContainer,
429 + id,
430 + type,
431 + heading,
432 + content,
433 + temp,
434 + kvps = null
435 +) {
436 + const messageContent = convertImageTags(content); // Convert image tags
437 + _drawMessage(
438 + messageContainer,
439 + heading,
440 + messageContent,
441 + temp,
442 + false,
443 + kvps,
444 + ["message-util"],
445 + ["msg-json"]
446 + );
447 + messageContainer.classList.add("center-container");
448 }
449
264 -export function drawMessageWarning(messageContainer, id, type, heading, content, temp, kvps = null) {
265 - return drawMessageAgentPlain(['message-warning'], messageContainer, id, type, heading, content, temp, kvps);
450 +export function drawMessageWarning(
451 + messageContainer,
452 + id,
453 + type,
454 + heading,
455 + content,
456 + temp,
457 + kvps = null
458 +) {
459 + return drawMessageAgentPlain(
460 + ["message-warning"],
461 + messageContainer,
462 + id,
463 + type,
464 + heading,
465 + content,
466 + temp,
467 + kvps
468 + );
469 }
470
268 -export function drawMessageError(messageContainer, id, type, heading, content, temp, kvps = null) {
269 - return drawMessageAgentPlain(['message-error'], messageContainer, id, type, heading, content, temp, kvps);
471 +export function drawMessageError(
472 + messageContainer,
473 + id,
474 + type,
475 + heading,
476 + content,
477 + temp,
478 + kvps = null
479 +) {
480 + return drawMessageAgentPlain(
481 + ["message-error"],
482 + messageContainer,
483 + id,
484 + type,
485 + heading,
486 + content,
487 + temp,
488 + kvps
489 + );
490 }
491
272 -function drawKvps(container, kvps) {
273 - if (kvps) {
274 - const table = document.createElement('table');
275 - table.classList.add('msg-kvps');
276 - for (let [key, value] of Object.entries(kvps)) {
277 - const row = table.insertRow();
278 - row.classList.add('kvps-row');
279 - if (key === "thoughts" || key === "reflection") row.classList.add('msg-thoughts');
280 -
281 - const th = row.insertCell();
282 - th.textContent = convertToTitleCase(key);
283 - th.classList.add('kvps-key');
284 -
285 - const td = row.insertCell();
286 - const pre = document.createElement('pre');
287 - pre.classList.add('kvps-val');
288 -
289 - if (Array.isArray(value)) value = value.join('\n');
290 -
291 - if (row.classList.contains('msg-thoughts')) {
292 - const span = document.createElement('span');
293 - span.innerHTML = value;
294 - pre.appendChild(span);
295 - td.appendChild(pre);
296 - addCopyButtonToElement(row);
297 -
298 - // Add click handler
299 - span.addEventListener('click', () => {
300 - copyText(span.textContent, span);
301 - });
302 -
303 - if (window.renderMathInElement) {
304 - renderMathInElement(span, {
305 - delimiters: [
306 - { left: "$$", right: "$$", display: true },
307 - { left: "\$$", right: "\$$", display: true },
308 - { left: "$", right: "$", display: false },
309 - { left: "\$$", right: "\$$", display: false }
310 - ],
311 - throwOnError: false
312 - });
313 - }
314 - } else {
315 - pre.textContent = value;
316 -
317 - // Add click handler
318 - pre.addEventListener('click', () => {
319 - copyText(value, pre);
320 - });
321 -
322 - td.appendChild(pre);
323 - addCopyButtonToElement(row);
324 - }
492 +function drawKvps(container, kvps, latex) {
493 + if (kvps) {
494 + const table = document.createElement("table");
495 + table.classList.add("msg-kvps");
496 + for (let [key, value] of Object.entries(kvps)) {
497 + const row = table.insertRow();
498 + row.classList.add("kvps-row");
499 + if (key === "thoughts" || key === "reflection")
500 + row.classList.add("msg-thoughts");
501 +
502 + const th = row.insertCell();
503 + th.textContent = convertToTitleCase(key);
504 + th.classList.add("kvps-key");
505 +
506 + const td = row.insertCell();
507 +
508 + if (Array.isArray(value)) {
509 + for (const item of value) {
510 + addValue(item);
511 + }
512 + } else {
513 + addValue(value);
514 + }
515 +
516 + function addValue(value) {
517 + if (typeof value === "object") value = JSON.stringify(value, null, 2);
518 +
519 + if (typeof value === "string" && value.startsWith("img://")) {
520 + const imgElement = document.createElement("img");
521 + imgElement.classList.add("kvps-img");
522 + imgElement.src = value.replace("img://", "/image_get?path=");
523 + imgElement.alt = "Image Attachment";
524 + td.appendChild(imgElement);
525 +
526 + // Add click handler and cursor change
527 + imgElement.style.cursor = "pointer";
528 + imgElement.addEventListener("click", () => {
529 + openImageModal(imgElement.src, 1000);
530 + });
531 +
532 + td.appendChild(imgElement);
533 + } else {
534 + const pre = document.createElement("pre");
535 + pre.classList.add("kvps-val");
536 + // if (row.classList.contains("msg-thoughts")) {
537 + const span = document.createElement("span");
538 + span.innerHTML = escapeHTML(value);
539 + pre.appendChild(span);
540 + td.appendChild(pre);
541 + addCopyButtonToElement(row);
542 +
543 + // Add click handler
544 + span.addEventListener("click", () => {
545 + copyText(span.textContent, span);
546 + });
547 +
548 + if (window.renderMathInElement && latex) {
549 + renderMathInElement(span, {
550 + delimiters: [
551 + { left: "$$", right: "$$", display: true },
552 + { left: "$$", right: "$$", display: true },
553 + { left: "$", right: "$", display: false },
554 + { left: "$$", right: "$$", display: false },
555 + ],
556 + throwOnError: false,
557 + });
558 + }
559 }
326 - container.appendChild(table);
560 + }
561 + // } else {
562 + // pre.textContent = value;
563 +
564 + // // Add click handler
565 + // pre.addEventListener("click", () => {
566 + // copyText(value, pre);
567 + // });
568 +
569 + // td.appendChild(pre);
570 + // addCopyButtonToElement(row);
571 + // }
572 }
573 + container.appendChild(table);
574 + }
575 }
576
577 function convertToTitleCase(str) {
331 - return str
332 - .replace(/_/g, ' ') // Replace underscores with spaces
333 - .toLowerCase() // Convert the entire string to lowercase
334 - .replace(/\b\w/g, function (match) {
335 - return match.toUpperCase(); // Capitalize the first letter of each word
336 - });
578 + return str
579 + .replace(/_/g, " ") // Replace underscores with spaces
580 + .toLowerCase() // Convert the entire string to lowercase
581 + .replace(/\b\w/g, function (match) {
582 + return match.toUpperCase(); // Capitalize the first letter of each word
583 + });
584 }
585
339 -
586 function convertImageTags(content) {
341 - // Regular expression to match <image> tags and extract base64 content
342 - const imageTagRegex = /<image>(.*?)<\/image>/g;
343 -
344 - // Replace <image> tags with <img> tags with base64 source
345 - const updatedContent = content.replace(imageTagRegex, (match, base64Content) => {
346 - return `<img src="data:image/jpeg;base64,${base64Content}" alt="Image Attachment" style="max-width: 250px !important;"/>`;
347 - });
587 + // Regular expression to match <image> tags and extract base64 content
588 + const imageTagRegex = /<image>(.*?)<\/image>/g;
589 +
590 + // Replace <image> tags with <img> tags with base64 source
591 + const updatedContent = content.replace(
592 + imageTagRegex,
593 + (match, base64Content) => {
594 + return `<img src="data:image/jpeg;base64,${base64Content}" alt="Image Attachment" style="max-width: 250px !important;"/>`;
595 + }
596 + );
597
349 - return updatedContent;
598 + return updatedContent;
599 }
600
601 async function copyText(text, element) {
353 - try {
354 - await navigator.clipboard.writeText(text);
355 - element.classList.add('copied');
356 - setTimeout(() => {
357 - element.classList.remove('copied');
358 - }, 2000);
359 - } catch (err) {
360 - console.error('Failed to copy text:', err);
361 - }
602 + try {
603 + await navigator.clipboard.writeText(text);
604 + element.classList.add("copied");
605 + setTimeout(() => {
606 + element.classList.remove("copied");
607 + }, 2000);
608 + } catch (err) {
609 + console.error("Failed to copy text:", err);
610 + }
611 +}
612 +
613 +function escapeHTML(str) {
614 + if (typeof str !== "string") {
615 + return str;
616 + }
617 +
618 + const escapeChars = {
619 + "&": "&amp;",
620 + "<": "&lt;",
621 + ">": "&gt;",
622 + "'": "&#39;",
623 + '"': "&quot;",
624 + };
625 + return str.replace(/[&<>'"]/g, (char) => escapeChars[char]);
626 }
webui/js/modal.js
+5 -1
@@ -39,14 +39,18 @@ const genericModalProxy = {
39 description: '',
40 html: '',
41
42 - async openModal(title, description, html) {
42 + async openModal(title, description, html, contentClasses = []) {
43 const modalEl = document.getElementById('genericModal');
44 + const modalContent = document.getElementById('viewer');
45 const modalAD = Alpine.$data(modalEl);
46
47 modalAD.isOpen = true;
48 modalAD.title = title
49 modalAD.description = description
50 modalAD.html = html
51 +
52 + modalContent.className = 'modal-content';
53 + modalContent.classList.add(...contentClasses);
54 },
55
56 handleClose() {
webui/js/settings.js
-15
@@ -97,18 +97,3 @@ const settingsModalProxy = {
97 // document.addEventListener('alpine:init', () => {
98 // Alpine.store('settingsModal', initSettingsModal());
99 // });
100 -
101 -function getIconName(title) {
102 - const iconMap = {
103 - 'Agent Config': 'agentconfig',
104 - 'Chat Model': 'chat-model',
105 - 'Utility model': 'utility-model',
106 - 'Embedding Model': 'embed-model',
107 - 'Speech to Text': 'voice',
108 - 'API Keys': 'api-keys',
109 - 'Authentication': 'auth',
110 - 'Development': 'dev'
111 - };
112 - return iconMap[title] || 'default';
113 -}
114 -
webui/public/agent.svg renamed
webui/public/api_keys.svg renamed
webui/public/chat_model.svg renamed
webui/public/embed_model.svg renamed
webui/public/stt.svg renamed
webui/public/util_model.svg renamed