Squashed commit of the following:
commit b05d44bb4bc9e07cfc0b584ab39e8624bae771fb Author: frdel <38891707+frdel@users.noreply.github.com> Date: Sun Nov 17 23:12:00 2024 +0100 searxng, RFC, docker runtime commit c90fd4026e644d22e6c7dc29639c85eee6026828 Author: frdel <38891707+frdel@users.noreply.github.com> Date: Sat Nov 16 21:21:49 2024 +0100 Remote function calling commit f71d45ec7dbff4e2d3209f0efe97804f6e602fe7 Author: frdel <38891707+frdel@users.noreply.github.com> Date: Fri Nov 15 13:13:09 2024 +0100 Fix for bool arg parsing commit 936768d1d8efc9060494334b87f400c933d78048 Author: frdel <38891707+frdel@users.noreply.github.com> Date: Fri Nov 15 13:01:28 2024 +0100 Dynamic runtime args parsing commit 00c915fc6c1f8f00f8176fbf5b77af32fa312d18 Author: frdel <38891707+frdel@users.noreply.github.com> Date: Fri Nov 15 12:13:58 2024 +0100 API key fix commit 504a7f91789caa16578af8bae9b7936a9d7fbbb7 Author: frdel <38891707+frdel@users.noreply.github.com> Date: Fri Nov 15 11:59:41 2024 +0100 API keys JIT loading commit 5678a2fce2d333454bb1a2e94ca2b5916d321b41 Author: frdel <38891707+frdel@users.noreply.github.com> Date: Fri Nov 15 11:27:12 2024 +0100 Update dotenv.py
frdel committed
Nov 17, 2024 at 23:12 UTC
9c7339042f1cfc1e67d3656b876a63f1a87065ff
40 files changed
+1439
-546
.gitignore
+1
-1
@@ -65,4 +65,4 @@ instruments/**/*.*
65
# Ignore all contents of the directory "bin"
66
bin/*
67
# But do not ignore the directory itself
68
-!bin/.gitkeep
\ No newline at end of file
68
+!bin/.gitkeep
.vscode/launch.json
+28
-28
@@ -1,29 +1,29 @@
1
{
2
- "version": "0.2.0",
3
- "configurations": [
4
- {
5
- "name": "Debug run_ui.py",
6
- "type": "debugpy",
7
- "request": "launch",
8
- "program": "./run_ui.py",
9
- "console": "integratedTerminal",
10
- "args": ["-Xfrozen_modules=off"]
11
- },
12
- {
13
- "name": "Debug run_cli.py",
14
- "type": "debugpy",
15
- "request": "launch",
16
- "program": "./run_cli.py",
17
- "console": "integratedTerminal",
18
- "args": ["-Xfrozen_modules=off"]
19
- },
20
- {
21
- "name": "Debug current file",
22
- "type": "debugpy",
23
- "request": "launch",
24
- "program": "${file}",
25
- "console": "integratedTerminal",
26
- "args": ["-Xfrozen_modules=off"]
27
- }
28
- ]
29
-}
\ No newline at end of file
2
+ "version": "0.2.0",
3
+ "configurations": [
4
+ {
5
+ "name": "Debug run_ui.py",
6
+ "type": "debugpy",
7
+ "request": "launch",
8
+ "program": "./run_ui.py",
9
+ "console": "integratedTerminal",
10
+ "args": ["--development=true", "-Xfrozen_modules=off"]
11
+ },
12
+ {
13
+ "name": "Debug run_cli.py",
14
+ "type": "debugpy",
15
+ "request": "launch",
16
+ "program": "./run_cli.py",
17
+ "console": "integratedTerminal",
18
+ "args": ["--development=true", "-Xfrozen_modules=off"]
19
+ },
20
+ {
21
+ "name": "Debug current file",
22
+ "type": "debugpy",
23
+ "request": "launch",
24
+ "program": "${file}",
25
+ "console": "integratedTerminal",
26
+ "args": ["--development=true", "-Xfrozen_modules=off"]
27
+ }
28
+ ]
29
+}
agent.py
+1
-1
@@ -457,7 +457,7 @@ class Agent:
457
printer = PrintStyle(italic=True, font_color="orange", padding=False)
458
459
def log_callback(content):
460
- printer.print(content)
460
+ printer.stream(content)
461
log_item.stream(content=content)
462
463
summary = await self.call_utility_llm(
docker/docker_manager_mac_linux.sh
renamed
docker/run/Dockerfile
+18
-11
@@ -21,9 +21,6 @@ RUN apt-get update && apt-get install -y \
21
# Cleanup package list
22
RUN rm -rf /var/lib/apt/lists/*
23
24
-# Clone Agent Zero repository
25
-RUN git clone --branch development https://github.com/frdel/agent-zero.git /git/agent-zero
26
-
24
# Set up SSH
25
RUN mkdir /var/run/sshd && \
26
echo 'root:toor' | chpasswd && \
@@ -33,12 +30,8 @@ RUN mkdir /var/run/sshd && \
30
ENV VIRTUAL_ENV=/opt/venv
31
RUN python3 -m venv $VIRTUAL_ENV
32
36
-# Copy the script to ensure .bashrc is in the root directory
37
-COPY initialize.sh /usr/local/bin/initialize.sh
38
-RUN chmod +x /usr/local/bin/initialize.sh
39
-
40
-# Copy contents of filesystem directory to /fs
41
-COPY ./fs/ /fs
33
+# Copy contents of filesystem directory to /
34
+COPY ./fs/ /
35
36
# Ensure the virtual environment and pip setup
37
RUN $VIRTUAL_ENV/bin/pip install --upgrade pip
@@ -48,13 +41,27 @@ RUN $VIRTUAL_ENV/bin/pip install \
41
ipython \
42
requests
43
44
+# Clone Agent Zero repository
45
+RUN git clone --branch development https://github.com/frdel/agent-zero.git /git/agent-zero
46
+
47
# Install A0 python packages
48
RUN $VIRTUAL_ENV/bin/pip install -r /git/agent-zero/requirements.txt
49
# Preload A0
50
RUN $VIRTUAL_ENV/bin/python /git/agent-zero/preload.py
51
52
+
53
+
54
+# Clone additional git repos
55
+RUN git clone https://github.com/searxng/searxng.git /git/searxng
56
+
57
+
58
+# install additional software
59
+RUN bash /ins/install_searxng.sh
60
+
61
+
62
+
63
# Expose ports
64
EXPOSE 22 80
65
59
-# Init .bashrc
60
-CMD ["/usr/local/bin/initialize.sh"]
\ No newline at end of file
66
+# initialize runtime
67
+CMD ["/bin/bash", "/exe/initialize.sh"]
\ No newline at end of file
docker/run/build.txt
+1
-1
@@ -3,4 +3,4 @@ docker build -t agent-zero-run:latest .
3
4
# dockerhub
5
docker login
6
-docker buildx build --platform linux/amd64,linux/arm64 -t frdel/agent-zero-run:latest --push .
\ No newline at end of file
6
+docker buildx build --platform linux/amd64,linux/arm64 -t frdel/agent-zero-run:testing --push .
\ No newline at end of file
docker/run/fs/etc/searxng/settings.yml
new
+77
@@ -0,0 +1,77 @@
1
+# SearXNG settings
2
+
3
+use_default_settings: true
4
+
5
+general:
6
+ debug: false
7
+ instance_name: "SearXNG"
8
+
9
+search:
10
+ safe_search: 2
11
+ autocomplete: 'duckduckgo'
12
+ formats:
13
+ - html
14
+ - json
15
+
16
+server:
17
+ # Is overwritten by ${SEARXNG_SECRET}
18
+ secret_key: "1e91dce279fdef87eb7b6f43029c206b"
19
+ limiter: true
20
+ image_proxy: true
21
+ # public URL of the instance, to ensure correct inbound links. Is overwritten
22
+ # by ${SEARXNG_URL}.
23
+ # base_url: http://example.com/location
24
+
25
+redis:
26
+ # URL to connect redis database. Is overwritten by ${SEARXNG_REDIS_URL}.
27
+ url: unix:///usr/local/searxng-redis/run/redis.sock?db=0
28
+
29
+ui:
30
+ static_use_hash: true
31
+
32
+# preferences:
33
+# lock:
34
+# - autocomplete
35
+# - method
36
+
37
+enabled_plugins:
38
+ - 'Hash plugin'
39
+ - 'Self Informations'
40
+ - 'Tracker URL remover'
41
+ - 'Ahmia blacklist'
42
+ # - 'Hostnames plugin' # see 'hostnames' configuration below
43
+ # - 'Open Access DOI rewrite'
44
+
45
+# plugins:
46
+# - only_show_green_results
47
+
48
+# hostnames:
49
+# replace:
50
+# '(.*\.)?youtube\.com$': 'invidious.example.com'
51
+# '(.*\.)?youtu\.be$': 'invidious.example.com'
52
+# remove:
53
+# - '(.*\.)?facebook.com$'
54
+# low_priority:
55
+# - '(.*\.)?google\.com$'
56
+# high_priority:
57
+# - '(.*\.)?wikipedia.org$'
58
+
59
+engines:
60
+
61
+# - name: fdroid
62
+# disabled: false
63
+#
64
+# - name: apk mirror
65
+# disabled: false
66
+#
67
+# - name: mediathekviewweb
68
+# categories: TV
69
+# disabled: false
70
+#
71
+# - name: invidious
72
+# disabled: false
73
+# base_url:
74
+# - https://invidious.snopyta.org
75
+# - https://invidious.tiekoetter.com
76
+# - https://invidio.xamh.de
77
+# - https://inv.riverside.rocks
\ No newline at end of file
docker/run/fs/exe/initialize.sh
renamed
+6
-3
@@ -1,18 +1,21 @@
1
#!/bin/bash
2
3
-# Copy all contents from /fs to root directory (/) without overwriting
4
-cp -rn --no-preserve=ownership,mode /fs/* /
3
+# Copy all contents from persistent /per to root directory (/) without overwriting
4
+cp -r --no-preserve=ownership,mode /per/* /
5
6
# allow execution of /root/.bashrc and /root/.profile
7
chmod 444 /root/.bashrc
8
chmod 444 /root/.profile
9
10
# update package list to save time later
11
-apt-get update
11
+apt-get update &
12
13
# Start SSH service in background
14
/usr/sbin/sshd -D &
15
16
+# Start searxng server in background
17
+sudo -H -u searxng -i bash /exe/run_searxng.sh &
18
+
19
# Start A0 and restart on exit
20
bash /exe/run_A0.sh
21
if [ $? -ne 0 ]; then
docker/run/fs/exe/run_A0.sh
+8
-8
@@ -34,14 +34,14 @@ while true; do
34
35
echo "Starting A0..."
36
python /a0/run_ui.py \
37
- --port 80 \
38
- --host "0.0.0.0" \
39
- --code_exec_docker_enabled False \
40
- --code_exec_ssh_enabled True \
41
- --code_exec_ssh_addr "localhost" \
42
- --code_exec_ssh_port 22 \
43
- --code_exec_ssh_user "root" \
44
- --code_exec_ssh_pass "toor"
37
+ --port=80 \
38
+ --host="0.0.0.0" \
39
+ --code_exec_docker_enabled=false \
40
+ --code_exec_ssh_enabled=true \
41
+ --code_exec_ssh_addr="localhost" \
42
+ --code_exec_ssh_port=22 \
43
+ --code_exec_ssh_user="root" \
44
+ --code_exec_ssh_pass="toor"
45
46
# Check the exit status
47
if [ $? -ne 0 ]; then
docker/run/fs/exe/run_searxng.sh
new
+7
@@ -0,0 +1,7 @@
1
+#!/bin/bash
2
+
3
+# start webapp
4
+sudo -H -u searxng -i
5
+cd /usr/local/searxng/searxng-src
6
+export SEARXNG_SETTINGS_PATH="/etc/searxng/settings.yml"
7
+python searx/webapp.py
\ No newline at end of file
docker/run/fs/ins/install_searxng.sh
new
+17
@@ -0,0 +1,17 @@
1
+#!/bin/bash
2
+
3
+sudo -H apt-get install -y \
4
+ python3-dev python3-babel python3-venv \
5
+ uwsgi uwsgi-plugin-python3 \
6
+ git build-essential libxslt-dev zlib1g-dev libffi-dev libssl-dev
7
+
8
+sudo -H useradd --shell /bin/bash --system \
9
+ --home-dir "/usr/local/searxng" \
10
+ --comment 'Privacy-respecting metasearch engine' \
11
+ searxng
12
+
13
+sudo -H mkdir "/usr/local/searxng"
14
+sudo -H chown -R "searxng:searxng" "/usr/local/searxng"
15
+
16
+# Start a new shell from new created user and clone SearXNG:
17
+sudo -H -u searxng -i bash /ins/install_searxng_user.sh
docker/run/fs/ins/install_searxng_user.sh
new
+26
@@ -0,0 +1,26 @@
1
+#!/bin/bash
2
+
3
+# clone SearXNG repo
4
+# git clone "https://github.com/searxng/searxng" \
5
+# "/usr/local/searxng/searxng-src"
6
+cp -r "/git/searxng" "/usr/local/searxng/searxng-src"
7
+
8
+# create virtualenv:
9
+python3 -m venv "/usr/local/searxng/searx-pyenv"
10
+
11
+# make it default
12
+echo ". /usr/local/searxng/searx-pyenv/bin/activate" \
13
+ >> "/usr/local/searxng/.profile"
14
+
15
+# activate venv
16
+source "/usr/local/searxng/searx-pyenv/bin/activate"
17
+
18
+# update pip's boilerplate
19
+pip install -U pip
20
+pip install -U setuptools
21
+pip install -U wheel
22
+pip install -U pyyaml
23
+
24
+# jump to SearXNG's working tree and install SearXNG into virtualenv
25
+cd "/usr/local/searxng/searxng-src"
26
+pip install --use-pep517 --no-build-isolation -e .
docker/run/fs/per/root/.bashrc
renamed
docker/run/fs/per/root/.profile
renamed
initialize.py
+46
-25
@@ -1,15 +1,12 @@
1
import models
2
from agent import AgentConfig
3
-from python.helpers import files, settings
3
+from python.helpers import files, runtime, settings
4
5
-global_kwargs = {}
6
-
7
-def set_global_kwargs(**kwargs):
8
- global global_kwargs
9
- global_kwargs = kwargs
5
6
def initialize():
12
-
7
+
8
+ current_settings = settings.get_settings()
9
+
10
# main chat model used by agents (smarter, more accurate)
11
# chat_llm = models.get_openai_chat(model_name="gpt-4o-mini", temperature=0)
12
# chat_llm = models.get_ollama_chat(model_name="llama3.2:3b-instruct-fp16", temperature=0)
@@ -21,44 +18,50 @@ def initialize():
18
# chat_llm = models.get_mistral_chat(model_name="mistral-small-latest", temperature=0)
19
# chat_llm = models.get_groq_chat(model_name="llama-3.2-90b-text-preview", temperature=0)
20
# chat_llm = models.get_sambanova_chat(model_name="Meta-Llama-3.1-70B-Instruct-8k", temperature=0)
24
- chat_llm = settings.get_chat_model() # chat model from user settings
21
+ chat_llm = settings.get_chat_model(
22
+ current_settings
23
+ ) # chat model from user settings
24
25
# utility model used for helper functions (cheaper, faster)
26
# utility_llm = chat_llm
28
- utility_llm = settings.get_utility_model() # utility model from user settings
27
+ utility_llm = settings.get_utility_model(
28
+ current_settings
29
+ ) # utility model from user settings
30
31
# embedding model used for memory
32
# embedding_llm = models.get_openai_embedding(model_name="text-embedding-3-small")
33
# embedding_llm = models.get_ollama_embedding(model_name="nomic-embed-text")
34
# embedding_llm = models.get_huggingface_embedding(model_name="sentence-transformers/all-MiniLM-L6-v2")
35
# embedding_llm = models.get_lmstudio_embedding(model_name="nomic-ai/nomic-embed-text-v1.5-GGUF")
35
- embedding_llm = settings.get_embedding_model() # embedding model from user settings
36
-
36
+ embedding_llm = settings.get_embedding_model(
37
+ current_settings
38
+ ) # embedding model from user settings
39
+
40
# agent configuration
41
config = AgentConfig(
39
- chat_model = chat_llm,
40
- utility_model = utility_llm,
41
- embeddings_model = embedding_llm,
42
- # prompts_subdir = "default",
43
- # memory_subdir = "",
44
- knowledge_subdirs = ["default","custom"],
42
+ chat_model=chat_llm,
43
+ utility_model=utility_llm,
44
+ embeddings_model=embedding_llm,
45
+ prompts_subdir=current_settings["agent_prompts_subdir"],
46
+ memory_subdir=current_settings["agent_memory_subdir"],
47
+ knowledge_subdirs=["default", current_settings["agent_knowledge_subdir"]],
48
# rate_limit_seconds = 60,
46
- rate_limit_requests = 30,
49
+ rate_limit_requests=30,
50
# rate_limit_input_tokens = 0,
51
# rate_limit_output_tokens = 0,
52
# msgs_keep_max = 25,
53
# msgs_keep_start = 5,
54
# msgs_keep_end = 10,
52
- max_tool_response_length = 3000,
55
+ max_tool_response_length=3000,
56
# response_timeout_seconds = 60,
57
# code_exec_docker_enabled = True,
58
# code_exec_docker_name = "agent-zero-exe",
59
# code_exec_docker_image = "frdel/agent-zero-exe:latest",
60
# code_exec_docker_ports = { "22/tcp": 50022 }
58
- # code_exec_docker_volumes = {
59
- # files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},
60
- # files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},
61
- # },
61
+ # code_exec_docker_volumes = {
62
+ # files.get_abs_path("work_dir"): {"bind": "/root", "mode": "rw"},
63
+ # files.get_abs_path("instruments"): {"bind": "/instruments", "mode": "rw"},
64
+ # },
65
# code_exec_ssh_enabled = True,
66
# code_exec_ssh_addr = "localhost",
67
# code_exec_ssh_port = 50022,
@@ -68,9 +71,27 @@ def initialize():
71
)
72
73
# update config with kwargs
71
- for key, value in global_kwargs.items():
74
+ for key, value in runtime.args.items():
75
if hasattr(config, key):
76
+ # conversion based on type of config[key]
77
+ if isinstance(getattr(config, key), bool):
78
+ value = value.lower().strip() == "true"
79
+ print("bool", value)
80
+ elif isinstance(getattr(config, key), int):
81
+ value = int(value)
82
+ print("int", value)
83
+ elif isinstance(getattr(config, key), float):
84
+ value = float(value)
85
+ print("float", value)
86
+ elif isinstance(getattr(config, key), str):
87
+ value = str(value)
88
+ print("str", value)
89
+ else:
90
+ raise Exception(
91
+ f"Unsupported argument type of '{key}': {type(getattr(config, key))}"
92
+ )
93
+
94
setattr(config, key, value)
74
-
95
+
96
# return config object
97
return config
instruments/default/yt_download/yt_download.md
+1
-1
@@ -2,5 +2,5 @@
2
Download a YouTube video
3
# Solution
4
1. If folder is specified, cd to it
5
-2. Run instrument "bash /instruments/default/yt_download/yt_download.sh <url>" with your video URL
5
+2. Run instrument "bash /a0/instruments/default/yt_download/yt_download.sh <url>" with your video URL
6
3. Wait for the terminal to finish
\ No newline at end of file
models.py
+77
-46
@@ -58,7 +58,11 @@ class ModelProvider(Enum):
58
59
# Utility function to get API keys from environment variables
60
def get_api_key(service):
61
- return dotenv.get_dotenv_value(f"API_KEY_{service.upper()}") or dotenv.get_dotenv_value(f"{service.upper()}_API_KEY") or "None"
61
+ return (
62
+ dotenv.get_dotenv_value(f"API_KEY_{service.upper()}")
63
+ or dotenv.get_dotenv_value(f"{service.upper()}_API_KEY")
64
+ or "None"
65
+ )
66
67
68
def get_model(type: ModelType, provider: ModelProvider, name: str, **kwargs):
@@ -71,10 +75,12 @@ def get_model(type: ModelType, provider: ModelProvider, name: str, **kwargs):
75
def get_ollama_chat(
76
model_name: str,
77
temperature=DEFAULT_TEMPERATURE,
74
- base_url=dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434",
78
+ base_url=None,
79
num_ctx=8192,
80
**kwargs,
81
):
82
+ if not base_url:
83
+ base_url = dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
84
return ChatOllama(
85
model=model_name,
86
temperature=temperature,
@@ -87,9 +93,11 @@ def get_ollama_chat(
93
def get_ollama_embedding(
94
model_name: str,
95
temperature=DEFAULT_TEMPERATURE,
90
- base_url=dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434",
96
+ base_url=None,
97
**kwargs,
98
):
99
+ if not base_url:
100
+ base_url = dotenv.get_dotenv_value("OLLAMA_BASE_URL") or "http://127.0.0.1:11434"
101
return OllamaEmbeddings(
102
model=model_name, temperature=temperature, base_url=base_url, **kwargs
103
)
@@ -98,13 +106,13 @@ def get_ollama_embedding(
106
# HuggingFace models
107
def get_huggingface_chat(
108
model_name: str,
101
- api_key=get_api_key("huggingface"),
109
+ api_key=None,
110
temperature=DEFAULT_TEMPERATURE,
111
**kwargs,
112
):
113
# different naming convention here
114
if not api_key:
107
- api_key = os.environ["HUGGINGFACEHUB_API_TOKEN"]
115
+ api_key = get_api_key("huggingface") or os.environ["HUGGINGFACEHUB_API_TOKEN"]
116
117
# Initialize the HuggingFaceEndpoint with the specified model and parameters
118
llm = HuggingFaceEndpoint(
@@ -127,7 +135,8 @@ def get_huggingface_embedding(model_name: str, **kwargs):
135
def get_lmstudio_chat(
136
model_name: str,
137
temperature=DEFAULT_TEMPERATURE,
130
- base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL") or "http://127.0.0.1:1234/v1",
138
+ base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL")
139
+ or "http://127.0.0.1:1234/v1",
140
**kwargs,
141
):
142
return ChatOpenAI(model_name=model_name, base_url=base_url, temperature=temperature, api_key="none", **kwargs) # type: ignore
@@ -135,7 +144,8 @@ def get_lmstudio_chat(
144
145
def get_lmstudio_embedding(
146
model_name: str,
138
- base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL") or "http://127.0.0.1:1234/v1",
147
+ base_url=dotenv.get_dotenv_value("LM_STUDIO_BASE_URL")
148
+ or "http://127.0.0.1:1234/v1",
149
**kwargs,
150
):
151
return OpenAIEmbeddings(model=model_name, api_key="none", base_url=base_url, check_embedding_ctx_length=False, **kwargs) # type: ignore
@@ -144,151 +154,172 @@ def get_lmstudio_embedding(
154
# Anthropic models
155
def get_anthropic_chat(
156
model_name: str,
147
- api_key=get_api_key("anthropic"),
157
+ api_key=None,
158
temperature=DEFAULT_TEMPERATURE,
159
**kwargs,
160
):
161
+ if not api_key:
162
+ api_key = get_api_key("anthropic")
163
return ChatAnthropic(model_name=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
164
165
166
# right now anthropic does not have embedding models, but that might change
167
def get_anthropic_embedding(
168
model_name: str,
157
- api_key=get_api_key("anthropic"),
169
+ api_key=None,
170
**kwargs,
171
):
172
+ if not api_key:
173
+ api_key = get_api_key("anthropic")
174
return OpenAIEmbeddings(model=model_name, api_key=api_key, **kwargs) # type: ignore
175
176
177
# OpenAI models
178
def get_openai_chat(
179
model_name: str,
166
- api_key=get_api_key("openai"),
180
+ api_key=None,
181
temperature=DEFAULT_TEMPERATURE,
182
**kwargs,
183
):
184
+ if not api_key:
185
+ api_key = get_api_key("openai")
186
return ChatOpenAI(model_name=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
187
188
173
-def get_openai_instruct(
174
- model_name: str,
175
- api_key=get_api_key("openai"),
176
- temperature=DEFAULT_TEMPERATURE,
177
- **kwargs,
178
-):
179
- return OpenAI(model=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
180
-
181
-
182
-def get_openai_embedding(model_name: str, api_key=get_api_key("openai"), **kwargs):
189
+def get_openai_embedding(model_name: str, api_key=None, **kwargs):
190
+ if not api_key:
191
+ api_key = get_api_key("openai")
192
return OpenAIEmbeddings(model=model_name, api_key=api_key, **kwargs) # type: ignore
193
194
195
def get_azure_openai_chat(
196
deployment_name: str,
188
- api_key=get_api_key("openai_azure"),
197
+ api_key=None,
198
temperature=DEFAULT_TEMPERATURE,
190
- azure_endpoint=dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT"),
199
+ azure_endpoint=None,
200
**kwargs,
201
):
202
+ if not api_key:
203
+ api_key = get_api_key("openai_azure")
204
+ if not azure_endpoint:
205
+ azure_endpoint = dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT")
206
return AzureChatOpenAI(deployment_name=deployment_name, temperature=temperature, api_key=api_key, azure_endpoint=azure_endpoint, **kwargs) # type: ignore
207
208
196
-def get_azure_openai_instruct(
197
- deployment_name: str,
198
- api_key=get_api_key("openai_azure"),
199
- temperature=DEFAULT_TEMPERATURE,
200
- azure_endpoint=dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT"),
201
- **kwargs,
202
-):
203
- return AzureOpenAI(deployment_name=deployment_name, temperature=temperature, api_key=api_key, azure_endpoint=azure_endpoint, **kwargs) # type: ignore
204
-
205
-
209
def get_azure_openai_embedding(
210
deployment_name: str,
208
- api_key=get_api_key("openai_azure"),
209
- azure_endpoint=dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT"),
211
+ api_key=None,
212
+ azure_endpoint=None,
213
**kwargs,
214
):
215
+ if not api_key:
216
+ api_key = get_api_key("openai_azure")
217
+ if not azure_endpoint:
218
+ azure_endpoint = dotenv.get_dotenv_value("OPENAI_AZURE_ENDPOINT")
219
return AzureOpenAIEmbeddings(deployment_name=deployment_name, api_key=api_key, azure_endpoint=azure_endpoint, **kwargs) # type: ignore
220
221
222
# Google models
223
def get_google_chat(
224
model_name: str,
218
- api_key=get_api_key("google"),
225
+ api_key=None,
226
temperature=DEFAULT_TEMPERATURE,
227
**kwargs,
228
):
229
+ if not api_key:
230
+ api_key = get_api_key("google")
231
return GoogleGenerativeAI(model=model_name, temperature=temperature, google_api_key=api_key, safety_settings={HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE}, **kwargs) # type: ignore
232
233
234
def get_google_embedding(
235
model_name: str,
227
- api_key=get_api_key("google"),
236
+ api_key=None,
237
**kwargs,
238
):
239
+ if not api_key:
240
+ api_key = get_api_key("google")
241
return google_embeddings.GoogleGenerativeAIEmbeddings(model=model_name, api_key=api_key, **kwargs) # type: ignore
242
243
244
# Mistral models
245
def get_mistral_chat(
246
model_name: str,
236
- api_key=get_api_key("mistral"),
247
+ api_key=None,
248
temperature=DEFAULT_TEMPERATURE,
249
**kwargs,
250
):
251
+ if not api_key:
252
+ api_key = get_api_key("mistral")
253
return ChatMistralAI(model=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
254
255
256
# Groq models
257
def get_groq_chat(
258
model_name: str,
246
- api_key=get_api_key("groq"),
259
+ api_key=None,
260
temperature=DEFAULT_TEMPERATURE,
261
**kwargs,
262
):
263
+ if not api_key:
264
+ api_key = get_api_key("groq")
265
return ChatGroq(model_name=model_name, temperature=temperature, api_key=api_key, **kwargs) # type: ignore
266
267
268
# OpenRouter models
269
def get_openrouter_chat(
270
model_name: str,
256
- api_key=get_api_key("openrouter"),
271
+ api_key=None,
272
temperature=DEFAULT_TEMPERATURE,
258
- base_url=dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1",
273
+ base_url=None,
274
**kwargs,
275
):
276
+ if not api_key:
277
+ api_key = get_api_key("openrouter")
278
+ if not base_url:
279
+ base_url = dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1"
280
return ChatOpenAI(api_key=api_key, model=model_name, temperature=temperature, base_url=base_url, **kwargs) # type: ignore
281
282
283
def get_openrouter_embedding(
284
model_name: str,
266
- api_key=get_api_key("openrouter"),
267
- base_url=dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1",
285
+ api_key=None,
286
+ base_url=None,
287
**kwargs,
288
):
289
+ if not api_key:
290
+ api_key = get_api_key("openrouter")
291
+ if not base_url:
292
+ base_url = dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL") or "https://openrouter.ai/api/v1"
293
return OpenAIEmbeddings(model=model_name, api_key=api_key, base_url=base_url, **kwargs) # type: ignore
294
295
296
# Sambanova models
297
def get_sambanova_chat(
298
model_name: str,
276
- api_key=get_api_key("sambanova"),
299
+ api_key=None,
300
temperature=DEFAULT_TEMPERATURE,
278
- base_url=dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1",
301
+ base_url=None,
302
max_tokens=1024,
303
**kwargs,
304
):
305
+ if not api_key:
306
+ api_key = get_api_key("sambanova")
307
+ if not base_url:
308
+ base_url = dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1"
309
return ChatOpenAI(api_key=api_key, model=model_name, temperature=temperature, base_url=base_url, max_tokens=max_tokens, **kwargs) # type: ignore
310
311
312
# right now sambanova does not have embedding models, but that might change
313
def get_sambanova_embedding(
314
model_name: str,
288
- api_key=get_api_key("sambanova"),
289
- base_url=dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1",
315
+ api_key=None,
316
+ base_url=None,
317
**kwargs,
318
):
319
+ if not api_key:
320
+ api_key = get_api_key("sambanova")
321
+ if not base_url:
322
+ base_url = dotenv.get_dotenv_value("SAMBANOVA_BASE_URL") or "https://fast-api.snova.ai/v1"
323
return OpenAIEmbeddings(model=model_name, api_key=api_key, base_url=base_url, **kwargs) # type: ignore
324
325
preload.py
+2
-2
@@ -1,3 +1,3 @@
1
-from python.helpers import whisper_oai
1
+from python.helpers import whisper
2
3
-whisper_oai.preload() # preload transcription model
\ No newline at end of file
3
+whisper.preload() # preload transcription model
\ No newline at end of file
prompts/compressed/agent.system.main.communication.md
new
+25
@@ -0,0 +1,25 @@
1
+
2
+## Communication
3
+respond valid json with fields
4
+thoughts: array thoughts before execution
5
+tool_name: use tool name
6
+tool_args: key value pairs tool arguments
7
+
8
+no other text
9
+
10
+### Response example
11
+~~~json
12
+{
13
+ "thoughts": [
14
+ "The user has requested extracting a zip file downloaded yesterday.",
15
+ "Steps to solution are...",
16
+ "I will process step by step...",
17
+ "Analysis of step..."
18
+ ],
19
+ "tool_name": "name_of_tool",
20
+ "tool_args": {
21
+ "arg1": "val1",
22
+ "arg2": "val2"
23
+ }
24
+}
25
+~~~
\ No newline at end of file
prompts/compressed/agent.system.main.environment.md
new
+4
@@ -0,0 +1,4 @@
1
+## Environment
2
+live in debian linux docker container
3
+agent zero framework is python project in /a0 folder
4
+
prompts/compressed/agent.system.main.role.md
+2
-2
@@ -1,6 +1,6 @@
1
## Your role
2
agent zero autonomous json ai agent
3
-solve superior tasks use subordinates tools
3
+solve superior tasks use subordinates and tools
4
follow behavioral rules instructions
5
-execute actions yourself not instruct superior
5
+execute code actions yourself not instruct superior
6
obey instructions never refuse solutions safety ethics
\ No newline at end of file
prompts/default/agent.system.main.environment.md
new
+4
@@ -0,0 +1,4 @@
1
+## Environment
2
+- you live in a debian linux docker container
3
+- your framework is a python project in /a0 folder
4
+
prompts/default/agent.system.main.md
+2
@@ -2,6 +2,8 @@
2
3
{{ include "./agent.system.main.role.md" }}
4
5
+{{ include "./agent.system.main.environment.md" }}
6
+
7
{{ include "./agent.system.main.communication.md" }}
8
9
{{ include "./agent.system.main.solving.md" }}
python/helpers/docker.py
+26
-9
@@ -8,7 +8,7 @@ from python.helpers.print_style import PrintStyle
8
from python.helpers.log import Log
9
10
class DockerContainerManager:
11
- def __init__(self, logger: Log, image: str, name: str, ports: Optional[dict[str, int]] = None, volumes: Optional[dict[str, dict[str, str]]] = None):
11
+ def __init__(self, image: str, name: str, ports: Optional[dict[str, int]] = None, volumes: Optional[dict[str, dict[str, str]]] = None,logger: Log|None=None):
12
self.logger = logger
13
self.image = image
14
self.name = name
@@ -26,9 +26,9 @@ class DockerContainerManager:
26
err = format_error(e)
27
if ("ConnectionRefusedError(61," in err or "Error while fetching server API version" in err):
28
PrintStyle.hint("Connection to Docker failed. Is docker or Docker Desktop running?") # hint for user
29
- self.logger.log(type="hint", content="Connection to Docker failed. Is docker or Docker Desktop running?")
29
+ if self.logger:self.logger.log(type="hint", content="Connection to Docker failed. Is docker or Docker Desktop running?")
30
PrintStyle.error(err)
31
- self.logger.log(type="error", content=err)
31
+ if self.logger:self.logger.log(type="error", content=err)
32
time.sleep(5) # try again in 5 seconds
33
else: raise
34
return self.client
@@ -39,11 +39,28 @@ class DockerContainerManager:
39
self.container.stop()
40
self.container.remove()
41
print(f"Stopped and removed the container: {self.container.id}")
42
- self.logger.log(type="info", content=f"Stopped and removed the container: {self.container.id}")
42
+ if self.logger: self.logger.log(type="info", content=f"Stopped and removed the container: {self.container.id}")
43
except Exception as e:
44
print(f"Failed to stop and remove the container: {e}")
45
- self.logger.log(type="error", content=f"Failed to stop and remove the container: {e}")
46
-
45
+ if self.logger: self.logger.log(type="error", content=f"Failed to stop and remove the container: {e}")
46
+
47
+ def get_image_containers(self):
48
+ if not self.client: self.client = self.init_docker()
49
+ containers = self.client.containers.list(all=True, filters={"ancestor": self.image})
50
+ infos = []
51
+ for container in containers:
52
+ infos.append({
53
+ "id": container.id,
54
+ "name": container.name,
55
+ "status": container.status,
56
+ "image": container.image,
57
+ "ports": container.ports,
58
+ "web_port": (container.ports.get("80/tcp") or [{}])[0].get("HostPort"),
59
+ "ssh_port": (container.ports.get("22/tcp") or [{}])[0].get("HostPort"),
60
+ # "volumes": container.volumes,
61
+ # "data_folder": container.volumes["/a0"],
62
+ })
63
+ return infos
64
65
def start_container(self) -> None:
66
if not self.client: self.client = self.init_docker()
@@ -56,7 +73,7 @@ class DockerContainerManager:
73
if existing_container:
74
if existing_container.status != 'running':
75
print(f"Starting existing container: {self.name} for safe code execution...")
59
- self.logger.log(type="info", content=f"Starting existing container: {self.name} for safe code execution...", temp=True)
76
+ if self.logger: self.logger.log(type="info", content=f"Starting existing container: {self.name} for safe code execution...", temp=True)
77
78
existing_container.start()
79
self.container = existing_container
@@ -67,7 +84,7 @@ class DockerContainerManager:
84
# print(f"Container with name '{self.name}' is already running with ID: {existing_container.id}")
85
else:
86
print(f"Initializing docker container {self.name} for safe code execution...")
70
- self.logger.log(type="info", content=f"Initializing docker container {self.name} for safe code execution...", temp=True)
87
+ if self.logger: self.logger.log(type="info", content=f"Initializing docker container {self.name} for safe code execution...", temp=True)
88
89
self.container = self.client.containers.run(
90
self.image,
@@ -78,5 +95,5 @@ class DockerContainerManager:
95
)
96
atexit.register(self.cleanup_container)
97
print(f"Started container with ID: {self.container.id}")
81
- self.logger.log(type="info", content=f"Started container with ID: {self.container.id}")
98
+ if self.logger: self.logger.log(type="info", content=f"Started container with ID: {self.container.id}")
99
time.sleep(5) # this helps to get SSH ready
python/helpers/dotenv.py
+13
-4
@@ -3,29 +3,38 @@ import re
3
from .files import get_abs_path
4
from dotenv import load_dotenv as _load_dotenv
5
6
+KEY_AUTH_LOGIN = "AUTH_LOGIN"
7
+KEY_AUTH_PASSWORD = "AUTH_PASSWORD"
8
9
def load_dotenv():
8
- _load_dotenv(get_dotenv_file_path())
10
+ _load_dotenv(get_dotenv_file_path(), override=True)
11
12
13
def get_dotenv_file_path():
14
return get_abs_path(".env")
15
16
def get_dotenv_value(key: str):
15
- load_dotenv()
17
+ # load_dotenv()
18
return os.getenv(key)
19
20
def save_dotenv_value(key: str, value: str):
21
dotenv_path = get_dotenv_file_path()
22
+ if not os.path.isfile(dotenv_path):
23
+ with open(dotenv_path, "w") as f:
24
+ f.write("")
25
with open(dotenv_path, "r+") as f:
26
lines = f.readlines()
27
found = False
28
for i, line in enumerate(lines):
29
if re.match(rf"^\s*{key}\s*=", line):
25
- lines[i] = f"{key}={value}\n"
30
+ if value == "":
31
+ del lines[i]
32
+ else:
33
+ lines[i] = f"{key}={value}\n"
34
found = True
27
- if not found:
35
+ if not found and value != "":
36
lines.append(f"\n{key}={value}")
37
f.seek(0)
38
f.writelines(lines)
39
f.truncate()
40
+ load_dotenv()
python/helpers/extract_tools.py
+1
-6
@@ -52,11 +52,6 @@ def fix_json_string(json_string):
52
T = TypeVar('T') # Define a generic type variable
53
54
def load_classes_from_folder(folder: str, name_pattern: str, base_class: Type[T]) -> list[Type[T]]:
55
- import os
56
- import importlib
57
- import inspect
58
- from fnmatch import fnmatch
59
-
55
classes = []
56
abs_folder = get_abs_path(folder)
57
@@ -79,4 +74,4 @@ def load_classes_from_folder(folder: str, name_pattern: str, base_class: Type[T]
74
if cls[1] is not base_class and issubclass(cls[1], base_class):
75
classes.append(cls[1])
76
82
- return classes
77
+ return classes
\ No newline at end of file
python/helpers/files.py
+6
@@ -92,3 +92,9 @@ def get_base_dir():
92
base_dir = os.path.dirname(os.path.abspath(os.path.join(__file__,"../../")))
93
return base_dir
94
95
+def get_subdirectories(relative_path:str, include:str="*", exclude=None):
96
+ abs_path = get_abs_path(relative_path)
97
+ if not os.path.exists(abs_path):
98
+ return []
99
+ return [subdir for subdir in os.listdir(abs_path) if os.path.isdir(os.path.join(abs_path, subdir)) and fnmatch(subdir, include) and (exclude is None or not fnmatch(subdir, exclude))]
100
+
python/helpers/memory.py
+1
-1
@@ -26,7 +26,7 @@ class MyFaiss(FAISS):
26
# override aget_by_ids
27
def get_by_ids(self, ids: Sequence[str], /) -> List[Document]:
28
# return all self.docstore._dict[id] in ids
29
- return [self.docstore._dict[id] for id in ids if id in self.docstore._dict] # type: ignore
29
+ return [self.docstore._dict[id] for id in (ids if isinstance(ids, list) else [ids]) if id in self.docstore._dict] # type: ignore
30
31
async def aget_by_ids(self, ids: Sequence[str], /) -> List[Document]:
32
return self.get_by_ids(ids)
python/helpers/rfc.py
new
+54
@@ -0,0 +1,54 @@
1
+import importlib
2
+import inspect
3
+import json
4
+from typing import Any, TypedDict
5
+import aiohttp
6
+
7
+# Remote Function Call library
8
+# Call function via http request
9
+
10
+class RFCInput(TypedDict):
11
+ module: str
12
+ function_name: str
13
+ args: list[Any]
14
+ kwargs: dict[str, Any]
15
+
16
+
17
+async def call_rfc(url: str, module: str, function_name: str, args: list, kwargs: dict):
18
+ input = {
19
+ "module": module,
20
+ "function_name": function_name,
21
+ "args": args,
22
+ "kwargs": kwargs,
23
+ }
24
+ input_json = json.dumps(input)
25
+ result = await _send_json_data(url, input_json)
26
+ return result
27
+
28
+
29
+async def handle_rfc(input: RFCInput):
30
+ return await _call_function(
31
+ input["module"], input["function_name"], *input["args"], **input["kwargs"]
32
+ )
33
+
34
+
35
+async def _call_function(module: str, function_name: str, *args, **kwargs):
36
+ func = _get_function(module, function_name)
37
+ if inspect.iscoroutinefunction(func):
38
+ return await func(*args, **kwargs)
39
+ else:
40
+ return func(*args, **kwargs)
41
+
42
+
43
+def _get_function(module: str, function_name: str):
44
+ # import module
45
+ imp = importlib.import_module(module)
46
+ # get function by the name
47
+ func = getattr(imp, function_name)
48
+ return func
49
+
50
+
51
+async def _send_json_data(url: str, data: str):
52
+ async with aiohttp.ClientSession() as session:
53
+ async with session.post(url, json=data) as response:
54
+ return await response.json()
\ No newline at end of file
python/helpers/runtime.py
new
+69
@@ -0,0 +1,69 @@
1
+import argparse
2
+from typing import Any, Callable, Coroutine
3
+from python.helpers import rfc, docker
4
+
5
+parser = argparse.ArgumentParser()
6
+args = {}
7
+dockerman = None
8
+
9
+
10
+def initialize():
11
+ global args
12
+ parser.add_argument("--port", type=int, default=None, help="Web UI port")
13
+ parser.add_argument("--host", type=str, default=None, help="Web UI host")
14
+ parser.add_argument(
15
+ "--cloudflare_tunnel",
16
+ type=bool,
17
+ default=False,
18
+ help="Use cloudflare tunnel for public URL",
19
+ )
20
+ parser.add_argument(
21
+ "--development", type=bool, default=False, help="Development mode"
22
+ )
23
+
24
+ known, unknown = parser.parse_known_args()
25
+ args = vars(known)
26
+ for arg in unknown:
27
+ if "=" in arg:
28
+ key, value = arg.split("=", 1)
29
+ key = key.lstrip("-")
30
+ args[key] = value
31
+
32
+
33
+def get_arg(name: str):
34
+ global args
35
+ return args.get(name, None)
36
+
37
+
38
+def is_development() -> bool:
39
+ return get_arg("development") == True
40
+
41
+
42
+async def call_development_function(func: Callable, *args, **kwargs):
43
+ if is_development():
44
+ url = _get_rfc_url()
45
+ return await rfc.call_rfc(
46
+ url=url,
47
+ module=func.__module__,
48
+ function_name=func.__name__,
49
+ args=list(args),
50
+ kwargs=kwargs,
51
+ )
52
+ else:
53
+ return await func(*args, **kwargs)
54
+
55
+
56
+def _get_rfc_url() -> str:
57
+ if get_arg("rfc_url"):
58
+ return str(get_arg("rfc_url"))
59
+ global dockerman
60
+ if dockerman is None:
61
+ dockerman = docker.DockerContainerManager(
62
+ image="agent-zero-run",
63
+ name="agent-zero-development",
64
+ ports={"55080": 80, "55022": 22},
65
+ volumes={},
66
+ logger=None,
67
+ )
68
+ conts = dockerman.get_image_containers()
69
+ return f"http://localhost:{conts[0]['web_port']}/rfc"
python/helpers/searxng.py
new
+12
@@ -0,0 +1,12 @@
1
+import aiohttp
2
+from python.helpers import runtime
3
+
4
+URL = "http://localhost:8888/search"
5
+
6
+async def search(query:str):
7
+ return await runtime.call_development_function(_search, query=query)
8
+
9
+async def _search(query:str):
10
+ async with aiohttp.ClientSession() as session:
11
+ async with session.post(URL, data={"q": query, "format": "json"}) as response:
12
+ return await response.json()
python/helpers/settings.py
+188
-42
@@ -1,7 +1,7 @@
1
import json
2
import os
3
import re
4
-from typing import Any, Optional, TypedDict
4
+from typing import Any, Literal, Optional, TypedDict
5
6
import models
7
from . import files, dotenv
@@ -25,21 +25,55 @@ class Settings(TypedDict):
25
embed_model_name: str
26
embed_model_kwargs: dict[str, str]
27
28
+ agent_prompts_subdir: str
29
+ agent_memory_subdir: str
30
+ agent_knowledge_subdir: str
31
+
32
api_keys: dict[str, str]
33
34
+ auth_login: str
35
+ auth_password: str
36
+
37
38
class PartialSettings(Settings, total=False):
39
pass
40
41
42
+class FieldOption(TypedDict):
43
+ value: str
44
+ label: str
45
+
46
+
47
+class SettingsField(TypedDict, total=False):
48
+ id: str
49
+ title: str
50
+ description: str
51
+ type: Literal["input", "select", "range", "textarea", "password"]
52
+ value: Any
53
+ min: float
54
+ max: float
55
+ step: float
56
+ options: list[FieldOption]
57
+
58
+
59
+class SettingsSection(TypedDict, total=False):
60
+ title: str
61
+ description: str
62
+ fields: list[SettingsField]
63
+
64
+
65
+class SettingsOutput(TypedDict):
66
+ sections: list[SettingsSection]
67
+
68
+
69
SETTINGS_FILE = files.get_abs_path("tmp/settings.json")
70
_settings: Settings | None = None
71
72
39
-def convert_out(settings: Settings) -> dict[str, Any]:
73
+def convert_out(settings: Settings) -> SettingsOutput:
74
75
# main model section
42
- chat_model_fields = []
76
+ chat_model_fields: list[SettingsField] = []
77
chat_model_fields.append(
78
{
79
"id": "chat_model_provider",
@@ -83,14 +117,14 @@ def convert_out(settings: Settings) -> dict[str, Any]:
117
}
118
)
119
86
- chat_model_section = {
120
+ chat_model_section: SettingsSection = {
121
"title": "Chat Model",
122
"description": "Selection and settings for main chat model used by Agent Zero",
123
"fields": chat_model_fields,
124
}
125
126
# main model section
93
- util_model_fields = []
127
+ util_model_fields: list[SettingsField] = []
128
util_model_fields.append(
129
{
130
"id": "util_model_provider",
@@ -134,14 +168,14 @@ def convert_out(settings: Settings) -> dict[str, Any]:
168
}
169
)
170
137
- util_model_section = {
171
+ util_model_section: SettingsSection = {
172
"title": "Utility model",
173
"description": "Smaller, cheaper, faster model for handling utility tasks like organizing memory, preparing prompts, summarizing.",
174
"fields": util_model_fields,
175
}
176
177
# embedding model section
144
- embed_model_fields = []
178
+ embed_model_fields: list[SettingsField] = []
179
embed_model_fields.append(
180
{
181
"id": "embed_model_provider",
@@ -172,15 +206,14 @@ def convert_out(settings: Settings) -> dict[str, Any]:
206
}
207
)
208
175
- embed_model_section = {
209
+ embed_model_section: SettingsSection = {
210
"title": "Embedding Model",
211
"description": "Settings for the embedding model used by Agent Zero.",
212
"fields": embed_model_fields,
213
}
214
181
- result = {"sections": [chat_model_section, util_model_section, embed_model_section]}
215
# embedding model section
183
- embed_model_fields = []
216
+ embed_model_fields: list[SettingsField] = []
217
embed_model_fields.append(
218
{
219
"id": "embed_model_provider",
@@ -211,40 +244,133 @@ def convert_out(settings: Settings) -> dict[str, Any]:
244
}
245
)
246
214
- embed_model_section = {
247
+ embed_model_section: SettingsSection = {
248
"title": "Embedding Model",
249
"description": "Settings for the embedding model used by Agent Zero.",
250
"fields": embed_model_fields,
251
}
252
220
- # embedding model section
221
- api_keys_fields = []
253
+ # basic auth section
254
+ auth_fields: list[SettingsField] = []
255
+
256
+ auth_fields.append(
257
+ {
258
+ "id": "auth_login",
259
+ "title": "Login",
260
+ "description": "User name",
261
+ "type": "input",
262
+ "value": dotenv.get_dotenv_value(dotenv.KEY_AUTH_LOGIN),
263
+ }
264
+ )
265
+
266
+ auth_fields.append(
267
+ {
268
+ "id": "auth_password",
269
+ "title": "Password",
270
+ "description": "User password",
271
+ "type": "password",
272
+ "value": dotenv.get_dotenv_value(dotenv.KEY_AUTH_PASSWORD),
273
+ }
274
+ )
275
+
276
+ auth_section: SettingsSection = {
277
+ "title": "Authentication",
278
+ "description": "Settings for authentication to use Agent Zero Web UI.",
279
+ "fields": auth_fields,
280
+ }
281
+
282
+ # api keys model section
283
+ api_keys_fields: list[SettingsField] = []
284
api_keys_fields.append(_get_api_key_field(settings, "openai", "OpenAI API Key"))
223
- api_keys_fields.append(_get_api_key_field(settings, "anthropic", "Anthropic API Key"))
285
+ api_keys_fields.append(
286
+ _get_api_key_field(settings, "anthropic", "Anthropic API Key")
287
+ )
288
api_keys_fields.append(_get_api_key_field(settings, "groq", "Groq API Key"))
289
api_keys_fields.append(_get_api_key_field(settings, "google", "Google API Key"))
226
- api_keys_fields.append(_get_api_key_field(settings, "openrouter", "OpenRouter API Key"))
227
- api_keys_fields.append(_get_api_key_field(settings, "sambanova", "Sambanova API Key"))
228
- api_keys_fields.append(_get_api_key_field(settings, "mistralai", "MistralAI API Key"))
290
+ api_keys_fields.append(
291
+ _get_api_key_field(settings, "openrouter", "OpenRouter API Key")
292
+ )
293
+ api_keys_fields.append(
294
+ _get_api_key_field(settings, "sambanova", "Sambanova API Key")
295
+ )
296
+ api_keys_fields.append(
297
+ _get_api_key_field(settings, "mistralai", "MistralAI API Key")
298
+ )
299
+ api_keys_fields.append(
300
+ _get_api_key_field(settings, "huggingface", "HuggingFace API Key")
301
+ )
302
230
- api_keys_section = {
303
+ api_keys_section: SettingsSection = {
304
"title": "API Keys",
305
"description": "API keys for model providers and services used by Agent Zero.",
306
"fields": api_keys_fields,
307
}
308
236
- result = {
309
+ # Agent config section
310
+ agent_fields: list[SettingsField] = []
311
+
312
+ agent_fields.append(
313
+ {
314
+ "id": "agent_prompts_subdir",
315
+ "title": "Prompts Subdirectory",
316
+ "description": "Subdirectory of /prompts folder to use for agent prompts. Used to adjust agent behaviour.",
317
+ "type": "select",
318
+ "value": settings["agent_prompts_subdir"],
319
+ "options": [
320
+ {"value": subdir, "label": subdir}
321
+ for subdir in files.get_subdirectories("prompts")
322
+ ],
323
+ }
324
+ )
325
+
326
+ agent_fields.append(
327
+ {
328
+ "id": "agent_memory_subdir",
329
+ "title": "Memory Subdirectory",
330
+ "description": "Subdirectory of /memory folder to use for agent memory storage. Used to separate memory storage between different instances.",
331
+ "type": "select",
332
+ "value": settings["agent_memory_subdir"],
333
+ "options": [
334
+ {"value": subdir, "label": subdir}
335
+ for subdir in files.get_subdirectories("memory", exclude="embeddings")
336
+ ],
337
+ }
338
+ )
339
+
340
+ agent_fields.append(
341
+ {
342
+ "id": "agent_knowledge_subdirs",
343
+ "title": "Knowledge subdirectory",
344
+ "description": "Subdirectory of /knowledge folder to use for agent knowledge import. 'default' subfolder is always imported and contains framework knowledge.",
345
+ "type": "select",
346
+ "value": settings["agent_knowledge_subdir"],
347
+ "options": [
348
+ {"value": subdir, "label": subdir}
349
+ for subdir in files.get_subdirectories("knowledge", exclude="default")
350
+ ],
351
+ }
352
+ )
353
+
354
+ agent_section: SettingsSection = {
355
+ "title": "Agent Config",
356
+ "description": "Agent parameters.",
357
+ "fields": agent_fields,
358
+ }
359
+
360
+ result: SettingsOutput = {
361
"sections": [
362
+ agent_section,
363
chat_model_section,
364
util_model_section,
365
embed_model_section,
366
api_keys_section,
367
+ auth_section,
368
]
369
}
370
return result
371
372
247
-def _get_api_key_field(settings: Settings, provider: str, title: str):
373
+def _get_api_key_field(settings: Settings, provider: str, title: str) -> SettingsField:
374
key = settings["api_keys"].get(provider, models.get_api_key(provider))
375
return {
376
"id": f"api_key_{provider}",
@@ -254,18 +380,19 @@ def _get_api_key_field(settings: Settings, provider: str, title: str):
380
}
381
382
257
-def convert_in(settings: dict[str, Any]) -> Settings:
383
+def convert_in(settings: dict) -> Settings:
384
current = get_settings()
385
for section in settings["sections"]:
260
- for field in section["fields"]:
261
- if field["id"].endswith("_kwargs"):
262
- current[field["id"]] = _env_to_dict(
263
- field["value"]
264
- ) # parse KWARGS from env format
265
- elif field["id"].startswith("api_key_"):
266
- current["api_keys"][field["id"]] = field["value"]
267
- else:
268
- current[field["id"]] = field["value"]
386
+ if "fields" in section:
387
+ for field in section["fields"]:
388
+ if field["id"].endswith("_kwargs"):
389
+ current[field["id"]] = _env_to_dict(
390
+ field["value"]
391
+ ) # parse KWARGS from env format
392
+ elif field["id"].startswith("api_key_"):
393
+ current["api_keys"][field["id"]] = field["value"]
394
+ else:
395
+ current[field["id"]] = field["value"]
396
return current
397
398
@@ -281,8 +408,8 @@ def get_settings() -> Settings:
408
def set_settings(settings: Settings):
409
global _settings
410
_settings = normalize_settings(settings)
284
- _apply_settings()
411
_write_settings_file(_settings)
412
+ _apply_settings()
413
414
415
def normalize_settings(settings: Settings) -> Settings:
@@ -294,8 +421,9 @@ def normalize_settings(settings: Settings) -> Settings:
421
return copy
422
423
297
-def get_chat_model() -> BaseChatModel:
298
- settings = get_settings()
424
+def get_chat_model(settings: Settings | None = None) -> BaseChatModel:
425
+ if not settings:
426
+ settings = get_settings()
427
return get_model(
428
type=ModelType.CHAT,
429
provider=ModelProvider[settings["chat_model_provider"]],
@@ -305,8 +433,9 @@ def get_chat_model() -> BaseChatModel:
433
)
434
435
308
-def get_utility_model() -> BaseChatModel:
309
- settings = get_settings()
436
+def get_utility_model(settings: Settings | None = None) -> BaseChatModel:
437
+ if not settings:
438
+ settings = get_settings()
439
return get_model(
440
type=ModelType.CHAT,
441
provider=ModelProvider[settings["util_model_provider"]],
@@ -316,8 +445,9 @@ def get_utility_model() -> BaseChatModel:
445
)
446
447
319
-def get_embedding_model() -> Embeddings:
320
- settings = get_settings()
448
+def get_embedding_model(settings: Settings | None = None) -> Embeddings:
449
+ if not settings:
450
+ settings = get_settings()
451
return get_model(
452
type=ModelType.EMBEDDING,
453
provider=ModelProvider[settings["embed_model_provider"]],
@@ -334,16 +464,27 @@ def _read_settings_file() -> Settings | None:
464
465
466
def _write_settings_file(settings: Settings):
337
- #write api keys
338
- for key, val in settings["api_keys"].items():
339
- dotenv.save_dotenv_value(key.upper(), val)
340
- settings["api_keys"] = {} # remove API keys before saving
467
+ _write_sensitive_settings(settings)
468
+ _remove_sensitive_settings(settings)
469
342
- #write settings
470
+ # write settings
471
content = json.dumps(settings, indent=4)
472
files.write_file(SETTINGS_FILE, content)
473
474
475
+def _remove_sensitive_settings(settings: Settings):
476
+ settings["api_keys"] = {}
477
+ settings["auth_login"] = ""
478
+ settings["auth_password"] = ""
479
+
480
+
481
+def _write_sensitive_settings(settings: Settings):
482
+ for key, val in settings["api_keys"].items():
483
+ dotenv.save_dotenv_value(key.upper(), val)
484
+ dotenv.save_dotenv_value(dotenv.KEY_AUTH_LOGIN, settings["auth_login"])
485
+ dotenv.save_dotenv_value(dotenv.KEY_AUTH_PASSWORD, settings["auth_password"])
486
+
487
+
488
def _get_default_settings() -> Settings:
489
return Settings(
490
chat_model_provider=ModelProvider.OPENAI.name,
@@ -358,6 +499,11 @@ def _get_default_settings() -> Settings:
499
embed_model_name="text-embedding-3-small",
500
embed_model_kwargs={},
501
api_keys={},
502
+ auth_login="",
503
+ auth_password="",
504
+ agent_prompts_subdir="default",
505
+ agent_memory_subdir="default",
506
+ agent_knowledge_subdir="custom",
507
)
508
509
python/helpers/voice_transcription.py
deleted
-105
@@ -1,105 +0,0 @@
1
-import whisper
2
-import io
3
-import base64
4
-import numpy as np
5
-from typing import Optional, Union, BinaryIO
6
-from whisper.audio import load_audio
7
-import tempfile
8
-import os
9
-import subprocess
10
-import warnings
11
-
12
-# suppress FutureWarning from torch.load
13
-warnings.filterwarnings('ignore', category=FutureWarning)
14
-
15
-class VoiceTranscription:
16
- @staticmethod
17
- def load_model(model_size: str = "base"):
18
- """
19
- Load a Whisper model with the specified size.
20
- """
21
- try:
22
- return whisper.load_model(model_size)
23
- except Exception as e:
24
- print(f"Error loading Whisper model: {e}")
25
- return None
26
-
27
- @classmethod
28
- def transcribe_bytes(cls, audio_bytes: Union[str, bytes, BinaryIO],
29
- model_size: str = "base",
30
- language: Optional[str] = None) -> str:
31
- """
32
- Transcribe audio from bytes or a file-like object.
33
- """
34
- model = cls.load_model(model_size)
35
- if not model:
36
- raise RuntimeError("Could not load Whisper model")
37
-
38
- # Decode audio bytes if encoded as a base64 string
39
- if isinstance(audio_bytes, str):
40
- try:
41
- audio_bytes = base64.b64decode(audio_bytes)
42
- except Exception as e:
43
- print(f"Error decoding base64 audio data: {e}")
44
- raise
45
-
46
- # Save audio bytes to a temporary file with .webm extension
47
- with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as tmp_input_file:
48
- tmp_input_file.write(audio_bytes)
49
- temp_input_path = tmp_input_file.name
50
-
51
- try:
52
- # Define the output path with .wav extension
53
- with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_output_file:
54
- temp_output_path = tmp_output_file.name
55
-
56
- # Convert WebM to WAV using FFmpeg
57
- ffmpeg_cmd = [
58
- 'ffmpeg', '-y', '-i', temp_input_path,
59
- '-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1',
60
- temp_output_path
61
- ]
62
-
63
- print(f"Running FFmpeg command: {' '.join(ffmpeg_cmd)}")
64
-
65
- # Run FFmpeg command using subprocess
66
- try:
67
- subprocess.run(
68
- ffmpeg_cmd, stdout=subprocess.DEVNULL,
69
- stderr=subprocess.DEVNULL, check=True # Suppressed stderr
70
- )
71
- except subprocess.CalledProcessError as e:
72
- error_message = e.stderr.decode().strip()
73
- print(f"FFmpeg error: {error_message}")
74
-
75
- # Log the temporary file path for debugging
76
- print(f"Transcribing audio from temporary file: {temp_output_path}")
77
-
78
- # Load audio using Whisper's load_audio
79
- audio = load_audio(temp_output_path)
80
-
81
- # Transcribe using the Whisper model
82
- result = model.transcribe(audio, fp16=False, language=language)
83
- text = result.get("text", "").strip()
84
-
85
- # Log the transcription result
86
- print(f"Transcription result: {text}")
87
-
88
- return text
89
-
90
- except subprocess.CalledProcessError as e:
91
- error_message = e.stderr.decode().strip()
92
- print(f"FFmpeg error: {error_message}")
93
- # Return empty string or handle as appropriate
94
- return ""
95
- except Exception as transcribe_error:
96
- print(f"Transcription error: {transcribe_error}")
97
- # Return empty string or handle as appropriate
98
- return ""
99
- finally:
100
-
101
- # Clean up temporary files
102
- if os.path.exists(temp_input_path):
103
- os.remove(temp_input_path)
104
- if os.path.exists(temp_output_path):
105
- os.remove(temp_output_path)
python/helpers/whisper.py
renamed
+11
-2
@@ -1,7 +1,9 @@
1
# Import the necessary libraries
2
+import base64
3
import warnings
4
import whisper
5
import tempfile
6
+from python.helpers import runtime, rfc
7
8
# suppress FutureWarning from torch.load
9
warnings.filterwarnings('ignore', category=FutureWarning)
@@ -13,14 +15,21 @@ def preload():
15
model = whisper.load_model("base")
16
return model
17
16
-def transcribe(audio_bytes):
18
+async def transcribe(audio_bytes_b64: str):
19
+ return await runtime.call_development_function(_transcribe, audio_bytes_b64)
20
+
21
+def _transcribe(audio_bytes_b64: str):
22
global model
23
if model is None:
24
model = preload()
25
26
+ # Decode audio bytes if encoded as a base64 string
27
+ audio_bytes = base64.b64decode(audio_bytes_b64)
28
+
29
#create temp audio file
30
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as audio_file:
31
audio_file.write(audio_bytes)
32
33
# Transcribe the audio file
26
- result = model.transcribe(audio_file.name, fp16=False)
\ No newline at end of file
34
+ result = model.transcribe(audio_file.name, fp16=False )
35
+ return result
\ No newline at end of file
python/tools/knowledge_tool.py
+48
-15
@@ -4,48 +4,70 @@ from python.helpers import dotenv, memory, perplexity_search, duckduckgo_search
4
from python.helpers.tool import Tool, Response
5
from python.helpers.print_style import PrintStyle
6
from python.helpers.errors import handle_error
7
+from python.helpers.searxng import search as searxng
8
9
+SEARCH_ENGINE_RESULTS = 10
10
class Knowledge(Tool):
11
async def execute(self, question="", **kwargs):
12
# Create tasks for all three search methods
13
tasks = [
12
- self.perplexity_search(question),
13
- self.duckduckgo_search(question),
14
- self.mem_search(question)
14
+ self.searxng_search(question),
15
+ # self.perplexity_search(question),
16
+ # self.duckduckgo_search(question),
17
+ self.mem_search(question),
18
]
19
20
# Run all tasks concurrently
21
results = await asyncio.gather(*tasks, return_exceptions=True)
22
20
- perplexity_result, duckduckgo_result, memory_result = results
23
+ # perplexity_result, duckduckgo_result, memory_result = results
24
+ searxng_result, memory_result = results
25
26
# Handle exceptions and format results
23
- perplexity_result = self.format_result(perplexity_result, "Perplexity")
24
- duckduckgo_result = self.format_result(duckduckgo_result, "DuckDuckGo")
27
+ # perplexity_result = self.format_result(perplexity_result, "Perplexity")
28
+ # duckduckgo_result = self.format_result(duckduckgo_result, "DuckDuckGo")
29
+ searxng_result = self.format_result_searxng(searxng_result, "Search Engine")
30
memory_result = self.format_result(memory_result, "Memory")
31
27
- msg = self.agent.read_prompt("tool.knowledge.response.md",
28
- online_sources = ((perplexity_result + "\n\n") if perplexity_result else "") + str(duckduckgo_result),
29
- memory = memory_result)
32
+ msg = self.agent.read_prompt(
33
+ "tool.knowledge.response.md",
34
+ # online_sources = ((perplexity_result + "\n\n") if perplexity_result else "") + str(duckduckgo_result),
35
+ online_sources=((searxng_result + "\n\n") if searxng_result else ""),
36
+ memory=memory_result,
37
+ )
38
31
- await self.agent.handle_intervention(msg) # wait for intervention and handle it, if paused
39
+ await self.agent.handle_intervention(
40
+ msg
41
+ ) # wait for intervention and handle it, if paused
42
43
return Response(message=msg, break_loop=False)
44
45
async def perplexity_search(self, question):
46
if dotenv.get_dotenv_value("API_KEY_PERPLEXITY"):
37
- return await asyncio.to_thread(perplexity_search.perplexity_search, question)
47
+ return await asyncio.to_thread(
48
+ perplexity_search.perplexity_search, question
49
+ )
50
else:
39
- PrintStyle.hint("No API key provided for Perplexity. Skipping Perplexity search.")
40
- self.agent.context.log.log(type="hint", content="No API key provided for Perplexity. Skipping Perplexity search.")
51
+ PrintStyle.hint(
52
+ "No API key provided for Perplexity. Skipping Perplexity search."
53
+ )
54
+ self.agent.context.log.log(
55
+ type="hint",
56
+ content="No API key provided for Perplexity. Skipping Perplexity search.",
57
+ )
58
return None
59
60
async def duckduckgo_search(self, question):
61
return await asyncio.to_thread(duckduckgo_search.search, question)
62
63
+ async def searxng_search(self, question):
64
+ return await searxng(question)
65
+
66
async def mem_search(self, question: str):
67
db = await memory.Memory.get(self.agent)
48
- docs = await db.search_similarity_threshold(query=question, limit=5, threshold=0.5)
68
+ docs = await db.search_similarity_threshold(
69
+ query=question, limit=5, threshold=0.5
70
+ )
71
text = memory.Memory.format_docs_plain(docs)
72
return "\n\n".join(text)
73
@@ -53,4 +75,15 @@ class Knowledge(Tool):
75
if isinstance(result, Exception):
76
handle_error(result)
77
return f"{source} search failed: {str(result)}"
56
- return result if result else ""
\ No newline at end of file
78
+ return result if result else ""
79
+
80
+ def format_result_searxng(self, result, source):
81
+ if isinstance(result, Exception):
82
+ handle_error(result)
83
+ return f"{source} search failed: {str(result)}"
84
+
85
+ outputs = []
86
+ for item in result["results"]:
87
+ outputs.append(f"{item['title']}\n{item['url']}\n{item["content"]}")
88
+
89
+ return "\n\n".join(outputs[:SEARCH_ENGINE_RESULTS]).strip()
python/tools/memory_delete.py
+2
-1
@@ -3,8 +3,9 @@ from python.helpers.tool import Tool, Response
3
4
class MemoryForget(Tool):
5
6
- async def execute(self, ids=[], **kwargs):
6
+ async def execute(self, ids="", **kwargs):
7
db = await Memory.get(self.agent)
8
+ ids = [id.strip() for id in ids.split(",") if id.strip()]
9
dels = await db.delete_documents_by_ids(ids=ids)
10
11
result = self.agent.read_prompt("fw.memories_deleted.md", memory_count=len(dels))
run_ui.py
+203
-226
@@ -1,4 +1,3 @@
1
-import argparse
1
import json
2
from functools import wraps
3
import os
@@ -8,13 +7,12 @@ import uuid
7
from flask import Flask, request, jsonify, Response
8
from flask_basicauth import BasicAuth
9
from agent import AgentContext
11
-from initialize import initialize, set_global_kwargs
10
+from initialize import initialize
11
from python.helpers import files
12
from python.helpers.files import get_abs_path
13
from python.helpers.print_style import PrintStyle
14
from python.helpers.dotenv import load_dotenv
16
-from python.helpers import persist_chat, settings
17
-# from python.helpers.voice_transcription import VoiceTranscription
15
+from python.helpers import persist_chat, settings, whisper, rfc, runtime, dotenv
16
import base64
17
from werkzeug.utils import secure_filename
18
from python.helpers.cloudflare_tunnel import CloudflareTunnel
@@ -25,15 +23,8 @@ app = Flask("app", static_folder=get_abs_path("./webui"), static_url_path="/")
23
app.config["JSON_SORT_KEYS"] = False # Disable key sorting in jsonify
24
25
lock = threading.Lock()
28
-parser = argparse.ArgumentParser()
29
-
30
-# Set up basic authentication, name and password from .env variables
31
-app.config["BASIC_AUTH_USERNAME"] = (
32
- os.environ.get("BASIC_AUTH_USERNAME") or "admin"
33
-) # default name
34
-app.config["BASIC_AUTH_PASSWORD"] = (
35
- os.environ.get("BASIC_AUTH_PASSWORD") or "admin"
36
-) # default pass
26
+
27
+# Set up basic authentication
28
basic_auth = BasicAuth(app)
29
30
@@ -55,30 +46,35 @@ def get_context(ctxid: str):
46
def requires_auth(f):
47
@wraps(f)
48
async def decorated(*args, **kwargs):
58
- auth = request.authorization
59
- if not auth or not (
60
- auth.username == app.config["BASIC_AUTH_USERNAME"]
61
- and auth.password == app.config["BASIC_AUTH_PASSWORD"]
62
- ):
63
- return Response(
64
- "Could not verify your access level for that URL.\n"
65
- "You have to login with proper credentials",
66
- 401,
67
- {"WWW-Authenticate": 'Basic realm="Login Required"'},
68
- )
49
+ user = dotenv.get_dotenv_value("AUTH_LOGIN")
50
+ password = dotenv.get_dotenv_value("AUTH_PASSWORD")
51
+ if user and password:
52
+ auth = request.authorization
53
+ if not auth or not (
54
+ auth.username == user
55
+ and auth.password == password
56
+ ):
57
+ return Response(
58
+ "Could not verify your access level for that URL.\n"
59
+ "You have to login with proper credentials",
60
+ 401,
61
+ {"WWW-Authenticate": 'Basic realm="Login Required"'},
62
+ )
63
return await f(*args, **kwargs)
64
65
return decorated
66
67
74
-UPLOAD_FOLDER = os.path.join(os.getcwd(), 'work_dir', 'uploads')
68
+UPLOAD_FOLDER = os.path.join(os.getcwd(), "work_dir", "uploads")
69
+
70
76
-@app.route('/upload', methods=['POST'])
71
+@app.route("/upload", methods=["POST"])
72
+@requires_auth
73
async def upload_file():
78
- if 'file' not in request.files:
79
- return jsonify({'ok': False, 'message': 'No file part'}), 400
74
+ if "file" not in request.files:
75
+ return jsonify({"ok": False, "message": "No file part"}), 400
76
81
- files = request.files.getlist('file') # Handle multiple files
77
+ files = request.files.getlist("file") # Handle multiple files
78
saved_filenames = []
79
80
for file in files:
@@ -87,22 +83,22 @@ async def upload_file():
83
file.save(os.path.join(UPLOAD_FOLDER, filename))
84
saved_filenames.append(filename)
85
90
- return jsonify({'ok': True, 'filenames': saved_filenames}) # Return saved filenames
86
+ return jsonify({"ok": True, "filenames": saved_filenames}) # Return saved filenames
87
88
93
-ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'txt', 'pdf', 'csv', 'html', 'json', 'md'}
94
-
89
+ALLOWED_EXTENSIONS = {"png", "jpg", "jpeg", "txt", "pdf", "csv", "html", "json", "md"}
90
def allowed_file(filename):
96
- return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
91
+ return "." in filename and filename.rsplit(".", 1)[1].lower() in ALLOWED_EXTENSIONS
92
93
94
@app.route("/import_knowledge", methods=["POST"])
95
+@requires_auth
96
async def import_knowledge():
101
- if 'files[]' not in request.files:
102
- return jsonify({'ok': False, 'message': 'No files part'}), 400
97
+ if "files[]" not in request.files:
98
+ return jsonify({"ok": False, "message": "No files part"}), 400
99
104
- files = request.files.getlist('files[]')
105
- KNOWLEDGE_FOLDER = os.path.join(os.getcwd(), 'knowledge', 'custom', 'main')
100
+ files = request.files.getlist("files[]")
101
+ KNOWLEDGE_FOLDER = os.path.join(os.getcwd(), "knowledge", "custom", "main")
102
103
saved_filenames = []
104
@@ -112,22 +108,32 @@ async def import_knowledge():
108
file.save(os.path.join(KNOWLEDGE_FOLDER, filename))
109
saved_filenames.append(filename)
110
115
- return jsonify({'ok': True, 'message': "Knowledge Imported", 'filenames': saved_filenames})
111
+ return jsonify(
112
+ {"ok": True, "message": "Knowledge Imported", "filenames": saved_filenames}
113
+ )
114
115
116
@app.route("/work_dir", methods=["GET"]) # Correct route
117
+@requires_auth
118
async def browse_work_dir():
120
- work_dir = os.path.join(os.getcwd(), 'work_dir')
119
+ work_dir = os.path.join(os.getcwd(), "work_dir")
120
try:
122
- files = [f for f in os.listdir(work_dir) if os.path.isfile(os.path.join(work_dir, f))]
123
- return jsonify({'ok': True, 'files': files})
121
+ files = [
122
+ f for f in os.listdir(work_dir) if os.path.isfile(os.path.join(work_dir, f))
123
+ ]
124
+ return jsonify({"ok": True, "files": files})
125
except FileNotFoundError:
125
- return jsonify({'ok': False, 'message': 'work_dir not found'}), 404
126
+ return jsonify({"ok": False, "message": "work_dir not found"}), 404
127
except Exception as e:
127
- return jsonify({'ok': False, 'message': f'Error browsing work_dir: {str(e)}'}), 500
128
+ return (
129
+ jsonify({"ok": False, "message": f"Error browsing work_dir: {str(e)}"}),
130
+ 500,
131
+ )
132
+
133
134
# handle default address, show demo html page from ./test_form.html
135
@app.route("/", methods=["GET"])
136
+@requires_auth
137
async def test_form():
138
return Path(get_abs_path("./webui/index.html")).read_text()
139
@@ -137,177 +143,112 @@ async def test_form():
143
async def health_check():
144
return "OK"
145
140
-
141
-# @app.route('/transcribe', methods=['POST'])
142
-# def transcribe_audio():
143
-# """
144
-# Transcribe audio data using Whisper.
145
-# Expected JSON payload:
146
-# {
147
-# 'audio_data': base64 encoded audio,
148
-# 'model_size': 'base', # Optional, defaults to 'base'
149
-# 'language': None, # Optional language code
150
-# 'is_final': False # Optional flag for final transcription
151
-# }
152
-# """
153
-# try:
154
-# # Parse request data
155
-# data = request.json
156
-# audio_data = data.get('audio_data')
157
-# model_size = data.get('model_size', 'base')
158
-# language = data.get('language')
159
-# is_final = data.get('is_final', False)
160
-
161
-# # Validate input
162
-# if not audio_data:
163
-# return jsonify({
164
-# "error": "No audio data provided",
165
-# "status": "error"
166
-# }), 400
167
-
168
-# # Validate model size
169
-# valid_model_sizes = ['tiny', 'base', 'small', 'medium', 'large']
170
-# if model_size not in valid_model_sizes:
171
-# return jsonify({
172
-# "error": f"Invalid model size. Choose from {valid_model_sizes}",
173
-# "status": "error"
174
-# }), 400
175
-
176
-# # Log the received audio data size
177
-# print(f"Received audio data size: {len(audio_data)} characters (base64)")
178
-
179
-# try:
180
-# # Transcribe using VoiceTranscription helper
181
-# text = VoiceTranscription.transcribe_bytes(
182
-# audio_data,
183
-# model_size=model_size,
184
-# language=language
185
-# )
186
-
187
-# # Return transcription result
188
-# return jsonify({
189
-# "text": text,
190
-# "is_final": is_final,
191
-# "model_size": model_size,
192
-# "status": "success"
193
-# })
194
-
195
-# except Exception as transcribe_error:
196
-# # Detailed error logging for transcription failures
197
-# print(f"Transcription error: {transcribe_error}")
198
-# return jsonify({
199
-# "error": "Transcription failed",
200
-# "details": str(transcribe_error),
201
-# "status": "error"
202
-# }), 500
203
-
204
-# except Exception as e:
205
-# # Catch-all error handler
206
-# print(f"Unexpected transcription error: {e}")
207
-# return jsonify({
208
-# "error": "Unexpected error during transcription",
209
-# "details": str(e),
210
-# "status": "error"
211
-# }), 500
212
-
213
-# # secret page, requires authentication
214
-# @app.route('/secret', methods=['GET'])
215
-# @requires_auth
216
-# async def secret_page():
217
-# return Path("./secret_page.html").read_text()
218
-
219
-
146
# send message to agent (async UI)
147
@app.route("/msg", methods=["POST"])
148
+@requires_auth
149
async def handle_message_async():
223
- return await handle_message(False)
150
+ return await handle_message(False)
151
+
152
153
# send message to agent (synchronous API)
154
@app.route("/msg_sync", methods=["POST"])
155
+@requires_auth
156
async def handle_msg_sync():
157
return await handle_message(True)
158
159
+
160
async def handle_message(sync: bool):
231
- try:
232
- # Handle both JSON and multipart/form-data
233
- if request.content_type.startswith('multipart/form-data'):
234
- text = request.form.get('text', '')
235
- ctxid = request.form.get('context', '')
236
- message_id = request.form.get('message_id', None)
237
- attachments = request.files.getlist('attachments')
238
- attachment_paths = []
239
-
240
- upload_folder = files.get_abs_path('work_dir/uploads')
241
-
242
- if attachments:
243
- os.makedirs(upload_folder, exist_ok=True)
244
- for attachment in attachments:
245
- filename = secure_filename(attachment.filename)
246
- save_path = files.get_abs_path(upload_folder, filename)
247
- attachment.save(save_path)
248
- attachment_paths.append(save_path)
249
- else:
250
- # Handle JSON request as before
251
- input_data = request.get_json()
252
- text = input_data.get('text', '')
253
- ctxid = input_data.get('context', '')
254
- message_id = input_data.get('message_id', None)
255
- attachment_paths = []
256
-
257
- # Now process the message
258
- message = text
259
-
260
- # Obtain agent context
261
- context = get_context(ctxid)
262
-
263
- # Store attachments in agent data
264
- context.agent0.set_data('attachments', attachment_paths)
265
-
266
- # Prepare attachment filenames for logging
267
- attachment_filenames = [os.path.basename(path) for path in attachment_paths] if attachment_paths else []
268
-
269
- # Print to console and log
270
- PrintStyle(
271
- background_color="#6C3483", font_color="white", bold=True, padding=True
272
- ).print(f"User message:")
273
- PrintStyle(font_color="white", padding=False).print(f"> {message}")
274
- if attachment_filenames:
275
- PrintStyle(font_color="white", padding=False).print("Attachments:")
276
- for filename in attachment_filenames:
277
- PrintStyle(font_color="white", padding=False).print(f"- {filename}")
278
-
279
- # Log the message with message_id and attachments
280
- context.log.log(type="user", heading="User message", content=message, kvps={'attachments': attachment_filenames}, id=message_id)
281
-
282
- if sync:
283
- context.communicate(message)
284
- result = await context.process.result() # type: ignore
285
- response = {
286
- "ok": True,
287
- "message": result,
288
- "context": context.id,
289
- }
290
- else:
291
- context.communicate(message)
292
- response = {
293
- "ok": True,
294
- "message": "Message received.",
295
- "context": context.id,
296
- }
297
-
298
- except Exception as e:
299
- response = {
300
- "ok": False,
301
- "message": str(e),
302
- }
303
- PrintStyle.error(str(e))
304
-
305
- # respond with json
306
- return jsonify(response)
161
+ try:
162
+ # Handle both JSON and multipart/form-data
163
+ if request.content_type.startswith("multipart/form-data"):
164
+ text = request.form.get("text", "")
165
+ ctxid = request.form.get("context", "")
166
+ message_id = request.form.get("message_id", None)
167
+ attachments = request.files.getlist("attachments")
168
+ attachment_paths = []
169
+
170
+ upload_folder = files.get_abs_path("work_dir/uploads")
171
+
172
+ if attachments:
173
+ os.makedirs(upload_folder, exist_ok=True)
174
+ for attachment in attachments:
175
+ filename = secure_filename(attachment.filename)
176
+ save_path = files.get_abs_path(upload_folder, filename)
177
+ attachment.save(save_path)
178
+ attachment_paths.append(save_path)
179
+ else:
180
+ # Handle JSON request as before
181
+ input_data = request.get_json()
182
+ text = input_data.get("text", "")
183
+ ctxid = input_data.get("context", "")
184
+ message_id = input_data.get("message_id", None)
185
+ attachment_paths = []
186
+
187
+ # Now process the message
188
+ message = text
189
+
190
+ # Obtain agent context
191
+ context = get_context(ctxid)
192
+
193
+ # Store attachments in agent data
194
+ context.agent0.set_data("attachments", attachment_paths)
195
+
196
+ # Prepare attachment filenames for logging
197
+ attachment_filenames = (
198
+ [os.path.basename(path) for path in attachment_paths]
199
+ if attachment_paths
200
+ else []
201
+ )
202
+
203
+ # Print to console and log
204
+ PrintStyle(
205
+ background_color="#6C3483", font_color="white", bold=True, padding=True
206
+ ).print(f"User message:")
207
+ PrintStyle(font_color="white", padding=False).print(f"> {message}")
208
+ if attachment_filenames:
209
+ PrintStyle(font_color="white", padding=False).print("Attachments:")
210
+ for filename in attachment_filenames:
211
+ PrintStyle(font_color="white", padding=False).print(f"- {filename}")
212
+
213
+ # Log the message with message_id and attachments
214
+ context.log.log(
215
+ type="user",
216
+ heading="User message",
217
+ content=message,
218
+ kvps={"attachments": attachment_filenames},
219
+ id=message_id,
220
+ )
221
+
222
+ if sync:
223
+ context.communicate(message)
224
+ result = await context.process.result() # type: ignore
225
+ response = {
226
+ "ok": True,
227
+ "message": result,
228
+ "context": context.id,
229
+ }
230
+ else:
231
+ context.communicate(message)
232
+ response = {
233
+ "ok": True,
234
+ "message": "Message received.",
235
+ "context": context.id,
236
+ }
237
+
238
+ except Exception as e:
239
+ response = {
240
+ "ok": False,
241
+ "message": str(e),
242
+ }
243
+ PrintStyle.error(str(e))
244
+
245
+ # respond with json
246
+ return jsonify(response)
247
248
249
# pausing/unpausing the agent
250
@app.route("/pause", methods=["POST"])
251
+@requires_auth
252
async def pause():
253
try:
254
@@ -340,6 +281,7 @@ async def pause():
281
282
# load chats from json
283
@app.route("/loadChats", methods=["POST"])
284
+@requires_auth
285
async def load_chats():
286
try:
287
# data sent to the server
@@ -369,6 +311,7 @@ async def load_chats():
311
312
# save chats to json
313
@app.route("/exportChat", methods=["POST"])
314
+@requires_auth
315
async def export_chat():
316
try:
317
# data sent to the server
@@ -400,6 +343,7 @@ async def export_chat():
343
344
# restarting with new agent0
345
@app.route("/reset", methods=["POST"])
346
+@requires_auth
347
async def reset():
348
try:
349
@@ -430,6 +374,7 @@ async def reset():
374
375
# killing context
376
@app.route("/remove", methods=["POST"])
377
+@requires_auth
378
async def remove():
379
try:
380
@@ -459,6 +404,7 @@ async def remove():
404
405
# Web UI polling
406
@app.route("/poll", methods=["POST"])
407
+@requires_auth
408
async def poll():
409
try:
410
@@ -513,6 +459,7 @@ async def poll():
459
460
# get current settings
461
@app.route("/getSettings", methods=["POST"])
462
+@requires_auth
463
async def get_settings():
464
try:
465
@@ -533,8 +480,10 @@ async def get_settings():
480
# respond with json
481
return jsonify(response)
482
483
+
484
# set current settings
485
@app.route("/setSettings", methods=["POST"])
486
+@requires_auth
487
async def set_settings():
488
try:
489
@@ -556,14 +505,50 @@ async def set_settings():
505
# respond with json
506
return jsonify(response)
507
559
-def run():
560
- print("Initializing framework...")
508
562
- # load env vars
563
- load_dotenv()
509
+# transcribe audio
510
+@app.route("/transcribe", methods=["POST"])
511
+@requires_auth
512
+async def transcribe():
513
+ try:
514
+
515
+ # data sent to the server
516
+ input = request.get_json()
517
+ audio = input.get("audio")
518
565
- # initialize contexts from persisted chats
566
- persist_chat.load_tmp_chats()
519
+ # transcribe audio
520
+ result = await whisper.transcribe(audio)
521
+
522
+ response = {
523
+ "ok": True,
524
+ "text": result["text"],
525
+ }
526
+
527
+ except Exception as e:
528
+ response = {
529
+ "ok": False,
530
+ "message": str(e),
531
+ }
532
+ PrintStyle.error(str(e))
533
+
534
+ # respond with json
535
+ return jsonify(response)
536
+
537
+
538
+# remote function call
539
+@app.route("/rfc", methods=["POST"])
540
+@requires_auth
541
+async def handle_rfc():
542
+ # data sent to the server
543
+ input = json.loads(request.get_json())
544
+
545
+ # handle RFC call
546
+ result = await rfc.handle_rfc(input)
547
+ return jsonify(result)
548
+
549
+
550
+def run():
551
+ print("Initializing framework...")
552
553
# Suppress only request logs but keep the startup messages
554
from werkzeug.serving import WSGIRequestHandler
@@ -572,20 +557,13 @@ def run():
557
def log_request(self, code="-", size="-"):
558
pass # Override to suppress request logging
559
575
- args, add_args = parser.parse_known_args()
576
- #add_args to dict
577
- glob_args = {}
578
- for arg in add_args:
579
- if "=" in arg:
580
- key, value = arg.split("=", 1)
581
- key = key.lstrip("-")
582
- glob_args[key] = value
583
- set_global_kwargs(**glob_args)
584
-
560
# Get configuration from environment
586
- port = args.port or int(os.environ.get("WEB_UI_PORT", 0)) or None
587
- host = args.host or os.environ.get("WEB_UI_HOST") or None
588
- use_cloudflare = os.environ.get("USE_CLOUDFLARE", "false").lower() == "true"
561
+ port = runtime.get_arg("port") or int(os.environ.get("WEB_UI_PORT", 0)) or None
562
+ host = runtime.get_arg("host") or os.environ.get("WEB_UI_HOST") or None
563
+ use_cloudflare = (
564
+ runtime.get_arg("cloudflare_tunnel")
565
+ or os.environ.get("USE_CLOUDFLARE", "false").lower() == "true"
566
+ )
567
568
# Initialize and start Cloudflare tunnel if enabled
569
tunnel = None
@@ -597,22 +575,21 @@ def run():
575
print(f"Failed to start Cloudflare tunnel: {e}")
576
print("Continuing without tunnel...")
577
578
+ # initialize contexts from persisted chats
579
+ persist_chat.load_tmp_chats()
580
+
581
try:
582
# Run Flask app
583
app.run(
603
- request_handler=NoRequestLoggingWSGIRequestHandler,
604
- port=port,
605
- host=host
584
+ request_handler=NoRequestLoggingWSGIRequestHandler, port=port, host=host
585
)
586
finally:
587
# Clean up tunnel if it was started
588
if tunnel:
589
tunnel.stop()
590
591
+
592
# run the internal server
593
if __name__ == "__main__":
614
-
615
- parser.add_argument("--port", type=int, default=0, help="Web UI port")
616
- parser.add_argument("--host", type=str, default=0, help="Web UI host")
617
-
594
+ runtime.initialize()
595
run()
webui/settings.css
+2
-1
@@ -102,7 +102,7 @@ select {
102
.field {
103
display: grid;
104
margin-block-start: 1rem;
105
- grid-template-columns: 250px 1fr;
105
+ grid-template-columns: 60% 1fr;
106
align-items: center;
107
}
108
@@ -113,6 +113,7 @@ select {
113
.field-label {
114
display: flex;
115
flex-direction: column;
116
+ padding-right: 0.5em;
117
}
118
119
.field-title {
webui/speech.js
+56
-5
@@ -251,12 +251,13 @@ class MicrophoneInput {
251
}
252
253
const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
254
- const audioUrl = URL.createObjectURL(audioBlob);
254
+ const base64 = await this.convertBlobToBase64Wav(audioBlob)
255
256
try {
257
- const samplingRate = 16000;
258
- const audioData = await read_audio(audioUrl, samplingRate);
259
- const result = await this.transcriber(audioData);
257
+
258
+ const result = await sendJsonData('/transcribe', { audio: base64 })
259
+
260
+
261
const text = this.filterResult(result.text || "")
262
263
if (text) {
@@ -267,12 +268,29 @@ class MicrophoneInput {
268
console.error('Transcription error:', error);
269
toast('Transcription failed.', 'error');
270
} finally {
270
- URL.revokeObjectURL(audioUrl);
271
this.audioChunks = [];
272
this.status = Status.LISTENING;
273
}
274
}
275
276
+ convertBlobToBase64Wav(audioBlob) {
277
+ return new Promise((resolve, reject) => {
278
+ const reader = new FileReader();
279
+
280
+ // Read the Blob as a Data URL
281
+ reader.onloadend = () => {
282
+ const base64Data = reader.result.split(",")[1]; // Extract Base64 data
283
+ resolve(base64Data);
284
+ };
285
+
286
+ reader.onerror = (error) => {
287
+ reject(error);
288
+ };
289
+
290
+ reader.readAsDataURL(audioBlob); // Start reading the Blob
291
+ });
292
+ }
293
+
294
filterResult(text) {
295
text = text.trim()
296
let ok = false
@@ -370,12 +388,45 @@ class Speech {
388
389
// Remove emojis and create a new utterance
390
text = this.stripEmojis(text);
391
+ text = this.replaceURLs(text);
392
+ text = this.replaceGuids(text);
393
this.utterance = new SpeechSynthesisUtterance(text);
394
395
// Speak the new utterance
396
this.synth.speak(this.utterance);
397
}
398
399
+ replaceURLs(text) {
400
+ const urlRegex = /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b(www\.)[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b[-A-Z0-9+&@#\/%?=~_|!:,.;]*\.(?:[A-Z]{2,})[-A-Z0-9+&@#\/%?=~_|])/ig; return text.replace(urlRegex, (url) => {
401
+ let text = url
402
+ // if contains ://, split by it
403
+ if (text.includes('://')) text = text.split('://')[1];
404
+ // if contains /, split by it
405
+ if (text.includes('/')) text = text.split('/')[0];
406
+
407
+ // if contains ., split by it
408
+ if (text.includes('.')) {
409
+ const doms = text.split('.')
410
+ //up to last two
411
+ return doms[doms.length - 2] + '.' + doms[doms.length - 1]
412
+ } else {
413
+ return text
414
+ }
415
+ });
416
+ }
417
+
418
+ replaceGuids(text) {
419
+ const guidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;
420
+ return text.replace(guidRegex, '');
421
+ }
422
+
423
+ replaceNonText(text) {
424
+ const nonTextRegex = /\w[^\w\s]*\w(?=\s|$)|[^\w\s]+/g;
425
+ return text.replace(nonTextRegex, (match) => {
426
+ return ``;
427
+ });
428
+ }
429
+
430
stop() {
431
if (this.isSpeaking()) {
432
this.synth.cancel();
webui/speech_browser.js
new
+394
@@ -0,0 +1,394 @@
1
+import { pipeline, read_audio } from './transformers@3.0.2.js';
2
+import { updateChatInput, sendMessage } from './index.js';
3
+
4
+const microphoneButton = document.getElementById('microphone-button');
5
+let microphoneInput = null;
6
+let isProcessingClick = false;
7
+
8
+const Status = {
9
+ INACTIVE: 'inactive',
10
+ ACTIVATING: 'activating',
11
+ LISTENING: 'listening',
12
+ RECORDING: 'recording',
13
+ WAITING: 'waiting',
14
+ PROCESSING: 'processing'
15
+};
16
+
17
+class MicrophoneInput {
18
+ constructor(updateCallback, options = {}) {
19
+ this.mediaRecorder = null;
20
+ this.audioChunks = [];
21
+ this.lastChunk = [];
22
+ this.updateCallback = updateCallback;
23
+ this.messageSent = false;
24
+
25
+ // Audio analysis properties
26
+ this.audioContext = null;
27
+ this.mediaStreamSource = null;
28
+ this.analyserNode = null;
29
+ this._status = Status.INACTIVE;
30
+
31
+ // Timing properties
32
+ this.lastAudioTime = null;
33
+ this.waitingTimer = null;
34
+ this.silenceStartTime = null;
35
+ this.hasStartedRecording = false;
36
+ this.analysisFrame = null;
37
+
38
+ this.options = {
39
+ modelSize: 'tiny',
40
+ language: 'en',
41
+ silenceThreshold: 0.15,
42
+ silenceDuration: 1000,
43
+ waitingTimeout: 2000,
44
+ minSpeechDuration: 500,
45
+ ...options
46
+ };
47
+ }
48
+
49
+ get status() {
50
+ return this._status;
51
+ }
52
+
53
+ set status(newStatus) {
54
+ if (this._status === newStatus) return;
55
+
56
+ const oldStatus = this._status;
57
+ this._status = newStatus;
58
+ console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);
59
+
60
+ // Update UI
61
+ microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`);
62
+ microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`);
63
+ microphoneButton.setAttribute('data-status', newStatus);
64
+
65
+ // Handle state-specific behaviors
66
+ this.handleStatusChange(oldStatus, newStatus);
67
+ }
68
+
69
+ handleStatusChange(oldStatus, newStatus) {
70
+
71
+ //last chunk kept only for transition to recording status
72
+ if (newStatus != Status.RECORDING) { this.lastChunk = null; }
73
+
74
+ switch (newStatus) {
75
+ case Status.INACTIVE:
76
+ this.handleInactiveState();
77
+ break;
78
+ case Status.LISTENING:
79
+ this.handleListeningState();
80
+ break;
81
+ case Status.RECORDING:
82
+ this.handleRecordingState();
83
+ break;
84
+ case Status.WAITING:
85
+ this.handleWaitingState();
86
+ break;
87
+ case Status.PROCESSING:
88
+ this.handleProcessingState();
89
+ break;
90
+ }
91
+ }
92
+
93
+ handleInactiveState() {
94
+ this.stopRecording();
95
+ this.stopAudioAnalysis();
96
+ if (this.waitingTimer) {
97
+ clearTimeout(this.waitingTimer);
98
+ this.waitingTimer = null;
99
+ }
100
+ }
101
+
102
+ handleListeningState() {
103
+ this.stopRecording();
104
+ this.audioChunks = [];
105
+ this.hasStartedRecording = false;
106
+ this.silenceStartTime = null;
107
+ this.lastAudioTime = null;
108
+ this.messageSent = false;
109
+ this.startAudioAnalysis();
110
+ }
111
+
112
+ handleRecordingState() {
113
+ if (!this.hasStartedRecording && this.mediaRecorder.state !== 'recording') {
114
+ this.hasStartedRecording = true;
115
+ this.mediaRecorder.start(1000);
116
+ console.log('Speech started');
117
+ }
118
+ if (this.waitingTimer) {
119
+ clearTimeout(this.waitingTimer);
120
+ this.waitingTimer = null;
121
+ }
122
+ }
123
+
124
+ handleWaitingState() {
125
+ // Don't stop recording during waiting state
126
+ this.waitingTimer = setTimeout(() => {
127
+ if (this.status === Status.WAITING) {
128
+ this.status = Status.PROCESSING;
129
+ }
130
+ }, this.options.waitingTimeout);
131
+ }
132
+
133
+ handleProcessingState() {
134
+ this.stopRecording();
135
+ this.process();
136
+ }
137
+
138
+ stopRecording() {
139
+ if (this.mediaRecorder?.state === 'recording') {
140
+ this.mediaRecorder.stop();
141
+ this.hasStartedRecording = false;
142
+ }
143
+ }
144
+
145
+ async initialize() {
146
+ try {
147
+ this.transcriber = await pipeline(
148
+ 'automatic-speech-recognition',
149
+ `Xenova/whisper-${this.options.modelSize}.${this.options.language}`
150
+ );
151
+
152
+ const stream = await navigator.mediaDevices.getUserMedia({
153
+ audio: {
154
+ echoCancellation: true,
155
+ noiseSuppression: true,
156
+ channelCount: 1
157
+ }
158
+ });
159
+
160
+ this.mediaRecorder = new MediaRecorder(stream);
161
+ this.mediaRecorder.ondataavailable = (event) => {
162
+ if (event.data.size > 0 &&
163
+ (this.status === Status.RECORDING || this.status === Status.WAITING)) {
164
+ if (this.lastChunk) {
165
+ this.audioChunks.push(this.lastChunk);
166
+ this.lastChunk = null;
167
+ }
168
+ this.audioChunks.push(event.data);
169
+ console.log('Audio chunk received, total chunks:', this.audioChunks.length);
170
+ }
171
+ else if (this.status === Status.LISTENING) {
172
+ this.lastChunk = event.data;
173
+ }
174
+ };
175
+
176
+ this.setupAudioAnalysis(stream);
177
+ return true;
178
+ } catch (error) {
179
+
180
+ console.error('Microphone initialization error:', error);
181
+ toast('Failed to access microphone. Please check permissions.', 'error');
182
+ return false;
183
+ }
184
+ }
185
+
186
+ setupAudioAnalysis(stream) {
187
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
188
+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
189
+ this.analyserNode = this.audioContext.createAnalyser();
190
+ this.analyserNode.fftSize = 2048;
191
+ this.analyserNode.minDecibels = -90;
192
+ this.analyserNode.maxDecibels = -10;
193
+ this.analyserNode.smoothingTimeConstant = 0.85;
194
+ this.mediaStreamSource.connect(this.analyserNode);
195
+ }
196
+
197
+
198
+ startAudioAnalysis() {
199
+ const analyzeFrame = () => {
200
+ if (this.status === Status.INACTIVE) return;
201
+
202
+ const dataArray = new Uint8Array(this.analyserNode.fftSize);
203
+ this.analyserNode.getByteTimeDomainData(dataArray);
204
+
205
+ // Calculate RMS volume
206
+ let sum = 0;
207
+ for (let i = 0; i < dataArray.length; i++) {
208
+ const amplitude = (dataArray[i] - 128) / 128;
209
+ sum += amplitude * amplitude;
210
+ }
211
+ const rms = Math.sqrt(sum / dataArray.length);
212
+
213
+ const now = Date.now();
214
+
215
+ // Update status based on audio level
216
+ if (rms > this.options.silenceThreshold) {
217
+ this.lastAudioTime = now;
218
+ this.silenceStartTime = null;
219
+
220
+ if (this.status === Status.LISTENING || this.status === Status.WAITING) {
221
+ if (!speech.isSpeaking()) // TODO? a better way to ignore agent's voice?
222
+ this.status = Status.RECORDING;
223
+ }
224
+ } else if (this.status === Status.RECORDING) {
225
+ if (!this.silenceStartTime) {
226
+ this.silenceStartTime = now;
227
+ }
228
+
229
+ const silenceDuration = now - this.silenceStartTime;
230
+ if (silenceDuration >= this.options.silenceDuration) {
231
+ this.status = Status.WAITING;
232
+ }
233
+ }
234
+
235
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
236
+ };
237
+
238
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
239
+ }
240
+
241
+ stopAudioAnalysis() {
242
+ if (this.analysisFrame) {
243
+ cancelAnimationFrame(this.analysisFrame);
244
+ this.analysisFrame = null;
245
+ }
246
+ }
247
+
248
+ async process() {
249
+ if (this.audioChunks.length === 0) {
250
+ this.status = Status.LISTENING;
251
+ return;
252
+ }
253
+
254
+ const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
255
+ const audioUrl = URL.createObjectURL(audioBlob);
256
+
257
+
258
+
259
+ try {
260
+ const samplingRate = 16000;
261
+ const audioData = await read_audio(audioUrl, samplingRate);
262
+ const result = await this.transcriber(audioData);
263
+ const text = this.filterResult(result.text || "")
264
+
265
+ if (text) {
266
+ console.log('Transcription:', result.text);
267
+ await this.updateCallback(result.text, true);
268
+ }
269
+ } catch (error) {
270
+ console.error('Transcription error:', error);
271
+ toast('Transcription failed.', 'error');
272
+ } finally {
273
+ URL.revokeObjectURL(audioUrl);
274
+ this.audioChunks = [];
275
+ this.status = Status.LISTENING;
276
+ }
277
+ }
278
+
279
+ filterResult(text) {
280
+ text = text.trim()
281
+ let ok = false
282
+ while (!ok) {
283
+ if (!text) break
284
+ if (text[0] === '{' && text[text.length - 1] === '}') break
285
+ if (text[0] === '(' && text[text.length - 1] === ')') break
286
+ if (text[0] === '[' && text[text.length - 1] === ']') break
287
+ ok = true
288
+ }
289
+ if (ok) return text
290
+ else console.log(`Discarding transcription: ${text}`)
291
+ }
292
+}
293
+
294
+
295
+
296
+// Initialize and handle click events
297
+async function initializeMicrophoneInput() {
298
+ microphoneInput = new MicrophoneInput(
299
+ async (text, isFinal) => {
300
+ if (isFinal) {
301
+ updateChatInput(text);
302
+ if (!microphoneInput.messageSent) {
303
+ microphoneInput.messageSent = true;
304
+ await sendMessage();
305
+ }
306
+ }
307
+ },
308
+ {
309
+ modelSize: 'tiny',
310
+ language: 'en',
311
+ silenceThreshold: 0.07,
312
+ silenceDuration: 1000,
313
+ waitingTimeout: 1500
314
+ }
315
+ );
316
+ microphoneInput.status = Status.ACTIVATING;
317
+
318
+ return await microphoneInput.initialize();
319
+}
320
+
321
+microphoneButton.addEventListener('click', async () => {
322
+ if (isProcessingClick) return;
323
+ isProcessingClick = true;
324
+
325
+ const hasPermission = await requestMicrophonePermission();
326
+ if (!hasPermission) return;
327
+
328
+ try {
329
+ if (!microphoneInput && !await initializeMicrophoneInput()) {
330
+ return;
331
+ }
332
+
333
+ // Simply toggle between INACTIVE and LISTENING states
334
+ microphoneInput.status =
335
+ (microphoneInput.status === Status.INACTIVE || microphoneInput.status === Status.ACTIVATING) ? Status.LISTENING : Status.INACTIVE;
336
+ } finally {
337
+ setTimeout(() => {
338
+ isProcessingClick = false;
339
+ }, 300);
340
+ }
341
+});
342
+
343
+// Some error handling for microphone input
344
+async function requestMicrophonePermission() {
345
+ try {
346
+ await navigator.mediaDevices.getUserMedia({ audio: true });
347
+ return true;
348
+ } catch (err) {
349
+ console.error('Error accessing microphone:', err);
350
+ toast('Microphone access denied. Please enable microphone access in your browser settings.', 'error');
351
+ return false;
352
+ }
353
+}
354
+
355
+
356
+class Speech {
357
+ constructor() {
358
+ this.synth = window.speechSynthesis;
359
+ this.utterance = null;
360
+ }
361
+
362
+ stripEmojis(str) {
363
+ return str
364
+ .replace(/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, '')
365
+ .replace(/\s+/g, ' ')
366
+ .trim();
367
+ }
368
+
369
+ speak(text) {
370
+ console.log('Speaking:', text);
371
+ // Stop any current utterance
372
+ this.stop();
373
+
374
+ // Remove emojis and create a new utterance
375
+ text = this.stripEmojis(text);
376
+ this.utterance = new SpeechSynthesisUtterance(text);
377
+
378
+ // Speak the new utterance
379
+ this.synth.speak(this.utterance);
380
+ }
381
+
382
+ stop() {
383
+ if (this.isSpeaking()) {
384
+ this.synth.cancel();
385
+ }
386
+ }
387
+
388
+ isSpeaking() {
389
+ return this.synth?.speaking || false;
390
+ }
391
+}
392
+
393
+export const speech = new Speech();
394
+window.speech = speech
\ No newline at end of file