Implement Kokoro TTS integration and settings updates
- Added Kokoro TTS support in preload and run_ui scripts. - Introduced a new API endpoint for text-to-speech synthesis. - Updated settings to include TTS enable/disable option. - Refactored speech handling to utilize a centralized speech store. - Enhanced UI with new speech button and SVG icon. - Updated dependencies in requirements.txt for Kokoro TTS.
TerminallyLazy committed
Jun 28, 2025 at 22:48 UTC
a100ee4143606c71c7ab62090adf1993c4eab2eb
13 files changed
+932
-514
preload.py
+17
-1
@@ -1,8 +1,15 @@
1
+
2
import asyncio
3
from python.helpers import runtime, whisper, settings
4
from python.helpers.print_style import PrintStyle
5
import models
6
7
+try:
8
+ from python.helpers import kokoro_tts
9
+ KOKORO_AVAILABLE = True
10
+except ImportError:
11
+ KOKORO_AVAILABLE = False
12
+
13
PrintStyle().print("Running preload...")
14
runtime.initialize()
15
@@ -28,9 +35,18 @@ async def preload():
35
except Exception as e:
36
PrintStyle().error(f"Error in preload_embedding: {e}")
37
38
+ # preload kokoro tts model if enabled
39
+ async def preload_kokoro():
40
+ if KOKORO_AVAILABLE and set.get("tts_enabled"):
41
+ try:
42
+ return await kokoro_tts.preload()
43
+ except Exception as e:
44
+ PrintStyle().error(f"Error in preload_kokoro: {e}")
45
46
# async tasks to preload
47
tasks = [preload_whisper(), preload_embedding()]
48
+ if KOKORO_AVAILABLE:
49
+ tasks.append(preload_kokoro())
50
51
await asyncio.gather(*tasks, return_exceptions=True)
52
PrintStyle().print("Preload completed")
@@ -39,4 +55,4 @@ async def preload():
55
56
57
# preload transcription model
42
-asyncio.run(preload())
58
+asyncio.run(preload())
\ No newline at end of file
python/api/synthesize.py
new
+95
@@ -0,0 +1,95 @@
1
+# api/synthesize.py
2
+
3
+import re
4
+from python.helpers.api import ApiHandler
5
+from flask import Request, Response
6
+
7
+from python.helpers import runtime, settings, kokoro_tts
8
+
9
+class Synthesize(ApiHandler):
10
+ async def process(self, input: dict, request: Request) -> dict | Response:
11
+ text = input.get("text", "")
12
+ ctxid = input.get("ctxid", "")
13
+
14
+ # DEBUG: Log what we received
15
+ print(f"[SYNTHESIZE DEBUG] Received text: '{text[:100]}...' (length: {len(text)})")
16
+
17
+ context = self.get_context(ctxid)
18
+ if await kokoro_tts.is_downloading():
19
+ context.log.log(type="info", content="Kokoro TTS model is currently being downloaded, please wait...")
20
+
21
+ try:
22
+ # Clean and chunk text for long responses
23
+ cleaned_text = self._clean_text(text)
24
+ chunks = self._chunk_text(cleaned_text)
25
+
26
+ if len(chunks) == 1:
27
+ # Single chunk - return as before
28
+ audio = await kokoro_tts.synthesize_sentences(chunks)
29
+ return {"audio": audio, "success": True}
30
+ else:
31
+ # Multiple chunks - return as sequence
32
+ audio_parts = []
33
+ for chunk in chunks:
34
+ chunk_audio = await kokoro_tts.synthesize_sentences([chunk])
35
+ audio_parts.append(chunk_audio)
36
+ return {"audio_parts": audio_parts, "success": True}
37
+ except Exception as e:
38
+ return {"error": str(e), "success": False}
39
+
40
+ def _clean_text(self, text: str) -> str:
41
+ """Clean text by removing markdown, tables, code blocks, and other formatting"""
42
+ # Remove code blocks
43
+ text = re.sub(r'```[\s\S]*?```', '', text)
44
+ text = re.sub(r'`[^`]*`', '', text)
45
+
46
+ # Remove markdown links
47
+ text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text)
48
+
49
+ # Remove markdown formatting
50
+ text = re.sub(r'[*_#]+', '', text)
51
+
52
+ # Remove tables (basic cleanup)
53
+ text = re.sub(r'\|[^\n]*\|', '', text)
54
+
55
+ # Remove extra whitespace and newlines
56
+ text = re.sub(r'\n+', ' ', text)
57
+ text = re.sub(r'\s+', ' ', text)
58
+
59
+ # Remove URLs
60
+ text = re.sub(r'https?://[^\s]+', '', text)
61
+
62
+ # Remove email addresses
63
+ text = re.sub(r'\S+@\S+', '', text)
64
+
65
+ return text.strip()
66
+
67
+ def _chunk_text(self, text: str) -> list[str]:
68
+ """Split text into manageable chunks for TTS"""
69
+ # If text is short enough, return as single chunk
70
+ if len(text) <= 300:
71
+ return [text]
72
+
73
+ # Split into sentences first
74
+ sentences = re.split(r'(?<=[.!?])\s+', text)
75
+
76
+ chunks = []
77
+ current_chunk = ""
78
+
79
+ for sentence in sentences:
80
+ sentence = sentence.strip()
81
+ if not sentence:
82
+ continue
83
+
84
+ # If adding this sentence would make chunk too long, start new chunk
85
+ if current_chunk and len(current_chunk + " " + sentence) > 300:
86
+ chunks.append(current_chunk.strip())
87
+ current_chunk = sentence
88
+ else:
89
+ current_chunk += (" " if current_chunk else "") + sentence
90
+
91
+ # Add the last chunk if it has content
92
+ if current_chunk.strip():
93
+ chunks.append(current_chunk.strip())
94
+
95
+ return chunks if chunks else [text]
\ No newline at end of file
python/helpers/kokoro_tts.py
new
+90
@@ -0,0 +1,90 @@
1
+# kokoro_tts.py
2
+
3
+import base64
4
+import io
5
+import warnings
6
+import asyncio
7
+import soundfile as sf
8
+from python.helpers import runtime
9
+from python.helpers.print_style import PrintStyle
10
+
11
+warnings.filterwarnings("ignore", category=FutureWarning)
12
+
13
+_pipeline = None
14
+_voice = "af_heart"
15
+is_updating_model = False
16
+
17
+async def preload():
18
+ try:
19
+ return await runtime.call_development_function(_preload)
20
+ except Exception as e:
21
+ if not runtime.is_development():
22
+ raise e
23
+ # Fallback to direct execution if RFC fails in development
24
+ PrintStyle.standard("RFC failed, falling back to direct execution...")
25
+ return await _preload()
26
+
27
+async def _preload():
28
+ global _pipeline, is_updating_model
29
+
30
+ while is_updating_model:
31
+ await asyncio.sleep(0.1)
32
+
33
+ try:
34
+ is_updating_model = True
35
+ if not _pipeline:
36
+ PrintStyle.standard("Loading Kokoro TTS model...")
37
+ from kokoro import KPipeline
38
+ _pipeline = KPipeline(lang_code='a')
39
+ finally:
40
+ is_updating_model = False
41
+
42
+async def is_downloading():
43
+ try:
44
+ return await runtime.call_development_function(_is_downloading)
45
+ except Exception as e:
46
+ if not runtime.is_development():
47
+ raise e
48
+ # Fallback to direct execution if RFC fails in development
49
+ return _is_downloading()
50
+
51
+def _is_downloading():
52
+ return is_updating_model
53
+
54
+async def synthesize_sentences(sentences: list[str]):
55
+ """Generate audio for multiple sentences and return concatenated base64 audio"""
56
+ try:
57
+ return await runtime.call_development_function(_synthesize_sentences, sentences)
58
+ except Exception as e:
59
+ if not runtime.is_development():
60
+ raise e
61
+ # Fallback to direct execution if RFC fails in development
62
+ return await _synthesize_sentences(sentences)
63
+
64
+async def _synthesize_sentences(sentences: list[str]):
65
+ await _preload()
66
+
67
+ combined_audio = []
68
+
69
+ try:
70
+ for sentence in sentences:
71
+ if sentence.strip():
72
+ segments = _pipeline(sentence.strip(), voice=_voice)
73
+ segment_list = list(segments)
74
+
75
+ for segment in segment_list:
76
+ audio_tensor = segment.audio
77
+ audio_numpy = audio_tensor.detach().cpu().numpy()
78
+ combined_audio.extend(audio_numpy)
79
+
80
+ # Convert combined audio to bytes
81
+ buffer = io.BytesIO()
82
+ sf.write(buffer, combined_audio, 24000, format='WAV')
83
+ audio_bytes = buffer.getvalue()
84
+
85
+ # Return base64 encoded audio
86
+ return base64.b64encode(audio_bytes).decode('utf-8')
87
+
88
+ except Exception as e:
89
+ PrintStyle.error(f"Error in Kokoro TTS synthesis: {e}")
90
+ raise
\ No newline at end of file
python/helpers/settings.py
+32
-32
@@ -7,14 +7,12 @@ import subprocess
7
from typing import Any, Literal, TypedDict
8
9
import models
10
-from python.helpers import runtime, whisper, defer, git
10
+from python.helpers import runtime, whisper, defer
11
from . import files, dotenv
12
from python.helpers.print_style import PrintStyle
13
14
15
class Settings(TypedDict):
16
- version: str
17
-
16
chat_model_provider: str
17
chat_model_name: str
18
chat_model_kwargs: dict[str, str]
@@ -67,6 +65,8 @@ class Settings(TypedDict):
65
stt_silence_duration: int
66
stt_waiting_timeout: int
67
68
+ tts_enabled: bool
69
+
70
mcp_servers: str
71
mcp_client_init_timeout: int
72
mcp_client_tool_timeout: int
@@ -504,8 +504,8 @@ def convert_out(settings: Settings) -> SettingsOutput:
504
agent_fields.append(
505
{
506
"id": "agent_prompts_subdir",
507
- "title": "A0 Prompts Subdirectory",
508
- "description": "Subdirectory of /prompts folder to be used by default agent no. 0. Subordinate agents can be spawned with other subdirectories, that is on their superior agent to decide. This setting affects the behaviour of the top level agent you communicate with.",
507
+ "title": "Prompts Subdirectory",
508
+ "description": "Subdirectory of /prompts folder to use for agent prompts. Used to adjust agent behaviour.",
509
"type": "select",
510
"value": settings["agent_prompts_subdir"],
511
"options": [
@@ -681,11 +681,24 @@ def convert_out(settings: Settings) -> SettingsOutput:
681
}
682
)
683
684
- stt_section: SettingsSection = {
685
- "id": "stt",
686
- "title": "Speech to Text",
687
- "description": "Voice transcription preferences and server turn detection settings.",
688
- "fields": stt_fields,
684
+ # TTS fields
685
+ tts_fields: list[SettingsField] = []
686
+
687
+ tts_fields.append(
688
+ {
689
+ "id": "tts_enabled",
690
+ "title": "Enable Kokoro TTS",
691
+ "description": "Enable server-side AI text-to-speech (Kokoro)",
692
+ "type": "switch",
693
+ "value": settings["tts_enabled"],
694
+ }
695
+ )
696
+
697
+ speech_section: SettingsSection = {
698
+ "id": "speech",
699
+ "title": "Speech",
700
+ "description": "Voice transcription and speech synthesis settings.",
701
+ "fields": stt_fields + tts_fields,
702
"tab": "agent",
703
}
704
@@ -815,7 +828,7 @@ def convert_out(settings: Settings) -> SettingsOutput:
828
embed_model_section,
829
browser_model_section,
830
# memory_section,
818
- stt_section,
831
+ speech_section,
832
api_keys_section,
833
auth_section,
834
mcp_client_section,
@@ -881,11 +894,6 @@ def normalize_settings(settings: Settings) -> Settings:
894
copy = settings.copy()
895
default = get_default_settings()
896
884
- # adjust settings values to match current version if needed
885
- if "version" not in copy or copy["version"] != default["version"]:
886
- _adjust_to_version(copy, default)
887
- copy["version"] = default["version"] # sync version
888
-
897
# remove keys that are not in default
898
keys_to_remove = [key for key in copy if key not in default]
899
for key in keys_to_remove:
@@ -907,13 +915,6 @@ def normalize_settings(settings: Settings) -> Settings:
915
return copy
916
917
910
-def _adjust_to_version(settings: Settings, default: Settings):
911
- # starting with 0.9, the default prompt subfolder for agent no. 0 is agent0
912
- # switch to agent0 if the old default is used from v0.8
913
- if "version" not in settings or settings["version"].startswith("v0.8"):
914
- if "agent_prompts_subdir" not in settings or settings["agent_prompts_subdir"] == "default":
915
- settings["agent_prompts_subdir"] = "agent0"
916
-
918
def _read_settings_file() -> Settings | None:
919
if os.path.exists(SETTINGS_FILE):
920
content = files.read_file(SETTINGS_FILE)
@@ -959,7 +960,6 @@ def get_default_settings() -> Settings:
960
from models import ModelProvider
961
962
return Settings(
962
- version=_get_version(),
963
chat_model_provider=ModelProvider.OPENAI.name,
964
chat_model_name="gpt-4.1",
965
chat_model_kwargs={"temperature": "0"},
@@ -990,7 +990,7 @@ def get_default_settings() -> Settings:
990
auth_login="",
991
auth_password="",
992
root_password="",
993
- agent_prompts_subdir="agent0",
993
+ agent_prompts_subdir="default",
994
agent_memory_subdir="default",
995
agent_knowledge_subdir="custom",
996
rfc_auto_docker=True,
@@ -1003,7 +1003,8 @@ def get_default_settings() -> Settings:
1003
stt_silence_threshold=0.3,
1004
stt_silence_duration=1000,
1005
stt_waiting_timeout=2000,
1006
- mcp_servers='{\n "mcpServers": {}\n}',
1006
+ tts_enabled=False,
1007
+ mcp_servers='{\n "mcpServers": {\n "render": {\n "command": "docker",\n "args": [\n "run",\n "-i",\n "--rm",\n "-e",\n "RENDER_API_KEY",\n "-v",\n "render-mcp-server-config:/config",\n "ghcr.io/render-oss/render-mcp-server"\n ],\n "env": {\n "RENDER_API_KEY": "rnd_C2reJ87CxaM8vQI8DqtBfL2ymwzA"\n }\n }\n }\n}',
1008
mcp_client_init_timeout=5,
1009
mcp_client_tool_timeout=120,
1010
mcp_server_enabled=False,
@@ -1091,14 +1092,14 @@ def _apply_settings(previous: Settings | None):
1092
) # TODO overkill, replace with background task
1093
1094
# update token in mcp server
1094
- current_token = (
1095
- create_auth_token()
1096
- ) # TODO - ugly, token in settings is generated from dotenv and does not always correspond
1097
- if not previous or current_token != previous["mcp_server_token"]:
1095
+ current_token = create_auth_token() #TODO - ugly, token in settings is generated from dotenv and does not always correspond
1096
+ if (
1097
+ not previous
1098
+ or current_token != previous["mcp_server_token"]
1099
+ ):
1100
1101
async def update_mcp_token(token: str):
1102
from python.helpers.mcp_server import DynamicMcpProxy
1101
-
1103
DynamicMcpProxy.get_instance().reconfigure(token=token)
1104
1105
task3 = defer.DeferredTask().start_task(
@@ -1177,7 +1178,6 @@ def create_auth_token() -> str:
1178
b64_token = base64.urlsafe_b64encode(hash_bytes).decode().replace("=", "")
1179
return b64_token[:16]
1180
1180
-
1181
def _get_version():
1182
try:
1183
git_info = git.get_git_info()
requirements.txt
+2
@@ -42,3 +42,5 @@ pdf2image==1.17.0
42
crontab==1.0.1
43
pathspec>=0.12.1
44
psutil>=7.0.0
45
+kokoro>=0.9.2
46
+soundfile
\ No newline at end of file
run_ui.py
+10
-1
@@ -245,10 +245,19 @@ def init_a0():
245
246
# only wait for init chats, otherwise they would seem to dissapear for a while on restart
247
init_chats.result_sync()
248
+
249
+ # preload Kokoro TTS model
250
+ try:
251
+ from python.helpers import kokoro_tts
252
+ import asyncio
253
+ asyncio.run(kokoro_tts.preload())
254
+ PrintStyle().debug("Kokoro TTS model preloaded successfully")
255
+ except Exception as e:
256
+ PrintStyle().error(f"Failed to preload Kokoro TTS model: {e}")
257
258
259
# run the internal server
260
if __name__ == "__main__":
261
runtime.initialize()
262
dotenv.load_dotenv()
254
- run()
263
+ run()
\ No newline at end of file
webui/index.html
+70
-23
@@ -21,13 +21,62 @@
21
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/5.15.4/css/all.min.css">
22
23
<!-- Flatpickr for datetime picker -->
24
- <link rel="stylesheet" href="vendor/flatpickr/flatpickr.min.css">
25
- <script src="vendor/flatpickr/flatpickr.min.js"></script>
24
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/flatpickr/dist/flatpickr.min.css">
25
+ <script src="https://cdn.jsdelivr.net/npm/flatpickr"></script>
26
27
<script>
28
window.safeCall = function (name, ...args) {
29
if (window[name]) window[name](...args)
30
}
31
+
32
+ // Pre-define global functions to ensure they're available to Alpine.js
33
+ window.newChat = function() {
34
+ console.log("newChat called before initialization");
35
+ };
36
+
37
+ // These will be replaced by proper implementations from index.js
38
+ window.resetChat = window.resetChat || function() { console.log("resetChat not ready"); };
39
+ window.loadChats = window.loadChats || function() { console.log("loadChats not ready"); };
40
+ window.saveChat = window.saveChat || function() { console.log("saveChat not ready"); };
41
+ window.restart = window.restart || function() { console.log("restart not ready"); };
42
+ window.killChat = window.killChat || function() { console.log("killChat not ready"); };
43
+ window.selectChat = window.selectChat || function() { console.log("selectChat not ready"); };
44
+ window.pauseAgent = window.pauseAgent || function() { console.log("pauseAgent not ready"); };
45
+ window.nudge = window.nudge || function() { console.log("nudge not ready"); };
46
+ window.toggleAutoScroll = window.toggleAutoScroll || function() { console.log("toggleAutoScroll not ready"); };
47
+ window.toggleJson = window.toggleJson || function() { console.log("toggleJson not ready"); };
48
+ window.toggleThoughts = window.toggleThoughts || function() { console.log("toggleThoughts not ready"); };
49
+ window.toggleUtils = window.toggleUtils || function() { console.log("toggleUtils not ready"); };
50
+ window.toggleDarkMode = window.toggleDarkMode || function() { console.log("toggleDarkMode not ready"); };
51
+ window.toggleSpeech = window.toggleSpeech || function() { console.log("toggleSpeech not ready"); };
52
+ window.loadKnowledge = window.loadKnowledge || function() { console.log("loadKnowledge not ready"); };
53
+ window.handleFileUpload = window.handleFileUpload || function() { console.log("handleFileUpload not ready"); };
54
+ window.openTaskDetail = window.openTaskDetail || function() { console.log("openTaskDetail not ready"); };
55
+
56
+ // Initialize speech object for isSpeaking check
57
+ window.speech = window.speech || {
58
+ isSpeaking: () => false,
59
+ speak: () => console.log("speech.speak not ready"),
60
+ stop: () => console.log("speech.stop not ready")
61
+ };
62
+
63
+ // Pre-define fetchApi for tunnel.js and other non-module scripts
64
+ window.fetchApi = window.fetchApi || function() {
65
+ console.log("fetchApi not ready");
66
+ return Promise.reject(new Error("fetchApi not initialized"));
67
+ };
68
+
69
+ // Pre-define settingsModalProxy
70
+ window.settingsModalProxy = window.settingsModalProxy || {
71
+ openModal: () => console.log("settingsModalProxy not ready"),
72
+ closeModal: () => console.log("settingsModalProxy not ready"),
73
+ switchTab: () => console.log("settingsModalProxy not ready"),
74
+ isOpen: false,
75
+ settings: {},
76
+ activeTab: 'agent',
77
+ provider: 'serveo',
78
+ filteredSections: []
79
+ };
80
</script>
81
82
<!-- Pre-initialize schedulerSettings to ensure Alpine doesn't miss it -->
@@ -103,19 +152,17 @@
152
<script type="module" src="js/history.js"></script>
153
<script type="module" src="index.js"></script>
154
106
- <!-- Then load Alpine.js -->
107
- <script defer src="vendor/alpine/alpine.collapse.min.js"></script>
108
- <!-- <script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.14.3/dist/cdn.min.js"></script> -->
155
+ <!-- Alpine.js is loaded via initFw.js -->
156
<script type="module" src="js/initFw.js"></script>
157
111
- <script src="vendor/ace/ace.js"></script>
112
- <link href="vendor/ace/ace.min.css" rel="stylesheet">
158
+ <script src="https://cdn.jsdelivr.net/npm/ace-builds@1.36.5/src-noconflict/ace.js"></script>
159
+ <link href="https://cdn.jsdelivr.net/npm/ace-builds@1.36.5/css/ace.min.css" rel="stylesheet">
160
<!-- KaTeX CSS -->
114
- <link rel="stylesheet" href="vendor/katex/katex.min.css" crossorigin="anonymous">
161
+ <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.8/dist/katex.min.css" crossorigin="anonymous">
162
163
<!-- KaTeX javascript -->
117
- <script defer src="vendor/katex/katex.min.js" crossorigin="anonymous"></script>
118
- <script defer src="vendor/katex/katex.auto-render.min.js"
164
+ <script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.8/dist/katex.min.js" crossorigin="anonymous"></script>
165
+ <script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.8/dist/contrib/auto-render.min.js"
166
crossorigin="anonymous"></script>
167
168
<!-- Non-module scripts after Alpine.js -->
@@ -719,20 +766,20 @@
766
<div id="section-tunnel" class="section" x-show="activeTab === 'external'">
767
<div class="section-title">Flare Tunnel</div>
768
<div class="section-description">Create a secure public URL to access your Agent Zero instance anytime, anywhere.</div>
722
- <div class="field">
723
- <div class="field-label">
724
- <div class="field-title">Tunnel provider</div>
725
- <div class="field-description">Select provider for public tunnel</div>
726
- </div>
727
- <div class="field-control">
728
- <select id="tunnel-provider" x-model="provider" :disabled="isLoading">
729
- <option value="serveo">Serveo</option>
730
- <option value="cloudflared">Cloudflare</option>
731
- </select>
732
- </div>
733
- </div>
769
<!-- Tunnel content UI -->
770
<div class="tunnel-container" x-data="tunnelSettings">
771
+ <div class="field">
772
+ <div class="field-label">
773
+ <div class="field-title">Tunnel provider</div>
774
+ <div class="field-description">Select provider for public tunnel</div>
775
+ </div>
776
+ <div class="field-control">
777
+ <select id="tunnel-provider" x-model="provider" :disabled="isLoading">
778
+ <option value="serveo">Serveo</option>
779
+ <option value="cloudflared">Cloudflare</option>
780
+ </select>
781
+ </div>
782
+ </div>
783
<!-- Loading spinner for tunnel operations -->
784
<div class="loading-spinner" x-show="isLoading">
785
<i class="fas fa-spinner fa-spin"></i>
@@ -1701,4 +1748,4 @@
1748
1749
</body>
1750
1704
-</html>
1751
+</html>
\ No newline at end of file
webui/index.js
+33
-5
@@ -347,6 +347,7 @@ function setConnectionStatus(connected) {
347
let lastLogVersion = 0;
348
let lastLogGuid = "";
349
let lastSpokenNo = 0;
350
+let pendingSpeechContent = new Map(); // Buffer for incomplete responses
351
352
async function poll() {
353
let updated = false;
@@ -516,14 +517,17 @@ function afterMessagesUpdate(logs) {
517
}
518
519
function speakMessages(logs) {
519
- // log.no, log.type, log.heading, log.content
520
+ // EXTENDED FIX: Only speak VERY long responses to ensure completeness
521
for (let i = logs.length - 1; i >= 0; i--) {
522
const log = logs[i];
522
- if (log.type == "response") {
523
- if (log.no > lastSpokenNo) {
523
+ if (log.type == "response" && !log.temp) {
524
+ if (log.no > lastSpokenNo && log.content.length > 300) { // Much higher threshold
525
lastSpokenNo = log.no;
526
+ console.log(`[SPEECH] Speaking full response (${log.content.length} chars): "${log.content.substring(0, 200)}..."`);
527
speech.speak(log.content);
528
return;
529
+ } else if (log.content.length <= 300) {
530
+ console.log(`[SPEECH] Skipping short response (${log.content.length} chars): "${log.content.substring(0, 100)}..."`);
531
}
532
}
533
}
@@ -713,6 +717,12 @@ export const setContext = function (id) {
717
lastLogGuid = "";
718
lastLogVersion = 0;
719
lastSpokenNo = 0;
720
+
721
+ // CRITICAL FIX: Clear speech buffer and stop any playing audio
722
+ pendingSpeechContent.clear();
723
+ if (window.speechStore) {
724
+ speechStore.stop();
725
+ }
726
727
// Clear the chat history immediately to avoid showing stale content
728
chatHistory.innerHTML = "";
@@ -1202,11 +1212,22 @@ window.handleFileUpload = function (event) {
1212
handleFiles(files, inputAD);
1213
};
1214
1205
-// Setup event handlers once the DOM is fully loaded
1206
-document.addEventListener("DOMContentLoaded", function () {
1215
+// Setup event handlers once Alpine.js is ready
1216
+function initializeUI() {
1217
setupSidebarToggle();
1218
setupTabs();
1219
initializeActiveTab();
1220
+}
1221
+
1222
+// Wait for Alpine.js to be ready
1223
+document.addEventListener("DOMContentLoaded", function () {
1224
+ // If Alpine is already available, initialize immediately
1225
+ if (window.Alpine) {
1226
+ initializeUI();
1227
+ } else {
1228
+ // Wait for Alpine to be loaded
1229
+ document.addEventListener('alpine:init', initializeUI);
1230
+ }
1231
});
1232
1233
// Setup tabs functionality
@@ -1229,6 +1250,13 @@ function setupTabs() {
1250
}
1251
1252
function activateTab(tabName) {
1253
+ // Safety check for Alpine.js
1254
+ if (!window.Alpine) {
1255
+ console.error("Alpine.js not ready, deferring tab activation");
1256
+ setTimeout(() => activateTab(tabName), 100);
1257
+ return;
1258
+ }
1259
+
1260
const chatsTab = document.getElementById("chats-tab");
1261
const tasksTab = document.getElementById("tasks-tab");
1262
const chatsSection = document.getElementById("chats-section");
webui/js/initFw.js
+25
@@ -2,6 +2,7 @@ import * as _modals from "./modals.js";
2
import * as _components from "./components.js";
3
4
await import("../vendor/alpine/alpine.min.js");
5
+await import("../vendor/alpine/alpine.collapse.min.js");
6
7
// add x-destroy directive
8
Alpine.directive(
@@ -11,3 +12,27 @@ Alpine.directive(
12
cleanup(() => onDestroy());
13
}
14
);
15
+
16
+// Wait for all modules to load before starting Alpine.js
17
+await new Promise(resolve => {
18
+ if (document.readyState === 'loading') {
19
+ document.addEventListener('DOMContentLoaded', resolve);
20
+ } else {
21
+ resolve();
22
+ }
23
+});
24
+
25
+// Give a small delay to ensure all module scripts have loaded
26
+await new Promise(resolve => setTimeout(resolve, 100));
27
+
28
+console.log("Starting Alpine.js...");
29
+// Start Alpine.js only if not already started
30
+try {
31
+ Alpine.start();
32
+} catch (error) {
33
+ if (error.message && error.message.includes('already been initialized')) {
34
+ console.log("Alpine.js already started, skipping initialization");
35
+ } else {
36
+ console.error("Error starting Alpine.js:", error);
37
+ }
38
+}
\ No newline at end of file
webui/js/settings.js
+4
-1
@@ -296,6 +296,9 @@ const settingsModalProxy = {
296
};
297
298
299
+// Make settingsModalProxy globally available
300
+window.settingsModalProxy = settingsModalProxy;
301
+
302
// function initSettingsModal() {
303
304
// window.openSettings = function () {
@@ -595,4 +598,4 @@ function showToast(message, type = 'info') {
598
document.body.removeChild(toast);
599
}, 300);
600
}, 3000);
598
-}
601
+}
\ No newline at end of file
webui/js/speech-store.js
new
+522
@@ -0,0 +1,522 @@
1
+import { createStore } from "./AlpineStore.js";
2
+import { updateChatInput, sendMessage } from "../index.js";
3
+
4
+const Status = {
5
+ INACTIVE: "inactive",
6
+ ACTIVATING: "activating",
7
+ LISTENING: "listening",
8
+ RECORDING: "recording",
9
+ WAITING: "waiting",
10
+ PROCESSING: "processing",
11
+};
12
+
13
+// Create the speech store
14
+export const speechStore = createStore('speech', {
15
+ // STT Settings
16
+ stt_model_size: "tiny",
17
+ stt_language: "en",
18
+ stt_silence_threshold: 0.05,
19
+ stt_silence_duration: 1000,
20
+ stt_waiting_timeout: 2000,
21
+
22
+ // TTS Settings
23
+ tts_enabled: false,
24
+
25
+ // TTS State
26
+ isSpeaking: false,
27
+ currentAudio: null,
28
+ audioContext: null,
29
+ userHasInteracted: false,
30
+
31
+ // STT State
32
+ microphoneInput: null,
33
+ micStatus: Status.INACTIVE,
34
+
35
+ // Initialize speech functionality
36
+ async init() {
37
+ await this.loadSettings();
38
+ this.setupBrowserTTS();
39
+ this.setupUserInteractionHandling();
40
+ },
41
+
42
+ // Load settings from server
43
+ async loadSettings() {
44
+ try {
45
+ const response = await fetchApi("/settings_get", { method: "POST" });
46
+ const data = await response.json();
47
+ const speechSection = data.settings.sections.find(s => s.title === "Speech");
48
+
49
+ if (speechSection) {
50
+ speechSection.fields.forEach(field => {
51
+ if (this.hasOwnProperty(field.id)) {
52
+ this[field.id] = field.value;
53
+ }
54
+ });
55
+ }
56
+ } catch (error) {
57
+ window.toastFetchError("Failed to load speech settings", error);
58
+ console.error("Failed to load speech settings:", error);
59
+ }
60
+ },
61
+
62
+ // Setup browser TTS
63
+ setupBrowserTTS() {
64
+ this.synth = window.speechSynthesis;
65
+ this.browserUtterance = null;
66
+ },
67
+
68
+ // Setup user interaction handling for autoplay policy
69
+ setupUserInteractionHandling() {
70
+ const enableAudio = () => {
71
+ if (!this.userHasInteracted) {
72
+ this.userHasInteracted = true;
73
+ console.log("User interaction detected - audio playback enabled");
74
+
75
+ // Create a dummy audio context to "unlock" audio
76
+ try {
77
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
78
+ this.audioContext.resume();
79
+ } catch (e) {
80
+ console.log("AudioContext not available");
81
+ }
82
+ }
83
+ };
84
+
85
+ // Listen for any user interaction
86
+ const events = ['click', 'touchstart', 'keydown', 'mousedown'];
87
+ events.forEach(event => {
88
+ document.addEventListener(event, enableAudio, { once: true, passive: true });
89
+ });
90
+ },
91
+
92
+ // Main speak function
93
+ async speak(text) {
94
+ if (!this.tts_enabled) return;
95
+ if (this.isSpeaking) return;
96
+
97
+ text = this.cleanText(text);
98
+ if (!text.trim()) return;
99
+
100
+ if (!this.userHasInteracted) {
101
+ this.showAudioPermissionPrompt();
102
+ return;
103
+ }
104
+
105
+ try {
106
+ await this.speakWithKokoro(text);
107
+ } catch (error) {
108
+ console.error("TTS error:", error);
109
+ this.speakWithBrowser(text);
110
+ }
111
+ },
112
+
113
+ // Show a prompt to user to enable audio
114
+ showAudioPermissionPrompt() {
115
+ if (window.toast) {
116
+ window.toast("Click anywhere to enable audio playback", "info", 5000);
117
+ } else {
118
+ console.log("Please click anywhere on the page to enable audio playback");
119
+ }
120
+ },
121
+
122
+ // Browser TTS
123
+ speakWithBrowser(text) {
124
+ this.browserUtterance = new SpeechSynthesisUtterance(text);
125
+ this.browserUtterance.onstart = () => { this.isSpeaking = true; };
126
+ this.browserUtterance.onend = () => { this.isSpeaking = false; };
127
+ this.synth.speak(this.browserUtterance);
128
+ },
129
+
130
+ // Kokoro TTS
131
+ async speakWithKokoro(text) {
132
+ try {
133
+ const response = await sendJsonData("/synthesize", { text });
134
+
135
+ if (response.success) {
136
+ if (response.audio_parts) {
137
+ // Multiple chunks - play sequentially
138
+ for (const audioPart of response.audio_parts) {
139
+ await this.playAudio(audioPart);
140
+ await new Promise(resolve => setTimeout(resolve, 100)); // Brief pause
141
+ }
142
+ } else if (response.audio) {
143
+ // Single audio
144
+ await this.playAudio(response.audio);
145
+ }
146
+ } else {
147
+ console.error("Kokoro TTS error:", response.error);
148
+ this.speakWithBrowser(text);
149
+ }
150
+ } catch (error) {
151
+ console.error("Kokoro TTS error:", error);
152
+ this.speakWithBrowser(text);
153
+ }
154
+ },
155
+
156
+
157
+ // Play base64 audio
158
+ async playAudio(base64Audio) {
159
+ return new Promise((resolve, reject) => {
160
+ const audio = new Audio();
161
+
162
+ audio.onplay = () => {
163
+ this.isSpeaking = true;
164
+ };
165
+ audio.onended = () => {
166
+ this.isSpeaking = false;
167
+ this.currentAudio = null;
168
+ resolve();
169
+ };
170
+ audio.onerror = (error) => {
171
+ this.isSpeaking = false;
172
+ this.currentAudio = null;
173
+ reject(error);
174
+ };
175
+
176
+ audio.src = `data:audio/wav;base64,${base64Audio}`;
177
+ this.currentAudio = audio;
178
+
179
+ audio.play().catch(error => {
180
+ this.isSpeaking = false;
181
+ this.currentAudio = null;
182
+
183
+ if (error.name === 'NotAllowedError') {
184
+ this.showAudioPermissionPrompt();
185
+ this.userHasInteracted = false;
186
+ }
187
+ reject(error);
188
+ });
189
+ });
190
+ },
191
+
192
+ // Stop all speech
193
+ stop() {
194
+ if (this.synth?.speaking) {
195
+ this.synth.cancel();
196
+ }
197
+
198
+ if (this.currentAudio) {
199
+ this.currentAudio.pause();
200
+ this.currentAudio.currentTime = 0;
201
+ this.currentAudio = null;
202
+ }
203
+
204
+ this.isSpeaking = false;
205
+ },
206
+
207
+ // Clean text for TTS
208
+ cleanText(text) {
209
+ return text
210
+ .replace(/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, "")
211
+ .replace(/https?:\/\/[^\s]+/g, "")
212
+ .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, "")
213
+ .replace(/\s+/g, " ")
214
+ .trim();
215
+ },
216
+
217
+ // Initialize microphone input
218
+ async initMicrophone() {
219
+ if (this.microphoneInput) return this.microphoneInput;
220
+
221
+ this.microphoneInput = new MicrophoneInput(
222
+ async (text, isFinal) => {
223
+ if (isFinal) {
224
+ updateChatInput(text);
225
+ if (!this.microphoneInput.messageSent) {
226
+ this.microphoneInput.messageSent = true;
227
+ await sendMessage();
228
+ }
229
+ }
230
+ }
231
+ );
232
+
233
+ this.micStatus = Status.ACTIVATING;
234
+ const initialized = await this.microphoneInput.initialize();
235
+ return initialized ? this.microphoneInput : null;
236
+ },
237
+
238
+ // Toggle microphone
239
+ async toggleMicrophone() {
240
+ const hasPermission = await this.requestMicrophonePermission();
241
+ if (!hasPermission) return;
242
+
243
+ if (!this.microphoneInput && !(await this.initMicrophone())) {
244
+ return;
245
+ }
246
+
247
+ this.micStatus = this.micStatus === Status.INACTIVE || this.micStatus === Status.ACTIVATING
248
+ ? Status.LISTENING
249
+ : Status.INACTIVE;
250
+ },
251
+
252
+ // Request microphone permission
253
+ async requestMicrophonePermission() {
254
+ try {
255
+ await navigator.mediaDevices.getUserMedia({ audio: true });
256
+ return true;
257
+ } catch (err) {
258
+ console.error("Error accessing microphone:", err);
259
+ toast("Microphone access denied. Please enable microphone access in your browser settings.", "error");
260
+ return false;
261
+ }
262
+ }
263
+});
264
+
265
+// Microphone Input Class (simplified for store integration)
266
+class MicrophoneInput {
267
+ constructor(updateCallback) {
268
+ this.mediaRecorder = null;
269
+ this.audioChunks = [];
270
+ this.lastChunk = [];
271
+ this.updateCallback = updateCallback;
272
+ this.messageSent = false;
273
+ this.audioContext = null;
274
+ this.mediaStreamSource = null;
275
+ this.analyserNode = null;
276
+ this._status = Status.INACTIVE;
277
+ this.lastAudioTime = null;
278
+ this.waitingTimer = null;
279
+ this.silenceStartTime = null;
280
+ this.hasStartedRecording = false;
281
+ this.analysisFrame = null;
282
+ }
283
+
284
+ get status() {
285
+ return this._status;
286
+ }
287
+
288
+ set status(newStatus) {
289
+ if (this._status === newStatus) return;
290
+
291
+ const oldStatus = this._status;
292
+ this._status = newStatus;
293
+ speechStore.micStatus = newStatus;
294
+ console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);
295
+
296
+ this.handleStatusChange(oldStatus, newStatus);
297
+ }
298
+
299
+ async initialize() {
300
+ try {
301
+ const stream = await navigator.mediaDevices.getUserMedia({
302
+ audio: { echoCancellation: true, noiseSuppression: true, channelCount: 1 }
303
+ });
304
+
305
+ this.mediaRecorder = new MediaRecorder(stream);
306
+ this.mediaRecorder.ondataavailable = (event) => {
307
+ if (event.data.size > 0 && (this.status === Status.RECORDING || this.status === Status.WAITING)) {
308
+ if (this.lastChunk) {
309
+ this.audioChunks.push(this.lastChunk);
310
+ this.lastChunk = null;
311
+ }
312
+ this.audioChunks.push(event.data);
313
+ } else if (this.status === Status.LISTENING) {
314
+ this.lastChunk = event.data;
315
+ }
316
+ };
317
+
318
+ this.setupAudioAnalysis(stream);
319
+ return true;
320
+ } catch (error) {
321
+ console.error("Microphone initialization error:", error);
322
+ toast("Failed to access microphone. Please check permissions.", "error");
323
+ return false;
324
+ }
325
+ }
326
+
327
+ handleStatusChange(oldStatus, newStatus) {
328
+ if (newStatus != Status.RECORDING) {
329
+ this.lastChunk = null;
330
+ }
331
+
332
+ switch (newStatus) {
333
+ case Status.INACTIVE:
334
+ this.handleInactiveState();
335
+ break;
336
+ case Status.LISTENING:
337
+ this.handleListeningState();
338
+ break;
339
+ case Status.RECORDING:
340
+ this.handleRecordingState();
341
+ break;
342
+ case Status.WAITING:
343
+ this.handleWaitingState();
344
+ break;
345
+ case Status.PROCESSING:
346
+ this.handleProcessingState();
347
+ break;
348
+ }
349
+ }
350
+
351
+ handleInactiveState() {
352
+ this.stopRecording();
353
+ this.stopAudioAnalysis();
354
+ if (this.waitingTimer) {
355
+ clearTimeout(this.waitingTimer);
356
+ this.waitingTimer = null;
357
+ }
358
+ }
359
+
360
+ handleListeningState() {
361
+ this.stopRecording();
362
+ this.audioChunks = [];
363
+ this.hasStartedRecording = false;
364
+ this.silenceStartTime = null;
365
+ this.lastAudioTime = null;
366
+ this.messageSent = false;
367
+ this.startAudioAnalysis();
368
+ }
369
+
370
+ handleRecordingState() {
371
+ if (!this.hasStartedRecording && this.mediaRecorder.state !== "recording") {
372
+ this.hasStartedRecording = true;
373
+ this.mediaRecorder.start(1000);
374
+ console.log("Speech started");
375
+ }
376
+ if (this.waitingTimer) {
377
+ clearTimeout(this.waitingTimer);
378
+ this.waitingTimer = null;
379
+ }
380
+ }
381
+
382
+ handleWaitingState() {
383
+ this.waitingTimer = setTimeout(() => {
384
+ if (this.status === Status.WAITING) {
385
+ this.status = Status.PROCESSING;
386
+ }
387
+ }, speechStore.stt_waiting_timeout);
388
+ }
389
+
390
+ handleProcessingState() {
391
+ this.stopRecording();
392
+ this.process();
393
+ }
394
+
395
+ setupAudioAnalysis(stream) {
396
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
397
+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
398
+ this.analyserNode = this.audioContext.createAnalyser();
399
+ this.analyserNode.fftSize = 2048;
400
+ this.analyserNode.minDecibels = -90;
401
+ this.analyserNode.maxDecibels = -10;
402
+ this.analyserNode.smoothingTimeConstant = 0.85;
403
+ this.mediaStreamSource.connect(this.analyserNode);
404
+ }
405
+
406
+ startAudioAnalysis() {
407
+ const analyzeFrame = () => {
408
+ if (this.status === Status.INACTIVE) return;
409
+
410
+ const dataArray = new Uint8Array(this.analyserNode.fftSize);
411
+ this.analyserNode.getByteTimeDomainData(dataArray);
412
+
413
+ let sum = 0;
414
+ for (let i = 0; i < dataArray.length; i++) {
415
+ const amplitude = (dataArray[i] - 128) / 128;
416
+ sum += amplitude * amplitude;
417
+ }
418
+ const rms = Math.sqrt(sum / dataArray.length);
419
+ const now = Date.now();
420
+
421
+ // Update status based on audio level (ignore if TTS is speaking)
422
+ if (rms > this.densify(speechStore.stt_silence_threshold)) {
423
+ this.lastAudioTime = now;
424
+ this.silenceStartTime = null;
425
+
426
+ if ((this.status === Status.LISTENING || this.status === Status.WAITING) && !speechStore.isSpeaking && !speechStore.isGenerating) {
427
+ this.status = Status.RECORDING;
428
+ }
429
+ } else if (this.status === Status.RECORDING) {
430
+ if (!this.silenceStartTime) {
431
+ this.silenceStartTime = now;
432
+ }
433
+
434
+ const silenceDuration = now - this.silenceStartTime;
435
+ if (silenceDuration >= speechStore.stt_silence_duration) {
436
+ this.status = Status.WAITING;
437
+ }
438
+ }
439
+
440
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
441
+ };
442
+
443
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
444
+ }
445
+
446
+ stopAudioAnalysis() {
447
+ if (this.analysisFrame) {
448
+ cancelAnimationFrame(this.analysisFrame);
449
+ this.analysisFrame = null;
450
+ }
451
+ }
452
+
453
+ stopRecording() {
454
+ if (this.mediaRecorder?.state === "recording") {
455
+ this.mediaRecorder.stop();
456
+ this.hasStartedRecording = false;
457
+ }
458
+ }
459
+
460
+ densify(x) {
461
+ return Math.exp(-5 * (1 - x));
462
+ }
463
+
464
+ async process() {
465
+ if (this.audioChunks.length === 0) {
466
+ this.status = Status.LISTENING;
467
+ return;
468
+ }
469
+
470
+ const audioBlob = new Blob(this.audioChunks, { type: "audio/wav" });
471
+ const base64 = await this.convertBlobToBase64Wav(audioBlob);
472
+
473
+ try {
474
+ const result = await sendJsonData("/transcribe", { audio: base64 });
475
+ const text = this.filterResult(result.text || "");
476
+
477
+ if (text) {
478
+ console.log("Transcription:", result.text);
479
+ await this.updateCallback(result.text, true);
480
+ }
481
+ } catch (error) {
482
+ window.toastFetchError("Transcription error", error);
483
+ console.error("Transcription error:", error);
484
+ } finally {
485
+ this.audioChunks = [];
486
+ this.status = Status.LISTENING;
487
+ }
488
+ }
489
+
490
+ convertBlobToBase64Wav(audioBlob) {
491
+ return new Promise((resolve, reject) => {
492
+ const reader = new FileReader();
493
+ reader.onloadend = () => {
494
+ const base64Data = reader.result.split(",")[1];
495
+ resolve(base64Data);
496
+ };
497
+ reader.onerror = (error) => reject(error);
498
+ reader.readAsDataURL(audioBlob);
499
+ });
500
+ }
501
+
502
+ filterResult(text) {
503
+ text = text.trim();
504
+ let ok = false;
505
+ while (!ok) {
506
+ if (!text) break;
507
+ if (text[0] === "{" && text[text.length - 1] === "}") break;
508
+ if (text[0] === "(" && text[text.length - 1] === ")") break;
509
+ if (text[0] === "[" && text[text.length - 1] === "]") break;
510
+ ok = true;
511
+ }
512
+ if (ok) return text;
513
+ else console.log(`Discarding transcription: ${text}`);
514
+ }
515
+}
516
+
517
+// Initialize speech store
518
+window.speechStore = speechStore;
519
+
520
+// Event listeners
521
+document.addEventListener("settings-updated", () => speechStore.loadSettings());
522
+document.addEventListener("DOMContentLoaded", () => speechStore.init());
\ No newline at end of file
webui/js/speech.js
+22
-451
@@ -1,376 +1,31 @@
1
-// import { pipeline, read_audio } from '../transformers@3.0.2.js';
2
-import { updateChatInput, sendMessage } from "../index.js";
1
+import { speechStore } from "./speech-store.js";
2
3
const microphoneButton = document.getElementById("microphone-button");
5
-let microphoneInput = null;
4
let isProcessingClick = false;
5
8
-const Status = {
9
- INACTIVE: "inactive",
10
- ACTIVATING: "activating",
11
- LISTENING: "listening",
12
- RECORDING: "recording",
13
- WAITING: "waiting",
14
- PROCESSING: "processing",
15
-};
16
-
17
-const micSettings = {
18
- stt_model_size: "tiny",
19
- stt_language: "en",
20
- stt_silence_threshold: 0.05,
21
- stt_silence_duration: 1000,
22
- stt_waiting_timeout: 2000,
23
-};
24
-window.micSettings = micSettings;
25
-loadMicSettings();
26
-
27
-function densify(x) {
28
- return Math.exp(-5 * (1 - x));
29
-}
30
-
31
-async function loadMicSettings() {
32
- try {
33
- const response = await fetchApi("/settings_get", {
34
- method: "POST",
35
- });
36
- const data = await response.json();
37
- const sttSettings = data.settings.sections.find(
38
- (s) => s.title === "Speech to Text"
39
- );
40
-
41
- if (sttSettings) {
42
- // Update options from server settings
43
- sttSettings.fields.forEach((field) => {
44
- const key = field.id; //.split('.')[1]; // speech_to_text.model_size -> model_size
45
- micSettings[key] = field.value;
46
- });
47
- }
48
- } catch (error) {
49
- window.toastFetchError("Failed to load speech settings", error);
50
- console.error("Failed to load speech settings:", error);
51
- }
52
-}
53
-
54
-class MicrophoneInput {
55
- constructor(updateCallback, options = {}) {
56
- this.mediaRecorder = null;
57
- this.audioChunks = [];
58
- this.lastChunk = [];
59
- this.updateCallback = updateCallback;
60
- this.messageSent = false;
61
-
62
- // Audio analysis properties
63
- this.audioContext = null;
64
- this.mediaStreamSource = null;
65
- this.analyserNode = null;
66
- this._status = Status.INACTIVE;
67
-
68
- // Timing properties
69
- this.lastAudioTime = null;
70
- this.waitingTimer = null;
71
- this.silenceStartTime = null;
72
- this.hasStartedRecording = false;
73
- this.analysisFrame = null;
74
- }
75
-
76
- get status() {
77
- return this._status;
78
- }
79
-
80
- set status(newStatus) {
81
- if (this._status === newStatus) return;
82
-
83
- const oldStatus = this._status;
84
- this._status = newStatus;
85
- console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);
86
-
87
- // Update UI
88
- microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`);
89
- microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`);
90
- microphoneButton.setAttribute("data-status", newStatus);
91
-
92
- // Handle state-specific behaviors
93
- this.handleStatusChange(oldStatus, newStatus);
94
- }
95
-
96
- handleStatusChange(oldStatus, newStatus) {
97
- //last chunk kept only for transition to recording status
98
- if (newStatus != Status.RECORDING) {
99
- this.lastChunk = null;
100
- }
101
-
102
- switch (newStatus) {
103
- case Status.INACTIVE:
104
- this.handleInactiveState();
105
- break;
106
- case Status.LISTENING:
107
- this.handleListeningState();
108
- break;
109
- case Status.RECORDING:
110
- this.handleRecordingState();
111
- break;
112
- case Status.WAITING:
113
- this.handleWaitingState();
114
- break;
115
- case Status.PROCESSING:
116
- this.handleProcessingState();
117
- break;
118
- }
119
- }
120
-
121
- handleInactiveState() {
122
- this.stopRecording();
123
- this.stopAudioAnalysis();
124
- if (this.waitingTimer) {
125
- clearTimeout(this.waitingTimer);
126
- this.waitingTimer = null;
127
- }
128
- }
129
-
130
- handleListeningState() {
131
- this.stopRecording();
132
- this.audioChunks = [];
133
- this.hasStartedRecording = false;
134
- this.silenceStartTime = null;
135
- this.lastAudioTime = null;
136
- this.messageSent = false;
137
- this.startAudioAnalysis();
138
- }
139
-
140
- handleRecordingState() {
141
- if (!this.hasStartedRecording && this.mediaRecorder.state !== "recording") {
142
- this.hasStartedRecording = true;
143
- this.mediaRecorder.start(1000);
144
- console.log("Speech started");
145
- }
146
- if (this.waitingTimer) {
147
- clearTimeout(this.waitingTimer);
148
- this.waitingTimer = null;
149
- }
150
- }
151
-
152
- handleWaitingState() {
153
- // Don't stop recording during waiting state
154
- this.waitingTimer = setTimeout(() => {
155
- if (this.status === Status.WAITING) {
156
- this.status = Status.PROCESSING;
157
- }
158
- }, micSettings.stt_waiting_timeout);
159
- }
160
-
161
- handleProcessingState() {
162
- this.stopRecording();
163
- this.process();
164
- }
165
-
166
- stopRecording() {
167
- if (this.mediaRecorder?.state === "recording") {
168
- this.mediaRecorder.stop();
169
- this.hasStartedRecording = false;
170
- }
171
- }
172
-
173
- async initialize() {
174
- try {
175
- const stream = await navigator.mediaDevices.getUserMedia({
176
- audio: {
177
- echoCancellation: true,
178
- noiseSuppression: true,
179
- channelCount: 1,
180
- },
181
- });
182
-
183
- this.mediaRecorder = new MediaRecorder(stream);
184
- this.mediaRecorder.ondataavailable = (event) => {
185
- if (
186
- event.data.size > 0 &&
187
- (this.status === Status.RECORDING || this.status === Status.WAITING)
188
- ) {
189
- if (this.lastChunk) {
190
- this.audioChunks.push(this.lastChunk);
191
- this.lastChunk = null;
192
- }
193
- this.audioChunks.push(event.data);
194
- console.log(
195
- "Audio chunk received, total chunks:",
196
- this.audioChunks.length
197
- );
198
- } else if (this.status === Status.LISTENING) {
199
- this.lastChunk = event.data;
200
- }
201
- };
202
-
203
- this.setupAudioAnalysis(stream);
204
- return true;
205
- } catch (error) {
206
- console.error("Microphone initialization error:", error);
207
- toast("Failed to access microphone. Please check permissions.", "error");
208
- return false;
209
- }
210
- }
211
-
212
- setupAudioAnalysis(stream) {
213
- this.audioContext = new (window.AudioContext ||
214
- window.webkitAudioContext)();
215
- this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
216
- this.analyserNode = this.audioContext.createAnalyser();
217
- this.analyserNode.fftSize = 2048;
218
- this.analyserNode.minDecibels = -90;
219
- this.analyserNode.maxDecibels = -10;
220
- this.analyserNode.smoothingTimeConstant = 0.85;
221
- this.mediaStreamSource.connect(this.analyserNode);
222
- }
223
-
224
- startAudioAnalysis() {
225
- const analyzeFrame = () => {
226
- if (this.status === Status.INACTIVE) return;
227
-
228
- const dataArray = new Uint8Array(this.analyserNode.fftSize);
229
- this.analyserNode.getByteTimeDomainData(dataArray);
230
-
231
- // Calculate RMS volume
232
- let sum = 0;
233
- for (let i = 0; i < dataArray.length; i++) {
234
- const amplitude = (dataArray[i] - 128) / 128;
235
- sum += amplitude * amplitude;
236
- }
237
- const rms = Math.sqrt(sum / dataArray.length);
238
-
239
- const now = Date.now();
240
-
241
- // Update status based on audio level
242
- if (rms > densify(micSettings.stt_silence_threshold)) {
243
- this.lastAudioTime = now;
244
- this.silenceStartTime = null;
245
-
246
- if (
247
- this.status === Status.LISTENING ||
248
- this.status === Status.WAITING
249
- ) {
250
- if (!speech.isSpeaking())
251
- // TODO? a better way to ignore agent's voice?
252
- this.status = Status.RECORDING;
253
- }
254
- } else if (this.status === Status.RECORDING) {
255
- if (!this.silenceStartTime) {
256
- this.silenceStartTime = now;
257
- }
258
-
259
- const silenceDuration = now - this.silenceStartTime;
260
- if (silenceDuration >= micSettings.stt_silence_duration) {
261
- this.status = Status.WAITING;
262
- }
263
- }
264
-
265
- this.analysisFrame = requestAnimationFrame(analyzeFrame);
266
- };
267
-
268
- this.analysisFrame = requestAnimationFrame(analyzeFrame);
269
- }
270
-
271
- stopAudioAnalysis() {
272
- if (this.analysisFrame) {
273
- cancelAnimationFrame(this.analysisFrame);
274
- this.analysisFrame = null;
275
- }
276
- }
277
-
278
- async process() {
279
- if (this.audioChunks.length === 0) {
280
- this.status = Status.LISTENING;
281
- return;
282
- }
283
-
284
- const audioBlob = new Blob(this.audioChunks, { type: "audio/wav" });
285
- const base64 = await this.convertBlobToBase64Wav(audioBlob);
286
-
287
- try {
288
- const result = await sendJsonData("/transcribe", { audio: base64 });
289
-
290
- const text = this.filterResult(result.text || "");
291
-
292
- if (text) {
293
- console.log("Transcription:", result.text);
294
- await this.updateCallback(result.text, true);
295
- }
296
- } catch (error) {
297
- window.toastFetchError("Transcription error", error);
298
- console.error("Transcription error:", error);
299
- } finally {
300
- this.audioChunks = [];
301
- this.status = Status.LISTENING;
302
- }
303
- }
304
-
305
- convertBlobToBase64Wav(audioBlob) {
306
- return new Promise((resolve, reject) => {
307
- const reader = new FileReader();
308
-
309
- // Read the Blob as a Data URL
310
- reader.onloadend = () => {
311
- const base64Data = reader.result.split(",")[1]; // Extract Base64 data
312
- resolve(base64Data);
313
- };
314
-
315
- reader.onerror = (error) => {
316
- reject(error);
317
- };
318
-
319
- reader.readAsDataURL(audioBlob); // Start reading the Blob
320
- });
321
- }
322
-
323
- filterResult(text) {
324
- text = text.trim();
325
- let ok = false;
326
- while (!ok) {
327
- if (!text) break;
328
- if (text[0] === "{" && text[text.length - 1] === "}") break;
329
- if (text[0] === "(" && text[text.length - 1] === ")") break;
330
- if (text[0] === "[" && text[text.length - 1] === "]") break;
331
- ok = true;
332
- }
333
- if (ok) return text;
334
- else console.log(`Discarding transcription: ${text}`);
335
- }
6
+// Update microphone button UI based on store status
7
+function updateMicrophoneButtonUI() {
8
+ const status = speechStore.micStatus;
9
+
10
+ microphoneButton.classList.remove('mic-inactive', 'mic-activating', 'mic-listening', 'mic-recording', 'mic-waiting', 'mic-processing');
11
+ microphoneButton.classList.add(`mic-${status.toLowerCase()}`);
12
+ microphoneButton.setAttribute("data-status", status);
13
}
14
338
-// Initialize and handle click events
339
-async function initializeMicrophoneInput() {
340
- window.microphoneInput = microphoneInput = new MicrophoneInput(
341
- async (text, isFinal) => {
342
- if (isFinal) {
343
- updateChatInput(text);
344
- if (!microphoneInput.messageSent) {
345
- microphoneInput.messageSent = true;
346
- await sendMessage();
347
- }
348
- }
349
- }
350
- );
351
- microphoneInput.status = Status.ACTIVATING;
352
-
353
- return await microphoneInput.initialize();
354
-}
15
+// Watch store for status changes
16
+document.addEventListener("alpine:init", () => {
17
+ Alpine.effect(() => {
18
+ updateMicrophoneButtonUI();
19
+ });
20
+});
21
22
+// Microphone button click handler
23
microphoneButton.addEventListener("click", async () => {
24
if (isProcessingClick) return;
25
isProcessingClick = true;
26
360
- const hasPermission = await requestMicrophonePermission();
361
- if (!hasPermission) return;
362
-
27
try {
364
- if (!microphoneInput && !(await initializeMicrophoneInput())) {
365
- return;
366
- }
367
-
368
- // Simply toggle between INACTIVE and LISTENING states
369
- microphoneInput.status =
370
- microphoneInput.status === Status.INACTIVE ||
371
- microphoneInput.status === Status.ACTIVATING
372
- ? Status.LISTENING
373
- : Status.INACTIVE;
28
+ await speechStore.toggleMicrophone();
29
} finally {
30
setTimeout(() => {
31
isProcessingClick = false;
@@ -378,104 +33,20 @@ microphoneButton.addEventListener("click", async () => {
33
}
34
});
35
381
-// Some error handling for microphone input
382
-async function requestMicrophonePermission() {
383
- try {
384
- await navigator.mediaDevices.getUserMedia({ audio: true });
385
- return true;
386
- } catch (err) {
387
- console.error("Error accessing microphone:", err);
388
- toast(
389
- "Microphone access denied. Please enable microphone access in your browser settings.",
390
- "error"
391
- );
392
- return false;
393
- }
394
-}
395
-
36
+// Create simplified Speech class for backward compatibility
37
class Speech {
397
- constructor() {
398
- this.synth = window.speechSynthesis;
399
- this.utterance = null;
400
- }
401
-
402
- stripEmojis(str) {
403
- return str
404
- .replace(
405
- /([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g,
406
- ""
407
- )
408
- .replace(/\s+/g, " ")
409
- .trim();
410
- }
411
-
412
- speak(text) {
413
- console.log("Speaking:", text);
414
- // Stop any current utterance
415
- this.stop();
416
-
417
- // Remove emojis and create a new utterance
418
- text = this.stripEmojis(text);
419
- text = this.replaceURLs(text);
420
- text = this.replaceGuids(text);
421
- this.utterance = new SpeechSynthesisUtterance(text);
422
-
423
- // Speak the new utterance
424
- this.synth.speak(this.utterance);
425
- }
426
-
427
- replaceURLs(text) {
428
- const urlRegex =
429
- /(\b(https?|ftp|file):\/\/[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b(www\.)[-A-Z0-9+&@#\/%?=~_|!:,.;]*[-A-Z0-9+&@#\/%=~_|])|(\b[-A-Z0-9+&@#\/%?=~_|!:,.;]*\.(?:[A-Z]{2,})[-A-Z0-9+&@#\/%?=~_|])/gi;
430
- return text.replace(urlRegex, (url) => {
431
- let text = url;
432
- // if contains ://, split by it
433
- if (text.includes("://")) text = text.split("://")[1];
434
- // if contains /, split by it
435
- if (text.includes("/")) text = text.split("/")[0];
436
-
437
- // if contains ., split by it
438
- if (text.includes(".")) {
439
- const doms = text.split(".");
440
- //up to last two
441
- return doms[doms.length - 2] + "." + doms[doms.length - 1];
442
- } else {
443
- return text;
444
- }
445
- });
446
- }
447
-
448
- replaceGuids(text) {
449
- const guidRegex =
450
- /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g;
451
- return text.replace(guidRegex, "");
452
- }
453
-
454
- replaceNonText(text) {
455
- const nonTextRegex = /\w[^\w\s]*\w(?=\s|$)|[^\w\s]+/g;
456
- text = text.replace(nonTextRegex, (match) => {
457
- return ``;
458
- });
459
- const longStringRegex = /\S{25,}/g;
460
- text = text.replace(longStringRegex, (match) => {
461
- return ``;
462
- });
463
- return text;
38
+ async speak(text) {
39
+ return speechStore.speak(text);
40
}
41
42
stop() {
467
- if (this.isSpeaking()) {
468
- this.synth.cancel();
469
- }
43
+ speechStore.stop();
44
}
45
46
isSpeaking() {
473
- return this.synth?.speaking || false;
47
+ return speechStore.isSpeaking;
48
}
49
}
50
51
export const speech = new Speech();
478
-window.speech = speech;
479
-
480
-// Add event listener for settings changes
481
-document.addEventListener("settings-updated", loadMicSettings);
52
+window.speech = speech;
\ No newline at end of file
webui/public/speech.svg
new
+10
@@ -0,0 +1,10 @@
1
+<svg width="100%" height="100%" data-name="Layer 2" viewBox="0 0 56 56" fill="none" stroke="#333" stroke-width="1.5" stroke-linecap="round" stroke-linejoin="round" xmlns="http://www.w3.org/2000/svg">
2
+ <!-- Speech bubble outline -->
3
+ <path d="M19 27c0-5 5-9 11-9h12c6 0 11 4 11 9v9c0 5-5 9-11 9h-6l-6 6v-6h-6c-6 0-11-4-11-9v-9z" fill="none"/>
4
+ <!-- Accent stroke (bubble base) -->
5
+ <path d="M20 35v-7c0-3 3-6 7-6" stroke="#5E8AF7"/>
6
+ <!-- Speech wave lines -->
7
+ <path d="M31 34h10" stroke="#5E8AF7"/>
8
+ <path d="M33 37h6" stroke="#5E8AF7"/>
9
+ <path d="M35 31h2" stroke="#5E8AF7"/>
10
+</svg>
\ No newline at end of file