Refactor speech stack into built-in Kokoro TTS and Whisper STT plugins
Split the legacy core speech stack into two built-in, independently toggleable plugins: `_kokoro_tts` for TTS and `_whisper_stt` for STT. This refactor keeps dependency installation and bootstrap concerns in Docker/bootstrap/preload, while moving speech-specific tooling, APIs, prompts, UI, and runtime behavior into the plugins. Core now exposes engine-agnostic `tts-service` and `stt-service` brokers, with browser-native TTS preserved as the fallback when Kokoro is disabled. Included in this change: - add built-in `_kokoro_tts` plugin with plugin-owned synth API, config, status UI, and provider registration - add built-in `_whisper_stt` plugin with plugin-owned transcribe API, mic runtime, device UI, prompt injection, and provider registration - remove legacy core speech APIs/helpers/settings/UI and delete unused `webui/js/speech_browser.js` - replace the old hardcoded speech settings section with a generic voice surface backed by plugin extensions - update preload/docs/tests to match the new plugin-owned speech architecture Behavioral intent: - both plugins are built-in but not `always_enabled` - users can now hot-switch TTS and STT independently - browser TTS remains available when `_kokoro_tts` is off - Whisper mic UI only appears when `_whisper_stt` is enabled
Alessandro committed
May 21, 2026 at 05:41 UTC
675afa8dee150f2670b0162291b4a36271b78532
59 files changed
+3331
-2088
README.md
+1
-1
@@ -203,7 +203,7 @@ Agent Zero supports plugins, MCP, A2A, custom tools, custom prompts, project-sco
203
204
- Fully Dockerized runtime with a clean Web UI.
205
- Real-time streamed output so you can interrupt, redirect, or refine the work as it happens.
206
-- Speech-to-text and text-to-speech support.
206
+- Speech-to-text and text-to-speech support through the built-in `_kokoro_tts` and `_whisper_stt` plugins, with browser-native speech synthesis as the fallback output path.
207
- Chat load/save, generated HTML logs, file browser, settings UI, and deployment-friendly `A0_SET_` configuration.
208
209
## Try These First
api/synthesize.py
deleted
-96
@@ -1,96 +0,0 @@
1
-# api/synthesize.py
2
-
3
-from helpers.api import ApiHandler, Request, Response
4
-
5
-from helpers import runtime, settings, kokoro_tts
6
-
7
-class Synthesize(ApiHandler):
8
- async def process(self, input: dict, request: Request) -> dict | Response:
9
- text = input.get("text", "")
10
- ctxid = input.get("ctxid", "")
11
-
12
- if ctxid:
13
- context = self.use_context(ctxid)
14
-
15
- # if not await kokoro_tts.is_downloaded():
16
- # context.log.log(type="info", content="Kokoro TTS model is currently being initialized, please wait...")
17
-
18
- try:
19
- # # Clean and chunk text for long responses
20
- # cleaned_text = self._clean_text(text)
21
- # chunks = self._chunk_text(cleaned_text)
22
-
23
- # if len(chunks) == 1:
24
- # # Single chunk - return as before
25
- # audio = await kokoro_tts.synthesize_sentences(chunks)
26
- # return {"audio": audio, "success": True}
27
- # else:
28
- # # Multiple chunks - return as sequence
29
- # audio_parts = []
30
- # for chunk in chunks:
31
- # chunk_audio = await kokoro_tts.synthesize_sentences([chunk])
32
- # audio_parts.append(chunk_audio)
33
- # return {"audio_parts": audio_parts, "success": True}
34
-
35
- # audio is chunked on the frontend for better flow
36
- audio = await kokoro_tts.synthesize_sentences([text])
37
- return {"audio": audio, "success": True}
38
- except Exception as e:
39
- return {"error": str(e), "success": False}
40
-
41
- # def _clean_text(self, text: str) -> str:
42
- # """Clean text by removing markdown, tables, code blocks, and other formatting"""
43
- # # Remove code blocks
44
- # text = re.sub(r'```[\s\S]*?```', '', text)
45
- # text = re.sub(r'`[^`]*`', '', text)
46
-
47
- # # Remove markdown links
48
- # text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text)
49
-
50
- # # Remove markdown formatting
51
- # text = re.sub(r'[*_#]+', '', text)
52
-
53
- # # Remove tables (basic cleanup)
54
- # text = re.sub(r'\|[^\n]*\|', '', text)
55
-
56
- # # Remove extra whitespace and newlines
57
- # text = re.sub(r'\n+', ' ', text)
58
- # text = re.sub(r'\s+', ' ', text)
59
-
60
- # # Remove URLs
61
- # text = re.sub(r'https?://[^\s]+', '', text)
62
-
63
- # # Remove email addresses
64
- # text = re.sub(r'\S+@\S+', '', text)
65
-
66
- # return text.strip()
67
-
68
- # def _chunk_text(self, text: str) -> list[str]:
69
- # """Split text into manageable chunks for TTS"""
70
- # # If text is short enough, return as single chunk
71
- # if len(text) <= 300:
72
- # return [text]
73
-
74
- # # Split into sentences first
75
- # sentences = re.split(r'(?<=[.!?])\s+', text)
76
-
77
- # chunks = []
78
- # current_chunk = ""
79
-
80
- # for sentence in sentences:
81
- # sentence = sentence.strip()
82
- # if not sentence:
83
- # continue
84
-
85
- # # If adding this sentence would make chunk too long, start new chunk
86
- # if current_chunk and len(current_chunk + " " + sentence) > 300:
87
- # chunks.append(current_chunk.strip())
88
- # current_chunk = sentence
89
- # else:
90
- # current_chunk += (" " if current_chunk else "") + sentence
91
-
92
- # # Add the last chunk if it has content
93
- # if current_chunk.strip():
94
- # chunks.append(current_chunk.strip())
95
-
96
- # return chunks if chunks else [text]
\ No newline at end of file
api/transcribe.py
deleted
-18
@@ -1,18 +0,0 @@
1
-from helpers.api import ApiHandler, Request, Response
2
-
3
-from helpers import runtime, settings, whisper
4
-
5
-class Transcribe(ApiHandler):
6
- async def process(self, input: dict, request: Request) -> dict | Response:
7
- audio = input.get("audio")
8
- ctxid = input.get("ctxid", "")
9
-
10
- if ctxid:
11
- context = self.use_context(ctxid)
12
-
13
- # if not await whisper.is_downloaded():
14
- # context.log.log(type="info", content="Whisper STT model is currently being initialized, please wait...")
15
-
16
- set = settings.get_settings()
17
- result = await whisper.transcribe(set["stt_model_size"], audio) # type: ignore
18
- return result
docs/guides/usage.md
+12
-3
@@ -314,18 +314,27 @@ Open **Settings -> External Services -> Flare Tunnel** to create or stop a tunne
314
315
## Voice Interface
316
317
-Agent Zero supports text-to-speech and speech-to-text.
317
+Agent Zero supports text-to-speech and speech-to-text through built-in voice plugins:
318
+
319
+- `_kokoro_tts` provides container-side Kokoro speech synthesis when enabled.
320
+- `_whisper_stt` provides local Whisper transcription and adds the microphone control when enabled.
321
+- Browser-native `speechSynthesis` remains the fallback output path when `_kokoro_tts` is disabled.
322
+
323
+Use the **Voice** section in Agent settings or the plugin settings in **Agent Plugins** to configure providers. Use the sidebar **Speech** preference when you want Agent Zero to read responses automatically.
324
325
Use speech when you want to listen while doing something else, dictate a prompt,
326
or make the interface more accessible.
327
328
329
324
-Speech-to-text settings live in Settings and include model size, language code,
325
-silence threshold, and recording behavior.
330
+Speech-to-text settings live in the Whisper STT plugin card and include model size, language code, voice message handling, silence threshold, and recording behavior. The microphone button appears in the chat input when `_whisper_stt` is enabled.
331
332
333
334
+> [!IMPORTANT]
335
+> Whisper STT and Kokoro TTS operate locally within the Docker/container runtime when their plugins are enabled.
336
+> Browser fallback TTS runs locally in the browser. No voice path requires OpenAI APIs.
337
+
338
## Mathematical Expressions
339
340
Agent Zero can render mathematical notation with KaTeX.
docs/setup/installation.md
+6
-4
@@ -364,11 +364,13 @@ Use `claude-sonnet-4-5` for Anthropic, but use `anthropic/claude-sonnet-4-5` for
364
> [!NOTE]
365
> Agent Zero uses a local embedding model by default (runs on CPU), but you can switch to OpenAI embeddings like `text-embedding-3-small` or `text-embedding-3-large` if preferred.
366
367
-### Speech to Text Options
367
+### Built-in Voice Plugins
368
369
-- **Model Size:** Choose the speech recognition model size
370
-- **Language Code:** Set the primary language for voice recognition
371
-- **Silence Settings:** Configure silence threshold, duration, and timeout parameters for voice input
369
+- Agent Zero ships Whisper STT as the built-in `_whisper_stt` plugin and Kokoro TTS as the built-in `_kokoro_tts` plugin.
370
+- Docker/bootstrap remains responsible for installing the required speech dependencies such as `ffmpeg`, Kokoro, Whisper, and `soundfile`.
371
+- Both plugins can be enabled or disabled independently from the Agent Plugins section in the Web UI.
372
+- Whisper model size, language, message handling, and silence behavior are configured from the plugin settings screen.
373
+- If `_kokoro_tts` is disabled, spoken output falls back to the browser's native speech synthesis instead of the container runtime.
374
375
### API Keys
376
helpers/kokoro_tts.py
deleted
-127
@@ -1,127 +0,0 @@
1
-# kokoro_tts.py
2
-
3
-import base64
4
-import io
5
-import warnings
6
-import asyncio
7
-import soundfile as sf
8
-from helpers import runtime
9
-from helpers.print_style import PrintStyle
10
-from helpers.notification import NotificationManager, NotificationType, NotificationPriority
11
-
12
-warnings.filterwarnings("ignore", category=FutureWarning)
13
-warnings.filterwarnings("ignore", category=UserWarning)
14
-
15
-_pipeline = None
16
-_voice = "am_puck,am_onyx"
17
-_speed = 1.1
18
-is_updating_model = False
19
-
20
-
21
-async def preload():
22
- try:
23
- # return await runtime.call_development_function(_preload)
24
- return await _preload()
25
- except Exception as e:
26
- # if not runtime.is_development():
27
- raise e
28
- # Fallback to direct execution if RFC fails in development
29
- # PrintStyle.standard("RFC failed, falling back to direct execution...")
30
- # return await _preload()
31
-
32
-
33
-async def _preload():
34
- global _pipeline, is_updating_model
35
-
36
- while is_updating_model:
37
- await asyncio.sleep(0.1)
38
-
39
- try:
40
- is_updating_model = True
41
- if not _pipeline:
42
- NotificationManager.send_notification(
43
- NotificationType.INFO,
44
- NotificationPriority.NORMAL,
45
- "Loading Kokoro TTS model...",
46
- display_time=99,
47
- group="kokoro-preload")
48
- PrintStyle.standard("Loading Kokoro TTS model...")
49
- from kokoro import KPipeline
50
- _pipeline = KPipeline(lang_code="a", repo_id="hexgrad/Kokoro-82M")
51
- NotificationManager.send_notification(
52
- NotificationType.INFO,
53
- NotificationPriority.NORMAL,
54
- "Kokoro TTS model loaded.",
55
- display_time=2,
56
- group="kokoro-preload")
57
- finally:
58
- is_updating_model = False
59
-
60
-
61
-async def is_downloading():
62
- try:
63
- # return await runtime.call_development_function(_is_downloading)
64
- return _is_downloading()
65
- except Exception as e:
66
- # if not runtime.is_development():
67
- raise e
68
- # Fallback to direct execution if RFC fails in development
69
- # return _is_downloading()
70
-
71
-
72
-def _is_downloading():
73
- return is_updating_model
74
-
75
-async def is_downloaded():
76
- try:
77
- # return await runtime.call_development_function(_is_downloaded)
78
- return _is_downloaded()
79
- except Exception as e:
80
- # if not runtime.is_development():
81
- raise e
82
- # Fallback to direct execution if RFC fails in development
83
- # return _is_downloaded()
84
-
85
-def _is_downloaded():
86
- return _pipeline is not None
87
-
88
-
89
-async def synthesize_sentences(sentences: list[str]):
90
- """Generate audio for multiple sentences and return concatenated base64 audio"""
91
- try:
92
- # return await runtime.call_development_function(_synthesize_sentences, sentences)
93
- return await _synthesize_sentences(sentences)
94
- except Exception as e:
95
- # if not runtime.is_development():
96
- raise e
97
- # Fallback to direct execution if RFC fails in development
98
- # return await _synthesize_sentences(sentences)
99
-
100
-
101
-async def _synthesize_sentences(sentences: list[str]):
102
- await _preload()
103
-
104
- combined_audio = []
105
-
106
- try:
107
- for sentence in sentences:
108
- if sentence.strip():
109
- segments = _pipeline(sentence.strip(), voice=_voice, speed=_speed) # type: ignore
110
- segment_list = list(segments)
111
-
112
- for segment in segment_list:
113
- audio_tensor = segment.audio
114
- audio_numpy = audio_tensor.detach().cpu().numpy() # type: ignore
115
- combined_audio.extend(audio_numpy)
116
-
117
- # Convert combined audio to bytes
118
- buffer = io.BytesIO()
119
- sf.write(buffer, combined_audio, 24000, format="WAV")
120
- audio_bytes = buffer.getvalue()
121
-
122
- # Return base64 encoded audio
123
- return base64.b64encode(audio_bytes).decode("utf-8")
124
-
125
- except Exception as e:
126
- PrintStyle.error(f"Error in Kokoro TTS synthesis: {e}")
127
- raise
\ No newline at end of file
helpers/settings.py
+1
-31
@@ -7,7 +7,7 @@ import subprocess
7
from typing import Any, Literal, TypedDict, cast, TypeVar
8
9
import models
10
-from helpers import runtime, whisper, defer, git, subagents
10
+from helpers import runtime, defer, git, subagents
11
from . import files, dotenv
12
from helpers.print_style import PrintStyle
13
from helpers.providers import get_providers, FieldOption as ProvidersFO
@@ -78,14 +78,6 @@ class Settings(TypedDict):
78
websocket_server_restart_enabled: bool
79
uvicorn_access_logs_enabled: bool
80
81
- stt_model_size: str
82
- stt_language: str
83
- stt_silence_threshold: float
84
- stt_silence_duration: int
85
- stt_waiting_timeout: int
86
-
87
- tts_kokoro: bool
88
-
81
mcp_servers: str
82
mcp_client_init_timeout: int
83
mcp_client_tool_timeout: int
@@ -151,7 +143,6 @@ class SettingsOutputAdditional(TypedDict):
143
embedding_providers: list[ModelProvider]
144
agent_subdirs: list[FieldOption]
145
knowledge_subdirs: list[FieldOption]
154
- stt_models: list[FieldOption]
146
is_dockerized: bool
147
runtime_settings: dict[str, Any]
148
@@ -196,14 +187,6 @@ def convert_out(settings: Settings) -> SettingsOutput:
187
if item["key"] != "_example"],
188
knowledge_subdirs=[{"value": subdir, "label": subdir}
189
for subdir in files.get_subdirectories("knowledge", exclude="default")],
199
- stt_models=[
200
- {"value": "tiny", "label": "Tiny (39M, English)"},
201
- {"value": "base", "label": "Base (74M, English)"},
202
- {"value": "small", "label": "Small (244M, English)"},
203
- {"value": "medium", "label": "Medium (769M, English)"},
204
- {"value": "large", "label": "Large (1.5B, Multilingual)"},
205
- {"value": "turbo", "label": "Turbo (Multilingual)"},
206
- ],
190
runtime_settings={},
191
),
192
)
@@ -225,7 +208,6 @@ def convert_out(settings: Settings) -> SettingsOutput:
208
209
additional["agent_subdirs"] = _ensure_option_present(additional.get("agent_subdirs"), current.get("agent_profile"))
210
additional["knowledge_subdirs"] = _ensure_option_present(additional.get("knowledge_subdirs"), current.get("agent_knowledge_subdir"))
228
- additional["stt_models"] = _ensure_option_present(additional.get("stt_models"), current.get("stt_model_size"))
211
212
# masked api keys
213
providers = get_providers("chat") + get_providers("embedding")
@@ -470,12 +452,6 @@ def get_default_settings() -> Settings:
452
rfc_port_http=get_default_value("rfc_port_http", 55080),
453
websocket_server_restart_enabled=get_default_value("websocket_server_restart_enabled", True),
454
uvicorn_access_logs_enabled=get_default_value("uvicorn_access_logs_enabled", False),
473
- stt_model_size=get_default_value("stt_model_size", "base"),
474
- stt_language=get_default_value("stt_language", "en"),
475
- stt_silence_threshold=get_default_value("stt_silence_threshold", 0.3),
476
- stt_silence_duration=get_default_value("stt_silence_duration", 1000),
477
- stt_waiting_timeout=get_default_value("stt_waiting_timeout", 2000),
478
- tts_kokoro=get_default_value("tts_kokoro", True),
455
mcp_servers=get_default_value("mcp_servers", '{\n "mcpServers": {}\n}'),
456
mcp_client_init_timeout=get_default_value("mcp_client_init_timeout", 10),
457
mcp_client_tool_timeout=get_default_value("mcp_client_tool_timeout", 120),
@@ -506,12 +482,6 @@ def _apply_settings(previous: Settings | None):
482
agent.config = ctx.config
483
agent = agent.get_data(agent.DATA_NAME_SUBORDINATE)
484
509
- # reload whisper model if necessary
510
- if not previous or _settings["stt_model_size"] != previous["stt_model_size"]:
511
- task = defer.DeferredTask().start_task(
512
- whisper.preload, _settings["stt_model_size"]
513
- ) # TODO overkill, replace with background task
514
-
485
# update mcp settings if necessary
486
if not previous or _settings["mcp_servers"] != previous["mcp_servers"]:
487
from helpers.mcp_handler import MCPConfig
helpers/whisper.py
deleted
-96
@@ -1,96 +0,0 @@
1
-import base64
2
-import warnings
3
-import whisper
4
-import tempfile
5
-import asyncio
6
-from helpers import runtime, rfc, settings, files
7
-from helpers.print_style import PrintStyle
8
-from helpers.notification import NotificationManager, NotificationType, NotificationPriority
9
-
10
-# Suppress FutureWarning from torch.load
11
-warnings.filterwarnings("ignore", category=FutureWarning)
12
-
13
-_model = None
14
-_model_name = ""
15
-is_updating_model = False # Tracks whether the model is currently updating
16
-
17
-async def preload(model_name:str):
18
- try:
19
- # return await runtime.call_development_function(_preload, model_name)
20
- return await _preload(model_name)
21
- except Exception as e:
22
- # if not runtime.is_development():
23
- raise e
24
-
25
-async def _preload(model_name:str):
26
- global _model, _model_name, is_updating_model
27
-
28
- while is_updating_model:
29
- await asyncio.sleep(0.1)
30
-
31
- try:
32
- is_updating_model = True
33
- if not _model or _model_name != model_name:
34
- NotificationManager.send_notification(
35
- NotificationType.INFO,
36
- NotificationPriority.NORMAL,
37
- "Loading Whisper model...",
38
- display_time=99,
39
- group="whisper-preload")
40
- PrintStyle.standard(f"Loading Whisper model: {model_name}")
41
- _model = whisper.load_model(name=model_name, download_root=files.get_abs_path("/tmp/models/whisper")) # type: ignore
42
- _model_name = model_name
43
- NotificationManager.send_notification(
44
- NotificationType.INFO,
45
- NotificationPriority.NORMAL,
46
- "Whisper model loaded.",
47
- display_time=2,
48
- group="whisper-preload")
49
- finally:
50
- is_updating_model = False
51
-
52
-async def is_downloading():
53
- # return await runtime.call_development_function(_is_downloading)
54
- return _is_downloading()
55
-
56
-def _is_downloading():
57
- return is_updating_model
58
-
59
-async def is_downloaded():
60
- try:
61
- # return await runtime.call_development_function(_is_downloaded)
62
- return _is_downloaded()
63
- except Exception as e:
64
- # if not runtime.is_development():
65
- raise e
66
- # Fallback to direct execution if RFC fails in development
67
- # return _is_downloaded()
68
-
69
-def _is_downloaded():
70
- return _model is not None
71
-
72
-async def transcribe(model_name:str, audio_bytes_b64: str):
73
- # return await runtime.call_development_function(_transcribe, model_name, audio_bytes_b64)
74
- return await _transcribe(model_name, audio_bytes_b64)
75
-
76
-
77
-async def _transcribe(model_name:str, audio_bytes_b64: str):
78
- await _preload(model_name)
79
-
80
- # Decode audio bytes if encoded as a base64 string
81
- audio_bytes = base64.b64decode(audio_bytes_b64)
82
-
83
- # Create temp audio file
84
- import os
85
- with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as audio_file:
86
- audio_file.write(audio_bytes)
87
- temp_path = audio_file.name
88
- try:
89
- # Transcribe the audio file
90
- result = _model.transcribe(temp_path, fp16=False) # type: ignore
91
- return result
92
- finally:
93
- try:
94
- os.remove(temp_path)
95
- except Exception:
96
- pass # ignore errors during cleanup
plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js
+2
-2
@@ -3,7 +3,7 @@ import {
3
copyToClipboard,
4
} from "/components/messages/action-buttons/simple-action-buttons.js";
5
import { store as stepDetailStore } from "/components/modals/process-step-detail/step-detail-store.js";
6
-import { store as speechStore } from "/components/chat/speech/speech-store.js";
6
+import { ttsService } from "/js/tts-service.js";
7
import { store as browserStore } from "/plugins/_browser/webui/browser-store.js";
8
import { getNamespacedClient } from "/js/websocket.js";
9
import { open as openSurface } from "/js/surfaces.js";
@@ -512,7 +512,7 @@ function drawBrowserTool({
512
buildDetailPayload(args, { headerLabels }),
513
),
514
),
515
- createActionButton("speak", "", () => speechStore.speak(contentText)),
515
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
516
createActionButton("copy", "", () => copyToClipboard(contentText)),
517
);
518
}
plugins/_kokoro_tts/README.md
new
+19
@@ -0,0 +1,19 @@
1
+# Kokoro TTS
2
+
3
+Built-in speech synthesis plugin backed by Kokoro.
4
+
5
+## Behavior
6
+
7
+- Registers Kokoro as the active TTS provider when the plugin is enabled.
8
+- Keeps browser-native `speechSynthesis` as the fallback path when disabled.
9
+- Keeps Python dependencies on the core Docker/bootstrap path. This plugin does not install packages or binaries on demand.
10
+
11
+## Config
12
+
13
+- `voice`: Kokoro voice identifier
14
+- `speed`: Kokoro playback speed multiplier
15
+
16
+## Routes
17
+
18
+- `POST /api/plugins/_kokoro_tts/synthesize`
19
+- `POST /api/plugins/_kokoro_tts/status`
plugins/_kokoro_tts/api/status.py
new
+31
@@ -0,0 +1,31 @@
1
+import importlib.metadata
2
+
3
+from helpers.api import ApiHandler, Request, Response
4
+from plugins._kokoro_tts.helpers import migration, runtime
5
+
6
+
7
+class Status(ApiHandler):
8
+ async def process(self, input: dict, request: Request) -> dict | Response:
9
+ migration.ensure_migrated()
10
+
11
+ package_version = ""
12
+ package_error = ""
13
+ try:
14
+ package_version = importlib.metadata.version("kokoro")
15
+ except Exception as e:
16
+ package_error = str(e)
17
+
18
+ return {
19
+ "plugin": "_kokoro_tts",
20
+ "enabled": runtime.is_globally_enabled(),
21
+ "config": runtime.get_config(),
22
+ "model": {
23
+ "ready": await runtime.is_downloaded(),
24
+ "loading": await runtime.is_downloading(),
25
+ },
26
+ "package": {
27
+ "version": package_version,
28
+ "error": package_error,
29
+ },
30
+ "fallback": "Browser-native speechSynthesis remains the fallback when Kokoro is disabled.",
31
+ }
plugins/_kokoro_tts/api/synthesize.py
new
+22
@@ -0,0 +1,22 @@
1
+from helpers.api import ApiHandler, Request, Response
2
+from plugins._kokoro_tts.helpers import runtime
3
+
4
+
5
+class Synthesize(ApiHandler):
6
+ async def process(self, input: dict, request: Request) -> dict | Response:
7
+ if not runtime.is_globally_enabled():
8
+ return Response(status=409, response="Kokoro TTS plugin is disabled")
9
+
10
+ text = str(input.get("text") or "").strip()
11
+ if not text:
12
+ return Response(status=400, response="Missing text")
13
+
14
+ try:
15
+ audio = await runtime.synthesize_sentences([text])
16
+ return {
17
+ "success": True,
18
+ "audio": audio,
19
+ "mime_type": "audio/wav",
20
+ }
21
+ except Exception as e:
22
+ return {"success": False, "error": str(e)}
plugins/_kokoro_tts/default_config.yaml
new
+2
@@ -0,0 +1,2 @@
1
+voice: am_puck,am_onyx
2
+speed: 1.1
plugins/_kokoro_tts/extensions/webui/page-head/runtime.html
new
+5
@@ -0,0 +1,5 @@
1
+<script type="module">
2
+ import { store } from "/plugins/_kokoro_tts/webui/kokoro-tts-store.js";
3
+
4
+ store.initRuntime();
5
+</script>
plugins/_kokoro_tts/extensions/webui/voice-settings-main/kokoro-card.html
new
+105
@@ -0,0 +1,105 @@
1
+<div x-data x-init="$store.kokoroTts?.ensureStatusLoaded?.()" x-show="$store.kokoroTts?.enabled">
2
+ <div class="voice-plugin-card">
3
+ <div class="voice-plugin-card-header">
4
+ <div>
5
+ <div class="voice-plugin-title">Kokoro TTS</div>
6
+ <div class="voice-plugin-description">
7
+ Server-side speech synthesis plugin. When disabled, Agent Zero falls back to browser-native TTS.
8
+ </div>
9
+ </div>
10
+ <span class="voice-plugin-badge" :class="`is-${$store.kokoroTts?.statusClass || 'warn'}`" x-text="$store.kokoroTts?.statusText || 'Idle'"></span>
11
+ </div>
12
+
13
+ <div class="voice-plugin-meta">
14
+ <div class="voice-plugin-meta-row">
15
+ <span>Voice</span>
16
+ <code x-text="$store.kokoroTts?.config?.voice || ''"></code>
17
+ </div>
18
+ <div class="voice-plugin-meta-row">
19
+ <span>Speed</span>
20
+ <span x-text="$store.kokoroTts?.config?.speed || 1.1"></span>
21
+ </div>
22
+ </div>
23
+
24
+ <div class="voice-plugin-actions">
25
+ <button class="btn btn-field" @click="$store.kokoroTts.openConfig()">Configure</button>
26
+ <button class="btn btn-field" @click="$store.kokoroTts.openPanel()">Status</button>
27
+ </div>
28
+ </div>
29
+
30
+ <style>
31
+ .voice-plugin-card {
32
+ display: flex;
33
+ flex-direction: column;
34
+ gap: 12px;
35
+ padding: 14px;
36
+ background: var(--color-input);
37
+ border: 1px solid var(--color-border);
38
+ border-radius: 10px;
39
+ }
40
+
41
+ .voice-plugin-card-header {
42
+ display: flex;
43
+ justify-content: space-between;
44
+ gap: 12px;
45
+ align-items: flex-start;
46
+ }
47
+
48
+ .voice-plugin-title {
49
+ font-weight: 700;
50
+ font-size: 0.98rem;
51
+ }
52
+
53
+ .voice-plugin-description {
54
+ color: var(--color-text-secondary);
55
+ font-size: var(--font-size-small);
56
+ margin-top: 0.25rem;
57
+ }
58
+
59
+ .voice-plugin-badge {
60
+ padding: 2px 8px;
61
+ border-radius: 999px;
62
+ font-size: 0.76rem;
63
+ font-weight: 600;
64
+ border: 1px solid transparent;
65
+ white-space: nowrap;
66
+ }
67
+
68
+ .voice-plugin-badge.is-ok {
69
+ color: #1b5e20;
70
+ background: rgba(46, 125, 50, 0.14);
71
+ border-color: rgba(46, 125, 50, 0.24);
72
+ }
73
+
74
+ .voice-plugin-badge.is-warn {
75
+ color: #8a6100;
76
+ background: rgba(191, 144, 0, 0.14);
77
+ border-color: rgba(191, 144, 0, 0.24);
78
+ }
79
+
80
+ .voice-plugin-meta {
81
+ display: flex;
82
+ flex-direction: column;
83
+ gap: 8px;
84
+ font-size: 0.84rem;
85
+ }
86
+
87
+ .voice-plugin-meta-row {
88
+ display: flex;
89
+ justify-content: space-between;
90
+ gap: 12px;
91
+ }
92
+
93
+ .voice-plugin-meta-row code {
94
+ font-family: var(--font-mono);
95
+ word-break: break-word;
96
+ text-align: right;
97
+ }
98
+
99
+ .voice-plugin-actions {
100
+ display: flex;
101
+ gap: 8px;
102
+ flex-wrap: wrap;
103
+ }
104
+ </style>
105
+</div>
plugins/_kokoro_tts/helpers/__init__.py
new
+3
@@ -0,0 +1,3 @@
1
+from . import runtime
2
+
3
+__all__ = ["runtime"]
plugins/_kokoro_tts/helpers/migration.py
new
+54
@@ -0,0 +1,54 @@
1
+from __future__ import annotations
2
+
3
+import json
4
+
5
+from helpers import files, plugins
6
+
7
+
8
+PLUGIN_NAME = "_kokoro_tts"
9
+LEGACY_SETTINGS_FILE = files.get_abs_path("usr/settings.json")
10
+
11
+
12
+def ensure_migrated() -> bool:
13
+ legacy_settings = _read_legacy_settings()
14
+ legacy_enabled = _coerce_bool(legacy_settings.get("tts_kokoro"), default=True)
15
+ if legacy_enabled or _has_explicit_toggle():
16
+ return False
17
+
18
+ disabled_path = plugins.determine_plugin_asset_path(
19
+ PLUGIN_NAME, "", "", plugins.DISABLED_FILE_NAME
20
+ )
21
+ files.write_file(disabled_path, "")
22
+ plugins.clear_plugin_cache([PLUGIN_NAME])
23
+ return True
24
+
25
+
26
+def _has_explicit_toggle() -> bool:
27
+ for root in plugins.get_plugin_roots(PLUGIN_NAME):
28
+ if files.exists(files.get_abs_path(root, plugins.ENABLED_FILE_NAME)):
29
+ return True
30
+ if files.exists(files.get_abs_path(root, plugins.DISABLED_FILE_NAME)):
31
+ return True
32
+ return False
33
+
34
+
35
+def _read_legacy_settings() -> dict:
36
+ if not files.exists(LEGACY_SETTINGS_FILE):
37
+ return {}
38
+
39
+ try:
40
+ return json.loads(files.read_file(LEGACY_SETTINGS_FILE))
41
+ except Exception:
42
+ return {}
43
+
44
+
45
+def _coerce_bool(value: object, default: bool) -> bool:
46
+ if isinstance(value, bool):
47
+ return value
48
+ if isinstance(value, str):
49
+ lowered = value.strip().lower()
50
+ if lowered in {"true", "1", "yes", "on"}:
51
+ return True
52
+ if lowered in {"false", "0", "no", "off"}:
53
+ return False
54
+ return default
plugins/_kokoro_tts/helpers/runtime.py
new
+146
@@ -0,0 +1,146 @@
1
+from __future__ import annotations
2
+
3
+import asyncio
4
+import base64
5
+import io
6
+import warnings
7
+from typing import Any
8
+
9
+import soundfile as sf
10
+
11
+from helpers import plugins
12
+from helpers.notification import (
13
+ NotificationManager,
14
+ NotificationPriority,
15
+ NotificationType,
16
+)
17
+from helpers.print_style import PrintStyle
18
+from plugins._kokoro_tts.helpers import migration
19
+
20
+
21
+warnings.filterwarnings("ignore", category=FutureWarning)
22
+warnings.filterwarnings("ignore", category=UserWarning)
23
+
24
+
25
+PLUGIN_NAME = "_kokoro_tts"
26
+DEFAULT_CONFIG = {
27
+ "voice": "am_puck,am_onyx",
28
+ "speed": 1.1,
29
+}
30
+
31
+_pipeline = None
32
+is_updating_model = False
33
+
34
+
35
+def normalize_config(config: dict[str, Any] | None) -> dict[str, Any]:
36
+ normalized = dict(DEFAULT_CONFIG)
37
+ if not isinstance(config, dict):
38
+ return normalized
39
+
40
+ voice = str(config.get("voice", normalized["voice"]) or "").strip()
41
+ if voice:
42
+ normalized["voice"] = voice
43
+
44
+ try:
45
+ speed = float(config.get("speed", normalized["speed"]))
46
+ if speed > 0:
47
+ normalized["speed"] = speed
48
+ except (TypeError, ValueError):
49
+ pass
50
+
51
+ return normalized
52
+
53
+
54
+def get_config() -> dict[str, Any]:
55
+ config = plugins.get_plugin_config(PLUGIN_NAME) or {}
56
+ return normalize_config(config)
57
+
58
+
59
+def is_globally_enabled() -> bool:
60
+ migration.ensure_migrated()
61
+ return plugins.determined_toggle_from_paths(
62
+ True, reversed(plugins.get_plugin_roots(PLUGIN_NAME))
63
+ )
64
+
65
+
66
+async def preload(config: dict[str, Any] | None = None):
67
+ return await _preload()
68
+
69
+
70
+async def _preload():
71
+ global _pipeline, is_updating_model
72
+
73
+ while is_updating_model:
74
+ await asyncio.sleep(0.1)
75
+
76
+ try:
77
+ is_updating_model = True
78
+ if not _pipeline:
79
+ NotificationManager.send_notification(
80
+ NotificationType.INFO,
81
+ NotificationPriority.NORMAL,
82
+ "Loading Kokoro TTS model...",
83
+ display_time=99,
84
+ group="kokoro-preload",
85
+ )
86
+ PrintStyle.standard("Loading Kokoro TTS model...")
87
+ from kokoro import KPipeline
88
+
89
+ _pipeline = KPipeline(lang_code="a", repo_id="hexgrad/Kokoro-82M")
90
+ NotificationManager.send_notification(
91
+ NotificationType.INFO,
92
+ NotificationPriority.NORMAL,
93
+ "Kokoro TTS model loaded.",
94
+ display_time=2,
95
+ group="kokoro-preload",
96
+ )
97
+ finally:
98
+ is_updating_model = False
99
+
100
+
101
+async def is_downloading() -> bool:
102
+ return is_updating_model
103
+
104
+
105
+async def is_downloaded() -> bool:
106
+ return _pipeline is not None
107
+
108
+
109
+async def synthesize_sentences(
110
+ sentences: list[str], config: dict[str, Any] | None = None
111
+) -> str:
112
+ cfg = normalize_config(config or get_config())
113
+ return await _synthesize_sentences(
114
+ sentences,
115
+ voice=str(cfg["voice"]),
116
+ speed=float(cfg["speed"]),
117
+ )
118
+
119
+
120
+async def _synthesize_sentences(
121
+ sentences: list[str], *, voice: str, speed: float
122
+) -> str:
123
+ await _preload()
124
+
125
+ combined_audio: list[float] = []
126
+
127
+ try:
128
+ for sentence in sentences:
129
+ if not sentence.strip():
130
+ continue
131
+
132
+ segments = _pipeline(sentence.strip(), voice=voice, speed=speed) # type: ignore[misc]
133
+ for segment in list(segments):
134
+ audio_tensor = segment.audio
135
+ audio_numpy = audio_tensor.detach().cpu().numpy() # type: ignore[union-attr]
136
+ combined_audio.extend(audio_numpy.tolist())
137
+
138
+ if not combined_audio:
139
+ return ""
140
+
141
+ buffer = io.BytesIO()
142
+ sf.write(buffer, combined_audio, 24000, format="WAV")
143
+ return base64.b64encode(buffer.getvalue()).decode("utf-8")
144
+ except Exception as e:
145
+ PrintStyle.error(f"Error in Kokoro TTS synthesis: {e}")
146
+ raise
plugins/_kokoro_tts/hooks.py
new
+12
@@ -0,0 +1,12 @@
1
+from __future__ import annotations
2
+
3
+from plugins._kokoro_tts.helpers import migration, runtime
4
+
5
+
6
+def get_plugin_config(default=None, **kwargs):
7
+ migration.ensure_migrated()
8
+ return runtime.normalize_config(default or {})
9
+
10
+
11
+def save_plugin_config(default=None, settings=None, **kwargs):
12
+ return runtime.normalize_config(settings or default or {})
plugins/_kokoro_tts/plugin.yaml
new
+9
@@ -0,0 +1,9 @@
1
+name: _kokoro_tts
2
+title: Kokoro TTS
3
+description: Built-in Kokoro text-to-speech plugin with browser TTS fallback when disabled.
4
+version: 1.0.0
5
+always_enabled: false
6
+settings_sections:
7
+ - agent
8
+per_project_config: false
9
+per_agent_config: false
plugins/_kokoro_tts/webui/config.html
new
+39
@@ -0,0 +1,39 @@
1
+<html>
2
+<head>
3
+ <title>Kokoro TTS</title>
4
+</head>
5
+
6
+<body>
7
+ <div x-data>
8
+ <template x-if="config">
9
+ <div class="plugin-config-page">
10
+ <div class="section-title">Kokoro TTS</div>
11
+ <div class="section-description">
12
+ Configure the built-in Kokoro voice provider. When this plugin is disabled,
13
+ spoken output falls back to the browser speech API.
14
+ </div>
15
+
16
+ <div class="field">
17
+ <div class="field-label">
18
+ <div class="field-title">Voice</div>
19
+ <div class="field-description">Kokoro voice identifier passed to the backend pipeline.</div>
20
+ </div>
21
+ <div class="field-control">
22
+ <input type="text" x-model="config.voice" />
23
+ </div>
24
+ </div>
25
+
26
+ <div class="field">
27
+ <div class="field-label">
28
+ <div class="field-title">Speed</div>
29
+ <div class="field-description">Playback speed multiplier for Kokoro synthesis.</div>
30
+ </div>
31
+ <div class="field-control">
32
+ <input type="number" min="0.1" step="0.1" x-model.number="config.speed" />
33
+ </div>
34
+ </div>
35
+ </div>
36
+ </template>
37
+ </div>
38
+</body>
39
+</html>
plugins/_kokoro_tts/webui/kokoro-tts-store.js
new
+116
@@ -0,0 +1,116 @@
1
+import { createStore } from "/js/AlpineStore.js";
2
+import { toastFrontendError } from "/components/notifications/notification-store.js";
3
+import { callJsonApi } from "/js/api.js";
4
+import { ttsService } from "/js/tts-service.js";
5
+
6
+const PLUGIN_NAME = "_kokoro_tts";
7
+
8
+const model = {
9
+ runtimeInitialized: false,
10
+ statusLoaded: false,
11
+ loading: false,
12
+ error: "",
13
+ enabled: false,
14
+ config: {
15
+ voice: "",
16
+ speed: 1.1,
17
+ },
18
+ modelReady: false,
19
+ modelLoading: false,
20
+ packageVersion: "",
21
+ providerCleanup: null,
22
+
23
+ async initRuntime() {
24
+ if (this.runtimeInitialized) return;
25
+ this.runtimeInitialized = true;
26
+ await this.refreshStatus({ suppressError: true });
27
+ },
28
+
29
+ async ensureStatusLoaded() {
30
+ if (this.statusLoaded || this.loading) return;
31
+ await this.refreshStatus({ suppressError: true });
32
+ },
33
+
34
+ async refreshStatus({ suppressError = false } = {}) {
35
+ this.loading = true;
36
+ this.error = "";
37
+
38
+ try {
39
+ const status = await callJsonApi(`/plugins/${PLUGIN_NAME}/status`, {});
40
+ this.statusLoaded = true;
41
+ this.enabled = !!status?.enabled;
42
+ this.config = {
43
+ voice: status?.config?.voice || "",
44
+ speed: Number(status?.config?.speed || 1.1),
45
+ };
46
+ this.modelReady = !!status?.model?.ready;
47
+ this.modelLoading = !!status?.model?.loading;
48
+ this.packageVersion = status?.package?.version || "";
49
+
50
+ if (this.enabled) {
51
+ this.registerProvider();
52
+ } else {
53
+ this.unregisterProvider();
54
+ }
55
+ } catch (error) {
56
+ this.error = error instanceof Error ? error.message : String(error);
57
+ this.unregisterProvider();
58
+ if (!suppressError) {
59
+ void toastFrontendError(this.error, "Kokoro TTS");
60
+ }
61
+ } finally {
62
+ this.loading = false;
63
+ }
64
+ },
65
+
66
+ registerProvider() {
67
+ if (this.providerCleanup || !this.enabled) return;
68
+
69
+ this.providerCleanup = ttsService.registerProvider(PLUGIN_NAME, {
70
+ synthesize: async (text) => {
71
+ const result = await callJsonApi(`/plugins/${PLUGIN_NAME}/synthesize`, {
72
+ text,
73
+ });
74
+ if (!result?.success) {
75
+ throw new Error(result?.error || "Kokoro TTS synthesis failed.");
76
+ }
77
+
78
+ return {
79
+ audioBase64: result.audio || "",
80
+ mimeType: result.mime_type || "audio/wav",
81
+ };
82
+ },
83
+ });
84
+ },
85
+
86
+ unregisterProvider() {
87
+ if (!this.providerCleanup) return;
88
+ this.providerCleanup();
89
+ this.providerCleanup = null;
90
+ },
91
+
92
+ get statusText() {
93
+ if (!this.enabled) return "Disabled";
94
+ if (this.modelLoading) return "Loading";
95
+ if (this.modelReady) return "Ready";
96
+ return "Idle";
97
+ },
98
+
99
+ get statusClass() {
100
+ if (!this.enabled) return "warn";
101
+ if (this.modelLoading) return "warn";
102
+ if (this.modelReady) return "ok";
103
+ return "warn";
104
+ },
105
+
106
+ async openConfig() {
107
+ const { store } = await import("/components/plugins/plugin-settings-store.js");
108
+ await store.openConfig(PLUGIN_NAME);
109
+ },
110
+
111
+ openPanel() {
112
+ window.openModal?.(`/plugins/${PLUGIN_NAME}/webui/main.html`);
113
+ },
114
+};
115
+
116
+export const store = createStore("kokoroTts", model);
plugins/_kokoro_tts/webui/main.html
new
+133
@@ -0,0 +1,133 @@
1
+<html>
2
+<head>
3
+ <title>Kokoro TTS</title>
4
+ <script type="module">
5
+ import { store } from "/plugins/_kokoro_tts/webui/kokoro-tts-store.js";
6
+ </script>
7
+</head>
8
+
9
+<body>
10
+ <div
11
+ x-data
12
+ x-init="$store.kokoroTts.ensureStatusLoaded()"
13
+ class="speech-plugin-page"
14
+ >
15
+ <template x-if="$store.kokoroTts">
16
+ <div>
17
+ <div class="section-title">Kokoro TTS</div>
18
+ <div class="section-description">
19
+ Built-in Kokoro speech synthesis. Dependency installation remains on the
20
+ Docker/bootstrap path; disabling this plugin returns spoken output to the
21
+ browser fallback.
22
+ </div>
23
+
24
+ <div class="speech-plugin-grid">
25
+ <div class="speech-plugin-card">
26
+ <div class="field-title">Provider State</div>
27
+ <div class="status-row">
28
+ <span class="status-key">Enabled</span>
29
+ <span class="status-badge" :class="$store.kokoroTts.enabled ? 'ok' : 'warn'" x-text="$store.kokoroTts.enabled ? 'Yes' : 'No'"></span>
30
+ </div>
31
+ <div class="status-row">
32
+ <span class="status-key">Model</span>
33
+ <span class="status-badge" :class="$store.kokoroTts.statusClass" x-text="$store.kokoroTts.statusText"></span>
34
+ </div>
35
+ <div class="status-row" x-show="$store.kokoroTts.packageVersion">
36
+ <span class="status-key">Package</span>
37
+ <span class="status-value" x-text="$store.kokoroTts.packageVersion"></span>
38
+ </div>
39
+ </div>
40
+
41
+ <div class="speech-plugin-card">
42
+ <div class="field-title">Resolved Config</div>
43
+ <div class="status-row">
44
+ <span class="status-key">Voice</span>
45
+ <span class="status-value mono" x-text="$store.kokoroTts.config.voice"></span>
46
+ </div>
47
+ <div class="status-row">
48
+ <span class="status-key">Speed</span>
49
+ <span class="status-value" x-text="$store.kokoroTts.config.speed"></span>
50
+ </div>
51
+ </div>
52
+ </div>
53
+
54
+ <div class="speech-plugin-actions">
55
+ <button class="btn btn-field" @click="$store.kokoroTts.openConfig()">Open Settings</button>
56
+ <button class="btn btn-field" @click="$store.kokoroTts.refreshStatus()">Refresh</button>
57
+ </div>
58
+ </div>
59
+ </template>
60
+ </div>
61
+
62
+ <style>
63
+ .speech-plugin-page {
64
+ display: flex;
65
+ flex-direction: column;
66
+ gap: 14px;
67
+ }
68
+
69
+ .speech-plugin-grid {
70
+ display: grid;
71
+ gap: 12px;
72
+ grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
73
+ }
74
+
75
+ .speech-plugin-card {
76
+ display: flex;
77
+ flex-direction: column;
78
+ gap: 10px;
79
+ padding: 14px;
80
+ background: var(--color-input);
81
+ }
82
+
83
+ .speech-plugin-actions {
84
+ display: flex;
85
+ gap: 8px;
86
+ flex-wrap: wrap;
87
+ }
88
+
89
+ .status-row {
90
+ display: flex;
91
+ align-items: flex-start;
92
+ justify-content: space-between;
93
+ gap: 12px;
94
+ font-size: 0.84rem;
95
+ }
96
+
97
+ .status-key {
98
+ opacity: 0.7;
99
+ min-width: 64px;
100
+ }
101
+
102
+ .status-value {
103
+ text-align: right;
104
+ word-break: break-word;
105
+ }
106
+
107
+ .status-badge {
108
+ padding: 2px 8px;
109
+ border-radius: 999px;
110
+ font-size: 0.76rem;
111
+ font-weight: 600;
112
+ border: 1px solid transparent;
113
+ }
114
+
115
+ .status-badge.ok {
116
+ color: #1b5e20;
117
+ background: rgba(46, 125, 50, 0.14);
118
+ border-color: rgba(46, 125, 50, 0.24);
119
+ }
120
+
121
+ .status-badge.warn {
122
+ color: #8a6100;
123
+ background: rgba(191, 144, 0, 0.14);
124
+ border-color: rgba(191, 144, 0, 0.24);
125
+ }
126
+
127
+ .mono {
128
+ font-family: var(--font-mono);
129
+ font-size: 0.78rem;
130
+ }
131
+ </style>
132
+</body>
133
+</html>
plugins/_kokoro_tts/webui/thumbnail.jpg
Binary files /dev/null and b/plugins/_kokoro_tts/webui/thumbnail.jpg differ
plugins/_whisper_stt/README.md
new
+23
@@ -0,0 +1,23 @@
1
+# Whisper STT
2
+
3
+Built-in speech-to-text plugin backed by Whisper.
4
+
5
+## Responsibilities
6
+
7
+- Registers Whisper as the active STT provider when the plugin is enabled.
8
+- Owns the microphone runtime, device selector UI, message delivery mode, and plugin APIs.
9
+- Keeps dependency installation and model bootstrap on the Docker/bootstrap path.
10
+
11
+## Config
12
+
13
+- `model_size`: Whisper model name
14
+- `language`: language hint or `auto`
15
+- `message_mode`: `send` to send final transcriptions immediately, or `draft` to leave them in the composer
16
+- `silence_threshold`: frontend threshold before recording starts
17
+- `silence_duration`: silence window before waiting state
18
+- `waiting_timeout`: delay before transcription dispatch
19
+
20
+## API
21
+
22
+- `POST /api/plugins/_whisper_stt/transcribe`
23
+- `POST /api/plugins/_whisper_stt/status`
plugins/_whisper_stt/api/status.py
new
+31
@@ -0,0 +1,31 @@
1
+import importlib.metadata
2
+
3
+from helpers.api import ApiHandler, Request, Response
4
+from plugins._whisper_stt.helpers import migration, runtime
5
+
6
+
7
+class Status(ApiHandler):
8
+ async def process(self, input: dict, request: Request) -> dict | Response:
9
+ migration.ensure_config_seeded()
10
+
11
+ package_version = ""
12
+ package_error = ""
13
+ try:
14
+ package_version = importlib.metadata.version("openai-whisper")
15
+ except Exception as e:
16
+ package_error = str(e)
17
+
18
+ return {
19
+ "plugin": "_whisper_stt",
20
+ "enabled": runtime.is_globally_enabled(),
21
+ "config": runtime.get_config(),
22
+ "model": {
23
+ "ready": await runtime.is_downloaded(),
24
+ "loading": await runtime.is_downloading(),
25
+ "loaded_model": runtime.get_loaded_model_name(),
26
+ },
27
+ "package": {
28
+ "version": package_version,
29
+ "error": package_error,
30
+ },
31
+ }
plugins/_whisper_stt/api/transcribe.py
new
+26
@@ -0,0 +1,26 @@
1
+from helpers.api import ApiHandler, Request, Response
2
+from plugins._whisper_stt.helpers import runtime
3
+
4
+
5
+class Transcribe(ApiHandler):
6
+ async def process(self, input: dict, request: Request) -> dict | Response:
7
+ if not runtime.is_globally_enabled():
8
+ return Response(status=409, response="Whisper STT plugin is disabled")
9
+
10
+ audio = str(input.get("audio") or "").strip()
11
+ if not audio:
12
+ return Response(status=400, response="Missing audio")
13
+
14
+ ctxid = str(input.get("ctxid") or "").strip()
15
+ if ctxid:
16
+ self.use_context(ctxid)
17
+
18
+ try:
19
+ result = await runtime.transcribe(audio)
20
+ return {
21
+ "success": True,
22
+ "text": str(result.get("text") or "").strip(),
23
+ "language": str(result.get("language") or "").strip(),
24
+ }
25
+ except Exception as e:
26
+ return {"success": False, "error": str(e), "text": ""}
plugins/_whisper_stt/default_config.yaml
new
+6
@@ -0,0 +1,6 @@
1
+model_size: base
2
+language: en
3
+message_mode: send
4
+silence_threshold: 0.3
5
+silence_duration: 1000
6
+waiting_timeout: 2000
plugins/_whisper_stt/extensions/webui/chat-input-box-end/microphone-button.html
new
+86
@@ -0,0 +1,86 @@
1
+<div x-data>
2
+ <style>
3
+ #microphone-button {
4
+ color: var(--color-background);
5
+ transition:
6
+ background-color 0.2s ease,
7
+ box-shadow 0.12s ease-in-out,
8
+ filter 0.12s ease-in-out,
9
+ opacity 0.12s ease-in-out;
10
+ }
11
+
12
+ #microphone-button.mic-disabled {
13
+ background-color: #5f6368;
14
+ cursor: not-allowed;
15
+ opacity: 0.58;
16
+ }
17
+
18
+ #microphone-button.mic-inactive {
19
+ background-color: grey;
20
+ }
21
+
22
+ #microphone-button.mic-activating {
23
+ background-color: silver;
24
+ animation: whisper-stt-mic-pulse 0.8s infinite;
25
+ }
26
+
27
+ #microphone-button.mic-listening {
28
+ background-color: red;
29
+ }
30
+
31
+ #microphone-button.mic-recording {
32
+ background-color: green;
33
+ }
34
+
35
+ #microphone-button.mic-waiting {
36
+ background-color: teal;
37
+ }
38
+
39
+ #microphone-button.mic-processing {
40
+ background-color: darkcyan;
41
+ animation: whisper-stt-mic-pulse 0.8s infinite;
42
+ transform-origin: center;
43
+ }
44
+
45
+ @media (hover: hover) {
46
+ #microphone-button:not(.mic-disabled):hover {
47
+ filter: brightness(1.08);
48
+ box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.08),
49
+ 0 6px 14px rgba(0, 0, 0, 0.18);
50
+ }
51
+ }
52
+
53
+ #microphone-button:not(.mic-disabled):active {
54
+ filter: brightness(0.92);
55
+ box-shadow: inset 0 0 0 1px rgba(0, 0, 0, 0.12);
56
+ }
57
+
58
+ @keyframes whisper-stt-mic-pulse {
59
+ 0% {
60
+ transform: scale(1);
61
+ }
62
+ 50% {
63
+ transform: scale(1.1);
64
+ }
65
+ 100% {
66
+ transform: scale(1);
67
+ }
68
+ }
69
+ </style>
70
+ <template x-if="$store.whisperStt">
71
+ <template x-teleport="#chat-buttons-wrapper">
72
+ <button
73
+ class="chat-button mic-inactive"
74
+ id="microphone-button"
75
+ aria-label="Start/Stop recording"
76
+ @click="$store.whisperStt.handleMicrophoneClick()"
77
+ x-init="$store.whisperStt.updateMicrophoneButtonUI()"
78
+ x-effect="$store.whisperStt.updateMicrophoneButtonUI()"
79
+ >
80
+ <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor">
81
+ <path d="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z" />
82
+ </svg>
83
+ </button>
84
+ </template>
85
+ </template>
86
+</div>
plugins/_whisper_stt/extensions/webui/page-head/runtime.html
new
+6
@@ -0,0 +1,6 @@
1
+<link rel="stylesheet" href="/plugins/_whisper_stt/webui/whisper-stt.css">
2
+<script type="module">
3
+ import { store } from "/plugins/_whisper_stt/webui/whisper-stt-store.js";
4
+
5
+ store.initRuntime();
6
+</script>
plugins/_whisper_stt/extensions/webui/voice-settings-main/whisper-card.html
new
+107
@@ -0,0 +1,107 @@
1
+<div x-data x-init="$store.whisperStt.ensureStatusLoaded()" x-show="$store.whisperStt">
2
+ <div class="voice-plugin-card">
3
+ <div class="voice-plugin-card-header">
4
+ <div>
5
+ <div class="voice-plugin-title">Whisper STT</div>
6
+ <div class="voice-plugin-description">
7
+ Browser microphone input routed into the built-in Whisper transcription backend.
8
+ </div>
9
+ </div>
10
+ <span class="voice-plugin-badge" :class="`is-${$store.whisperStt.statusClass || 'warn'}`" x-text="$store.whisperStt.statusText || 'Idle'"></span>
11
+ </div>
12
+
13
+ <div class="voice-plugin-meta">
14
+ <div class="voice-plugin-meta-row">
15
+ <span>Model size</span>
16
+ <span x-text="$store.whisperStt.config.model_size || 'base'"></span>
17
+ </div>
18
+ <div class="voice-plugin-meta-row">
19
+ <span>Language</span>
20
+ <span x-text="$store.whisperStt.config.language || 'en'"></span>
21
+ </div>
22
+ <div class="voice-plugin-meta-row">
23
+ <span>Message</span>
24
+ <span x-text="$store.whisperStt.messageModeLabel"></span>
25
+ </div>
26
+ <div class="voice-plugin-meta-row">
27
+ <span>Microphone</span>
28
+ <span x-text="$store.whisperStt.selectedDeviceLabel"></span>
29
+ </div>
30
+ </div>
31
+
32
+ <div class="voice-plugin-actions">
33
+ <button class="btn btn-field" @click="$store.whisperStt.openConfig()">Configure</button>
34
+ <button class="btn btn-field" @click="$store.whisperStt.openPanel()">Status</button>
35
+ </div>
36
+ </div>
37
+
38
+ <style>
39
+ .voice-plugin-card {
40
+ display: flex;
41
+ flex-direction: column;
42
+ gap: 12px;
43
+ padding: 14px;
44
+ background: var(--color-input);
45
+ border: 1px solid var(--color-border);
46
+ border-radius: 10px;
47
+ }
48
+
49
+ .voice-plugin-card-header {
50
+ display: flex;
51
+ justify-content: space-between;
52
+ gap: 12px;
53
+ align-items: flex-start;
54
+ }
55
+
56
+ .voice-plugin-title {
57
+ font-weight: 700;
58
+ font-size: 0.98rem;
59
+ }
60
+
61
+ .voice-plugin-description {
62
+ color: var(--color-text-secondary);
63
+ font-size: var(--font-size-small);
64
+ margin-top: 0.25rem;
65
+ }
66
+
67
+ .voice-plugin-badge {
68
+ padding: 2px 8px;
69
+ border-radius: 999px;
70
+ font-size: 0.76rem;
71
+ font-weight: 600;
72
+ border: 1px solid transparent;
73
+ white-space: nowrap;
74
+ }
75
+
76
+ .voice-plugin-badge.is-ok {
77
+ color: #1b5e20;
78
+ background: rgba(46, 125, 50, 0.14);
79
+ border-color: rgba(46, 125, 50, 0.24);
80
+ }
81
+
82
+ .voice-plugin-badge.is-warn {
83
+ color: #8a6100;
84
+ background: rgba(191, 144, 0, 0.14);
85
+ border-color: rgba(191, 144, 0, 0.24);
86
+ }
87
+
88
+ .voice-plugin-meta {
89
+ display: flex;
90
+ flex-direction: column;
91
+ gap: 8px;
92
+ font-size: 0.84rem;
93
+ }
94
+
95
+ .voice-plugin-meta-row {
96
+ display: flex;
97
+ justify-content: space-between;
98
+ gap: 12px;
99
+ }
100
+
101
+ .voice-plugin-actions {
102
+ display: flex;
103
+ gap: 8px;
104
+ flex-wrap: wrap;
105
+ }
106
+ </style>
107
+</div>
plugins/_whisper_stt/helpers/__init__.py
new
+1
@@ -0,0 +1 @@
1
+
plugins/_whisper_stt/helpers/migration.py
new
+98
@@ -0,0 +1,98 @@
1
+from __future__ import annotations
2
+
3
+import json
4
+from typing import Any
5
+
6
+from helpers import files, plugins
7
+
8
+
9
+PLUGIN_NAME = "_whisper_stt"
10
+LEGACY_SETTINGS_FILE = files.get_abs_path("usr/settings.json")
11
+DEFAULT_CONFIG = {
12
+ "model_size": "base",
13
+ "language": "en",
14
+ "message_mode": "send",
15
+ "silence_threshold": 0.3,
16
+ "silence_duration": 1000,
17
+ "waiting_timeout": 2000,
18
+}
19
+
20
+
21
+def ensure_config_seeded() -> bool:
22
+ config_path = get_config_path()
23
+ if files.exists(config_path):
24
+ return False
25
+
26
+ config = build_seed_config(_read_legacy_settings())
27
+ files.write_file(config_path, json.dumps(config, indent=2))
28
+ return True
29
+
30
+
31
+def get_config_path() -> str:
32
+ return plugins.determine_plugin_asset_path(
33
+ PLUGIN_NAME, "", "", plugins.CONFIG_FILE_NAME
34
+ )
35
+
36
+
37
+def read_saved_config() -> dict[str, Any]:
38
+ config_path = get_config_path()
39
+ if not files.exists(config_path):
40
+ return {}
41
+
42
+ try:
43
+ return json.loads(files.read_file(config_path))
44
+ except Exception:
45
+ return {}
46
+
47
+
48
+def build_seed_config(legacy_settings: dict[str, Any]) -> dict[str, Any]:
49
+ seeded = dict(DEFAULT_CONFIG)
50
+
51
+ model_size = str(
52
+ legacy_settings.get("stt_model_size", seeded["model_size"]) or ""
53
+ ).strip()
54
+ if model_size:
55
+ seeded["model_size"] = model_size
56
+
57
+ language = str(legacy_settings.get("stt_language", seeded["language"]) or "").strip()
58
+ if language:
59
+ seeded["language"] = language
60
+
61
+ seeded["silence_threshold"] = _coerce_float(
62
+ legacy_settings.get("stt_silence_threshold"),
63
+ seeded["silence_threshold"],
64
+ )
65
+ seeded["silence_duration"] = _coerce_int(
66
+ legacy_settings.get("stt_silence_duration"),
67
+ seeded["silence_duration"],
68
+ )
69
+ seeded["waiting_timeout"] = _coerce_int(
70
+ legacy_settings.get("stt_waiting_timeout"),
71
+ seeded["waiting_timeout"],
72
+ )
73
+
74
+ return seeded
75
+
76
+
77
+def _read_legacy_settings() -> dict[str, Any]:
78
+ if not files.exists(LEGACY_SETTINGS_FILE):
79
+ return {}
80
+
81
+ try:
82
+ return json.loads(files.read_file(LEGACY_SETTINGS_FILE))
83
+ except Exception:
84
+ return {}
85
+
86
+
87
+def _coerce_float(value: Any, default: float) -> float:
88
+ try:
89
+ return float(value)
90
+ except (TypeError, ValueError):
91
+ return default
92
+
93
+
94
+def _coerce_int(value: Any, default: int) -> int:
95
+ try:
96
+ return int(value)
97
+ except (TypeError, ValueError):
98
+ return default
plugins/_whisper_stt/helpers/runtime.py
new
+193
@@ -0,0 +1,193 @@
1
+from __future__ import annotations
2
+
3
+import asyncio
4
+import base64
5
+import os
6
+import tempfile
7
+import warnings
8
+from typing import Any
9
+
10
+import whisper
11
+
12
+from helpers import files, plugins
13
+from helpers.notification import (
14
+ NotificationManager,
15
+ NotificationPriority,
16
+ NotificationType,
17
+)
18
+from helpers.print_style import PrintStyle
19
+from plugins._whisper_stt.helpers import migration
20
+
21
+
22
+warnings.filterwarnings("ignore", category=FutureWarning)
23
+
24
+
25
+PLUGIN_NAME = "_whisper_stt"
26
+DEFAULT_CONFIG = {
27
+ "model_size": "base",
28
+ "language": "en",
29
+ "message_mode": "send",
30
+ "silence_threshold": 0.3,
31
+ "silence_duration": 1000,
32
+ "waiting_timeout": 2000,
33
+}
34
+VALID_MODEL_SIZES = {"tiny", "base", "small", "medium", "large", "turbo"}
35
+VALID_MESSAGE_MODES = {"send", "draft"}
36
+
37
+_model = None
38
+_model_name = ""
39
+is_updating_model = False
40
+
41
+
42
+def normalize_config(config: dict[str, Any] | None) -> dict[str, Any]:
43
+ normalized = dict(DEFAULT_CONFIG)
44
+ if not isinstance(config, dict):
45
+ return normalized
46
+
47
+ model_size = str(config.get("model_size", normalized["model_size"]) or "").strip()
48
+ if model_size in VALID_MODEL_SIZES:
49
+ normalized["model_size"] = model_size
50
+
51
+ language = str(config.get("language", normalized["language"]) or "").strip()
52
+ if language:
53
+ normalized["language"] = language
54
+
55
+ message_mode = (
56
+ str(config.get("message_mode", normalized["message_mode"]) or "")
57
+ .strip()
58
+ .lower()
59
+ )
60
+ if message_mode in VALID_MESSAGE_MODES:
61
+ normalized["message_mode"] = message_mode
62
+
63
+ try:
64
+ silence_threshold = float(
65
+ config.get("silence_threshold", normalized["silence_threshold"])
66
+ )
67
+ normalized["silence_threshold"] = min(max(silence_threshold, 0.0), 1.0)
68
+ except (TypeError, ValueError):
69
+ pass
70
+
71
+ try:
72
+ silence_duration = int(
73
+ config.get("silence_duration", normalized["silence_duration"])
74
+ )
75
+ if silence_duration > 0:
76
+ normalized["silence_duration"] = silence_duration
77
+ except (TypeError, ValueError):
78
+ pass
79
+
80
+ try:
81
+ waiting_timeout = int(config.get("waiting_timeout", normalized["waiting_timeout"]))
82
+ if waiting_timeout > 0:
83
+ normalized["waiting_timeout"] = waiting_timeout
84
+ except (TypeError, ValueError):
85
+ pass
86
+
87
+ return normalized
88
+
89
+
90
+def get_config() -> dict[str, Any]:
91
+ migration.ensure_config_seeded()
92
+ config = plugins.get_plugin_config(PLUGIN_NAME) or {}
93
+ return normalize_config(config)
94
+
95
+
96
+def get_loaded_model_name() -> str:
97
+ return _model_name
98
+
99
+
100
+def is_globally_enabled() -> bool:
101
+ return plugins.determined_toggle_from_paths(
102
+ True, reversed(plugins.get_plugin_roots(PLUGIN_NAME))
103
+ )
104
+
105
+
106
+async def preload(model_name: str | None = None):
107
+ cfg = get_config()
108
+ resolved_model = str(model_name or cfg["model_size"])
109
+ return await _preload(resolved_model)
110
+
111
+
112
+async def _preload(model_name: str):
113
+ global _model, _model_name, is_updating_model
114
+
115
+ while is_updating_model:
116
+ await asyncio.sleep(0.1)
117
+
118
+ try:
119
+ is_updating_model = True
120
+ if not _model or _model_name != model_name:
121
+ NotificationManager.send_notification(
122
+ NotificationType.INFO,
123
+ NotificationPriority.NORMAL,
124
+ "Loading Whisper model...",
125
+ display_time=99,
126
+ group="whisper-preload",
127
+ )
128
+ PrintStyle.standard(f"Loading Whisper model: {model_name}")
129
+ _model = whisper.load_model(
130
+ name=model_name,
131
+ download_root=files.get_abs_path("/tmp/models/whisper"),
132
+ )
133
+ _model_name = model_name
134
+ NotificationManager.send_notification(
135
+ NotificationType.INFO,
136
+ NotificationPriority.NORMAL,
137
+ "Whisper model loaded.",
138
+ display_time=2,
139
+ group="whisper-preload",
140
+ )
141
+ finally:
142
+ is_updating_model = False
143
+
144
+
145
+async def is_downloading() -> bool:
146
+ return is_updating_model
147
+
148
+
149
+async def is_downloaded() -> bool:
150
+ return _model is not None
151
+
152
+
153
+async def transcribe(
154
+ audio_bytes_b64: str, config: dict[str, Any] | None = None
155
+) -> dict[str, Any]:
156
+ cfg = normalize_config(config or get_config())
157
+ return await _transcribe(
158
+ str(cfg["model_size"]),
159
+ audio_bytes_b64,
160
+ language=_resolve_language(str(cfg["language"])),
161
+ )
162
+
163
+
164
+async def _transcribe(
165
+ model_name: str, audio_bytes_b64: str, *, language: str | None = None
166
+) -> dict[str, Any]:
167
+ await _preload(model_name)
168
+
169
+ audio_bytes = base64.b64decode(audio_bytes_b64)
170
+
171
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as audio_file:
172
+ audio_file.write(audio_bytes)
173
+ temp_path = audio_file.name
174
+
175
+ try:
176
+ kwargs: dict[str, Any] = {"fp16": False}
177
+ if language:
178
+ kwargs["language"] = language
179
+
180
+ result = _model.transcribe(temp_path, **kwargs) # type: ignore[union-attr]
181
+ return result if isinstance(result, dict) else {}
182
+ finally:
183
+ try:
184
+ os.remove(temp_path)
185
+ except Exception:
186
+ pass
187
+
188
+
189
+def _resolve_language(language: str) -> str | None:
190
+ value = language.strip().lower()
191
+ if not value or value == "auto":
192
+ return None
193
+ return value
plugins/_whisper_stt/hooks.py
new
+23
@@ -0,0 +1,23 @@
1
+from __future__ import annotations
2
+
3
+from helpers.defer import DeferredTask
4
+from plugins._whisper_stt.helpers import migration, runtime
5
+
6
+
7
+def get_plugin_config(default=None, **kwargs):
8
+ migration.ensure_config_seeded()
9
+ return runtime.normalize_config(default or {})
10
+
11
+
12
+def save_plugin_config(default=None, settings=None, **kwargs):
13
+ migration.ensure_config_seeded()
14
+
15
+ normalized = runtime.normalize_config(settings or default or {})
16
+ previous = runtime.normalize_config(migration.read_saved_config())
17
+
18
+ previous_model = str(previous.get("model_size") or "")
19
+ next_model = str(normalized.get("model_size") or "")
20
+ if next_model and next_model != previous_model:
21
+ DeferredTask().start_task(runtime.preload, next_model)
22
+
23
+ return normalized
plugins/_whisper_stt/plugin.yaml
new
+9
@@ -0,0 +1,9 @@
1
+name: _whisper_stt
2
+title: Whisper STT
3
+description: Built-in Whisper speech-to-text plugin.
4
+version: 1.0.0
5
+always_enabled: false
6
+settings_sections:
7
+ - agent
8
+per_project_config: false
9
+per_agent_config: false
plugins/_whisper_stt/webui/config.html
new
+102
@@ -0,0 +1,102 @@
1
+<html>
2
+<head>
3
+ <title>Whisper STT</title>
4
+</head>
5
+
6
+<body>
7
+ <div x-data>
8
+ <template x-if="config">
9
+ <div class="plugin-config-page">
10
+ <div class="section-title">Whisper STT</div>
11
+ <div class="section-description">
12
+ Configure the built-in Whisper speech-to-text provider. Browser microphone
13
+ device selection stays local to the Web UI and is managed from the plugin status panel.
14
+ </div>
15
+
16
+ <div class="field">
17
+ <div class="field-label">
18
+ <div class="field-title">Model Size</div>
19
+ <div class="field-description">
20
+ Whisper model variant loaded by the backend runtime.
21
+ </div>
22
+ </div>
23
+ <div class="field-control">
24
+ <select x-model="config.model_size">
25
+ <option value="tiny">Tiny</option>
26
+ <option value="base">Base</option>
27
+ <option value="small">Small</option>
28
+ <option value="medium">Medium</option>
29
+ <option value="large">Large</option>
30
+ <option value="turbo">Turbo</option>
31
+ </select>
32
+ </div>
33
+ </div>
34
+
35
+ <div class="field">
36
+ <div class="field-label">
37
+ <div class="field-title">Language</div>
38
+ <div class="field-description">
39
+ Language hint forwarded to Whisper. Use <code>auto</code> to let Whisper detect it.
40
+ </div>
41
+ </div>
42
+ <div class="field-control">
43
+ <input type="text" x-model="config.language" placeholder="en" />
44
+ </div>
45
+ </div>
46
+
47
+ <div class="field">
48
+ <div class="field-label">
49
+ <div class="field-title">Voice Message Handling</div>
50
+ <div class="field-description">
51
+ Choose whether final transcriptions are sent immediately or left in the composer for review.
52
+ </div>
53
+ </div>
54
+ <div class="field-control">
55
+ <select x-model="config.message_mode">
56
+ <option value="send">Send immediately</option>
57
+ <option value="draft">Draft in composer</option>
58
+ </select>
59
+ </div>
60
+ </div>
61
+
62
+ <div class="field">
63
+ <div class="field-label">
64
+ <div class="field-title">Silence Threshold</div>
65
+ <div class="field-description">
66
+ Minimum detected signal before voice input switches from listening to recording.
67
+ </div>
68
+ </div>
69
+ <div class="field-control">
70
+ <input type="range" min="0" max="1" step="0.01" x-model.number="config.silence_threshold" />
71
+ <span class="range-value" x-text="config.silence_threshold"></span>
72
+ </div>
73
+ </div>
74
+
75
+ <div class="field">
76
+ <div class="field-label">
77
+ <div class="field-title">Silence Duration</div>
78
+ <div class="field-description">
79
+ Milliseconds of silence required before the recording enters the waiting phase.
80
+ </div>
81
+ </div>
82
+ <div class="field-control">
83
+ <input type="number" min="100" step="100" x-model.number="config.silence_duration" />
84
+ </div>
85
+ </div>
86
+
87
+ <div class="field">
88
+ <div class="field-label">
89
+ <div class="field-title">Waiting Timeout</div>
90
+ <div class="field-description">
91
+ Milliseconds to wait after silence before audio is sent for transcription.
92
+ </div>
93
+ </div>
94
+ <div class="field-control">
95
+ <input type="number" min="100" step="100" x-model.number="config.waiting_timeout" />
96
+ </div>
97
+ </div>
98
+ </div>
99
+ </template>
100
+ </div>
101
+</body>
102
+</html>
plugins/_whisper_stt/webui/main.html
new
+178
@@ -0,0 +1,178 @@
1
+<html>
2
+<head>
3
+ <title>Whisper STT</title>
4
+ <script type="module">
5
+ import { store } from "/plugins/_whisper_stt/webui/whisper-stt-store.js";
6
+ </script>
7
+</head>
8
+
9
+<body>
10
+ <div
11
+ x-data
12
+ x-init="$store.whisperStt.initRuntime()"
13
+ class="speech-plugin-page"
14
+ >
15
+ <template x-if="$store.whisperStt">
16
+ <div>
17
+ <div class="section-title">Whisper STT</div>
18
+ <div class="section-description">
19
+ Built-in Whisper transcription. Dependency installation stays on the Docker/bootstrap path;
20
+ this plugin only owns the speech-to-text behavior, UI, and routing.
21
+ </div>
22
+
23
+ <div class="speech-plugin-grid">
24
+ <div class="speech-plugin-card">
25
+ <div class="field-title">Provider State</div>
26
+ <div class="status-row">
27
+ <span class="status-key">Enabled</span>
28
+ <span class="status-badge" :class="$store.whisperStt.enabled ? 'ok' : 'warn'" x-text="$store.whisperStt.enabled ? 'Yes' : 'No'"></span>
29
+ </div>
30
+ <div class="status-row">
31
+ <span class="status-key">Model</span>
32
+ <span class="status-badge" :class="$store.whisperStt.statusClass" x-text="$store.whisperStt.statusText"></span>
33
+ </div>
34
+ <div class="status-row" x-show="$store.whisperStt.loadedModel">
35
+ <span class="status-key">Loaded</span>
36
+ <span class="status-value" x-text="$store.whisperStt.loadedModel"></span>
37
+ </div>
38
+ <div class="status-row" x-show="$store.whisperStt.packageVersion">
39
+ <span class="status-key">Package</span>
40
+ <span class="status-value" x-text="$store.whisperStt.packageVersion"></span>
41
+ </div>
42
+ </div>
43
+
44
+ <div class="speech-plugin-card">
45
+ <div class="field-title">Resolved Config</div>
46
+ <div class="status-row">
47
+ <span class="status-key">Model size</span>
48
+ <span class="status-value" x-text="$store.whisperStt.config.model_size"></span>
49
+ </div>
50
+ <div class="status-row">
51
+ <span class="status-key">Language</span>
52
+ <span class="status-value" x-text="$store.whisperStt.config.language"></span>
53
+ </div>
54
+ <div class="status-row">
55
+ <span class="status-key">Message</span>
56
+ <span class="status-value" x-text="$store.whisperStt.messageModeLabel"></span>
57
+ </div>
58
+ <div class="status-row">
59
+ <span class="status-key">Threshold</span>
60
+ <span class="status-value" x-text="$store.whisperStt.config.silence_threshold"></span>
61
+ </div>
62
+ <div class="status-row">
63
+ <span class="status-key">Silence</span>
64
+ <span class="status-value" x-text="`${$store.whisperStt.config.silence_duration} ms`"></span>
65
+ </div>
66
+ <div class="status-row">
67
+ <span class="status-key">Wait</span>
68
+ <span class="status-value" x-text="`${$store.whisperStt.config.waiting_timeout} ms`"></span>
69
+ </div>
70
+ </div>
71
+
72
+ <div class="speech-plugin-card">
73
+ <div class="field-title">Microphone</div>
74
+ <div class="status-row">
75
+ <span class="status-key">Status</span>
76
+ <span class="status-badge warn" x-text="$store.whisperStt.micStatus"></span>
77
+ </div>
78
+ <label class="device-picker">
79
+ <span class="status-key">Device</span>
80
+ <select
81
+ :value="$store.whisperStt.selectedDevice"
82
+ @change="$store.whisperStt.selectDevice($event.target.value)"
83
+ >
84
+ <option value="">System default</option>
85
+ <template x-for="device in $store.whisperStt.devices" :key="device.deviceId">
86
+ <option :value="device.deviceId" x-text="device.label || `Microphone ${device.deviceId}`"></option>
87
+ </template>
88
+ </select>
89
+ </label>
90
+ <div class="field-description">
91
+ Device selection is browser-local and applies to the microphone button injected by this plugin.
92
+ </div>
93
+ </div>
94
+ </div>
95
+
96
+ <div class="speech-plugin-actions">
97
+ <button class="btn btn-field" @click="$store.whisperStt.requestMicrophonePermission()">Request Mic Permission</button>
98
+ <button class="btn btn-field" @click="$store.whisperStt.openConfig()">Open Settings</button>
99
+ <button class="btn btn-field" @click="$store.whisperStt.refreshStatus()">Refresh</button>
100
+ </div>
101
+ </div>
102
+ </template>
103
+ </div>
104
+
105
+ <style>
106
+ .speech-plugin-page {
107
+ display: flex;
108
+ flex-direction: column;
109
+ gap: 14px;
110
+ }
111
+
112
+ .speech-plugin-grid {
113
+ display: grid;
114
+ gap: 12px;
115
+ grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
116
+ }
117
+
118
+ .speech-plugin-card {
119
+ display: flex;
120
+ flex-direction: column;
121
+ gap: 10px;
122
+ padding: 14px;
123
+ background: var(--color-input);
124
+ }
125
+
126
+ .speech-plugin-actions {
127
+ display: flex;
128
+ gap: 8px;
129
+ flex-wrap: wrap;
130
+ }
131
+
132
+ .status-row,
133
+ .device-picker {
134
+ display: flex;
135
+ align-items: flex-start;
136
+ justify-content: space-between;
137
+ gap: 12px;
138
+ font-size: 0.84rem;
139
+ }
140
+
141
+ .device-picker {
142
+ flex-direction: column;
143
+ align-items: stretch;
144
+ }
145
+
146
+ .status-key {
147
+ opacity: 0.7;
148
+ min-width: 64px;
149
+ }
150
+
151
+ .status-value {
152
+ text-align: right;
153
+ word-break: break-word;
154
+ }
155
+
156
+ .status-badge {
157
+ padding: 2px 8px;
158
+ border-radius: 999px;
159
+ font-size: 0.76rem;
160
+ font-weight: 600;
161
+ border: 1px solid transparent;
162
+ text-transform: capitalize;
163
+ }
164
+
165
+ .status-badge.ok {
166
+ color: #1b5e20;
167
+ background: rgba(46, 125, 50, 0.14);
168
+ border-color: rgba(46, 125, 50, 0.24);
169
+ }
170
+
171
+ .status-badge.warn {
172
+ color: #8a6100;
173
+ background: rgba(191, 144, 0, 0.14);
174
+ border-color: rgba(191, 144, 0, 0.24);
175
+ }
176
+ </style>
177
+</body>
178
+</html>
plugins/_whisper_stt/webui/thumbnail.jpg
Binary files /dev/null and b/plugins/_whisper_stt/webui/thumbnail.jpg differ
plugins/_whisper_stt/webui/whisper-stt-store.js
new
+721
@@ -0,0 +1,721 @@
1
+import { createStore } from "/js/AlpineStore.js";
2
+import { toastFrontendError } from "/components/notifications/notification-store.js";
3
+import { callJsonApi } from "/js/api.js";
4
+import { sttService } from "/js/stt-service.js";
5
+import { ttsService } from "/js/tts-service.js";
6
+import { sendMessage, updateChatInput } from "/index.js";
7
+
8
+const PLUGIN_NAME = "_whisper_stt";
9
+
10
+const Status = {
11
+ INACTIVE: "inactive",
12
+ ACTIVATING: "activating",
13
+ LISTENING: "listening",
14
+ RECORDING: "recording",
15
+ WAITING: "waiting",
16
+ PROCESSING: "processing",
17
+};
18
+
19
+const MicButtonClasses = [
20
+ "mic-disabled",
21
+ "mic-inactive",
22
+ "mic-activating",
23
+ "mic-listening",
24
+ "mic-recording",
25
+ "mic-waiting",
26
+ "mic-processing",
27
+];
28
+
29
+const MicStatusLabels = {
30
+ disabled: "Whisper STT disabled",
31
+ inactive: "Microphone standby",
32
+ activating: "Microphone activating",
33
+ listening: "Listening for speech",
34
+ recording: "Recording voice",
35
+ waiting: "Waiting for final silence",
36
+ processing: "Transcribing voice",
37
+};
38
+
39
+function clearMicrophoneTooltip(element) {
40
+ const tooltip = globalThis.bootstrap?.Tooltip?.getInstance?.(element);
41
+ tooltip?.dispose?.();
42
+ element.removeAttribute("title");
43
+ element.removeAttribute("data-bs-original-title");
44
+ element.removeAttribute("data-bs-toggle");
45
+ element.removeAttribute("data-bs-trigger");
46
+ element.removeAttribute("data-bs-tooltip-initialized");
47
+}
48
+
49
+const model = {
50
+ runtimeInitialized: false,
51
+ statusLoaded: false,
52
+ loading: false,
53
+ error: "",
54
+ enabled: false,
55
+ config: {
56
+ model_size: "base",
57
+ language: "en",
58
+ message_mode: "send",
59
+ silence_threshold: 0.3,
60
+ silence_duration: 1000,
61
+ waiting_timeout: 2000,
62
+ },
63
+ modelReady: false,
64
+ modelLoading: false,
65
+ loadedModel: "",
66
+ packageVersion: "",
67
+ providerCleanup: null,
68
+ microphoneInput: null,
69
+ isProcessingClick: false,
70
+ devices: [],
71
+ selectedDevice: "",
72
+ requestingPermission: false,
73
+ _ttsListener: null,
74
+ _deviceChangeListenerBound: false,
75
+
76
+ async initRuntime() {
77
+ if (this.runtimeInitialized) return;
78
+
79
+ this.runtimeInitialized = true;
80
+ await this.loadDevices();
81
+ await this.refreshStatus({ suppressError: true });
82
+
83
+ if (!this._deviceChangeListenerBound) {
84
+ navigator.mediaDevices?.addEventListener?.("devicechange", () => {
85
+ void this.loadDevices();
86
+ });
87
+ this._deviceChangeListenerBound = true;
88
+ }
89
+
90
+ if (!this._ttsListener) {
91
+ this._ttsListener = (event) => {
92
+ if (event?.detail?.isSpeaking && this.micStatus !== Status.INACTIVE) {
93
+ this.stop();
94
+ }
95
+ };
96
+ ttsService.addEventListener("statechange", this._ttsListener);
97
+ }
98
+ },
99
+
100
+ async ensureStatusLoaded({ force = false, suppressError = true } = {}) {
101
+ if ((!this.statusLoaded || force) && !this.loading) {
102
+ await this.refreshStatus({ suppressError });
103
+ }
104
+ },
105
+
106
+ async refreshStatus({ suppressError = false } = {}) {
107
+ this.loading = true;
108
+ this.error = "";
109
+
110
+ try {
111
+ const status = await callJsonApi(`/plugins/${PLUGIN_NAME}/status`, {});
112
+ this.statusLoaded = true;
113
+ this.enabled = !!status?.enabled;
114
+ this.config = {
115
+ model_size: status?.config?.model_size || "base",
116
+ language: status?.config?.language || "en",
117
+ message_mode:
118
+ status?.config?.message_mode === "draft" ? "draft" : "send",
119
+ silence_threshold: Number(status?.config?.silence_threshold ?? 0.3),
120
+ silence_duration: Number(status?.config?.silence_duration ?? 1000),
121
+ waiting_timeout: Number(status?.config?.waiting_timeout ?? 2000),
122
+ };
123
+ this.modelReady = !!status?.model?.ready;
124
+ this.modelLoading = !!status?.model?.loading;
125
+ this.loadedModel = status?.model?.loaded_model || "";
126
+ this.packageVersion = status?.package?.version || "";
127
+
128
+ if (this.enabled) {
129
+ this.registerProvider();
130
+ } else {
131
+ this.unregisterProvider();
132
+ }
133
+ } catch (error) {
134
+ this.error = error instanceof Error ? error.message : String(error);
135
+ this.unregisterProvider();
136
+ if (!suppressError) {
137
+ void toastFrontendError(this.error, "Whisper STT");
138
+ }
139
+ } finally {
140
+ this.loading = false;
141
+ this.updateMicrophoneButtonUI();
142
+ }
143
+ },
144
+
145
+ registerProvider() {
146
+ if (this.providerCleanup || !this.enabled) return;
147
+
148
+ this.providerCleanup = sttService.registerProvider(PLUGIN_NAME, {
149
+ handleMicrophoneClick: async () => await this.handleMicrophoneClick(),
150
+ requestMicrophonePermission: async () =>
151
+ await this.requestMicrophonePermission(),
152
+ updateMicrophoneButtonUI: () => this.updateMicrophoneButtonUI(),
153
+ stop: () => this.stop(),
154
+ getStatus: () => this.micStatus,
155
+ });
156
+
157
+ sttService.emitStatusChange(this.micStatus);
158
+ this.updateMicrophoneButtonUI();
159
+ },
160
+
161
+ unregisterProvider() {
162
+ if (!this.providerCleanup) return;
163
+
164
+ this.stop();
165
+ this.providerCleanup();
166
+ this.providerCleanup = null;
167
+ },
168
+
169
+ async openConfig() {
170
+ const { store } = await import("/components/plugins/plugin-settings-store.js");
171
+ await store.openConfig(PLUGIN_NAME);
172
+ },
173
+
174
+ openPanel() {
175
+ window.openModal?.(`/plugins/${PLUGIN_NAME}/webui/main.html`);
176
+ },
177
+
178
+ updateMicrophoneButtonUI() {
179
+ const microphoneButton = document.getElementById("microphone-button");
180
+ if (!microphoneButton) return;
181
+
182
+ const status = this.enabled ? this.micStatus : "disabled";
183
+ const label = MicStatusLabels[status] || "Microphone";
184
+ clearMicrophoneTooltip(microphoneButton);
185
+ microphoneButton.classList.remove(...MicButtonClasses);
186
+ microphoneButton.classList.add(`mic-${status}`);
187
+ microphoneButton.setAttribute("data-status", status);
188
+ microphoneButton.setAttribute("aria-label", label);
189
+ microphoneButton.setAttribute(
190
+ "aria-pressed",
191
+ String(
192
+ status !== "disabled" &&
193
+ status !== Status.INACTIVE &&
194
+ status !== Status.ACTIVATING,
195
+ ),
196
+ );
197
+ },
198
+
199
+ async loadDevices() {
200
+ try {
201
+ const devices = await navigator.mediaDevices.enumerateDevices();
202
+ this.devices = devices.filter(
203
+ (device) => device.kind === "audioinput" && device.deviceId,
204
+ );
205
+
206
+ const saved = localStorage.getItem("whisperSttSelectedDevice") || "";
207
+ const savedStillExists = this.devices.some(
208
+ (device) => device.deviceId === saved,
209
+ );
210
+
211
+ if (savedStillExists) {
212
+ this.selectedDevice = saved;
213
+ return;
214
+ }
215
+
216
+ const defaultDevice =
217
+ this.devices.find((device) => device.deviceId === "default") ||
218
+ this.devices[0];
219
+ this.selectedDevice = defaultDevice?.deviceId || "";
220
+ } catch (error) {
221
+ console.error("[Whisper STT] Failed to enumerate audio devices", error);
222
+ this.devices = [];
223
+ this.selectedDevice = "";
224
+ }
225
+ },
226
+
227
+ async selectDevice(deviceId) {
228
+ this.selectedDevice = deviceId || "";
229
+ localStorage.setItem("whisperSttSelectedDevice", this.selectedDevice);
230
+
231
+ if (this.microphoneInput?.selectedDeviceId !== this.selectedDevice) {
232
+ this.stop();
233
+ this.microphoneInput = null;
234
+ }
235
+ },
236
+
237
+ getSelectedDevice() {
238
+ let device = this.devices.find(
239
+ (candidate) => candidate.deviceId === this.selectedDevice,
240
+ );
241
+
242
+ if (!device && this.devices.length > 0) {
243
+ device =
244
+ this.devices.find((candidate) => candidate.deviceId === "default") ||
245
+ this.devices[0];
246
+ }
247
+
248
+ return device || null;
249
+ },
250
+
251
+ async requestMicrophonePermission() {
252
+ this.requestingPermission = true;
253
+
254
+ try {
255
+ const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
256
+ stream.getTracks().forEach((track) => track.stop());
257
+ await this.loadDevices();
258
+ return true;
259
+ } catch (error) {
260
+ console.error("[Whisper STT] Microphone permission denied", error);
261
+ globalThis.toast?.(
262
+ "Microphone access denied. Please enable microphone access in your browser settings.",
263
+ "error",
264
+ );
265
+ return false;
266
+ } finally {
267
+ this.requestingPermission = false;
268
+ }
269
+ },
270
+
271
+ async handleMicrophoneClick() {
272
+ if (this.isProcessingClick) return;
273
+
274
+ this.isProcessingClick = true;
275
+ try {
276
+ await this.ensureStatusLoaded({ force: true, suppressError: false });
277
+ if (!this.enabled) {
278
+ globalThis.justToast?.("Whisper STT is disabled.", "info");
279
+ return;
280
+ }
281
+
282
+ ttsService.stop();
283
+
284
+ const selectedDevice = this.getSelectedDevice();
285
+ if (
286
+ this.microphoneInput &&
287
+ this.microphoneInput.selectedDeviceId !== (selectedDevice?.deviceId || "")
288
+ ) {
289
+ this.stop();
290
+ this.microphoneInput = null;
291
+ }
292
+
293
+ if (!this.microphoneInput) {
294
+ await this.initMicrophone();
295
+ }
296
+
297
+ if (this.microphoneInput) {
298
+ await this.microphoneInput.toggle();
299
+ }
300
+ } finally {
301
+ setTimeout(() => {
302
+ this.isProcessingClick = false;
303
+ }, 300);
304
+ }
305
+ },
306
+
307
+ async initMicrophone() {
308
+ if (this.microphoneInput) return this.microphoneInput;
309
+
310
+ const input = new MicrophoneInput(this, async (text, isFinal) => {
311
+ if (isFinal) {
312
+ await this.sendVoiceMessage(text);
313
+ }
314
+ });
315
+
316
+ const initialized = await input.initialize();
317
+ this.microphoneInput = initialized ? input : null;
318
+ return this.microphoneInput;
319
+ },
320
+
321
+ async sendVoiceMessage(text) {
322
+ const message = String(text || "").trim();
323
+ if (!message) return;
324
+
325
+ updateChatInput(message);
326
+
327
+ if (!this.sendsImmediately) {
328
+ this.stop();
329
+ return;
330
+ }
331
+
332
+ if (!this.microphoneInput?.messageSent) {
333
+ this.microphoneInput.messageSent = true;
334
+ await sendMessage();
335
+ }
336
+ },
337
+
338
+ notifyStatusChange() {
339
+ this.updateMicrophoneButtonUI();
340
+ sttService.emitStatusChange(this.micStatus);
341
+ },
342
+
343
+ stop() {
344
+ if (this.microphoneInput) {
345
+ this.microphoneInput.status = Status.INACTIVE;
346
+ this.microphoneInput.dispose();
347
+ this.microphoneInput = null;
348
+ }
349
+
350
+ this.notifyStatusChange();
351
+ },
352
+
353
+ get micStatus() {
354
+ return this.microphoneInput?.status || Status.INACTIVE;
355
+ },
356
+
357
+ get sendsImmediately() {
358
+ return this.config.message_mode !== "draft";
359
+ },
360
+
361
+ get messageModeLabel() {
362
+ return this.sendsImmediately ? "Send immediately" : "Draft in composer";
363
+ },
364
+
365
+ get statusText() {
366
+ if (!this.enabled) return "Disabled";
367
+ if (this.modelLoading) return "Loading";
368
+ if (this.modelReady) return "Ready";
369
+ return "Idle";
370
+ },
371
+
372
+ get statusClass() {
373
+ if (!this.enabled) return "warn";
374
+ if (this.modelLoading) return "warn";
375
+ if (this.modelReady) return "ok";
376
+ return "warn";
377
+ },
378
+
379
+ get selectedDeviceLabel() {
380
+ const device = this.getSelectedDevice();
381
+ if (!device) return "System default";
382
+ return device.label || "System default";
383
+ },
384
+};
385
+
386
+class MicrophoneInput {
387
+ constructor(owner, updateCallback) {
388
+ this.owner = owner;
389
+ this.updateCallback = updateCallback;
390
+ this.mediaStream = null;
391
+ this.mediaRecorder = null;
392
+ this.audioContext = null;
393
+ this.mediaStreamSource = null;
394
+ this.analyserNode = null;
395
+ this.audioChunks = [];
396
+ this.lastChunk = null;
397
+ this.messageSent = false;
398
+ this.lastAudioTime = null;
399
+ this.waitingTimer = null;
400
+ this.silenceStartTime = null;
401
+ this.hasStartedRecording = false;
402
+ this.analysisFrame = null;
403
+ this.selectedDeviceId = "";
404
+ this._status = Status.INACTIVE;
405
+ }
406
+
407
+ get status() {
408
+ return this._status;
409
+ }
410
+
411
+ set status(nextStatus) {
412
+ if (this._status === nextStatus) return;
413
+
414
+ const previousStatus = this._status;
415
+ this._status = nextStatus;
416
+ this.handleStatusChange(previousStatus, nextStatus);
417
+ this.owner.notifyStatusChange();
418
+ }
419
+
420
+ async initialize() {
421
+ this.status = Status.ACTIVATING;
422
+
423
+ try {
424
+ const selectedDevice = this.owner.getSelectedDevice();
425
+ const stream = await navigator.mediaDevices.getUserMedia({
426
+ audio: {
427
+ deviceId:
428
+ selectedDevice?.deviceId
429
+ ? { exact: selectedDevice.deviceId }
430
+ : undefined,
431
+ echoCancellation: true,
432
+ noiseSuppression: true,
433
+ channelCount: 1,
434
+ },
435
+ });
436
+
437
+ this.selectedDeviceId = selectedDevice?.deviceId || "";
438
+ this.mediaStream = stream;
439
+ this.mediaRecorder = new MediaRecorder(stream);
440
+ this.mediaRecorder.ondataavailable = (event) => {
441
+ if (
442
+ event.data.size > 0 &&
443
+ (this.status === Status.RECORDING || this.status === Status.WAITING)
444
+ ) {
445
+ if (this.lastChunk) {
446
+ this.audioChunks.push(this.lastChunk);
447
+ this.lastChunk = null;
448
+ }
449
+ this.audioChunks.push(event.data);
450
+ } else if (this.status === Status.LISTENING) {
451
+ this.lastChunk = event.data;
452
+ }
453
+ };
454
+
455
+ this.setupAudioAnalysis(stream);
456
+ return true;
457
+ } catch (error) {
458
+ console.error("[Whisper STT] Microphone initialization failed", error);
459
+ globalThis.toast?.(
460
+ "Failed to access the microphone. Please check browser permissions.",
461
+ "error",
462
+ );
463
+ this.status = Status.INACTIVE;
464
+ this.dispose();
465
+ return false;
466
+ }
467
+ }
468
+
469
+ handleStatusChange(previousStatus, nextStatus) {
470
+ if (nextStatus !== Status.RECORDING) {
471
+ this.lastChunk = null;
472
+ }
473
+
474
+ switch (nextStatus) {
475
+ case Status.INACTIVE:
476
+ this.handleInactiveState();
477
+ break;
478
+ case Status.LISTENING:
479
+ this.handleListeningState();
480
+ break;
481
+ case Status.RECORDING:
482
+ this.handleRecordingState();
483
+ break;
484
+ case Status.WAITING:
485
+ this.handleWaitingState();
486
+ break;
487
+ case Status.PROCESSING:
488
+ this.handleProcessingState();
489
+ break;
490
+ }
491
+ }
492
+
493
+ handleInactiveState() {
494
+ this.stopRecording();
495
+ this.stopAudioAnalysis();
496
+ clearTimeout(this.waitingTimer);
497
+ this.waitingTimer = null;
498
+ }
499
+
500
+ handleListeningState() {
501
+ this.stopRecording();
502
+ this.audioChunks = [];
503
+ this.hasStartedRecording = false;
504
+ this.silenceStartTime = null;
505
+ this.lastAudioTime = null;
506
+ this.messageSent = false;
507
+ this.startAudioAnalysis();
508
+ }
509
+
510
+ handleRecordingState() {
511
+ if (!this.mediaRecorder) return;
512
+
513
+ if (!this.hasStartedRecording && this.mediaRecorder.state !== "recording") {
514
+ this.hasStartedRecording = true;
515
+ this.mediaRecorder.start(1000);
516
+ }
517
+
518
+ clearTimeout(this.waitingTimer);
519
+ this.waitingTimer = null;
520
+ }
521
+
522
+ handleWaitingState() {
523
+ clearTimeout(this.waitingTimer);
524
+ this.waitingTimer = setTimeout(() => {
525
+ if (this.status === Status.WAITING) {
526
+ this.status = Status.PROCESSING;
527
+ }
528
+ }, this.owner.config.waiting_timeout);
529
+ }
530
+
531
+ handleProcessingState() {
532
+ this.stopRecording();
533
+ void this.process();
534
+ }
535
+
536
+ setupAudioAnalysis(stream) {
537
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
538
+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
539
+ this.analyserNode = this.audioContext.createAnalyser();
540
+ this.analyserNode.fftSize = 2048;
541
+ this.analyserNode.minDecibels = -90;
542
+ this.analyserNode.maxDecibels = -10;
543
+ this.analyserNode.smoothingTimeConstant = 0.85;
544
+ this.mediaStreamSource.connect(this.analyserNode);
545
+ }
546
+
547
+ startAudioAnalysis() {
548
+ const analyzeFrame = () => {
549
+ if (this.status === Status.INACTIVE || !this.analyserNode) return;
550
+
551
+ const dataArray = new Uint8Array(this.analyserNode.fftSize);
552
+ this.analyserNode.getByteTimeDomainData(dataArray);
553
+
554
+ let sum = 0;
555
+ for (let index = 0; index < dataArray.length; index += 1) {
556
+ const amplitude = (dataArray[index] - 128) / 128;
557
+ sum += amplitude * amplitude;
558
+ }
559
+
560
+ const rms = Math.sqrt(sum / dataArray.length);
561
+ const now = Date.now();
562
+ const silenceThreshold = this.densify(this.owner.config.silence_threshold);
563
+
564
+ if (rms > silenceThreshold) {
565
+ this.lastAudioTime = now;
566
+ this.silenceStartTime = null;
567
+
568
+ if (
569
+ (this.status === Status.LISTENING || this.status === Status.WAITING) &&
570
+ !ttsService.isSpeaking()
571
+ ) {
572
+ this.status = Status.RECORDING;
573
+ }
574
+ } else if (this.status === Status.RECORDING) {
575
+ if (!this.silenceStartTime) {
576
+ this.silenceStartTime = now;
577
+ }
578
+
579
+ const silenceDuration = now - this.silenceStartTime;
580
+ if (silenceDuration >= this.owner.config.silence_duration) {
581
+ this.status = Status.WAITING;
582
+ }
583
+ }
584
+
585
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
586
+ };
587
+
588
+ this.stopAudioAnalysis();
589
+ this.analysisFrame = requestAnimationFrame(analyzeFrame);
590
+ }
591
+
592
+ stopAudioAnalysis() {
593
+ if (this.analysisFrame) {
594
+ cancelAnimationFrame(this.analysisFrame);
595
+ this.analysisFrame = null;
596
+ }
597
+ }
598
+
599
+ stopRecording() {
600
+ if (this.mediaRecorder?.state === "recording") {
601
+ this.mediaRecorder.stop();
602
+ this.hasStartedRecording = false;
603
+ }
604
+ }
605
+
606
+ densify(value) {
607
+ return Math.exp(-5 * (1 - value));
608
+ }
609
+
610
+ async process() {
611
+ if (this.audioChunks.length === 0) {
612
+ if (this.status === Status.PROCESSING) {
613
+ this.status = Status.LISTENING;
614
+ }
615
+ return;
616
+ }
617
+
618
+ const audioBlob = new Blob(this.audioChunks, { type: "audio/wav" });
619
+ const audio = await this.convertBlobToBase64(audioBlob);
620
+
621
+ try {
622
+ const result = await callJsonApi(`/plugins/${PLUGIN_NAME}/transcribe`, {
623
+ audio,
624
+ });
625
+ const text = this.filterResult(result?.text || "");
626
+ if (text) {
627
+ await this.updateCallback(text, true);
628
+ }
629
+ } catch (error) {
630
+ console.error("[Whisper STT] Transcription failed", error);
631
+ window.toastFetchError?.("Transcription error", error);
632
+ } finally {
633
+ this.audioChunks = [];
634
+ if (this.status === Status.PROCESSING) {
635
+ this.status = Status.LISTENING;
636
+ }
637
+ }
638
+ }
639
+
640
+ convertBlobToBase64(audioBlob) {
641
+ return new Promise((resolve, reject) => {
642
+ const reader = new FileReader();
643
+ reader.onloadend = () => {
644
+ const result = String(reader.result || "");
645
+ resolve(result.split(",")[1] || "");
646
+ };
647
+ reader.onerror = (error) => reject(error);
648
+ reader.readAsDataURL(audioBlob);
649
+ });
650
+ }
651
+
652
+ filterResult(text) {
653
+ const normalized = String(text || "").trim();
654
+ if (!normalized) return "";
655
+
656
+ const wrapped =
657
+ (normalized.startsWith("{") && normalized.endsWith("}")) ||
658
+ (normalized.startsWith("(") && normalized.endsWith(")")) ||
659
+ (normalized.startsWith("[") && normalized.endsWith("]"));
660
+
661
+ if (wrapped) {
662
+ console.log(`[Whisper STT] Discarding transcription: ${normalized}`);
663
+ return "";
664
+ }
665
+
666
+ return normalized;
667
+ }
668
+
669
+ async toggle() {
670
+ const hasPermission = await this.requestPermission();
671
+ if (!hasPermission) return;
672
+
673
+ if (
674
+ this.status === Status.INACTIVE ||
675
+ this.status === Status.ACTIVATING
676
+ ) {
677
+ this.status = Status.LISTENING;
678
+ } else {
679
+ this.owner.stop();
680
+ }
681
+ }
682
+
683
+ async requestPermission() {
684
+ return await this.owner.requestMicrophonePermission();
685
+ }
686
+
687
+ dispose() {
688
+ clearTimeout(this.waitingTimer);
689
+ this.waitingTimer = null;
690
+ this.stopAudioAnalysis();
691
+
692
+ try {
693
+ this.mediaRecorder?.stream?.getTracks?.().forEach((track) => track.stop());
694
+ } catch (_error) {
695
+ // Ignore media cleanup failures.
696
+ }
697
+
698
+ try {
699
+ this.mediaStream?.getTracks?.().forEach((track) => track.stop());
700
+ } catch (_error) {
701
+ // Ignore media cleanup failures.
702
+ }
703
+
704
+ try {
705
+ this.audioContext?.close?.();
706
+ } catch (_error) {
707
+ // Ignore audio context cleanup failures.
708
+ }
709
+
710
+ this.mediaStream = null;
711
+ this.mediaRecorder = null;
712
+ this.mediaStreamSource = null;
713
+ this.analyserNode = null;
714
+ this.audioContext = null;
715
+ this.audioChunks = [];
716
+ this.lastChunk = null;
717
+ this.hasStartedRecording = false;
718
+ }
719
+}
720
+
721
+export const store = createStore("whisperStt", model);
plugins/_whisper_stt/webui/whisper-stt.css
new
+66
@@ -0,0 +1,66 @@
1
+#microphone-button {
2
+ color: var(--color-background);
3
+ transition:
4
+ background-color 0.2s ease,
5
+ box-shadow 0.12s ease-in-out,
6
+ filter 0.12s ease-in-out,
7
+ opacity 0.12s ease-in-out;
8
+}
9
+
10
+#microphone-button.mic-disabled {
11
+ background-color: #5f6368;
12
+ cursor: not-allowed;
13
+ opacity: 0.58;
14
+}
15
+
16
+#microphone-button.mic-inactive {
17
+ background-color: grey;
18
+}
19
+
20
+#microphone-button.mic-activating {
21
+ background-color: silver;
22
+ animation: whisper-stt-mic-pulse 0.8s infinite;
23
+}
24
+
25
+#microphone-button.mic-listening {
26
+ background-color: red;
27
+}
28
+
29
+#microphone-button.mic-recording {
30
+ background-color: green;
31
+}
32
+
33
+#microphone-button.mic-waiting {
34
+ background-color: teal;
35
+}
36
+
37
+#microphone-button.mic-processing {
38
+ background-color: darkcyan;
39
+ animation: whisper-stt-mic-pulse 0.8s infinite;
40
+ transform-origin: center;
41
+}
42
+
43
+@media (hover: hover) {
44
+ #microphone-button:not(.mic-disabled):hover {
45
+ filter: brightness(1.08);
46
+ box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.08),
47
+ 0 6px 14px rgba(0, 0, 0, 0.18);
48
+ }
49
+}
50
+
51
+#microphone-button:not(.mic-disabled):active {
52
+ filter: brightness(0.92);
53
+ box-shadow: inset 0 0 0 1px rgba(0, 0, 0, 0.12);
54
+}
55
+
56
+@keyframes whisper-stt-mic-pulse {
57
+ 0% {
58
+ transform: scale(1);
59
+ }
60
+ 50% {
61
+ transform: scale(1.1);
62
+ }
63
+ 100% {
64
+ transform: scale(1);
65
+ }
66
+}
preload.py
+13
-10
@@ -1,18 +1,20 @@
1
import asyncio
2
-from helpers import runtime, whisper, settings
2
+from helpers import runtime
3
from helpers.print_style import PrintStyle
4
-from helpers import kokoro_tts
4
import models
5
+from plugins._kokoro_tts.helpers import runtime as kokoro_tts_runtime
6
+from plugins._whisper_stt.helpers import runtime as whisper_stt_runtime
7
8
9
async def preload():
10
try:
10
- set = settings.get_default_settings()
11
-
11
# preload whisper model
12
async def preload_whisper():
13
+ if not whisper_stt_runtime.is_globally_enabled():
14
+ return None
15
try:
15
- return await whisper.preload(set["stt_model_size"])
16
+ config = whisper_stt_runtime.get_config()
17
+ return await whisper_stt_runtime.preload(str(config["model_size"]))
18
except Exception as e:
19
PrintStyle().error(f"Error in preload_whisper: {e}")
20
@@ -32,11 +34,12 @@ async def preload():
34
35
# preload kokoro tts model if enabled
36
async def preload_kokoro():
35
- if set["tts_kokoro"]:
36
- try:
37
- return await kokoro_tts.preload()
38
- except Exception as e:
39
- PrintStyle().error(f"Error in preload_kokoro: {e}")
37
+ if not kokoro_tts_runtime.is_globally_enabled():
38
+ return None
39
+ try:
40
+ return await kokoro_tts_runtime.preload()
41
+ except Exception as e:
42
+ PrintStyle().error(f"Error in preload_kokoro: {e}")
43
44
# async tasks to preload
45
tasks = [
tests/test_speech_plugin_split.py
new
+223
@@ -0,0 +1,223 @@
1
+from __future__ import annotations
2
+
3
+import importlib
4
+import sys
5
+import types
6
+from pathlib import Path
7
+
8
+
9
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
10
+if str(PROJECT_ROOT) not in sys.path:
11
+ sys.path.insert(0, str(PROJECT_ROOT))
12
+
13
+
14
+from helpers import plugins, settings
15
+
16
+
17
+def test_builtin_speech_plugins_are_discoverable_and_toggleable() -> None:
18
+ discovered = {
19
+ item.name: item
20
+ for item in plugins.get_enhanced_plugins_list(
21
+ custom=True,
22
+ builtin=True,
23
+ plugin_names=["_kokoro_tts", "_whisper_stt"],
24
+ )
25
+ }
26
+
27
+ assert "_kokoro_tts" in discovered
28
+ assert "_whisper_stt" in discovered
29
+
30
+ assert discovered["_kokoro_tts"].always_enabled is False
31
+ assert discovered["_whisper_stt"].always_enabled is False
32
+ assert "agent" in discovered["_kokoro_tts"].settings_sections
33
+ assert "agent" in discovered["_whisper_stt"].settings_sections
34
+
35
+
36
+def test_legacy_core_speech_artifacts_are_removed() -> None:
37
+ removed_paths = [
38
+ "api/synthesize.py",
39
+ "api/transcribe.py",
40
+ "helpers/kokoro_tts.py",
41
+ "helpers/whisper.py",
42
+ "webui/components/chat/speech/speech-store.js",
43
+ "webui/components/settings/agent/speech.html",
44
+ "webui/components/settings/speech/microphone-setting-store.js",
45
+ "webui/components/settings/speech/microphone.html",
46
+ "webui/css/speech.css",
47
+ "webui/js/speech_browser.js",
48
+ ]
49
+
50
+ for relative_path in removed_paths:
51
+ assert not (PROJECT_ROOT / relative_path).exists(), relative_path
52
+
53
+
54
+def test_plugin_owned_voice_files_exist() -> None:
55
+ expected_paths = [
56
+ "plugins/_kokoro_tts/plugin.yaml",
57
+ "plugins/_kokoro_tts/api/synthesize.py",
58
+ "plugins/_kokoro_tts/extensions/webui/page-head/runtime.html",
59
+ "plugins/_kokoro_tts/extensions/webui/voice-settings-main/kokoro-card.html",
60
+ "plugins/_whisper_stt/plugin.yaml",
61
+ "plugins/_whisper_stt/api/transcribe.py",
62
+ "plugins/_whisper_stt/extensions/webui/page-head/runtime.html",
63
+ "plugins/_whisper_stt/extensions/webui/chat-input-box-end/microphone-button.html",
64
+ "plugins/_whisper_stt/extensions/webui/voice-settings-main/whisper-card.html",
65
+ "plugins/_whisper_stt/webui/whisper-stt-store.js",
66
+ ]
67
+
68
+ for relative_path in expected_paths:
69
+ assert (PROJECT_ROOT / relative_path).exists(), relative_path
70
+
71
+
72
+def test_core_settings_no_longer_expose_legacy_speech_keys() -> None:
73
+ defaults = settings.get_default_settings()
74
+ output = settings.convert_out(defaults)
75
+
76
+ legacy_keys = {
77
+ "tts_kokoro",
78
+ "stt_model_size",
79
+ "stt_language",
80
+ "stt_silence_threshold",
81
+ "stt_silence_duration",
82
+ "stt_waiting_timeout",
83
+ }
84
+
85
+ assert legacy_keys.isdisjoint(defaults.keys())
86
+ assert legacy_keys.isdisjoint(output["settings"].keys())
87
+ assert "stt_models" not in output["additional"]
88
+
89
+
90
+def test_voice_prefix_prompt_rule_is_removed() -> None:
91
+ core_prompt = (PROJECT_ROOT / "prompts/agent.system.main.communication_additions.md").read_text(
92
+ encoding="utf-8"
93
+ )
94
+ whisper_store = (
95
+ PROJECT_ROOT / "plugins/_whisper_stt/webui/whisper-stt-store.js"
96
+ ).read_text(encoding="utf-8")
97
+ voice_surface = (PROJECT_ROOT / "webui/components/settings/agent/voice.html").read_text(
98
+ encoding="utf-8"
99
+ )
100
+
101
+ assert "if starts (voice) then transcribed can contain errors consider compensation" not in core_prompt
102
+ assert "(voice)" not in whisper_store
103
+ assert not (
104
+ PROJECT_ROOT / "plugins/_whisper_stt/prompts/agent.system.voice_transcription.md"
105
+ ).exists()
106
+ assert not (
107
+ PROJECT_ROOT
108
+ / "plugins/_whisper_stt/extensions/python/system_prompt/_20_voice_transcription.py"
109
+ ).exists()
110
+ assert '<x-extension id="voice-settings-start"></x-extension>' in voice_surface
111
+ assert '<x-extension id="voice-settings-main"></x-extension>' in voice_surface
112
+ assert '<x-extension id="voice-settings-end"></x-extension>' in voice_surface
113
+
114
+
115
+def test_whisper_message_mode_defaults_to_send_and_supports_draft() -> None:
116
+ sys.modules.setdefault(
117
+ "whisper",
118
+ types.SimpleNamespace(load_model=lambda *args, **kwargs: None),
119
+ )
120
+ runtime = importlib.import_module("plugins._whisper_stt.helpers.runtime")
121
+
122
+ assert runtime.normalize_config({})["message_mode"] == "send"
123
+ assert runtime.normalize_config({"message_mode": "draft"})["message_mode"] == "draft"
124
+ assert runtime.normalize_config({"message_mode": "DRAFT"})["message_mode"] == "draft"
125
+ assert runtime.normalize_config({"message_mode": "invalid"})["message_mode"] == "send"
126
+
127
+ default_config = (
128
+ PROJECT_ROOT / "plugins/_whisper_stt/default_config.yaml"
129
+ ).read_text(encoding="utf-8")
130
+ migration = (
131
+ PROJECT_ROOT / "plugins/_whisper_stt/helpers/migration.py"
132
+ ).read_text(encoding="utf-8")
133
+ config_ui = (
134
+ PROJECT_ROOT / "plugins/_whisper_stt/webui/config.html"
135
+ ).read_text(encoding="utf-8")
136
+ status_ui = (
137
+ PROJECT_ROOT / "plugins/_whisper_stt/webui/main.html"
138
+ ).read_text(encoding="utf-8")
139
+ voice_card = (
140
+ PROJECT_ROOT
141
+ / "plugins/_whisper_stt/extensions/webui/voice-settings-main/whisper-card.html"
142
+ ).read_text(encoding="utf-8")
143
+ whisper_store = (
144
+ PROJECT_ROOT / "plugins/_whisper_stt/webui/whisper-stt-store.js"
145
+ ).read_text(encoding="utf-8")
146
+
147
+ assert "message_mode: send" in default_config
148
+ assert '"message_mode": "send"' in migration
149
+ assert '<option value="send">Send immediately</option>' in config_ui
150
+ assert '<option value="draft">Draft in composer</option>' in config_ui
151
+ assert "messageModeLabel" in status_ui
152
+ assert "messageModeLabel" in voice_card
153
+ assert 'message_mode: "send"' in whisper_store
154
+ assert 'status?.config?.message_mode === "draft" ? "draft" : "send"' in whisper_store
155
+ assert "updateChatInput(message)" in whisper_store
156
+ assert "sendMessage()" in whisper_store
157
+
158
+
159
+def test_browser_tool_speech_action_uses_shared_tts_service() -> None:
160
+ browser_handler = (
161
+ PROJECT_ROOT
162
+ / "plugins/_browser/extensions/webui/get_tool_message_handler/browser-tool-handler.js"
163
+ ).read_text(encoding="utf-8")
164
+
165
+ assert "/components/chat/speech/speech-store.js" not in browser_handler
166
+ assert "/js/tts-service.js" in browser_handler
167
+ assert "ttsService.speak(contentText)" in browser_handler
168
+
169
+
170
+def test_chat_bar_keeps_existing_send_and_mic_icon_contract() -> None:
171
+ chat_bar = (
172
+ PROJECT_ROOT / "webui/components/chat/input/chat-bar-input.html"
173
+ ).read_text(encoding="utf-8")
174
+ mic_extension = (
175
+ PROJECT_ROOT
176
+ / "plugins/_whisper_stt/extensions/webui/chat-input-box-end/microphone-button.html"
177
+ ).read_text(encoding="utf-8")
178
+ whisper_store = (
179
+ PROJECT_ROOT / "plugins/_whisper_stt/webui/whisper-stt-store.js"
180
+ ).read_text(encoding="utf-8")
181
+ whisper_css = (
182
+ PROJECT_ROOT / "plugins/_whisper_stt/webui/whisper-stt.css"
183
+ ).read_text(encoding="utf-8")
184
+
185
+ assert 'id="send-button"' in chat_bar
186
+ assert 'x-text="$store.chatInput.sendButtonIcon"' in chat_bar
187
+ assert ':class="$store.chatInput.sendButtonClass"' in chat_bar
188
+ assert ':title="$store.chatInput.sendButtonTitle"' in chat_bar
189
+
190
+ assert 'id="microphone-button"' in mic_extension
191
+ assert "<svg" in mic_extension
192
+ assert "material-symbols-outlined" not in mic_extension
193
+ assert "buttonIcon" not in mic_extension
194
+ assert 'title=' not in mic_extension
195
+ assert 'x-effect="$store.whisperStt.updateMicrophoneButtonUI()"' in mic_extension
196
+ assert 'x-init="$store.whisperStt.updateMicrophoneButtonUI()"' in mic_extension
197
+ assert "updateMicrophoneButtonUI()" in whisper_store
198
+ assert "data-status" in whisper_store
199
+ assert 'setAttribute("title"' not in whisper_store
200
+ assert 'removeAttribute("title")' in whisper_store
201
+ assert 'removeAttribute("data-bs-original-title")' in whisper_store
202
+ assert "this.updateMicrophoneButtonUI();" in whisper_store
203
+ assert "sttService.emitStatusChange(this.micStatus)" in whisper_store
204
+
205
+ for state in [
206
+ "disabled",
207
+ "inactive",
208
+ "activating",
209
+ "listening",
210
+ "recording",
211
+ "waiting",
212
+ "processing",
213
+ ]:
214
+ assert f'"mic-{state}"' in whisper_store
215
+ assert f"#microphone-button.mic-{state}" in whisper_css
216
+ assert f"#microphone-button.mic-{state}" in mic_extension
217
+
218
+ assert "background-color: red;" in whisper_css
219
+ assert "background-color: green;" in whisper_css
220
+ assert "background-color: teal;" in whisper_css
221
+ assert "background-color: grey;" in mic_extension
222
+ assert "whisper-stt-mic-pulse 0.8s infinite" in mic_extension
223
+ assert "whisper-stt-mic-pulse 0.8s infinite" in whisper_css
webui/components/chat/input/chat-bar-input.html
-9
@@ -1,7 +1,6 @@
1
<html>
2
<head>
3
<script type="module">
4
- import { store as speechStore } from "/components/chat/speech/speech-store.js";
4
import { store as fullScreenStore } from "/components/modals/full-screen-input/full-screen-store.js";
5
import { store as messageQueueStore } from "/components/chat/message-queue/message-queue-store.js";
6
import { store as chatInputStore } from "/components/chat/input/input-store.js";
@@ -65,14 +64,6 @@
64
:title="$store.chatInput.sendButtonTitle">
65
<span class="material-symbols-outlined" x-text="$store.chatInput.sendButtonIcon"></span>
66
</button>
68
-
69
- <!-- Microphone button -->
70
- <button class="chat-button mic-inactive" id="microphone-button" aria-label="Start/Stop recording"
71
- @click="$store.speech.handleMicrophoneClick()" x-effect="$store.speech.updateMicrophoneButtonUI()">
72
- <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor">
73
- <path d="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z" />
74
- </svg>
75
- </button>
67
</div>
68
</div>
69
webui/components/chat/input/progress.html
+23
-3
@@ -1,7 +1,6 @@
1
<html>
2
<head>
3
<script type="module">
4
- import { store as speechStore } from "/components/chat/speech/speech-store.js";
4
import { store as chatNavStore } from "/components/chat/navigation/chat-navigation-store.js";
5
</script>
6
</head>
@@ -13,8 +12,29 @@
12
<span id="progress-bar" style="display:none;"></span>
13
14
<div id="progress-bar-right">
16
- <h4 id="progress-bar-stop-speech" x-data x-cloak x-show="$store.speech.isSpeaking">
17
- <span id="stop-speech" @click="$store.speech.stop()" style="cursor: pointer" title="Stop Speech" aria-label="Stop Speech">
15
+ <h4
16
+ id="progress-bar-stop-speech"
17
+ x-data="{
18
+ speaking: window.ttsService?.isSpeaking?.() || false,
19
+ _listener: null,
20
+ init() {
21
+ this._listener = (event) => {
22
+ this.speaking = !!event?.detail?.isSpeaking;
23
+ };
24
+ window.ttsService?.addEventListener?.('statechange', this._listener);
25
+ },
26
+ cleanup() {
27
+ if (this._listener) {
28
+ window.ttsService?.removeEventListener?.('statechange', this._listener);
29
+ }
30
+ },
31
+ }"
32
+ x-init="init()"
33
+ x-destroy="cleanup()"
34
+ x-cloak
35
+ x-show="speaking"
36
+ >
37
+ <span id="stop-speech" @click="window.ttsService?.stop?.()" style="cursor: pointer" title="Stop Speech" aria-label="Stop Speech">
38
<span class="icon material-symbols-outlined">volume_off</span>
39
</span>
40
</h4>
webui/components/chat/speech/speech-store.js
deleted
-962
@@ -1,962 +0,0 @@
1
-import { createStore } from "/js/AlpineStore.js";
2
-import { updateChatInput, sendMessage } from "/index.js";
3
-import { sleep } from "/js/sleep.js";
4
-import { store as microphoneSettingStore } from "/components/settings/speech/microphone-setting-store.js";
5
-import * as shortcuts from "/js/shortcuts.js";
6
-
7
-const Status = {
8
- INACTIVE: "inactive",
9
- ACTIVATING: "activating",
10
- LISTENING: "listening",
11
- RECORDING: "recording",
12
- WAITING: "waiting",
13
- PROCESSING: "processing",
14
-};
15
-
16
-// Create the speech store
17
-const model = {
18
- // Initialization guard
19
- _initialized: false,
20
-
21
- // STT Settings
22
- stt_model_size: "tiny",
23
- stt_language: "en",
24
- stt_silence_threshold: 0.05,
25
- stt_silence_duration: 1000,
26
- stt_waiting_timeout: 2000,
27
-
28
- // TTS Settings
29
- tts_kokoro: false,
30
-
31
- // TTS State
32
- isSpeaking: false,
33
- speakingId: "",
34
- speakingText: "",
35
- currentAudio: null,
36
- audioEl: null,
37
- audioContext: null,
38
- userHasInteracted: false,
39
- stopSpeechChain: false,
40
- ttsStream: null,
41
-
42
- // STT State
43
- microphoneInput: null,
44
- isProcessingClick: false,
45
- selectedDevice: null,
46
-
47
- // Getter for micStatus - delegates to microphoneInput
48
- get micStatus() {
49
- return this.microphoneInput?.status || Status.INACTIVE;
50
- },
51
-
52
- updateMicrophoneButtonUI() {
53
- const microphoneButton = document.getElementById("microphone-button");
54
- if (!microphoneButton) return;
55
- const status = this.micStatus;
56
- microphoneButton.classList.remove(
57
- "mic-inactive",
58
- "mic-activating",
59
- "mic-listening",
60
- "mic-recording",
61
- "mic-waiting",
62
- "mic-processing"
63
- );
64
- microphoneButton.classList.add(`mic-${status.toLowerCase()}`);
65
- microphoneButton.setAttribute("data-status", status);
66
- },
67
-
68
- async handleMicrophoneClick() {
69
- if (this.isProcessingClick) return;
70
- this.isProcessingClick = true;
71
- try {
72
- // reset mic input if device has changed in settings
73
- const device = microphoneSettingStore.getSelectedDevice();
74
- if (device != this.selectedDevice) {
75
- this.selectedDevice = device;
76
- this.microphoneInput = null;
77
- console.log("Device changed, microphoneInput reset");
78
- }
79
-
80
- if (!this.microphoneInput) {
81
- await this.initMicrophone();
82
- }
83
-
84
- if (this.microphoneInput) {
85
- await this.microphoneInput.toggle();
86
- }
87
- } finally {
88
- setTimeout(() => {
89
- this.isProcessingClick = false;
90
- }, 300);
91
- }
92
- },
93
-
94
- // Initialize speech functionality
95
- async init() {
96
- // Guard against multiple initializations
97
- if (this._initialized) {
98
- console.log(
99
- "[Speech Store] Already initialized, skipping duplicate init()"
100
- );
101
- return;
102
- }
103
-
104
- this._initialized = true;
105
- await this.loadSettings();
106
- this.setupBrowserTTS();
107
- this.setupUserInteractionHandling();
108
- },
109
-
110
- // Load settings from server
111
- async loadSettings() {
112
- try {
113
- const response = await fetchApi("/settings_get", { method: "POST" });
114
- const data = await response.json();
115
- const settings = data?.settings || {};
116
-
117
- if (settings) {
118
- this.stt_model_size = settings.stt_model_size ?? this.stt_model_size;
119
- this.stt_language = settings.stt_language ?? this.stt_language;
120
- this.stt_silence_threshold =
121
- settings.stt_silence_threshold ?? this.stt_silence_threshold;
122
- this.stt_silence_duration =
123
- settings.stt_silence_duration ?? this.stt_silence_duration;
124
- this.stt_waiting_timeout =
125
- settings.stt_waiting_timeout ?? this.stt_waiting_timeout;
126
- this.tts_kokoro = settings.tts_kokoro ?? this.tts_kokoro;
127
- }
128
- } catch (error) {
129
- window.toastFetchError("Failed to load speech settings", error);
130
- console.error("Failed to load speech settings:", error);
131
- }
132
- },
133
-
134
- // Setup browser TTS
135
- setupBrowserTTS() {
136
- this.synth = window.speechSynthesis;
137
- this.browserUtterance = null;
138
- },
139
-
140
- // Setup user interaction handling for autoplay policy
141
- setupUserInteractionHandling() {
142
- const enableAudio = () => {
143
- if (!this.userHasInteracted) {
144
- this.userHasInteracted = true;
145
- console.log("User interaction detected - audio playback enabled");
146
-
147
- // Create a dummy audio context to "unlock" audio
148
- try {
149
- this.audioContext = new (window.AudioContext ||
150
- window.webkitAudioContext)();
151
- this.audioContext.resume();
152
- } catch (e) {
153
- console.log("AudioContext not available");
154
- }
155
- }
156
- };
157
-
158
- // Listen for any user interaction
159
- const events = ["click", "touchstart", "keydown", "mousedown"];
160
- events.forEach((event) => {
161
- document.addEventListener(event, enableAudio, {
162
- once: true,
163
- passive: true,
164
- });
165
- });
166
- },
167
-
168
- // main speak function, allows to speak a stream of text that is generated piece by piece
169
- async speakStream(id, text, finished = false) {
170
- // if already running the same stream, do nothing
171
- if (
172
- this.ttsStream &&
173
- this.ttsStream.id === id &&
174
- this.ttsStream.text === text &&
175
- this.ttsStream.finished === finished
176
- )
177
- return;
178
-
179
- // if user has not interacted (after reload), do not play audio
180
- if (!this.userHasInteracted) return this.showAudioPermissionPrompt();
181
-
182
- // new stream
183
- if (!this.ttsStream || this.ttsStream.id !== id) {
184
- // this.stop(); // stop potential previous stream
185
- // create new stream data
186
- this.ttsStream = {
187
- id,
188
- text,
189
- finished,
190
- running: false,
191
- lastChunkIndex: -1,
192
- stopped: false,
193
- chunks: [],
194
- };
195
- } else {
196
- // update existing stream data
197
- this.ttsStream.finished = finished;
198
- this.ttsStream.text = text;
199
- }
200
-
201
- // cleanup text
202
- const cleanText = this.cleanText(text);
203
- if (!cleanText.trim()) return;
204
-
205
- // chunk it for faster processing
206
- this.ttsStream.chunks = this.chunkText(cleanText);
207
- if (this.ttsStream.chunks.length == 0) return;
208
-
209
- // if stream was already running, just updating chunks is enough
210
- // The running loop will pick up the new chunks automatically
211
- if (this.ttsStream.running) return;
212
- else this.ttsStream.running = true; // proceed to running phase
213
-
214
- // terminator function to kill the stream if new stream has started
215
- const terminator = () =>
216
- this.ttsStream?.id !== id || this.ttsStream?.stopped;
217
-
218
- const spoken = [];
219
-
220
- // continuously loop until all chunks are spoken and stream is finished
221
- while (true) {
222
- // check if we should stop
223
- if (terminator()) break;
224
-
225
- // get the next chunk index to speak
226
- const nextIndex = this.ttsStream.lastChunkIndex + 1;
227
-
228
- // if no more chunks available, check if we should wait or exit
229
- if (nextIndex >= this.ttsStream.chunks.length) {
230
- // if stream is finished, we're done
231
- if (this.ttsStream.finished) break;
232
- // otherwise wait a bit for more chunks to arrive
233
- await new Promise((resolve) => setTimeout(resolve, 50));
234
- continue;
235
- }
236
-
237
- // do not speak the last chunk until finished (it is being generated)
238
- if (
239
- nextIndex == this.ttsStream.chunks.length - 1 &&
240
- !this.ttsStream.finished
241
- ) {
242
- // wait a bit for more content or finish signal
243
- await new Promise((resolve) => setTimeout(resolve, 50));
244
- continue;
245
- }
246
-
247
- // set the index of last spoken chunk
248
- this.ttsStream.lastChunkIndex = nextIndex;
249
-
250
- // speak the chunk
251
- const chunk = this.ttsStream.chunks[nextIndex];
252
- spoken.push(chunk);
253
- await this._speak(chunk, nextIndex > 0, () => terminator());
254
- }
255
-
256
- // at the end, finish stream data
257
- this.ttsStream.running = false;
258
- },
259
-
260
- // simplified speak function, speak a single finished piece of text
261
- async speak(text) {
262
- const id = Math.random();
263
- return await this.speakStream(id, text, true);
264
- },
265
-
266
- // speak wrapper
267
- async _speak(text, waitForPrevious, terminator) {
268
- // default browser speech
269
- if (!this.tts_kokoro)
270
- return await this.speakWithBrowser(text, waitForPrevious, terminator);
271
-
272
- // kokoro tts
273
- try {
274
- await await this.speakWithKokoro(text, waitForPrevious, terminator);
275
- } catch (error) {
276
- console.error(error);
277
- return await this.speakWithBrowser(text, waitForPrevious, terminator);
278
- }
279
- },
280
-
281
- chunkText(text, { maxChunkLength = 135, lineSeparator = "..." } = {}) {
282
- const INC_LIMIT = maxChunkLength * 2;
283
- const MIN_CHUNK_LENGTH = 20; // minimum length for a chunk before merging
284
-
285
- // Only split by ,/word if needed (unchanged)
286
- const splitDeep = (seg) => {
287
- if (seg.length <= INC_LIMIT) return [seg];
288
- const byComma = seg.match(/[^,]+(?:,|$)/g);
289
- if (byComma.length > 1)
290
- return byComma.flatMap((p, i) =>
291
- splitDeep(i < byComma.length - 1 ? p : p.replace(/,$/, ""))
292
- );
293
- const out = [];
294
- let part = "";
295
- for (const word of seg.split(/\s+/)) {
296
- const need = part ? part.length + 1 + word.length : word.length;
297
- if (need <= maxChunkLength) {
298
- part += (part ? " " : "") + word;
299
- } else {
300
- if (part) out.push(part);
301
- if (word.length > maxChunkLength) {
302
- for (let i = 0; i < word.length; i += maxChunkLength)
303
- out.push(word.slice(i, i + maxChunkLength));
304
- part = "";
305
- } else {
306
- part = word;
307
- }
308
- }
309
- }
310
- if (part) out.push(part);
311
- return out;
312
- };
313
-
314
- // Only split on [.!?] followed by space
315
- const sentenceTokens = (line) => {
316
- const toks = [];
317
- let start = 0;
318
- for (let i = 0; i < line.length; i++) {
319
- const c = line[i];
320
- if (
321
- (c === "." || c === "!" || c === "?") &&
322
- /\s/.test(line[i + 1] || "")
323
- ) {
324
- toks.push(line.slice(start, i + 1));
325
- i += 1;
326
- start = i + 1;
327
- }
328
- }
329
- if (start < line.length) toks.push(line.slice(start));
330
- return toks;
331
- };
332
-
333
- // Step 1: Split all newlines into individual chunks first
334
- let initialChunks = [];
335
- const lines = text.split(/\n+/).filter((l) => l.trim());
336
-
337
- for (const line of lines) {
338
- if (!line.trim()) continue;
339
- // Process each line into sentence tokens and add to chunks
340
- const sentences = sentenceTokens(line.trim());
341
- initialChunks.push(...sentences);
342
- }
343
-
344
- // Step 2: Merge short chunks until they meet minimum length criteria
345
- const finalChunks = [];
346
- let currentChunk = "";
347
-
348
- for (let i = 0; i < initialChunks.length; i++) {
349
- const chunk = initialChunks[i];
350
-
351
- // If current chunk is empty, start with this chunk
352
- if (!currentChunk) {
353
- currentChunk = chunk;
354
- // If this is the last chunk or it's already long enough, add it
355
- if (
356
- i === initialChunks.length - 1 ||
357
- currentChunk.length >= MIN_CHUNK_LENGTH
358
- ) {
359
- finalChunks.push(currentChunk);
360
- currentChunk = "";
361
- }
362
- continue;
363
- }
364
-
365
- // Current chunk exists, check if we should merge
366
- if (currentChunk.length < MIN_CHUNK_LENGTH) {
367
- // Try to merge with separator
368
- const merged = currentChunk + " " + lineSeparator + " " + chunk;
369
-
370
- // Check if merged chunk fits within max length
371
- if (merged.length <= maxChunkLength) {
372
- currentChunk = merged;
373
- } else {
374
- // Doesn't fit, add current chunk and start new one
375
- finalChunks.push(currentChunk);
376
- currentChunk = chunk;
377
- }
378
- } else {
379
- // Current chunk is already long enough, add it and start new one
380
- finalChunks.push(currentChunk);
381
- currentChunk = chunk;
382
- }
383
-
384
- // If this is the last chunk, add whatever is in the buffer
385
- if (i === initialChunks.length - 1 && currentChunk) {
386
- finalChunks.push(currentChunk);
387
- }
388
- }
389
-
390
- return finalChunks.map((chunk) => chunk.trimEnd());
391
- },
392
-
393
- // Show a prompt to user to enable audio
394
- showAudioPermissionPrompt() {
395
- shortcuts.frontendNotification({
396
- type: "info",
397
- message: "Click anywhere to enable audio playback",
398
- displayTime: 5000,
399
- frontendOnly: true,
400
- });
401
- console.log("Please click anywhere on the page to enable audio playback");
402
- },
403
-
404
- // Browser TTS
405
- async speakWithBrowser(text, waitForPrevious = false, terminator = null) {
406
- // wait for previous to finish if requested
407
- while (waitForPrevious && this.isSpeaking) await sleep(25);
408
- if (terminator && terminator()) return;
409
-
410
- // stop previous only if not waiting for it
411
- if (!waitForPrevious) this.stopAudio();
412
-
413
- this.browserUtterance = new SpeechSynthesisUtterance(text);
414
- this.browserUtterance.onstart = () => {
415
- this.isSpeaking = true;
416
- };
417
- this.browserUtterance.onend = () => {
418
- this.isSpeaking = false;
419
- };
420
-
421
- this.synth.speak(this.browserUtterance);
422
- },
423
-
424
- // Kokoro TTS
425
- async speakWithKokoro(text, waitForPrevious = false, terminator = null) {
426
- try {
427
- // synthesize on the backend
428
- const response = await sendJsonData("/synthesize", { text });
429
-
430
- // wait for previous to finish if requested
431
- while (waitForPrevious && this.isSpeaking) await sleep(25);
432
- if (terminator && terminator()) return;
433
-
434
- // stop previous only if not waiting for it
435
- if (!waitForPrevious) this.stopAudio();
436
-
437
- if (response.success) {
438
- if (response.audio_parts) {
439
- // Multiple chunks - play sequentially
440
- for (const audioPart of response.audio_parts) {
441
- if (terminator && terminator()) return;
442
- await this.playAudio(audioPart);
443
- await sleep(100); // Brief pause
444
- }
445
- } else if (response.audio) {
446
- // Single audio
447
- this.playAudio(response.audio);
448
- }
449
- } else {
450
- throw new Error("Kokoro TTS error:", response.error);
451
- }
452
- } catch (error) {
453
- throw new Error("Kokoro TTS error:", error);
454
- }
455
- },
456
-
457
- // Play base64 audio
458
- async playAudio(base64Audio) {
459
- return new Promise((resolve, reject) => {
460
- const audio = this.audioEl ? this.audioEl : (this.audioEl = new Audio());
461
-
462
- // Reset any previous playback state
463
- audio.pause();
464
- audio.currentTime = 0;
465
-
466
- audio.onplay = () => {
467
- this.isSpeaking = true;
468
- };
469
- audio.onended = () => {
470
- this.isSpeaking = false;
471
- this.currentAudio = null;
472
- resolve();
473
- };
474
- audio.onerror = (error) => {
475
- this.isSpeaking = false;
476
- this.currentAudio = null;
477
- reject(error);
478
- };
479
-
480
- audio.src = `data:audio/wav;base64,${base64Audio}`;
481
- this.currentAudio = audio;
482
-
483
- audio.play().catch((error) => {
484
- this.isSpeaking = false;
485
- this.currentAudio = null;
486
-
487
- if (error.name === "NotAllowedError") {
488
- this.showAudioPermissionPrompt();
489
- this.userHasInteracted = false;
490
- }
491
- reject(error);
492
- });
493
- });
494
- },
495
-
496
- // Stop current speech chain
497
- stop() {
498
- this.stopAudio(); // stop current audio immediately
499
- if (this.ttsStream) this.ttsStream.stopped = true; // set stop on current stream
500
- },
501
-
502
- // Stop current speech audio
503
- stopAudio() {
504
- if (this.synth?.speaking) {
505
- this.synth.cancel();
506
- }
507
-
508
- if (this.audioEl) {
509
- this.audioEl.pause();
510
- this.audioEl.currentTime = 0;
511
- }
512
- this.currentAudio = null;
513
- this.isSpeaking = false;
514
- },
515
-
516
- // Clean text for TTS
517
- cleanText(text) {
518
- // Use SUB character (ASCII 26, 0x1A) for placeholders to avoid conflicts with actual text
519
- const SUB = "\x1A"; // non-printable substitute character
520
- const codePlaceholder = SUB + "code" + SUB;
521
- const tablePlaceholder = SUB + "table" + SUB;
522
-
523
- // Handle code blocks BEFORE HTML parsing (markdown code blocks)
524
- text = text.replace(/```(?:[a-zA-Z0-9]*\n)?[\s\S]*?```/g, codePlaceholder); // closed code blocks
525
- text = text.replace(/```(?:[a-zA-Z0-9]*\n)?[\s\S]*$/g, codePlaceholder); // unclosed code blocks
526
-
527
- // Replace inline code ticks with content preserved
528
- text = text.replace(/`([^`]*)`/g, "$1"); // remove backticks but keep content
529
-
530
- // Parse HTML using browser's DOMParser to properly extract text content
531
- try {
532
- const parser = new DOMParser();
533
- // Wrap in a div to handle fragments
534
- const doc = parser.parseFromString(`<div>${text}</div>`, 'text/html');
535
-
536
- // Replace <pre> and <code> tags with placeholder before extracting text
537
- doc.querySelectorAll('pre, code').forEach(el => {
538
- el.textContent = codePlaceholder;
539
- });
540
-
541
- // Extract text content (this strips all HTML tags properly)
542
- text = doc.body.textContent || "";
543
- } catch (e) {
544
- // Fallback: simple tag stripping if DOMParser fails
545
- console.warn("[Speech Store] DOMParser failed, using fallback:", e);
546
- text = text.replace(/<pre[^>]*>[\s\S]*?<\/pre>/gi, codePlaceholder);
547
- text = text.replace(/<code[^>]*>[\s\S]*?<\/code>/gi, codePlaceholder);
548
- text = text.replace(/<[^>]+>/g, ''); // strip remaining tags
549
- }
550
-
551
- // Remove markdown links: [label](url) → label
552
- text = text.replace(/\[([^\]]+)\]\([^\)]+\)/g, "$1");
553
-
554
- // Remove markdown formatting: *, _, #
555
- text = text.replace(/[*_#]+/g, "");
556
-
557
- // Handle tables - both complete and partial
558
- // Check if text contains a table-like pattern
559
- if (text.includes("|")) {
560
- // Find consecutive lines with | characters (table rows)
561
- const tableLines = text
562
- .split("\n")
563
- .filter((line) => line.includes("|") && line.trim().startsWith("|"));
564
- if (tableLines.length > 0) {
565
- // Replace each table line with a placeholder
566
- for (const line of tableLines) {
567
- text = text.replace(line, tablePlaceholder);
568
- }
569
- } else {
570
- // Just handle individual table rows
571
- text = text.replace(/\|[^\n]*\|/g, tablePlaceholder);
572
- }
573
- }
574
-
575
- // Remove emojis and private unicode blocks
576
- text = text.replace(
577
- /([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g,
578
- ""
579
- );
580
-
581
- // Replace URLs with just the domain name
582
- text = text.replace(/https?:\/\/[^\s]+/g, (match) => {
583
- try {
584
- return new URL(match).hostname;
585
- } catch {
586
- return "";
587
- }
588
- });
589
-
590
- // Remove email addresses
591
- // text = text.replace(/\S+@\S+/g, "");
592
-
593
- // Replace UUIDs with 'UUID'
594
- text = text.replace(
595
- /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g,
596
- "UUID"
597
- );
598
-
599
- // Collapse multiple spaces/tabs to a single space, but preserve newlines
600
- text = text.replace(/[ \t]+/g, " ");
601
-
602
- // Function to merge consecutive placeholders of any type
603
- function mergePlaceholders(txt, placeholder, replacement) {
604
- // Create regex for consecutive placeholders (with possible whitespace between)
605
- const regex = new RegExp(placeholder + "\\s*" + placeholder, "g");
606
- // Merge consecutive placeholders until no more found
607
- while (regex.test(txt)) {
608
- txt = txt.replace(regex, placeholder);
609
- }
610
- // Replace all remaining placeholders with human-readable text
611
- return txt.replace(new RegExp(placeholder, "g"), replacement);
612
- }
613
-
614
- // Apply placeholder merging for both types
615
- text = mergePlaceholders(text, codePlaceholder, "See code attached ...");
616
- text = mergePlaceholders(text, tablePlaceholder, "See table attached ...");
617
-
618
- // Trim leading/trailing whitespace
619
- text = text.trim();
620
-
621
- return text;
622
- },
623
-
624
- // Initialize microphone input
625
- async initMicrophone() {
626
- if (this.microphoneInput) return this.microphoneInput;
627
-
628
- this.microphoneInput = new MicrophoneInput(async (text, isFinal) => {
629
- if (isFinal) {
630
- this.sendMessage(text);
631
- }
632
- });
633
-
634
- const initialized = await this.microphoneInput.initialize();
635
- return initialized ? this.microphoneInput : null;
636
- },
637
-
638
- async sendMessage(text) {
639
- text = "(voice) " + text;
640
- updateChatInput(text);
641
- if (!this.microphoneInput.messageSent) {
642
- this.microphoneInput.messageSent = true;
643
- await sendMessage();
644
- }
645
- },
646
-
647
- // Request microphone permission - delegate to MicrophoneInput
648
- async requestMicrophonePermission() {
649
- return this.microphoneInput
650
- ? this.microphoneInput.requestPermission()
651
- : MicrophoneInput.prototype.requestPermission.call(null);
652
- },
653
-};
654
-
655
-// Microphone Input Class (simplified for store integration)
656
-class MicrophoneInput {
657
- constructor(updateCallback) {
658
- this.mediaRecorder = null;
659
- this.audioChunks = [];
660
- this.lastChunk = [];
661
- this.updateCallback = updateCallback;
662
- this.messageSent = false;
663
- this.audioContext = null;
664
- this.mediaStreamSource = null;
665
- this.analyserNode = null;
666
- this._status = Status.INACTIVE;
667
- this.lastAudioTime = null;
668
- this.waitingTimer = null;
669
- this.silenceStartTime = null;
670
- this.hasStartedRecording = false;
671
- this.analysisFrame = null;
672
- }
673
-
674
- get status() {
675
- return this._status;
676
- }
677
-
678
- set status(newStatus) {
679
- if (this._status === newStatus) return;
680
-
681
- const oldStatus = this._status;
682
- this._status = newStatus;
683
- console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);
684
-
685
- this.handleStatusChange(oldStatus, newStatus);
686
- }
687
-
688
- async initialize() {
689
- // Set status to activating at the start of initialization
690
- this.status = Status.ACTIVATING;
691
- try {
692
- // get selected device from microphone settings
693
- const selectedDevice = microphoneSettingStore.getSelectedDevice();
694
-
695
- const stream = await navigator.mediaDevices.getUserMedia({
696
- audio: {
697
- deviceId:
698
- selectedDevice && selectedDevice.deviceId
699
- ? { exact: selectedDevice.deviceId }
700
- : undefined,
701
- echoCancellation: true,
702
- noiseSuppression: true,
703
- channelCount: 1,
704
- },
705
- });
706
-
707
- this.mediaRecorder = new MediaRecorder(stream);
708
- this.mediaRecorder.ondataavailable = (event) => {
709
- if (
710
- event.data.size > 0 &&
711
- (this.status === Status.RECORDING || this.status === Status.WAITING)
712
- ) {
713
- if (this.lastChunk) {
714
- this.audioChunks.push(this.lastChunk);
715
- this.lastChunk = null;
716
- }
717
- this.audioChunks.push(event.data);
718
- } else if (this.status === Status.LISTENING) {
719
- this.lastChunk = event.data;
720
- }
721
- };
722
-
723
- this.setupAudioAnalysis(stream);
724
- return true;
725
- } catch (error) {
726
- console.error("Microphone initialization error:", error);
727
- toast("Failed to access microphone. Please check permissions.", "error");
728
- return false;
729
- }
730
- }
731
-
732
- handleStatusChange(oldStatus, newStatus) {
733
- if (newStatus != Status.RECORDING) {
734
- this.lastChunk = null;
735
- }
736
-
737
- switch (newStatus) {
738
- case Status.INACTIVE:
739
- this.handleInactiveState();
740
- break;
741
- case Status.LISTENING:
742
- this.handleListeningState();
743
- break;
744
- case Status.RECORDING:
745
- this.handleRecordingState();
746
- break;
747
- case Status.WAITING:
748
- this.handleWaitingState();
749
- break;
750
- case Status.PROCESSING:
751
- this.handleProcessingState();
752
- break;
753
- }
754
- }
755
-
756
- handleInactiveState() {
757
- this.stopRecording();
758
- this.stopAudioAnalysis();
759
- if (this.waitingTimer) {
760
- clearTimeout(this.waitingTimer);
761
- this.waitingTimer = null;
762
- }
763
- }
764
-
765
- handleListeningState() {
766
- this.stopRecording();
767
- this.audioChunks = [];
768
- this.hasStartedRecording = false;
769
- this.silenceStartTime = null;
770
- this.lastAudioTime = null;
771
- this.messageSent = false;
772
- this.startAudioAnalysis();
773
- }
774
-
775
- handleRecordingState() {
776
- if (!this.hasStartedRecording && this.mediaRecorder.state !== "recording") {
777
- this.hasStartedRecording = true;
778
- this.mediaRecorder.start(1000);
779
- console.log("Speech started");
780
- }
781
- if (this.waitingTimer) {
782
- clearTimeout(this.waitingTimer);
783
- this.waitingTimer = null;
784
- }
785
- }
786
-
787
- handleWaitingState() {
788
- this.waitingTimer = setTimeout(() => {
789
- if (this.status === Status.WAITING) {
790
- this.status = Status.PROCESSING;
791
- }
792
- }, store.stt_waiting_timeout);
793
- }
794
-
795
- handleProcessingState() {
796
- this.stopRecording();
797
- this.process();
798
- }
799
-
800
- setupAudioAnalysis(stream) {
801
- this.audioContext = new (window.AudioContext ||
802
- window.webkitAudioContext)();
803
- this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
804
- this.analyserNode = this.audioContext.createAnalyser();
805
- this.analyserNode.fftSize = 2048;
806
- this.analyserNode.minDecibels = -90;
807
- this.analyserNode.maxDecibels = -10;
808
- this.analyserNode.smoothingTimeConstant = 0.85;
809
- this.mediaStreamSource.connect(this.analyserNode);
810
- }
811
-
812
- startAudioAnalysis() {
813
- const analyzeFrame = () => {
814
- if (this.status === Status.INACTIVE) return;
815
-
816
- const dataArray = new Uint8Array(this.analyserNode.fftSize);
817
- this.analyserNode.getByteTimeDomainData(dataArray);
818
-
819
- let sum = 0;
820
- for (let i = 0; i < dataArray.length; i++) {
821
- const amplitude = (dataArray[i] - 128) / 128;
822
- sum += amplitude * amplitude;
823
- }
824
- const rms = Math.sqrt(sum / dataArray.length);
825
- const now = Date.now();
826
-
827
- // Update status based on audio level (ignore if TTS is speaking)
828
- if (rms > this.densify(store.stt_silence_threshold)) {
829
- this.lastAudioTime = now;
830
- this.silenceStartTime = null;
831
-
832
- if (
833
- (this.status === Status.LISTENING ||
834
- this.status === Status.WAITING) &&
835
- !store.isSpeaking
836
- ) {
837
- this.status = Status.RECORDING;
838
- }
839
- } else if (this.status === Status.RECORDING) {
840
- if (!this.silenceStartTime) {
841
- this.silenceStartTime = now;
842
- }
843
-
844
- const silenceDuration = now - this.silenceStartTime;
845
- if (silenceDuration >= store.stt_silence_duration) {
846
- this.status = Status.WAITING;
847
- }
848
- }
849
-
850
- this.analysisFrame = requestAnimationFrame(analyzeFrame);
851
- };
852
-
853
- this.analysisFrame = requestAnimationFrame(analyzeFrame);
854
- }
855
-
856
- stopAudioAnalysis() {
857
- if (this.analysisFrame) {
858
- cancelAnimationFrame(this.analysisFrame);
859
- this.analysisFrame = null;
860
- }
861
- }
862
-
863
- stopRecording() {
864
- if (this.mediaRecorder?.state === "recording") {
865
- this.mediaRecorder.stop();
866
- this.hasStartedRecording = false;
867
- }
868
- }
869
-
870
- densify(x) {
871
- return Math.exp(-5 * (1 - x));
872
- }
873
-
874
- async process() {
875
- if (this.audioChunks.length === 0) {
876
- this.status = Status.LISTENING;
877
- return;
878
- }
879
-
880
- const audioBlob = new Blob(this.audioChunks, { type: "audio/wav" });
881
- const base64 = await this.convertBlobToBase64Wav(audioBlob);
882
-
883
- try {
884
- const result = await sendJsonData("/transcribe", { audio: base64 });
885
- const text = this.filterResult(result.text || "");
886
-
887
- if (text) {
888
- console.log("Transcription:", result.text);
889
- await this.updateCallback(result.text, true);
890
- }
891
- } catch (error) {
892
- window.toastFetchError("Transcription error", error);
893
- console.error("Transcription error:", error);
894
- } finally {
895
- this.audioChunks = [];
896
- this.status = Status.LISTENING;
897
- }
898
- }
899
-
900
- convertBlobToBase64Wav(audioBlob) {
901
- return new Promise((resolve, reject) => {
902
- const reader = new FileReader();
903
- reader.onloadend = () => {
904
- const base64Data = reader.result.split(",")[1];
905
- resolve(base64Data);
906
- };
907
- reader.onerror = (error) => reject(error);
908
- reader.readAsDataURL(audioBlob);
909
- });
910
- }
911
-
912
- filterResult(text) {
913
- text = text.trim();
914
- let ok = false;
915
- while (!ok) {
916
- if (!text) break;
917
- if (text[0] === "{" && text[text.length - 1] === "}") break;
918
- if (text[0] === "(" && text[text.length - 1] === ")") break;
919
- if (text[0] === "[" && text[text.length - 1] === "]") break;
920
- ok = true;
921
- }
922
- if (ok) return text;
923
- else console.log(`Discarding transcription: ${text}`);
924
- }
925
-
926
- // Toggle microphone between active and inactive states
927
- async toggle() {
928
- const hasPermission = await this.requestPermission();
929
- if (!hasPermission) return;
930
-
931
- // Toggle between listening and inactive
932
- if (this.status === Status.INACTIVE || this.status === Status.ACTIVATING) {
933
- this.status = Status.LISTENING;
934
- } else {
935
- this.status = Status.INACTIVE;
936
- }
937
- }
938
-
939
- // Request microphone permission
940
- async requestPermission() {
941
- try {
942
- await navigator.mediaDevices.getUserMedia({ audio: true });
943
- return true;
944
- } catch (err) {
945
- console.error("Error accessing microphone:", err);
946
- toast(
947
- "Microphone access denied. Please enable microphone access in your browser settings.",
948
- "error"
949
- );
950
- return false;
951
- }
952
- }
953
-}
954
-
955
-export const store = createStore("speech", model);
956
-
957
-// Initialize speech store
958
-// window.speechStore = speechStore;
959
-
960
-// Event listeners
961
-document.addEventListener("settings-updated", () => store.loadSettings());
962
-// document.addEventListener("DOMContentLoaded", () => speechStore.init());
webui/components/settings/agent/agent-settings.html
+5
-5
@@ -22,9 +22,9 @@
22
</a>
23
</li>
24
<li>
25
- <a href="#section-speech">
26
- <img src="/public/speech.svg" alt="Speech" />
27
- <span>Speech</span>
25
+ <a href="#section-voice">
26
+ <img src="/public/speech.svg" alt="Voice" />
27
+ <span>Voice</span>
28
</a>
29
</li>
30
<li>
@@ -44,8 +44,8 @@
44
<x-component path="/plugins/_model_config/webui/models-summary.html"></x-component>
45
</div>
46
47
- <div id="section-speech" class="section">
48
- <x-component path="settings/agent/speech.html"></x-component>
47
+ <div id="section-voice" class="section">
48
+ <x-component path="settings/agent/voice.html"></x-component>
49
</div>
50
51
<div id="section-workdir" class="section">
webui/components/settings/agent/speech.html
deleted
-108
@@ -1,108 +0,0 @@
1
-<html>
2
- <head>
3
- <title>Speech</title>
4
- </head>
5
-
6
- <body>
7
- <div x-data>
8
- <template x-if="$store.settings.settings">
9
- <div>
10
- <div class="section-title">Speech</div>
11
- <div class="section-description">
12
- Pick the microphone, transcription model, and voice output behavior.
13
- </div>
14
-
15
- <div class="field field-full">
16
- <div class="field-label">
17
- <div class="field-title">Microphone device</div>
18
- <div class="field-description">Choose the input device Agent Zero listens to.</div>
19
- </div>
20
- <div class="field-control">
21
- <x-component path="settings/speech/microphone.html"></x-component>
22
- </div>
23
- </div>
24
-
25
- <div class="field">
26
- <div class="field-label">
27
- <div class="field-title">Speech-to-text model size</div>
28
- <div class="field-description">Larger models can hear more accurately, but need more time and memory.</div>
29
- </div>
30
- <div class="field-control">
31
- <select x-model="$store.settings.settings.stt_model_size">
32
- <template x-for="option in $store.settings.additional?.stt_models" :key="option.value">
33
- <option :value="option.value" :selected="option.value === $store.settings.settings.stt_model_size" x-text="option.label"></option>
34
- </template>
35
- </select>
36
- </div>
37
- </div>
38
-
39
- <div class="field">
40
- <div class="field-label">
41
- <div class="field-title">Enable Kokoro TTS</div>
42
- <div class="field-description">
43
- Use higher-quality server-side speech instead of the browser voice.
44
- </div>
45
- </div>
46
- <div class="field-control">
47
- <label class="toggle">
48
- <input type="checkbox" x-model="$store.settings.settings.tts_kokoro" />
49
- <span class="toggler"></span>
50
- </label>
51
- </div>
52
- </div>
53
-
54
- <div class="settings-advanced-section" x-data="{ advOpen: false }">
55
- <button type="button" class="settings-advanced-toggle" @click="advOpen = !advOpen">
56
- <span class="material-symbols-outlined settings-advanced-toggle-icon"
57
- :style="advOpen ? 'transform:rotate(90deg)' : ''">chevron_right</span>
58
- <span>Advanced Settings</span>
59
- </button>
60
-
61
- <div class="settings-advanced-body" x-show="advOpen" x-transition.opacity>
62
- <div class="field">
63
- <div class="field-label">
64
- <div class="field-title">Language hint</div>
65
- <div class="field-description">Use a short language code such as en, fr, or it when automatic detection needs guidance.</div>
66
- </div>
67
- <div class="field-control">
68
- <input type="text" x-model="$store.settings.settings.stt_language" />
69
- </div>
70
- </div>
71
-
72
- <div class="field">
73
- <div class="field-label">
74
- <div class="field-title">Silence threshold</div>
75
- <div class="field-description">Lower values catch softer speech; higher values ignore more room noise.</div>
76
- </div>
77
- <div class="field-control">
78
- <input type="range" min="0" max="1" step="0.01" x-model.number="$store.settings.settings.stt_silence_threshold" />
79
- <span class="range-value" x-text="$store.settings.settings.stt_silence_threshold"></span>
80
- </div>
81
- </div>
82
-
83
- <div class="field">
84
- <div class="field-label">
85
- <div class="field-title">End-of-speech delay</div>
86
- <div class="field-description">How long silence must last before Agent Zero treats your sentence as complete.</div>
87
- </div>
88
- <div class="field-control">
89
- <input type="number" x-model.number="$store.settings.settings.stt_silence_duration" />
90
- </div>
91
- </div>
92
-
93
- <div class="field">
94
- <div class="field-label">
95
- <div class="field-title">Microphone close delay</div>
96
- <div class="field-description">How long Agent Zero waits before closing the microphone after speech stops.</div>
97
- </div>
98
- <div class="field-control">
99
- <input type="number" x-model.number="$store.settings.settings.stt_waiting_timeout" />
100
- </div>
101
- </div>
102
- </div>
103
- </div>
104
- </div>
105
- </template>
106
- </div>
107
- </body>
108
-</html>
webui/components/settings/agent/voice.html
new
+35
@@ -0,0 +1,35 @@
1
+<html>
2
+ <head>
3
+ <title>Voice</title>
4
+ </head>
5
+
6
+ <body>
7
+ <div x-data>
8
+ <div class="section-title">Voice</div>
9
+ <div class="section-description">
10
+ Voice capabilities are provided by built-in plugins. Browser-native
11
+ speech remains available as the fallback output path when no TTS plugin
12
+ is active. Enable or disable providers from the Agent Plugins section
13
+ below.
14
+ </div>
15
+
16
+ <x-extension id="voice-settings-start"></x-extension>
17
+ <div class="voice-settings-grid">
18
+ <x-extension id="voice-settings-main"></x-extension>
19
+ </div>
20
+ <x-extension id="voice-settings-end"></x-extension>
21
+ </div>
22
+
23
+ <style>
24
+ .voice-settings-grid {
25
+ display: grid;
26
+ gap: 12px;
27
+ grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
28
+ }
29
+
30
+ .voice-settings-grid > x-extension {
31
+ display: contents;
32
+ }
33
+ </style>
34
+ </body>
35
+</html>
webui/components/settings/speech/microphone-setting-store.js
deleted
-88
@@ -1,88 +0,0 @@
1
-import { createStore } from "/js/AlpineStore.js";
2
-
3
-const model = {
4
-
5
-
6
- devices: [],
7
- selectedDevice: "",
8
-
9
- async init() {
10
- // Load selected device from localStorage if present
11
- const saved = localStorage.getItem('microphoneSelectedDevice');
12
- await this.loadDevices();
13
- if (saved && this.devices.some(d => d.deviceId === saved)) {
14
- this.selectedDevice = saved;
15
- }
16
- },
17
-
18
- async loadDevices() {
19
- // Get media devices
20
- const devices = await navigator.mediaDevices.enumerateDevices();
21
- // Filter for audio input (microphones)
22
- this.devices = devices.filter(d => d.kind === "audioinput" && d.deviceId);
23
- // Set selected device to first available, if any
24
- this.selectedDevice = this.devices.length > 0 ? this.devices[0].deviceId : "";
25
- },
26
-
27
- // track permission request state
28
- requestingPermission: false,
29
- permissionTimer: null,
30
- permissionAttempts: 0,
31
-
32
- // request microphone permission and poll for devices
33
- async requestPermission() {
34
- // set flag first so UI can update immediately
35
- clearTimeout(this.permissionTimer);
36
- this.requestingPermission = true;
37
- this.permissionAttempts = 0;
38
-
39
- // request permission in next tick to allow UI to update
40
- setTimeout(async () => {
41
- try {
42
- await navigator.mediaDevices.getUserMedia({ audio: true });
43
- // start polling for devices
44
- this.pollForDevices();
45
- } catch (err) {
46
- console.error("Microphone permission denied");
47
- this.requestingPermission = false;
48
- }
49
- }, 0);
50
- },
51
-
52
- // poll for devices until found or timeout (60s)
53
- async pollForDevices() {
54
- await this.loadDevices();
55
-
56
- // check if we found devices with valid IDs
57
- if (this.devices.some(d => d.deviceId && d.deviceId !== "") || this.permissionAttempts >= 60) {
58
- this.requestingPermission = false;
59
- return;
60
- }
61
-
62
- // continue polling
63
- this.permissionAttempts++;
64
- this.permissionTimer = setTimeout(() => this.pollForDevices(), 1000);
65
- },
66
-
67
- async selectDevice(deviceId) {
68
- this.selectedDevice = deviceId;
69
- this.onSelectDevice();
70
- },
71
-
72
- async onSelectDevice() {
73
- localStorage.setItem('microphoneSelectedDevice', this.selectedDevice);
74
- },
75
-
76
- getSelectedDevice() {
77
- let device = this.devices.find(d => d.deviceId === this.selectedDevice);
78
- if (!device && this.devices.length > 0) {
79
- device = this.devices.find(d => d.deviceId === "default") || this.devices[0];
80
- }
81
- return device;
82
- }
83
-
84
-};
85
-
86
-const store = createStore("microphoneSetting", model);
87
-
88
-export { store };
webui/components/settings/speech/microphone.html
deleted
-47
@@ -1,47 +0,0 @@
1
-<html>
2
-
3
-<head>
4
- <title>Microhone settings</title>
5
-
6
- <!-- Import the alpine store -->
7
- <script type="module">
8
- import { store } from "/components/settings/speech/microphone-setting-store.js";
9
- console.log("microphone-setting-store.js loaded");
10
- </script>
11
-</head>
12
-
13
-<body>
14
-
15
- <!-- This construct of x-data + x-if is used to ensure the component is only rendered when the store is available -->
16
- <div x-data>
17
- <template x-if="$store.microphoneSetting">
18
-
19
- <div>
20
- <select x-model="$store.microphoneSetting.selectedDevice"
21
- @change="$store.microphoneSetting.onSelectDevice()"
22
- x-show="$store.microphoneSetting.devices.length > 0">
23
- <template x-for="option in $store.microphoneSetting.devices" :key="option.deviceId">
24
- <option :value="option.deviceId" x-text="option.label"
25
- :selected="option.deviceId === $store.microphoneSetting.selectedDevice"></option>
26
- </template>
27
- </select>
28
- <button class="btn btn-field"
29
- x-show="$store.microphoneSetting.devices.length == 0 && !$store.microphoneSetting.requestingPermission"
30
- @click="$store.microphoneSetting.requestPermission()">Request permission to select device</button>
31
- <button class="btn btn-field"
32
- x-show="$store.microphoneSetting.requestingPermission"
33
- @click="$store.microphoneSetting.requestPermission()">
34
- <span>Waiting for devices... [retry]</span>
35
- </button>
36
- </div>
37
-
38
- </template>
39
- </div>
40
-
41
- <!-- Optional style for the component -->
42
- <style>
43
- </style>
44
-
45
-</body>
46
-
47
-</html>
\ No newline at end of file
webui/components/sidebar/bottom/preferences/preferences-store.js
+2
-2
@@ -1,6 +1,6 @@
1
import { createStore } from "/js/AlpineStore.js";
2
import * as css from "/js/css.js";
3
-import { store as speechStore } from "/components/chat/speech/speech-store.js";
3
+import { ttsService } from "/js/tts-service.js";
4
import { applyModeSteps } from "/components/messages/process-group/process-group-dom.js";
5
6
// Preferences store centralizes user preference toggles and side-effects
@@ -153,7 +153,7 @@ const model = {
153
154
_applySpeech(value) {
155
localStorage.setItem("speech", value);
156
- if (!value) speechStore.stopAudio();
156
+ if (!value) ttsService.stop();
157
},
158
159
webui/css/speech.css
deleted
-64
@@ -1,64 +0,0 @@
1
-/* MIC BUTTON */
2
-
3
-/* Only apply hover effects on devices that support hover */
4
-@media (hover: hover) {
5
- #microphone-button:hover {
6
- background-color: #636363;
7
- box-shadow: 0 0 0 1px rgba(255, 255, 255, 0.08),
8
- 0 6px 14px rgba(0, 0, 0, 0.18);
9
- }
10
-}
11
-
12
-#microphone-button:active {
13
- background-color: #444444;
14
- box-shadow: inset 0 0 0 1px rgba(0, 0, 0, 0.12);
15
-}
16
-
17
-#microphone-button.recording {
18
- background-color: #ff4136; /* Red color for recording */
19
- transition: background-color 0.3s ease;
20
-}
21
-
22
-@keyframes pulse {
23
- 0% {
24
- transform: scale(1);
25
- }
26
- 50% {
27
- transform: scale(1.1);
28
- }
29
- 100% {
30
- transform: scale(1);
31
- }
32
- }
33
-
34
-.mic-pulse {
35
- animation: pulse 1.5s infinite;
36
-}
37
-
38
-
39
-.mic-inactive{
40
- background-color: grey;
41
-}
42
-
43
-.mic-activating{
44
- background-color: silver;
45
- animation: pulse 0.8s infinite;
46
-}
47
-
48
-.mic-listening {
49
- background-color: red;
50
-}
51
-
52
-.mic-recording {
53
- background-color: green;
54
-}
55
-
56
-.mic-waiting {
57
- background-color: teal;
58
-}
59
-
60
-.mic-processing {
61
- background-color: darkcyan;
62
- animation: pulse 0.8s infinite;
63
- transform-origin: center;
64
-}
\ No newline at end of file
webui/index.html
-1
@@ -14,7 +14,6 @@
14
<link rel="stylesheet" href="css/settings.css">
15
<link rel="stylesheet" href="css/modals.css">
16
<link rel="stylesheet" href="css/surfaces.css">
17
- <link rel="stylesheet" href="css/speech.css">
17
<link rel="stylesheet" href="css/scheduler-datepicker.css">
18
<link rel="stylesheet" href="css/scheduler.css">
19
<link rel="stylesheet" href="css/notification.css">
webui/index.js
+4
-4
@@ -3,8 +3,8 @@ import * as api from "/js/api.js";
3
import { callJsExtensions } from "/js/extensions.js";
4
import * as css from "/js/css.js";
5
import { sleep } from "/js/sleep.js";
6
+import { ttsService } from "/js/tts-service.js";
7
import { store as attachmentsStore } from "/components/chat/attachments/attachmentsStore.js";
7
-import { store as speechStore } from "/components/chat/speech/speech-store.js";
8
import { store as notificationStore } from "/components/notifications/notification-store.js";
9
import { store as preferencesStore } from "/components/sidebar/bottom/preferences/preferences-store.js";
10
import { store as inputStore } from "/components/chat/input/input-store.js";
@@ -458,7 +458,7 @@ function speakMessages(logs) {
458
// finished response
459
if (log.type == "response") {
460
// lastSpokenNo = log.no;
461
- speechStore.speakStream(
461
+ ttsService.speakStream(
462
getChatBasedId(log.no),
463
log.content,
464
log.kvps?.finished
@@ -474,7 +474,7 @@ function speakMessages(logs) {
474
log.kvps.tool_name != "response"
475
) {
476
// lastSpokenNo = log.no;
477
- speechStore.speakStream(getChatBasedId(log.no), log.kvps.headline, true);
477
+ ttsService.speakStream(getChatBasedId(log.no), log.kvps.headline, true);
478
return;
479
}
480
}
@@ -550,7 +550,7 @@ export const setContext = function (id) {
550
lastSpokenNo = 0;
551
552
// Stop speech when switching chats
553
- speechStore.stopAudio();
553
+ ttsService.stop();
554
555
// Clear the chat history immediately to avoid showing stale content
556
const chatHistoryEl = document.getElementById("chat-history");
webui/js/messages.js
+13
-13
@@ -3,7 +3,7 @@ import { store as imageViewerStore } from "../components/modals/image-viewer/ima
3
import { marked } from "../vendor/marked/marked.esm.js";
4
import { store as _messageResizeStore } from "/components/messages/resize/message-resize-store.js"; // keep here, required in html
5
import { store as attachmentsStore } from "/components/chat/attachments/attachmentsStore.js";
6
-import { store as speechStore } from "/components/chat/speech/speech-store.js";
6
+import { ttsService } from "/js/tts-service.js";
7
import {
8
createActionButton,
9
copyToClipboard,
@@ -784,7 +784,7 @@ export function drawMessageDefault({
784
const contentText = String(content ?? "");
785
const actionButtons = contentText.trim()
786
? [
787
- createActionButton("speak", "", () => speechStore.speak(contentText)),
787
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
788
createActionButton("copy", "", () => copyToClipboard(contentText)),
789
].filter(Boolean)
790
: [];
@@ -837,7 +837,7 @@ export function drawMessageAgent({
837
838
if (thoughtsText.trim()) {
839
actionButtons.push(
840
- createActionButton("speak", "", () => speechStore.speak(thoughtsText)),
840
+ createActionButton("speak", "", () => ttsService.speak(thoughtsText)),
841
);
842
actionButtons.push(
843
createActionButton("copy", "", () => copyToClipboard(thoughtsText)),
@@ -875,7 +875,7 @@ export function drawMessageResponse({
875
const contentText = String(content ?? "");
876
const actionButtons = contentText.trim()
877
? [
878
- createActionButton("speak", "", () => speechStore.speak(contentText)),
878
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
879
createActionButton("copy", "", () => copyToClipboard(contentText)),
880
].filter(Boolean)
881
: [];
@@ -942,7 +942,7 @@ export function drawMessageResponse({
942
const responseText = String(content ?? "");
943
const responseActionButtons = responseText.trim()
944
? [
945
- createActionButton("speak", "", () => speechStore.speak(responseText)),
945
+ createActionButton("speak", "", () => ttsService.speak(responseText)),
946
createActionButton("copy", "", () => copyToClipboard(responseText)),
947
].filter(Boolean)
948
: [];
@@ -1085,7 +1085,7 @@ export function drawMessageUser({
1085
const userText = String(content ?? "");
1086
const userActionButtons = userText.trim()
1087
? [
1088
- createActionButton("speak", "", () => speechStore.speak(userText)),
1088
+ createActionButton("speak", "", () => ttsService.speak(userText)),
1089
createActionButton("copy", "", () => copyToClipboard(userText)),
1090
].filter(Boolean)
1091
: [];
@@ -1175,7 +1175,7 @@ export function drawMessageToolSimple({
1175
buildDetailPayload(arguments[0], { headerLabels }),
1176
),
1177
),
1178
- createActionButton("speak", "", () => speechStore.speak(contentText)),
1178
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
1179
createActionButton("copy", "", () => copyToClipboard(contentText)),
1180
].filter(Boolean)
1181
: [];
@@ -1220,7 +1220,7 @@ export function drawMessageMcp({
1220
buildDetailPayload(arguments[0], { headerLabels }),
1221
),
1222
),
1223
- createActionButton("speak", "", () => speechStore.speak(contentText)),
1223
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
1224
createActionButton("copy", "", () => copyToClipboard(contentText)),
1225
].filter(Boolean)
1226
: [];
@@ -1265,7 +1265,7 @@ export function drawMessageSubagent({
1265
buildDetailPayload(arguments[0], { headerLabels }),
1266
),
1267
),
1268
- createActionButton("speak", "", () => speechStore.speak(contentText)),
1268
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
1269
createActionButton("copy", "", () => copyToClipboard(contentText)),
1270
].filter(Boolean)
1271
: [];
@@ -1299,7 +1299,7 @@ export function drawMessageInfo({
1299
const contentText = String(content ?? "");
1300
const actionButtons = contentText.trim()
1301
? [
1302
- createActionButton("speak", "", () => speechStore.speak(contentText)),
1302
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
1303
createActionButton("copy", "", () => copyToClipboard(contentText)),
1304
].filter(Boolean)
1305
: [];
@@ -1335,7 +1335,7 @@ export function drawMessageUtil({
1335
const contentText = String(content ?? "");
1336
const actionButtons = contentText.trim()
1337
? [
1338
- createActionButton("speak", "", () => speechStore.speak(contentText)),
1338
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
1339
createActionButton("copy", "", () => copyToClipboard(contentText)),
1340
].filter(Boolean)
1341
: [];
@@ -1374,7 +1374,7 @@ export function drawMessageHint({
1374
const contentText = String(content ?? "");
1375
const actionButtons = contentText.trim()
1376
? [
1377
- createActionButton("speak", "", () => speechStore.speak(contentText)),
1377
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
1378
createActionButton("copy", "", () => copyToClipboard(contentText)),
1379
].filter(Boolean)
1380
: [];
@@ -1442,7 +1442,7 @@ export function drawMessageWarning({
1442
const contentText = String(content ?? "");
1443
const actionButtons = contentText.trim()
1444
? [
1445
- createActionButton("speak", "", () => speechStore.speak(contentText)),
1445
+ createActionButton("speak", "", () => ttsService.speak(contentText)),
1446
createActionButton("copy", "", () => copyToClipboard(contentText)),
1447
].filter(Boolean)
1448
: [];
webui/js/speech_browser.js
deleted
-394
@@ -1,394 +0,0 @@
1
-import { pipeline, read_audio } from './transformers@3.0.2.js';
2
-import { updateChatInput, sendMessage } from '../index.js';
3
-
4
-const microphoneButton = document.getElementById('microphone-button');
5
-let microphoneInput = null;
6
-let isProcessingClick = false;
7
-
8
-const Status = {
9
- INACTIVE: 'inactive',
10
- ACTIVATING: 'activating',
11
- LISTENING: 'listening',
12
- RECORDING: 'recording',
13
- WAITING: 'waiting',
14
- PROCESSING: 'processing'
15
-};
16
-
17
-class MicrophoneInput {
18
- constructor(updateCallback, options = {}) {
19
- this.mediaRecorder = null;
20
- this.audioChunks = [];
21
- this.lastChunk = [];
22
- this.updateCallback = updateCallback;
23
- this.messageSent = false;
24
-
25
- // Audio analysis properties
26
- this.audioContext = null;
27
- this.mediaStreamSource = null;
28
- this.analyserNode = null;
29
- this._status = Status.INACTIVE;
30
-
31
- // Timing properties
32
- this.lastAudioTime = null;
33
- this.waitingTimer = null;
34
- this.silenceStartTime = null;
35
- this.hasStartedRecording = false;
36
- this.analysisFrame = null;
37
-
38
- this.options = {
39
- modelSize: 'tiny',
40
- language: 'en',
41
- silenceThreshold: 0.15,
42
- silenceDuration: 1000,
43
- waitingTimeout: 2000,
44
- minSpeechDuration: 500,
45
- ...options
46
- };
47
- }
48
-
49
- get status() {
50
- return this._status;
51
- }
52
-
53
- set status(newStatus) {
54
- if (this._status === newStatus) return;
55
-
56
- const oldStatus = this._status;
57
- this._status = newStatus;
58
- console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);
59
-
60
- // Update UI
61
- microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`);
62
- microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`);
63
- microphoneButton.setAttribute('data-status', newStatus);
64
-
65
- // Handle state-specific behaviors
66
- this.handleStatusChange(oldStatus, newStatus);
67
- }
68
-
69
- handleStatusChange(oldStatus, newStatus) {
70
-
71
- //last chunk kept only for transition to recording status
72
- if (newStatus != Status.RECORDING) { this.lastChunk = null; }
73
-
74
- switch (newStatus) {
75
- case Status.INACTIVE:
76
- this.handleInactiveState();
77
- break;
78
- case Status.LISTENING:
79
- this.handleListeningState();
80
- break;
81
- case Status.RECORDING:
82
- this.handleRecordingState();
83
- break;
84
- case Status.WAITING:
85
- this.handleWaitingState();
86
- break;
87
- case Status.PROCESSING:
88
- this.handleProcessingState();
89
- break;
90
- }
91
- }
92
-
93
- handleInactiveState() {
94
- this.stopRecording();
95
- this.stopAudioAnalysis();
96
- if (this.waitingTimer) {
97
- clearTimeout(this.waitingTimer);
98
- this.waitingTimer = null;
99
- }
100
- }
101
-
102
- handleListeningState() {
103
- this.stopRecording();
104
- this.audioChunks = [];
105
- this.hasStartedRecording = false;
106
- this.silenceStartTime = null;
107
- this.lastAudioTime = null;
108
- this.messageSent = false;
109
- this.startAudioAnalysis();
110
- }
111
-
112
- handleRecordingState() {
113
- if (!this.hasStartedRecording && this.mediaRecorder.state !== 'recording') {
114
- this.hasStartedRecording = true;
115
- this.mediaRecorder.start(1000);
116
- console.log('Speech started');
117
- }
118
- if (this.waitingTimer) {
119
- clearTimeout(this.waitingTimer);
120
- this.waitingTimer = null;
121
- }
122
- }
123
-
124
- handleWaitingState() {
125
- // Don't stop recording during waiting state
126
- this.waitingTimer = setTimeout(() => {
127
- if (this.status === Status.WAITING) {
128
- this.status = Status.PROCESSING;
129
- }
130
- }, this.options.waitingTimeout);
131
- }
132
-
133
- handleProcessingState() {
134
- this.stopRecording();
135
- this.process();
136
- }
137
-
138
- stopRecording() {
139
- if (this.mediaRecorder?.state === 'recording') {
140
- this.mediaRecorder.stop();
141
- this.hasStartedRecording = false;
142
- }
143
- }
144
-
145
- async initialize() {
146
- try {
147
- this.transcriber = await pipeline(
148
- 'automatic-speech-recognition',
149
- `Xenova/whisper-${this.options.modelSize}.${this.options.language}`
150
- );
151
-
152
- const stream = await navigator.mediaDevices.getUserMedia({
153
- audio: {
154
- echoCancellation: true,
155
- noiseSuppression: true,
156
- channelCount: 1
157
- }
158
- });
159
-
160
- this.mediaRecorder = new MediaRecorder(stream);
161
- this.mediaRecorder.ondataavailable = (event) => {
162
- if (event.data.size > 0 &&
163
- (this.status === Status.RECORDING || this.status === Status.WAITING)) {
164
- if (this.lastChunk) {
165
- this.audioChunks.push(this.lastChunk);
166
- this.lastChunk = null;
167
- }
168
- this.audioChunks.push(event.data);
169
- console.log('Audio chunk received, total chunks:', this.audioChunks.length);
170
- }
171
- else if (this.status === Status.LISTENING) {
172
- this.lastChunk = event.data;
173
- }
174
- };
175
-
176
- this.setupAudioAnalysis(stream);
177
- return true;
178
- } catch (error) {
179
-
180
- console.error('Microphone initialization error:', error);
181
- window.toastFrontendError('Failed to access microphone. Please check permissions.', 'Microphone Error');
182
- return false;
183
- }
184
- }
185
-
186
- setupAudioAnalysis(stream) {
187
- this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
188
- this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
189
- this.analyserNode = this.audioContext.createAnalyser();
190
- this.analyserNode.fftSize = 2048;
191
- this.analyserNode.minDecibels = -90;
192
- this.analyserNode.maxDecibels = -10;
193
- this.analyserNode.smoothingTimeConstant = 0.85;
194
- this.mediaStreamSource.connect(this.analyserNode);
195
- }
196
-
197
-
198
- startAudioAnalysis() {
199
- const analyzeFrame = () => {
200
- if (this.status === Status.INACTIVE) return;
201
-
202
- const dataArray = new Uint8Array(this.analyserNode.fftSize);
203
- this.analyserNode.getByteTimeDomainData(dataArray);
204
-
205
- // Calculate RMS volume
206
- let sum = 0;
207
- for (let i = 0; i < dataArray.length; i++) {
208
- const amplitude = (dataArray[i] - 128) / 128;
209
- sum += amplitude * amplitude;
210
- }
211
- const rms = Math.sqrt(sum / dataArray.length);
212
-
213
- const now = Date.now();
214
-
215
- // Update status based on audio level
216
- if (rms > this.options.silenceThreshold) {
217
- this.lastAudioTime = now;
218
- this.silenceStartTime = null;
219
-
220
- if (this.status === Status.LISTENING || this.status === Status.WAITING) {
221
- if (!speech.isSpeaking()) // TODO? a better way to ignore agent's voice?
222
- this.status = Status.RECORDING;
223
- }
224
- } else if (this.status === Status.RECORDING) {
225
- if (!this.silenceStartTime) {
226
- this.silenceStartTime = now;
227
- }
228
-
229
- const silenceDuration = now - this.silenceStartTime;
230
- if (silenceDuration >= this.options.silenceDuration) {
231
- this.status = Status.WAITING;
232
- }
233
- }
234
-
235
- this.analysisFrame = requestAnimationFrame(analyzeFrame);
236
- };
237
-
238
- this.analysisFrame = requestAnimationFrame(analyzeFrame);
239
- }
240
-
241
- stopAudioAnalysis() {
242
- if (this.analysisFrame) {
243
- cancelAnimationFrame(this.analysisFrame);
244
- this.analysisFrame = null;
245
- }
246
- }
247
-
248
- async process() {
249
- if (this.audioChunks.length === 0) {
250
- this.status = Status.LISTENING;
251
- return;
252
- }
253
-
254
- const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
255
- const audioUrl = URL.createObjectURL(audioBlob);
256
-
257
-
258
-
259
- try {
260
- const samplingRate = 16000;
261
- const audioData = await read_audio(audioUrl, samplingRate);
262
- const result = await this.transcriber(audioData);
263
- const text = this.filterResult(result.text || "")
264
-
265
- if (text) {
266
- console.log('Transcription:', result.text);
267
- await this.updateCallback(result.text, true);
268
- }
269
- } catch (error) {
270
- console.error('Transcription error:', error);
271
- window.toastFrontendError('Transcription failed.', 'Speech Recognition Error');
272
- } finally {
273
- URL.revokeObjectURL(audioUrl);
274
- this.audioChunks = [];
275
- this.status = Status.LISTENING;
276
- }
277
- }
278
-
279
- filterResult(text) {
280
- text = text.trim()
281
- let ok = false
282
- while (!ok) {
283
- if (!text) break
284
- if (text[0] === '{' && text[text.length - 1] === '}') break
285
- if (text[0] === '(' && text[text.length - 1] === ')') break
286
- if (text[0] === '[' && text[text.length - 1] === ']') break
287
- ok = true
288
- }
289
- if (ok) return text
290
- else console.log(`Discarding transcription: ${text}`)
291
- }
292
-}
293
-
294
-
295
-
296
-// Initialize and handle click events
297
-async function initializeMicrophoneInput() {
298
- microphoneInput = new MicrophoneInput(
299
- async (text, isFinal) => {
300
- if (isFinal) {
301
- updateChatInput(text);
302
- if (!microphoneInput.messageSent) {
303
- microphoneInput.messageSent = true;
304
- await sendMessage();
305
- }
306
- }
307
- },
308
- {
309
- modelSize: 'tiny',
310
- language: 'en',
311
- silenceThreshold: 0.07,
312
- silenceDuration: 1000,
313
- waitingTimeout: 1500
314
- }
315
- );
316
- microphoneInput.status = Status.ACTIVATING;
317
-
318
- return await microphoneInput.initialize();
319
-}
320
-
321
-microphoneButton.addEventListener('click', async () => {
322
- if (isProcessingClick) return;
323
- isProcessingClick = true;
324
-
325
- const hasPermission = await requestMicrophonePermission();
326
- if (!hasPermission) return;
327
-
328
- try {
329
- if (!microphoneInput && !await initializeMicrophoneInput()) {
330
- return;
331
- }
332
-
333
- // Simply toggle between INACTIVE and LISTENING states
334
- microphoneInput.status =
335
- (microphoneInput.status === Status.INACTIVE || microphoneInput.status === Status.ACTIVATING) ? Status.LISTENING : Status.INACTIVE;
336
- } finally {
337
- setTimeout(() => {
338
- isProcessingClick = false;
339
- }, 300);
340
- }
341
-});
342
-
343
-// Some error handling for microphone input
344
-async function requestMicrophonePermission() {
345
- try {
346
- await navigator.mediaDevices.getUserMedia({ audio: true });
347
- return true;
348
- } catch (err) {
349
- console.error('Error accessing microphone:', err);
350
- window.toastFrontendError('Microphone access denied. Please enable microphone access in your browser settings.', 'Microphone Error');
351
- return false;
352
- }
353
-}
354
-
355
-
356
-class Speech {
357
- constructor() {
358
- this.synth = window.speechSynthesis;
359
- this.utterance = null;
360
- }
361
-
362
- stripEmojis(str) {
363
- return str
364
- .replace(/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g, '')
365
- .replace(/\s+/g, ' ')
366
- .trim();
367
- }
368
-
369
- speak(text) {
370
- console.log('Speaking:', text);
371
- // Stop any current utterance
372
- this.stop();
373
-
374
- // Remove emojis and create a new utterance
375
- text = this.stripEmojis(text);
376
- this.utterance = new SpeechSynthesisUtterance(text);
377
-
378
- // Speak the new utterance
379
- this.synth.speak(this.utterance);
380
- }
381
-
382
- stop() {
383
- if (this.isSpeaking()) {
384
- this.synth.cancel();
385
- }
386
- }
387
-
388
- isSpeaking() {
389
- return this.synth?.speaking || false;
390
- }
391
-}
392
-
393
-export const speech = new Speech();
394
-window.speech = speech
webui/js/stt-service.js
new
+87
@@ -0,0 +1,87 @@
1
+class SttService extends EventTarget {
2
+ constructor() {
3
+ super();
4
+ this.providers = new Map();
5
+ }
6
+
7
+ registerProvider(id, provider) {
8
+ if (!id || !provider) {
9
+ throw new Error("STT providers must define an id and provider object.");
10
+ }
11
+
12
+ this.providers.set(id, provider);
13
+ this.emitProvidersChange();
14
+
15
+ return () => this.unregisterProvider(id);
16
+ }
17
+
18
+ unregisterProvider(id) {
19
+ if (!this.providers.has(id)) return;
20
+ const activeProviderId = this.getActiveProviderId();
21
+ if (activeProviderId === id) {
22
+ this.stop();
23
+ this.emitStatusChange("inactive");
24
+ }
25
+ this.providers.delete(id);
26
+ this.emitProvidersChange();
27
+ }
28
+
29
+ getActiveProviderId() {
30
+ const next = this.providers.keys().next();
31
+ return next.done ? "" : String(next.value || "");
32
+ }
33
+
34
+ getActiveProvider() {
35
+ const providerId = this.getActiveProviderId();
36
+ return providerId ? this.providers.get(providerId) || null : null;
37
+ }
38
+
39
+ hasProvider() {
40
+ return !!this.getActiveProvider();
41
+ }
42
+
43
+ emitProvidersChange() {
44
+ this.dispatchEvent(
45
+ new CustomEvent("providerschange", {
46
+ detail: {
47
+ activeProviderId: this.getActiveProviderId(),
48
+ providerIds: Array.from(this.providers.keys()),
49
+ },
50
+ }),
51
+ );
52
+ }
53
+
54
+ emitStatusChange(status) {
55
+ this.dispatchEvent(
56
+ new CustomEvent("statuschange", {
57
+ detail: {
58
+ activeProviderId: this.getActiveProviderId(),
59
+ status,
60
+ },
61
+ }),
62
+ );
63
+ }
64
+
65
+ async handleMicrophoneClick() {
66
+ return await this.getActiveProvider()?.handleMicrophoneClick?.();
67
+ }
68
+
69
+ async requestMicrophonePermission() {
70
+ return await this.getActiveProvider()?.requestMicrophonePermission?.();
71
+ }
72
+
73
+ updateMicrophoneButtonUI() {
74
+ this.getActiveProvider()?.updateMicrophoneButtonUI?.();
75
+ }
76
+
77
+ stop() {
78
+ this.getActiveProvider()?.stop?.();
79
+ }
80
+
81
+ getStatus() {
82
+ return this.getActiveProvider()?.getStatus?.() || "inactive";
83
+ }
84
+}
85
+
86
+export const sttService = new SttService();
87
+globalThis.sttService = sttService;
webui/js/tts-service.js
new
+532
@@ -0,0 +1,532 @@
1
+import { sleep } from "/js/sleep.js";
2
+import * as shortcuts from "/js/shortcuts.js";
3
+
4
+class TtsService extends EventTarget {
5
+ constructor() {
6
+ super();
7
+ this.providers = new Map();
8
+ this.synth = window.speechSynthesis;
9
+ this.browserUtterance = null;
10
+ this.audioEl = null;
11
+ this.currentAudio = null;
12
+ this.audioContext = null;
13
+ this.userHasInteracted = false;
14
+ this.ttsStream = null;
15
+ this._isSpeaking = false;
16
+
17
+ this.setupUserInteractionHandling();
18
+ }
19
+
20
+ registerProvider(id, provider) {
21
+ if (!id || !provider || typeof provider.synthesize !== "function") {
22
+ throw new Error("TTS providers must define an id and synthesize(text).");
23
+ }
24
+
25
+ this.providers.set(id, provider);
26
+ this.emitProvidersChange();
27
+
28
+ return () => this.unregisterProvider(id);
29
+ }
30
+
31
+ unregisterProvider(id) {
32
+ if (!this.providers.has(id)) return;
33
+
34
+ const activeProviderId = this.getActiveProviderId();
35
+ this.providers.delete(id);
36
+
37
+ if (activeProviderId === id) {
38
+ this.stop();
39
+ }
40
+
41
+ this.emitProvidersChange();
42
+ }
43
+
44
+ getActiveProviderId() {
45
+ const next = this.providers.keys().next();
46
+ return next.done ? "" : String(next.value || "");
47
+ }
48
+
49
+ getActiveProvider() {
50
+ const providerId = this.getActiveProviderId();
51
+ return providerId ? this.providers.get(providerId) || null : null;
52
+ }
53
+
54
+ hasProvider() {
55
+ return !!this.getActiveProvider();
56
+ }
57
+
58
+ isSpeaking() {
59
+ return this._isSpeaking;
60
+ }
61
+
62
+ getState() {
63
+ return {
64
+ activeProviderId: this.getActiveProviderId(),
65
+ isSpeaking: this.isSpeaking(),
66
+ userHasInteracted: this.userHasInteracted,
67
+ };
68
+ }
69
+
70
+ emitProvidersChange() {
71
+ this.dispatchEvent(
72
+ new CustomEvent("providerschange", {
73
+ detail: {
74
+ activeProviderId: this.getActiveProviderId(),
75
+ providerIds: Array.from(this.providers.keys()),
76
+ },
77
+ }),
78
+ );
79
+ this.emitStateChange();
80
+ }
81
+
82
+ emitStateChange() {
83
+ this.dispatchEvent(
84
+ new CustomEvent("statechange", {
85
+ detail: this.getState(),
86
+ }),
87
+ );
88
+ }
89
+
90
+ setSpeaking(value) {
91
+ const next = !!value;
92
+ if (this._isSpeaking === next) return;
93
+ this._isSpeaking = next;
94
+ this.emitStateChange();
95
+ }
96
+
97
+ setupUserInteractionHandling() {
98
+ const enableAudio = () => {
99
+ if (this.userHasInteracted) return;
100
+
101
+ this.userHasInteracted = true;
102
+ try {
103
+ this.audioContext = new (window.AudioContext ||
104
+ window.webkitAudioContext)();
105
+ this.audioContext.resume();
106
+ } catch (_error) {
107
+ // AudioContext is unavailable in some browsers/modes.
108
+ }
109
+
110
+ this.emitStateChange();
111
+ };
112
+
113
+ const events = ["click", "touchstart", "keydown", "mousedown"];
114
+ events.forEach((eventName) => {
115
+ document.addEventListener(eventName, enableAudio, {
116
+ once: true,
117
+ passive: true,
118
+ });
119
+ });
120
+ }
121
+
122
+ showAudioPermissionPrompt() {
123
+ shortcuts.frontendNotification({
124
+ type: "info",
125
+ message: "Click anywhere to enable audio playback",
126
+ displayTime: 5000,
127
+ frontendOnly: true,
128
+ });
129
+ }
130
+
131
+ async speak(text) {
132
+ const id = Math.random();
133
+ return await this.speakStream(id, text, true);
134
+ }
135
+
136
+ async speakStream(id, text, finished = false) {
137
+ if (
138
+ this.ttsStream &&
139
+ this.ttsStream.id === id &&
140
+ this.ttsStream.text === text &&
141
+ this.ttsStream.finished === finished
142
+ ) {
143
+ return;
144
+ }
145
+
146
+ if (!this.userHasInteracted) {
147
+ this.showAudioPermissionPrompt();
148
+ return;
149
+ }
150
+
151
+ if (!this.ttsStream || this.ttsStream.id !== id) {
152
+ this.ttsStream = {
153
+ id,
154
+ text,
155
+ finished,
156
+ running: false,
157
+ lastChunkIndex: -1,
158
+ stopped: false,
159
+ chunks: [],
160
+ };
161
+ } else {
162
+ this.ttsStream.finished = finished;
163
+ this.ttsStream.text = text;
164
+ }
165
+
166
+ const cleanText = this.cleanText(text);
167
+ if (!cleanText.trim()) return;
168
+
169
+ this.ttsStream.chunks = this.chunkText(cleanText);
170
+ if (this.ttsStream.chunks.length === 0) return;
171
+
172
+ if (this.ttsStream.running) return;
173
+ this.ttsStream.running = true;
174
+
175
+ const terminator = () =>
176
+ this.ttsStream?.id !== id || this.ttsStream?.stopped;
177
+
178
+ while (true) {
179
+ if (terminator()) break;
180
+
181
+ const nextIndex = this.ttsStream.lastChunkIndex + 1;
182
+ if (nextIndex >= this.ttsStream.chunks.length) {
183
+ if (this.ttsStream.finished) break;
184
+ await new Promise((resolve) => setTimeout(resolve, 50));
185
+ continue;
186
+ }
187
+
188
+ if (
189
+ nextIndex === this.ttsStream.chunks.length - 1 &&
190
+ !this.ttsStream.finished
191
+ ) {
192
+ await new Promise((resolve) => setTimeout(resolve, 50));
193
+ continue;
194
+ }
195
+
196
+ this.ttsStream.lastChunkIndex = nextIndex;
197
+ const chunk = this.ttsStream.chunks[nextIndex];
198
+ await this.speakChunk(chunk, nextIndex > 0, terminator);
199
+ }
200
+
201
+ this.ttsStream.running = false;
202
+ }
203
+
204
+ async speakChunk(text, waitForPrevious = false, terminator = null) {
205
+ const provider = this.getActiveProvider();
206
+
207
+ if (provider) {
208
+ try {
209
+ return await this.speakWithProvider(
210
+ provider,
211
+ text,
212
+ waitForPrevious,
213
+ terminator,
214
+ );
215
+ } catch (error) {
216
+ console.error("TTS provider failed, falling back to browser TTS", error);
217
+ }
218
+ }
219
+
220
+ return await this.speakWithBrowser(text, waitForPrevious, terminator);
221
+ }
222
+
223
+ async speakWithProvider(provider, text, waitForPrevious = false, terminator = null) {
224
+ const payload = await provider.synthesize(text, {
225
+ providerId: this.getActiveProviderId(),
226
+ });
227
+
228
+ while (waitForPrevious && this.isSpeaking()) {
229
+ await sleep(25);
230
+ }
231
+ if (terminator && terminator()) return;
232
+
233
+ if (!waitForPrevious) {
234
+ this.stopAudio();
235
+ }
236
+
237
+ if (!payload) return;
238
+
239
+ if (Array.isArray(payload.audioParts)) {
240
+ for (const part of payload.audioParts) {
241
+ if (terminator && terminator()) return;
242
+ await this.playAudioBase64(part, payload.mimeType);
243
+ await sleep(100);
244
+ }
245
+ return;
246
+ }
247
+
248
+ const audioBase64 = payload.audioBase64 || payload.audio;
249
+ if (audioBase64) {
250
+ await this.playAudioBase64(audioBase64, payload.mimeType);
251
+ }
252
+ }
253
+
254
+ async speakWithBrowser(text, waitForPrevious = false, terminator = null) {
255
+ while (waitForPrevious && this.isSpeaking()) {
256
+ await sleep(25);
257
+ }
258
+ if (terminator && terminator()) return;
259
+
260
+ if (!waitForPrevious) {
261
+ this.stopAudio();
262
+ }
263
+
264
+ return await new Promise((resolve, reject) => {
265
+ const utterance = new SpeechSynthesisUtterance(text);
266
+ this.browserUtterance = utterance;
267
+
268
+ utterance.onstart = () => {
269
+ this.setSpeaking(true);
270
+ };
271
+ utterance.onend = () => {
272
+ if (this.browserUtterance === utterance) {
273
+ this.browserUtterance = null;
274
+ }
275
+ this.setSpeaking(false);
276
+ resolve();
277
+ };
278
+ utterance.onerror = (error) => {
279
+ if (this.browserUtterance === utterance) {
280
+ this.browserUtterance = null;
281
+ }
282
+ this.setSpeaking(false);
283
+ reject(error);
284
+ };
285
+
286
+ this.synth.speak(utterance);
287
+ });
288
+ }
289
+
290
+ async playAudioBase64(base64Audio, mimeType = "audio/wav") {
291
+ return await new Promise((resolve, reject) => {
292
+ const audio = this.audioEl ? this.audioEl : (this.audioEl = new Audio());
293
+
294
+ audio.pause();
295
+ audio.currentTime = 0;
296
+
297
+ audio.onplay = () => {
298
+ this.setSpeaking(true);
299
+ };
300
+ audio.onended = () => {
301
+ this.setSpeaking(false);
302
+ this.currentAudio = null;
303
+ resolve();
304
+ };
305
+ audio.onerror = (error) => {
306
+ this.setSpeaking(false);
307
+ this.currentAudio = null;
308
+ reject(error);
309
+ };
310
+
311
+ audio.src = `data:${mimeType};base64,${base64Audio}`;
312
+ this.currentAudio = audio;
313
+
314
+ audio.play().catch((error) => {
315
+ this.setSpeaking(false);
316
+ this.currentAudio = null;
317
+ if (error?.name === "NotAllowedError") {
318
+ this.showAudioPermissionPrompt();
319
+ this.userHasInteracted = false;
320
+ this.emitStateChange();
321
+ }
322
+ reject(error);
323
+ });
324
+ });
325
+ }
326
+
327
+ stop() {
328
+ this.stopAudio();
329
+ if (this.ttsStream) {
330
+ this.ttsStream.stopped = true;
331
+ }
332
+
333
+ const provider = this.getActiveProvider();
334
+ try {
335
+ provider?.stop?.();
336
+ } catch (error) {
337
+ console.error("Failed to stop TTS provider cleanly", error);
338
+ }
339
+ }
340
+
341
+ stopAudio() {
342
+ if (this.synth?.speaking) {
343
+ this.synth.cancel();
344
+ }
345
+
346
+ if (this.audioEl) {
347
+ this.audioEl.pause();
348
+ this.audioEl.currentTime = 0;
349
+ }
350
+
351
+ this.currentAudio = null;
352
+ this.setSpeaking(false);
353
+ }
354
+
355
+ chunkText(text, { maxChunkLength = 135, lineSeparator = "..." } = {}) {
356
+ const INC_LIMIT = maxChunkLength * 2;
357
+ const MIN_CHUNK_LENGTH = 20;
358
+
359
+ const splitDeep = (segment) => {
360
+ if (segment.length <= INC_LIMIT) return [segment];
361
+ const byComma = segment.match(/[^,]+(?:,|$)/g);
362
+ if (byComma.length > 1) {
363
+ return byComma.flatMap((part, index) =>
364
+ splitDeep(
365
+ index < byComma.length - 1 ? part : part.replace(/,$/, ""),
366
+ ),
367
+ );
368
+ }
369
+
370
+ const out = [];
371
+ let part = "";
372
+ for (const word of segment.split(/\s+/)) {
373
+ const need = part ? part.length + 1 + word.length : word.length;
374
+ if (need <= maxChunkLength) {
375
+ part += (part ? " " : "") + word;
376
+ } else {
377
+ if (part) out.push(part);
378
+ if (word.length > maxChunkLength) {
379
+ for (let index = 0; index < word.length; index += maxChunkLength) {
380
+ out.push(word.slice(index, index + maxChunkLength));
381
+ }
382
+ part = "";
383
+ } else {
384
+ part = word;
385
+ }
386
+ }
387
+ }
388
+ if (part) out.push(part);
389
+ return out;
390
+ };
391
+
392
+ const sentenceTokens = (line) => {
393
+ const tokens = [];
394
+ let start = 0;
395
+ for (let index = 0; index < line.length; index++) {
396
+ const character = line[index];
397
+ if (
398
+ (character === "." || character === "!" || character === "?") &&
399
+ /\s/.test(line[index + 1] || "")
400
+ ) {
401
+ tokens.push(line.slice(start, index + 1));
402
+ index += 1;
403
+ start = index + 1;
404
+ }
405
+ }
406
+ if (start < line.length) {
407
+ tokens.push(line.slice(start));
408
+ }
409
+ return tokens.flatMap((token) => splitDeep(token.trim())).filter(Boolean);
410
+ };
411
+
412
+ const initialChunks = [];
413
+ const lines = text.split(/\n+/).filter((line) => line.trim());
414
+ for (const line of lines) {
415
+ initialChunks.push(...sentenceTokens(line.trim()));
416
+ }
417
+
418
+ const finalChunks = [];
419
+ let currentChunk = "";
420
+
421
+ for (let index = 0; index < initialChunks.length; index++) {
422
+ const chunk = initialChunks[index];
423
+ if (!currentChunk) {
424
+ currentChunk = chunk;
425
+ if (
426
+ index === initialChunks.length - 1 ||
427
+ currentChunk.length >= MIN_CHUNK_LENGTH
428
+ ) {
429
+ finalChunks.push(currentChunk);
430
+ currentChunk = "";
431
+ }
432
+ continue;
433
+ }
434
+
435
+ if (currentChunk.length < MIN_CHUNK_LENGTH) {
436
+ const merged = `${currentChunk} ${lineSeparator} ${chunk}`;
437
+ if (merged.length <= maxChunkLength) {
438
+ currentChunk = merged;
439
+ } else {
440
+ finalChunks.push(currentChunk);
441
+ currentChunk = chunk;
442
+ }
443
+ } else {
444
+ finalChunks.push(currentChunk);
445
+ currentChunk = chunk;
446
+ }
447
+
448
+ if (index === initialChunks.length - 1 && currentChunk) {
449
+ finalChunks.push(currentChunk);
450
+ }
451
+ }
452
+
453
+ return finalChunks.map((chunk) => chunk.trimEnd());
454
+ }
455
+
456
+ cleanText(text) {
457
+ const SUB = "\x1A";
458
+ const codePlaceholder = `${SUB}code${SUB}`;
459
+ const tablePlaceholder = `${SUB}table${SUB}`;
460
+
461
+ text = text.replace(
462
+ /```(?:[a-zA-Z0-9]*\n)?[\s\S]*?```/g,
463
+ codePlaceholder,
464
+ );
465
+ text = text.replace(/```(?:[a-zA-Z0-9]*\n)?[\s\S]*$/g, codePlaceholder);
466
+ text = text.replace(/`([^`]*)`/g, "$1");
467
+
468
+ try {
469
+ const parser = new DOMParser();
470
+ const doc = parser.parseFromString(`<div>${text}</div>`, "text/html");
471
+ doc.querySelectorAll("pre, code").forEach((element) => {
472
+ element.textContent = codePlaceholder;
473
+ });
474
+ text = doc.body.textContent || "";
475
+ } catch (_error) {
476
+ text = text.replace(/<pre[^>]*>[\s\S]*?<\/pre>/gi, codePlaceholder);
477
+ text = text.replace(/<code[^>]*>[\s\S]*?<\/code>/gi, codePlaceholder);
478
+ text = text.replace(/<[^>]+>/g, "");
479
+ }
480
+
481
+ text = text.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1");
482
+ text = text.replace(/[*_#]+/g, "");
483
+
484
+ if (text.includes("|")) {
485
+ const tableLines = text
486
+ .split("\n")
487
+ .filter((line) => line.includes("|") && line.trim().startsWith("|"));
488
+ if (tableLines.length > 0) {
489
+ for (const line of tableLines) {
490
+ text = text.replace(line, tablePlaceholder);
491
+ }
492
+ } else {
493
+ text = text.replace(/\|[^\n]*\|/g, tablePlaceholder);
494
+ }
495
+ }
496
+
497
+ text = text.replace(
498
+ /([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g,
499
+ "",
500
+ );
501
+
502
+ text = text.replace(/https?:\/\/[^\s]+/g, (match) => {
503
+ try {
504
+ return new URL(match).hostname;
505
+ } catch {
506
+ return "";
507
+ }
508
+ });
509
+
510
+ text = text.replace(
511
+ /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g,
512
+ "UUID",
513
+ );
514
+ text = text.replace(/[ \t]+/g, " ");
515
+
516
+ const mergePlaceholders = (value, placeholder, replacement) => {
517
+ const pattern = new RegExp(`${placeholder}\\s*${placeholder}`, "g");
518
+ while (pattern.test(value)) {
519
+ value = value.replace(pattern, placeholder);
520
+ }
521
+ return value.replace(new RegExp(placeholder, "g"), replacement);
522
+ };
523
+
524
+ text = mergePlaceholders(text, codePlaceholder, "See code attached ...");
525
+ text = mergePlaceholders(text, tablePlaceholder, "See table attached ...");
526
+
527
+ return text.trim();
528
+ }
529
+}
530
+
531
+export const ttsService = new TtsService();
532
+globalThis.ttsService = ttsService;