Alpine fix version, STT fixes
frdel committed
Nov 29, 2024 at 08:55 UTC
c99b1a47d4f25d8184661a77418ebfafa5c00ee9
5 files changed
+52
-64
python/helpers/settings.py
+5
-18
@@ -4,6 +4,7 @@ import re
4
from typing import Any, Literal, Optional, TypedDict
5
6
import models
7
+from python.helpers import whisper
8
from . import files, dotenv
9
from models import get_model, ModelProvider, ModelType
10
from langchain_core.language_models.chat_models import BaseChatModel
@@ -44,7 +45,6 @@ class Settings(TypedDict):
45
stt_silence_threshold: float
46
stt_silence_duration: int
47
stt_waiting_timeout: int
47
- stt_min_speech_duration: int
48
49
50
class PartialSettings(Settings, total=False):
@@ -474,14 +474,6 @@ def convert_out(settings: Settings) -> SettingsOutput:
474
"value": settings["stt_waiting_timeout"]
475
})
476
477
- stt_fields.append({
478
- "id": "stt_min_speech_duration",
479
- "title": "Prefix padding (ms)",
480
- "description": "Minimum duration of audio to be included in the stream before speech was recognized.",
481
- "type": "input",
482
- "value": settings["stt_min_speech_duration"]
483
- })
484
-
477
stt_section: SettingsSection = {
478
"title": "Speech to Text",
479
"description": "Voice transcription preferences and server turn detection settings.",
@@ -593,12 +585,6 @@ def get_embedding_model(settings: Settings | None = None) -> Embeddings:
585
**settings["embed_model_kwargs"],
586
)
587
596
-def get_speech_settings(settings: Settings | None = None) -> dict:
597
- if not settings:
598
- settings = get_settings()
599
- return settings["speech_to_text"]
600
-
601
-
588
def _read_settings_file() -> Settings | None:
589
if os.path.exists(SETTINGS_FILE):
590
content = files.read_file(SETTINGS_FILE)
@@ -653,12 +639,11 @@ def _get_default_settings() -> Settings:
639
agent_knowledge_subdir="custom",
640
rfc_url="http://localhost:55080",
641
rfc_password="",
656
- stt_model_size="tiny",
642
+ stt_model_size="base",
643
stt_language="en",
658
- stt_silence_threshold=0.15,
644
+ stt_silence_threshold=0.3,
645
stt_silence_duration=1000,
646
stt_waiting_timeout=2000,
661
- stt_min_speech_duration=500
647
)
648
649
@@ -676,6 +661,8 @@ def _apply_settings():
661
agent.config = ctx.config
662
agent = agent.get_data(agent.DATA_NAME_SUBORDINATE)
663
664
+ # reload whisper model if necessary
665
+ whisper.preload()
666
667
def _env_to_dict(data: str):
668
env_dict = {}
python/helpers/whisper.py
+7
-3
@@ -3,16 +3,20 @@ import base64
3
import warnings
4
import whisper
5
import tempfile
6
-from python.helpers import runtime, rfc
6
+from python.helpers import runtime, rfc, settings
7
8
# suppress FutureWarning from torch.load
9
warnings.filterwarnings('ignore', category=FutureWarning)
10
11
model = None
12
+model_name = ""
13
14
def preload():
14
- global model
15
- model = whisper.load_model("base")
15
+ global model, model_name
16
+ set = settings.get_settings()
17
+ if not model or model_name != set["stt_model_size"]:
18
+ model = whisper.load_model(set["stt_model_size"])
19
+ model_name = set["stt_model_size"]
20
return model
21
22
async def transcribe(audio_bytes_b64: str):
webui/index.html
+2
-2
@@ -19,8 +19,8 @@
19
}
20
</script>
21
22
- <script defer src="https://cdn.jsdelivr.net/npm/@alpinejs/collapse@3.x.x/dist/cdn.min.js"></script>
23
- <script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.x.x/dist/cdn.min.js"></script>
22
+ <script defer src="https://cdn.jsdelivr.net/npm/@alpinejs/collapse@3.14.3/dist/cdn.min.js"></script>
23
+ <script defer src="https://cdn.jsdelivr.net/npm/alpinejs@3.14.3/dist/cdn.min.js"></script>
24
25
<script src="https://cdn.jsdelivr.net/npm/ace-builds@1.36.5/src-noconflict/ace.js"></script>
26
<link href="https://cdn.jsdelivr.net/npm/ace-builds@1.36.5/css/ace.min.css" rel="stylesheet">
webui/js/settings.js
+1
-1
@@ -44,7 +44,7 @@ const settingsModalProxy = {
44
const modalEl = document.getElementById('settingsModal');
45
const modalAD = Alpine.$data(modalEl);
46
resp = await window.sendJsonData("/settings_set", modalAD.settings);
47
-
47
+ document.dispatchEvent(new CustomEvent('settings-updated', { detail: resp.settings }));
48
this.resolvePromise({
49
status: 'saved',
50
data: resp.settings
webui/js/speech.js
+37
-40
@@ -14,6 +14,38 @@ const Status = {
14
PROCESSING: 'processing'
15
};
16
17
+const micSettings = {
18
+ stt_model_size: 'tiny',
19
+ stt_language: 'en',
20
+ stt_silence_threshold: 0.05,
21
+ stt_silence_duration: 1000,
22
+ stt_waiting_timeout: 2000,
23
+};
24
+window.micSettings = micSettings
25
+loadMicSettings()
26
+
27
+function densify(x) {
28
+ return Math.exp(-5 * (1 - x));
29
+}
30
+
31
+async function loadMicSettings() {
32
+ try {
33
+ const response = await fetch('/settings_get');
34
+ const data = await response.json();
35
+ const sttSettings = data.settings.sections.find(s => s.title === 'Speech to Text');
36
+
37
+ if (sttSettings) {
38
+ // Update options from server settings
39
+ sttSettings.fields.forEach(field => {
40
+ const key = field.id //.split('.')[1]; // speech_to_text.model_size -> model_size
41
+ micSettings[key] = field.value;
42
+ });
43
+ }
44
+ } catch (error) {
45
+ console.error('Failed to load speech settings:', error);
46
+ }
47
+}
48
+
49
class MicrophoneInput {
50
constructor(updateCallback, options = {}) {
51
this.mediaRecorder = null;
@@ -34,37 +66,6 @@ class MicrophoneInput {
66
this.silenceStartTime = null;
67
this.hasStartedRecording = false;
68
this.analysisFrame = null;
37
-
38
- // Initialize with defaults
39
- this.options = {
40
- modelSize: 'tiny',
41
- language: 'en',
42
- silenceThreshold: 0.15,
43
- silenceDuration: 1000,
44
- waitingTimeout: 2000,
45
- minSpeechDuration: 500
46
- };
47
-
48
- // Fetch settings from server
49
- this.loadSettings();
50
- }
51
-
52
- async loadSettings() {
53
- try {
54
- const response = await fetch('/settings_get');
55
- const data = await response.json();
56
- const sttSettings = data.settings.sections.find(s => s.title === 'Speech to Text');
57
-
58
- if (sttSettings) {
59
- // Update options from server settings
60
- sttSettings.fields.forEach(field => {
61
- const key = field.id.split('.')[1]; // speech_to_text.model_size -> model_size
62
- this.options[key] = field.value;
63
- });
64
- }
65
- } catch (error) {
66
- console.error('Failed to load speech settings:', error);
67
- }
69
}
70
71
get status() {
@@ -148,7 +149,7 @@ class MicrophoneInput {
149
if (this.status === Status.WAITING) {
150
this.status = Status.PROCESSING;
151
}
151
- }, this.options.waitingTimeout);
152
+ }, micSettings.stt_waiting_timeout);
153
}
154
155
handleProcessingState() {
@@ -228,7 +229,7 @@ class MicrophoneInput {
229
const now = Date.now();
230
231
// Update status based on audio level
231
- if (rms > this.options.silenceThreshold) {
232
+ if (rms > densify(micSettings.stt_silence_threshold)) {
233
this.lastAudioTime = now;
234
this.silenceStartTime = null;
235
@@ -242,7 +243,7 @@ class MicrophoneInput {
243
}
244
245
const silenceDuration = now - this.silenceStartTime;
245
- if (silenceDuration >= this.options.silenceDuration) {
246
+ if (silenceDuration >= micSettings.stt_silence_duration) {
247
this.status = Status.WAITING;
248
}
249
}
@@ -326,7 +327,7 @@ class MicrophoneInput {
327
328
// Initialize and handle click events
329
async function initializeMicrophoneInput() {
329
- microphoneInput = new MicrophoneInput(
330
+ window.microphoneInput = microphoneInput = new MicrophoneInput(
331
async (text, isFinal) => {
332
if (isFinal) {
333
updateChatInput(text);
@@ -456,8 +457,4 @@ export const speech = new Speech();
457
window.speech = speech
458
459
// Add event listener for settings changes
459
-document.addEventListener('settings-updated', async () => {
460
- if (microphoneInput) {
461
- await microphoneInput.loadSettings();
462
- }
463
-});
\ No newline at end of file
460
+document.addEventListener('settings-updated', loadMicSettings);
\ No newline at end of file