native models, log trunc fix, speech skip
moved whisper and kokoro to native runtime log kvps truncation fix skip speech on context switch
frdel committed
Jul 14, 2025 at 14:30 UTC
dcae2b74ac8e05e8ebb2f49ef2aa3a88dcb6cf6e
7 files changed
+126
-58
preload.py
+5
-1
@@ -40,7 +40,11 @@ async def preload():
40
PrintStyle().error(f"Error in preload_kokoro: {e}")
41
42
# async tasks to preload
43
- tasks = [preload_whisper(), preload_embedding(), preload_kokoro()]
43
+ tasks = [
44
+ preload_embedding(),
45
+ # preload_whisper(),
46
+ # preload_kokoro()
47
+ ] # no longer preload kokoro and whisper, do it JIT
48
49
await asyncio.gather(*tasks, return_exceptions=True)
50
PrintStyle().print("Preload completed")
python/api/synthesize.py
+1
-1
@@ -12,7 +12,7 @@ class Synthesize(ApiHandler):
12
ctxid = input.get("ctxid", "")
13
14
context = self.get_context(ctxid)
15
- if await kokoro_tts.is_downloading():
15
+ if not await kokoro_tts.is_downloaded():
16
context.log.log(type="info", content="Kokoro TTS model is currently being downloaded, please wait...")
17
18
try:
python/api/transcribe.py
+2
-2
@@ -9,8 +9,8 @@ class Transcribe(ApiHandler):
9
ctxid = input.get("ctxid", "")
10
11
context = self.get_context(ctxid)
12
- if await whisper.is_downloading():
13
- context.log.log(type="info", content="Whisper model is currently being downloaded, please wait...")
12
+ if not await whisper.is_downloaded():
13
+ context.log.log(type="info", content="Whisper STT model is currently being downloaded, please wait...")
14
15
set = settings.get_settings()
16
result = await whisper.transcribe(set["stt_model_size"], audio) # type: ignore
python/helpers/kokoro_tts.py
+44
-22
@@ -15,52 +15,74 @@ _voice = "am_puck,am_onyx"
15
_speed = 1.1
16
is_updating_model = False
17
18
+
19
async def preload():
20
try:
20
- return await runtime.call_development_function(_preload)
21
+ # return await runtime.call_development_function(_preload)
22
+ return await _preload()
23
except Exception as e:
22
- if not runtime.is_development():
23
- raise e
24
+ # if not runtime.is_development():
25
+ raise e
26
# Fallback to direct execution if RFC fails in development
25
- PrintStyle.standard("RFC failed, falling back to direct execution...")
26
- return await _preload()
27
+ # PrintStyle.standard("RFC failed, falling back to direct execution...")
28
+ # return await _preload()
29
+
30
31
async def _preload():
32
global _pipeline, is_updating_model
30
-
33
+
34
while is_updating_model:
35
await asyncio.sleep(0.1)
33
-
36
+
37
try:
38
is_updating_model = True
39
if not _pipeline:
40
PrintStyle.standard("Loading Kokoro TTS model...")
41
from kokoro import KPipeline
39
- _pipeline = KPipeline(lang_code='a')
42
+ _pipeline = KPipeline(lang_code="a")
43
finally:
44
is_updating_model = False
45
46
+
47
async def is_downloading():
48
try:
45
- return await runtime.call_development_function(_is_downloading)
49
+ # return await runtime.call_development_function(_is_downloading)
50
+ return _is_downloading()
51
except Exception as e:
47
- if not runtime.is_development():
48
- raise e
52
+ # if not runtime.is_development():
53
+ raise e
54
# Fallback to direct execution if RFC fails in development
50
- return _is_downloading()
55
+ # return _is_downloading()
56
+
57
58
def _is_downloading():
59
return is_updating_model
60
61
+async def is_downloaded():
62
+ try:
63
+ # return await runtime.call_development_function(_is_downloaded)
64
+ return _is_downloaded()
65
+ except Exception as e:
66
+ # if not runtime.is_development():
67
+ raise e
68
+ # Fallback to direct execution if RFC fails in development
69
+ # return _is_downloaded()
70
+
71
+def _is_downloaded():
72
+ return _pipeline is not None
73
+
74
+
75
async def synthesize_sentences(sentences: list[str]):
76
"""Generate audio for multiple sentences and return concatenated base64 audio"""
77
try:
58
- return await runtime.call_development_function(_synthesize_sentences, sentences)
78
+ # return await runtime.call_development_function(_synthesize_sentences, sentences)
79
+ return await _synthesize_sentences(sentences)
80
except Exception as e:
60
- if not runtime.is_development():
61
- raise e
81
+ # if not runtime.is_development():
82
+ raise e
83
# Fallback to direct execution if RFC fails in development
63
- return await _synthesize_sentences(sentences)
84
+ # return await _synthesize_sentences(sentences)
85
+
86
87
async def _synthesize_sentences(sentences: list[str]):
88
await _preload()
@@ -70,22 +92,22 @@ async def _synthesize_sentences(sentences: list[str]):
92
try:
93
for sentence in sentences:
94
if sentence.strip():
73
- segments = _pipeline(sentence.strip(), voice=_voice, speed=_speed)
95
+ segments = _pipeline(sentence.strip(), voice=_voice, speed=_speed) # type: ignore
96
segment_list = list(segments)
75
-
97
+
98
for segment in segment_list:
99
audio_tensor = segment.audio
78
- audio_numpy = audio_tensor.detach().cpu().numpy()
100
+ audio_numpy = audio_tensor.detach().cpu().numpy() # type: ignore
101
combined_audio.extend(audio_numpy)
102
103
# Convert combined audio to bytes
104
buffer = io.BytesIO()
83
- sf.write(buffer, combined_audio, 24000, format='WAV')
105
+ sf.write(buffer, combined_audio, 24000, format="WAV")
106
audio_bytes = buffer.getvalue()
107
108
# Return base64 encoded audio
87
- return base64.b64encode(audio_bytes).decode('utf-8')
109
+ return base64.b64encode(audio_bytes).decode("utf-8")
110
111
except Exception as e:
112
PrintStyle.error(f"Error in Kokoro TTS synthesis: {e}")
91
- raise
\ No newline at end of file
113
+ raise
\ No newline at end of file
python/helpers/log.py
+31
-17
@@ -4,6 +4,7 @@ from typing import Any, Literal, Optional, Dict
4
import uuid
5
from collections import OrderedDict # Import OrderedDict
6
from python.helpers.strings import truncate_text_by_ratio
7
+import copy
8
9
Type = Literal[
10
"agent",
@@ -25,9 +26,9 @@ ProgressUpdate = Literal["persistent", "temporary", "none"]
26
27
28
HEADING_MAX_LEN: int = 120
28
-CONTENT_MAX_LEN: int = 4000
29
+CONTENT_MAX_LEN: int = 10000
30
KEY_MAX_LEN: int = 60
30
-VALUE_MAX_LEN: int = 1000
31
+VALUE_MAX_LEN: int = 3000
32
PROGRESS_MAX_LEN: int = 120
33
34
@@ -44,29 +45,36 @@ def _truncate_progress(text: str | None) -> str:
45
def _truncate_key(text: str) -> str:
46
return truncate_text_by_ratio(str(text), KEY_MAX_LEN, "...", ratio=1.0)
47
47
-def _truncate_value(text: Any) -> Any:
48
+def _truncate_value(val: Any) -> Any:
49
+ # If dict, recursively truncate each value
50
+ if isinstance(val, dict):
51
+ for k in list(val.keys()):
52
+ val[k] = _truncate_value(val[k])
53
+ return val
54
+ # If list or tuple, recursively truncate each item
55
+ if isinstance(val, list):
56
+ for i in range(len(val)):
57
+ val[i] = _truncate_value(val[i])
58
+ return val
59
+ if isinstance(val, tuple):
60
+ return tuple(_truncate_value(x) for x in val)
61
+
62
# Convert non-str values to json for consistent length measurement
49
- if isinstance(text, str):
50
- raw = text
63
+ if isinstance(val, str):
64
+ raw = val
65
else:
66
try:
53
- raw = json.dumps(text, ensure_ascii=False)
67
+ raw = json.dumps(val, ensure_ascii=False)
68
except Exception:
55
- raw = str(text)
69
+ raw = str(val)
70
71
if len(raw) <= VALUE_MAX_LEN:
58
- return text # No truncation needed, preserve original type
72
+ return val # No truncation needed, preserve original type
73
60
- # Determine removed characters dynamically to build replacement string
74
+ # Do a single truncation calculation
75
removed = len(raw) - VALUE_MAX_LEN
62
- while True:
63
- replacement = f"\n\n<< {removed} Characters hidden >>\n\n"
64
- truncated = truncate_text_by_ratio(raw, VALUE_MAX_LEN, replacement, ratio=0.3)
65
- new_removed = len(raw) - (len(truncated) - len(replacement))
66
- if new_removed == removed:
67
- break
68
- removed = new_removed
69
-
76
+ replacement = f"\n\n<< {removed} Characters hidden >>\n\n"
77
+ truncated = truncate_text_by_ratio(raw, VALUE_MAX_LEN, replacement, ratio=0.3)
78
return truncated
79
80
def _truncate_content(text: str | None) -> str:
@@ -177,10 +185,13 @@ class Log:
185
186
# Truncate kvps
187
if kvps is not None:
188
+ kvps = copy.deepcopy(kvps) # deep copy to avoid modifying the original kvps
189
kvps = OrderedDict({
190
_truncate_key(k): _truncate_value(v) for k, v in kvps.items()
191
})
192
# Apply truncation to kwargs merged into kvps later
193
+ if kwargs is not None:
194
+ kwargs = copy.deepcopy(kwargs) # deep copy to avoid modifying the original kwargs
195
kwargs = { _truncate_key(k): _truncate_value(v) for k, v in (kwargs or {}).items() }
196
197
# Ensure kvps is OrderedDict even if None
@@ -231,6 +242,7 @@ class Log:
242
item.content = _truncate_content(content)
243
244
if kvps is not None:
245
+ kvps = copy.deepcopy(kvps) # deep copy to avoid modifying the original kvps
246
item.kvps = OrderedDict({
247
_truncate_key(k): _truncate_value(v) for k, v in kvps.items()
248
}) # Ensure order
@@ -239,11 +251,13 @@ class Log:
251
item.temp = temp
252
253
if kwargs:
254
+ kwargs = copy.deepcopy(kwargs) # deep copy to avoid modifying the original kwargs
255
if item.kvps is None:
256
item.kvps = OrderedDict() # Ensure kvps is an OrderedDict
257
for k, v in kwargs.items():
258
item.kvps[_truncate_key(k)] = _truncate_value(v)
259
260
+
261
self.updates += [item.no]
262
self._update_progress_from_item(item)
263
python/helpers/whisper.py
+23
-7
@@ -3,7 +3,7 @@ import warnings
3
import whisper
4
import tempfile
5
import asyncio
6
-from python.helpers import runtime, rfc, settings
6
+from python.helpers import runtime, rfc, settings, files
7
from python.helpers.print_style import PrintStyle
8
9
# Suppress FutureWarning from torch.load
@@ -15,10 +15,11 @@ is_updating_model = False # Tracks whether the model is currently updating
15
16
async def preload(model_name:str):
17
try:
18
- return await runtime.call_development_function(_preload, model_name)
18
+ # return await runtime.call_development_function(_preload, model_name)
19
+ return await _preload(model_name)
20
except Exception as e:
20
- if not runtime.is_development():
21
- raise e
21
+ # if not runtime.is_development():
22
+ raise e
23
24
async def _preload(model_name:str):
25
global _model, _model_name, is_updating_model
@@ -30,19 +31,34 @@ async def _preload(model_name:str):
31
is_updating_model = True
32
if not _model or _model_name != model_name:
33
PrintStyle.standard(f"Loading Whisper model: {model_name}")
33
- _model = whisper.load_model(name=model_name) # type: ignore
34
+ _model = whisper.load_model(name=model_name, download_root=files.get_abs_path("/tmp/models/whisper")) # type: ignore
35
_model_name = model_name
36
finally:
37
is_updating_model = False
38
39
async def is_downloading():
39
- return await runtime.call_development_function(_is_downloading)
40
+ # return await runtime.call_development_function(_is_downloading)
41
+ return _is_downloading()
42
43
def _is_downloading():
44
return is_updating_model
45
46
+async def is_downloaded():
47
+ try:
48
+ # return await runtime.call_development_function(_is_downloaded)
49
+ return _is_downloaded()
50
+ except Exception as e:
51
+ # if not runtime.is_development():
52
+ raise e
53
+ # Fallback to direct execution if RFC fails in development
54
+ # return _is_downloaded()
55
+
56
+def _is_downloaded():
57
+ return _model is not None
58
+
59
async def transcribe(model_name:str, audio_bytes_b64: str):
45
- return await runtime.call_development_function(_transcribe, model_name, audio_bytes_b64)
60
+ # return await runtime.call_development_function(_transcribe, model_name, audio_bytes_b64)
61
+ return await _transcribe(model_name, audio_bytes_b64)
62
63
64
async def _transcribe(model_name:str, audio_bytes_b64: str):
webui/index.js
+20
-8
@@ -24,6 +24,7 @@ const timeDate = document.getElementById("time-date-container");
24
let autoScroll = true;
25
let context = "";
26
let resetCounter = 0;
27
+let skipOneSpeech = false;
28
let connectionStatus = false;
29
30
// Initialize the toggle button
@@ -503,6 +504,10 @@ function afterMessagesUpdate(logs) {
504
}
505
506
function speakMessages(logs) {
507
+ if (skipOneSpeech) {
508
+ skipOneSpeech = false;
509
+ return;
510
+ }
511
// log.no, log.type, log.heading, log.content
512
for (let i = logs.length - 1; i >= 0; i--) {
513
const log = logs[i];
@@ -512,10 +517,14 @@ function speakMessages(logs) {
517
518
// finished response
519
if (log.type == "response") {
515
- // lastSpokenNo = log.no;
516
- speechStore.speakStream(getChatBasedId(log.no), log.content, log.kvps?.finished);
517
- return;
518
-
520
+ // lastSpokenNo = log.no;
521
+ speechStore.speakStream(
522
+ getChatBasedId(log.no),
523
+ log.content,
524
+ log.kvps?.finished
525
+ );
526
+ return;
527
+
528
// finished LLM headline, not response
529
} else if (
530
log.type == "agent" &&
@@ -524,9 +533,9 @@ function speakMessages(logs) {
533
log.kvps.tool_args &&
534
log.kvps.tool_name != "response"
535
) {
527
- // lastSpokenNo = log.no;
528
- speechStore.speakStream(getChatBasedId(log.no), log.kvps.headline, true);
529
- return;
536
+ // lastSpokenNo = log.no;
537
+ speechStore.speakStream(getChatBasedId(log.no), log.kvps.headline, true);
538
+ return;
539
}
540
}
541
}
@@ -736,6 +745,9 @@ export const setContext = function (id) {
745
if (tasksAD) tasksAD.selected = id;
746
}
747
}
748
+
749
+ //skip one speech if enabled when switching context
750
+ if (localStorage.getItem("speech") == "true") skipOneSpeech = true;
751
};
752
753
export const getContext = function () {
@@ -743,7 +755,7 @@ export const getContext = function () {
755
};
756
757
export const getChatBasedId = function (id) {
746
- return context+"-"+resetCounter+"-"+id;
758
+ return context + "-" + resetCounter + "-" + id;
759
};
760
761
window.toggleAutoScroll = async function (_autoScroll) {