feature: openai-whisper voice input
This also reverts commit 92a904d4411a203c482bc1231dee1438d7279b62.
Alessandro committed
Nov 10, 2024 at 00:37 UTC
82ca0d800ab6e5abf99eda57f31f474a5de8d2d8
5 files changed
+500
-16
python/helpers/voice_transcription.py
new
+105
@@ -0,0 +1,105 @@
1
+import whisper
2
+import io
3
+import base64
4
+import numpy as np
5
+from typing import Optional, Union, BinaryIO
6
+from whisper.audio import load_audio
7
+import tempfile
8
+import os
9
+import subprocess
10
+import warnings
11
+
12
+# suppress FutureWarning from torch.load
13
+warnings.filterwarnings('ignore', category=FutureWarning)
14
+
15
+class VoiceTranscription:
16
+ @staticmethod
17
+ def load_model(model_size: str = "base"):
18
+ """
19
+ Load a Whisper model with the specified size.
20
+ """
21
+ try:
22
+ return whisper.load_model(model_size)
23
+ except Exception as e:
24
+ print(f"Error loading Whisper model: {e}")
25
+ return None
26
+
27
+ @classmethod
28
+ def transcribe_bytes(cls, audio_bytes: Union[str, bytes, BinaryIO],
29
+ model_size: str = "base",
30
+ language: Optional[str] = None) -> str:
31
+ """
32
+ Transcribe audio from bytes or a file-like object.
33
+ """
34
+ model = cls.load_model(model_size)
35
+ if not model:
36
+ raise RuntimeError("Could not load Whisper model")
37
+
38
+ # Decode audio bytes if encoded as a base64 string
39
+ if isinstance(audio_bytes, str):
40
+ try:
41
+ audio_bytes = base64.b64decode(audio_bytes)
42
+ except Exception as e:
43
+ print(f"Error decoding base64 audio data: {e}")
44
+ raise
45
+
46
+ # Save audio bytes to a temporary file with .webm extension
47
+ with tempfile.NamedTemporaryFile(suffix=".webm", delete=False) as tmp_input_file:
48
+ tmp_input_file.write(audio_bytes)
49
+ temp_input_path = tmp_input_file.name
50
+
51
+ try:
52
+ # Define the output path with .wav extension
53
+ with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp_output_file:
54
+ temp_output_path = tmp_output_file.name
55
+
56
+ # Convert WebM to WAV using FFmpeg
57
+ ffmpeg_cmd = [
58
+ 'ffmpeg', '-y', '-i', temp_input_path,
59
+ '-acodec', 'pcm_s16le', '-ar', '16000', '-ac', '1',
60
+ temp_output_path
61
+ ]
62
+
63
+ print(f"Running FFmpeg command: {' '.join(ffmpeg_cmd)}")
64
+
65
+ # Run FFmpeg command using subprocess
66
+ try:
67
+ subprocess.run(
68
+ ffmpeg_cmd, stdout=subprocess.DEVNULL,
69
+ stderr=subprocess.DEVNULL, check=True # Suppressed stderr
70
+ )
71
+ except subprocess.CalledProcessError as e:
72
+ error_message = e.stderr.decode().strip()
73
+ print(f"FFmpeg error: {error_message}")
74
+
75
+ # Log the temporary file path for debugging
76
+ print(f"Transcribing audio from temporary file: {temp_output_path}")
77
+
78
+ # Load audio using Whisper's load_audio
79
+ audio = load_audio(temp_output_path)
80
+
81
+ # Transcribe using the Whisper model
82
+ result = model.transcribe(audio, fp16=False, language=language)
83
+ text = result.get("text", "").strip()
84
+
85
+ # Log the transcription result
86
+ print(f"Transcription result: {text}")
87
+
88
+ return text
89
+
90
+ except subprocess.CalledProcessError as e:
91
+ error_message = e.stderr.decode().strip()
92
+ print(f"FFmpeg error: {error_message}")
93
+ # Return empty string or handle as appropriate
94
+ return ""
95
+ except Exception as transcribe_error:
96
+ print(f"Transcription error: {transcribe_error}")
97
+ # Return empty string or handle as appropriate
98
+ return ""
99
+ finally:
100
+
101
+ # Clean up temporary files
102
+ if os.path.exists(temp_input_path):
103
+ os.remove(temp_input_path)
104
+ if os.path.exists(temp_output_path):
105
+ os.remove(temp_output_path)
requirements.txt
+1
@@ -14,6 +14,7 @@ langchain-huggingface==0.0.3
14
langchain-mistralai==0.1.8
15
langchain-ollama==0.1.3
16
langchain-openai==0.1.15
17
+openai-whisper==20240930
18
lxml_html_clean==0.3.1
19
markdown==3.7
20
newspaper3k==0.2.8
run_ui.py
+74
@@ -13,6 +13,8 @@ from python.helpers.files import get_abs_path
13
from python.helpers.print_style import PrintStyle
14
from python.helpers.dotenv import load_dotenv
15
from python.helpers import persist_chat, settings
16
+from python.helpers.voice_transcription import VoiceTranscription
17
+import base64
18
from werkzeug.utils import secure_filename
19
20
@@ -133,6 +135,78 @@ async def health_check():
135
return "OK"
136
137
138
+@app.route('/transcribe', methods=['POST'])
139
+def transcribe_audio():
140
+ """
141
+ Transcribe audio data using Whisper.
142
+ Expected JSON payload:
143
+ {
144
+ 'audio_data': base64 encoded audio,
145
+ 'model_size': 'base', # Optional, defaults to 'base'
146
+ 'language': None, # Optional language code
147
+ 'is_final': False # Optional flag for final transcription
148
+ }
149
+ """
150
+ try:
151
+ # Parse request data
152
+ data = request.json
153
+ audio_data = data.get('audio_data')
154
+ model_size = data.get('model_size', 'base')
155
+ language = data.get('language')
156
+ is_final = data.get('is_final', False)
157
+
158
+ # Validate input
159
+ if not audio_data:
160
+ return jsonify({
161
+ "error": "No audio data provided",
162
+ "status": "error"
163
+ }), 400
164
+
165
+ # Validate model size
166
+ valid_model_sizes = ['tiny', 'base', 'small', 'medium', 'large']
167
+ if model_size not in valid_model_sizes:
168
+ return jsonify({
169
+ "error": f"Invalid model size. Choose from {valid_model_sizes}",
170
+ "status": "error"
171
+ }), 400
172
+
173
+ # Log the received audio data size
174
+ print(f"Received audio data size: {len(audio_data)} characters (base64)")
175
+
176
+ try:
177
+ # Transcribe using VoiceTranscription helper
178
+ text = VoiceTranscription.transcribe_bytes(
179
+ audio_data,
180
+ model_size=model_size,
181
+ language=language
182
+ )
183
+
184
+ # Return transcription result
185
+ return jsonify({
186
+ "text": text,
187
+ "is_final": is_final,
188
+ "model_size": model_size,
189
+ "status": "success"
190
+ })
191
+
192
+ except Exception as transcribe_error:
193
+ # Detailed error logging for transcription failures
194
+ print(f"Transcription error: {transcribe_error}")
195
+ return jsonify({
196
+ "error": "Transcription failed",
197
+ "details": str(transcribe_error),
198
+ "status": "error"
199
+ }), 500
200
+
201
+ except Exception as e:
202
+ # Catch-all error handler
203
+ print(f"Unexpected transcription error: {e}")
204
+ return jsonify({
205
+ "error": "Unexpected error during transcription",
206
+ "details": str(e),
207
+ "status": "error"
208
+ }), 500
209
+
210
# # secret page, requires authentication
211
# @app.route('/secret', methods=['GET'])
212
# @requires_auth
webui/index.css
+17
@@ -917,6 +917,23 @@ pre {
917
}
918
#microphone-button.recording {
919
background-color: #ff4136; /* Red color for recording */
920
+ transition: background-color 0.3s ease;
921
+}
922
+
923
+.mic-pulse {
924
+ animation: pulse 1.5s infinite;
925
+}
926
+
927
+@keyframes pulse {
928
+ 0% {
929
+ transform: scale(1);
930
+ }
931
+ 50% {
932
+ transform: scale(1.1);
933
+ }
934
+ 100% {
935
+ transform: scale(1);
936
+ }
937
}
938
939
.chat-button svg {
webui/index.js
+303
-16
@@ -18,6 +18,7 @@ const microphoneButton = document.getElementById('microphone-button');
18
let autoScroll = true;
19
let context = "";
20
let microphoneInput = null;
21
+let isProcessingClick = false;
22
23
24
// Initialize the toggle button
@@ -150,32 +151,308 @@ sendButton.addEventListener('click', sendMessage);
151
152
// MICROPHONE INPUT
153
154
+
155
+class MicrophoneInput {
156
+ /**
157
+ * Voice Input Handler with Whisper Transcription
158
+ *
159
+ * Whisper Model Size Configuration:
160
+ * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)
161
+ * - Best for: Quick prototyping, low-resource environments
162
+ * - Pros: Very fast, low memory usage
163
+ * - Cons: Lowest transcription accuracy
164
+ *
165
+ * - 'base': Small model, good balance of speed and accuracy (~74MB)
166
+ * - Best for: General-purpose voice input
167
+ * - Pros: Reasonable accuracy, moderate resource usage
168
+ * - Cons: Less accurate than larger models
169
+ *
170
+ * - 'small': Medium-sized model, better accuracy (~244MB)
171
+ * - Best for: More precise transcription needs
172
+ * - Pros: Improved accuracy over base model
173
+ * - Cons: Slower, more memory-intensive
174
+ *
175
+ * - 'medium': Large model with high accuracy (~769MB)
176
+ * - Best for: Professional transcription, multi-language support
177
+ * - Pros: Very high accuracy
178
+ * - Cons: Significant computational resources required
179
+ *
180
+ * - 'large': Largest model, highest accuracy (~1.5GB)
181
+ * - Best for: Professional, multi-language transcription
182
+ * - Pros: Highest possible accuracy
183
+ * - Cons: Slowest, most resource-intensive
184
+ *
185
+ * Recommended Default: 'base' for most web applications
186
+ */
187
+ constructor(updateCallback, options = {}) {
188
+ this.mediaRecorder = null;
189
+ this.audioChunks = [];
190
+ this.isRecording = false;
191
+ this.updateCallback = updateCallback;
192
+ this.isFinalizing = false;
193
+ this.messageSent = false; // move messageSent into class
194
+
195
+ // New properties for silence detection
196
+ this.audioContext = null;
197
+ this.mediaStreamSource = null;
198
+ this.analyserNode = null;
199
+ this.silenceTimer = null;
200
+ this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed
201
+ this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds
202
+
203
+ this.options = {
204
+ modelSize: 'base',
205
+ language: null,
206
+ chunkDuration: 3000,
207
+ ...options
208
+ };
209
+ }
210
+
211
+ async initialize() {
212
+ try {
213
+ const stream = await navigator.mediaDevices.getUserMedia({
214
+ audio: {
215
+ echoCancellation: true,
216
+ noiseSuppression: true,
217
+ channelCount: 1
218
+ }
219
+ });
220
+
221
+ // Configure MediaRecorder
222
+ this.mediaRecorder = new MediaRecorder(stream, {
223
+ mimeType: 'audio/webm;codecs=opus'
224
+ });
225
+
226
+ // Handle audio data availability
227
+ this.mediaRecorder.ondataavailable = async (event) => {
228
+ if (event.data.size > 0) {
229
+ this.audioChunks.push(event.data);
230
+ // await this.processAudioChunk(event.data);
231
+ }
232
+ };
233
+
234
+ // Handle recording stop
235
+ this.mediaRecorder.onstop = async () => {
236
+ await this.finalizeRecording();
237
+ };
238
+
239
+ // Set up AudioContext and AnalyserNode for silence detection
240
+ this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
241
+ this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
242
+ this.analyserNode = this.audioContext.createAnalyser();
243
+ this.analyserNode.minDecibels = -90;
244
+ this.analyserNode.maxDecibels = -10;
245
+ this.analyserNode.smoothingTimeConstant = 0.85;
246
+
247
+ this.mediaStreamSource.connect(this.analyserNode);
248
+ } catch (error) {
249
+ console.error('Microphone initialization error:', error);
250
+ toast('Failed to access microphone. Please check permissions.', 'error');
251
+ }
252
+ }
253
+
254
+ startSilenceDetection() {
255
+ const dataArray = new Uint8Array(this.analyserNode.fftSize);
256
+ const checkSilence = () => {
257
+ this.analyserNode.getByteTimeDomainData(dataArray);
258
+
259
+ let sum = 0;
260
+ for (let i = 0; i < dataArray.length; i++) {
261
+ const amplitude = (dataArray[i] - 128) / 128;
262
+ sum += amplitude * amplitude;
263
+ }
264
+ const rms = Math.sqrt(sum / dataArray.length);
265
+
266
+ if (rms < this.silenceThreshold) {
267
+ if (!this.silenceTimer) {
268
+ this.silenceTimer = setTimeout(() => {
269
+ if (this.isRecording) {
270
+ console.log('Silence detected. Stopping recording.');
271
+ this.stopRecording();
272
+ microphoneButton.classList.remove('recording');
273
+ microphoneButton.classList.remove('mic-pulse');
274
+ }
275
+ }, this.silenceDuration);
276
+ }
277
+ } else {
278
+ if (this.silenceTimer) {
279
+ clearTimeout(this.silenceTimer);
280
+ this.silenceTimer = null;
281
+ }
282
+ }
283
+
284
+ if (this.isRecording) {
285
+ requestAnimationFrame(checkSilence);
286
+ }
287
+ };
288
+
289
+ if (this.isRecording) {
290
+ requestAnimationFrame(checkSilence);
291
+ }
292
+ }
293
+
294
+ startRecording() {
295
+ if (this.mediaRecorder && this.audioContext) {
296
+ this.isRecording = true;
297
+ this.audioChunks = [];
298
+ this.messageSent = false;
299
+ this.mediaRecorder.start(this.options.chunkDuration);
300
+ this.audioContext.resume();
301
+ this.startSilenceDetection();
302
+ }
303
+ }
304
+
305
+ stopRecording() {
306
+ if (this.mediaRecorder && this.isRecording) {
307
+ this.isRecording = false;
308
+ if (!this.isFinalizing) {
309
+ this.isFinalizing = true;
310
+ this.mediaRecorder.stop();
311
+ this.audioContext.suspend();
312
+ if (this.silenceTimer) {
313
+ clearTimeout(this.silenceTimer);
314
+ this.silenceTimer = null;
315
+ }
316
+ }
317
+ }
318
+ }
319
+
320
+
321
+ async finalizeRecording() {
322
+ if (this.isFinalizing) {
323
+ this.isFinalizing = false;
324
+
325
+ if (this.audioChunks.length > 0) {
326
+ const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });
327
+ this.audioChunks = []; // Clear for next recording
328
+
329
+ const reader = new FileReader();
330
+ reader.onloadend = async () => {
331
+ const base64Data = reader.result.split(',')[1];
332
+
333
+ try {
334
+ const response = await fetch('/transcribe', {
335
+ method: 'POST',
336
+ headers: {
337
+ 'Content-Type': 'application/json'
338
+ },
339
+ body: JSON.stringify({
340
+ audio_data: base64Data,
341
+ model_size: this.options.modelSize,
342
+ language: this.options.language,
343
+ is_final: true
344
+ })
345
+ });
346
+
347
+ const result = await response.json();
348
+
349
+ if (result.text) {
350
+ console.log('Final transcription received:', result.text);
351
+ await this.updateCallback(result.text, true);
352
+ } else {
353
+ console.warn('Final transcription returned empty text.');
354
+ }
355
+ } catch (transcribeError) {
356
+ console.error('Final transcription error:', transcribeError);
357
+ toast('Final transcription failed.', 'error');
358
+ } finally {
359
+ // Reset the microphone button state
360
+ microphoneButton.classList.remove('recording');
361
+ microphoneButton.classList.remove('mic-pulse');
362
+ microphoneButton.style.backgroundColor = '';
363
+ }
364
+ };
365
+ reader.readAsDataURL(audioBlob);
366
+ }
367
+ }
368
+ }
369
+}
370
+
371
+export default MicrophoneInput;
372
+
373
+
374
async function initializeMicrophoneInput() {
154
- microphoneInput = new MicrophoneInput(updateChatInput);
375
+ console.log('Initializing microphone input');
376
+
377
+ microphoneInput = new MicrophoneInput(
378
+ async (text, isFinal) => {
379
+ if (isFinal) {
380
+ console.log('Final transcription callback received:', text);
381
+ chatInput.value = text;
382
+ adjustTextareaHeight();
383
+
384
+ if (!microphoneInput.messageSent) {
385
+ microphoneInput.messageSent = true;
386
+ console.log('Sending message');
387
+ await sendMessage();
388
+
389
+ // Clear the chat input after sending the message
390
+ chatInput.value = '';
391
+ adjustTextareaHeight();
392
+ }
393
+ }
394
+ },
395
+ {
396
+ modelSize: 'base',
397
+ language: 'en',
398
+ silenceThreshold: 0.07, // Adjust as needed
399
+ silenceDuration: 2000, // Adjust as needed
400
+ onError: (error) => {
401
+ console.error('Microphone input error:', error);
402
+ toast('Microphone error: ' + error.message, 'error');
403
+ // Reset recording state
404
+ if (microphoneButton.classList.contains('recording')) {
405
+ microphoneButton.classList.remove('recording');
406
+ }
407
+ }
408
+ }
409
+ );
410
+
411
await microphoneInput.initialize();
412
}
413
+
414
function updateChatInput(text) {
158
- chatInput.value += text + ' ';
415
+ console.log('updateChatInput called with:', text);
416
+
417
+ // Ensure the text is not undefined or null
418
+ if (!text) {
419
+ console.warn('Received empty transcription text');
420
+ return;
421
+ }
422
+
423
+ // Append text with proper spacing
424
+ const currentValue = chatInput.value;
425
+ const needsSpace = currentValue.length > 0 && !currentValue.endsWith(' ');
426
+ chatInput.value = currentValue + (needsSpace ? ' ' : '') + text + ' ';
427
+
428
+ // Adjust height and trigger input event
429
adjustTextareaHeight();
430
+ chatInput.dispatchEvent(new Event('input'));
431
+
432
+ console.log('Updated chat input value:', chatInput.value);
433
}
161
-microphoneButton.addEventListener('click', () => {
162
- if (!microphoneInput) {
163
- initializeMicrophoneInput().then(() => {
164
- toggleRecording();
165
- });
166
- } else {
167
- toggleRecording();
168
- }
169
-});
434
+
435
+
436
function toggleRecording() {
437
+ console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);
438
+
439
if (microphoneInput.isRecording) {
440
microphoneInput.stopRecording();
441
microphoneButton.classList.remove('recording');
442
+ // Add pulsing animation class
443
+ microphoneButton.classList.remove('mic-pulse');
444
} else {
445
microphoneInput.startRecording();
446
microphoneButton.classList.add('recording');
447
+ // Add pulsing animation class
448
+ microphoneButton.classList.add('mic-pulse');
449
}
450
+
451
+ // Add visual feedback
452
+ microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';
453
+ console.log('New recording state:', microphoneInput.isRecording);
454
}
455
+
456
// Some error handling for microphone input
457
async function requestMicrophonePermission() {
458
try {
@@ -189,15 +466,25 @@ async function requestMicrophonePermission() {
466
}
467
// microphoneButton click event listener modifier
468
microphoneButton.addEventListener('click', async () => {
469
+ console.log('Microphone button clicked');
470
+ if (isProcessingClick) {
471
+ console.log('Click already being processed, ignoring');
472
+ return;
473
+ }
474
+ isProcessingClick = true;
475
+
476
const hasPermission = await requestMicrophonePermission();
477
if (!hasPermission) return;
478
+
479
if (!microphoneInput) {
195
- initializeMicrophoneInput().then(() => {
196
- toggleRecording();
197
- });
198
- } else {
199
- toggleRecording();
480
+ await initializeMicrophoneInput();
481
}
482
+
483
+ await toggleRecording();
484
+
485
+ setTimeout(() => {
486
+ isProcessingClick = false;
487
+ }, 300); // Add a 300ms delay before allowing another click
488
});
489
490
function updateUserTime() {