@samitouri / QOSami_Agents_Zero-1 / commits / 1a91334a

STT continued

Dialogue mode and state managed for STT

frdel committed Nov 10, 2024 at 20:57 UTC 1a91334a420f605101e6aecdbd34f1dca33c192b
6 files changed +757 -259
webui/index.css
-29
@@ -907,35 +907,6 @@ pre {
907 background-color: var(--color-primary-light);
908 }
909
910 -/* MIC BUTTON */
911 -#microphone-button {
912 - background-color: #3270e2;
913 -}
914 -
915 -#microphone-button:hover {
916 - background-color: #4382e8;
917 -}
918 -#microphone-button.recording {
919 - background-color: #ff4136; /* Red color for recording */
920 - transition: background-color 0.3s ease;
921 -}
922 -
923 -.mic-pulse {
924 - animation: pulse 1.5s infinite;
925 -}
926 -
927 -@keyframes pulse {
928 - 0% {
929 - transform: scale(1);
930 - }
931 - 50% {
932 - transform: scale(1.1);
933 - }
934 - 100% {
935 - transform: scale(1);
936 - }
937 -}
938 -
910 .chat-button svg {
911 width: 1.5rem;
912 height: 1.5rem;
webui/index.html
+2 -1
@@ -8,6 +8,7 @@
8 <link rel="stylesheet" href="index.css">
9 <link rel="stylesheet" href="toast.css">
10 <link rel="stylesheet" href="settings.css">
11 + <link rel="stylesheet" href="speech.css">
12
13 <script>
14 window.safeCall = function (name, ...args) {
@@ -236,7 +237,7 @@
237 </button>
238
239 <!-- Microphone button -->
239 - <button class="chat-button" id="microphone-button" aria-label="Start/Stop recording">
240 + <button class="chat-button mic-inactive" id="microphone-button" aria-label="Start/Stop recording">
241 <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 18" fill="currentColor" width="24" height="24">
242 <path d="m8,12c1.66,0,3-1.34,3-3V3c0-1.66-1.34-3-3-3s-3,1.34-3,3v6c0,1.66,1.34,3,3,3Zm-1,1.9c-2.7-.4-4.8-2.6-5-5.4H0c.2,3.8,3.1,6.9,7,7.5v2h2v-2c3.9-.6,6.8-3.7,7-7.5h-2c-.2,2.8-2.3,5-5,5.4h-2Z"/>
243 </svg>
webui/speech copy.js new
+337
@@ -0,0 +1,337 @@
1 +import { pipeline, read_audio } from './transformers@3.0.2.js';
2 +import { updateChatInput, sendMessage } from './index.js';
3 +
4 +const microphoneButton = document.getElementById('microphone-button');
5 +let microphoneInput = null;
6 +let isProcessingClick = false;
7 +
8 +class MicrophoneInput {
9 + /**
10 + * Voice Input Handler with Whisper Transcription
11 + *
12 + * Whisper Model Size Configuration:
13 + * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)
14 + * - Best for: Quick prototyping, low-resource environments
15 + * - Pros: Very fast, low memory usage
16 + * - Cons: Lowest transcription accuracy
17 + *
18 + * - 'base': Small model, good balance of speed and accuracy (~74MB)
19 + * - Best for: General-purpose voice input
20 + * - Pros: Reasonable accuracy, moderate resource usage
21 + * - Cons: Less accurate than larger models
22 + *
23 + * - 'small': Medium-sized model, better accuracy (~244MB)
24 + * - Best for: More precise transcription needs
25 + * - Pros: Improved accuracy over base model
26 + * - Cons: Slower, more memory-intensive
27 + *
28 + * - 'medium': Large model with high accuracy (~769MB)
29 + * - Best for: Professional transcription, multi-language support
30 + * - Pros: Very high accuracy
31 + * - Cons: Significant computational resources required
32 + *
33 + * - 'large': Largest model, highest accuracy (~1.5GB)
34 + * - Best for: Professional, multi-language transcription
35 + * - Pros: Highest possible accuracy
36 + * - Cons: Slowest, most resource-intensive
37 + *
38 + * Recommended Default: 'base' for most web applications
39 + */
40 + constructor(updateCallback, options = {}) {
41 + this.mediaRecorder = null;
42 + this.audioChunks = [];
43 + this.isRecording = false;
44 + this.updateCallback = updateCallback;
45 + this.isFinalizing = false;
46 + this.messageSent = false; // move messageSent into class
47 +
48 + // New properties for silence detection
49 + this.audioContext = null;
50 + this.mediaStreamSource = null;
51 + this.analyserNode = null;
52 + this.silenceTimer = null;
53 + this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed
54 + this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds
55 +
56 + this.options = {
57 + modelSize: 'tiny',
58 + language: 'en',
59 + chunkDuration: 3000,
60 + ...options
61 + };
62 + }
63 +
64 + async initialize() {
65 + try {
66 +
67 + this.transcriber = await pipeline(`automatic-speech-recognition`, `Xenova/whisper-${this.options.modelSize}.${this.options.language}`);
68 +
69 + const stream = await navigator.mediaDevices.getUserMedia({
70 + audio: {
71 + echoCancellation: true,
72 + noiseSuppression: true,
73 + channelCount: 1
74 + }
75 + });
76 +
77 + // Configure MediaRecorder
78 + this.mediaRecorder = new MediaRecorder(stream);
79 +
80 + // Handle audio data availability
81 + this.mediaRecorder.ondataavailable = async (event) => {
82 + if (event.data.size > 0) {
83 + this.audioChunks.push(event.data);
84 + // await this.processAudioChunk(event.data);
85 + }
86 + };
87 +
88 + // Handle recording stop
89 + this.mediaRecorder.onstop = async () => {
90 + await this.finalizeRecording();
91 + };
92 +
93 + // Set up AudioContext and AnalyserNode for silence detection
94 + this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
95 + this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
96 + this.analyserNode = this.audioContext.createAnalyser();
97 + this.analyserNode.minDecibels = -90;
98 + this.analyserNode.maxDecibels = -10;
99 + this.analyserNode.smoothingTimeConstant = 0.85;
100 +
101 + this.mediaStreamSource.connect(this.analyserNode);
102 + } catch (error) {
103 + console.error('Microphone initialization error:', error);
104 + toast('Failed to access microphone. Please check permissions.', 'error');
105 + }
106 + }
107 +
108 + startSilenceDetection() {
109 + const dataArray = new Uint8Array(this.analyserNode.fftSize);
110 + const checkSilence = () => {
111 + this.analyserNode.getByteTimeDomainData(dataArray);
112 +
113 + let sum = 0;
114 + for (let i = 0; i < dataArray.length; i++) {
115 + const amplitude = (dataArray[i] - 128) / 128;
116 + sum += amplitude * amplitude;
117 + }
118 + const rms = Math.sqrt(sum / dataArray.length);
119 +
120 + if (rms < this.silenceThreshold) {
121 + if (!this.silenceTimer) {
122 + this.silenceTimer = setTimeout(() => {
123 + if (this.isRecording) {
124 + console.log('Silence detected. Stopping recording.');
125 + this.stopRecording();
126 + microphoneButton.classList.remove('recording');
127 + microphoneButton.classList.remove('mic-pulse');
128 + }
129 + }, this.silenceDuration);
130 + }
131 + } else {
132 + if (this.silenceTimer) {
133 + clearTimeout(this.silenceTimer);
134 + this.silenceTimer = null;
135 + }
136 + }
137 +
138 + if (this.isRecording) {
139 + requestAnimationFrame(checkSilence);
140 + }
141 + };
142 +
143 + if (this.isRecording) {
144 + requestAnimationFrame(checkSilence);
145 + }
146 + }
147 +
148 + startRecording() {
149 + if (this.mediaRecorder && this.audioContext) {
150 + this.isRecording = true;
151 + this.audioChunks = [];
152 + this.messageSent = false;
153 + this.mediaRecorder.start(this.options.chunkDuration);
154 + this.audioContext.resume();
155 + this.startSilenceDetection();
156 + }
157 + }
158 +
159 + stopRecording() {
160 + if (this.mediaRecorder && this.isRecording) {
161 + this.isRecording = false;
162 + if (!this.isFinalizing) {
163 + this.isFinalizing = true;
164 + this.mediaRecorder.stop();
165 + this.audioContext.suspend();
166 + if (this.silenceTimer) {
167 + clearTimeout(this.silenceTimer);
168 + this.silenceTimer = null;
169 + }
170 + }
171 + }
172 + }
173 +
174 +
175 + async finalizeRecording() {
176 + if (this.isFinalizing) {
177 + this.isFinalizing = false;
178 +
179 + if (this.audioChunks.length > 0) {
180 +
181 + const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
182 + const audioUrl = URL.createObjectURL(audioBlob);
183 + const samplingRate = 16000; // Adjust as needed for the model
184 + const audioData = await read_audio(audioUrl, samplingRate);
185 + URL.revokeObjectURL(audioUrl);
186 +
187 + // Transcribe the audio
188 + const result = await this.transcriber(audioData);
189 +
190 + if (result.text) {
191 + console.log('Final transcription received:', result.text);
192 + await this.updateCallback(result.text, true);
193 + } else {
194 + console.warn('Final transcription returned empty text.');
195 + }
196 +
197 +
198 + // Release the object URL after use
199 +
200 + // const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });
201 + // this.audioChunks = []; // Clear for next recording
202 +
203 + // const reader = new FileReader();
204 + // reader.onloadend = async () => {
205 + // const base64Data = reader.result.split(',')[1];
206 +
207 + // try {
208 + // const response = await fetch('/transcribe', {
209 + // method: 'POST',
210 + // headers: {
211 + // 'Content-Type': 'application/json'
212 + // },
213 + // body: JSON.stringify({
214 + // audio_data: base64Data,
215 + // model_size: this.options.modelSize,
216 + // language: this.options.language,
217 + // is_final: true
218 + // })
219 + // });
220 +
221 + // const result = await response.json();
222 +
223 + // if (result.text) {
224 + // console.log('Final transcription received:', result.text);
225 + // await this.updateCallback(result.text, true);
226 + // } else {
227 + // console.warn('Final transcription returned empty text.');
228 + // }
229 + // } catch (transcribeError) {
230 + // console.error('Final transcription error:', transcribeError);
231 + // toast('Final transcription failed.', 'error');
232 + // } finally {
233 + // // Reset the microphone button state
234 + // microphoneButton.classList.remove('recording');
235 + // microphoneButton.classList.remove('mic-pulse');
236 + // microphoneButton.style.backgroundColor = '';
237 + // }
238 + // };
239 + // reader.readAsDataURL(audioBlob);
240 + }
241 + }
242 + }
243 +}
244 +
245 +export default MicrophoneInput;
246 +
247 +async function initializeMicrophoneInput() {
248 + console.log('Initializing microphone input');
249 +
250 + microphoneInput = new MicrophoneInput(
251 + async (text, isFinal) => {
252 + if (isFinal) {
253 + console.log('Final transcription callback received:', text);
254 + updateChatInput(text)
255 + // chatInput.value = text;
256 + // adjustTextareaHeight();
257 +
258 + if (!microphoneInput.messageSent) {
259 + microphoneInput.messageSent = true;
260 + console.log('Sending message');
261 + await sendMessage();
262 + }
263 + }
264 + },
265 + {
266 + modelSize: 'tiny',
267 + language: 'en',
268 + silenceThreshold: 0.07, // Adjust as needed
269 + silenceDuration: 2000, // Adjust as needed
270 + onError: (error) => {
271 + console.error('Microphone input error:', error);
272 + toast('Microphone error: ' + error.message, 'error');
273 + // Reset recording state
274 + if (microphoneButton.classList.contains('recording')) {
275 + microphoneButton.classList.remove('recording');
276 + }
277 + }
278 + }
279 + );
280 +
281 + await microphoneInput.initialize();
282 +}
283 +
284 +
285 +function toggleRecording() {
286 + console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);
287 +
288 + if (microphoneInput.isRecording) {
289 + microphoneInput.stopRecording();
290 + microphoneButton.classList.remove('recording');
291 + // Add pulsing animation class
292 + microphoneButton.classList.remove('mic-pulse');
293 + } else {
294 + microphoneInput.startRecording();
295 + microphoneButton.classList.add('recording');
296 + // Add pulsing animation class
297 + microphoneButton.classList.add('mic-pulse');
298 + }
299 +
300 + // Add visual feedback
301 + microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';
302 + console.log('New recording state:', microphoneInput.isRecording);
303 +}
304 +
305 +// Some error handling for microphone input
306 +async function requestMicrophonePermission() {
307 + try {
308 + await navigator.mediaDevices.getUserMedia({ audio: true });
309 + return true;
310 + } catch (err) {
311 + console.error('Error accessing microphone:', err);
312 + toast('Microphone access denied. Please enable microphone access in your browser settings.', 'error');
313 + return false;
314 + }
315 +}
316 +// microphoneButton click event listener modifier
317 +microphoneButton.addEventListener('click', async () => {
318 + console.log('Microphone button clicked');
319 + if (isProcessingClick) {
320 + console.log('Click already being processed, ignoring');
321 + return;
322 + }
323 + isProcessingClick = true;
324 +
325 + const hasPermission = await requestMicrophonePermission();
326 + if (!hasPermission) return;
327 +
328 + if (!microphoneInput) {
329 + await initializeMicrophoneInput();
330 + }
331 +
332 + await toggleRecording();
333 +
334 + setTimeout(() => {
335 + isProcessingClick = false;
336 + }, 300); // Add a 300ms delay before allowing another click
337 +});
webui/speech.css new
+55
@@ -0,0 +1,55 @@
1 +/* MIC BUTTON */
2 +#microphone-button {
3 +
4 +}
5 +
6 +#microphone-button:hover {
7 +}
8 +
9 +#microphone-button.recording {
10 + background-color: #ff4136; /* Red color for recording */
11 + transition: background-color 0.3s ease;
12 +}
13 +
14 +@keyframes pulse {
15 + 0% {
16 + transform: scale(1);
17 + }
18 + 50% {
19 + transform: scale(1.1);
20 + }
21 + 100% {
22 + transform: scale(1);
23 + }
24 + }
25 +
26 +.mic-pulse {
27 + animation: pulse 1.5s infinite;
28 +}
29 +
30 +
31 +.mic-inactive{
32 + background-color: grey;
33 +}
34 +
35 +.mic-activating{
36 + background-color: silver;
37 + animation: pulse 0.8s infinite;
38 +}
39 +
40 +.mic-listening {
41 + background-color: red;
42 +}
43 +
44 +.mic-recording {
45 + background-color: green;
46 +}
47 +
48 +.mic-waiting {
49 + background-color: teal;
50 +}
51 +
52 +.mic-processing {
53 + background-color: darkcyan;
54 + animation: pulse 0.8s infinite;
55 +}
\ No newline at end of file
webui/speech.js
+241 -229
@@ -5,66 +5,149 @@ const microphoneButton = document.getElementById('microphone-button');
5 let microphoneInput = null;
6 let isProcessingClick = false;
7
8 +const Status = {
9 + INACTIVE: 'inactive',
10 + ACTIVATING: 'activating',
11 + LISTENING: 'listening',
12 + RECORDING: 'recording',
13 + WAITING: 'waiting',
14 + PROCESSING: 'processing'
15 +};
16 +
17 class MicrophoneInput {
9 - /**
10 - * Voice Input Handler with Whisper Transcription
11 - *
12 - * Whisper Model Size Configuration:
13 - * - 'tiny': Smallest model, fastest, lowest accuracy (~32MB)
14 - * - Best for: Quick prototyping, low-resource environments
15 - * - Pros: Very fast, low memory usage
16 - * - Cons: Lowest transcription accuracy
17 - *
18 - * - 'base': Small model, good balance of speed and accuracy (~74MB)
19 - * - Best for: General-purpose voice input
20 - * - Pros: Reasonable accuracy, moderate resource usage
21 - * - Cons: Less accurate than larger models
22 - *
23 - * - 'small': Medium-sized model, better accuracy (~244MB)
24 - * - Best for: More precise transcription needs
25 - * - Pros: Improved accuracy over base model
26 - * - Cons: Slower, more memory-intensive
27 - *
28 - * - 'medium': Large model with high accuracy (~769MB)
29 - * - Best for: Professional transcription, multi-language support
30 - * - Pros: Very high accuracy
31 - * - Cons: Significant computational resources required
32 - *
33 - * - 'large': Largest model, highest accuracy (~1.5GB)
34 - * - Best for: Professional, multi-language transcription
35 - * - Pros: Highest possible accuracy
36 - * - Cons: Slowest, most resource-intensive
37 - *
38 - * Recommended Default: 'base' for most web applications
39 - */
18 constructor(updateCallback, options = {}) {
19 this.mediaRecorder = null;
20 this.audioChunks = [];
43 - this.isRecording = false;
21 + this.lastChunk = [];
22 this.updateCallback = updateCallback;
45 - this.isFinalizing = false;
46 - this.messageSent = false; // move messageSent into class
23 + this.messageSent = false;
24
48 - // New properties for silence detection
25 + // Audio analysis properties
26 this.audioContext = null;
27 this.mediaStreamSource = null;
28 this.analyserNode = null;
52 - this.silenceTimer = null;
53 - this.silenceThreshold = options.silenceThreshold || 0.01; // Adjust as needed
54 - this.silenceDuration = options.silenceDuration || 2000; // Duration in milliseconds
29 + this._status = Status.INACTIVE;
30 +
31 + // Timing properties
32 + this.lastAudioTime = null;
33 + this.waitingTimer = null;
34 + this.silenceStartTime = null;
35 + this.hasStartedRecording = false;
36 + this.analysisFrame = null;
37
38 this.options = {
39 modelSize: 'tiny',
40 language: 'en',
59 - chunkDuration: 3000,
41 + silenceThreshold: 0.15,
42 + silenceDuration: 1000,
43 + waitingTimeout: 2000,
44 + minSpeechDuration: 500,
45 ...options
46 };
47 }
48
49 + get status() {
50 + return this._status;
51 + }
52 +
53 + set status(newStatus) {
54 + if (this._status === newStatus) return;
55 +
56 + const oldStatus = this._status;
57 + this._status = newStatus;
58 + console.log(`Mic status changed from ${oldStatus} to ${newStatus}`);
59 +
60 + // Update UI
61 + microphoneButton.classList.remove(`mic-${oldStatus.toLowerCase()}`);
62 + microphoneButton.classList.add(`mic-${newStatus.toLowerCase()}`);
63 + microphoneButton.setAttribute('data-status', newStatus);
64 +
65 + // Handle state-specific behaviors
66 + this.handleStatusChange(oldStatus, newStatus);
67 + }
68 +
69 + handleStatusChange(oldStatus, newStatus) {
70 +
71 + //last chunk kept only for transition to recording status
72 + if (newStatus != Status.RECORDING) { this.lastChunk = null; }
73 +
74 + switch (newStatus) {
75 + case Status.INACTIVE:
76 + this.handleInactiveState();
77 + break;
78 + case Status.LISTENING:
79 + this.handleListeningState();
80 + break;
81 + case Status.RECORDING:
82 + this.handleRecordingState();
83 + break;
84 + case Status.WAITING:
85 + this.handleWaitingState();
86 + break;
87 + case Status.PROCESSING:
88 + this.handleProcessingState();
89 + break;
90 + }
91 + }
92 +
93 + handleInactiveState() {
94 + this.stopRecording();
95 + this.stopAudioAnalysis();
96 + if (this.waitingTimer) {
97 + clearTimeout(this.waitingTimer);
98 + this.waitingTimer = null;
99 + }
100 + }
101 +
102 + handleListeningState() {
103 + this.stopRecording();
104 + this.audioChunks = [];
105 + this.hasStartedRecording = false;
106 + this.silenceStartTime = null;
107 + this.lastAudioTime = null;
108 + this.messageSent = false;
109 + this.startAudioAnalysis();
110 + }
111 +
112 + handleRecordingState() {
113 + if (!this.hasStartedRecording && this.mediaRecorder.state !== 'recording') {
114 + this.hasStartedRecording = true;
115 + this.mediaRecorder.start(1000);
116 + console.log('Speech started');
117 + }
118 + if (this.waitingTimer) {
119 + clearTimeout(this.waitingTimer);
120 + this.waitingTimer = null;
121 + }
122 + }
123 +
124 + handleWaitingState() {
125 + // Don't stop recording during waiting state
126 + this.waitingTimer = setTimeout(() => {
127 + if (this.status === Status.WAITING) {
128 + this.status = Status.PROCESSING;
129 + }
130 + }, this.options.waitingTimeout);
131 + }
132 +
133 + handleProcessingState() {
134 + this.stopRecording();
135 + this.process();
136 + }
137 +
138 + stopRecording() {
139 + if (this.mediaRecorder?.state === 'recording') {
140 + this.mediaRecorder.stop();
141 + this.hasStartedRecording = false;
142 + }
143 + }
144 +
145 async initialize() {
146 try {
66 -
67 - this.transcriber = await pipeline(`automatic-speech-recognition`, `Xenova/whisper-${this.options.modelSize}.${this.options.language}`);
147 + this.transcriber = await pipeline(
148 + 'automatic-speech-recognition',
149 + `Xenova/whisper-${this.options.modelSize}.${this.options.language}`
150 + );
151
152 const stream = await navigator.mediaDevices.getUserMedia({
153 audio: {
@@ -74,42 +157,51 @@ class MicrophoneInput {
157 }
158 });
159
77 - // Configure MediaRecorder
160 this.mediaRecorder = new MediaRecorder(stream);
79 -
80 - // Handle audio data availability
81 - this.mediaRecorder.ondataavailable = async (event) => {
82 - if (event.data.size > 0) {
161 + this.mediaRecorder.ondataavailable = (event) => {
162 + if (event.data.size > 0 &&
163 + (this.status === Status.RECORDING || this.status === Status.WAITING)) {
164 + if (this.lastChunk) {
165 + this.audioChunks.push(this.lastChunk);
166 + this.lastChunk = null;
167 + }
168 this.audioChunks.push(event.data);
84 - // await this.processAudioChunk(event.data);
169 + console.log('Audio chunk received, total chunks:', this.audioChunks.length);
170 + }
171 + else if (this.status === Status.LISTENING) {
172 + this.lastChunk = event.data;
173 }
174 };
175
88 - // Handle recording stop
89 - this.mediaRecorder.onstop = async () => {
90 - await this.finalizeRecording();
91 - };
92 -
93 - // Set up AudioContext and AnalyserNode for silence detection
94 - this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
95 - this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
96 - this.analyserNode = this.audioContext.createAnalyser();
97 - this.analyserNode.minDecibels = -90;
98 - this.analyserNode.maxDecibels = -10;
99 - this.analyserNode.smoothingTimeConstant = 0.85;
100 -
101 - this.mediaStreamSource.connect(this.analyserNode);
176 + this.setupAudioAnalysis(stream);
177 + return true;
178 } catch (error) {
179 +
180 console.error('Microphone initialization error:', error);
181 toast('Failed to access microphone. Please check permissions.', 'error');
182 + return false;
183 }
184 }
185
108 - startSilenceDetection() {
109 - const dataArray = new Uint8Array(this.analyserNode.fftSize);
110 - const checkSilence = () => {
186 + setupAudioAnalysis(stream) {
187 + this.audioContext = new (window.AudioContext || window.webkitAudioContext)();
188 + this.mediaStreamSource = this.audioContext.createMediaStreamSource(stream);
189 + this.analyserNode = this.audioContext.createAnalyser();
190 + this.analyserNode.fftSize = 2048;
191 + this.analyserNode.minDecibels = -90;
192 + this.analyserNode.maxDecibels = -10;
193 + this.analyserNode.smoothingTimeConstant = 0.85;
194 + this.mediaStreamSource.connect(this.analyserNode);
195 + }
196 +
197 + startAudioAnalysis() {
198 + const analyzeFrame = () => {
199 + if (this.status === Status.INACTIVE) return;
200 +
201 + const dataArray = new Uint8Array(this.analyserNode.fftSize);
202 this.analyserNode.getByteTimeDomainData(dataArray);
203
204 + // Calculate RMS volume
205 let sum = 0;
206 for (let i = 0; i < dataArray.length; i++) {
207 const amplitude = (dataArray[i] - 128) / 128;
@@ -117,147 +209,94 @@ class MicrophoneInput {
209 }
210 const rms = Math.sqrt(sum / dataArray.length);
211
120 - if (rms < this.silenceThreshold) {
121 - if (!this.silenceTimer) {
122 - this.silenceTimer = setTimeout(() => {
123 - if (this.isRecording) {
124 - console.log('Silence detected. Stopping recording.');
125 - this.stopRecording();
126 - microphoneButton.classList.remove('recording');
127 - microphoneButton.classList.remove('mic-pulse');
128 - }
129 - }, this.silenceDuration);
212 + const now = Date.now();
213 +
214 + // Update status based on audio level
215 + if (rms > this.options.silenceThreshold) {
216 + this.lastAudioTime = now;
217 + this.silenceStartTime = null;
218 +
219 + if (this.status === Status.LISTENING || this.status === Status.WAITING) {
220 + this.status = Status.RECORDING;
221 }
131 - } else {
132 - if (this.silenceTimer) {
133 - clearTimeout(this.silenceTimer);
134 - this.silenceTimer = null;
222 + } else if (this.status === Status.RECORDING) {
223 + if (!this.silenceStartTime) {
224 + this.silenceStartTime = now;
225 }
136 - }
226
138 - if (this.isRecording) {
139 - requestAnimationFrame(checkSilence);
227 + const silenceDuration = now - this.silenceStartTime;
228 + if (silenceDuration >= this.options.silenceDuration) {
229 + this.status = Status.WAITING;
230 + }
231 }
232 +
233 + this.analysisFrame = requestAnimationFrame(analyzeFrame);
234 };
235
143 - if (this.isRecording) {
144 - requestAnimationFrame(checkSilence);
145 - }
236 + this.analysisFrame = requestAnimationFrame(analyzeFrame);
237 }
238
148 - startRecording() {
149 - if (this.mediaRecorder && this.audioContext) {
150 - this.isRecording = true;
151 - this.audioChunks = [];
152 - this.messageSent = false;
153 - this.mediaRecorder.start(this.options.chunkDuration);
154 - this.audioContext.resume();
155 - this.startSilenceDetection();
239 + stopAudioAnalysis() {
240 + if (this.analysisFrame) {
241 + cancelAnimationFrame(this.analysisFrame);
242 + this.analysisFrame = null;
243 }
244 }
245
159 - stopRecording() {
160 - if (this.mediaRecorder && this.isRecording) {
161 - this.isRecording = false;
162 - if (!this.isFinalizing) {
163 - this.isFinalizing = true;
164 - this.mediaRecorder.stop();
165 - this.audioContext.suspend();
166 - if (this.silenceTimer) {
167 - clearTimeout(this.silenceTimer);
168 - this.silenceTimer = null;
169 - }
170 - }
246 + async process() {
247 + if (this.audioChunks.length === 0) {
248 + this.status = Status.LISTENING;
249 + return;
250 }
172 - }
251
252 + const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
253 + const audioUrl = URL.createObjectURL(audioBlob);
254
175 - async finalizeRecording() {
176 - if (this.isFinalizing) {
177 - this.isFinalizing = false;
178 -
179 - if (this.audioChunks.length > 0) {
180 -
181 - const audioBlob = new Blob(this.audioChunks, { type: 'audio/wav' });
182 - const audioUrl = URL.createObjectURL(audioBlob);
183 - const samplingRate = 16000; // Adjust as needed for the model
184 - const audioData = await read_audio(audioUrl, samplingRate);
185 - URL.revokeObjectURL(audioUrl);
186 -
187 - // Transcribe the audio
188 - const result = await this.transcriber(audioData);
189 -
190 - if (result.text) {
191 - console.log('Final transcription received:', result.text);
192 - await this.updateCallback(result.text, true);
193 - } else {
194 - console.warn('Final transcription returned empty text.');
195 - }
196 -
197 -
198 - // Release the object URL after use
199 -
200 - // const audioBlob = new Blob(this.audioChunks, { type: 'audio/webm' });
201 - // this.audioChunks = []; // Clear for next recording
202 -
203 - // const reader = new FileReader();
204 - // reader.onloadend = async () => {
205 - // const base64Data = reader.result.split(',')[1];
206 -
207 - // try {
208 - // const response = await fetch('/transcribe', {
209 - // method: 'POST',
210 - // headers: {
211 - // 'Content-Type': 'application/json'
212 - // },
213 - // body: JSON.stringify({
214 - // audio_data: base64Data,
215 - // model_size: this.options.modelSize,
216 - // language: this.options.language,
217 - // is_final: true
218 - // })
219 - // });
220 -
221 - // const result = await response.json();
222 -
223 - // if (result.text) {
224 - // console.log('Final transcription received:', result.text);
225 - // await this.updateCallback(result.text, true);
226 - // } else {
227 - // console.warn('Final transcription returned empty text.');
228 - // }
229 - // } catch (transcribeError) {
230 - // console.error('Final transcription error:', transcribeError);
231 - // toast('Final transcription failed.', 'error');
232 - // } finally {
233 - // // Reset the microphone button state
234 - // microphoneButton.classList.remove('recording');
235 - // microphoneButton.classList.remove('mic-pulse');
236 - // microphoneButton.style.backgroundColor = '';
237 - // }
238 - // };
239 - // reader.readAsDataURL(audioBlob);
255 + try {
256 + const samplingRate = 16000;
257 + const audioData = await read_audio(audioUrl, samplingRate);
258 + const result = await this.transcriber(audioData);
259 + const text = this.filterResult(result.text || "")
260 +
261 + if (text) {
262 + console.log('Transcription:', result.text);
263 + await this.updateCallback(result.text, true);
264 }
265 + } catch (error) {
266 + console.error('Transcription error:', error);
267 + toast('Transcription failed.', 'error');
268 + } finally {
269 + URL.revokeObjectURL(audioUrl);
270 + this.audioChunks = [];
271 + this.status = Status.LISTENING;
272 }
273 }
274 +
275 + filterResult(text) {
276 + text = text.trim()
277 + let ok = false
278 + while (!ok) {
279 + if (!text) break
280 + if (text[0] === '{' && text[text.length - 1] === '}') break
281 + if (text[0] === '(' && text[text.length - 1] === ')') break
282 + if (text[0] === '[' && text[text.length - 1] === ']') break
283 + ok = true
284 + }
285 + if (ok) return text
286 + else console.log(`Discarding transcription: ${text}`)
287 + }
288 }
289
245 -export default MicrophoneInput;
290
247 -async function initializeMicrophoneInput() {
248 - console.log('Initializing microphone input');
291
292 +// Initialize and handle click events
293 +async function initializeMicrophoneInput() {
294 microphoneInput = new MicrophoneInput(
295 async (text, isFinal) => {
296 if (isFinal) {
253 - console.log('Final transcription callback received:', text);
254 - updateChatInput(text)
255 - // chatInput.value = text;
256 - // adjustTextareaHeight();
257 -
297 + updateChatInput(text);
298 if (!microphoneInput.messageSent) {
299 microphoneInput.messageSent = true;
260 - console.log('Sending message');
300 await sendMessage();
301 }
302 }
@@ -265,42 +304,37 @@ async function initializeMicrophoneInput() {
304 {
305 modelSize: 'tiny',
306 language: 'en',
268 - silenceThreshold: 0.07, // Adjust as needed
269 - silenceDuration: 2000, // Adjust as needed
270 - onError: (error) => {
271 - console.error('Microphone input error:', error);
272 - toast('Microphone error: ' + error.message, 'error');
273 - // Reset recording state
274 - if (microphoneButton.classList.contains('recording')) {
275 - microphoneButton.classList.remove('recording');
276 - }
277 - }
307 + silenceThreshold: 0.07,
308 + silenceDuration: 1000,
309 + waitingTimeout: 1500
310 }
311 );
312 + microphoneInput.status = Status.ACTIVATING;
313
281 - await microphoneInput.initialize();
314 + return await microphoneInput.initialize();
315 }
316
317 +microphoneButton.addEventListener('click', async () => {
318 + if (isProcessingClick) return;
319 + isProcessingClick = true;
320
285 -function toggleRecording() {
286 - console.log('toggleRecording called, isRecording:', microphoneInput.isRecording);
321 + const hasPermission = await requestMicrophonePermission();
322 + if (!hasPermission) return;
323
288 - if (microphoneInput.isRecording) {
289 - microphoneInput.stopRecording();
290 - microphoneButton.classList.remove('recording');
291 - // Add pulsing animation class
292 - microphoneButton.classList.remove('mic-pulse');
293 - } else {
294 - microphoneInput.startRecording();
295 - microphoneButton.classList.add('recording');
296 - // Add pulsing animation class
297 - microphoneButton.classList.add('mic-pulse');
298 - }
324 + try {
325 + if (!microphoneInput && !await initializeMicrophoneInput()) {
326 + return;
327 + }
328
300 - // Add visual feedback
301 - microphoneButton.style.backgroundColor = microphoneInput.isRecording ? '#ff4444' : '';
302 - console.log('New recording state:', microphoneInput.isRecording);
303 -}
329 + // Simply toggle between INACTIVE and LISTENING states
330 + microphoneInput.status =
331 + (microphoneInput.status === Status.INACTIVE || microphoneInput.status === Status.ACTIVATING) ? Status.LISTENING : Status.INACTIVE;
332 + } finally {
333 + setTimeout(() => {
334 + isProcessingClick = false;
335 + }, 300);
336 + }
337 +});
338
339 // Some error handling for microphone input
340 async function requestMicrophonePermission() {
@@ -313,25 +347,3 @@ async function requestMicrophonePermission() {
347 return false;
348 }
349 }
316 -// microphoneButton click event listener modifier
317 -microphoneButton.addEventListener('click', async () => {
318 - console.log('Microphone button clicked');
319 - if (isProcessingClick) {
320 - console.log('Click already being processed, ignoring');
321 - return;
322 - }
323 - isProcessingClick = true;
324 -
325 - const hasPermission = await requestMicrophonePermission();
326 - if (!hasPermission) return;
327 -
328 - if (!microphoneInput) {
329 - await initializeMicrophoneInput();
330 - }
331 -
332 - await toggleRecording();
333 -
334 - setTimeout(() => {
335 - isProcessingClick = false;
336 - }, 300); // Add a 300ms delay before allowing another click
337 -});
webui/test2.html new
+122
@@ -0,0 +1,122 @@
1 +<!DOCTYPE html>
2 +<html lang="en">
3 +<head>
4 + <meta charset="UTF-8">
5 + <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1">
6 + <title>Agent Zero</title>
7 +
8 + <script type="module">
9 + import { pipeline, read_audio } from './transformers@3.0.2.js';
10 +
11 + let transcriber;
12 + let mediaRecorder;
13 + let isRecording = false;
14 + let audioChunks = [];
15 + let chunks_to_process = [{ tokens: [], finalized: false }];
16 +
17 + async function initTranscriber() {
18 + try {
19 + transcriber = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en');
20 + } catch (error) {
21 + console.error("Failed to initialize transcriber:", error);
22 + }
23 + }
24 +
25 + async function toggleRecording() {
26 + if (isRecording) {
27 + stopRecording();
28 + } else {
29 + startRecording();
30 + }
31 + }
32 +
33 + async function startRecording() {
34 + isRecording = true;
35 + document.getElementById("micButton").innerText = "Stop Recording";
36 +
37 + const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
38 + mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/webm;codecs=opus' });
39 +
40 + // Handle data as it’s available
41 + mediaRecorder.ondataavailable = (event) => {
42 + audioChunks.push(event.data);
43 +
44 + // Process chunks every 3 seconds
45 + if (audioChunks.length >= 3) {
46 + processAudioChunks();
47 + }
48 + };
49 +
50 + // Capture small chunks every second
51 + mediaRecorder.start(1000);
52 + }
53 +
54 + async function processAudioChunks() {
55 + const audioBlob = new Blob(audioChunks, { type: 'audio/webm' });
56 + const audioUrl = URL.createObjectURL(audioBlob);
57 +
58 + try {
59 + const audioData = await read_audio(audioUrl, 16000);
60 +
61 + if (transcriber) {
62 + await transcriber(audioData, {
63 + callback_function: mergeChunksCallback,
64 + chunk_callback: processChunkCallback,
65 + return_timestamps: true,
66 + force_full_sequences: false,
67 + });
68 + } else {
69 + console.warn("Transcriber is not ready yet.");
70 + }
71 + } catch (error) {
72 + console.error("Error during transcription:", error);
73 + }
74 +
75 + // Cleanup and reset buffer
76 + URL.revokeObjectURL(audioUrl);
77 + audioChunks = [];
78 + }
79 +
80 + function processChunkCallback(chunk) {
81 + let lastChunk = chunks_to_process[chunks_to_process.length - 1];
82 + Object.assign(lastChunk, chunk);
83 + lastChunk.finalized = true;
84 +
85 + if (!chunk.is_last) {
86 + chunks_to_process.push({ tokens: [], finalized: false });
87 + }
88 + }
89 +
90 + function mergeChunksCallback(item) {
91 + let lastChunk = chunks_to_process[chunks_to_process.length - 1];
92 + lastChunk.tokens = [...item[0].output_token_ids];
93 +
94 + // Merge text chunks and update transcript
95 + const mergedText = transcriber.tokenizer._decode_asr(chunks_to_process, {
96 + time_precision: 1.0 / transcriber.processor.feature_extractor.config.chunk_length,
97 + return_timestamps: true,
98 + force_full_sequences: false,
99 + });
100 +
101 + document.getElementById("transcript").innerText += mergedText + ' ';
102 + }
103 +
104 + function stopRecording() {
105 + isRecording = false;
106 + document.getElementById("micButton").innerText = "Start Recording";
107 + if (mediaRecorder && mediaRecorder.state !== 'inactive') {
108 + mediaRecorder.stop();
109 + }
110 + }
111 +
112 + window.toggleRecording = toggleRecording;
113 + window.onload = initTranscriber;
114 + </script>
115 +</head>
116 +
117 +<body>
118 + <h1>Agent Zero Speech Transcription</h1>
119 + <button id="micButton" onclick="toggleRecording()">Start Recording</button>
120 + <p id="transcript">Transcript will appear here...</p>
121 +</body>
122 +</html>