@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 236 lines 8.19 KB
Raw
1 import {
2 VoiceTranscriptionError,
3 type PreparedVoiceTranscription,
4 type VoiceRecorder,
5 type VoiceStreamingSession,
6 type VoiceInputErrorCode,
7 type VoiceTranscriptionOptions,
8 } from "@t3tools/client-runtime/voice-input";
9 import * as Schema from "effect/Schema";
10 import { requestMicrophone } from "./requestMicrophone";
11 import workletUrl from "./pcmWorklet.ts?worker&url";
12 import { msg } from "gt-react";
13 import { m } from "../i18n/messages";
14
15 const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
16
17 type PreparedBrowserTranscription = PreparedVoiceTranscription & {
18 readonly streaming?: VoiceStreamingSession & { readonly feed: (pcm: Float32Array) => void };
19 };
20
21 export function createBrowserPcmCapture(
22 input: {
23 readonly getMicrophoneId: () => string | Promise<string>;
24 readonly onLevel: (level: number) => void;
25 readonly onDurationLimit: () => void;
26 readonly onError: (message: string, code?: VoiceInputErrorCode) => void;
27 },
28 prepare: (
29 options: VoiceTranscriptionOptions,
30 canStream: () => boolean,
31 ) => Promise<PreparedBrowserTranscription>,
32 ) {
33 let stream: MediaStream | undefined;
34 let recordingUri: string | null = null;
35 let chunks: Float32Array<ArrayBuffer>[] = [];
36 let audioContext: AudioContext | undefined;
37 let worklet: AudioWorkletNode | undefined;
38 let durationTimer: ReturnType<typeof setTimeout> | undefined;
39 let signal: AbortSignal | undefined;
40 let stopAcceptingAudio: (() => void) | undefined;
41 let feed: ((pcm: Float32Array) => void) | undefined;
42 let stopped:
43 | { readonly resolve: () => void; readonly reject: (error: Error) => void }
44 | undefined;
45
46 const cleanupCapture = () => {
47 if (durationTimer) clearTimeout(durationTimer);
48 durationTimer = undefined;
49 worklet?.disconnect();
50 worklet?.port.close();
51 worklet = undefined;
52 stream?.getTracks().forEach((track) => track.stop());
53 stream = undefined;
54 void audioContext?.close();
55 audioContext = undefined;
56 stopped?.resolve();
57 stopped = undefined;
58 chunks = [];
59 stopAcceptingAudio?.();
60 stopAcceptingAudio = undefined;
61 feed = undefined;
62 input.onLevel(0);
63 };
64
65 const recorder: VoiceRecorder = {
66 get uri() {
67 return recordingUri;
68 },
69 prepareToRecordAsync: async () => {
70 const captureSignal = signal;
71 if (!captureSignal) throw new Error(m(msg("Voice transcription is not prepared.")));
72 captureSignal.throwIfAborted();
73 if (recordingUri) URL.revokeObjectURL(recordingUri);
74 recordingUri = null;
75 chunks = [];
76 const microphoneId = await input.getMicrophoneId();
77 captureSignal.throwIfAborted();
78 const captured = await new Promise<MediaStream>((resolve, reject) => {
79 const onAbort = () => reject(captureSignal.reason);
80 const permission = requestMicrophone(microphoneId, captureSignal);
81 captureSignal.addEventListener("abort", onAbort, { once: true });
82 void permission.then(
83 (media) => {
84 captureSignal.removeEventListener("abort", onAbort);
85 if (captureSignal.aborted) media.getTracks().forEach((track) => track.stop());
86 else resolve(media);
87 },
88 (error: unknown) => {
89 captureSignal.removeEventListener("abort", onAbort);
90 reject(error);
91 },
92 );
93 });
94 if (captureSignal.aborted) {
95 captured.getTracks().forEach((track) => track.stop());
96 captureSignal.throwIfAborted();
97 }
98 stream = captured;
99 try {
100 const context = new AudioContext();
101 audioContext = context;
102 await context.audioWorklet.addModule(workletUrl);
103 captureSignal.throwIfAborted();
104 const node = new AudioWorkletNode(context, "t3-pcm-capture", { channelCount: 1 });
105 worklet = node;
106 node.port.onmessage = ({
107 data,
108 }: MessageEvent<Float32Array<ArrayBuffer> | number | "stopped" | "limit">) => {
109 if (captureSignal.aborted) return;
110 if (data === "stopped") {
111 stopped?.resolve();
112 return;
113 }
114 if (data === "limit") {
115 input.onDurationLimit();
116 return;
117 }
118 if (typeof data === "number") {
119 input.onLevel(data);
120 return;
121 }
122 if (feed) feed(data);
123 else chunks.push(data);
124 };
125 node.onprocessorerror = () => {
126 stopped?.reject(new Error(m(msg("Microphone processing failed."))));
127 input.onError(m(msg("Microphone processing failed.")), "microphone-failed");
128 };
129 for (const track of captured.getTracks())
130 track.addEventListener(
131 "ended",
132 () => input.onError(m(msg("The microphone disconnected.")), "microphone-disconnected"),
133 {
134 once: true,
135 },
136 );
137 context.createMediaStreamSource(captured).connect(node);
138 // The processor emits silence, keeping the graph active without microphone playback.
139 node.connect(context.destination);
140 await context.resume();
141 captureSignal.throwIfAborted();
142 } catch (error) {
143 cleanupCapture();
144 throw error;
145 }
146 },
147 record: ({ forDuration }) => {
148 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
149 worklet?.port.postMessage("start");
150 durationTimer = setTimeout(input.onDurationLimit, forDuration * 1_000);
151 },
152 stop: async () => {
153 if (!worklet) return;
154 if (signal?.aborted) {
155 cleanupCapture();
156 return;
157 }
158 const pending = new Promise<void>((resolve, reject) => {
159 stopped = { resolve, reject };
160 });
161 const timeout = setTimeout(
162 () => stopped?.reject(new Error(m(msg("Microphone capture did not stop.")))),
163 5_000,
164 );
165 // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
166 worklet.port.postMessage("stop");
167 try {
168 await pending;
169 if (!feed && chunks.length) {
170 const blob = new Blob(chunks, { type: "application/octet-stream" });
171 recordingUri = URL.createObjectURL(blob);
172 }
173 } finally {
174 clearTimeout(timeout);
175 cleanupCapture();
176 }
177 },
178 };
179
180 return {
181 recorder,
182 cancelRecording: cleanupCapture,
183 deleteRecording: (uri: string) => {
184 URL.revokeObjectURL(uri);
185 if (recordingUri === uri) recordingUri = null;
186 },
187 transcriber: {
188 prepare: async (options: VoiceTranscriptionOptions): Promise<PreparedVoiceTranscription> => {
189 signal = options.signal;
190 let acceptingAudio = true;
191 stopAcceptingAudio = () => {
192 acceptingAudio = false;
193 };
194 const canStream = () => acceptingAudio && !options.signal.aborted;
195 let streaming: VoiceStreamingSession | undefined;
196 let destination: PreparedBrowserTranscription | undefined;
197 const pending = prepare(options, canStream)
198 .then((prepared) => {
199 options.signal.throwIfAborted();
200 destination = prepared;
201 if (canStream() && prepared.streaming) {
202 streaming = prepared.streaming;
203 feed = prepared.streaming.feed;
204 for (const chunk of chunks) feed(chunk);
205 chunks = [];
206 }
207 return prepared;
208 })
209 .catch((cause: unknown) => {
210 const error = new VoiceTranscriptionError({
211 code: "preparation-failed",
212 message: isVoiceTranscriptionError(cause)
213 ? cause.message
214 : m(msg("Could not prepare voice transcription.")),
215 cause,
216 });
217 if (canStream()) input.onError(error.message);
218 throw error;
219 });
220 void pending.catch(() => {});
221 return {
222 get locale() {
223 return destination?.locale ?? "auto";
224 },
225 get streaming() {
226 return streaming;
227 },
228 async transcribe(uri, transcriptionOptions) {
229 const prepared = await pending;
230 return prepared.transcribe(uri, transcriptionOptions);
231 },
232 };
233 },
234 },
235 };
236 }