The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 04:45 UTC
| 1 | import { |
| 2 | VoiceTranscriptionError, |
| 3 | type PreparedVoiceTranscription, |
| 4 | type VoiceRecorder, |
| 5 | type VoiceStreamingSession, |
| 6 | type VoiceInputErrorCode, |
| 7 | type VoiceTranscriptionOptions, |
| 8 | } from "@t3tools/client-runtime/voice-input"; |
| 9 | import * as Schema from "effect/Schema"; |
| 10 | import { requestMicrophone } from "./requestMicrophone"; |
| 11 | import workletUrl from "./pcmWorklet.ts?worker&url"; |
| 12 | import { msg } from "gt-react"; |
| 13 | import { m } from "../i18n/messages"; |
| 14 | |
| 15 | const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError); |
| 16 | |
| 17 | type PreparedBrowserTranscription = PreparedVoiceTranscription & { |
| 18 | readonly streaming?: VoiceStreamingSession & { readonly feed: (pcm: Float32Array) => void }; |
| 19 | }; |
| 20 | |
| 21 | export function createBrowserPcmCapture( |
| 22 | input: { |
| 23 | readonly getMicrophoneId: () => string | Promise<string>; |
| 24 | readonly onLevel: (level: number) => void; |
| 25 | readonly onDurationLimit: () => void; |
| 26 | readonly onError: (message: string, code?: VoiceInputErrorCode) => void; |
| 27 | }, |
| 28 | prepare: ( |
| 29 | options: VoiceTranscriptionOptions, |
| 30 | canStream: () => boolean, |
| 31 | ) => Promise<PreparedBrowserTranscription>, |
| 32 | ) { |
| 33 | let stream: MediaStream | undefined; |
| 34 | let recordingUri: string | null = null; |
| 35 | let chunks: Float32Array<ArrayBuffer>[] = []; |
| 36 | let audioContext: AudioContext | undefined; |
| 37 | let worklet: AudioWorkletNode | undefined; |
| 38 | let durationTimer: ReturnType<typeof setTimeout> | undefined; |
| 39 | let signal: AbortSignal | undefined; |
| 40 | let stopAcceptingAudio: (() => void) | undefined; |
| 41 | let feed: ((pcm: Float32Array) => void) | undefined; |
| 42 | let stopped: |
| 43 | | { readonly resolve: () => void; readonly reject: (error: Error) => void } |
| 44 | | undefined; |
| 45 | |
| 46 | const cleanupCapture = () => { |
| 47 | if (durationTimer) clearTimeout(durationTimer); |
| 48 | durationTimer = undefined; |
| 49 | worklet?.disconnect(); |
| 50 | worklet?.port.close(); |
| 51 | worklet = undefined; |
| 52 | stream?.getTracks().forEach((track) => track.stop()); |
| 53 | stream = undefined; |
| 54 | void audioContext?.close(); |
| 55 | audioContext = undefined; |
| 56 | stopped?.resolve(); |
| 57 | stopped = undefined; |
| 58 | chunks = []; |
| 59 | stopAcceptingAudio?.(); |
| 60 | stopAcceptingAudio = undefined; |
| 61 | feed = undefined; |
| 62 | input.onLevel(0); |
| 63 | }; |
| 64 | |
| 65 | const recorder: VoiceRecorder = { |
| 66 | get uri() { |
| 67 | return recordingUri; |
| 68 | }, |
| 69 | prepareToRecordAsync: async () => { |
| 70 | const captureSignal = signal; |
| 71 | if (!captureSignal) throw new Error(m(msg("Voice transcription is not prepared."))); |
| 72 | captureSignal.throwIfAborted(); |
| 73 | if (recordingUri) URL.revokeObjectURL(recordingUri); |
| 74 | recordingUri = null; |
| 75 | chunks = []; |
| 76 | const microphoneId = await input.getMicrophoneId(); |
| 77 | captureSignal.throwIfAborted(); |
| 78 | const captured = await new Promise<MediaStream>((resolve, reject) => { |
| 79 | const onAbort = () => reject(captureSignal.reason); |
| 80 | const permission = requestMicrophone(microphoneId, captureSignal); |
| 81 | captureSignal.addEventListener("abort", onAbort, { once: true }); |
| 82 | void permission.then( |
| 83 | (media) => { |
| 84 | captureSignal.removeEventListener("abort", onAbort); |
| 85 | if (captureSignal.aborted) media.getTracks().forEach((track) => track.stop()); |
| 86 | else resolve(media); |
| 87 | }, |
| 88 | (error: unknown) => { |
| 89 | captureSignal.removeEventListener("abort", onAbort); |
| 90 | reject(error); |
| 91 | }, |
| 92 | ); |
| 93 | }); |
| 94 | if (captureSignal.aborted) { |
| 95 | captured.getTracks().forEach((track) => track.stop()); |
| 96 | captureSignal.throwIfAborted(); |
| 97 | } |
| 98 | stream = captured; |
| 99 | try { |
| 100 | const context = new AudioContext(); |
| 101 | audioContext = context; |
| 102 | await context.audioWorklet.addModule(workletUrl); |
| 103 | captureSignal.throwIfAborted(); |
| 104 | const node = new AudioWorkletNode(context, "t3-pcm-capture", { channelCount: 1 }); |
| 105 | worklet = node; |
| 106 | node.port.onmessage = ({ |
| 107 | data, |
| 108 | }: MessageEvent<Float32Array<ArrayBuffer> | number | "stopped" | "limit">) => { |
| 109 | if (captureSignal.aborted) return; |
| 110 | if (data === "stopped") { |
| 111 | stopped?.resolve(); |
| 112 | return; |
| 113 | } |
| 114 | if (data === "limit") { |
| 115 | input.onDurationLimit(); |
| 116 | return; |
| 117 | } |
| 118 | if (typeof data === "number") { |
| 119 | input.onLevel(data); |
| 120 | return; |
| 121 | } |
| 122 | if (feed) feed(data); |
| 123 | else chunks.push(data); |
| 124 | }; |
| 125 | node.onprocessorerror = () => { |
| 126 | stopped?.reject(new Error(m(msg("Microphone processing failed.")))); |
| 127 | input.onError(m(msg("Microphone processing failed.")), "microphone-failed"); |
| 128 | }; |
| 129 | for (const track of captured.getTracks()) |
| 130 | track.addEventListener( |
| 131 | "ended", |
| 132 | () => input.onError(m(msg("The microphone disconnected.")), "microphone-disconnected"), |
| 133 | { |
| 134 | once: true, |
| 135 | }, |
| 136 | ); |
| 137 | context.createMediaStreamSource(captured).connect(node); |
| 138 | // The processor emits silence, keeping the graph active without microphone playback. |
| 139 | node.connect(context.destination); |
| 140 | await context.resume(); |
| 141 | captureSignal.throwIfAborted(); |
| 142 | } catch (error) { |
| 143 | cleanupCapture(); |
| 144 | throw error; |
| 145 | } |
| 146 | }, |
| 147 | record: ({ forDuration }) => { |
| 148 | // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin |
| 149 | worklet?.port.postMessage("start"); |
| 150 | durationTimer = setTimeout(input.onDurationLimit, forDuration * 1_000); |
| 151 | }, |
| 152 | stop: async () => { |
| 153 | if (!worklet) return; |
| 154 | if (signal?.aborted) { |
| 155 | cleanupCapture(); |
| 156 | return; |
| 157 | } |
| 158 | const pending = new Promise<void>((resolve, reject) => { |
| 159 | stopped = { resolve, reject }; |
| 160 | }); |
| 161 | const timeout = setTimeout( |
| 162 | () => stopped?.reject(new Error(m(msg("Microphone capture did not stop.")))), |
| 163 | 5_000, |
| 164 | ); |
| 165 | // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin |
| 166 | worklet.port.postMessage("stop"); |
| 167 | try { |
| 168 | await pending; |
| 169 | if (!feed && chunks.length) { |
| 170 | const blob = new Blob(chunks, { type: "application/octet-stream" }); |
| 171 | recordingUri = URL.createObjectURL(blob); |
| 172 | } |
| 173 | } finally { |
| 174 | clearTimeout(timeout); |
| 175 | cleanupCapture(); |
| 176 | } |
| 177 | }, |
| 178 | }; |
| 179 | |
| 180 | return { |
| 181 | recorder, |
| 182 | cancelRecording: cleanupCapture, |
| 183 | deleteRecording: (uri: string) => { |
| 184 | URL.revokeObjectURL(uri); |
| 185 | if (recordingUri === uri) recordingUri = null; |
| 186 | }, |
| 187 | transcriber: { |
| 188 | prepare: async (options: VoiceTranscriptionOptions): Promise<PreparedVoiceTranscription> => { |
| 189 | signal = options.signal; |
| 190 | let acceptingAudio = true; |
| 191 | stopAcceptingAudio = () => { |
| 192 | acceptingAudio = false; |
| 193 | }; |
| 194 | const canStream = () => acceptingAudio && !options.signal.aborted; |
| 195 | let streaming: VoiceStreamingSession | undefined; |
| 196 | let destination: PreparedBrowserTranscription | undefined; |
| 197 | const pending = prepare(options, canStream) |
| 198 | .then((prepared) => { |
| 199 | options.signal.throwIfAborted(); |
| 200 | destination = prepared; |
| 201 | if (canStream() && prepared.streaming) { |
| 202 | streaming = prepared.streaming; |
| 203 | feed = prepared.streaming.feed; |
| 204 | for (const chunk of chunks) feed(chunk); |
| 205 | chunks = []; |
| 206 | } |
| 207 | return prepared; |
| 208 | }) |
| 209 | .catch((cause: unknown) => { |
| 210 | const error = new VoiceTranscriptionError({ |
| 211 | code: "preparation-failed", |
| 212 | message: isVoiceTranscriptionError(cause) |
| 213 | ? cause.message |
| 214 | : m(msg("Could not prepare voice transcription.")), |
| 215 | cause, |
| 216 | }); |
| 217 | if (canStream()) input.onError(error.message); |
| 218 | throw error; |
| 219 | }); |
| 220 | void pending.catch(() => {}); |
| 221 | return { |
| 222 | get locale() { |
| 223 | return destination?.locale ?? "auto"; |
| 224 | }, |
| 225 | get streaming() { |
| 226 | return streaming; |
| 227 | }, |
| 228 | async transcribe(uri, transcriptionOptions) { |
| 229 | const prepared = await pending; |
| 230 | return prepared.transcribe(uri, transcriptionOptions); |
| 231 | }, |
| 232 | }; |
| 233 | }, |
| 234 | }, |
| 235 | }; |
| 236 | } |