@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 218 lines 8.59 KB
Raw
1 import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech";
2 import {
3 requireCloudSpeechKey,
4 transcribeCloudPcm,
5 getEnvironmentSpeechStatus,
6 getEnvironmentSpeechStreamUrl,
7 openSpeechStream,
8 throwIfVoiceTranscriptionAborted,
9 transcribeEnvironmentPcm,
10 type VoiceInputErrorCode,
11 VoiceTranscriptionError,
12 type VoiceTranscriptionOptions,
13 } from "@t3tools/client-runtime/voice-input";
14 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
15 import type {
16 SpeechTranscriptionOptions,
17 SpeechStreamText,
18 EnvironmentSpeechStatus,
19 } from "@t3tools/contracts";
20 import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts";
21 import * as Schema from "effect/Schema";
22
23 import { readAppleSpeechLocales, resolveAppleSpeechLocale } from "./appleSpeech";
24 import { createBrowserPcmCapture } from "./browserPcmCapture";
25 import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
26 import { msg } from "gt-react";
27 import { m } from "../i18n/messages";
28
29 const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
30
31 export function createBrowserVoiceInputPlatform(input: {
32 readonly prepared: PreparedConnection;
33 readonly cloudProvider?: CloudSpeechProvider | undefined;
34 /** The exact OpenRouter model the grant names; required with that provider. */
35 readonly cloudModel?: CloudSpeechModelId | undefined;
36 readonly hasAudioConsent: () => boolean;
37 readonly getTranscriptionOptions: () => SpeechTranscriptionOptions;
38 readonly getStatusListener?: () => ((status: EnvironmentSpeechStatus) => void) | undefined;
39 readonly getStreamingEnabled: () => boolean;
40 readonly getMicrophoneId: () => string | Promise<string>;
41 readonly onLevel: (level: number) => void;
42 readonly onDurationLimit: () => void;
43 readonly onText: (text: SpeechStreamText) => void;
44 readonly onError: (message: string, code?: VoiceInputErrorCode) => void;
45 }) {
46 return createBrowserPcmCapture(input, async (options, canStream) => {
47 const onStatus = input.getStatusListener?.();
48 const recordingOptions = input.getTranscriptionOptions();
49 const streamingEnabled = input.getStreamingEnabled();
50 const transcribeRecording = async (
51 uri: string,
52 { signal: transcriptionSignal }: VoiceTranscriptionOptions,
53 ) => {
54 try {
55 const response = await fetch(uri, { signal: transcriptionSignal });
56 const pcm = new Uint8Array(await response.arrayBuffer());
57 const result = await runEnvironmentSpeech(
58 input.cloudProvider
59 ? transcribeCloudPcm(
60 input.prepared,
61 input.cloudProvider,
62 pcm,
63 recordingOptions,
64 input.hasAudioConsent,
65 input.cloudModel,
66 )
67 : transcribeEnvironmentPcm(
68 input.prepared,
69 pcm,
70 recordingOptions,
71 input.hasAudioConsent,
72 ),
73 { signal: transcriptionSignal },
74 );
75 throwIfVoiceTranscriptionAborted(transcriptionSignal);
76 return result.text;
77 } catch (cause) {
78 throwIfVoiceTranscriptionAborted(transcriptionSignal);
79 if (isVoiceTranscriptionError(cause)) throw cause;
80 throw new VoiceTranscriptionError({
81 code: "transcription-failed",
82 message: m(msg("Voice transcription failed. Check the destination in Settings → Voice.")),
83 cause,
84 });
85 }
86 };
87 if (input.cloudProvider) {
88 if (input.cloudProvider === "openrouter" && !input.cloudModel)
89 throw new VoiceTranscriptionError({
90 code: "unavailable",
91 message: m(msg("Choose an OpenRouter model in Settings → Voice.")),
92 });
93 await runEnvironmentSpeech(
94 requireCloudSpeechKey(input.prepared, input.cloudProvider),
95 options,
96 );
97 throwIfVoiceTranscriptionAborted(options.signal);
98 return {
99 locale: effectiveSpeechLanguage(cloudSpeechLanguages, recordingOptions.speechLanguage),
100 transcribe: transcribeRecording,
101 };
102 }
103 const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
104 throwIfVoiceTranscriptionAborted(options.signal);
105 onStatus?.(status);
106 throwIfVoiceTranscriptionAborted(options.signal);
107 if (!status.supported)
108 throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
109 if (status.state === "missing-model")
110 throw new VoiceTranscriptionError({
111 code: "unavailable",
112 message: m(msg("Could not start voice input")),
113 });
114 const language = status.languages
115 ? effectiveSpeechLanguage(
116 {
117 languages: status.languages,
118 supportsLanguageDetection: status.supportsLanguageDetection ?? false,
119 },
120 recordingOptions.speechLanguage,
121 )
122 : "auto";
123 const nativeStreaming = status.supportsStreaming && streamingEnabled;
124 const batchLocale =
125 status.supportsTranslation && recordingOptions.speechTranslateToEnglish ? "en" : language;
126 const batch = { locale: batchLocale, transcribe: transcribeRecording };
127 if (
128 !canStream() ||
129 (status.supportsStreaming ? !nativeStreaming : !status.supportsBufferedStreaming)
130 ) {
131 return batch;
132 }
133 const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options);
134 if (!canStream()) return batch;
135 return {
136 locale: nativeStreaming ? language : batchLocale,
137 transcribe: transcribeRecording,
138 streaming: openSpeechStream({
139 url,
140 canSend: input.hasAudioConsent,
141 options: recordingOptions,
142 signal: options.signal,
143 onText: input.onText,
144 onError: (error) => input.onError(error.message),
145 }),
146 };
147 });
148 }
149
150 async function waitForAppleSpeech<T>(request: Promise<T>, signal: AbortSignal): Promise<T> {
151 const aborted = Promise.withResolvers<never>();
152 const onAbort = () => aborted.reject(signal.reason);
153 signal.addEventListener("abort", onAbort, { once: true });
154 if (signal.aborted) onAbort();
155 try {
156 return await Promise.race([request, aborted.promise]);
157 } finally {
158 signal.removeEventListener("abort", onAbort);
159 }
160 }
161
162 /** Records like the environment path, then transcribes in the desktop main process; audio stays on this Mac. */
163 export function createAppleSpeechVoiceInputPlatform(input: {
164 readonly hasAudioConsent: () => boolean;
165 readonly getTranscriptionOptions: () => SpeechTranscriptionOptions;
166 readonly getMicrophoneId: () => string | Promise<string>;
167 readonly onLevel: (level: number) => void;
168 readonly onDurationLimit: () => void;
169 readonly onError: (message: string, code?: VoiceInputErrorCode) => void;
170 }) {
171 return createBrowserPcmCapture(input, async ({ signal }) => {
172 const appleSpeech = window.desktopBridge?.appleSpeech;
173 const { speechLanguage } = input.getTranscriptionOptions();
174 // The packaged app pins `navigator.language` to en-US, so "auto" asks the OS.
175 const system = window.desktopBridge?.getPreferredSystemLanguages?.() ?? [];
176 if (!appleSpeech) throw new Error("Apple Speech is not available.");
177 signal.throwIfAborted();
178 const prepared = await waitForAppleSpeech(
179 appleSpeech.prepare(
180 resolveAppleSpeechLocale(readAppleSpeechLocales(), speechLanguage, system) ??
181 navigator.language,
182 ),
183 signal,
184 );
185 throwIfVoiceTranscriptionAborted(signal);
186 if (!prepared.ok) {
187 if (prepared.code === "failed") throw new Error("Apple Speech could not prepare.");
188 throw new VoiceTranscriptionError({
189 code: prepared.code,
190 message:
191 prepared.code === "unsupported-locale"
192 ? m(msg("Apple Speech does not support this language."))
193 : m(msg("Apple Speech is not available on this Mac.")),
194 });
195 }
196 const locale = prepared.value;
197 return {
198 locale,
199 transcribe: async (uri, { signal: transcriptionSignal }) => {
200 const response = await fetch(uri, { signal: transcriptionSignal });
201 const pcm = new Uint8Array(await response.arrayBuffer());
202 throwIfVoiceTranscriptionAborted(transcriptionSignal);
203 if (!input.hasAudioConsent())
204 throw new VoiceTranscriptionError({
205 code: "cancelled",
206 message: m(msg("Voice transcription was cancelled.")),
207 });
208 const result = await waitForAppleSpeech(
209 appleSpeech.transcribe({ locale, pcm }),
210 transcriptionSignal,
211 );
212 throwIfVoiceTranscriptionAborted(transcriptionSignal);
213 if (!result.ok) throw new Error("Apple Speech could not transcribe.");
214 return result.value;
215 },
216 };
217 });
218 }