@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 261 lines 10.7 KB
Raw
1 import {
2 CloudSpeechApiKey,
3 CloudSpeechError,
4 CloudSpeechModel,
5 CloudSpeechModelId,
6 SettingsSecretValue,
7 isSecretReference,
8 type SecretReference,
9 type ServerSettingsError,
10 type CloudSpeechKeys,
11 type CloudSpeechProvider,
12 EnvironmentSpeechTranscriptionResult,
13 type SpeechTranscriptionOptions,
14 } from "@t3tools/contracts";
15 import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
16 import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech";
17 import * as Context from "effect/Context";
18 import * as Effect from "effect/Effect";
19 import * as Exit from "effect/Exit";
20 import * as Layer from "effect/Layer";
21 import * as Redacted from "effect/Redacted";
22 import * as Schema from "effect/Schema";
23 import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
24 import * as ServerSettings from "../serverSettings.ts";
25 import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts";
26 import { applySpeechAliases, speechRecognitionWords } from "./customWords.ts";
27 import { removeSpeechFillerWords } from "./fillerWords.ts";
28 import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts";
29
30 const endpoints = {
31 openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" },
32 xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" },
33 openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" },
34 };
35 const OPENROUTER_CATALOG_URL =
36 "https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true";
37 const isApiKey = Schema.is(CloudSpeechApiKey);
38 const isModelId = Schema.is(CloudSpeechModelId);
39 const sameCredential = Schema.toEquivalence(SettingsSecretValue);
40 const OpenRouterCatalog = Schema.Struct({
41 data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optionalKey(Schema.String) })),
42 });
43
44 function wave(samples: Float32Array) {
45 const buffer = new ArrayBuffer(44 + samples.length * 2);
46 const view = new DataView(buffer);
47 const tag = (at: number, value: string) =>
48 new Uint8Array(buffer, at, 4).set(new TextEncoder().encode(value));
49 tag(0, "RIFF");
50 view.setUint32(4, buffer.byteLength - 8, true);
51 tag(8, "WAVE");
52 tag(12, "fmt ");
53 view.setUint32(16, 16, true);
54 view.setUint16(20, 1, true);
55 view.setUint16(22, 1, true);
56 view.setUint32(24, 16_000, true);
57 view.setUint32(28, 32_000, true);
58 view.setUint16(32, 2, true);
59 view.setUint16(34, 16, true);
60 tag(36, "data");
61 view.setUint32(40, buffer.byteLength - 44, true);
62 for (let index = 0; index < samples.length; index++) {
63 const sample = samples[index]!;
64 view.setInt16(44 + index * 2, Math.round(sample * (sample < 0 ? 32768 : 32767)), true);
65 }
66 return buffer;
67 }
68
69 function audioBody(
70 provider: "openai" | "xai",
71 audio: ArrayBuffer,
72 language: string,
73 options: SpeechTranscriptionOptions,
74 ) {
75 const form = new FormData();
76 form.set("model", endpoints[provider].model);
77 if (language !== "auto") form.set(provider === "openai" ? "languages[]" : "language", language);
78 for (const term of speechRecognitionWords(
79 options.speechHints,
80 options.speechCustomWords.map(({ term }) => term),
81 )) {
82 if (provider === "openai" && /[<>\r\n]/.test(term)) continue;
83 form.append(provider === "openai" ? "keywords[]" : "keyterm", term);
84 }
85 if (provider === "xai") {
86 if (language !== "auto") form.set("format", "true");
87 form.set(
88 "filler_words",
89 String(!options.speechRemoveFillerWords || Boolean(options.speechCorrectionWord)),
90 );
91 if (!options.speechSilenceFiltering) form.set("vad_threshold", "0");
92 }
93 // xAI ignores options that follow the file part.
94 form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav");
95 return HttpClientRequest.bodyFormData(form);
96 }
97
98 function openRouterBody(model: string, audio: ArrayBuffer, language: string) {
99 return HttpClientRequest.bodyJsonUnsafe({
100 model,
101 input_audio: { data: Buffer.from(audio).toString("base64"), format: "wav" },
102 ...(language !== "auto" ? { language } : {}),
103 provider: { zdr: true },
104 });
105 }
106
107 export class CloudSpeech extends Context.Service<
108 CloudSpeech,
109 {
110 readonly configured: Effect.Effect<CloudSpeechKeys, ServerSettingsError>;
111 /** OpenRouter's public transcription catalog, cached briefly on this server. */
112 readonly models: Effect.Effect<ReadonlyArray<CloudSpeechModel>, CloudSpeechError>;
113 readonly setKey: (
114 provider: CloudSpeechProvider,
115 key: Redacted.Redacted<string> | SecretReference | null,
116 ) => Effect.Effect<void, ServerSettingsError>;
117 readonly transcribe: (
118 provider: CloudSpeechProvider,
119 pcm: Uint8Array,
120 options: SpeechTranscriptionOptions,
121 model?: string | undefined,
122 ) => Effect.Effect<string, CloudSpeechError | SpeechInvalidAudioError | ServerSettingsError>;
123 }
124 >()("t3/speech/CloudSpeech") {}
125
126 export const layer = Layer.effect(
127 CloudSpeech,
128 Effect.gen(function* () {
129 const settings = yield* ServerSettings.ServerSettingsService;
130 const resolver = yield* SecretResolver.SecretResolver;
131 const http = yield* HttpClient.HttpClient;
132 const credentials = settings.getSettings.pipe(
133 Effect.map((value) => value.cloudSpeechCredentials),
134 );
135 const configured = credentials.pipe(
136 Effect.map(({ openai, xai, openrouter }) => ({
137 openai: Boolean(openai),
138 xai: Boolean(xai),
139 openrouter: Boolean(openrouter),
140 })),
141 );
142 const execute = (request: HttpClientRequest.HttpClientRequest) =>
143 http
144 .execute(request)
145 .pipe(
146 Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
147 Effect.provideService(HttpClient.TracerPropagationEnabled, false),
148 );
149 const fetchCatalog = Effect.fn("CloudSpeech.catalog")(function* () {
150 const unavailable = () =>
151 new CloudSpeechError({ provider: "openrouter", reason: "unavailable" });
152 const response = yield* execute(HttpClientRequest.get(OPENROUTER_CATALOG_URL)).pipe(
153 Effect.mapError(unavailable),
154 );
155 if (response.status < 200 || response.status >= 300) return yield* unavailable();
156 const { data } = yield* HttpClientResponse.schemaBodyJson(OpenRouterCatalog)(response).pipe(
157 Effect.mapError(unavailable),
158 );
159 return data.flatMap(({ id, name }) =>
160 isModelId(id) ? [{ id, name: name?.trim() || id }] : [],
161 );
162 });
163 // The catalog is public, so the cache holds no credential or consent state.
164 const models = yield* Effect.cachedWithTTL(
165 fetchCatalog().pipe(
166 Effect.timeoutOrElse({
167 duration: "10 seconds",
168 orElse: () =>
169 Effect.fail(new CloudSpeechError({ provider: "openrouter", reason: "unavailable" })),
170 }),
171 ),
172 (exit) => (Exit.isSuccess(exit) ? "10 minutes" : 0),
173 );
174 const setKey = Effect.fn("CloudSpeech.setKey")(function* (
175 provider: CloudSpeechProvider,
176 key: Redacted.Redacted<string> | SecretReference | null,
177 ) {
178 yield* settings.updateSettings({
179 cloudSpeechCredentials: {
180 [provider]: Redacted.isRedacted(key) ? Redacted.value(key) : (key ?? ""),
181 },
182 });
183 });
184 const transcribe = Effect.fn("CloudSpeech.transcribe")(
185 function* (
186 provider: CloudSpeechProvider,
187 pcm: Uint8Array,
188 options: SpeechTranscriptionOptions,
189 model?: string,
190 ) {
191 const source = (yield* credentials)[provider];
192 if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" });
193 if (provider === "openrouter" && (!model || !isModelId(model)))
194 return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
195 const samples = yield* Effect.try({
196 try: () => decodeSpeechPcm(pcm),
197 catch: () =>
198 new SpeechInvalidAudioError({
199 byteLength: pcm.byteLength,
200 message: "Invalid PCM audio.",
201 }),
202 });
203 const language = effectiveSpeechLanguage(cloudSpeechLanguages, options.speechLanguage);
204 if (provider === "openrouter" && !(yield* models).some((entry) => entry.id === model))
205 return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
206 // A vault that locked since the last recording must not authorize a new upload.
207 if (isSecretReference(source)) yield* resolver.invalidate([source]);
208 const unreadable = () => new CloudSpeechError({ provider, reason: "unreadable-key" });
209 const key = yield* requireSettingsSecret(resolver, source).pipe(
210 Effect.mapError(unreadable),
211 );
212 if (!isApiKey(key)) return yield* unreadable();
213 const audio = wave(samples);
214 const request = HttpClientRequest.post(endpoints[provider].url).pipe(
215 HttpClientRequest.bearerToken(Redacted.make(key)),
216 provider === "openrouter"
217 ? openRouterBody(model!, audio, language)
218 : audioBody(provider, audio, language, options),
219 );
220 const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" });
221 // A catalog fetch or vault lookup may have outlived the credential it started with.
222 const current = (yield* credentials)[provider];
223 if (!current || !sameCredential(source, current))
224 return yield* new CloudSpeechError({
225 provider,
226 reason: current ? "key-changed" : "missing-key",
227 });
228 const response = yield* execute(request).pipe(Effect.mapError(unavailable));
229 if (response.status < 200 || response.status >= 300)
230 return yield* new CloudSpeechError({
231 provider,
232 reason:
233 response.status === 429
234 ? "rate-limit"
235 : [401, 402, 403].includes(response.status)
236 ? "rejected-key"
237 : response.status === 404 && provider === "openrouter"
238 ? "model-unavailable"
239 : "unavailable",
240 });
241 const { text } = yield* HttpClientResponse.schemaBodyJson(
242 EnvironmentSpeechTranscriptionResult,
243 )(response).pipe(Effect.mapError(unavailable));
244 const corrected = applySpeechAliases(text, options.speechCustomWords);
245 return (
246 options.speechRemoveFillerWords
247 ? removeSpeechFillerWords(corrected, language, options.speechCorrectionWord)
248 : corrected
249 ).trim();
250 },
251 (effect, provider) =>
252 effect.pipe(
253 Effect.timeoutOrElse({
254 duration: "2 minutes",
255 orElse: () => Effect.fail(new CloudSpeechError({ provider, reason: "unavailable" })),
256 }),
257 ),
258 );
259 return CloudSpeech.of({ configured, models, setKey, transcribe });
260 }),
261 );