@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 989 lines 37.3 KB
Raw
1 import {
2 SPEECH_STREAM_MAX_CHUNK_BYTES,
3 type EnvironmentSpeechModel,
4 type EnvironmentSpeechModelDownload,
5 type EnvironmentSpeechStatus,
6 type SpeechAcceleration,
7 type SpeechModelUnloadTimeout,
8 type SpeechTranscriptionOptions,
9 } from "@t3tools/contracts";
10 import * as HostProcess from "@t3tools/shared/HostProcess";
11 import * as Cause from "effect/Cause";
12 import * as Clock from "effect/Clock";
13 import * as Context from "effect/Context";
14 import * as Deferred from "effect/Deferred";
15 import * as Duration from "effect/Duration";
16 import * as Effect from "effect/Effect";
17 import * as Exit from "effect/Exit";
18 import * as Fiber from "effect/Fiber";
19 import * as Layer from "effect/Layer";
20 import * as Queue from "effect/Queue";
21 import * as Schema from "effect/Schema";
22 import * as Scope from "effect/Scope";
23 import * as Stream from "effect/Stream";
24
25 import * as ServerSettings from "../serverSettings.ts";
26 import {
27 makeSpeechTextCorrector,
28 normalizeSpeechCustomWords,
29 speechRecognitionWords,
30 } from "./customWords.ts";
31 import { removeSpeechFillerWords } from "./fillerWords.ts";
32 import {
33 DEFAULT_SPEECH_MODEL_ID,
34 effectiveSpeechLanguage,
35 getSpeechModel,
36 SPEECH_MODELS,
37 type SpeechModel,
38 } from "./model.ts";
39 import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts";
40 import { nativeSpeechLanguage } from "./nativeLanguage.ts";
41 import { SILERO_VAD } from "./silero.ts";
42 import { isSpeechPlatformSupported } from "./platform.ts";
43 import * as SpeechModelStore from "./SpeechModelStore.ts";
44
45 const SAMPLE_RATE = 16_000;
46 const MAX_SPEECH_DURATION_SECONDS = 5 * 60;
47 export const MAX_SPEECH_BYTES =
48 SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * MAX_SPEECH_DURATION_SECONDS;
49
50 export class SpeechInvalidAudioError extends Schema.TaggedError<SpeechInvalidAudioError>()(
51 "SpeechInvalidAudioError",
52 { byteLength: Schema.Number, message: Schema.String },
53 ) {}
54
55 export function decodeSpeechPcm(pcmBytes: Uint8Array): Float32Array {
56 if (
57 pcmBytes.byteLength === 0 ||
58 pcmBytes.byteLength > MAX_SPEECH_BYTES ||
59 pcmBytes.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0
60 ) {
61 throw new SpeechInvalidAudioError({
62 byteLength: pcmBytes.byteLength,
63 message: "audio must contain at most five minutes of 16 kHz mono Float32 PCM",
64 });
65 }
66 // WebSocket frames can be pooled Node Buffers whose backing storage is larger
67 // than the frame. Copy the view so only the received bytes are decoded.
68 const pcm = new Float32Array(Uint8Array.from(pcmBytes).buffer);
69 for (const sample of pcm) {
70 if (!Number.isFinite(sample) || sample < -1 || sample > 1) {
71 throw new SpeechInvalidAudioError({
72 byteLength: pcmBytes.byteLength,
73 message: "audio contains an invalid PCM sample",
74 });
75 }
76 }
77 return pcm;
78 }
79
80 export class SpeechOperationError extends Schema.TaggedError<SpeechOperationError>()(
81 "SpeechOperationError",
82 { operation: Schema.String, cause: Schema.Defect() },
83 ) {
84 override get message(): string {
85 return `Environment speech ${this.operation} failed.`;
86 }
87 }
88
89 export class SpeechUnsupportedPlatformError extends Schema.TaggedError<SpeechUnsupportedPlatformError>()(
90 "SpeechUnsupportedPlatformError",
91 { platform: Schema.String, architecture: Schema.String },
92 ) {}
93
94 export class SpeechBusyError extends Schema.TaggedError<SpeechBusyError>()("SpeechBusyError", {
95 operation: Schema.String,
96 }) {}
97
98 export class SpeechModelNotFoundError extends Schema.TaggedError<SpeechModelNotFoundError>()(
99 "SpeechModelNotFoundError",
100 { modelId: Schema.String },
101 ) {}
102
103 export class SpeechDownloadCancelledError extends Schema.TaggedError<SpeechDownloadCancelledError>()(
104 "SpeechDownloadCancelledError",
105 { modelId: Schema.String },
106 ) {}
107
108 export type SpeechError =
109 | SpeechOperationError
110 | SpeechInvalidAudioError
111 | SpeechUnsupportedPlatformError
112 | SpeechModelNotFoundError
113 | SpeechDownloadCancelledError
114 | SpeechBusyError;
115
116 const isSpeechError = Schema.is(
117 Schema.Union([
118 SpeechInvalidAudioError,
119 SpeechUnsupportedPlatformError,
120 SpeechBusyError,
121 SpeechModelNotFoundError,
122 SpeechDownloadCancelledError,
123 SpeechOperationError,
124 ]),
125 );
126
127 type LoadedModel = Awaited<ReturnType<typeof loadNativeSpeechModel>>;
128 type SpeechModels = { readonly models: ReadonlyArray<EnvironmentSpeechModel> };
129
130 export type SpeechStream = {
131 readonly feed: (
132 pcm: Uint8Array,
133 ) => Effect.Effect<Awaited<ReturnType<LoadedModel["feed"]>>, SpeechError>;
134 readonly finish: Effect.Effect<string, SpeechError>;
135 };
136
137 export class SpeechService extends Context.Service<
138 SpeechService,
139 {
140 readonly status: Effect.Effect<EnvironmentSpeechStatus, SpeechError>;
141 readonly models: (includeBenchmarks: boolean) => Effect.Effect<SpeechModels, SpeechError>;
142 readonly downloadModel: (
143 modelId: string,
144 ) => Effect.Effect<Required<EnvironmentSpeechModelDownload>, SpeechError>;
145 readonly selectModel: (modelId: string) => Effect.Effect<SpeechModels, SpeechError>;
146 readonly cancelDownload: (modelId: string) => Effect.Effect<SpeechModels, SpeechError>;
147 readonly removeModel: (modelId: string) => Effect.Effect<SpeechModels, SpeechError>;
148 /** Uses catalog model files already on this machine, from a file or folder path. */
149 readonly linkModels: (source: string) => Effect.Effect<SpeechModels, SpeechError>;
150 readonly prepareModel: Effect.Effect<void, SpeechError>;
151 readonly transcribe: (
152 pcmBytes: Uint8Array,
153 options: SpeechTranscriptionOptions,
154 ) => Effect.Effect<string, SpeechError>;
155 readonly startStream: (
156 options: SpeechTranscriptionOptions,
157 ) => Effect.Effect<SpeechStream, SpeechError, Scope.Scope>;
158 readonly updateAcceleration: (
159 acceleration: SpeechAcceleration,
160 ) => Effect.Effect<EnvironmentSpeechStatus, SpeechError>;
161 readonly updateModelUnloadTimeout: (
162 timeout: SpeechModelUnloadTimeout,
163 ) => Effect.Effect<EnvironmentSpeechStatus, SpeechError>;
164 }
165 >()("t3/speech/SpeechService") {}
166
167 const UNLOAD_MILLISECONDS: Record<Exclude<SpeechModelUnloadTimeout, "never">, number> = {
168 immediately: 0,
169 min_2: 2 * 60_000,
170 min_5: 5 * 60_000,
171 min_10: 10 * 60_000,
172 min_15: 15 * 60_000,
173 hour_1: 60 * 60_000,
174 };
175
176 const make = Effect.gen(function* () {
177 const platform = yield* HostProcess.Platform;
178 const architecture = yield* HostProcess.Architecture;
179 const serverSettings = yield* ServerSettings.ServerSettingsService;
180 const store = yield* SpeechModelStore.SpeechModelStore;
181 const scope = yield* Effect.scope;
182 const clock = yield* Clock.Clock;
183 const now = () => clock.currentTimeMillisUnsafe();
184 const libc = platform === "linux" ? yield* HostProcess.LinuxLibc : undefined;
185 const unsupportedReason = isSpeechPlatformSupported(platform, architecture, libc)
186 ? null
187 : `voice transcription is not available on ${platform}-${architecture}`;
188
189 let model: LoadedModel | undefined;
190 let loadedModelId: string | undefined;
191 let loadedAcceleration: string | undefined;
192 let autoCpuModel: string | undefined;
193 let loading: Promise<LoadedModel> | undefined;
194 const discoverGpuDevices = yield* Effect.tryPromise({
195 try: () => listNativeSpeechGpuDevices(),
196 catch: (cause) => new SpeechOperationError({ operation: "discover GPU devices", cause }),
197 }).pipe(Effect.cachedWithTTL((exit) => (Exit.isSuccess(exit) ? Duration.infinity : "1 minute")));
198 let downloading:
199 | {
200 modelId: string;
201 downloaded: number;
202 verifying: boolean;
203 fiber: Deferred.Deferred<Fiber.Fiber<string, SpeechError>>;
204 }
205 | undefined;
206 let activeTranscriptions = 0;
207 // One native request at a time; the slot is a promise so unload waits for it.
208 let activeOperation: Promise<unknown> | undefined;
209 const operationBusy = () => Boolean(activeOperation || downloading);
210 let closing = false;
211 // @effect-diagnostics-next-line abortControllerInEffect:off - Cached native workers must also stop on service shutdown.
212 const lifetime = new AbortController();
213 let unloadTimeout: SpeechModelUnloadTimeout = "min_15";
214
215 // Deadlines are absolute: the consumer fiber registers its sleep a few scheduler
216 // hops after the request, so a relative delay would drift.
217 const unloadRequests = yield* Queue.sliding<{ deadline: number } | null>(1);
218 const clearUnloadTimer = () => Queue.offerUnsafe(unloadRequests, null);
219 const scheduleUnload = (minimumDelayMs = 0) => {
220 clearUnloadTimer();
221 if (!model || closing || unloadTimeout === "never") return;
222 Queue.offerUnsafe(unloadRequests, {
223 deadline: now() + Math.max(minimumDelayMs, UNLOAD_MILLISECONDS[unloadTimeout]),
224 });
225 };
226
227 const forgetModel = (target: LoadedModel) => {
228 if (model !== target) return;
229 model = undefined;
230 loadedModelId = undefined;
231 loadedAcceleration = undefined;
232 };
233 const disposeModel = async () => {
234 const target = model;
235 if (!target) return;
236 forgetModel(target);
237 await target.dispose();
238 };
239 // Stops the isolated process without waiting for native work to settle.
240 const dropModel = (target: LoadedModel) => {
241 forgetModel(target);
242 void Promise.resolve()
243 .then(() => target.dispose())
244 .catch(() => undefined);
245 };
246
247 // Auto placement may land on a GPU the native engine cannot actually run; one retry on
248 // this host's CPU recovers it. A GPU the person chose, and CPU itself, report the error.
249 const canRetryOnCpu = (acceleration: SpeechAcceleration, loaded: LoadedModel) =>
250 acceleration === "auto" && loaded.backend.toLowerCase() !== "cpu";
251
252 yield* Effect.addFinalizer(() =>
253 Effect.promise(async () => {
254 closing = true;
255 clearUnloadTimer();
256 lifetime.abort();
257 await disposeModel();
258 }),
259 );
260
261 yield* Stream.fromQueue(unloadRequests).pipe(
262 Stream.switchMap((request) =>
263 request === null
264 ? Stream.empty
265 : Stream.fromEffect(
266 Effect.gen(function* () {
267 yield* Effect.sleep(Math.max(0, request.deadline - now()));
268 if (closing) return;
269 if (operationBusy() || activeTranscriptions || loading) {
270 scheduleUnload(1000);
271 return;
272 }
273 yield* Effect.promise(() => disposeModel().catch(() => undefined)).pipe(
274 Effect.uninterruptible,
275 );
276 }),
277 ),
278 ),
279 Stream.runDrain,
280 Effect.forkScoped,
281 );
282
283 type SettingsSnapshot = Effect.Success<
284 ServerSettings.ServerSettingsService["Service"]["getSettings"]
285 >;
286
287 const selectedModel = (settings: SettingsSnapshot): SpeechModel =>
288 getSpeechModel(settings.speechModelId) ?? getSpeechModel(DEFAULT_SPEECH_MODEL_ID)!;
289
290 let silenceRetryAt = 0;
291 const acquireSilenceModel = Effect.fn("SpeechService.acquireSilenceModel")(function* (
292 purpose: "installation" | "recording",
293 ) {
294 if (purpose === "recording" && now() < silenceRetryAt) return undefined;
295 const path = yield* store.download(SILERO_VAD).pipe(
296 Effect.timeout(purpose === "recording" ? "15 seconds" : "2 minutes"),
297 Effect.orElseSucceed(() => undefined),
298 );
299 if (path !== undefined) {
300 silenceRetryAt = 0;
301 return path;
302 }
303 silenceRetryAt = now() + 10 * 60_000;
304 yield* Effect.logWarning("Silero unavailable; using raw audio with bounded ASR processing.");
305 });
306
307 // Publish the reservation before forking: a cached transfer can finish before forkIn returns.
308 const downloadFiber = Effect.fn("SpeechService.downloadFiber")(function* (modelId: string) {
309 const definition = getSpeechModel(modelId);
310 if (!definition) return yield* new SpeechModelNotFoundError({ modelId });
311 if (closing || activeOperation || (downloading && downloading.modelId !== modelId))
312 return yield* new SpeechBusyError({ operation: "model download" });
313 if (downloading) return yield* Deferred.await(downloading.fiber);
314 const transfer = {
315 modelId,
316 downloaded: 0,
317 verifying: false,
318 fiber: Deferred.makeUnsafe<Fiber.Fiber<string, SpeechError>>(),
319 };
320 downloading = transfer;
321 const fiber = yield* store
322 .download(definition, {
323 onProgress: (downloaded) => {
324 transfer.downloaded = downloaded;
325 transfer.verifying = false;
326 },
327 onVerificationStart: () => {
328 transfer.verifying = true;
329 },
330 })
331 .pipe(
332 (modelDownload) =>
333 Effect.all([modelDownload, acquireSilenceModel("installation")], { concurrency: 2 }),
334 Effect.map(([modelPath]) => modelPath),
335 Effect.mapError((cause) => speechError("model download", cause)),
336 Effect.forkIn(scope),
337 );
338 fiber.addObserver(() => {
339 if (downloading !== transfer) return;
340 downloading = undefined;
341 scheduleUnload();
342 });
343 yield* Deferred.succeed(transfer.fiber, fiber);
344 return fiber;
345 }, Effect.uninterruptible);
346
347 const download = Effect.fn("SpeechService.download")(function* (modelId: string) {
348 const exit = yield* Fiber.await(yield* downloadFiber(modelId));
349 if (Exit.isSuccess(exit)) return exit.value;
350 return yield* Cause.hasInterruptsOnly(exit.cause)
351 ? new SpeechDownloadCancelledError({ modelId })
352 : Effect.failCause(exit.cause);
353 });
354
355 const installedModelPath = (definition: SpeechModel) =>
356 store.installedPath(definition).pipe(
357 Effect.filterOrFail(
358 (modelPath) => modelPath !== undefined,
359 () => new SpeechModelNotFoundError({ modelId: definition.id }),
360 ),
361 );
362
363 const loadModel = async (
364 modelPath: string,
365 definition: SpeechModel,
366 signal = lifetime.signal,
367 requestedAcceleration = "auto",
368 ) => {
369 const acceleration =
370 requestedAcceleration === "auto" && autoCpuModel === definition.id
371 ? "cpu"
372 : requestedAcceleration;
373 if (model && loadedModelId === definition.id && loadedAcceleration === acceleration) {
374 scheduleUnload();
375 return model;
376 }
377 if (loading) return loading;
378 await disposeModel();
379 signal.throwIfAborted();
380 const pending = loadNativeSpeechModel(modelPath, signal, undefined, acceleration).then(
381 async (loaded) => {
382 if (closing || signal.aborted) {
383 await loaded.dispose();
384 signal.throwIfAborted();
385 throw new SpeechBusyError({ operation: "model preparation" });
386 }
387 model = loaded;
388 loadedModelId = definition.id;
389 loadedAcceleration = acceleration;
390 scheduleUnload();
391 return loaded;
392 },
393 );
394 const release = () => {
395 if (loading === pending) loading = undefined;
396 };
397 loading = pending;
398 signal.addEventListener("abort", release, { once: true });
399 try {
400 return await pending;
401 } finally {
402 signal.removeEventListener("abort", release);
403 release();
404 }
405 };
406
407 const speechError = (operation: string, cause: unknown): SpeechError =>
408 isSpeechError(cause) ? cause : new SpeechOperationError({ operation, cause });
409
410 const attempt = <A>(operation: string, run: (signal: AbortSignal) => Promise<A>) =>
411 Effect.tryPromise({ try: run, catch: (cause) => speechError(operation, cause) });
412
413 // The busy slot is released by a finalizer, so interrupting the fiber frees it at
414 // once instead of waiting for native work that may never settle.
415 const exclusive = <A, E, R>(operation: string, effect: Effect.Effect<A, E, R>) =>
416 Effect.uninterruptibleMask((restore) =>
417 Effect.suspend<A, E | SpeechBusyError, R>(() => {
418 if (closing || operationBusy()) return Effect.fail(new SpeechBusyError({ operation }));
419 const gate = Promise.withResolvers<void>();
420 activeOperation = gate.promise;
421 return restore(effect).pipe(
422 Effect.ensuring(
423 Effect.sync(() => {
424 if (activeOperation === gate.promise) activeOperation = undefined;
425 scheduleUnload(
426 operation === "model preparation" ? MAX_SPEECH_DURATION_SECONDS * 1000 : 0,
427 );
428 gate.resolve();
429 }),
430 ),
431 );
432 }),
433 );
434
435 const readSettings = (operation: string) =>
436 serverSettings.getSettings.pipe(
437 Effect.tap((settings) =>
438 Effect.sync(() => {
439 if (settings.speechModelId !== autoCpuModel || settings.speechAcceleration !== "auto")
440 autoCpuModel = undefined;
441 if (unloadTimeout !== settings.speechModelUnloadTimeout) {
442 unloadTimeout = settings.speechModelUnloadTimeout;
443 scheduleUnload();
444 }
445 }),
446 ),
447 Effect.mapError((cause) => new SpeechOperationError({ operation, cause })),
448 );
449
450 const writeSettings = (
451 operation: string,
452 patch: Parameters<ServerSettings.ServerSettingsService["Service"]["updateSettings"]>[0],
453 ) =>
454 serverSettings
455 .updateSettings(patch)
456 .pipe(Effect.mapError((cause) => new SpeechOperationError({ operation, cause })));
457
458 const currentStatus = Effect.fn("SpeechService.currentStatus")(function* (
459 settings: SettingsSnapshot,
460 ): Effect.fn.Return<EnvironmentSpeechStatus> {
461 if (unsupportedReason) return { supported: false, reason: unsupportedReason };
462 const definition = selectedModel(settings);
463 const ready = yield* store.isReady(definition);
464 const gpuDevices = yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => []));
465 const autoUsesCpu =
466 gpuDevices.length === 0 ||
467 autoCpuModel === definition.id ||
468 (loadedModelId === definition.id &&
469 loadedAcceleration === "auto" &&
470 model?.backend.toLowerCase() === "cpu");
471 return {
472 supported: true,
473 state: activeTranscriptions > 0 ? "transcribing" : ready ? "ready" : "missing-model",
474 modelId: definition.id,
475 model: definition.name,
476 size: definition.size,
477 supportsStreaming: definition.supportsStreaming,
478 supportsBufferedStreaming: true,
479 supportsModelLinking: true,
480 supportsTranslation: definition.supportsTranslation,
481 languages: definition.languages,
482 supportsLanguageDetection: definition.supportsLanguageDetection,
483 acceleration: settings.speechAcceleration,
484 benchmarkTier:
485 settings.speechAcceleration === "cpu" ||
486 (settings.speechAcceleration === "auto" && autoUsesCpu)
487 ? "cpu"
488 : "accelerated",
489 modelUnloadTimeout: settings.speechModelUnloadTimeout,
490 gpuDevices,
491 };
492 });
493
494 const listModels = Effect.fn("SpeechService.listModels")(function* (
495 settings: SettingsSnapshot,
496 includeBenchmarks: boolean,
497 ): Effect.fn.Return<SpeechModels> {
498 const selected = selectedModel(settings);
499 const models = yield* Effect.forEach(
500 SPEECH_MODELS,
501 (definition) =>
502 Effect.all([store.isReady(definition), store.linkedFrom(definition)]).pipe(
503 Effect.map(([ready, linkedFrom]): EnvironmentSpeechModel => {
504 const operation = downloading?.modelId === definition.id ? downloading : undefined;
505 return {
506 id: definition.id,
507 name: definition.name,
508 description: definition.description,
509 size: definition.size,
510 languages: definition.languages,
511 accuracy: definition.accuracy,
512 speed: definition.speed,
513 ...(includeBenchmarks ? { benchmark: definition.benchmark } : {}),
514 recommended: definition.recommended,
515 supportsStreaming: definition.supportsStreaming,
516 supportsLanguageDetection: definition.supportsLanguageDetection,
517 supportsTranslation: definition.supportsTranslation,
518 active: ready && selected.id === definition.id,
519 state: operation
520 ? operation.verifying
521 ? "verifying"
522 : "downloading"
523 : ready
524 ? "installed"
525 : "downloadable",
526 ...(operation ? { downloaded: operation.downloaded } : {}),
527 ...(ready && linkedFrom ? { linkedFrom } : {}),
528 };
529 }),
530 ),
531 { concurrency: "unbounded" },
532 );
533 return { models };
534 });
535
536 const freshStatus = (operation: string) =>
537 readSettings(operation).pipe(Effect.flatMap(currentStatus));
538 const readModels = (includeBenchmarks: boolean) =>
539 readSettings("model listing").pipe(
540 Effect.flatMap((settings) => listModels(settings, includeBenchmarks)),
541 );
542 const modelsEffect = readModels(false);
543
544 const requireSupported = Effect.suspend(() =>
545 unsupportedReason
546 ? Effect.fail(new SpeechUnsupportedPlatformError({ platform, architecture }))
547 : Effect.void,
548 );
549
550 const finalizeTranscript = (
551 text: string,
552 options: SpeechTranscriptionOptions,
553 language: string | undefined,
554 ) => {
555 return (
556 options.speechRemoveFillerWords
557 ? removeSpeechFillerWords(text, language, options.speechCorrectionWord)
558 : text
559 ).trim();
560 };
561
562 const prepareModel = readSettings("model preparation").pipe(
563 Effect.flatMap((settings) =>
564 exclusive(
565 "model preparation",
566 requireSupported.pipe(
567 Effect.andThen(installedModelPath(selectedModel(settings))),
568 Effect.flatMap((modelPath) =>
569 attempt("model preparation", (signal) =>
570 loadModel(
571 modelPath,
572 selectedModel(settings),
573 AbortSignal.any([signal, lifetime.signal]),
574 settings.speechAcceleration,
575 ),
576 ).pipe(
577 Effect.timeout("2 minutes"),
578 Effect.catchTags({
579 TimeoutError: (cause) =>
580 Effect.fail(new SpeechOperationError({ operation: "model preparation", cause })),
581 }),
582 ),
583 ),
584 ),
585 ),
586 ),
587 );
588
589 const silenceModelPath = Effect.fn("SpeechService.silenceModelPath")(function* (
590 loaded: LoadedModel,
591 options: SpeechTranscriptionOptions,
592 nativeStreaming: boolean,
593 ) {
594 if (!options.speechSilenceFiltering && (nativeStreaming || loaded.supportsLongForm))
595 return undefined;
596 return yield* acquireSilenceModel("recording");
597 });
598
599 const transcribe: SpeechService["Service"]["transcribe"] = Effect.fn("SpeechService.transcribe")(
600 function* (pcmBytes, options) {
601 const settings = yield* readSettings("transcription");
602 return yield* exclusive(
603 "transcription",
604 Effect.gen(function* () {
605 yield* requireSupported;
606 const pcm = yield* Effect.try({
607 try: () => decodeSpeechPcm(pcmBytes),
608 catch: (cause) => speechError("transcription", cause),
609 });
610 if (pcm.length === 0) return "";
611 activeTranscriptions += 1;
612 return yield* Effect.gen(function* () {
613 const definition = selectedModel(settings);
614 const language = effectiveSpeechLanguage(definition, options.speechLanguage);
615 const translateToEnglish =
616 options.speechTranslateToEnglish &&
617 definition.supportsTranslation &&
618 language !== "en";
619 const modelPath = yield* installedModelPath(definition);
620 const loaded = yield* Effect.tryPromise({
621 try: (signal) =>
622 loadModel(
623 modelPath,
624 definition,
625 AbortSignal.any([signal, lifetime.signal]),
626 settings.speechAcceleration,
627 ),
628 catch: (cause) => speechError("model preparation", cause),
629 });
630 const vadPath = yield* silenceModelPath(loaded, options, false);
631 const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
632 const customWords = dictionary.map(({ term }) => term);
633 const recognitionWords = speechRecognitionWords(options.speechHints, customWords);
634 // The vocabulary capability guarantees plain transcription, not translation.
635 const vocabularyFor = (inference: LoadedModel) =>
636 inference.supportsVocabulary && !translateToEnglish ? recognitionWords : [];
637 const transcribeWith = (inference: LoadedModel) =>
638 Effect.tryPromise({
639 try: (signal) => {
640 // Interruption kills the isolated process; the next request reloads the model.
641 signal.addEventListener("abort", () => dropModel(inference), { once: true });
642 const hint = nativeSpeechLanguage(inference.languages, language);
643 return inference.transcribe(
644 pcm,
645 {
646 timestamps: "none",
647 ...(hint ? { language: hint } : {}),
648 ...(translateToEnglish
649 ? { task: "translate" as const, targetLanguage: "en" as const }
650 : {}),
651 ...(vocabularyFor(inference).length > 0
652 ? { vocabulary: recognitionWords }
653 : {}),
654 },
655 options.speechSilenceFiltering,
656 vadPath,
657 );
658 },
659 catch: (cause) => {
660 dropModel(inference);
661 return speechError("inference", cause);
662 },
663 });
664 let inference = loaded;
665 const result = yield* transcribeWith(loaded).pipe(
666 Effect.catch((error) => {
667 if (!canRetryOnCpu(settings.speechAcceleration, loaded)) return Effect.fail(error);
668 return Effect.tryPromise({
669 try: (signal) =>
670 loadModel(
671 modelPath,
672 definition,
673 AbortSignal.any([signal, lifetime.signal]),
674 "cpu",
675 ),
676 catch: (cause) => speechError("model preparation", cause),
677 }).pipe(
678 Effect.flatMap((cpu) => {
679 autoCpuModel = definition.id;
680 inference = cpu;
681 return transcribeWith(cpu);
682 }),
683 );
684 }),
685 );
686 const correct = makeSpeechTextCorrector(
687 vocabularyFor(inference).length > 0 ? [] : customWords,
688 dictionary,
689 );
690 return finalizeTranscript(
691 correct(result.text),
692 options,
693 translateToEnglish ? "en" : language === "auto" ? undefined : language,
694 );
695 }).pipe(
696 Effect.ensuring(
697 Effect.sync(() => {
698 activeTranscriptions -= 1;
699 }),
700 ),
701 );
702 }),
703 );
704 },
705 );
706
707 // Stream setup shares the public startStream span.
708 const openStream = Effect.fnUntraced(function* (options: SpeechTranscriptionOptions) {
709 const operation = "streaming transcription";
710 yield* requireSupported;
711 const released = Promise.withResolvers<void>();
712 // @effect-diagnostics-next-line abortControllerInEffect:off - Stream cancellation must abort native work independently of the caller fiber.
713 const controller = new AbortController();
714 const signal = AbortSignal.any([controller.signal, lifetime.signal]);
715 let finished = false;
716 let closed = false;
717 let started = false;
718 let loaded: LoadedModel | undefined;
719 yield* Effect.acquireRelease(
720 Effect.suspend(() => {
721 if (closing || operationBusy()) return Effect.fail(new SpeechBusyError({ operation }));
722 activeOperation = released.promise;
723 activeTranscriptions += 1;
724 return Effect.void;
725 }),
726 () =>
727 Effect.sync(() => {
728 activeTranscriptions -= 1;
729 if (activeOperation === released.promise) activeOperation = undefined;
730 scheduleUnload();
731 released.resolve();
732 }),
733 );
734 yield* Effect.addFinalizer(() =>
735 Effect.gen(function* () {
736 closed = true;
737 if (!finished) {
738 const dispose = Effect.promise(async () => {
739 const pending = loading;
740 controller.abort();
741 await pending?.catch(() => undefined);
742 if (loaded) {
743 forgetModel(loaded);
744 await loaded.dispose();
745 }
746 });
747 if (loaded && started) {
748 yield* attempt("reset stream", () => loaded!.reset()).pipe(
749 Effect.timeout("2500 millis"),
750 Effect.catch(() => dispose),
751 );
752 } else {
753 yield* dispose;
754 }
755 }
756 }),
757 );
758 const settings = yield* readSettings(operation);
759 const definition = selectedModel(settings);
760 const language = effectiveSpeechLanguage(definition, options.speechLanguage);
761 const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
762 const customWords = dictionary.map(({ term }) => term);
763 const recognitionWords = speechRecognitionWords(options.speechHints, customWords);
764 let correct = makeSpeechTextCorrector(customWords, dictionary);
765 const modelPath = yield* installedModelPath(definition);
766 let streamModel = yield* attempt(operation, () =>
767 loadModel(modelPath, definition, signal, settings.speechAcceleration),
768 );
769 loaded = streamModel;
770 const vadPath = yield* silenceModelPath(streamModel, options, streamModel.supportsStreaming);
771 let translateToEnglish = false;
772 const beginStream = async () => {
773 translateToEnglish =
774 options.speechTranslateToEnglish &&
775 !streamModel.supportsStreaming &&
776 streamModel.supportsTranslation &&
777 language !== "en";
778 const vocabulary =
779 streamModel.supportsVocabulary && !translateToEnglish && recognitionWords.length
780 ? recognitionWords
781 : undefined;
782 const hint = nativeSpeechLanguage(streamModel.languages, language);
783 await streamModel.begin(
784 {
785 timestamps: "none",
786 ...(hint ? { language: hint } : {}),
787 ...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}),
788 ...(vocabulary ? { vocabulary } : {}),
789 },
790 options.speechSilenceFiltering,
791 vadPath,
792 );
793 correct = makeSpeechTextCorrector(vocabulary ? [] : customWords, dictionary);
794 started = true;
795 };
796 // Audio already fed is kept so a CPU restart can replay it; the limit caps it at one recording.
797 const received: Float32Array[] = [];
798 let retriedOnCpu = false;
799 const recoverOnCpu = async (cause: unknown) => {
800 started = false;
801 if (closed || signal.aborted || retriedOnCpu) throw cause;
802 if (!canRetryOnCpu(settings.speechAcceleration, streamModel)) throw cause;
803 retriedOnCpu = true;
804 forgetModel(streamModel);
805 await streamModel.dispose();
806 if (closed || signal.aborted) throw cause;
807 streamModel = await loadModel(modelPath, definition, signal, "cpu");
808 autoCpuModel = definition.id;
809 loaded = streamModel;
810 await beginStream();
811 let update: Awaited<ReturnType<LoadedModel["feed"]>> | undefined;
812 for (const chunk of received) {
813 if (closed || signal.aborted) throw new Error("Speech stream is closed.");
814 update = await streamModel.feed(chunk);
815 }
816 return update;
817 };
818 yield* attempt(operation, async () => {
819 try {
820 await beginStream();
821 } catch (cause) {
822 await recoverOnCpu(cause);
823 }
824 });
825 let byteLength = 0;
826 let busy = false;
827 // Client revisions stay monotonic across a native restart, whose own counter resets.
828 let revisionBase = 0;
829 let lastNativeRevision = 0;
830 const run = <A>(work: () => Promise<A>) =>
831 attempt(operation, async () => {
832 if (busy || finished || closed || !started || signal.aborted)
833 throw new Error("Speech stream is not ready.");
834 busy = true;
835 try {
836 return await work();
837 } finally {
838 busy = false;
839 }
840 });
841 return {
842 feed: (bytes: Uint8Array) =>
843 run(async () => {
844 if (
845 bytes.byteLength > SPEECH_STREAM_MAX_CHUNK_BYTES ||
846 byteLength + bytes.byteLength > MAX_SPEECH_BYTES
847 )
848 throw new SpeechInvalidAudioError({
849 byteLength: bytes.byteLength,
850 message: "Speech stream audio limit exceeded.",
851 });
852 const pcm = decodeSpeechPcm(bytes);
853 byteLength += bytes.byteLength;
854 received.push(pcm);
855 let update: Awaited<ReturnType<LoadedModel["feed"]>> | undefined;
856 try {
857 update = await streamModel.feed(pcm);
858 } catch (cause) {
859 update = await recoverOnCpu(cause);
860 revisionBase += lastNativeRevision;
861 lastNativeRevision = 0;
862 }
863 if (update) lastNativeRevision = update.revision;
864 return {
865 revision: revisionBase + (update?.revision ?? lastNativeRevision),
866 text: update?.text
867 ? {
868 committed: correct(update.text.committed),
869 tentative: correct(update.text.tentative),
870 }
871 : null,
872 };
873 }),
874 finish: run(async () => {
875 let text: string;
876 try {
877 text = await streamModel.finish().catch(async (cause: unknown) => {
878 await recoverOnCpu(cause);
879 if (closed || signal.aborted) throw cause;
880 return streamModel.finish();
881 });
882 } catch (cause) {
883 started = false;
884 throw cause;
885 }
886 finished = true;
887 return finalizeTranscript(
888 correct(text),
889 options,
890 translateToEnglish ? "en" : language === "auto" ? undefined : language,
891 );
892 }),
893 };
894 });
895
896 return SpeechService.of({
897 status: freshStatus("status"),
898 models: readModels,
899 prepareModel,
900 transcribe,
901 startStream: Effect.fn("SpeechService.startStream")(function* (options) {
902 const session = yield* Scope.fork(yield* Effect.scope);
903 return yield* openStream(options).pipe(
904 Scope.provide(session),
905 Effect.onError((cause) => Scope.close(session, Exit.failCause(cause))),
906 );
907 }),
908 downloadModel: Effect.fn("SpeechService.downloadModel")(function* (modelId) {
909 yield* requireSupported;
910 const cancelled = yield* download(modelId).pipe(
911 Effect.as(false),
912 Effect.catchTags({ SpeechDownloadCancelledError: () => Effect.succeed(true) }),
913 );
914 return { ...(yield* modelsEffect), cancelled };
915 }),
916 selectModel: Effect.fn("SpeechService.selectModel")((modelId) =>
917 exclusive(
918 "model selection",
919 Effect.gen(function* () {
920 const definition = getSpeechModel(modelId);
921 if (!definition || !(yield* store.isReady(definition)))
922 return yield* new SpeechModelNotFoundError({ modelId });
923 yield* attempt("model selection", () => disposeModel());
924 yield* writeSettings("model selection", { speechModelId: modelId });
925 }),
926 ).pipe(Effect.andThen(modelsEffect)),
927 ),
928 cancelDownload: Effect.fn("SpeechService.cancelDownload")((modelId) =>
929 Effect.suspend(() =>
930 downloading?.modelId === modelId
931 ? Deferred.await(downloading.fiber).pipe(Effect.flatMap(Fiber.interrupt))
932 : Effect.void,
933 ).pipe(Effect.andThen(modelsEffect)),
934 ),
935 removeModel: Effect.fn("SpeechService.removeModel")((modelId) =>
936 exclusive(
937 "model removal",
938 Effect.gen(function* () {
939 const settings = yield* readSettings("model removal");
940 const definition = getSpeechModel(modelId);
941 if (!definition) return yield* new SpeechModelNotFoundError({ modelId });
942 yield* attempt("model removal", async () => {
943 await loading?.catch(() => undefined);
944 if (loadedModelId === modelId) await disposeModel();
945 });
946 yield* store
947 .remove(definition)
948 .pipe(Effect.mapError((cause) => speechError("model removal", cause)));
949 if (selectedModel(settings).id !== modelId) return;
950 const candidates = yield* Effect.forEach(
951 SPEECH_MODELS.filter((candidate) => candidate.id !== modelId),
952 (candidate) =>
953 store.isReady(candidate).pipe(Effect.map((ready) => ({ candidate, ready }))),
954 { concurrency: "unbounded" },
955 );
956 const replacement = candidates.find(({ ready }) => ready)?.candidate;
957 if (replacement) yield* writeSettings("model removal", { speechModelId: replacement.id });
958 }),
959 ).pipe(Effect.andThen(modelsEffect)),
960 ),
961 linkModels: Effect.fn("SpeechService.linkModels")(function* (source) {
962 yield* requireSupported;
963 const settings = yield* readSettings("model linking");
964 const linked = yield* exclusive(
965 "model linking",
966 store
967 .link(SPEECH_MODELS, source)
968 .pipe(Effect.mapError((cause) => speechError("model linking", cause))),
969 );
970 const first = linked[0];
971 if (first === undefined) return yield* new SpeechModelNotFoundError({ modelId: source });
972 if (!(yield* store.isReady(selectedModel(settings))))
973 yield* writeSettings("model linking", { speechModelId: first });
974 return yield* modelsEffect;
975 }),
976 updateAcceleration: Effect.fn("SpeechService.updateAcceleration")((acceleration) =>
977 writeSettings("acceleration update", { speechAcceleration: acceleration }).pipe(
978 Effect.andThen(freshStatus("acceleration update")),
979 ),
980 ),
981 updateModelUnloadTimeout: Effect.fn("SpeechService.updateModelUnloadTimeout")((timeout) =>
982 writeSettings("model unload timeout update", { speechModelUnloadTimeout: timeout }).pipe(
983 Effect.andThen(freshStatus("model unload timeout update")),
984 ),
985 ),
986 });
987 });
988
989 export const layer = Layer.effect(SpeechService, make);