The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { |
| 2 | SPEECH_STREAM_MAX_CHUNK_BYTES, |
| 3 | type EnvironmentSpeechModel, |
| 4 | type EnvironmentSpeechModelDownload, |
| 5 | type EnvironmentSpeechStatus, |
| 6 | type SpeechAcceleration, |
| 7 | type SpeechModelUnloadTimeout, |
| 8 | type SpeechTranscriptionOptions, |
| 9 | } from "@t3tools/contracts"; |
| 10 | import * as HostProcess from "@t3tools/shared/HostProcess"; |
| 11 | import * as Cause from "effect/Cause"; |
| 12 | import * as Clock from "effect/Clock"; |
| 13 | import * as Context from "effect/Context"; |
| 14 | import * as Deferred from "effect/Deferred"; |
| 15 | import * as Duration from "effect/Duration"; |
| 16 | import * as Effect from "effect/Effect"; |
| 17 | import * as Exit from "effect/Exit"; |
| 18 | import * as Fiber from "effect/Fiber"; |
| 19 | import * as Layer from "effect/Layer"; |
| 20 | import * as Queue from "effect/Queue"; |
| 21 | import * as Schema from "effect/Schema"; |
| 22 | import * as Scope from "effect/Scope"; |
| 23 | import * as Stream from "effect/Stream"; |
| 24 | |
| 25 | import * as ServerSettings from "../serverSettings.ts"; |
| 26 | import { |
| 27 | makeSpeechTextCorrector, |
| 28 | normalizeSpeechCustomWords, |
| 29 | speechRecognitionWords, |
| 30 | } from "./customWords.ts"; |
| 31 | import { removeSpeechFillerWords } from "./fillerWords.ts"; |
| 32 | import { |
| 33 | DEFAULT_SPEECH_MODEL_ID, |
| 34 | effectiveSpeechLanguage, |
| 35 | getSpeechModel, |
| 36 | SPEECH_MODELS, |
| 37 | type SpeechModel, |
| 38 | } from "./model.ts"; |
| 39 | import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts"; |
| 40 | import { nativeSpeechLanguage } from "./nativeLanguage.ts"; |
| 41 | import { SILERO_VAD } from "./silero.ts"; |
| 42 | import { isSpeechPlatformSupported } from "./platform.ts"; |
| 43 | import * as SpeechModelStore from "./SpeechModelStore.ts"; |
| 44 | |
| 45 | const SAMPLE_RATE = 16_000; |
| 46 | const MAX_SPEECH_DURATION_SECONDS = 5 * 60; |
| 47 | export const MAX_SPEECH_BYTES = |
| 48 | SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * MAX_SPEECH_DURATION_SECONDS; |
| 49 | |
| 50 | export class SpeechInvalidAudioError extends Schema.TaggedError<SpeechInvalidAudioError>()( |
| 51 | "SpeechInvalidAudioError", |
| 52 | { byteLength: Schema.Number, message: Schema.String }, |
| 53 | ) {} |
| 54 | |
| 55 | export function decodeSpeechPcm(pcmBytes: Uint8Array): Float32Array { |
| 56 | if ( |
| 57 | pcmBytes.byteLength === 0 || |
| 58 | pcmBytes.byteLength > MAX_SPEECH_BYTES || |
| 59 | pcmBytes.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0 |
| 60 | ) { |
| 61 | throw new SpeechInvalidAudioError({ |
| 62 | byteLength: pcmBytes.byteLength, |
| 63 | message: "audio must contain at most five minutes of 16 kHz mono Float32 PCM", |
| 64 | }); |
| 65 | } |
| 66 | // WebSocket frames can be pooled Node Buffers whose backing storage is larger |
| 67 | // than the frame. Copy the view so only the received bytes are decoded. |
| 68 | const pcm = new Float32Array(Uint8Array.from(pcmBytes).buffer); |
| 69 | for (const sample of pcm) { |
| 70 | if (!Number.isFinite(sample) || sample < -1 || sample > 1) { |
| 71 | throw new SpeechInvalidAudioError({ |
| 72 | byteLength: pcmBytes.byteLength, |
| 73 | message: "audio contains an invalid PCM sample", |
| 74 | }); |
| 75 | } |
| 76 | } |
| 77 | return pcm; |
| 78 | } |
| 79 | |
| 80 | export class SpeechOperationError extends Schema.TaggedError<SpeechOperationError>()( |
| 81 | "SpeechOperationError", |
| 82 | { operation: Schema.String, cause: Schema.Defect() }, |
| 83 | ) { |
| 84 | override get message(): string { |
| 85 | return `Environment speech ${this.operation} failed.`; |
| 86 | } |
| 87 | } |
| 88 | |
| 89 | export class SpeechUnsupportedPlatformError extends Schema.TaggedError<SpeechUnsupportedPlatformError>()( |
| 90 | "SpeechUnsupportedPlatformError", |
| 91 | { platform: Schema.String, architecture: Schema.String }, |
| 92 | ) {} |
| 93 | |
| 94 | export class SpeechBusyError extends Schema.TaggedError<SpeechBusyError>()("SpeechBusyError", { |
| 95 | operation: Schema.String, |
| 96 | }) {} |
| 97 | |
| 98 | export class SpeechModelNotFoundError extends Schema.TaggedError<SpeechModelNotFoundError>()( |
| 99 | "SpeechModelNotFoundError", |
| 100 | { modelId: Schema.String }, |
| 101 | ) {} |
| 102 | |
| 103 | export class SpeechDownloadCancelledError extends Schema.TaggedError<SpeechDownloadCancelledError>()( |
| 104 | "SpeechDownloadCancelledError", |
| 105 | { modelId: Schema.String }, |
| 106 | ) {} |
| 107 | |
| 108 | export type SpeechError = |
| 109 | | SpeechOperationError |
| 110 | | SpeechInvalidAudioError |
| 111 | | SpeechUnsupportedPlatformError |
| 112 | | SpeechModelNotFoundError |
| 113 | | SpeechDownloadCancelledError |
| 114 | | SpeechBusyError; |
| 115 | |
| 116 | const isSpeechError = Schema.is( |
| 117 | Schema.Union([ |
| 118 | SpeechInvalidAudioError, |
| 119 | SpeechUnsupportedPlatformError, |
| 120 | SpeechBusyError, |
| 121 | SpeechModelNotFoundError, |
| 122 | SpeechDownloadCancelledError, |
| 123 | SpeechOperationError, |
| 124 | ]), |
| 125 | ); |
| 126 | |
| 127 | type LoadedModel = Awaited<ReturnType<typeof loadNativeSpeechModel>>; |
| 128 | type SpeechModels = { readonly models: ReadonlyArray<EnvironmentSpeechModel> }; |
| 129 | |
| 130 | export type SpeechStream = { |
| 131 | readonly feed: ( |
| 132 | pcm: Uint8Array, |
| 133 | ) => Effect.Effect<Awaited<ReturnType<LoadedModel["feed"]>>, SpeechError>; |
| 134 | readonly finish: Effect.Effect<string, SpeechError>; |
| 135 | }; |
| 136 | |
| 137 | export class SpeechService extends Context.Service< |
| 138 | SpeechService, |
| 139 | { |
| 140 | readonly status: Effect.Effect<EnvironmentSpeechStatus, SpeechError>; |
| 141 | readonly models: (includeBenchmarks: boolean) => Effect.Effect<SpeechModels, SpeechError>; |
| 142 | readonly downloadModel: ( |
| 143 | modelId: string, |
| 144 | ) => Effect.Effect<Required<EnvironmentSpeechModelDownload>, SpeechError>; |
| 145 | readonly selectModel: (modelId: string) => Effect.Effect<SpeechModels, SpeechError>; |
| 146 | readonly cancelDownload: (modelId: string) => Effect.Effect<SpeechModels, SpeechError>; |
| 147 | readonly removeModel: (modelId: string) => Effect.Effect<SpeechModels, SpeechError>; |
| 148 | /** Uses catalog model files already on this machine, from a file or folder path. */ |
| 149 | readonly linkModels: (source: string) => Effect.Effect<SpeechModels, SpeechError>; |
| 150 | readonly prepareModel: Effect.Effect<void, SpeechError>; |
| 151 | readonly transcribe: ( |
| 152 | pcmBytes: Uint8Array, |
| 153 | options: SpeechTranscriptionOptions, |
| 154 | ) => Effect.Effect<string, SpeechError>; |
| 155 | readonly startStream: ( |
| 156 | options: SpeechTranscriptionOptions, |
| 157 | ) => Effect.Effect<SpeechStream, SpeechError, Scope.Scope>; |
| 158 | readonly updateAcceleration: ( |
| 159 | acceleration: SpeechAcceleration, |
| 160 | ) => Effect.Effect<EnvironmentSpeechStatus, SpeechError>; |
| 161 | readonly updateModelUnloadTimeout: ( |
| 162 | timeout: SpeechModelUnloadTimeout, |
| 163 | ) => Effect.Effect<EnvironmentSpeechStatus, SpeechError>; |
| 164 | } |
| 165 | >()("t3/speech/SpeechService") {} |
| 166 | |
| 167 | const UNLOAD_MILLISECONDS: Record<Exclude<SpeechModelUnloadTimeout, "never">, number> = { |
| 168 | immediately: 0, |
| 169 | min_2: 2 * 60_000, |
| 170 | min_5: 5 * 60_000, |
| 171 | min_10: 10 * 60_000, |
| 172 | min_15: 15 * 60_000, |
| 173 | hour_1: 60 * 60_000, |
| 174 | }; |
| 175 | |
| 176 | const make = Effect.gen(function* () { |
| 177 | const platform = yield* HostProcess.Platform; |
| 178 | const architecture = yield* HostProcess.Architecture; |
| 179 | const serverSettings = yield* ServerSettings.ServerSettingsService; |
| 180 | const store = yield* SpeechModelStore.SpeechModelStore; |
| 181 | const scope = yield* Effect.scope; |
| 182 | const clock = yield* Clock.Clock; |
| 183 | const now = () => clock.currentTimeMillisUnsafe(); |
| 184 | const libc = platform === "linux" ? yield* HostProcess.LinuxLibc : undefined; |
| 185 | const unsupportedReason = isSpeechPlatformSupported(platform, architecture, libc) |
| 186 | ? null |
| 187 | : `voice transcription is not available on ${platform}-${architecture}`; |
| 188 | |
| 189 | let model: LoadedModel | undefined; |
| 190 | let loadedModelId: string | undefined; |
| 191 | let loadedAcceleration: string | undefined; |
| 192 | let autoCpuModel: string | undefined; |
| 193 | let loading: Promise<LoadedModel> | undefined; |
| 194 | const discoverGpuDevices = yield* Effect.tryPromise({ |
| 195 | try: () => listNativeSpeechGpuDevices(), |
| 196 | catch: (cause) => new SpeechOperationError({ operation: "discover GPU devices", cause }), |
| 197 | }).pipe(Effect.cachedWithTTL((exit) => (Exit.isSuccess(exit) ? Duration.infinity : "1 minute"))); |
| 198 | let downloading: |
| 199 | | { |
| 200 | modelId: string; |
| 201 | downloaded: number; |
| 202 | verifying: boolean; |
| 203 | fiber: Deferred.Deferred<Fiber.Fiber<string, SpeechError>>; |
| 204 | } |
| 205 | | undefined; |
| 206 | let activeTranscriptions = 0; |
| 207 | // One native request at a time; the slot is a promise so unload waits for it. |
| 208 | let activeOperation: Promise<unknown> | undefined; |
| 209 | const operationBusy = () => Boolean(activeOperation || downloading); |
| 210 | let closing = false; |
| 211 | // @effect-diagnostics-next-line abortControllerInEffect:off - Cached native workers must also stop on service shutdown. |
| 212 | const lifetime = new AbortController(); |
| 213 | let unloadTimeout: SpeechModelUnloadTimeout = "min_15"; |
| 214 | |
| 215 | // Deadlines are absolute: the consumer fiber registers its sleep a few scheduler |
| 216 | // hops after the request, so a relative delay would drift. |
| 217 | const unloadRequests = yield* Queue.sliding<{ deadline: number } | null>(1); |
| 218 | const clearUnloadTimer = () => Queue.offerUnsafe(unloadRequests, null); |
| 219 | const scheduleUnload = (minimumDelayMs = 0) => { |
| 220 | clearUnloadTimer(); |
| 221 | if (!model || closing || unloadTimeout === "never") return; |
| 222 | Queue.offerUnsafe(unloadRequests, { |
| 223 | deadline: now() + Math.max(minimumDelayMs, UNLOAD_MILLISECONDS[unloadTimeout]), |
| 224 | }); |
| 225 | }; |
| 226 | |
| 227 | const forgetModel = (target: LoadedModel) => { |
| 228 | if (model !== target) return; |
| 229 | model = undefined; |
| 230 | loadedModelId = undefined; |
| 231 | loadedAcceleration = undefined; |
| 232 | }; |
| 233 | const disposeModel = async () => { |
| 234 | const target = model; |
| 235 | if (!target) return; |
| 236 | forgetModel(target); |
| 237 | await target.dispose(); |
| 238 | }; |
| 239 | // Stops the isolated process without waiting for native work to settle. |
| 240 | const dropModel = (target: LoadedModel) => { |
| 241 | forgetModel(target); |
| 242 | void Promise.resolve() |
| 243 | .then(() => target.dispose()) |
| 244 | .catch(() => undefined); |
| 245 | }; |
| 246 | |
| 247 | // Auto placement may land on a GPU the native engine cannot actually run; one retry on |
| 248 | // this host's CPU recovers it. A GPU the person chose, and CPU itself, report the error. |
| 249 | const canRetryOnCpu = (acceleration: SpeechAcceleration, loaded: LoadedModel) => |
| 250 | acceleration === "auto" && loaded.backend.toLowerCase() !== "cpu"; |
| 251 | |
| 252 | yield* Effect.addFinalizer(() => |
| 253 | Effect.promise(async () => { |
| 254 | closing = true; |
| 255 | clearUnloadTimer(); |
| 256 | lifetime.abort(); |
| 257 | await disposeModel(); |
| 258 | }), |
| 259 | ); |
| 260 | |
| 261 | yield* Stream.fromQueue(unloadRequests).pipe( |
| 262 | Stream.switchMap((request) => |
| 263 | request === null |
| 264 | ? Stream.empty |
| 265 | : Stream.fromEffect( |
| 266 | Effect.gen(function* () { |
| 267 | yield* Effect.sleep(Math.max(0, request.deadline - now())); |
| 268 | if (closing) return; |
| 269 | if (operationBusy() || activeTranscriptions || loading) { |
| 270 | scheduleUnload(1000); |
| 271 | return; |
| 272 | } |
| 273 | yield* Effect.promise(() => disposeModel().catch(() => undefined)).pipe( |
| 274 | Effect.uninterruptible, |
| 275 | ); |
| 276 | }), |
| 277 | ), |
| 278 | ), |
| 279 | Stream.runDrain, |
| 280 | Effect.forkScoped, |
| 281 | ); |
| 282 | |
| 283 | type SettingsSnapshot = Effect.Success< |
| 284 | ServerSettings.ServerSettingsService["Service"]["getSettings"] |
| 285 | >; |
| 286 | |
| 287 | const selectedModel = (settings: SettingsSnapshot): SpeechModel => |
| 288 | getSpeechModel(settings.speechModelId) ?? getSpeechModel(DEFAULT_SPEECH_MODEL_ID)!; |
| 289 | |
| 290 | let silenceRetryAt = 0; |
| 291 | const acquireSilenceModel = Effect.fn("SpeechService.acquireSilenceModel")(function* ( |
| 292 | purpose: "installation" | "recording", |
| 293 | ) { |
| 294 | if (purpose === "recording" && now() < silenceRetryAt) return undefined; |
| 295 | const path = yield* store.download(SILERO_VAD).pipe( |
| 296 | Effect.timeout(purpose === "recording" ? "15 seconds" : "2 minutes"), |
| 297 | Effect.orElseSucceed(() => undefined), |
| 298 | ); |
| 299 | if (path !== undefined) { |
| 300 | silenceRetryAt = 0; |
| 301 | return path; |
| 302 | } |
| 303 | silenceRetryAt = now() + 10 * 60_000; |
| 304 | yield* Effect.logWarning("Silero unavailable; using raw audio with bounded ASR processing."); |
| 305 | }); |
| 306 | |
| 307 | // Publish the reservation before forking: a cached transfer can finish before forkIn returns. |
| 308 | const downloadFiber = Effect.fn("SpeechService.downloadFiber")(function* (modelId: string) { |
| 309 | const definition = getSpeechModel(modelId); |
| 310 | if (!definition) return yield* new SpeechModelNotFoundError({ modelId }); |
| 311 | if (closing || activeOperation || (downloading && downloading.modelId !== modelId)) |
| 312 | return yield* new SpeechBusyError({ operation: "model download" }); |
| 313 | if (downloading) return yield* Deferred.await(downloading.fiber); |
| 314 | const transfer = { |
| 315 | modelId, |
| 316 | downloaded: 0, |
| 317 | verifying: false, |
| 318 | fiber: Deferred.makeUnsafe<Fiber.Fiber<string, SpeechError>>(), |
| 319 | }; |
| 320 | downloading = transfer; |
| 321 | const fiber = yield* store |
| 322 | .download(definition, { |
| 323 | onProgress: (downloaded) => { |
| 324 | transfer.downloaded = downloaded; |
| 325 | transfer.verifying = false; |
| 326 | }, |
| 327 | onVerificationStart: () => { |
| 328 | transfer.verifying = true; |
| 329 | }, |
| 330 | }) |
| 331 | .pipe( |
| 332 | (modelDownload) => |
| 333 | Effect.all([modelDownload, acquireSilenceModel("installation")], { concurrency: 2 }), |
| 334 | Effect.map(([modelPath]) => modelPath), |
| 335 | Effect.mapError((cause) => speechError("model download", cause)), |
| 336 | Effect.forkIn(scope), |
| 337 | ); |
| 338 | fiber.addObserver(() => { |
| 339 | if (downloading !== transfer) return; |
| 340 | downloading = undefined; |
| 341 | scheduleUnload(); |
| 342 | }); |
| 343 | yield* Deferred.succeed(transfer.fiber, fiber); |
| 344 | return fiber; |
| 345 | }, Effect.uninterruptible); |
| 346 | |
| 347 | const download = Effect.fn("SpeechService.download")(function* (modelId: string) { |
| 348 | const exit = yield* Fiber.await(yield* downloadFiber(modelId)); |
| 349 | if (Exit.isSuccess(exit)) return exit.value; |
| 350 | return yield* Cause.hasInterruptsOnly(exit.cause) |
| 351 | ? new SpeechDownloadCancelledError({ modelId }) |
| 352 | : Effect.failCause(exit.cause); |
| 353 | }); |
| 354 | |
| 355 | const installedModelPath = (definition: SpeechModel) => |
| 356 | store.installedPath(definition).pipe( |
| 357 | Effect.filterOrFail( |
| 358 | (modelPath) => modelPath !== undefined, |
| 359 | () => new SpeechModelNotFoundError({ modelId: definition.id }), |
| 360 | ), |
| 361 | ); |
| 362 | |
| 363 | const loadModel = async ( |
| 364 | modelPath: string, |
| 365 | definition: SpeechModel, |
| 366 | signal = lifetime.signal, |
| 367 | requestedAcceleration = "auto", |
| 368 | ) => { |
| 369 | const acceleration = |
| 370 | requestedAcceleration === "auto" && autoCpuModel === definition.id |
| 371 | ? "cpu" |
| 372 | : requestedAcceleration; |
| 373 | if (model && loadedModelId === definition.id && loadedAcceleration === acceleration) { |
| 374 | scheduleUnload(); |
| 375 | return model; |
| 376 | } |
| 377 | if (loading) return loading; |
| 378 | await disposeModel(); |
| 379 | signal.throwIfAborted(); |
| 380 | const pending = loadNativeSpeechModel(modelPath, signal, undefined, acceleration).then( |
| 381 | async (loaded) => { |
| 382 | if (closing || signal.aborted) { |
| 383 | await loaded.dispose(); |
| 384 | signal.throwIfAborted(); |
| 385 | throw new SpeechBusyError({ operation: "model preparation" }); |
| 386 | } |
| 387 | model = loaded; |
| 388 | loadedModelId = definition.id; |
| 389 | loadedAcceleration = acceleration; |
| 390 | scheduleUnload(); |
| 391 | return loaded; |
| 392 | }, |
| 393 | ); |
| 394 | const release = () => { |
| 395 | if (loading === pending) loading = undefined; |
| 396 | }; |
| 397 | loading = pending; |
| 398 | signal.addEventListener("abort", release, { once: true }); |
| 399 | try { |
| 400 | return await pending; |
| 401 | } finally { |
| 402 | signal.removeEventListener("abort", release); |
| 403 | release(); |
| 404 | } |
| 405 | }; |
| 406 | |
| 407 | const speechError = (operation: string, cause: unknown): SpeechError => |
| 408 | isSpeechError(cause) ? cause : new SpeechOperationError({ operation, cause }); |
| 409 | |
| 410 | const attempt = <A>(operation: string, run: (signal: AbortSignal) => Promise<A>) => |
| 411 | Effect.tryPromise({ try: run, catch: (cause) => speechError(operation, cause) }); |
| 412 | |
| 413 | // The busy slot is released by a finalizer, so interrupting the fiber frees it at |
| 414 | // once instead of waiting for native work that may never settle. |
| 415 | const exclusive = <A, E, R>(operation: string, effect: Effect.Effect<A, E, R>) => |
| 416 | Effect.uninterruptibleMask((restore) => |
| 417 | Effect.suspend<A, E | SpeechBusyError, R>(() => { |
| 418 | if (closing || operationBusy()) return Effect.fail(new SpeechBusyError({ operation })); |
| 419 | const gate = Promise.withResolvers<void>(); |
| 420 | activeOperation = gate.promise; |
| 421 | return restore(effect).pipe( |
| 422 | Effect.ensuring( |
| 423 | Effect.sync(() => { |
| 424 | if (activeOperation === gate.promise) activeOperation = undefined; |
| 425 | scheduleUnload( |
| 426 | operation === "model preparation" ? MAX_SPEECH_DURATION_SECONDS * 1000 : 0, |
| 427 | ); |
| 428 | gate.resolve(); |
| 429 | }), |
| 430 | ), |
| 431 | ); |
| 432 | }), |
| 433 | ); |
| 434 | |
| 435 | const readSettings = (operation: string) => |
| 436 | serverSettings.getSettings.pipe( |
| 437 | Effect.tap((settings) => |
| 438 | Effect.sync(() => { |
| 439 | if (settings.speechModelId !== autoCpuModel || settings.speechAcceleration !== "auto") |
| 440 | autoCpuModel = undefined; |
| 441 | if (unloadTimeout !== settings.speechModelUnloadTimeout) { |
| 442 | unloadTimeout = settings.speechModelUnloadTimeout; |
| 443 | scheduleUnload(); |
| 444 | } |
| 445 | }), |
| 446 | ), |
| 447 | Effect.mapError((cause) => new SpeechOperationError({ operation, cause })), |
| 448 | ); |
| 449 | |
| 450 | const writeSettings = ( |
| 451 | operation: string, |
| 452 | patch: Parameters<ServerSettings.ServerSettingsService["Service"]["updateSettings"]>[0], |
| 453 | ) => |
| 454 | serverSettings |
| 455 | .updateSettings(patch) |
| 456 | .pipe(Effect.mapError((cause) => new SpeechOperationError({ operation, cause }))); |
| 457 | |
| 458 | const currentStatus = Effect.fn("SpeechService.currentStatus")(function* ( |
| 459 | settings: SettingsSnapshot, |
| 460 | ): Effect.fn.Return<EnvironmentSpeechStatus> { |
| 461 | if (unsupportedReason) return { supported: false, reason: unsupportedReason }; |
| 462 | const definition = selectedModel(settings); |
| 463 | const ready = yield* store.isReady(definition); |
| 464 | const gpuDevices = yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => [])); |
| 465 | const autoUsesCpu = |
| 466 | gpuDevices.length === 0 || |
| 467 | autoCpuModel === definition.id || |
| 468 | (loadedModelId === definition.id && |
| 469 | loadedAcceleration === "auto" && |
| 470 | model?.backend.toLowerCase() === "cpu"); |
| 471 | return { |
| 472 | supported: true, |
| 473 | state: activeTranscriptions > 0 ? "transcribing" : ready ? "ready" : "missing-model", |
| 474 | modelId: definition.id, |
| 475 | model: definition.name, |
| 476 | size: definition.size, |
| 477 | supportsStreaming: definition.supportsStreaming, |
| 478 | supportsBufferedStreaming: true, |
| 479 | supportsModelLinking: true, |
| 480 | supportsTranslation: definition.supportsTranslation, |
| 481 | languages: definition.languages, |
| 482 | supportsLanguageDetection: definition.supportsLanguageDetection, |
| 483 | acceleration: settings.speechAcceleration, |
| 484 | benchmarkTier: |
| 485 | settings.speechAcceleration === "cpu" || |
| 486 | (settings.speechAcceleration === "auto" && autoUsesCpu) |
| 487 | ? "cpu" |
| 488 | : "accelerated", |
| 489 | modelUnloadTimeout: settings.speechModelUnloadTimeout, |
| 490 | gpuDevices, |
| 491 | }; |
| 492 | }); |
| 493 | |
| 494 | const listModels = Effect.fn("SpeechService.listModels")(function* ( |
| 495 | settings: SettingsSnapshot, |
| 496 | includeBenchmarks: boolean, |
| 497 | ): Effect.fn.Return<SpeechModels> { |
| 498 | const selected = selectedModel(settings); |
| 499 | const models = yield* Effect.forEach( |
| 500 | SPEECH_MODELS, |
| 501 | (definition) => |
| 502 | Effect.all([store.isReady(definition), store.linkedFrom(definition)]).pipe( |
| 503 | Effect.map(([ready, linkedFrom]): EnvironmentSpeechModel => { |
| 504 | const operation = downloading?.modelId === definition.id ? downloading : undefined; |
| 505 | return { |
| 506 | id: definition.id, |
| 507 | name: definition.name, |
| 508 | description: definition.description, |
| 509 | size: definition.size, |
| 510 | languages: definition.languages, |
| 511 | accuracy: definition.accuracy, |
| 512 | speed: definition.speed, |
| 513 | ...(includeBenchmarks ? { benchmark: definition.benchmark } : {}), |
| 514 | recommended: definition.recommended, |
| 515 | supportsStreaming: definition.supportsStreaming, |
| 516 | supportsLanguageDetection: definition.supportsLanguageDetection, |
| 517 | supportsTranslation: definition.supportsTranslation, |
| 518 | active: ready && selected.id === definition.id, |
| 519 | state: operation |
| 520 | ? operation.verifying |
| 521 | ? "verifying" |
| 522 | : "downloading" |
| 523 | : ready |
| 524 | ? "installed" |
| 525 | : "downloadable", |
| 526 | ...(operation ? { downloaded: operation.downloaded } : {}), |
| 527 | ...(ready && linkedFrom ? { linkedFrom } : {}), |
| 528 | }; |
| 529 | }), |
| 530 | ), |
| 531 | { concurrency: "unbounded" }, |
| 532 | ); |
| 533 | return { models }; |
| 534 | }); |
| 535 | |
| 536 | const freshStatus = (operation: string) => |
| 537 | readSettings(operation).pipe(Effect.flatMap(currentStatus)); |
| 538 | const readModels = (includeBenchmarks: boolean) => |
| 539 | readSettings("model listing").pipe( |
| 540 | Effect.flatMap((settings) => listModels(settings, includeBenchmarks)), |
| 541 | ); |
| 542 | const modelsEffect = readModels(false); |
| 543 | |
| 544 | const requireSupported = Effect.suspend(() => |
| 545 | unsupportedReason |
| 546 | ? Effect.fail(new SpeechUnsupportedPlatformError({ platform, architecture })) |
| 547 | : Effect.void, |
| 548 | ); |
| 549 | |
| 550 | const finalizeTranscript = ( |
| 551 | text: string, |
| 552 | options: SpeechTranscriptionOptions, |
| 553 | language: string | undefined, |
| 554 | ) => { |
| 555 | return ( |
| 556 | options.speechRemoveFillerWords |
| 557 | ? removeSpeechFillerWords(text, language, options.speechCorrectionWord) |
| 558 | : text |
| 559 | ).trim(); |
| 560 | }; |
| 561 | |
| 562 | const prepareModel = readSettings("model preparation").pipe( |
| 563 | Effect.flatMap((settings) => |
| 564 | exclusive( |
| 565 | "model preparation", |
| 566 | requireSupported.pipe( |
| 567 | Effect.andThen(installedModelPath(selectedModel(settings))), |
| 568 | Effect.flatMap((modelPath) => |
| 569 | attempt("model preparation", (signal) => |
| 570 | loadModel( |
| 571 | modelPath, |
| 572 | selectedModel(settings), |
| 573 | AbortSignal.any([signal, lifetime.signal]), |
| 574 | settings.speechAcceleration, |
| 575 | ), |
| 576 | ).pipe( |
| 577 | Effect.timeout("2 minutes"), |
| 578 | Effect.catchTags({ |
| 579 | TimeoutError: (cause) => |
| 580 | Effect.fail(new SpeechOperationError({ operation: "model preparation", cause })), |
| 581 | }), |
| 582 | ), |
| 583 | ), |
| 584 | ), |
| 585 | ), |
| 586 | ), |
| 587 | ); |
| 588 | |
| 589 | const silenceModelPath = Effect.fn("SpeechService.silenceModelPath")(function* ( |
| 590 | loaded: LoadedModel, |
| 591 | options: SpeechTranscriptionOptions, |
| 592 | nativeStreaming: boolean, |
| 593 | ) { |
| 594 | if (!options.speechSilenceFiltering && (nativeStreaming || loaded.supportsLongForm)) |
| 595 | return undefined; |
| 596 | return yield* acquireSilenceModel("recording"); |
| 597 | }); |
| 598 | |
| 599 | const transcribe: SpeechService["Service"]["transcribe"] = Effect.fn("SpeechService.transcribe")( |
| 600 | function* (pcmBytes, options) { |
| 601 | const settings = yield* readSettings("transcription"); |
| 602 | return yield* exclusive( |
| 603 | "transcription", |
| 604 | Effect.gen(function* () { |
| 605 | yield* requireSupported; |
| 606 | const pcm = yield* Effect.try({ |
| 607 | try: () => decodeSpeechPcm(pcmBytes), |
| 608 | catch: (cause) => speechError("transcription", cause), |
| 609 | }); |
| 610 | if (pcm.length === 0) return ""; |
| 611 | activeTranscriptions += 1; |
| 612 | return yield* Effect.gen(function* () { |
| 613 | const definition = selectedModel(settings); |
| 614 | const language = effectiveSpeechLanguage(definition, options.speechLanguage); |
| 615 | const translateToEnglish = |
| 616 | options.speechTranslateToEnglish && |
| 617 | definition.supportsTranslation && |
| 618 | language !== "en"; |
| 619 | const modelPath = yield* installedModelPath(definition); |
| 620 | const loaded = yield* Effect.tryPromise({ |
| 621 | try: (signal) => |
| 622 | loadModel( |
| 623 | modelPath, |
| 624 | definition, |
| 625 | AbortSignal.any([signal, lifetime.signal]), |
| 626 | settings.speechAcceleration, |
| 627 | ), |
| 628 | catch: (cause) => speechError("model preparation", cause), |
| 629 | }); |
| 630 | const vadPath = yield* silenceModelPath(loaded, options, false); |
| 631 | const dictionary = normalizeSpeechCustomWords(options.speechCustomWords); |
| 632 | const customWords = dictionary.map(({ term }) => term); |
| 633 | const recognitionWords = speechRecognitionWords(options.speechHints, customWords); |
| 634 | // The vocabulary capability guarantees plain transcription, not translation. |
| 635 | const vocabularyFor = (inference: LoadedModel) => |
| 636 | inference.supportsVocabulary && !translateToEnglish ? recognitionWords : []; |
| 637 | const transcribeWith = (inference: LoadedModel) => |
| 638 | Effect.tryPromise({ |
| 639 | try: (signal) => { |
| 640 | // Interruption kills the isolated process; the next request reloads the model. |
| 641 | signal.addEventListener("abort", () => dropModel(inference), { once: true }); |
| 642 | const hint = nativeSpeechLanguage(inference.languages, language); |
| 643 | return inference.transcribe( |
| 644 | pcm, |
| 645 | { |
| 646 | timestamps: "none", |
| 647 | ...(hint ? { language: hint } : {}), |
| 648 | ...(translateToEnglish |
| 649 | ? { task: "translate" as const, targetLanguage: "en" as const } |
| 650 | : {}), |
| 651 | ...(vocabularyFor(inference).length > 0 |
| 652 | ? { vocabulary: recognitionWords } |
| 653 | : {}), |
| 654 | }, |
| 655 | options.speechSilenceFiltering, |
| 656 | vadPath, |
| 657 | ); |
| 658 | }, |
| 659 | catch: (cause) => { |
| 660 | dropModel(inference); |
| 661 | return speechError("inference", cause); |
| 662 | }, |
| 663 | }); |
| 664 | let inference = loaded; |
| 665 | const result = yield* transcribeWith(loaded).pipe( |
| 666 | Effect.catch((error) => { |
| 667 | if (!canRetryOnCpu(settings.speechAcceleration, loaded)) return Effect.fail(error); |
| 668 | return Effect.tryPromise({ |
| 669 | try: (signal) => |
| 670 | loadModel( |
| 671 | modelPath, |
| 672 | definition, |
| 673 | AbortSignal.any([signal, lifetime.signal]), |
| 674 | "cpu", |
| 675 | ), |
| 676 | catch: (cause) => speechError("model preparation", cause), |
| 677 | }).pipe( |
| 678 | Effect.flatMap((cpu) => { |
| 679 | autoCpuModel = definition.id; |
| 680 | inference = cpu; |
| 681 | return transcribeWith(cpu); |
| 682 | }), |
| 683 | ); |
| 684 | }), |
| 685 | ); |
| 686 | const correct = makeSpeechTextCorrector( |
| 687 | vocabularyFor(inference).length > 0 ? [] : customWords, |
| 688 | dictionary, |
| 689 | ); |
| 690 | return finalizeTranscript( |
| 691 | correct(result.text), |
| 692 | options, |
| 693 | translateToEnglish ? "en" : language === "auto" ? undefined : language, |
| 694 | ); |
| 695 | }).pipe( |
| 696 | Effect.ensuring( |
| 697 | Effect.sync(() => { |
| 698 | activeTranscriptions -= 1; |
| 699 | }), |
| 700 | ), |
| 701 | ); |
| 702 | }), |
| 703 | ); |
| 704 | }, |
| 705 | ); |
| 706 | |
| 707 | // Stream setup shares the public startStream span. |
| 708 | const openStream = Effect.fnUntraced(function* (options: SpeechTranscriptionOptions) { |
| 709 | const operation = "streaming transcription"; |
| 710 | yield* requireSupported; |
| 711 | const released = Promise.withResolvers<void>(); |
| 712 | // @effect-diagnostics-next-line abortControllerInEffect:off - Stream cancellation must abort native work independently of the caller fiber. |
| 713 | const controller = new AbortController(); |
| 714 | const signal = AbortSignal.any([controller.signal, lifetime.signal]); |
| 715 | let finished = false; |
| 716 | let closed = false; |
| 717 | let started = false; |
| 718 | let loaded: LoadedModel | undefined; |
| 719 | yield* Effect.acquireRelease( |
| 720 | Effect.suspend(() => { |
| 721 | if (closing || operationBusy()) return Effect.fail(new SpeechBusyError({ operation })); |
| 722 | activeOperation = released.promise; |
| 723 | activeTranscriptions += 1; |
| 724 | return Effect.void; |
| 725 | }), |
| 726 | () => |
| 727 | Effect.sync(() => { |
| 728 | activeTranscriptions -= 1; |
| 729 | if (activeOperation === released.promise) activeOperation = undefined; |
| 730 | scheduleUnload(); |
| 731 | released.resolve(); |
| 732 | }), |
| 733 | ); |
| 734 | yield* Effect.addFinalizer(() => |
| 735 | Effect.gen(function* () { |
| 736 | closed = true; |
| 737 | if (!finished) { |
| 738 | const dispose = Effect.promise(async () => { |
| 739 | const pending = loading; |
| 740 | controller.abort(); |
| 741 | await pending?.catch(() => undefined); |
| 742 | if (loaded) { |
| 743 | forgetModel(loaded); |
| 744 | await loaded.dispose(); |
| 745 | } |
| 746 | }); |
| 747 | if (loaded && started) { |
| 748 | yield* attempt("reset stream", () => loaded!.reset()).pipe( |
| 749 | Effect.timeout("2500 millis"), |
| 750 | Effect.catch(() => dispose), |
| 751 | ); |
| 752 | } else { |
| 753 | yield* dispose; |
| 754 | } |
| 755 | } |
| 756 | }), |
| 757 | ); |
| 758 | const settings = yield* readSettings(operation); |
| 759 | const definition = selectedModel(settings); |
| 760 | const language = effectiveSpeechLanguage(definition, options.speechLanguage); |
| 761 | const dictionary = normalizeSpeechCustomWords(options.speechCustomWords); |
| 762 | const customWords = dictionary.map(({ term }) => term); |
| 763 | const recognitionWords = speechRecognitionWords(options.speechHints, customWords); |
| 764 | let correct = makeSpeechTextCorrector(customWords, dictionary); |
| 765 | const modelPath = yield* installedModelPath(definition); |
| 766 | let streamModel = yield* attempt(operation, () => |
| 767 | loadModel(modelPath, definition, signal, settings.speechAcceleration), |
| 768 | ); |
| 769 | loaded = streamModel; |
| 770 | const vadPath = yield* silenceModelPath(streamModel, options, streamModel.supportsStreaming); |
| 771 | let translateToEnglish = false; |
| 772 | const beginStream = async () => { |
| 773 | translateToEnglish = |
| 774 | options.speechTranslateToEnglish && |
| 775 | !streamModel.supportsStreaming && |
| 776 | streamModel.supportsTranslation && |
| 777 | language !== "en"; |
| 778 | const vocabulary = |
| 779 | streamModel.supportsVocabulary && !translateToEnglish && recognitionWords.length |
| 780 | ? recognitionWords |
| 781 | : undefined; |
| 782 | const hint = nativeSpeechLanguage(streamModel.languages, language); |
| 783 | await streamModel.begin( |
| 784 | { |
| 785 | timestamps: "none", |
| 786 | ...(hint ? { language: hint } : {}), |
| 787 | ...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}), |
| 788 | ...(vocabulary ? { vocabulary } : {}), |
| 789 | }, |
| 790 | options.speechSilenceFiltering, |
| 791 | vadPath, |
| 792 | ); |
| 793 | correct = makeSpeechTextCorrector(vocabulary ? [] : customWords, dictionary); |
| 794 | started = true; |
| 795 | }; |
| 796 | // Audio already fed is kept so a CPU restart can replay it; the limit caps it at one recording. |
| 797 | const received: Float32Array[] = []; |
| 798 | let retriedOnCpu = false; |
| 799 | const recoverOnCpu = async (cause: unknown) => { |
| 800 | started = false; |
| 801 | if (closed || signal.aborted || retriedOnCpu) throw cause; |
| 802 | if (!canRetryOnCpu(settings.speechAcceleration, streamModel)) throw cause; |
| 803 | retriedOnCpu = true; |
| 804 | forgetModel(streamModel); |
| 805 | await streamModel.dispose(); |
| 806 | if (closed || signal.aborted) throw cause; |
| 807 | streamModel = await loadModel(modelPath, definition, signal, "cpu"); |
| 808 | autoCpuModel = definition.id; |
| 809 | loaded = streamModel; |
| 810 | await beginStream(); |
| 811 | let update: Awaited<ReturnType<LoadedModel["feed"]>> | undefined; |
| 812 | for (const chunk of received) { |
| 813 | if (closed || signal.aborted) throw new Error("Speech stream is closed."); |
| 814 | update = await streamModel.feed(chunk); |
| 815 | } |
| 816 | return update; |
| 817 | }; |
| 818 | yield* attempt(operation, async () => { |
| 819 | try { |
| 820 | await beginStream(); |
| 821 | } catch (cause) { |
| 822 | await recoverOnCpu(cause); |
| 823 | } |
| 824 | }); |
| 825 | let byteLength = 0; |
| 826 | let busy = false; |
| 827 | // Client revisions stay monotonic across a native restart, whose own counter resets. |
| 828 | let revisionBase = 0; |
| 829 | let lastNativeRevision = 0; |
| 830 | const run = <A>(work: () => Promise<A>) => |
| 831 | attempt(operation, async () => { |
| 832 | if (busy || finished || closed || !started || signal.aborted) |
| 833 | throw new Error("Speech stream is not ready."); |
| 834 | busy = true; |
| 835 | try { |
| 836 | return await work(); |
| 837 | } finally { |
| 838 | busy = false; |
| 839 | } |
| 840 | }); |
| 841 | return { |
| 842 | feed: (bytes: Uint8Array) => |
| 843 | run(async () => { |
| 844 | if ( |
| 845 | bytes.byteLength > SPEECH_STREAM_MAX_CHUNK_BYTES || |
| 846 | byteLength + bytes.byteLength > MAX_SPEECH_BYTES |
| 847 | ) |
| 848 | throw new SpeechInvalidAudioError({ |
| 849 | byteLength: bytes.byteLength, |
| 850 | message: "Speech stream audio limit exceeded.", |
| 851 | }); |
| 852 | const pcm = decodeSpeechPcm(bytes); |
| 853 | byteLength += bytes.byteLength; |
| 854 | received.push(pcm); |
| 855 | let update: Awaited<ReturnType<LoadedModel["feed"]>> | undefined; |
| 856 | try { |
| 857 | update = await streamModel.feed(pcm); |
| 858 | } catch (cause) { |
| 859 | update = await recoverOnCpu(cause); |
| 860 | revisionBase += lastNativeRevision; |
| 861 | lastNativeRevision = 0; |
| 862 | } |
| 863 | if (update) lastNativeRevision = update.revision; |
| 864 | return { |
| 865 | revision: revisionBase + (update?.revision ?? lastNativeRevision), |
| 866 | text: update?.text |
| 867 | ? { |
| 868 | committed: correct(update.text.committed), |
| 869 | tentative: correct(update.text.tentative), |
| 870 | } |
| 871 | : null, |
| 872 | }; |
| 873 | }), |
| 874 | finish: run(async () => { |
| 875 | let text: string; |
| 876 | try { |
| 877 | text = await streamModel.finish().catch(async (cause: unknown) => { |
| 878 | await recoverOnCpu(cause); |
| 879 | if (closed || signal.aborted) throw cause; |
| 880 | return streamModel.finish(); |
| 881 | }); |
| 882 | } catch (cause) { |
| 883 | started = false; |
| 884 | throw cause; |
| 885 | } |
| 886 | finished = true; |
| 887 | return finalizeTranscript( |
| 888 | correct(text), |
| 889 | options, |
| 890 | translateToEnglish ? "en" : language === "auto" ? undefined : language, |
| 891 | ); |
| 892 | }), |
| 893 | }; |
| 894 | }); |
| 895 | |
| 896 | return SpeechService.of({ |
| 897 | status: freshStatus("status"), |
| 898 | models: readModels, |
| 899 | prepareModel, |
| 900 | transcribe, |
| 901 | startStream: Effect.fn("SpeechService.startStream")(function* (options) { |
| 902 | const session = yield* Scope.fork(yield* Effect.scope); |
| 903 | return yield* openStream(options).pipe( |
| 904 | Scope.provide(session), |
| 905 | Effect.onError((cause) => Scope.close(session, Exit.failCause(cause))), |
| 906 | ); |
| 907 | }), |
| 908 | downloadModel: Effect.fn("SpeechService.downloadModel")(function* (modelId) { |
| 909 | yield* requireSupported; |
| 910 | const cancelled = yield* download(modelId).pipe( |
| 911 | Effect.as(false), |
| 912 | Effect.catchTags({ SpeechDownloadCancelledError: () => Effect.succeed(true) }), |
| 913 | ); |
| 914 | return { ...(yield* modelsEffect), cancelled }; |
| 915 | }), |
| 916 | selectModel: Effect.fn("SpeechService.selectModel")((modelId) => |
| 917 | exclusive( |
| 918 | "model selection", |
| 919 | Effect.gen(function* () { |
| 920 | const definition = getSpeechModel(modelId); |
| 921 | if (!definition || !(yield* store.isReady(definition))) |
| 922 | return yield* new SpeechModelNotFoundError({ modelId }); |
| 923 | yield* attempt("model selection", () => disposeModel()); |
| 924 | yield* writeSettings("model selection", { speechModelId: modelId }); |
| 925 | }), |
| 926 | ).pipe(Effect.andThen(modelsEffect)), |
| 927 | ), |
| 928 | cancelDownload: Effect.fn("SpeechService.cancelDownload")((modelId) => |
| 929 | Effect.suspend(() => |
| 930 | downloading?.modelId === modelId |
| 931 | ? Deferred.await(downloading.fiber).pipe(Effect.flatMap(Fiber.interrupt)) |
| 932 | : Effect.void, |
| 933 | ).pipe(Effect.andThen(modelsEffect)), |
| 934 | ), |
| 935 | removeModel: Effect.fn("SpeechService.removeModel")((modelId) => |
| 936 | exclusive( |
| 937 | "model removal", |
| 938 | Effect.gen(function* () { |
| 939 | const settings = yield* readSettings("model removal"); |
| 940 | const definition = getSpeechModel(modelId); |
| 941 | if (!definition) return yield* new SpeechModelNotFoundError({ modelId }); |
| 942 | yield* attempt("model removal", async () => { |
| 943 | await loading?.catch(() => undefined); |
| 944 | if (loadedModelId === modelId) await disposeModel(); |
| 945 | }); |
| 946 | yield* store |
| 947 | .remove(definition) |
| 948 | .pipe(Effect.mapError((cause) => speechError("model removal", cause))); |
| 949 | if (selectedModel(settings).id !== modelId) return; |
| 950 | const candidates = yield* Effect.forEach( |
| 951 | SPEECH_MODELS.filter((candidate) => candidate.id !== modelId), |
| 952 | (candidate) => |
| 953 | store.isReady(candidate).pipe(Effect.map((ready) => ({ candidate, ready }))), |
| 954 | { concurrency: "unbounded" }, |
| 955 | ); |
| 956 | const replacement = candidates.find(({ ready }) => ready)?.candidate; |
| 957 | if (replacement) yield* writeSettings("model removal", { speechModelId: replacement.id }); |
| 958 | }), |
| 959 | ).pipe(Effect.andThen(modelsEffect)), |
| 960 | ), |
| 961 | linkModels: Effect.fn("SpeechService.linkModels")(function* (source) { |
| 962 | yield* requireSupported; |
| 963 | const settings = yield* readSettings("model linking"); |
| 964 | const linked = yield* exclusive( |
| 965 | "model linking", |
| 966 | store |
| 967 | .link(SPEECH_MODELS, source) |
| 968 | .pipe(Effect.mapError((cause) => speechError("model linking", cause))), |
| 969 | ); |
| 970 | const first = linked[0]; |
| 971 | if (first === undefined) return yield* new SpeechModelNotFoundError({ modelId: source }); |
| 972 | if (!(yield* store.isReady(selectedModel(settings)))) |
| 973 | yield* writeSettings("model linking", { speechModelId: first }); |
| 974 | return yield* modelsEffect; |
| 975 | }), |
| 976 | updateAcceleration: Effect.fn("SpeechService.updateAcceleration")((acceleration) => |
| 977 | writeSettings("acceleration update", { speechAcceleration: acceleration }).pipe( |
| 978 | Effect.andThen(freshStatus("acceleration update")), |
| 979 | ), |
| 980 | ), |
| 981 | updateModelUnloadTimeout: Effect.fn("SpeechService.updateModelUnloadTimeout")((timeout) => |
| 982 | writeSettings("model unload timeout update", { speechModelUnloadTimeout: timeout }).pipe( |
| 983 | Effect.andThen(freshStatus("model unload timeout update")), |
| 984 | ), |
| 985 | ), |
| 986 | }); |
| 987 | }); |
| 988 | |
| 989 | export const layer = Layer.effect(SpeechService, make); |