@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 998 lines 39.5 KB
Raw
1 import { beforeEach, expect, vi } from "vite-plus/test";
2 import { it } from "@effect/vitest";
3 import * as NodeServices from "@effect/platform-node/NodeServices";
4 import * as Effect from "effect/Effect";
5 import * as Deferred from "effect/Deferred";
6 import * as TestClock from "effect/testing/TestClock";
7 import * as Fiber from "effect/Fiber";
8 import * as Layer from "effect/Layer";
9 import * as Result from "effect/Result";
10 import * as Scheduler from "effect/Scheduler";
11 import * as Scope from "effect/Scope";
12 import * as Exit from "effect/Exit";
13 import * as PlatformError from "effect/PlatformError";
14 import * as HostProcess from "@t3tools/shared/HostProcess";
15 import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS } from "@t3tools/contracts";
16 import * as ServerSettings from "../serverSettings.ts";
17 import { listNativeSpeechGpuDevices } from "./native.ts";
18 import * as SpeechModelStore from "./SpeechModelStore.ts";
19 import * as SpeechService from "./SpeechService.ts";
20 import { SILERO_VAD } from "./silero.ts";
21
22 const native = vi.hoisted(() => ({
23 backend: "Vulkan0",
24 supportsStreaming: true,
25 supportsLongForm: false,
26 supportsTranslation: true,
27 supportsVocabulary: false,
28 languages: ["en", "fr"] as readonly string[],
29 begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}),
30 feed: vi.fn(async (_pcm: Float32Array) => ({
31 revision: 1,
32 text: { committed: "", tentative: "hello" },
33 })),
34 finish: vi.fn(async () => "hello"),
35 reset: vi.fn(async () => {}),
36 dispose: vi.fn(),
37 transcribe: vi.fn(async (_pcm: Float32Array, _options?: unknown) => ({ text: "hello" })),
38 }));
39 const loadNative = vi.hoisted(() =>
40 vi.fn(
41 async (
42 _path: string,
43 _signal: AbortSignal,
44 _moduleUrl?: string,
45 backend: "auto" | "cpu" = "auto",
46 ) => {
47 native.backend = backend === "cpu" ? "CPU" : "Vulkan0";
48 return native;
49 },
50 ),
51 );
52 const readyModels = vi.hoisted(() => new Set(["test-model", "fallback-model"]));
53 const modelDefinitions = vi.hoisted(() => [
54 {
55 id: "test-model",
56 name: "test",
57 description: "test",
58 size: 4,
59 languages: ["en", "fr"],
60 accuracy: 1,
61 benchmark: {
62 accuracy: { en: { grade: "A-", errorPercent: 4 } },
63 secondsPer30s: { cpu: 3, accelerated: 1.2 },
64 },
65 speed: 1,
66 recommended: true,
67 supportsStreaming: true,
68 supportsLanguageDetection: false,
69 supportsTranslation: true,
70 },
71 {
72 id: "fallback-model",
73 name: "fallback",
74 description: "fallback",
75 size: 8,
76 languages: ["en", "fr"],
77 accuracy: 2,
78 speed: 2,
79 recommended: false,
80 supportsStreaming: false,
81 supportsLanguageDetection: false,
82 supportsTranslation: true,
83 },
84 ]);
85 vi.mock("./native.ts", () => ({
86 loadNativeSpeechModel: loadNative,
87 listNativeSpeechGpuDevices: vi.fn(async () => [{ id: '["vulkan","gpu-1"]', name: "Test GPU" }]),
88 }));
89 vi.mock("./model.ts", () => ({
90 DEFAULT_SPEECH_MODEL_ID: "test-model",
91 SPEECH_MODELS: modelDefinitions,
92 getSpeechModel: (modelId: string) => modelDefinitions.find((model) => model.id === modelId),
93 effectiveSpeechLanguage: (model: { languages: string[] }, intent: string) =>
94 intent !== "auto" && model.languages.includes(intent) ? intent : "en",
95 }));
96 const download = vi.fn<SpeechModelStore.SpeechModelStore["Service"]["download"]>(() =>
97 Effect.succeed("test.gguf"),
98 );
99 const store = Layer.succeed(
100 SpeechModelStore.SpeechModelStore,
101 SpeechModelStore.SpeechModelStore.of({
102 isReady: (model) => Effect.sync(() => readyModels.has(model.id)),
103 installedPath: (model) =>
104 Effect.sync(() => (readyModels.has(model.id) ? "test.gguf" : undefined)),
105 download,
106 link: () => Effect.succeed([]),
107 linkedFrom: () => Effect.undefined,
108 remove: (model) =>
109 Effect.sync(() => {
110 readyModels.delete(model.id);
111 }),
112 }),
113 );
114 const layer = SpeechService.layer.pipe(
115 Layer.provide(store),
116 Layer.provideMerge(ServerSettings.layerTest({ speechModelId: "test-model" })),
117 Layer.provide(NodeServices.layer),
118 );
119 const pcm = () => new Uint8Array(new Float32Array([0.25]).buffer);
120
121 beforeEach(() => {
122 vi.clearAllMocks();
123 loadNative.mockReset();
124 native.finish.mockReset();
125 native.backend = "Vulkan0";
126 native.supportsVocabulary = false;
127 native.supportsStreaming = true;
128 native.languages = ["en", "fr"];
129 readyModels.clear();
130 readyModels.add("test-model");
131 readyModels.add("fallback-model");
132 });
133
134 it("decodes a pooled Node Buffer without reading adjacent bytes", () => {
135 const pool = Buffer.allocUnsafe(64);
136 pool.fill(0xff);
137 Buffer.from(new Float32Array([0.25]).buffer).copy(pool, 20);
138 expect([...SpeechService.decodeSpeechPcm(pool.subarray(20, 24))]).toEqual([0.25]);
139 });
140
141 it("rejects empty, misaligned, non-finite, and oversized PCM as invalid audio", () => {
142 for (const input of [
143 new Uint8Array(),
144 new Uint8Array(3),
145 new Uint8Array(new Float32Array([Number.NaN]).buffer),
146 new Uint8Array(SpeechService.MAX_SPEECH_BYTES + 4),
147 ]) {
148 expect(() => SpeechService.decodeSpeechPcm(input)).toThrow(
149 expect.objectContaining({ _tag: "SpeechInvalidAudioError" }),
150 );
151 }
152 });
153
154 it.effect.each(["CPU", "Vulkan0"])(
155 "prepares a batch model on %s before audio arrives and reuses it for transcription",
156 (backend) =>
157 Effect.gen(function* () {
158 const speech = yield* SpeechService.SpeechService;
159 yield* speech.selectModel("fallback-model");
160 loadNative.mockImplementationOnce(async () => {
161 native.backend = backend;
162 return native;
163 });
164 yield* speech.prepareModel;
165 expect(yield* speech.status).toMatchObject({
166 benchmarkTier: backend === "CPU" ? "cpu" : "accelerated",
167 });
168 expect(loadNative).toHaveBeenCalledTimes(1);
169 expect(native.transcribe).not.toHaveBeenCalled();
170 native.supportsStreaming = false;
171 yield* Effect.gen(function* () {
172 const stream = yield* speech.startStream({
173 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
174 speechLanguage: "fr",
175 speechTranslateToEnglish: true,
176 speechSilenceFiltering: false,
177 });
178 expect(native.begin).toHaveBeenLastCalledWith(
179 { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
180 false,
181 "test.gguf",
182 );
183 expect(yield* stream.finish).toBe("hello");
184 }).pipe(Effect.scoped);
185 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
186 expect(loadNative).toHaveBeenCalledTimes(1);
187 }).pipe(Effect.provide(layer)),
188 );
189
190 it.effect("sends the chosen language in the spelling the loaded model advertises", () =>
191 Effect.gen(function* () {
192 const speech = yield* SpeechService.SpeechService;
193 // Nemotron Streaming 3.5 advertises regional tags while the catalog lists bare codes.
194 native.languages = ["en-US", "en-GB", "fr-FR"];
195 yield* Effect.gen(function* () {
196 const stream = yield* speech.startStream({
197 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
198 speechLanguage: "fr",
199 });
200 expect(native.begin).toHaveBeenLastCalledWith(
201 { timestamps: "none", language: "fr-FR" },
202 true,
203 "test.gguf",
204 );
205 expect(yield* stream.finish).toBe("hello");
206 }).pipe(Effect.scoped);
207 native.supportsStreaming = false;
208 expect(
209 yield* speech.transcribe(pcm(), {
210 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
211 speechLanguage: "fr",
212 }),
213 ).toBe("hello");
214 expect(native.transcribe).toHaveBeenLastCalledWith(
215 expect.any(Float32Array),
216 { timestamps: "none", language: "fr-FR" },
217 true,
218 "test.gguf",
219 );
220 // A language the model does not advertise reaches it unchanged, so the model reports it.
221 yield* speech.updateAcceleration("cpu");
222 native.languages = ["en-US"];
223 native.transcribe.mockRejectedValueOnce(new Error("transcribe_run: unsupported language"));
224 const rejected = yield* Effect.result(
225 speech.transcribe(pcm(), { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "fr" }),
226 );
227 expect(Result.isFailure(rejected) && rejected.failure).toMatchObject({
228 _tag: "SpeechOperationError",
229 operation: "inference",
230 });
231 expect(native.transcribe).toHaveBeenLastCalledWith(
232 expect.any(Float32Array),
233 { timestamps: "none", language: "fr" },
234 true,
235 "test.gguf",
236 );
237 expect(loadNative.mock.calls.map(([path, , , acceleration]) => [path, acceleration])).toEqual([
238 ["test.gguf", "auto"],
239 ["test.gguf", "cpu"],
240 ]);
241 }).pipe(Effect.provide(layer)),
242 );
243
244 it.effect.each(["cancellation", "timeout"] as const)(
245 "releases a stalled model preparation on %s",
246 (reason) =>
247 Effect.gen(function* () {
248 const speech = yield* SpeechService.SpeechService;
249 const started = Promise.withResolvers<AbortSignal>();
250 const stalled = Promise.withResolvers<typeof native>();
251 const disposed = Promise.withResolvers<void>();
252 const abandoned = { ...native, dispose: vi.fn(() => disposed.resolve()) };
253 loadNative.mockImplementationOnce((_path, signal) => {
254 started.resolve(signal);
255 return stalled.promise;
256 });
257 const request = yield* speech.prepareModel.pipe(Effect.forkChild);
258 const signal = yield* Effect.promise(() => started.promise);
259 try {
260 if (reason === "cancellation") {
261 yield* Fiber.interrupt(request).pipe(Effect.forkChild({ startImmediately: true }));
262 }
263 yield* TestClock.adjust(reason === "timeout" ? "2 minutes" : 0);
264 expect(signal.aborted).toBe(true);
265 const exit = yield* Fiber.await(request);
266 expect(Exit.isFailure(exit)).toBe(true);
267 if (reason === "timeout") {
268 expect(Exit.findErrorOption(exit)).toMatchObject({
269 value: { _tag: "SpeechOperationError", operation: "model preparation" },
270 });
271 }
272 yield* speech.selectModel("fallback-model");
273 const transcribe = speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
274 expect(yield* transcribe).toBe("hello");
275 stalled.resolve(abandoned);
276 yield* Effect.promise(() => disposed.promise);
277 expect(yield* transcribe).toBe("hello");
278 expect(loadNative).toHaveBeenCalledTimes(2);
279 } finally {
280 stalled.resolve(abandoned);
281 }
282 }).pipe(Effect.provide(layer)),
283 );
284
285 it.effect("rejects model mutations outside the offered registry", () =>
286 Effect.gen(function* () {
287 const speech = yield* SpeechService.SpeechService;
288 for (const id of [
289 "handy-computer/canary-1b-gguf",
290 "handy-computer/granite-speech-5.0-470m-turboctc-nc-gguf",
291 "unknown-model",
292 ]) {
293 for (const mutation of [speech.downloadModel, speech.selectModel, speech.removeModel]) {
294 const result = yield* Effect.result(mutation(id));
295 expect(Result.isFailure(result) && result.failure).toMatchObject({
296 _tag: "SpeechModelNotFoundError",
297 modelId: id,
298 });
299 }
300 }
301 expect(download).not.toHaveBeenCalled();
302 const current = (yield* speech.models(false)).models.find((model) => model.active);
303 expect(current?.id).toBe("test-model");
304 expect(current).not.toHaveProperty("benchmark");
305 expect(
306 (yield* speech.models(true)).models.find((model) => model.active)?.benchmark?.secondsPer30s,
307 ).toEqual({ cpu: 3, accelerated: 1.2 });
308 }).pipe(Effect.provide(layer)),
309 );
310
311 it.effect("keeps a detached download exclusive until the transfer finishes", () =>
312 Effect.gen(function* () {
313 const speech = yield* SpeechService.SpeechService;
314 const started = yield* Deferred.make<void>();
315 const finish = yield* Deferred.make<void>();
316 const helperStarted = yield* Deferred.make<void>();
317 let helperStartedDuringInstall = false;
318 let helperInterrupted = false;
319 download
320 .mockImplementationOnce(() =>
321 Deferred.succeed(started, undefined).pipe(
322 Effect.andThen(Deferred.await(finish)),
323 Effect.as("test.gguf"),
324 ),
325 )
326 .mockImplementationOnce(() =>
327 Deferred.isDone(finish).pipe(
328 Effect.tap((done) =>
329 Effect.sync(() => {
330 helperStartedDuringInstall = !done;
331 }),
332 ),
333 Effect.andThen(Deferred.succeed(helperStarted, undefined)),
334 Effect.andThen(Effect.never),
335 Effect.onInterrupt(() =>
336 Effect.sync(() => {
337 helperInterrupted = true;
338 }),
339 ),
340 ),
341 );
342 const request = yield* speech.downloadModel("test-model").pipe(Effect.forkChild);
343 yield* Deferred.await(started);
344 yield* Deferred.await(helperStarted);
345 yield* Fiber.interrupt(request);
346 yield* Deferred.succeed(finish, undefined);
347 yield* TestClock.adjust("16 seconds");
348 expect(helperInterrupted).toBe(false);
349 const removed = yield* Effect.result(speech.removeModel("test-model"));
350 expect(Result.isFailure(removed) && removed.failure).toMatchObject({ _tag: "SpeechBusyError" });
351 const join = yield* speech
352 .downloadModel("test-model")
353 .pipe(Effect.forkChild({ startImmediately: true }));
354 yield* TestClock.adjust("2 minutes");
355 expect(helperInterrupted).toBe(true);
356 yield* Fiber.join(join);
357 expect(download.mock.calls.map(([model]) => model.id)).toEqual(["test-model", SILERO_VAD.id]);
358 expect(helperStartedDuringInstall).toBe(true);
359 yield* speech.removeModel("test-model");
360 expect(readyModels.has("test-model")).toBe(false);
361 }).pipe(Effect.provide(layer)),
362 );
363
364 it.effect("reports cancellation even after the speech weights are installed", () =>
365 Effect.gen(function* () {
366 const speech = yield* SpeechService.SpeechService;
367 readyModels.delete("fallback-model");
368 const weightsInstalled = yield* Deferred.make<void>();
369 const helperStarted = yield* Deferred.make<void>();
370 download
371 .mockImplementationOnce(() =>
372 Effect.sync(() => {
373 readyModels.add("fallback-model");
374 }).pipe(
375 Effect.andThen(Deferred.succeed(weightsInstalled, undefined)),
376 Effect.as("test.gguf"),
377 ),
378 )
379 .mockImplementationOnce(() =>
380 Deferred.succeed(helperStarted, undefined).pipe(Effect.andThen(Effect.never)),
381 );
382
383 const request = yield* speech.downloadModel("fallback-model").pipe(Effect.forkChild);
384 yield* Deferred.await(weightsInstalled);
385 yield* Deferred.await(helperStarted);
386 yield* speech.cancelDownload("fallback-model");
387
388 expect(yield* Fiber.join(request)).toMatchObject({
389 cancelled: true,
390 models: expect.arrayContaining([
391 expect.objectContaining({ id: "fallback-model", state: "installed", active: false }),
392 expect.objectContaining({ id: "test-model", active: true }),
393 ]),
394 });
395 }).pipe(Effect.provide(layer)),
396 );
397
398 it.effect("preserves the current model when another client removes a selection candidate", () =>
399 Effect.gen(function* () {
400 const speech = yield* SpeechService.SpeechService;
401 yield* speech.removeModel("fallback-model");
402 const selection = yield* Effect.result(speech.selectModel("fallback-model"));
403 expect(Result.isFailure(selection) && selection.failure).toMatchObject({
404 _tag: "SpeechModelNotFoundError",
405 modelId: "fallback-model",
406 });
407 expect(yield* speech.status).toMatchObject({ supported: true, modelId: "test-model" });
408 }).pipe(Effect.provide(layer)),
409 );
410
411 it.effect("bounds missing Silero recovery during recording and continues with raw audio", () =>
412 Effect.gen(function* () {
413 const helperStarted = yield* Deferred.make<void>();
414 const helperInterrupted = yield* Deferred.make<void>();
415 download.mockImplementationOnce(() =>
416 Deferred.succeed(helperStarted, undefined).pipe(
417 Effect.andThen(Effect.never),
418 Effect.onInterrupt(() => Deferred.succeed(helperInterrupted, undefined)),
419 ),
420 );
421 const speech = yield* SpeechService.SpeechService;
422 const recording = yield* speech
423 .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
424 .pipe(Effect.forkChild);
425 yield* Deferred.await(helperStarted);
426 yield* TestClock.adjust("14999 millis");
427 expect(native.begin).not.toHaveBeenCalled();
428 yield* TestClock.adjust("1 millis");
429 yield* Deferred.await(helperInterrupted);
430 const stream = yield* Fiber.join(recording);
431 expect(native.begin).toHaveBeenLastCalledWith(expect.anything(), true, undefined);
432 yield* stream.feed(pcm());
433 expect(yield* stream.finish).toBe("hello");
434 }).pipe(Effect.scoped, Effect.provide(layer)),
435 );
436
437 it.effect("keeps the installed speech model usable when optional Silero acquisition fails", () =>
438 Effect.gen(function* () {
439 download
440 .mockImplementationOnce(() => Effect.succeed("test.gguf"))
441 .mockImplementationOnce(() =>
442 Effect.fail(
443 new PlatformError.PlatformError(
444 new PlatformError.SystemError({
445 _tag: "NotFound",
446 module: "FileSystem",
447 method: "open",
448 }),
449 ),
450 ),
451 );
452 const speech = yield* SpeechService.SpeechService;
453 const installed = yield* speech.downloadModel("test-model");
454 expect(installed).toHaveProperty("cancelled", false);
455 expect(installed.models.find((model) => model.id === "test-model")?.state).toBe("installed");
456 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
457 yield* Effect.scoped(
458 Effect.gen(function* () {
459 const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
460 expect(yield* stream.finish).toBe("hello");
461 }),
462 );
463 expect(download).toHaveBeenCalledTimes(2);
464 yield* TestClock.adjust("10 minutes");
465 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
466 expect(download).toHaveBeenCalledTimes(3);
467 }).pipe(Effect.provide(layer)),
468 );
469
470 it.effect(
471 "allows another model operation after an already-installed download completes immediately",
472 () =>
473 Effect.gen(function* () {
474 const speech = yield* SpeechService.SpeechService;
475 const models = yield* speech.downloadModel("test-model").pipe(
476 Effect.provideService(Scheduler.Scheduler, {
477 executionMode: "sync",
478 shouldYield: () => false,
479 makeDispatcher: () => ({ scheduleTask: (task) => task(), flush: () => {} }),
480 }),
481 );
482 expect(models.models.find((model) => model.id === "test-model")?.state).toBe("installed");
483 yield* speech.selectModel("fallback-model");
484 }).pipe(Effect.provide(layer)),
485 );
486
487 it.effect("honors exact aliases and phrase boundaries in batch and streamed text", () =>
488 Effect.gen(function* () {
489 const speech = yield* SpeechService.SpeechService;
490 const options = {
491 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
492 speechCustomWords: [
493 { term: "AI", aliases: ["ay eye"] },
494 { term: "Bob", aliases: ["Robert"] },
495 { term: "Roberts", aliases: [] },
496 { term: "O'Reilly", aliases: ['"the publisher"'] },
497 { term: "GitHub", aliases: [] },
498 { term: "McDonald’s", aliases: [] },
499 { term: "King’s Cross", aliases: [] },
500 { term: "C#", aliases: [] },
501 { term: "React", aliases: ["瑞安特"] },
502 { term: "Kubernetes", aliases: [] },
503 ],
504 };
505 const spoken =
506 'a test with a i and ay eye. Robert, Roberts. "the publisher" and the publisher. (git hub), git (hub). ' +
507 "git hub's API, git hub’s API, git’s hub, mac donald's menu, king's cross. vitamin c and plan c. " +
508 "我用github和瑞安特,部署到kubernetis上。Robert用git hub。githubで管理する。";
509 const expected =
510 "a test with AI and AI. Bob, Roberts. O'Reilly and the publisher. (GitHub), git (hub). " +
511 "GitHub's API, GitHub’s API, git’s hub, McDonald’s menu, King’s Cross. vitamin c and plan c. " +
512 "我用GitHub和React,部署到Kubernetes上。Bob用GitHub。GitHubで管理する。";
513 native.transcribe.mockResolvedValueOnce({ text: spoken });
514 native.finish.mockResolvedValueOnce(spoken);
515 expect(yield* speech.transcribe(pcm(), options)).toBe(expected);
516 const stream = yield* speech.startStream(options);
517 expect(yield* stream.finish).toBe(expected);
518 }).pipe(Effect.scoped, Effect.provide(layer)),
519 );
520
521 it.effect("uses vocabulary only for supported transcription and always applies aliases", () =>
522 Effect.gen(function* () {
523 for (const supportsVocabulary of [false, true]) {
524 for (const translate of [false, true]) {
525 yield* Effect.gen(function* () {
526 native.supportsVocabulary = supportsVocabulary;
527 const speech = yield* SpeechService.SpeechService;
528 const options = {
529 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
530 speechLanguage: "fr" as const,
531 speechTranslateToEnglish: translate,
532 speechCustomWords: [
533 { term: "GitHub", aliases: [] },
534 { term: "Bob", aliases: ["Robert"] },
535 { term: "T3 Code", aliases: [] },
536 ],
537 speechHints: ["Server"],
538 };
539 const vocabulary =
540 supportsVocabulary && !translate ? ["Server", "GitHub", "Bob", "T3 Code"] : undefined;
541 native.transcribe.mockResolvedValueOnce({ text: "git hub and Robert on the server" });
542 expect(yield* speech.transcribe(pcm(), options)).toBe(
543 vocabulary ? "git hub and Bob on the server" : "GitHub and Bob on the server",
544 );
545 const delivered = native.transcribe.mock.lastCall?.[1];
546 expect(delivered).toEqual({
547 timestamps: "none",
548 language: "fr",
549 ...(translate ? { task: "translate", targetLanguage: "en" } : {}),
550 ...(vocabulary ? { vocabulary } : {}),
551 });
552 if (translate) return;
553 native.feed.mockResolvedValueOnce({
554 revision: 1,
555 text: { committed: "git hub", tentative: "Robert" },
556 });
557 native.finish.mockResolvedValueOnce("git hub and Robert on the server");
558 const stream = yield* speech.startStream(options);
559 expect(yield* stream.feed(pcm())).toEqual({
560 revision: 1,
561 text: { committed: vocabulary ? "git hub" : "GitHub", tentative: "Bob" },
562 });
563 expect(yield* stream.finish).toBe(
564 vocabulary ? "git hub and Bob on the server" : "GitHub and Bob on the server",
565 );
566 expect(native.begin.mock.lastCall).toEqual([
567 { timestamps: "none", language: "fr", ...(vocabulary ? { vocabulary } : {}) },
568 true,
569 "test.gguf",
570 ]);
571 }).pipe(Effect.scoped, Effect.provide(layer));
572 }
573 }
574 }),
575 );
576
577 it.effect("unloads after the chosen idle period and reloads on the next recording", () =>
578 Effect.gen(function* () {
579 const speech = yield* SpeechService.SpeechService;
580 yield* speech.updateModelUnloadTimeout("min_2");
581 yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
582 yield* TestClock.adjust(2 * 60_000 - 1);
583 expect(native.dispose).not.toHaveBeenCalled();
584 yield* TestClock.adjust(1);
585 expect(native.dispose).toHaveBeenCalledOnce();
586 yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
587 expect(loadNative).toHaveBeenCalledTimes(2);
588 }).pipe(Effect.provide(layer)),
589 );
590
591 it.effect.each([
592 {
593 cue: undefined,
594 remove: true,
595 text: "First paragraph.\n\nSecond paragraph.",
596 expected: "First paragraph.\n\nSecond paragraph.",
597 },
598 {
599 cue: undefined,
600 remove: true,
601 text: "First um\n\nuh Second.",
602 expected: "First\n\nSecond.",
603 },
604 {
605 cue: undefined,
606 remove: true,
607 text: "First um\r\n\r\nuh Second.",
608 expected: "First\r\n\r\nSecond.",
609 },
610 { cue: "um", remove: true, text: "Make it red, um, blue.", expected: "Make it red, um, blue." },
611 {
612 cue: "uh no",
613 remove: true,
614 text: "Make it red, uh no, blue.",
615 expected: "Make it red, uh no, blue.",
616 },
617 {
618 cue: "uh no",
619 remove: true,
620 text: "Uh, Make it red, UH NO, blue. Uh nope.",
621 expected: "Make it red, UH NO, blue. nope.",
622 },
623 {
624 cue: " UM ",
625 remove: true,
626 text: "Uh, Make it red, um, blue.",
627 expected: "Make it red, um, blue.",
628 },
629 {
630 cue: undefined,
631 remove: true,
632 text: "Uh, Make it red, um, blue.",
633 expected: "Make it red, blue.",
634 },
635 {
636 cue: "um",
637 remove: false,
638 text: "Uh, Make it red, um, blue.",
639 expected: "Uh, Make it red, um, blue.",
640 },
641 ])(
642 "preserves only an explicit correction cue during batch and stream filler removal: $cue / $remove",
643 ({ cue, remove, text, expected }) =>
644 Effect.gen(function* () {
645 const speech = yield* SpeechService.SpeechService;
646 const options = {
647 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
648 speechLanguage: "en",
649 speechRemoveFillerWords: remove,
650 speechCustomWords: [{ term: "um", aliases: [] }],
651 ...(cue === undefined ? {} : { speechCorrectionWord: cue }),
652 };
653 native.transcribe.mockResolvedValueOnce({ text });
654 native.finish.mockResolvedValueOnce(text);
655 expect(yield* speech.transcribe(pcm(), options)).toBe(expected);
656 const stream = yield* speech.startStream(options);
657 expect(yield* stream.finish).toBe(expected);
658 }).pipe(Effect.scoped, Effect.provide(layer)),
659 );
660
661 it.effect("holds model ownership until the stream scope closes and preserves silence", () =>
662 Effect.gen(function* () {
663 const speech = yield* SpeechService.SpeechService;
664 yield* Effect.gen(function* () {
665 const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
666 const busy = yield* Effect.result(speech.selectModel("fallback-model"));
667 expect(Result.isFailure(busy) && busy.failure).toMatchObject({ _tag: "SpeechBusyError" });
668 yield* stream.feed(new Uint8Array(new Float32Array(160).buffer));
669 expect(native.feed.mock.calls[0]?.[0]).toHaveLength(160);
670 expect(yield* stream.finish).toBe("hello");
671 }).pipe(Effect.scoped);
672 expect(native.dispose).not.toHaveBeenCalled();
673 yield* speech.selectModel("fallback-model");
674 expect(native.dispose).toHaveBeenCalledOnce();
675 }).pipe(Effect.provide(layer)),
676 );
677
678 it.effect("resets cancelled streams and reuses the loaded model", () =>
679 Effect.gen(function* () {
680 const speech = yield* SpeechService.SpeechService;
681 yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS).pipe(Effect.scoped);
682 expect(native.reset).toHaveBeenCalledOnce();
683 expect(native.dispose).not.toHaveBeenCalled();
684 yield* Effect.gen(function* () {
685 const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
686 yield* stream.feed(pcm());
687 yield* stream.finish;
688 }).pipe(Effect.scoped);
689 expect(loadNative).toHaveBeenCalledOnce();
690 }).pipe(Effect.provide(layer)),
691 );
692
693 it.effect("releases a failed stream start without waiting for its socket scope to close", () =>
694 Effect.gen(function* () {
695 const speech = yield* SpeechService.SpeechService;
696 const socket = yield* Scope.fork(yield* Effect.scope);
697 yield* speech.updateAcceleration("cpu");
698 native.begin.mockRejectedValueOnce(new Error("cannot start streaming"));
699 const error = yield* speech
700 .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
701 .pipe(Scope.provide(socket), Effect.flip);
702 expect(error._tag).toBe("SpeechOperationError");
703 expect(yield* speech.status).toMatchObject({ state: "ready" });
704 expect(native.dispose).toHaveBeenCalledOnce();
705 const next = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
706 yield* Scope.close(socket, Exit.void);
707 expect(yield* speech.status).toMatchObject({ state: "transcribing" });
708 expect(yield* next.finish).toBe("hello");
709 }).pipe(Effect.scoped, Effect.provide(layer)),
710 );
711
712 it.effect("closes its scope without waiting for hung native inference", () =>
713 Effect.gen(function* () {
714 const started = Promise.withResolvers<void>();
715 native.transcribe.mockImplementationOnce(() => {
716 started.resolve();
717 return new Promise(() => {});
718 });
719 yield* Effect.gen(function* () {
720 const speech = yield* SpeechService.SpeechService;
721 const request = yield* speech
722 .transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
723 .pipe(Effect.forkChild);
724 yield* Effect.promise(() => started.promise);
725 yield* Fiber.interrupt(request);
726 }).pipe(Effect.provide(layer));
727 expect(native.dispose).toHaveBeenCalledOnce();
728 }),
729 );
730
731 it.effect("reports unsupported hosts without wrapping a synthetic cause", () =>
732 Effect.gen(function* () {
733 const result = yield* Effect.gen(function* () {
734 const speech = yield* SpeechService.SpeechService;
735 return yield* Effect.result(speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS));
736 }).pipe(
737 Effect.provide(layer),
738 Effect.provideService(HostProcess.Platform, "win32"),
739 Effect.provideService(HostProcess.Architecture, "arm64"),
740 );
741 expect(Result.isFailure(result) && result.failure).toMatchObject({
742 _tag: "SpeechUnsupportedPlatformError",
743 platform: "win32",
744 architecture: "arm64",
745 });
746 }),
747 );
748
749 it.effect.each([{ devices: [] }, { devices: [{ id: "gpu", name: "Test GPU" }] }])(
750 "retries failed GPU discovery without repeatedly probing on status polls: $devices",
751 ({ devices }) =>
752 Effect.gen(function* () {
753 const speech = yield* SpeechService.SpeechService;
754 const discover = vi.mocked(listNativeSpeechGpuDevices).mockReset();
755 discover.mockRejectedValueOnce(new Error("driver not ready")).mockResolvedValueOnce(devices);
756 expect(yield* speech.status).toMatchObject({ gpuDevices: [], benchmarkTier: "cpu" });
757 yield* speech.status;
758 expect(discover).toHaveBeenCalledOnce();
759 yield* TestClock.adjust("1 minute");
760 expect(yield* speech.status).toMatchObject({
761 gpuDevices: devices,
762 benchmarkTier: devices.length ? "accelerated" : "cpu",
763 });
764 expect(yield* speech.updateAcceleration("cpu")).toMatchObject({ benchmarkTier: "cpu" });
765 expect(yield* speech.updateAcceleration('gpu:["vulkan","gpu-1"]')).toMatchObject({
766 benchmarkTier: "accelerated",
767 });
768 yield* speech.updateAcceleration("auto");
769 yield* TestClock.adjust("1 minute");
770 expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
771 expect(discover).toHaveBeenCalledTimes(2);
772 }).pipe(Effect.provide(layer)),
773 );
774
775 it.effect(
776 "retries Auto-placed inference with the dictionary and the CPU's vocabulary capability",
777 () =>
778 Effect.gen(function* () {
779 for (const cpuSupportsVocabulary of [true, false]) {
780 yield* Effect.gen(function* () {
781 native.supportsVocabulary = true;
782 loadNative.mockImplementationOnce(async () => native);
783 loadNative.mockImplementationOnce(async () => ({
784 ...native,
785 backend: "CPU",
786 supportsVocabulary: cpuSupportsVocabulary,
787 }));
788 native.transcribe
789 .mockRejectedValueOnce(new Error("worker stopped"))
790 .mockResolvedValueOnce({ text: "git hub Robert" });
791 const options = {
792 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
793 speechCustomWords: [
794 { term: "GitHub", aliases: [] },
795 { term: "Bob", aliases: ["Robert"] },
796 ],
797 };
798 const speech = yield* SpeechService.SpeechService;
799 expect(yield* speech.status).toMatchObject({ benchmarkTier: "accelerated" });
800 expect(yield* speech.transcribe(pcm(), options)).toBe(
801 cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
802 );
803 expect(yield* speech.status).toMatchObject({ benchmarkTier: "cpu" });
804 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe(
805 "hello",
806 );
807 expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]);
808 expect(native.transcribe.mock.calls.slice(-3, -1).map((call) => call[1])).toEqual([
809 { timestamps: "none", language: "en", vocabulary: ["GitHub", "Bob"] },
810 {
811 timestamps: "none",
812 language: "en",
813 ...(cpuSupportsVocabulary ? { vocabulary: ["GitHub", "Bob"] } : {}),
814 },
815 ]);
816 }).pipe(Effect.provide(layer));
817 }
818 }),
819 );
820
821 it.effect("reports a failure on a GPU the person chose instead of switching to CPU", () =>
822 Effect.gen(function* () {
823 const failure = new Error("GPU failed");
824 native.transcribe.mockRejectedValueOnce(failure);
825 const speech = yield* SpeechService.SpeechService;
826 const acceleration = 'gpu:["vulkan","gpu-1"]';
827 yield* speech.updateAcceleration(acceleration);
828 const result = yield* Effect.result(
829 speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS),
830 );
831 expect(Result.isFailure(result) && result.failure).toMatchObject({
832 _tag: "SpeechOperationError",
833 operation: "inference",
834 cause: failure,
835 });
836 expect(loadNative).toHaveBeenCalledTimes(1);
837 expect(loadNative.mock.calls[0]?.[3]).toBe(acceleration);
838 }).pipe(Effect.provide(layer)),
839 );
840
841 it.effect("replays streamed audio on CPU after an Auto-placed feed crashes, once", () =>
842 Effect.gen(function* () {
843 for (const cpuSupportsVocabulary of [true, false]) {
844 yield* Effect.gen(function* () {
845 native.supportsVocabulary = true;
846 const feedCalls = native.feed.mock.calls.length;
847 const text = { committed: "", tentative: "git hub Robert" };
848 const corrected = { committed: "", tentative: "git hub Bob" };
849 const options = {
850 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
851 speechCustomWords: [
852 { term: "GitHub", aliases: [] },
853 { term: "Bob", aliases: ["Robert"] },
854 ],
855 };
856 native.feed
857 .mockResolvedValueOnce({ revision: 3, text })
858 .mockRejectedValueOnce(new Error("worker stopped"))
859 .mockResolvedValueOnce({ revision: 1, text })
860 .mockResolvedValueOnce({ revision: 2, text })
861 .mockResolvedValueOnce({ revision: 3, text });
862 loadNative.mockImplementationOnce(async () => native);
863 loadNative.mockImplementationOnce(async () => ({
864 ...native,
865 backend: "CPU",
866 supportsVocabulary: cpuSupportsVocabulary,
867 }));
868 const speech = yield* SpeechService.SpeechService;
869 const stream = yield* speech.startStream(options);
870 expect(yield* stream.feed(pcm())).toEqual({ revision: 3, text: corrected });
871 const recovered = {
872 committed: "",
873 tentative: cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
874 };
875 expect(yield* stream.feed(pcm())).toEqual({ revision: 5, text: recovered });
876 expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]);
877 expect(native.begin.mock.calls.slice(-2).map(([options]) => options)).toEqual([
878 { timestamps: "none", language: "en", vocabulary: ["GitHub", "Bob"] },
879 {
880 timestamps: "none",
881 language: "en",
882 ...(cpuSupportsVocabulary ? { vocabulary: ["GitHub", "Bob"] } : {}),
883 },
884 ]);
885 expect(native.feed).toHaveBeenCalledTimes(feedCalls + 4);
886 expect(yield* stream.feed(pcm())).toEqual({ revision: 6, text: recovered });
887 const loads = loadNative.mock.calls.length;
888 native.finish.mockRejectedValueOnce(new Error("CPU finish failed"));
889 expect(Result.isFailure(yield* Effect.result(stream.finish))).toBe(true);
890 expect(Result.isFailure(yield* Effect.result(stream.feed(pcm())))).toBe(true);
891 expect(loadNative).toHaveBeenCalledTimes(loads);
892 }).pipe(Effect.scoped, Effect.provide(layer));
893 }
894 }),
895 );
896
897 it.effect.each([false, true])(
898 "recovers Auto stream finalization on a fresh CPU worker with retained audio: %s",
899 (withAudio) =>
900 Effect.gen(function* () {
901 const cpu = {
902 ...native,
903 backend: "CPU",
904 begin: vi.fn(async (_options?: unknown) => {}),
905 feed: vi.fn(async (_pcm: Float32Array) => ({
906 revision: 1,
907 text: { committed: "", tentative: "" },
908 })),
909 finish: vi.fn(async () => "Robert"),
910 dispose: vi.fn(),
911 };
912 const gpu = {
913 ...native,
914 finish: vi.fn(async () => {
915 throw new Error("GPU finish failed");
916 }),
917 };
918 loadNative.mockResolvedValueOnce(gpu).mockResolvedValueOnce(cpu);
919 const speech = yield* SpeechService.SpeechService;
920 const chunks = withAudio ? [new Float32Array([0.25, 0]), new Float32Array([-0.5])] : [];
921 yield* Effect.gen(function* () {
922 const stream = yield* speech.startStream({
923 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
924 speechLanguage: "fr",
925 speechCustomWords: [{ term: "Bob", aliases: ["Robert"] }],
926 });
927 for (const chunk of chunks) yield* stream.feed(new Uint8Array(chunk.buffer));
928 expect(yield* stream.finish).toBe("Bob");
929 expect(cpu.feed.mock.calls.map(([chunk]) => [...chunk])).toEqual(
930 chunks.map((chunk) => [...chunk]),
931 );
932 expect(cpu.begin).toHaveBeenCalledWith(
933 { timestamps: "none", language: "fr" },
934 true,
935 "test.gguf",
936 );
937 expect(gpu.dispose).toHaveBeenCalledOnce();
938 expect(cpu.dispose).not.toHaveBeenCalled();
939 const busy = yield* Effect.result(speech.selectModel("fallback-model"));
940 expect(Result.isFailure(busy) && busy.failure).toMatchObject({ _tag: "SpeechBusyError" });
941 }).pipe(Effect.scoped);
942 expect(loadNative.mock.calls.map(([path, , , acceleration]) => [path, acceleration])).toEqual(
943 [
944 ["test.gguf", "auto"],
945 ["test.gguf", "cpu"],
946 ],
947 );
948 yield* speech.selectModel("fallback-model");
949 expect(cpu.dispose).toHaveBeenCalledOnce();
950 }).pipe(Effect.provide(layer)),
951 );
952
953 it.effect.each(["auto", 'gpu:["vulkan","gpu-1"]', "cpu"] as const)(
954 "reports finalization failure without repeated or explicit-backend fallback: %s",
955 (acceleration) =>
956 Effect.gen(function* () {
957 const failure = new Error("finish failed");
958 const worker = {
959 ...native,
960 backend: acceleration === "cpu" ? "CPU" : "Vulkan0",
961 finish: vi.fn(async () => {
962 throw failure;
963 }),
964 };
965 const cpu = { ...worker, backend: "CPU", dispose: vi.fn() };
966 loadNative.mockResolvedValueOnce(worker);
967 if (acceleration === "auto") loadNative.mockResolvedValueOnce(cpu);
968 const speech = yield* SpeechService.SpeechService;
969 yield* speech.updateAcceleration(acceleration);
970 const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
971 expect(Result.isFailure(yield* Effect.result(stream.finish))).toBe(true);
972 expect(worker.finish).toHaveBeenCalledTimes(acceleration === "auto" ? 2 : 1);
973 expect(loadNative).toHaveBeenCalledTimes(acceleration === "auto" ? 2 : 1);
974 }).pipe(Effect.scoped, Effect.provide(layer)),
975 );
976
977 it.effect("does not restart a stream cancelled while native finalization is pending", () =>
978 Effect.gen(function* () {
979 const pending = Promise.withResolvers<string>();
980 const started = Promise.withResolvers<void>();
981 native.finish.mockImplementationOnce(() => {
982 started.resolve();
983 return pending.promise;
984 });
985 const speech = yield* SpeechService.SpeechService;
986 const streamScope = yield* Scope.make();
987 const stream = yield* speech
988 .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
989 .pipe(Effect.provideService(Scope.Scope, streamScope));
990 const request = yield* Effect.result(stream.finish).pipe(Effect.forkChild);
991 yield* Effect.promise(() => started.promise);
992 yield* Scope.close(streamScope, Exit.void);
993 pending.reject(new Error("cancelled worker stopped"));
994 expect(Result.isFailure(yield* Fiber.join(request))).toBe(true);
995 expect(loadNative).toHaveBeenCalledOnce();
996 expect(native.reset).toHaveBeenCalledOnce();
997 }).pipe(Effect.provide(layer)),
998 );