@devries-cameron20 / t3code Mirror · read-only / commits / 6b02cbff3a
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 00:45 UTC

fix(speech): match reference benchmarks to transcription hardware (#31956)

Co-authored-by: maria-rcks <maria@kuuro.net>

Utkarsh Patil committed Nov 10, 2026 at 00:27 UTC 6b02cbff3a0cd9a0f2c97007562a1ce74fd35aa0
15 files changed +501 -138
apps/mobile/src/features/settings/VoiceModelSection.tsx
+5 -2
@@ -143,12 +143,15 @@ function ModelStats(props: { readonly model: EnvironmentSpeechModel; readonly la
143 ],
144 ] as const)
145 : []),
146 - ...(data && data.secondsPer30s !== null
146 + ...(data && data.secondsPer30s.accelerated !== null
147 ? ([
148 [
149 "speed",
150 <>
151 - <Text className="font-t3-medium text-foreground">{data.secondsPer30s} s</Text> / 30 s
151 + <Text className="font-t3-medium text-foreground">
152 + {data.secondsPer30s.accelerated} s
153 + </Text>{" "}
154 + / 30 s
155 </>,
156 ],
157 ] as const)
apps/mobile/src/features/voice-input/voiceDestination.test.ts
+1
@@ -158,6 +158,7 @@ it("keeps local as the default and never reuses consent for an offline or resele
158 acceleration: "cpu",
159 modelUnloadTimeout: "never",
160 gpuDevices: [],
161 + benchmarkTier: "cpu",
162 }),
163 )
164 .mockResolvedValueOnce(Response.json({ text: "hello" }));
apps/server/scripts/speech-catalog.test.ts
+17 -2
@@ -45,9 +45,24 @@ it("reproduces the bundled registry byte for byte without upstream requests", as
45 );
46 expect(withoutFrenchOrSpeed.models[0]?.benchmark).toEqual({
47 accuracy: { en: { grade: "A-", errorPercent: 3.99 } },
48 - secondsPer30s: null,
48 + secondsPer30s: { cpu: null, accelerated: null },
49 });
50 - expect(generate().models[0]?.benchmark.secondsPer30s).toBeCloseTo(1.19, 2);
50 + expect(generate().models[0]?.benchmark.secondsPer30s.accelerated).toBeCloseTo(1.19, 2);
51 + for (const performance of [{ cpu: 2 }, { accelerated: 10 }, { cpu: 2, accelerated: 10 }]) {
52 + const result = generateSpeechCatalog(
53 + {
54 + ...catalog.data,
55 + models: [parakeet],
56 + results: { [parakeet.id]: { accuracy: {}, performance } },
57 + },
58 + "0.2.3",
59 + "0.4.1",
60 + );
61 + expect(result.models[0]?.benchmark.secondsPer30s).toEqual({
62 + cpu: performance.cpu ? 15 : null,
63 + accelerated: performance.accelerated ? 3 : null,
64 + });
65 + }
66 });
67
68 it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {
apps/server/scripts/speech-catalog.ts
+7 -4
@@ -81,7 +81,9 @@ export function generateSpeechCatalog(
81 ids.add(model.repository);
82 filenames.add(file.filename);
83 const languages = [...new Set(model.languages.map(canonicalLanguage))];
84 - const realtimeFactor = data.results[model.id]?.performance.accelerated;
84 + const performance = data.results[model.id]?.performance;
85 + const secondsPer30s = (realtimeFactor: number | undefined) =>
86 + realtimeFactor ? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2)) : null;
87 const accuracy = Object.fromEntries(
88 languages.flatMap((language) => {
89 const cell = data.results[model.id]?.accuracy[languageIdentity(language)];
@@ -109,9 +111,10 @@ export function generateSpeechCatalog(
111 languages,
112 benchmark: {
113 accuracy,
112 - secondsPer30s: realtimeFactor
113 - ? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2))
114 - : null,
114 + secondsPer30s: {
115 + cpu: secondsPer30s(performance?.cpu),
116 + accelerated: secondsPer30s(performance?.accelerated),
117 + },
118 },
119 recommended: RECOMMENDED.has(model.id),
120 supportsStreaming: model.capabilities.streaming,
apps/server/src/speech/SpeechService.test.ts
+50 -28
@@ -58,7 +58,10 @@ const modelDefinitions = vi.hoisted(() => [
58 size: 4,
59 languages: ["en", "fr"],
60 accuracy: 1,
61 - benchmark: { accuracy: { en: { grade: "A-", errorPercent: 4 } }, secondsPer30s: 1.2 },
61 + benchmark: {
62 + accuracy: { en: { grade: "A-", errorPercent: 4 } },
63 + secondsPer30s: { cpu: 3, accelerated: 1.2 },
64 + },
65 speed: 1,
66 recommended: true,
67 supportsStreaming: true,
@@ -146,31 +149,40 @@ it("rejects empty, misaligned, non-finite, and oversized PCM as invalid audio",
149 }
150 });
151
149 -it.effect("prepares a batch model before audio arrives and reuses it for transcription", () =>
150 - Effect.gen(function* () {
151 - const speech = yield* SpeechService.SpeechService;
152 - yield* speech.selectModel("fallback-model");
153 - yield* speech.prepareModel;
154 - expect(loadNative).toHaveBeenCalledTimes(1);
155 - expect(native.transcribe).not.toHaveBeenCalled();
156 - native.supportsStreaming = false;
157 - yield* Effect.gen(function* () {
158 - const stream = yield* speech.startStream({
159 - ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
160 - speechLanguage: "fr",
161 - speechTranslateToEnglish: true,
162 - speechSilenceFiltering: false,
152 +it.effect.each(["CPU", "Vulkan0"])(
153 + "prepares a batch model on %s before audio arrives and reuses it for transcription",
154 + (backend) =>
155 + Effect.gen(function* () {
156 + const speech = yield* SpeechService.SpeechService;
157 + yield* speech.selectModel("fallback-model");
158 + loadNative.mockImplementationOnce(async () => {
159 + native.backend = backend;
160 + return native;
161 });
164 - expect(native.begin).toHaveBeenLastCalledWith(
165 - { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
166 - false,
167 - "test.gguf",
168 - );
169 - expect(yield* stream.finish).toBe("hello");
170 - }).pipe(Effect.scoped);
171 - expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
172 - expect(loadNative).toHaveBeenCalledTimes(1);
173 - }).pipe(Effect.provide(layer)),
162 + yield* speech.prepareModel;
163 + expect(yield* speech.status).toMatchObject({
164 + benchmarkTier: backend === "CPU" ? "cpu" : "accelerated",
165 + });
166 + expect(loadNative).toHaveBeenCalledTimes(1);
167 + expect(native.transcribe).not.toHaveBeenCalled();
168 + native.supportsStreaming = false;
169 + yield* Effect.gen(function* () {
170 + const stream = yield* speech.startStream({
171 + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
172 + speechLanguage: "fr",
173 + speechTranslateToEnglish: true,
174 + speechSilenceFiltering: false,
175 + });
176 + expect(native.begin).toHaveBeenLastCalledWith(
177 + { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
178 + false,
179 + "test.gguf",
180 + );
181 + expect(yield* stream.finish).toBe("hello");
182 + }).pipe(Effect.scoped);
183 + expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
184 + expect(loadNative).toHaveBeenCalledTimes(1);
185 + }).pipe(Effect.provide(layer)),
186 );
187
188 it.effect("sends the chosen language in the spelling the loaded model advertises", () =>
@@ -290,7 +302,7 @@ it.effect("rejects model mutations outside the offered registry", () =>
302 expect(current).not.toHaveProperty("benchmark");
303 expect(
304 (yield* speech.models(true)).models.find((model) => model.active)?.benchmark?.secondsPer30s,
293 - ).toBe(1.2);
305 + ).toEqual({ cpu: 3, accelerated: 1.2 });
306 }).pipe(Effect.provide(layer)),
307 );
308
@@ -665,11 +677,19 @@ it.effect.each([{ devices: [] }, { devices: [{ id: "gpu", name: "Test GPU" }] }]
677 const speech = yield* SpeechService.SpeechService;
678 const discover = vi.mocked(listNativeSpeechGpuDevices).mockReset();
679 discover.mockRejectedValueOnce(new Error("driver not ready")).mockResolvedValueOnce(devices);
668 - expect(yield* speech.status).toMatchObject({ gpuDevices: [] });
680 + expect(yield* speech.status).toMatchObject({ gpuDevices: [], benchmarkTier: "cpu" });
681 yield* speech.status;
682 expect(discover).toHaveBeenCalledOnce();
683 yield* TestClock.adjust("1 minute");
672 - expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
684 + expect(yield* speech.status).toMatchObject({
685 + gpuDevices: devices,
686 + benchmarkTier: devices.length ? "accelerated" : "cpu",
687 + });
688 + expect(yield* speech.updateAcceleration("cpu")).toMatchObject({ benchmarkTier: "cpu" });
689 + expect(yield* speech.updateAcceleration('gpu:["vulkan","gpu-1"]')).toMatchObject({
690 + benchmarkTier: "accelerated",
691 + });
692 + yield* speech.updateAcceleration("auto");
693 yield* TestClock.adjust("1 minute");
694 expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
695 expect(discover).toHaveBeenCalledTimes(2);
@@ -700,9 +720,11 @@ it.effect(
720 ],
721 };
722 const speech = yield* SpeechService.SpeechService;
723 + expect(yield* speech.status).toMatchObject({ benchmarkTier: "accelerated" });
724 expect(yield* speech.transcribe(pcm(), options)).toBe(
725 cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
726 );
727 + expect(yield* speech.status).toMatchObject({ benchmarkTier: "cpu" });
728 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe(
729 "hello",
730 );
apps/server/src/speech/SpeechService.ts
+13 -1
@@ -452,6 +452,13 @@ const make = Effect.gen(function* () {
452 if (unsupportedReason) return { supported: false, reason: unsupportedReason };
453 const definition = selectedModel(settings);
454 const ready = yield* store.isReady(definition);
455 + const gpuDevices = yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => []));
456 + const autoUsesCpu =
457 + gpuDevices.length === 0 ||
458 + autoCpuModel === definition.id ||
459 + (loadedModelId === definition.id &&
460 + loadedAcceleration === "auto" &&
461 + model?.backend.toLowerCase() === "cpu");
462 return {
463 supported: true,
464 state: activeTranscriptions > 0 ? "transcribing" : ready ? "ready" : "missing-model",
@@ -464,8 +471,13 @@ const make = Effect.gen(function* () {
471 languages: definition.languages,
472 supportsLanguageDetection: definition.supportsLanguageDetection,
473 acceleration: settings.speechAcceleration,
474 + benchmarkTier:
475 + settings.speechAcceleration === "cpu" ||
476 + (settings.speechAcceleration === "auto" && autoUsesCpu)
477 + ? "cpu"
478 + : "accelerated",
479 modelUnloadTimeout: settings.speechModelUnloadTimeout,
468 - gpuDevices: yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => [])),
480 + gpuDevices,
481 };
482 });
483
apps/server/src/speech/catalog.json
+59 -59
@@ -15,7 +15,7 @@
15 "languages": ["en"],
16 "benchmark": {
17 "accuracy": { "en": { "grade": "A-", "errorPercent": 3.99 } },
18 - "secondsPer30s": 1.19
18 + "secondsPer30s": { "cpu": 2.15, "accelerated": 1.19 }
19 },
20 "recommended": true,
21 "supportsStreaming": true,
@@ -92,7 +92,7 @@
92 "ro": { "grade": "D", "errorPercent": 28.28 },
93 "et": { "grade": "F", "errorPercent": 31.84 }
94 },
95 - "secondsPer30s": 1.75
95 + "secondsPer30s": { "cpu": 2.83, "accelerated": 1.75 }
96 },
97 "recommended": true,
98 "supportsStreaming": true,
@@ -116,7 +116,7 @@
116 "es": { "grade": "B", "errorPercent": 6.54 },
117 "fr": { "grade": "B-", "errorPercent": 8.53 }
118 },
119 - "secondsPer30s": 0.91
119 + "secondsPer30s": { "cpu": 1.38, "accelerated": 0.91 }
120 },
121 "recommended": true,
122 "supportsStreaming": false,
@@ -165,7 +165,7 @@
165 "vi": { "grade": "B", "errorPercent": 7.39 },
166 "ko": { "grade": "B", "errorPercent": 6.57 }
167 },
168 - "secondsPer30s": 3.71
168 + "secondsPer30s": { "cpu": 6.96, "accelerated": 3.71 }
169 },
170 "recommended": true,
171 "supportsStreaming": false,
@@ -367,7 +367,7 @@
367 "yo": { "grade": "F", "errorPercent": 109 },
368 "zh": { "grade": "C", "errorPercent": 13.13 }
369 },
370 - "secondsPer30s": 7.34
370 + "secondsPer30s": { "cpu": 15.58, "accelerated": 7.34 }
371 },
372 "recommended": true,
373 "supportsStreaming": false,
@@ -400,7 +400,7 @@
400 "hi": { "grade": "C", "errorPercent": 14.44 },
401 "ko": { "grade": "B+", "errorPercent": 5.27 }
402 },
403 - "secondsPer30s": 39.47
403 + "secondsPer30s": { "cpu": 51.28, "accelerated": 39.47 }
404 },
405 "recommended": false,
406 "supportsStreaming": true,
@@ -471,7 +471,7 @@
471 "el": { "grade": "F", "errorPercent": 32.25 },
472 "mt": { "grade": "C-", "errorPercent": 17.01 }
473 },
474 - "secondsPer30s": 1.3
474 + "secondsPer30s": { "cpu": 2.26, "accelerated": 1.3 }
475 },
476 "recommended": false,
477 "supportsStreaming": false,
@@ -542,7 +542,7 @@
542 "sv": { "grade": "C", "errorPercent": 15.25 },
543 "uk": { "grade": "B", "errorPercent": 6.84 }
544 },
545 - "secondsPer30s": 1.2
545 + "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
546 },
547 "recommended": false,
548 "supportsStreaming": false,
@@ -561,7 +561,7 @@
561 "languages": ["en"],
562 "benchmark": {
563 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.11 } },
564 - "secondsPer30s": 1.09
564 + "secondsPer30s": { "cpu": 2.1, "accelerated": 1.09 }
565 },
566 "recommended": false,
567 "supportsStreaming": false,
@@ -642,7 +642,7 @@
642 "hu": { "grade": "F", "errorPercent": 56.24 },
643 "mk": { "grade": "F", "errorPercent": 35.09 }
644 },
645 - "secondsPer30s": 3.09
645 + "secondsPer30s": { "cpu": 5.67, "accelerated": 3.09 }
646 },
647 "recommended": false,
648 "supportsStreaming": false,
@@ -725,7 +725,7 @@
725 "sl": { "grade": "F", "errorPercent": 75.22 },
726 "sv": { "grade": "F", "errorPercent": 75.36 }
727 },
728 - "secondsPer30s": 3.4
728 + "secondsPer30s": { "cpu": 5.48, "accelerated": 3.4 }
729 },
730 "recommended": false,
731 "supportsStreaming": false,
@@ -796,7 +796,7 @@
796 "ru": { "grade": "B", "errorPercent": 7.83 },
797 "uk": { "grade": "C+", "errorPercent": 10.58 }
798 },
799 - "secondsPer30s": 2.12
799 + "secondsPer30s": { "cpu": 3.6, "accelerated": 2.12 }
800 },
801 "recommended": false,
802 "supportsStreaming": false,
@@ -815,7 +815,7 @@
815 "languages": ["en"],
816 "benchmark": {
817 "accuracy": { "en": { "grade": "A", "errorPercent": 3.58 } },
818 - "secondsPer30s": 7.55
818 + "secondsPer30s": { "cpu": 11.95, "accelerated": 7.55 }
819 },
820 "recommended": false,
821 "supportsStreaming": false,
@@ -837,7 +837,7 @@
837 "en": { "grade": "A-", "errorPercent": 4.88 },
838 "ar": { "grade": "C+", "errorPercent": 11.06 }
839 },
840 - "secondsPer30s": 3.61
840 + "secondsPer30s": { "cpu": 6.96, "accelerated": 3.61 }
841 },
842 "recommended": false,
843 "supportsStreaming": false,
@@ -860,7 +860,7 @@
860 "en": { "grade": "B+", "errorPercent": 5.49 },
861 "ja": { "grade": "B-", "errorPercent": 8.5 }
862 },
863 - "secondsPer30s": 2.8
863 + "secondsPer30s": { "cpu": 3.78, "accelerated": 2.8 }
864 },
865 "recommended": false,
866 "supportsStreaming": false,
@@ -879,7 +879,7 @@
879 "languages": ["ru"],
880 "benchmark": {
881 "accuracy": { "ru": { "grade": "B+", "errorPercent": 5.53 } },
882 - "secondsPer30s": 0.85
882 + "secondsPer30s": { "cpu": 2.62, "accelerated": 0.85 }
883 },
884 "recommended": false,
885 "supportsStreaming": false,
@@ -898,7 +898,7 @@
898 "languages": ["ru"],
899 "benchmark": {
900 "accuracy": { "ru": { "grade": "B+", "errorPercent": 5.35 } },
901 - "secondsPer30s": 2
901 + "secondsPer30s": { "cpu": 3.81, "accelerated": 2 }
902 },
903 "recommended": false,
904 "supportsStreaming": false,
@@ -917,7 +917,7 @@
917 "languages": ["en"],
918 "benchmark": {
919 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.61 } },
920 - "secondsPer30s": 1.31
920 + "secondsPer30s": { "cpu": 1.74, "accelerated": 1.31 }
921 },
922 "recommended": false,
923 "supportsStreaming": false,
@@ -943,7 +943,7 @@
943 "pt": { "grade": "B-", "errorPercent": 9.8 },
944 "ja": { "grade": "B", "errorPercent": 6.3 }
945 },
946 - "secondsPer30s": 6.5
946 + "secondsPer30s": { "cpu": 10.81, "accelerated": 6.5 }
947 },
948 "recommended": false,
949 "supportsStreaming": false,
@@ -962,7 +962,7 @@
962 "languages": ["en"],
963 "benchmark": {
964 "accuracy": { "en": { "grade": "F", "errorPercent": 37.48 } },
965 - "secondsPer30s": 0.42
965 + "secondsPer30s": { "cpu": 1.16, "accelerated": 0.42 }
966 },
967 "recommended": false,
968 "supportsStreaming": false,
@@ -981,7 +981,7 @@
981 "languages": ["en"],
982 "benchmark": {
983 "accuracy": { "en": { "grade": "B+", "errorPercent": 6.11 } },
984 - "secondsPer30s": 0.39
984 + "secondsPer30s": { "cpu": 0.6, "accelerated": 0.39 }
985 },
986 "recommended": false,
987 "supportsStreaming": false,
@@ -1000,7 +1000,7 @@
1000 "languages": ["en"],
1001 "benchmark": {
1002 "accuracy": { "en": { "grade": "B", "errorPercent": 6.52 } },
1003 - "secondsPer30s": 1.14
1003 + "secondsPer30s": { "cpu": 2.24, "accelerated": 1.14 }
1004 },
1005 "recommended": false,
1006 "supportsStreaming": true,
@@ -1081,7 +1081,7 @@
1081 "hu": { "grade": "F", "errorPercent": 32.84 },
1082 "mk": { "grade": "C-", "errorPercent": 18.22 }
1083 },
1084 - "secondsPer30s": 7.34
1084 + "secondsPer30s": { "cpu": 12.15, "accelerated": 7.34 }
1085 },
1086 "recommended": false,
1087 "supportsStreaming": false,
@@ -1106,7 +1106,7 @@
1106 "ja": { "grade": "B", "errorPercent": 7.63 },
1107 "ko": { "grade": "B-", "errorPercent": 8.27 }
1108 },
1109 - "secondsPer30s": 1.08
1109 + "secondsPer30s": { "cpu": 1.52, "accelerated": 1.08 }
1110 },
1111 "recommended": false,
1112 "supportsStreaming": false,
@@ -1134,7 +1134,7 @@
1134 "nl": { "grade": "B", "errorPercent": 6.57 },
1135 "hi": { "grade": "B-", "errorPercent": 8.93 }
1136 },
1137 - "secondsPer30s": 27.65
1137 + "secondsPer30s": { "cpu": 53.57, "accelerated": 27.65 }
1138 },
1139 "recommended": false,
1140 "supportsStreaming": false,
@@ -1336,7 +1336,7 @@
1336 "yo": { "grade": "F", "errorPercent": 118.5 },
1337 "zh": { "grade": "D", "errorPercent": 23.06 }
1338 },
1339 - "secondsPer30s": 2.57
1339 + "secondsPer30s": { "cpu": 5.12, "accelerated": 2.57 }
1340 },
1341 "recommended": false,
1342 "supportsStreaming": false,
@@ -1540,7 +1540,7 @@
1540 "yue": { "grade": "F", "errorPercent": 34.62 },
1541 "zh": { "grade": "B-", "errorPercent": 8.5 }
1542 },
1543 - "secondsPer30s": 9.69
1543 + "secondsPer30s": { "cpu": 20.98, "accelerated": 9.69 }
1544 },
1545 "recommended": false,
1546 "supportsStreaming": false,
@@ -1562,7 +1562,7 @@
1562 "zh": { "grade": "B-", "errorPercent": 8.1 },
1563 "en": { "grade": "A-", "errorPercent": 4.14 }
1564 },
1565 - "secondsPer30s": 14.85
1565 + "secondsPer30s": { "cpu": 31.25, "accelerated": 14.85 }
1566 },
1567 "recommended": false,
1568 "supportsStreaming": false,
@@ -1766,7 +1766,7 @@
1766 "yue": { "grade": "D", "errorPercent": 22.06 },
1767 "zh": { "grade": "B-", "errorPercent": 7.98 }
1768 },
1769 - "secondsPer30s": 15.04
1769 + "secondsPer30s": { "cpu": 30.93, "accelerated": 15.04 }
1770 },
1771 "recommended": false,
1772 "supportsStreaming": false,
@@ -1790,7 +1790,7 @@
1790 "es": { "grade": "B", "errorPercent": 6.73 },
1791 "fr": { "grade": "B", "errorPercent": 7.22 }
1792 },
1793 - "secondsPer30s": 1.94
1793 + "secondsPer30s": { "cpu": 3.39, "accelerated": 1.94 }
1794 },
1795 "recommended": false,
1796 "supportsStreaming": false,
@@ -1809,7 +1809,7 @@
1809 "languages": ["ru"],
1810 "benchmark": {
1811 "accuracy": { "ru": { "grade": "B-", "errorPercent": 8.42 } },
1812 - "secondsPer30s": 0.75
1812 + "secondsPer30s": { "cpu": 2.54, "accelerated": 0.75 }
1813 },
1814 "recommended": false,
1815 "supportsStreaming": false,
@@ -1828,7 +1828,7 @@
1828 "languages": ["ru"],
1829 "benchmark": {
1830 "accuracy": { "ru": { "grade": "B-", "errorPercent": 8.07 } },
1831 - "secondsPer30s": 1.12
1831 + "secondsPer30s": { "cpu": 2.9, "accelerated": 1.12 }
1832 },
1833 "recommended": false,
1834 "supportsStreaming": false,
@@ -1854,7 +1854,7 @@
1854 "pt": { "grade": "B-", "errorPercent": 9.6 },
1855 "ja": { "grade": "B", "errorPercent": 6.44 }
1856 },
1857 - "secondsPer30s": 6.47
1857 + "secondsPer30s": { "cpu": 10.89, "accelerated": 6.47 }
1858 },
1859 "recommended": false,
1860 "supportsStreaming": false,
@@ -1879,7 +1879,7 @@
1879 "es": { "grade": "A-", "errorPercent": 4.08 },
1880 "pt": { "grade": "B+", "errorPercent": 5.57 }
1881 },
1882 - "secondsPer30s": 7.51
1882 + "secondsPer30s": { "cpu": 13.99, "accelerated": 7.51 }
1883 },
1884 "recommended": false,
1885 "supportsStreaming": false,
@@ -1904,7 +1904,7 @@
1904 "es": { "grade": "B", "errorPercent": 6.53 },
1905 "pt": { "grade": "C+", "errorPercent": 10.61 }
1906 },
1907 - "secondsPer30s": 6.46
1907 + "secondsPer30s": { "cpu": 11.54, "accelerated": 6.46 }
1908 },
1909 "recommended": false,
1910 "supportsStreaming": false,
@@ -1923,7 +1923,7 @@
1923 "languages": ["en"],
1924 "benchmark": {
1925 "accuracy": { "en": { "grade": "C+", "errorPercent": 12.25 } },
1926 - "secondsPer30s": 0.84
1926 + "secondsPer30s": { "cpu": 1.28, "accelerated": 0.84 }
1927 },
1928 "recommended": false,
1929 "supportsStreaming": false,
@@ -1942,7 +1942,7 @@
1942 "languages": ["en"],
1943 "benchmark": {
1944 "accuracy": { "en": { "grade": "B", "errorPercent": 7.87 } },
1945 - "secondsPer30s": 2.1
1945 + "secondsPer30s": { "cpu": 3.1, "accelerated": 2.1 }
1946 },
1947 "recommended": false,
1948 "supportsStreaming": true,
@@ -1961,7 +1961,7 @@
1961 "languages": ["en"],
1962 "benchmark": {
1963 "accuracy": { "en": { "grade": "B-", "errorPercent": 8.55 } },
1964 - "secondsPer30s": 1.38
1964 + "secondsPer30s": { "cpu": 2.22, "accelerated": 1.38 }
1965 },
1966 "recommended": false,
1967 "supportsStreaming": true,
@@ -1980,7 +1980,7 @@
1980 "languages": ["en"],
1981 "benchmark": {
1982 "accuracy": { "en": { "grade": "C-", "errorPercent": 18.18 } },
1983 - "secondsPer30s": 0.51
1983 + "secondsPer30s": { "cpu": 0.53, "accelerated": 0.51 }
1984 },
1985 "recommended": false,
1986 "supportsStreaming": true,
@@ -1999,7 +1999,7 @@
1999 "languages": ["en"],
2000 "benchmark": {
2001 "accuracy": { "en": { "grade": "C", "errorPercent": 14.13 } },
2002 - "secondsPer30s": 0.5
2002 + "secondsPer30s": { "cpu": 0.73, "accelerated": 0.5 }
2003 },
2004 "recommended": false,
2005 "supportsStreaming": false,
@@ -2021,7 +2021,7 @@
2021 "en": { "grade": "B+", "errorPercent": 5.13 },
2022 "zh": { "grade": "B-", "errorPercent": 9.23 }
2023 },
2024 - "secondsPer30s": 9.79
2024 + "secondsPer30s": { "cpu": 19.11, "accelerated": 9.79 }
2025 },
2026 "recommended": false,
2027 "supportsStreaming": false,
@@ -2040,7 +2040,7 @@
2040 "languages": ["en"],
2041 "benchmark": {
2042 "accuracy": { "en": { "grade": "B", "errorPercent": 6.43 } },
2043 - "secondsPer30s": 1.11
2043 + "secondsPer30s": { "cpu": 2.21, "accelerated": 1.11 }
2044 },
2045 "recommended": false,
2046 "supportsStreaming": true,
@@ -2111,7 +2111,7 @@
2111 "sv": { "grade": "C", "errorPercent": 13.26 },
2112 "uk": { "grade": "B", "errorPercent": 6.53 }
2113 },
2114 - "secondsPer30s": 1.2
2114 + "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
2115 },
2116 "recommended": false,
2117 "supportsStreaming": false,
@@ -2130,7 +2130,7 @@
2130 "languages": ["en"],
2131 "benchmark": {
2132 "accuracy": { "en": { "grade": "B+", "errorPercent": 5.53 } },
2133 - "secondsPer30s": 1.05
2133 + "secondsPer30s": { "cpu": 1.98, "accelerated": 1.05 }
2134 },
2135 "recommended": false,
2136 "supportsStreaming": false,
@@ -2149,7 +2149,7 @@
2149 "languages": ["en"],
2150 "benchmark": {
2151 "accuracy": { "en": { "grade": "B+", "errorPercent": 5.61 } },
2152 - "secondsPer30s": 1.79
2152 + "secondsPer30s": { "cpu": 3.36, "accelerated": 1.79 }
2153 },
2154 "recommended": false,
2155 "supportsStreaming": false,
@@ -2220,7 +2220,7 @@
2220 "sv": { "grade": "C-", "errorPercent": 16.42 },
2221 "uk": { "grade": "B-", "errorPercent": 8.11 }
2222 },
2223 - "secondsPer30s": 1.2
2223 + "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
2224 },
2225 "recommended": false,
2226 "supportsStreaming": false,
@@ -2239,7 +2239,7 @@
2239 "languages": ["en"],
2240 "benchmark": {
2241 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.57 } },
2242 - "secondsPer30s": 1.11
2242 + "secondsPer30s": { "cpu": 2.02, "accelerated": 1.11 }
2243 },
2244 "recommended": false,
2245 "supportsStreaming": false,
@@ -2258,7 +2258,7 @@
2258 "languages": ["en"],
2259 "benchmark": {
2260 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.45 } },
2261 - "secondsPer30s": 1.87
2261 + "secondsPer30s": { "cpu": 3.49, "accelerated": 1.87 }
2262 },
2263 "recommended": false,
2264 "supportsStreaming": false,
@@ -2277,7 +2277,7 @@
2277 "languages": ["en"],
2278 "benchmark": {
2279 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.68 } },
2280 - "secondsPer30s": 1.76
2280 + "secondsPer30s": { "cpu": 3.22, "accelerated": 1.76 }
2281 },
2282 "recommended": false,
2283 "supportsStreaming": false,
@@ -2296,7 +2296,7 @@
2296 "languages": ["en"],
2297 "benchmark": {
2298 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.24 } },
2299 - "secondsPer30s": 1.82
2299 + "secondsPer30s": { "cpu": 3.41, "accelerated": 1.82 }
2300 },
2301 "recommended": false,
2302 "supportsStreaming": false,
@@ -2324,7 +2324,7 @@
2324 "nl": { "grade": "B+", "errorPercent": 5.12 },
2325 "hi": { "grade": "B", "errorPercent": 7.4 }
2326 },
2327 - "secondsPer30s": null
2327 + "secondsPer30s": { "cpu": null, "accelerated": null }
2328 },
2329 "recommended": false,
2330 "supportsStreaming": false,
@@ -2526,7 +2526,7 @@
2526 "yo": { "grade": "F", "errorPercent": 103.28 },
2527 "zh": { "grade": "F", "errorPercent": 36.21 }
2528 },
2529 - "secondsPer30s": 1.03
2529 + "secondsPer30s": { "cpu": 1.55, "accelerated": 1.03 }
2530 },
2531 "recommended": false,
2532 "supportsStreaming": false,
@@ -2545,7 +2545,7 @@
2545 "languages": ["en"],
2546 "benchmark": {
2547 "accuracy": { "en": { "grade": "B", "errorPercent": 7.6 } },
2548 - "secondsPer30s": 0.94
2548 + "secondsPer30s": { "cpu": 1.46, "accelerated": 0.94 }
2549 },
2550 "recommended": false,
2551 "supportsStreaming": false,
@@ -2747,7 +2747,7 @@
2747 "yo": { "grade": "F", "errorPercent": 113.72 },
2748 "zh": { "grade": "C-", "errorPercent": 19.32 }
2749 },
2750 - "secondsPer30s": 14.96
2750 + "secondsPer30s": { "cpu": 31.25, "accelerated": 14.96 }
2751 },
2752 "recommended": false,
2753 "supportsStreaming": false,
@@ -2949,7 +2949,7 @@
2949 "yo": { "grade": "F", "errorPercent": 96 },
2950 "zh": { "grade": "C", "errorPercent": 15.39 }
2951 },
2952 - "secondsPer30s": 14.93
2952 + "secondsPer30s": { "cpu": 31.25, "accelerated": 14.93 }
2953 },
2954 "recommended": false,
2955 "supportsStreaming": false,
@@ -2968,7 +2968,7 @@
2968 "languages": ["en"],
2969 "benchmark": {
2970 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.88 } },
2971 - "secondsPer30s": 6.81
2971 + "secondsPer30s": { "cpu": 14.53, "accelerated": 6.81 }
2972 },
2973 "recommended": false,
2974 "supportsStreaming": false,
@@ -2987,7 +2987,7 @@
2987 "languages": ["en"],
2988 "benchmark": {
2989 "accuracy": { "en": { "grade": "B+", "errorPercent": 6.14 } },
2990 - "secondsPer30s": 2.4
2990 + "secondsPer30s": { "cpu": 4.79, "accelerated": 2.4 }
2991 },
2992 "recommended": false,
2993 "supportsStreaming": false,
@@ -3189,7 +3189,7 @@
3189 "yo": { "grade": "F", "errorPercent": 102.29 },
3190 "zh": { "grade": "F", "errorPercent": 41.91 }
3191 },
3192 - "secondsPer30s": 0.7
3192 + "secondsPer30s": { "cpu": 0.75, "accelerated": 0.7 }
3193 },
3194 "recommended": false,
3195 "supportsStreaming": false,
@@ -3208,7 +3208,7 @@
3208 "languages": ["en"],
3209 "benchmark": {
3210 "accuracy": { "en": { "grade": "C+", "errorPercent": 10.72 } },
3211 - "secondsPer30s": 0.63
3211 + "secondsPer30s": { "cpu": 0.73, "accelerated": 0.63 }
3212 },
3213 "recommended": false,
3214 "supportsStreaming": false,
apps/web/src/components/settings/SpeechModelComparison.tsx
+16 -13
@@ -1,4 +1,4 @@
1 -import type { EnvironmentSpeechModel } from "@t3tools/contracts";
1 +import type { EnvironmentSpeechModel, SpeechBenchmarkTier } from "@t3tools/contracts";
2 import { CheckIcon } from "lucide-react";
3
4 import { cn } from "../../lib/utils";
@@ -73,14 +73,17 @@ export function LanguageRow({
73 export function ModelBenchmarks({
74 model,
75 languages,
76 + tier,
77 facts,
78 }: {
79 model: EnvironmentSpeechModel;
80 languages: readonly string[];
81 + tier: SpeechBenchmarkTier;
82 /** Plain facts, such as language count and size, shown after the scores. */
83 facts: readonly string[];
84 }) {
85 const data = model.benchmark;
86 + const seconds = data?.secondsPer30s[tier];
87 const visible = !data
88 ? []
89 : languages.length
@@ -106,18 +109,18 @@ export function ModelBenchmarks({
109 </span>
110 );
111 })}
109 - {data ? (
110 - <span>
111 - Speed:{" "}
112 - {data.secondsPer30s === null ? (
113 - "not benchmarked"
114 - ) : (
115 - <>
116 - <strong className="font-medium text-foreground">{data.secondsPer30s} s</strong> / 30 s
117 - </>
118 - )}
119 - </span>
120 - ) : null}
112 + <span>
113 + Speed:{" "}
114 + {tier === "cpu" && data && data.secondsPer30s.cpu === undefined ? (
115 + "update server"
116 + ) : seconds == null ? (
117 + "not benchmarked"
118 + ) : (
119 + <>
120 + <strong className="font-medium text-foreground">{seconds} s</strong> / 30 s
121 + </>
122 + )}
123 + </span>
124 {facts.map((fact) => (
125 <span key={fact}>{fact}</span>
126 ))}
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+132 -2
@@ -1,4 +1,9 @@
1 -import { PrimaryConnectionTarget, RelayConnectionTarget } from "@t3tools/client-runtime/connection";
1 +import {
2 + BearerConnectionTarget,
3 + PrimaryConnectionTarget,
4 + RelayConnectionTarget,
5 + type PreparedConnection,
6 +} from "@t3tools/client-runtime/connection";
7 import type * as Effect from "effect/Effect";
8 import {
9 EnvironmentId,
@@ -71,6 +76,128 @@ vi.mock("../../connection/runtime", async () => {
76 };
77 });
78
79 +function deferred<T>() {
80 + let resolve!: (value: T) => void;
81 + const promise = new Promise<T>((done) => {
82 + resolve = done;
83 + });
84 + return { promise, resolve };
85 +}
86 +
87 +it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accelerated: 2.1 }])(
88 + "uses the destination environment's benchmark tier across switches with measurements %j",
89 + async (secondsPer30s) => {
90 + const target = new PrimaryConnectionTarget({
91 + environmentId: EnvironmentId.make("voice-local"),
92 + label: "Local CPU",
93 + httpBaseUrl: window.location.origin,
94 + wsBaseUrl: "ws://localhost",
95 + });
96 + const local: PreparedConnection = {
97 + ...target,
98 + target,
99 + socketUrl: target.wsBaseUrl,
100 + httpAuthorization: null,
101 + };
102 + const remote: PreparedConnection = {
103 + environmentId: EnvironmentId.make("voice-remote"),
104 + label: "Remote GPU",
105 + httpBaseUrl: "https://voice.example.test",
106 + socketUrl: "wss://voice.example.test/ws",
107 + httpAuthorization: { _tag: "Bearer", token: "test-token" },
108 + target: new BearerConnectionTarget({
109 + environmentId: EnvironmentId.make("voice-remote"),
110 + label: "Remote GPU",
111 + connectionId: "remote",
112 + }),
113 + };
114 + let prepared = local;
115 + const remoteStatus = deferred<Response>();
116 + const remoteRequested = deferred<void>();
117 + const status = {
118 + supported: true,
119 + state: "ready",
120 + modelId: "both",
121 + model: "Bilingual",
122 + size: 100,
123 + supportsStreaming: false,
124 + supportsTranslation: false,
125 + acceleration: "auto",
126 + modelUnloadTimeout: "never",
127 + gpuDevices: [],
128 + benchmarkTier: "cpu",
129 + } satisfies EnvironmentSpeechStatus;
130 + const model: EnvironmentSpeechModel = {
131 + id: "both",
132 + name: "Bilingual",
133 + description: "",
134 + size: 100,
135 + languages: ["en"],
136 + accuracy: 1,
137 + speed: 1,
138 + recommended: true,
139 + supportsStreaming: false,
140 + supportsTranslation: false,
141 + supportsLanguageDetection: false,
142 + active: false,
143 + state: "downloadable",
144 + benchmark: {
145 + accuracy: { en: { grade: "A-", errorPercent: 4 } },
146 + secondsPer30s,
147 + },
148 + };
149 + const cpuBenchmark = `Speed: ${secondsPer30s.cpu === undefined ? "update server" : secondsPer30s.cpu === null ? "not benchmarked" : `${secondsPer30s.cpu} s / 30 s`}`;
150 + const gpuBenchmark = `Speed: ${secondsPer30s.accelerated === null ? "not benchmarked" : `${secondsPer30s.accelerated} s / 30 s`}`;
151 + const fetch = globalThis.fetch;
152 + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
153 + const url = new URL(input instanceof Request ? input.url : input, window.location.href);
154 + if (url.pathname === "/api/voice/status") {
155 + if (url.origin === remote.httpBaseUrl) {
156 + remoteRequested.resolve();
157 + return remoteStatus.promise;
158 + }
159 + return Promise.resolve(Response.json(status));
160 + }
161 + if (url.pathname === "/api/voice/models")
162 + return Promise.resolve(Response.json({ models: [model] }));
163 + return fetch(input, init);
164 + });
165 + await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
166 + () => ({
167 + environmentId: prepared.environmentId,
168 + label: prepared.label,
169 + grantId: "test",
170 + prepared,
171 + }),
172 + async () => {
173 + const panel = () => (
174 + <SettingsScopeProvider search={{}} onChange={() => {}}>
175 + <VoiceSettingsPanel />
176 + </SettingsScopeProvider>
177 + );
178 + await using screen = await render(panel());
179 + await expect.element(screen.getByText(cpuBenchmark)).toBeVisible();
180 + prepared = remote;
181 + await screen.rerender(panel());
182 + await remoteRequested.promise;
183 + await expect.element(screen.getByText(cpuBenchmark)).not.toBeInTheDocument();
184 + remoteStatus.resolve(
185 + Response.json({
186 + ...status,
187 + benchmarkTier: "accelerated",
188 + gpuDevices: [{ id: "gpu", name: "Remote GPU" }],
189 + }),
190 + );
191 + await expect.element(screen.getByText(gpuBenchmark)).toBeVisible();
192 + prepared = local;
193 + await screen.rerender(panel());
194 + await expect.element(screen.getByText(cpuBenchmark)).toBeVisible();
195 + await expect.element(screen.getByText(gpuBenchmark)).not.toBeInTheDocument();
196 + },
197 + );
198 + },
199 +);
200 +
201 it("tests the selected model after download and preserves its transcript across filters", async () => {
202 const target = new PrimaryConnectionTarget({
203 environmentId: EnvironmentId.make("voice-filter"),
@@ -105,6 +232,7 @@ it("tests the selected model after download and preserves its transcript across
232 acceleration: "cpu",
233 modelUnloadTimeout: "never",
234 gpuDevices: [],
235 + benchmarkTier: "cpu",
236 } satisfies EnvironmentSpeechStatus;
237 let installed = false;
238 const fetch = globalThis.fetch;
@@ -136,7 +264,7 @@ it("tests the selected model after download and preserves its transcript across
264 en: { grade: "A-", errorPercent: 4 },
265 fr: { grade: "B+", errorPercent: 5.5 },
266 },
139 - secondsPer30s: 2.1,
267 + secondsPer30s: { cpu: 5, accelerated: 2.1 },
268 },
269 }
270 : {}),
@@ -269,6 +397,7 @@ it("loads the models of a relay-connected destination", async () => {
397 acceleration: "cpu",
398 modelUnloadTimeout: "never",
399 gpuDevices: [],
400 + benchmarkTier: "cpu",
401 } satisfies EnvironmentSpeechStatus;
402 const authorizations: Array<string | null> = [];
403 const fetch = globalThis.fetch;
@@ -335,6 +464,7 @@ it("recovers from a failed download progress poll without notifying, but reports
464 acceleration: "cpu",
465 modelUnloadTimeout: "never",
466 gpuDevices: [],
467 + benchmarkTier: "cpu",
468 };
469 const unavailable = () => new Response(null, { status: 503 });
470 const readModels = vi
apps/web/src/components/settings/VoiceSettingsPanel.tsx
+6 -1
@@ -21,6 +21,7 @@ import {
21 type ProjectId,
22 ProviderDriverKind,
23 type SpeechAcceleration,
24 + type SpeechBenchmarkTier,
25 type SpeechCustomWords,
26 type SpeechModelUnloadTimeout,
27 type VoiceShortcutMode,
@@ -159,6 +160,7 @@ const spellingsOf = (words: SpeechCustomWords) =>
160 type ModelCardProps = {
161 readonly model: EnvironmentSpeechModel;
162 readonly comparisonLanguages: readonly string[];
163 + readonly benchmarkTier: SpeechBenchmarkTier;
164 readonly busy: boolean;
165 readonly canCancel: boolean;
166 readonly onDownload: () => void;
@@ -216,6 +218,7 @@ function ModelCard(props: ModelCardProps) {
218 <ModelBenchmarks
219 model={model}
220 languages={props.comparisonLanguages}
221 + tier={props.benchmarkTier}
222 facts={[
223 ...(model.languages.length > 1
224 ? [`${model.languages.length} languages`]
@@ -801,6 +804,7 @@ function SpeechModelBrowser(props: {
804 downloadedOnly,
805 sort,
806 accuracyLanguage,
807 + benchmarkTier: status?.supported ? status.benchmarkTier : "accelerated",
808 });
809 const activeModel = models.find((model) => model.active);
810 const renderedModels =
@@ -823,7 +827,7 @@ function SpeechModelBrowser(props: {
827 <p className="border-b border-border/60 px-3 py-2 text-xs text-muted-foreground sm:px-4">
828 Pick languages to see models that support all of them. Accuracy is word or character
829 error rate: lower is better. Speed is the time to transcribe 30 seconds of audio on
826 - Handy’s benchmark GPU.
830 + Handy’s benchmark {status.benchmarkTier === "cpu" ? "CPU" : "GPU"}.
831 </p>
832 <div className="@min-[32rem]/voice-models:grid @min-[32rem]/voice-models:grid-cols-[11rem_minmax(0,1fr)]">
833 <div className="flex min-h-0 flex-col border-b border-border/60 bg-muted/10 @min-[32rem]/voice-models:border-r @min-[32rem]/voice-models:border-b-0">
@@ -940,6 +944,7 @@ function SpeechModelBrowser(props: {
944 busy: controlsDisabled,
945 canCancel,
946 comparisonLanguages: selectedLanguages,
947 + benchmarkTier: status.benchmarkTier,
948 onDownload: () =>
949 void run(
950 "Could not download the model",
apps/web/src/components/settings/speechModelBrowser.test.ts
+5 -3
@@ -33,7 +33,7 @@ const models: readonly EnvironmentSpeechModel[] = [
33 en: { grade: "A", errorPercent: 3.8 },
34 fr: { grade: "B", errorPercent: 8 },
35 },
36 - secondsPer30s: 2,
36 + secondsPer30s: { cpu: 2, accelerated: 2 },
37 },
38 },
39 {
@@ -47,7 +47,7 @@ const models: readonly EnvironmentSpeechModel[] = [
47 en: { grade: "A", errorPercent: 3.2 },
48 fr: { grade: "B", errorPercent: 9 },
49 },
50 - secondsPer30s: 1,
50 + secondsPer30s: { cpu: 1, accelerated: 1 },
51 },
52 },
53 {
@@ -58,7 +58,7 @@ const models: readonly EnvironmentSpeechModel[] = [
58 active: false,
59 state: "verifying",
60 languages: ["en"],
61 - benchmark: { accuracy: {}, secondsPer30s: null },
61 + benchmark: { accuracy: {}, secondsPer30s: { cpu: null, accelerated: null } },
62 },
63 ];
64
@@ -78,6 +78,7 @@ it.each([
78 downloadedOnly: false,
79 sort,
80 accuracyLanguage,
81 + benchmarkTier: "accelerated",
82 }).map((entry) => entry.id),
83 ).toEqual(expected);
84 expect(models[0]?.id).toBe("active");
@@ -91,6 +92,7 @@ it("combines text, all selected languages, and completed downloads without retai
92 downloadedOnly: true,
93 sort: "size",
94 accuracyLanguage: "en",
95 + benchmarkTier: "accelerated",
96 } as const;
97 expect(filterSpeechModels(models, options).map((entry) => entry.id)).toEqual(["fast", "active"]);
98 expect(
apps/web/src/components/settings/speechModelBrowser.ts
+3 -2
@@ -1,4 +1,4 @@
1 -import type { EnvironmentSpeechModel } from "@t3tools/contracts";
1 +import type { EnvironmentSpeechModel, SpeechBenchmarkTier } from "@t3tools/contracts";
2
3 export type SpeechModelSort = "recommended" | "size" | "accuracy" | "speed";
4
@@ -17,6 +17,7 @@ export function filterSpeechModels(
17 readonly downloadedOnly: boolean;
18 readonly sort: SpeechModelSort;
19 readonly accuracyLanguage: string;
20 + readonly benchmarkTier: SpeechBenchmarkTier;
21 },
22 ) {
23 const query = options.search.trim().toLocaleLowerCase();
@@ -27,7 +28,7 @@ export function filterSpeechModels(
28 case "accuracy":
29 return model.benchmark?.accuracy[options.accuracyLanguage]?.errorPercent ?? Infinity;
30 case "speed":
30 - return model.benchmark?.secondsPer30s ?? Infinity;
31 + return model.benchmark?.secondsPer30s[options.benchmarkTier] ?? Infinity;
32 case "recommended":
33 return modelSortOrder(model);
34 }
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+1
@@ -128,6 +128,7 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
128 acceleration: "auto",
129 modelUnloadTimeout: "min_15",
130 gpuDevices: [],
131 + benchmarkTier: "cpu",
132 };
133 }),
134 downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
packages/contracts/src/speech.test.ts new
+97
@@ -0,0 +1,97 @@
1 +import * as Schema from "effect/Schema";
2 +import { expect, it } from "vite-plus/test";
3 +
4 +import { EnvironmentSpeechModel, EnvironmentSpeechStatus } from "./speech.ts";
5 +
6 +const decodeStatus = Schema.decodeUnknownSync(EnvironmentSpeechStatus);
7 +const encodeStatus = Schema.encodeSync(EnvironmentSpeechStatus);
8 +const decodeModel = Schema.decodeUnknownSync(EnvironmentSpeechModel);
9 +const encodeModel = Schema.encodeSync(EnvironmentSpeechModel);
10 +
11 +const legacyStatus = {
12 + supported: true,
13 + state: "ready",
14 + modelId: "test",
15 + model: "Test",
16 + size: 100,
17 + supportsStreaming: false,
18 + supportsTranslation: false,
19 + acceleration: "auto",
20 + modelUnloadTimeout: "never",
21 + gpuDevices: [],
22 +};
23 +const gpuDevices = [{ id: "gpu", name: "Remote GPU" }];
24 +
25 +it.each([
26 + { acceleration: "cpu", gpuDevices, tier: "cpu" },
27 + { acceleration: "auto", gpuDevices: [], tier: "cpu" },
28 + { acceleration: "auto", gpuDevices, tier: "accelerated" },
29 + { acceleration: "gpu:gpu", gpuDevices: [], tier: "accelerated" },
30 +])("decodes legacy speech status for $acceleration with $gpuDevices", ({ tier, ...settings }) => {
31 + const status = decodeStatus({
32 + ...legacyStatus,
33 + ...settings,
34 + });
35 + expect(status).toMatchObject({ ...settings, benchmarkTier: tier });
36 + expect(encodeStatus(status)).toMatchObject({ benchmarkTier: tier });
37 +
38 + const fallback = { ...legacyStatus, ...settings, benchmarkTier: "cpu" };
39 + expect(decodeStatus(fallback)).toEqual(fallback);
40 + expect(() =>
41 + decodeStatus({
42 + ...legacyStatus,
43 + benchmarkTier: "unknown",
44 + }),
45 + ).toThrow();
46 +});
47 +
48 +it.each([
49 + { wire: { secondsPer30s: 2.1 }, expected: { accelerated: 2.1 } },
50 + { wire: { secondsPer30s: null }, expected: { accelerated: null } },
51 + { wire: { secondsPer30s: 2.1, cpuSecondsPer30s: 5 }, expected: { cpu: 5, accelerated: 2.1 } },
52 + {
53 + wire: { secondsPer30s: null, cpuSecondsPer30s: null },
54 + expected: { cpu: null, accelerated: null },
55 + },
56 + {
57 + wire: { secondsPer30s: { cpu: 5, accelerated: 2.1 } },
58 + expected: { cpu: 5, accelerated: 2.1 },
59 + },
60 + { wire: { secondsPer30s: { accelerated: 2.1 } }, expected: { accelerated: 2.1 } },
61 +])("keeps speech benchmark $wire compatible with old and new clients", ({ wire, expected }) => {
62 + const model = {
63 + id: "test",
64 + name: "Test",
65 + description: "",
66 + size: 100,
67 + languages: ["en"],
68 + accuracy: 0,
69 + speed: 0,
70 + benchmark: { accuracy: { en: { grade: "A-", errorPercent: 4 } }, ...wire },
71 + recommended: false,
72 + supportsStreaming: false,
73 + supportsLanguageDetection: false,
74 + supportsTranslation: false,
75 + active: true,
76 + state: "installed",
77 + };
78 + const decoded = decodeModel(model);
79 + expect(decoded.benchmark).toEqual({
80 + accuracy: model.benchmark.accuracy,
81 + secondsPer30s: expected,
82 + });
83 + const encoded = encodeModel(decoded);
84 + const legacyBenchmark = Schema.Struct({ secondsPer30s: Schema.NullOr(Schema.Finite) });
85 + expect(Schema.decodeUnknownSync(legacyBenchmark)(encoded.benchmark)).toEqual({
86 + secondsPer30s: expected.accelerated,
87 + });
88 + expect(decodeModel(encoded)).toEqual(decoded);
89 + for (const secondsPer30s of ["2.1", { cpu: 5 }, Number.NaN]) {
90 + expect(() =>
91 + decodeModel({
92 + ...model,
93 + benchmark: { ...model.benchmark, secondsPer30s },
94 + }),
95 + ).toThrow();
96 + }
97 +});
packages/contracts/src/speech.ts
+89 -21
@@ -1,5 +1,6 @@
1 import * as Effect from "effect/Effect";
2 import * as Schema from "effect/Schema";
3 +import * as SchemaTransformation from "effect/SchemaTransformation";
4 import { TrimmedNonEmptyString } from "./baseSchemas.ts";
5
6 export const CloudSpeechApiKey = TrimmedNonEmptyString.check(
@@ -120,6 +121,9 @@ export type SpeechLanguage = typeof SpeechLanguage.Type;
121
122 export const SpeechGpuDevice = Schema.Struct({ id: Schema.String, name: Schema.String });
123
124 +export const SpeechBenchmarkTier = Schema.Literals(["cpu", "accelerated"]);
125 +export type SpeechBenchmarkTier = typeof SpeechBenchmarkTier.Type;
126 +
127 export const EnvironmentSpeechModelState = Schema.Literals([
128 "downloadable",
129 "downloading",
@@ -128,6 +132,25 @@ export const EnvironmentSpeechModelState = Schema.Literals([
132 ]);
133 export type EnvironmentSpeechModelState = typeof EnvironmentSpeechModelState.Type;
134
135 +// A missing CPU field identifies a legacy server; null means a current catalog has no measurement.
136 +const SpeechBenchmarkTimes = Schema.Struct({
137 + cpu: Schema.optionalKey(Schema.NullOr(Schema.Finite)),
138 + accelerated: Schema.NullOr(Schema.Finite),
139 +});
140 +const SpeechBenchmarkAccuracy = Schema.Record(
141 + Schema.String,
142 + Schema.Struct({ grade: Schema.String, errorPercent: Schema.Finite }),
143 +);
144 +const SpeechBenchmark = Schema.Struct({
145 + accuracy: SpeechBenchmarkAccuracy,
146 + secondsPer30s: SpeechBenchmarkTimes,
147 +});
148 +const SpeechBenchmarkWire = Schema.Struct({
149 + accuracy: SpeechBenchmarkAccuracy,
150 + secondsPer30s: Schema.Union([SpeechBenchmarkTimes, Schema.NullOr(Schema.Finite)]),
151 + cpuSecondsPer30s: Schema.optionalKey(Schema.NullOr(Schema.Finite)),
152 +});
153 +
154 export const EnvironmentSpeechModel = Schema.Struct({
155 id: SpeechModelId,
156 name: Schema.String,
@@ -137,13 +160,32 @@ export const EnvironmentSpeechModel = Schema.Struct({
160 accuracy: Schema.Finite,
161 speed: Schema.Finite,
162 benchmark: Schema.optionalKey(
140 - Schema.Struct({
141 - accuracy: Schema.Record(
142 - Schema.String,
143 - Schema.Struct({ grade: Schema.String, errorPercent: Schema.Finite }),
163 + SpeechBenchmarkWire.pipe(
164 + Schema.decodeTo(
165 + SpeechBenchmark,
166 + SchemaTransformation.transform<
167 + typeof SpeechBenchmark.Type,
168 + typeof SpeechBenchmarkWire.Type
169 + >({
170 + decode: ({ accuracy, secondsPer30s, cpuSecondsPer30s }) => ({
171 + accuracy,
172 + secondsPer30s:
173 + typeof secondsPer30s === "number" || secondsPer30s === null
174 + ? {
175 + accelerated: secondsPer30s,
176 + ...(cpuSecondsPer30s !== undefined ? { cpu: cpuSecondsPer30s } : {}),
177 + }
178 + : secondsPer30s,
179 + }),
180 + // Older clients require the scalar; newer clients normalize both wire representations.
181 + encode: ({ accuracy, secondsPer30s }) => ({
182 + accuracy,
183 + secondsPer30s: secondsPer30s.accelerated,
184 + ...(secondsPer30s.cpu !== undefined ? { cpuSecondsPer30s: secondsPer30s.cpu } : {}),
185 + }),
186 + }),
187 ),
145 - secondsPer30s: Schema.NullOr(Schema.Finite),
146 - }),
188 + ),
189 ),
190 recommended: Schema.Boolean,
191 supportsStreaming: Schema.Boolean,
@@ -165,26 +207,52 @@ export const EnvironmentSpeechModelRequest = Schema.Struct({ modelId: SpeechMode
207 export const EnvironmentSpeechState = Schema.Literals(["missing-model", "ready", "transcribing"]);
208 export type EnvironmentSpeechState = typeof EnvironmentSpeechState.Type;
209
210 +const SupportedSpeechStatus = Schema.Struct({
211 + supported: Schema.Literal(true),
212 + state: EnvironmentSpeechState,
213 + modelId: SpeechModelId,
214 + model: Schema.String,
215 + size: Schema.Finite,
216 + supportsStreaming: Schema.Boolean,
217 + supportsBufferedStreaming: Schema.optionalKey(Schema.Boolean),
218 + supportsTranslation: Schema.Boolean,
219 + languages: Schema.optionalKey(Schema.Array(Schema.String)),
220 + supportsLanguageDetection: Schema.optionalKey(Schema.Boolean),
221 + acceleration: SpeechAcceleration,
222 + benchmarkTier: SpeechBenchmarkTier,
223 + modelUnloadTimeout: SpeechModelUnloadTimeout,
224 + gpuDevices: Schema.Array(SpeechGpuDevice),
225 +});
226 +const SupportedSpeechStatusWire = SupportedSpeechStatus.mapFields((fields) => ({
227 + ...fields,
228 + benchmarkTier: Schema.optionalKey(SpeechBenchmarkTier),
229 +}));
230 +
231 export const EnvironmentSpeechStatus = Schema.Union([
232 Schema.Struct({
233 supported: Schema.Literal(false),
234 reason: Schema.String,
235 }),
173 - Schema.Struct({
174 - supported: Schema.Literal(true),
175 - state: EnvironmentSpeechState,
176 - modelId: SpeechModelId,
177 - model: Schema.String,
178 - size: Schema.Finite,
179 - supportsStreaming: Schema.Boolean,
180 - supportsBufferedStreaming: Schema.optionalKey(Schema.Boolean),
181 - supportsTranslation: Schema.Boolean,
182 - languages: Schema.optionalKey(Schema.Array(Schema.String)),
183 - supportsLanguageDetection: Schema.optionalKey(Schema.Boolean),
184 - acceleration: SpeechAcceleration,
185 - modelUnloadTimeout: SpeechModelUnloadTimeout,
186 - gpuDevices: Schema.Array(SpeechGpuDevice),
187 - }),
236 + SupportedSpeechStatusWire.pipe(
237 + Schema.decodeTo(
238 + Schema.toType(SupportedSpeechStatus),
239 + SchemaTransformation.transform<
240 + typeof SupportedSpeechStatus.Type,
241 + typeof SupportedSpeechStatusWire.Type
242 + >({
243 + decode: (status) => ({
244 + ...status,
245 + benchmarkTier:
246 + status.benchmarkTier ??
247 + (status.acceleration === "cpu" ||
248 + (status.acceleration === "auto" && status.gpuDevices.length === 0)
249 + ? "cpu"
250 + : "accelerated"),
251 + }),
252 + encode: (status) => status,
253 + }),
254 + ),
255 + ),
256 ]);
257 export type EnvironmentSpeechStatus = typeof EnvironmentSpeechStatus.Type;
258