The best way to control Claude Code, Codex and more from any computer or phone
fix(speech): match reference benchmarks to transcription hardware (#31956)
Co-authored-by: maria-rcks <maria@kuuro.net>
Utkarsh Patil committed
Nov 10, 2026 at 00:27 UTC
6b02cbff3a0cd9a0f2c97007562a1ce74fd35aa0
15 files changed
+501
-138
apps/mobile/src/features/settings/VoiceModelSection.tsx
+5
-2
@@ -143,12 +143,15 @@ function ModelStats(props: { readonly model: EnvironmentSpeechModel; readonly la
143
],
144
] as const)
145
: []),
146
- ...(data && data.secondsPer30s !== null
146
+ ...(data && data.secondsPer30s.accelerated !== null
147
? ([
148
[
149
"speed",
150
<>
151
- <Text className="font-t3-medium text-foreground">{data.secondsPer30s} s</Text> / 30 s
151
+ <Text className="font-t3-medium text-foreground">
152
+ {data.secondsPer30s.accelerated} s
153
+ </Text>{" "}
154
+ / 30 s
155
</>,
156
],
157
] as const)
apps/mobile/src/features/voice-input/voiceDestination.test.ts
+1
@@ -158,6 +158,7 @@ it("keeps local as the default and never reuses consent for an offline or resele
158
acceleration: "cpu",
159
modelUnloadTimeout: "never",
160
gpuDevices: [],
161
+ benchmarkTier: "cpu",
162
}),
163
)
164
.mockResolvedValueOnce(Response.json({ text: "hello" }));
apps/server/scripts/speech-catalog.test.ts
+17
-2
@@ -45,9 +45,24 @@ it("reproduces the bundled registry byte for byte without upstream requests", as
45
);
46
expect(withoutFrenchOrSpeed.models[0]?.benchmark).toEqual({
47
accuracy: { en: { grade: "A-", errorPercent: 3.99 } },
48
- secondsPer30s: null,
48
+ secondsPer30s: { cpu: null, accelerated: null },
49
});
50
- expect(generate().models[0]?.benchmark.secondsPer30s).toBeCloseTo(1.19, 2);
50
+ expect(generate().models[0]?.benchmark.secondsPer30s.accelerated).toBeCloseTo(1.19, 2);
51
+ for (const performance of [{ cpu: 2 }, { accelerated: 10 }, { cpu: 2, accelerated: 10 }]) {
52
+ const result = generateSpeechCatalog(
53
+ {
54
+ ...catalog.data,
55
+ models: [parakeet],
56
+ results: { [parakeet.id]: { accuracy: {}, performance } },
57
+ },
58
+ "0.2.3",
59
+ "0.4.1",
60
+ );
61
+ expect(result.models[0]?.benchmark.secondsPer30s).toEqual({
62
+ cpu: performance.cpu ? 15 : null,
63
+ accelerated: performance.accelerated ? 3 : null,
64
+ });
65
+ }
66
});
67
68
it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {
apps/server/scripts/speech-catalog.ts
+7
-4
@@ -81,7 +81,9 @@ export function generateSpeechCatalog(
81
ids.add(model.repository);
82
filenames.add(file.filename);
83
const languages = [...new Set(model.languages.map(canonicalLanguage))];
84
- const realtimeFactor = data.results[model.id]?.performance.accelerated;
84
+ const performance = data.results[model.id]?.performance;
85
+ const secondsPer30s = (realtimeFactor: number | undefined) =>
86
+ realtimeFactor ? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2)) : null;
87
const accuracy = Object.fromEntries(
88
languages.flatMap((language) => {
89
const cell = data.results[model.id]?.accuracy[languageIdentity(language)];
@@ -109,9 +111,10 @@ export function generateSpeechCatalog(
111
languages,
112
benchmark: {
113
accuracy,
112
- secondsPer30s: realtimeFactor
113
- ? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2))
114
- : null,
114
+ secondsPer30s: {
115
+ cpu: secondsPer30s(performance?.cpu),
116
+ accelerated: secondsPer30s(performance?.accelerated),
117
+ },
118
},
119
recommended: RECOMMENDED.has(model.id),
120
supportsStreaming: model.capabilities.streaming,
apps/server/src/speech/SpeechService.test.ts
+50
-28
@@ -58,7 +58,10 @@ const modelDefinitions = vi.hoisted(() => [
58
size: 4,
59
languages: ["en", "fr"],
60
accuracy: 1,
61
- benchmark: { accuracy: { en: { grade: "A-", errorPercent: 4 } }, secondsPer30s: 1.2 },
61
+ benchmark: {
62
+ accuracy: { en: { grade: "A-", errorPercent: 4 } },
63
+ secondsPer30s: { cpu: 3, accelerated: 1.2 },
64
+ },
65
speed: 1,
66
recommended: true,
67
supportsStreaming: true,
@@ -146,31 +149,40 @@ it("rejects empty, misaligned, non-finite, and oversized PCM as invalid audio",
149
}
150
});
151
149
-it.effect("prepares a batch model before audio arrives and reuses it for transcription", () =>
150
- Effect.gen(function* () {
151
- const speech = yield* SpeechService.SpeechService;
152
- yield* speech.selectModel("fallback-model");
153
- yield* speech.prepareModel;
154
- expect(loadNative).toHaveBeenCalledTimes(1);
155
- expect(native.transcribe).not.toHaveBeenCalled();
156
- native.supportsStreaming = false;
157
- yield* Effect.gen(function* () {
158
- const stream = yield* speech.startStream({
159
- ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
160
- speechLanguage: "fr",
161
- speechTranslateToEnglish: true,
162
- speechSilenceFiltering: false,
152
+it.effect.each(["CPU", "Vulkan0"])(
153
+ "prepares a batch model on %s before audio arrives and reuses it for transcription",
154
+ (backend) =>
155
+ Effect.gen(function* () {
156
+ const speech = yield* SpeechService.SpeechService;
157
+ yield* speech.selectModel("fallback-model");
158
+ loadNative.mockImplementationOnce(async () => {
159
+ native.backend = backend;
160
+ return native;
161
});
164
- expect(native.begin).toHaveBeenLastCalledWith(
165
- { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
166
- false,
167
- "test.gguf",
168
- );
169
- expect(yield* stream.finish).toBe("hello");
170
- }).pipe(Effect.scoped);
171
- expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
172
- expect(loadNative).toHaveBeenCalledTimes(1);
173
- }).pipe(Effect.provide(layer)),
162
+ yield* speech.prepareModel;
163
+ expect(yield* speech.status).toMatchObject({
164
+ benchmarkTier: backend === "CPU" ? "cpu" : "accelerated",
165
+ });
166
+ expect(loadNative).toHaveBeenCalledTimes(1);
167
+ expect(native.transcribe).not.toHaveBeenCalled();
168
+ native.supportsStreaming = false;
169
+ yield* Effect.gen(function* () {
170
+ const stream = yield* speech.startStream({
171
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
172
+ speechLanguage: "fr",
173
+ speechTranslateToEnglish: true,
174
+ speechSilenceFiltering: false,
175
+ });
176
+ expect(native.begin).toHaveBeenLastCalledWith(
177
+ { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
178
+ false,
179
+ "test.gguf",
180
+ );
181
+ expect(yield* stream.finish).toBe("hello");
182
+ }).pipe(Effect.scoped);
183
+ expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
184
+ expect(loadNative).toHaveBeenCalledTimes(1);
185
+ }).pipe(Effect.provide(layer)),
186
);
187
188
it.effect("sends the chosen language in the spelling the loaded model advertises", () =>
@@ -290,7 +302,7 @@ it.effect("rejects model mutations outside the offered registry", () =>
302
expect(current).not.toHaveProperty("benchmark");
303
expect(
304
(yield* speech.models(true)).models.find((model) => model.active)?.benchmark?.secondsPer30s,
293
- ).toBe(1.2);
305
+ ).toEqual({ cpu: 3, accelerated: 1.2 });
306
}).pipe(Effect.provide(layer)),
307
);
308
@@ -665,11 +677,19 @@ it.effect.each([{ devices: [] }, { devices: [{ id: "gpu", name: "Test GPU" }] }]
677
const speech = yield* SpeechService.SpeechService;
678
const discover = vi.mocked(listNativeSpeechGpuDevices).mockReset();
679
discover.mockRejectedValueOnce(new Error("driver not ready")).mockResolvedValueOnce(devices);
668
- expect(yield* speech.status).toMatchObject({ gpuDevices: [] });
680
+ expect(yield* speech.status).toMatchObject({ gpuDevices: [], benchmarkTier: "cpu" });
681
yield* speech.status;
682
expect(discover).toHaveBeenCalledOnce();
683
yield* TestClock.adjust("1 minute");
672
- expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
684
+ expect(yield* speech.status).toMatchObject({
685
+ gpuDevices: devices,
686
+ benchmarkTier: devices.length ? "accelerated" : "cpu",
687
+ });
688
+ expect(yield* speech.updateAcceleration("cpu")).toMatchObject({ benchmarkTier: "cpu" });
689
+ expect(yield* speech.updateAcceleration('gpu:["vulkan","gpu-1"]')).toMatchObject({
690
+ benchmarkTier: "accelerated",
691
+ });
692
+ yield* speech.updateAcceleration("auto");
693
yield* TestClock.adjust("1 minute");
694
expect(yield* speech.status).toMatchObject({ gpuDevices: devices });
695
expect(discover).toHaveBeenCalledTimes(2);
@@ -700,9 +720,11 @@ it.effect(
720
],
721
};
722
const speech = yield* SpeechService.SpeechService;
723
+ expect(yield* speech.status).toMatchObject({ benchmarkTier: "accelerated" });
724
expect(yield* speech.transcribe(pcm(), options)).toBe(
725
cpuSupportsVocabulary ? "git hub Bob" : "GitHub Bob",
726
);
727
+ expect(yield* speech.status).toMatchObject({ benchmarkTier: "cpu" });
728
expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe(
729
"hello",
730
);
apps/server/src/speech/SpeechService.ts
+13
-1
@@ -452,6 +452,13 @@ const make = Effect.gen(function* () {
452
if (unsupportedReason) return { supported: false, reason: unsupportedReason };
453
const definition = selectedModel(settings);
454
const ready = yield* store.isReady(definition);
455
+ const gpuDevices = yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => []));
456
+ const autoUsesCpu =
457
+ gpuDevices.length === 0 ||
458
+ autoCpuModel === definition.id ||
459
+ (loadedModelId === definition.id &&
460
+ loadedAcceleration === "auto" &&
461
+ model?.backend.toLowerCase() === "cpu");
462
return {
463
supported: true,
464
state: activeTranscriptions > 0 ? "transcribing" : ready ? "ready" : "missing-model",
@@ -464,8 +471,13 @@ const make = Effect.gen(function* () {
471
languages: definition.languages,
472
supportsLanguageDetection: definition.supportsLanguageDetection,
473
acceleration: settings.speechAcceleration,
474
+ benchmarkTier:
475
+ settings.speechAcceleration === "cpu" ||
476
+ (settings.speechAcceleration === "auto" && autoUsesCpu)
477
+ ? "cpu"
478
+ : "accelerated",
479
modelUnloadTimeout: settings.speechModelUnloadTimeout,
468
- gpuDevices: yield* discoverGpuDevices.pipe(Effect.orElseSucceed(() => [])),
480
+ gpuDevices,
481
};
482
});
483
apps/server/src/speech/catalog.json
+59
-59
@@ -15,7 +15,7 @@
15
"languages": ["en"],
16
"benchmark": {
17
"accuracy": { "en": { "grade": "A-", "errorPercent": 3.99 } },
18
- "secondsPer30s": 1.19
18
+ "secondsPer30s": { "cpu": 2.15, "accelerated": 1.19 }
19
},
20
"recommended": true,
21
"supportsStreaming": true,
@@ -92,7 +92,7 @@
92
"ro": { "grade": "D", "errorPercent": 28.28 },
93
"et": { "grade": "F", "errorPercent": 31.84 }
94
},
95
- "secondsPer30s": 1.75
95
+ "secondsPer30s": { "cpu": 2.83, "accelerated": 1.75 }
96
},
97
"recommended": true,
98
"supportsStreaming": true,
@@ -116,7 +116,7 @@
116
"es": { "grade": "B", "errorPercent": 6.54 },
117
"fr": { "grade": "B-", "errorPercent": 8.53 }
118
},
119
- "secondsPer30s": 0.91
119
+ "secondsPer30s": { "cpu": 1.38, "accelerated": 0.91 }
120
},
121
"recommended": true,
122
"supportsStreaming": false,
@@ -165,7 +165,7 @@
165
"vi": { "grade": "B", "errorPercent": 7.39 },
166
"ko": { "grade": "B", "errorPercent": 6.57 }
167
},
168
- "secondsPer30s": 3.71
168
+ "secondsPer30s": { "cpu": 6.96, "accelerated": 3.71 }
169
},
170
"recommended": true,
171
"supportsStreaming": false,
@@ -367,7 +367,7 @@
367
"yo": { "grade": "F", "errorPercent": 109 },
368
"zh": { "grade": "C", "errorPercent": 13.13 }
369
},
370
- "secondsPer30s": 7.34
370
+ "secondsPer30s": { "cpu": 15.58, "accelerated": 7.34 }
371
},
372
"recommended": true,
373
"supportsStreaming": false,
@@ -400,7 +400,7 @@
400
"hi": { "grade": "C", "errorPercent": 14.44 },
401
"ko": { "grade": "B+", "errorPercent": 5.27 }
402
},
403
- "secondsPer30s": 39.47
403
+ "secondsPer30s": { "cpu": 51.28, "accelerated": 39.47 }
404
},
405
"recommended": false,
406
"supportsStreaming": true,
@@ -471,7 +471,7 @@
471
"el": { "grade": "F", "errorPercent": 32.25 },
472
"mt": { "grade": "C-", "errorPercent": 17.01 }
473
},
474
- "secondsPer30s": 1.3
474
+ "secondsPer30s": { "cpu": 2.26, "accelerated": 1.3 }
475
},
476
"recommended": false,
477
"supportsStreaming": false,
@@ -542,7 +542,7 @@
542
"sv": { "grade": "C", "errorPercent": 15.25 },
543
"uk": { "grade": "B", "errorPercent": 6.84 }
544
},
545
- "secondsPer30s": 1.2
545
+ "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
546
},
547
"recommended": false,
548
"supportsStreaming": false,
@@ -561,7 +561,7 @@
561
"languages": ["en"],
562
"benchmark": {
563
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.11 } },
564
- "secondsPer30s": 1.09
564
+ "secondsPer30s": { "cpu": 2.1, "accelerated": 1.09 }
565
},
566
"recommended": false,
567
"supportsStreaming": false,
@@ -642,7 +642,7 @@
642
"hu": { "grade": "F", "errorPercent": 56.24 },
643
"mk": { "grade": "F", "errorPercent": 35.09 }
644
},
645
- "secondsPer30s": 3.09
645
+ "secondsPer30s": { "cpu": 5.67, "accelerated": 3.09 }
646
},
647
"recommended": false,
648
"supportsStreaming": false,
@@ -725,7 +725,7 @@
725
"sl": { "grade": "F", "errorPercent": 75.22 },
726
"sv": { "grade": "F", "errorPercent": 75.36 }
727
},
728
- "secondsPer30s": 3.4
728
+ "secondsPer30s": { "cpu": 5.48, "accelerated": 3.4 }
729
},
730
"recommended": false,
731
"supportsStreaming": false,
@@ -796,7 +796,7 @@
796
"ru": { "grade": "B", "errorPercent": 7.83 },
797
"uk": { "grade": "C+", "errorPercent": 10.58 }
798
},
799
- "secondsPer30s": 2.12
799
+ "secondsPer30s": { "cpu": 3.6, "accelerated": 2.12 }
800
},
801
"recommended": false,
802
"supportsStreaming": false,
@@ -815,7 +815,7 @@
815
"languages": ["en"],
816
"benchmark": {
817
"accuracy": { "en": { "grade": "A", "errorPercent": 3.58 } },
818
- "secondsPer30s": 7.55
818
+ "secondsPer30s": { "cpu": 11.95, "accelerated": 7.55 }
819
},
820
"recommended": false,
821
"supportsStreaming": false,
@@ -837,7 +837,7 @@
837
"en": { "grade": "A-", "errorPercent": 4.88 },
838
"ar": { "grade": "C+", "errorPercent": 11.06 }
839
},
840
- "secondsPer30s": 3.61
840
+ "secondsPer30s": { "cpu": 6.96, "accelerated": 3.61 }
841
},
842
"recommended": false,
843
"supportsStreaming": false,
@@ -860,7 +860,7 @@
860
"en": { "grade": "B+", "errorPercent": 5.49 },
861
"ja": { "grade": "B-", "errorPercent": 8.5 }
862
},
863
- "secondsPer30s": 2.8
863
+ "secondsPer30s": { "cpu": 3.78, "accelerated": 2.8 }
864
},
865
"recommended": false,
866
"supportsStreaming": false,
@@ -879,7 +879,7 @@
879
"languages": ["ru"],
880
"benchmark": {
881
"accuracy": { "ru": { "grade": "B+", "errorPercent": 5.53 } },
882
- "secondsPer30s": 0.85
882
+ "secondsPer30s": { "cpu": 2.62, "accelerated": 0.85 }
883
},
884
"recommended": false,
885
"supportsStreaming": false,
@@ -898,7 +898,7 @@
898
"languages": ["ru"],
899
"benchmark": {
900
"accuracy": { "ru": { "grade": "B+", "errorPercent": 5.35 } },
901
- "secondsPer30s": 2
901
+ "secondsPer30s": { "cpu": 3.81, "accelerated": 2 }
902
},
903
"recommended": false,
904
"supportsStreaming": false,
@@ -917,7 +917,7 @@
917
"languages": ["en"],
918
"benchmark": {
919
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.61 } },
920
- "secondsPer30s": 1.31
920
+ "secondsPer30s": { "cpu": 1.74, "accelerated": 1.31 }
921
},
922
"recommended": false,
923
"supportsStreaming": false,
@@ -943,7 +943,7 @@
943
"pt": { "grade": "B-", "errorPercent": 9.8 },
944
"ja": { "grade": "B", "errorPercent": 6.3 }
945
},
946
- "secondsPer30s": 6.5
946
+ "secondsPer30s": { "cpu": 10.81, "accelerated": 6.5 }
947
},
948
"recommended": false,
949
"supportsStreaming": false,
@@ -962,7 +962,7 @@
962
"languages": ["en"],
963
"benchmark": {
964
"accuracy": { "en": { "grade": "F", "errorPercent": 37.48 } },
965
- "secondsPer30s": 0.42
965
+ "secondsPer30s": { "cpu": 1.16, "accelerated": 0.42 }
966
},
967
"recommended": false,
968
"supportsStreaming": false,
@@ -981,7 +981,7 @@
981
"languages": ["en"],
982
"benchmark": {
983
"accuracy": { "en": { "grade": "B+", "errorPercent": 6.11 } },
984
- "secondsPer30s": 0.39
984
+ "secondsPer30s": { "cpu": 0.6, "accelerated": 0.39 }
985
},
986
"recommended": false,
987
"supportsStreaming": false,
@@ -1000,7 +1000,7 @@
1000
"languages": ["en"],
1001
"benchmark": {
1002
"accuracy": { "en": { "grade": "B", "errorPercent": 6.52 } },
1003
- "secondsPer30s": 1.14
1003
+ "secondsPer30s": { "cpu": 2.24, "accelerated": 1.14 }
1004
},
1005
"recommended": false,
1006
"supportsStreaming": true,
@@ -1081,7 +1081,7 @@
1081
"hu": { "grade": "F", "errorPercent": 32.84 },
1082
"mk": { "grade": "C-", "errorPercent": 18.22 }
1083
},
1084
- "secondsPer30s": 7.34
1084
+ "secondsPer30s": { "cpu": 12.15, "accelerated": 7.34 }
1085
},
1086
"recommended": false,
1087
"supportsStreaming": false,
@@ -1106,7 +1106,7 @@
1106
"ja": { "grade": "B", "errorPercent": 7.63 },
1107
"ko": { "grade": "B-", "errorPercent": 8.27 }
1108
},
1109
- "secondsPer30s": 1.08
1109
+ "secondsPer30s": { "cpu": 1.52, "accelerated": 1.08 }
1110
},
1111
"recommended": false,
1112
"supportsStreaming": false,
@@ -1134,7 +1134,7 @@
1134
"nl": { "grade": "B", "errorPercent": 6.57 },
1135
"hi": { "grade": "B-", "errorPercent": 8.93 }
1136
},
1137
- "secondsPer30s": 27.65
1137
+ "secondsPer30s": { "cpu": 53.57, "accelerated": 27.65 }
1138
},
1139
"recommended": false,
1140
"supportsStreaming": false,
@@ -1336,7 +1336,7 @@
1336
"yo": { "grade": "F", "errorPercent": 118.5 },
1337
"zh": { "grade": "D", "errorPercent": 23.06 }
1338
},
1339
- "secondsPer30s": 2.57
1339
+ "secondsPer30s": { "cpu": 5.12, "accelerated": 2.57 }
1340
},
1341
"recommended": false,
1342
"supportsStreaming": false,
@@ -1540,7 +1540,7 @@
1540
"yue": { "grade": "F", "errorPercent": 34.62 },
1541
"zh": { "grade": "B-", "errorPercent": 8.5 }
1542
},
1543
- "secondsPer30s": 9.69
1543
+ "secondsPer30s": { "cpu": 20.98, "accelerated": 9.69 }
1544
},
1545
"recommended": false,
1546
"supportsStreaming": false,
@@ -1562,7 +1562,7 @@
1562
"zh": { "grade": "B-", "errorPercent": 8.1 },
1563
"en": { "grade": "A-", "errorPercent": 4.14 }
1564
},
1565
- "secondsPer30s": 14.85
1565
+ "secondsPer30s": { "cpu": 31.25, "accelerated": 14.85 }
1566
},
1567
"recommended": false,
1568
"supportsStreaming": false,
@@ -1766,7 +1766,7 @@
1766
"yue": { "grade": "D", "errorPercent": 22.06 },
1767
"zh": { "grade": "B-", "errorPercent": 7.98 }
1768
},
1769
- "secondsPer30s": 15.04
1769
+ "secondsPer30s": { "cpu": 30.93, "accelerated": 15.04 }
1770
},
1771
"recommended": false,
1772
"supportsStreaming": false,
@@ -1790,7 +1790,7 @@
1790
"es": { "grade": "B", "errorPercent": 6.73 },
1791
"fr": { "grade": "B", "errorPercent": 7.22 }
1792
},
1793
- "secondsPer30s": 1.94
1793
+ "secondsPer30s": { "cpu": 3.39, "accelerated": 1.94 }
1794
},
1795
"recommended": false,
1796
"supportsStreaming": false,
@@ -1809,7 +1809,7 @@
1809
"languages": ["ru"],
1810
"benchmark": {
1811
"accuracy": { "ru": { "grade": "B-", "errorPercent": 8.42 } },
1812
- "secondsPer30s": 0.75
1812
+ "secondsPer30s": { "cpu": 2.54, "accelerated": 0.75 }
1813
},
1814
"recommended": false,
1815
"supportsStreaming": false,
@@ -1828,7 +1828,7 @@
1828
"languages": ["ru"],
1829
"benchmark": {
1830
"accuracy": { "ru": { "grade": "B-", "errorPercent": 8.07 } },
1831
- "secondsPer30s": 1.12
1831
+ "secondsPer30s": { "cpu": 2.9, "accelerated": 1.12 }
1832
},
1833
"recommended": false,
1834
"supportsStreaming": false,
@@ -1854,7 +1854,7 @@
1854
"pt": { "grade": "B-", "errorPercent": 9.6 },
1855
"ja": { "grade": "B", "errorPercent": 6.44 }
1856
},
1857
- "secondsPer30s": 6.47
1857
+ "secondsPer30s": { "cpu": 10.89, "accelerated": 6.47 }
1858
},
1859
"recommended": false,
1860
"supportsStreaming": false,
@@ -1879,7 +1879,7 @@
1879
"es": { "grade": "A-", "errorPercent": 4.08 },
1880
"pt": { "grade": "B+", "errorPercent": 5.57 }
1881
},
1882
- "secondsPer30s": 7.51
1882
+ "secondsPer30s": { "cpu": 13.99, "accelerated": 7.51 }
1883
},
1884
"recommended": false,
1885
"supportsStreaming": false,
@@ -1904,7 +1904,7 @@
1904
"es": { "grade": "B", "errorPercent": 6.53 },
1905
"pt": { "grade": "C+", "errorPercent": 10.61 }
1906
},
1907
- "secondsPer30s": 6.46
1907
+ "secondsPer30s": { "cpu": 11.54, "accelerated": 6.46 }
1908
},
1909
"recommended": false,
1910
"supportsStreaming": false,
@@ -1923,7 +1923,7 @@
1923
"languages": ["en"],
1924
"benchmark": {
1925
"accuracy": { "en": { "grade": "C+", "errorPercent": 12.25 } },
1926
- "secondsPer30s": 0.84
1926
+ "secondsPer30s": { "cpu": 1.28, "accelerated": 0.84 }
1927
},
1928
"recommended": false,
1929
"supportsStreaming": false,
@@ -1942,7 +1942,7 @@
1942
"languages": ["en"],
1943
"benchmark": {
1944
"accuracy": { "en": { "grade": "B", "errorPercent": 7.87 } },
1945
- "secondsPer30s": 2.1
1945
+ "secondsPer30s": { "cpu": 3.1, "accelerated": 2.1 }
1946
},
1947
"recommended": false,
1948
"supportsStreaming": true,
@@ -1961,7 +1961,7 @@
1961
"languages": ["en"],
1962
"benchmark": {
1963
"accuracy": { "en": { "grade": "B-", "errorPercent": 8.55 } },
1964
- "secondsPer30s": 1.38
1964
+ "secondsPer30s": { "cpu": 2.22, "accelerated": 1.38 }
1965
},
1966
"recommended": false,
1967
"supportsStreaming": true,
@@ -1980,7 +1980,7 @@
1980
"languages": ["en"],
1981
"benchmark": {
1982
"accuracy": { "en": { "grade": "C-", "errorPercent": 18.18 } },
1983
- "secondsPer30s": 0.51
1983
+ "secondsPer30s": { "cpu": 0.53, "accelerated": 0.51 }
1984
},
1985
"recommended": false,
1986
"supportsStreaming": true,
@@ -1999,7 +1999,7 @@
1999
"languages": ["en"],
2000
"benchmark": {
2001
"accuracy": { "en": { "grade": "C", "errorPercent": 14.13 } },
2002
- "secondsPer30s": 0.5
2002
+ "secondsPer30s": { "cpu": 0.73, "accelerated": 0.5 }
2003
},
2004
"recommended": false,
2005
"supportsStreaming": false,
@@ -2021,7 +2021,7 @@
2021
"en": { "grade": "B+", "errorPercent": 5.13 },
2022
"zh": { "grade": "B-", "errorPercent": 9.23 }
2023
},
2024
- "secondsPer30s": 9.79
2024
+ "secondsPer30s": { "cpu": 19.11, "accelerated": 9.79 }
2025
},
2026
"recommended": false,
2027
"supportsStreaming": false,
@@ -2040,7 +2040,7 @@
2040
"languages": ["en"],
2041
"benchmark": {
2042
"accuracy": { "en": { "grade": "B", "errorPercent": 6.43 } },
2043
- "secondsPer30s": 1.11
2043
+ "secondsPer30s": { "cpu": 2.21, "accelerated": 1.11 }
2044
},
2045
"recommended": false,
2046
"supportsStreaming": true,
@@ -2111,7 +2111,7 @@
2111
"sv": { "grade": "C", "errorPercent": 13.26 },
2112
"uk": { "grade": "B", "errorPercent": 6.53 }
2113
},
2114
- "secondsPer30s": 1.2
2114
+ "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
2115
},
2116
"recommended": false,
2117
"supportsStreaming": false,
@@ -2130,7 +2130,7 @@
2130
"languages": ["en"],
2131
"benchmark": {
2132
"accuracy": { "en": { "grade": "B+", "errorPercent": 5.53 } },
2133
- "secondsPer30s": 1.05
2133
+ "secondsPer30s": { "cpu": 1.98, "accelerated": 1.05 }
2134
},
2135
"recommended": false,
2136
"supportsStreaming": false,
@@ -2149,7 +2149,7 @@
2149
"languages": ["en"],
2150
"benchmark": {
2151
"accuracy": { "en": { "grade": "B+", "errorPercent": 5.61 } },
2152
- "secondsPer30s": 1.79
2152
+ "secondsPer30s": { "cpu": 3.36, "accelerated": 1.79 }
2153
},
2154
"recommended": false,
2155
"supportsStreaming": false,
@@ -2220,7 +2220,7 @@
2220
"sv": { "grade": "C-", "errorPercent": 16.42 },
2221
"uk": { "grade": "B-", "errorPercent": 8.11 }
2222
},
2223
- "secondsPer30s": 1.2
2223
+ "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
2224
},
2225
"recommended": false,
2226
"supportsStreaming": false,
@@ -2239,7 +2239,7 @@
2239
"languages": ["en"],
2240
"benchmark": {
2241
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.57 } },
2242
- "secondsPer30s": 1.11
2242
+ "secondsPer30s": { "cpu": 2.02, "accelerated": 1.11 }
2243
},
2244
"recommended": false,
2245
"supportsStreaming": false,
@@ -2258,7 +2258,7 @@
2258
"languages": ["en"],
2259
"benchmark": {
2260
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.45 } },
2261
- "secondsPer30s": 1.87
2261
+ "secondsPer30s": { "cpu": 3.49, "accelerated": 1.87 }
2262
},
2263
"recommended": false,
2264
"supportsStreaming": false,
@@ -2277,7 +2277,7 @@
2277
"languages": ["en"],
2278
"benchmark": {
2279
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.68 } },
2280
- "secondsPer30s": 1.76
2280
+ "secondsPer30s": { "cpu": 3.22, "accelerated": 1.76 }
2281
},
2282
"recommended": false,
2283
"supportsStreaming": false,
@@ -2296,7 +2296,7 @@
2296
"languages": ["en"],
2297
"benchmark": {
2298
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.24 } },
2299
- "secondsPer30s": 1.82
2299
+ "secondsPer30s": { "cpu": 3.41, "accelerated": 1.82 }
2300
},
2301
"recommended": false,
2302
"supportsStreaming": false,
@@ -2324,7 +2324,7 @@
2324
"nl": { "grade": "B+", "errorPercent": 5.12 },
2325
"hi": { "grade": "B", "errorPercent": 7.4 }
2326
},
2327
- "secondsPer30s": null
2327
+ "secondsPer30s": { "cpu": null, "accelerated": null }
2328
},
2329
"recommended": false,
2330
"supportsStreaming": false,
@@ -2526,7 +2526,7 @@
2526
"yo": { "grade": "F", "errorPercent": 103.28 },
2527
"zh": { "grade": "F", "errorPercent": 36.21 }
2528
},
2529
- "secondsPer30s": 1.03
2529
+ "secondsPer30s": { "cpu": 1.55, "accelerated": 1.03 }
2530
},
2531
"recommended": false,
2532
"supportsStreaming": false,
@@ -2545,7 +2545,7 @@
2545
"languages": ["en"],
2546
"benchmark": {
2547
"accuracy": { "en": { "grade": "B", "errorPercent": 7.6 } },
2548
- "secondsPer30s": 0.94
2548
+ "secondsPer30s": { "cpu": 1.46, "accelerated": 0.94 }
2549
},
2550
"recommended": false,
2551
"supportsStreaming": false,
@@ -2747,7 +2747,7 @@
2747
"yo": { "grade": "F", "errorPercent": 113.72 },
2748
"zh": { "grade": "C-", "errorPercent": 19.32 }
2749
},
2750
- "secondsPer30s": 14.96
2750
+ "secondsPer30s": { "cpu": 31.25, "accelerated": 14.96 }
2751
},
2752
"recommended": false,
2753
"supportsStreaming": false,
@@ -2949,7 +2949,7 @@
2949
"yo": { "grade": "F", "errorPercent": 96 },
2950
"zh": { "grade": "C", "errorPercent": 15.39 }
2951
},
2952
- "secondsPer30s": 14.93
2952
+ "secondsPer30s": { "cpu": 31.25, "accelerated": 14.93 }
2953
},
2954
"recommended": false,
2955
"supportsStreaming": false,
@@ -2968,7 +2968,7 @@
2968
"languages": ["en"],
2969
"benchmark": {
2970
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.88 } },
2971
- "secondsPer30s": 6.81
2971
+ "secondsPer30s": { "cpu": 14.53, "accelerated": 6.81 }
2972
},
2973
"recommended": false,
2974
"supportsStreaming": false,
@@ -2987,7 +2987,7 @@
2987
"languages": ["en"],
2988
"benchmark": {
2989
"accuracy": { "en": { "grade": "B+", "errorPercent": 6.14 } },
2990
- "secondsPer30s": 2.4
2990
+ "secondsPer30s": { "cpu": 4.79, "accelerated": 2.4 }
2991
},
2992
"recommended": false,
2993
"supportsStreaming": false,
@@ -3189,7 +3189,7 @@
3189
"yo": { "grade": "F", "errorPercent": 102.29 },
3190
"zh": { "grade": "F", "errorPercent": 41.91 }
3191
},
3192
- "secondsPer30s": 0.7
3192
+ "secondsPer30s": { "cpu": 0.75, "accelerated": 0.7 }
3193
},
3194
"recommended": false,
3195
"supportsStreaming": false,
@@ -3208,7 +3208,7 @@
3208
"languages": ["en"],
3209
"benchmark": {
3210
"accuracy": { "en": { "grade": "C+", "errorPercent": 10.72 } },
3211
- "secondsPer30s": 0.63
3211
+ "secondsPer30s": { "cpu": 0.73, "accelerated": 0.63 }
3212
},
3213
"recommended": false,
3214
"supportsStreaming": false,
apps/web/src/components/settings/SpeechModelComparison.tsx
+16
-13
@@ -1,4 +1,4 @@
1
-import type { EnvironmentSpeechModel } from "@t3tools/contracts";
1
+import type { EnvironmentSpeechModel, SpeechBenchmarkTier } from "@t3tools/contracts";
2
import { CheckIcon } from "lucide-react";
3
4
import { cn } from "../../lib/utils";
@@ -73,14 +73,17 @@ export function LanguageRow({
73
export function ModelBenchmarks({
74
model,
75
languages,
76
+ tier,
77
facts,
78
}: {
79
model: EnvironmentSpeechModel;
80
languages: readonly string[];
81
+ tier: SpeechBenchmarkTier;
82
/** Plain facts, such as language count and size, shown after the scores. */
83
facts: readonly string[];
84
}) {
85
const data = model.benchmark;
86
+ const seconds = data?.secondsPer30s[tier];
87
const visible = !data
88
? []
89
: languages.length
@@ -106,18 +109,18 @@ export function ModelBenchmarks({
109
</span>
110
);
111
})}
109
- {data ? (
110
- <span>
111
- Speed:{" "}
112
- {data.secondsPer30s === null ? (
113
- "not benchmarked"
114
- ) : (
115
- <>
116
- <strong className="font-medium text-foreground">{data.secondsPer30s} s</strong> / 30 s
117
- </>
118
- )}
119
- </span>
120
- ) : null}
112
+ <span>
113
+ Speed:{" "}
114
+ {tier === "cpu" && data && data.secondsPer30s.cpu === undefined ? (
115
+ "update server"
116
+ ) : seconds == null ? (
117
+ "not benchmarked"
118
+ ) : (
119
+ <>
120
+ <strong className="font-medium text-foreground">{seconds} s</strong> / 30 s
121
+ </>
122
+ )}
123
+ </span>
124
{facts.map((fact) => (
125
<span key={fact}>{fact}</span>
126
))}
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+132
-2
@@ -1,4 +1,9 @@
1
-import { PrimaryConnectionTarget, RelayConnectionTarget } from "@t3tools/client-runtime/connection";
1
+import {
2
+ BearerConnectionTarget,
3
+ PrimaryConnectionTarget,
4
+ RelayConnectionTarget,
5
+ type PreparedConnection,
6
+} from "@t3tools/client-runtime/connection";
7
import type * as Effect from "effect/Effect";
8
import {
9
EnvironmentId,
@@ -71,6 +76,128 @@ vi.mock("../../connection/runtime", async () => {
76
};
77
});
78
79
+function deferred<T>() {
80
+ let resolve!: (value: T) => void;
81
+ const promise = new Promise<T>((done) => {
82
+ resolve = done;
83
+ });
84
+ return { promise, resolve };
85
+}
86
+
87
+it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accelerated: 2.1 }])(
88
+ "uses the destination environment's benchmark tier across switches with measurements %j",
89
+ async (secondsPer30s) => {
90
+ const target = new PrimaryConnectionTarget({
91
+ environmentId: EnvironmentId.make("voice-local"),
92
+ label: "Local CPU",
93
+ httpBaseUrl: window.location.origin,
94
+ wsBaseUrl: "ws://localhost",
95
+ });
96
+ const local: PreparedConnection = {
97
+ ...target,
98
+ target,
99
+ socketUrl: target.wsBaseUrl,
100
+ httpAuthorization: null,
101
+ };
102
+ const remote: PreparedConnection = {
103
+ environmentId: EnvironmentId.make("voice-remote"),
104
+ label: "Remote GPU",
105
+ httpBaseUrl: "https://voice.example.test",
106
+ socketUrl: "wss://voice.example.test/ws",
107
+ httpAuthorization: { _tag: "Bearer", token: "test-token" },
108
+ target: new BearerConnectionTarget({
109
+ environmentId: EnvironmentId.make("voice-remote"),
110
+ label: "Remote GPU",
111
+ connectionId: "remote",
112
+ }),
113
+ };
114
+ let prepared = local;
115
+ const remoteStatus = deferred<Response>();
116
+ const remoteRequested = deferred<void>();
117
+ const status = {
118
+ supported: true,
119
+ state: "ready",
120
+ modelId: "both",
121
+ model: "Bilingual",
122
+ size: 100,
123
+ supportsStreaming: false,
124
+ supportsTranslation: false,
125
+ acceleration: "auto",
126
+ modelUnloadTimeout: "never",
127
+ gpuDevices: [],
128
+ benchmarkTier: "cpu",
129
+ } satisfies EnvironmentSpeechStatus;
130
+ const model: EnvironmentSpeechModel = {
131
+ id: "both",
132
+ name: "Bilingual",
133
+ description: "",
134
+ size: 100,
135
+ languages: ["en"],
136
+ accuracy: 1,
137
+ speed: 1,
138
+ recommended: true,
139
+ supportsStreaming: false,
140
+ supportsTranslation: false,
141
+ supportsLanguageDetection: false,
142
+ active: false,
143
+ state: "downloadable",
144
+ benchmark: {
145
+ accuracy: { en: { grade: "A-", errorPercent: 4 } },
146
+ secondsPer30s,
147
+ },
148
+ };
149
+ const cpuBenchmark = `Speed: ${secondsPer30s.cpu === undefined ? "update server" : secondsPer30s.cpu === null ? "not benchmarked" : `${secondsPer30s.cpu} s / 30 s`}`;
150
+ const gpuBenchmark = `Speed: ${secondsPer30s.accelerated === null ? "not benchmarked" : `${secondsPer30s.accelerated} s / 30 s`}`;
151
+ const fetch = globalThis.fetch;
152
+ using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
153
+ const url = new URL(input instanceof Request ? input.url : input, window.location.href);
154
+ if (url.pathname === "/api/voice/status") {
155
+ if (url.origin === remote.httpBaseUrl) {
156
+ remoteRequested.resolve();
157
+ return remoteStatus.promise;
158
+ }
159
+ return Promise.resolve(Response.json(status));
160
+ }
161
+ if (url.pathname === "/api/voice/models")
162
+ return Promise.resolve(Response.json({ models: [model] }));
163
+ return fetch(input, init);
164
+ });
165
+ await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
166
+ () => ({
167
+ environmentId: prepared.environmentId,
168
+ label: prepared.label,
169
+ grantId: "test",
170
+ prepared,
171
+ }),
172
+ async () => {
173
+ const panel = () => (
174
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
175
+ <VoiceSettingsPanel />
176
+ </SettingsScopeProvider>
177
+ );
178
+ await using screen = await render(panel());
179
+ await expect.element(screen.getByText(cpuBenchmark)).toBeVisible();
180
+ prepared = remote;
181
+ await screen.rerender(panel());
182
+ await remoteRequested.promise;
183
+ await expect.element(screen.getByText(cpuBenchmark)).not.toBeInTheDocument();
184
+ remoteStatus.resolve(
185
+ Response.json({
186
+ ...status,
187
+ benchmarkTier: "accelerated",
188
+ gpuDevices: [{ id: "gpu", name: "Remote GPU" }],
189
+ }),
190
+ );
191
+ await expect.element(screen.getByText(gpuBenchmark)).toBeVisible();
192
+ prepared = local;
193
+ await screen.rerender(panel());
194
+ await expect.element(screen.getByText(cpuBenchmark)).toBeVisible();
195
+ await expect.element(screen.getByText(gpuBenchmark)).not.toBeInTheDocument();
196
+ },
197
+ );
198
+ },
199
+);
200
+
201
it("tests the selected model after download and preserves its transcript across filters", async () => {
202
const target = new PrimaryConnectionTarget({
203
environmentId: EnvironmentId.make("voice-filter"),
@@ -105,6 +232,7 @@ it("tests the selected model after download and preserves its transcript across
232
acceleration: "cpu",
233
modelUnloadTimeout: "never",
234
gpuDevices: [],
235
+ benchmarkTier: "cpu",
236
} satisfies EnvironmentSpeechStatus;
237
let installed = false;
238
const fetch = globalThis.fetch;
@@ -136,7 +264,7 @@ it("tests the selected model after download and preserves its transcript across
264
en: { grade: "A-", errorPercent: 4 },
265
fr: { grade: "B+", errorPercent: 5.5 },
266
},
139
- secondsPer30s: 2.1,
267
+ secondsPer30s: { cpu: 5, accelerated: 2.1 },
268
},
269
}
270
: {}),
@@ -269,6 +397,7 @@ it("loads the models of a relay-connected destination", async () => {
397
acceleration: "cpu",
398
modelUnloadTimeout: "never",
399
gpuDevices: [],
400
+ benchmarkTier: "cpu",
401
} satisfies EnvironmentSpeechStatus;
402
const authorizations: Array<string | null> = [];
403
const fetch = globalThis.fetch;
@@ -335,6 +464,7 @@ it("recovers from a failed download progress poll without notifying, but reports
464
acceleration: "cpu",
465
modelUnloadTimeout: "never",
466
gpuDevices: [],
467
+ benchmarkTier: "cpu",
468
};
469
const unavailable = () => new Response(null, { status: 503 });
470
const readModels = vi
apps/web/src/components/settings/VoiceSettingsPanel.tsx
+6
-1
@@ -21,6 +21,7 @@ import {
21
type ProjectId,
22
ProviderDriverKind,
23
type SpeechAcceleration,
24
+ type SpeechBenchmarkTier,
25
type SpeechCustomWords,
26
type SpeechModelUnloadTimeout,
27
type VoiceShortcutMode,
@@ -159,6 +160,7 @@ const spellingsOf = (words: SpeechCustomWords) =>
160
type ModelCardProps = {
161
readonly model: EnvironmentSpeechModel;
162
readonly comparisonLanguages: readonly string[];
163
+ readonly benchmarkTier: SpeechBenchmarkTier;
164
readonly busy: boolean;
165
readonly canCancel: boolean;
166
readonly onDownload: () => void;
@@ -216,6 +218,7 @@ function ModelCard(props: ModelCardProps) {
218
<ModelBenchmarks
219
model={model}
220
languages={props.comparisonLanguages}
221
+ tier={props.benchmarkTier}
222
facts={[
223
...(model.languages.length > 1
224
? [`${model.languages.length} languages`]
@@ -801,6 +804,7 @@ function SpeechModelBrowser(props: {
804
downloadedOnly,
805
sort,
806
accuracyLanguage,
807
+ benchmarkTier: status?.supported ? status.benchmarkTier : "accelerated",
808
});
809
const activeModel = models.find((model) => model.active);
810
const renderedModels =
@@ -823,7 +827,7 @@ function SpeechModelBrowser(props: {
827
<p className="border-b border-border/60 px-3 py-2 text-xs text-muted-foreground sm:px-4">
828
Pick languages to see models that support all of them. Accuracy is word or character
829
error rate: lower is better. Speed is the time to transcribe 30 seconds of audio on
826
- Handy’s benchmark GPU.
830
+ Handy’s benchmark {status.benchmarkTier === "cpu" ? "CPU" : "GPU"}.
831
</p>
832
<div className="@min-[32rem]/voice-models:grid @min-[32rem]/voice-models:grid-cols-[11rem_minmax(0,1fr)]">
833
<div className="flex min-h-0 flex-col border-b border-border/60 bg-muted/10 @min-[32rem]/voice-models:border-r @min-[32rem]/voice-models:border-b-0">
@@ -940,6 +944,7 @@ function SpeechModelBrowser(props: {
944
busy: controlsDisabled,
945
canCancel,
946
comparisonLanguages: selectedLanguages,
947
+ benchmarkTier: status.benchmarkTier,
948
onDownload: () =>
949
void run(
950
"Could not download the model",
apps/web/src/components/settings/speechModelBrowser.test.ts
+5
-3
@@ -33,7 +33,7 @@ const models: readonly EnvironmentSpeechModel[] = [
33
en: { grade: "A", errorPercent: 3.8 },
34
fr: { grade: "B", errorPercent: 8 },
35
},
36
- secondsPer30s: 2,
36
+ secondsPer30s: { cpu: 2, accelerated: 2 },
37
},
38
},
39
{
@@ -47,7 +47,7 @@ const models: readonly EnvironmentSpeechModel[] = [
47
en: { grade: "A", errorPercent: 3.2 },
48
fr: { grade: "B", errorPercent: 9 },
49
},
50
- secondsPer30s: 1,
50
+ secondsPer30s: { cpu: 1, accelerated: 1 },
51
},
52
},
53
{
@@ -58,7 +58,7 @@ const models: readonly EnvironmentSpeechModel[] = [
58
active: false,
59
state: "verifying",
60
languages: ["en"],
61
- benchmark: { accuracy: {}, secondsPer30s: null },
61
+ benchmark: { accuracy: {}, secondsPer30s: { cpu: null, accelerated: null } },
62
},
63
];
64
@@ -78,6 +78,7 @@ it.each([
78
downloadedOnly: false,
79
sort,
80
accuracyLanguage,
81
+ benchmarkTier: "accelerated",
82
}).map((entry) => entry.id),
83
).toEqual(expected);
84
expect(models[0]?.id).toBe("active");
@@ -91,6 +92,7 @@ it("combines text, all selected languages, and completed downloads without retai
92
downloadedOnly: true,
93
sort: "size",
94
accuracyLanguage: "en",
95
+ benchmarkTier: "accelerated",
96
} as const;
97
expect(filterSpeechModels(models, options).map((entry) => entry.id)).toEqual(["fast", "active"]);
98
expect(
apps/web/src/components/settings/speechModelBrowser.ts
+3
-2
@@ -1,4 +1,4 @@
1
-import type { EnvironmentSpeechModel } from "@t3tools/contracts";
1
+import type { EnvironmentSpeechModel, SpeechBenchmarkTier } from "@t3tools/contracts";
2
3
export type SpeechModelSort = "recommended" | "size" | "accuracy" | "speed";
4
@@ -17,6 +17,7 @@ export function filterSpeechModels(
17
readonly downloadedOnly: boolean;
18
readonly sort: SpeechModelSort;
19
readonly accuracyLanguage: string;
20
+ readonly benchmarkTier: SpeechBenchmarkTier;
21
},
22
) {
23
const query = options.search.trim().toLocaleLowerCase();
@@ -27,7 +28,7 @@ export function filterSpeechModels(
28
case "accuracy":
29
return model.benchmark?.accuracy[options.accuracyLanguage]?.errorPercent ?? Infinity;
30
case "speed":
30
- return model.benchmark?.secondsPer30s ?? Infinity;
31
+ return model.benchmark?.secondsPer30s[options.benchmarkTier] ?? Infinity;
32
case "recommended":
33
return modelSortOrder(model);
34
}
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+1
@@ -128,6 +128,7 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
128
acceleration: "auto",
129
modelUnloadTimeout: "min_15",
130
gpuDevices: [],
131
+ benchmarkTier: "cpu",
132
};
133
}),
134
downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
packages/contracts/src/speech.test.ts
new
+97
@@ -0,0 +1,97 @@
1
+import * as Schema from "effect/Schema";
2
+import { expect, it } from "vite-plus/test";
3
+
4
+import { EnvironmentSpeechModel, EnvironmentSpeechStatus } from "./speech.ts";
5
+
6
+const decodeStatus = Schema.decodeUnknownSync(EnvironmentSpeechStatus);
7
+const encodeStatus = Schema.encodeSync(EnvironmentSpeechStatus);
8
+const decodeModel = Schema.decodeUnknownSync(EnvironmentSpeechModel);
9
+const encodeModel = Schema.encodeSync(EnvironmentSpeechModel);
10
+
11
+const legacyStatus = {
12
+ supported: true,
13
+ state: "ready",
14
+ modelId: "test",
15
+ model: "Test",
16
+ size: 100,
17
+ supportsStreaming: false,
18
+ supportsTranslation: false,
19
+ acceleration: "auto",
20
+ modelUnloadTimeout: "never",
21
+ gpuDevices: [],
22
+};
23
+const gpuDevices = [{ id: "gpu", name: "Remote GPU" }];
24
+
25
+it.each([
26
+ { acceleration: "cpu", gpuDevices, tier: "cpu" },
27
+ { acceleration: "auto", gpuDevices: [], tier: "cpu" },
28
+ { acceleration: "auto", gpuDevices, tier: "accelerated" },
29
+ { acceleration: "gpu:gpu", gpuDevices: [], tier: "accelerated" },
30
+])("decodes legacy speech status for $acceleration with $gpuDevices", ({ tier, ...settings }) => {
31
+ const status = decodeStatus({
32
+ ...legacyStatus,
33
+ ...settings,
34
+ });
35
+ expect(status).toMatchObject({ ...settings, benchmarkTier: tier });
36
+ expect(encodeStatus(status)).toMatchObject({ benchmarkTier: tier });
37
+
38
+ const fallback = { ...legacyStatus, ...settings, benchmarkTier: "cpu" };
39
+ expect(decodeStatus(fallback)).toEqual(fallback);
40
+ expect(() =>
41
+ decodeStatus({
42
+ ...legacyStatus,
43
+ benchmarkTier: "unknown",
44
+ }),
45
+ ).toThrow();
46
+});
47
+
48
+it.each([
49
+ { wire: { secondsPer30s: 2.1 }, expected: { accelerated: 2.1 } },
50
+ { wire: { secondsPer30s: null }, expected: { accelerated: null } },
51
+ { wire: { secondsPer30s: 2.1, cpuSecondsPer30s: 5 }, expected: { cpu: 5, accelerated: 2.1 } },
52
+ {
53
+ wire: { secondsPer30s: null, cpuSecondsPer30s: null },
54
+ expected: { cpu: null, accelerated: null },
55
+ },
56
+ {
57
+ wire: { secondsPer30s: { cpu: 5, accelerated: 2.1 } },
58
+ expected: { cpu: 5, accelerated: 2.1 },
59
+ },
60
+ { wire: { secondsPer30s: { accelerated: 2.1 } }, expected: { accelerated: 2.1 } },
61
+])("keeps speech benchmark $wire compatible with old and new clients", ({ wire, expected }) => {
62
+ const model = {
63
+ id: "test",
64
+ name: "Test",
65
+ description: "",
66
+ size: 100,
67
+ languages: ["en"],
68
+ accuracy: 0,
69
+ speed: 0,
70
+ benchmark: { accuracy: { en: { grade: "A-", errorPercent: 4 } }, ...wire },
71
+ recommended: false,
72
+ supportsStreaming: false,
73
+ supportsLanguageDetection: false,
74
+ supportsTranslation: false,
75
+ active: true,
76
+ state: "installed",
77
+ };
78
+ const decoded = decodeModel(model);
79
+ expect(decoded.benchmark).toEqual({
80
+ accuracy: model.benchmark.accuracy,
81
+ secondsPer30s: expected,
82
+ });
83
+ const encoded = encodeModel(decoded);
84
+ const legacyBenchmark = Schema.Struct({ secondsPer30s: Schema.NullOr(Schema.Finite) });
85
+ expect(Schema.decodeUnknownSync(legacyBenchmark)(encoded.benchmark)).toEqual({
86
+ secondsPer30s: expected.accelerated,
87
+ });
88
+ expect(decodeModel(encoded)).toEqual(decoded);
89
+ for (const secondsPer30s of ["2.1", { cpu: 5 }, Number.NaN]) {
90
+ expect(() =>
91
+ decodeModel({
92
+ ...model,
93
+ benchmark: { ...model.benchmark, secondsPer30s },
94
+ }),
95
+ ).toThrow();
96
+ }
97
+});
packages/contracts/src/speech.ts
+89
-21
@@ -1,5 +1,6 @@
1
import * as Effect from "effect/Effect";
2
import * as Schema from "effect/Schema";
3
+import * as SchemaTransformation from "effect/SchemaTransformation";
4
import { TrimmedNonEmptyString } from "./baseSchemas.ts";
5
6
export const CloudSpeechApiKey = TrimmedNonEmptyString.check(
@@ -120,6 +121,9 @@ export type SpeechLanguage = typeof SpeechLanguage.Type;
121
122
export const SpeechGpuDevice = Schema.Struct({ id: Schema.String, name: Schema.String });
123
124
+export const SpeechBenchmarkTier = Schema.Literals(["cpu", "accelerated"]);
125
+export type SpeechBenchmarkTier = typeof SpeechBenchmarkTier.Type;
126
+
127
export const EnvironmentSpeechModelState = Schema.Literals([
128
"downloadable",
129
"downloading",
@@ -128,6 +132,25 @@ export const EnvironmentSpeechModelState = Schema.Literals([
132
]);
133
export type EnvironmentSpeechModelState = typeof EnvironmentSpeechModelState.Type;
134
135
+// A missing CPU field identifies a legacy server; null means a current catalog has no measurement.
136
+const SpeechBenchmarkTimes = Schema.Struct({
137
+ cpu: Schema.optionalKey(Schema.NullOr(Schema.Finite)),
138
+ accelerated: Schema.NullOr(Schema.Finite),
139
+});
140
+const SpeechBenchmarkAccuracy = Schema.Record(
141
+ Schema.String,
142
+ Schema.Struct({ grade: Schema.String, errorPercent: Schema.Finite }),
143
+);
144
+const SpeechBenchmark = Schema.Struct({
145
+ accuracy: SpeechBenchmarkAccuracy,
146
+ secondsPer30s: SpeechBenchmarkTimes,
147
+});
148
+const SpeechBenchmarkWire = Schema.Struct({
149
+ accuracy: SpeechBenchmarkAccuracy,
150
+ secondsPer30s: Schema.Union([SpeechBenchmarkTimes, Schema.NullOr(Schema.Finite)]),
151
+ cpuSecondsPer30s: Schema.optionalKey(Schema.NullOr(Schema.Finite)),
152
+});
153
+
154
export const EnvironmentSpeechModel = Schema.Struct({
155
id: SpeechModelId,
156
name: Schema.String,
@@ -137,13 +160,32 @@ export const EnvironmentSpeechModel = Schema.Struct({
160
accuracy: Schema.Finite,
161
speed: Schema.Finite,
162
benchmark: Schema.optionalKey(
140
- Schema.Struct({
141
- accuracy: Schema.Record(
142
- Schema.String,
143
- Schema.Struct({ grade: Schema.String, errorPercent: Schema.Finite }),
163
+ SpeechBenchmarkWire.pipe(
164
+ Schema.decodeTo(
165
+ SpeechBenchmark,
166
+ SchemaTransformation.transform<
167
+ typeof SpeechBenchmark.Type,
168
+ typeof SpeechBenchmarkWire.Type
169
+ >({
170
+ decode: ({ accuracy, secondsPer30s, cpuSecondsPer30s }) => ({
171
+ accuracy,
172
+ secondsPer30s:
173
+ typeof secondsPer30s === "number" || secondsPer30s === null
174
+ ? {
175
+ accelerated: secondsPer30s,
176
+ ...(cpuSecondsPer30s !== undefined ? { cpu: cpuSecondsPer30s } : {}),
177
+ }
178
+ : secondsPer30s,
179
+ }),
180
+ // Older clients require the scalar; newer clients normalize both wire representations.
181
+ encode: ({ accuracy, secondsPer30s }) => ({
182
+ accuracy,
183
+ secondsPer30s: secondsPer30s.accelerated,
184
+ ...(secondsPer30s.cpu !== undefined ? { cpuSecondsPer30s: secondsPer30s.cpu } : {}),
185
+ }),
186
+ }),
187
),
145
- secondsPer30s: Schema.NullOr(Schema.Finite),
146
- }),
188
+ ),
189
),
190
recommended: Schema.Boolean,
191
supportsStreaming: Schema.Boolean,
@@ -165,26 +207,52 @@ export const EnvironmentSpeechModelRequest = Schema.Struct({ modelId: SpeechMode
207
export const EnvironmentSpeechState = Schema.Literals(["missing-model", "ready", "transcribing"]);
208
export type EnvironmentSpeechState = typeof EnvironmentSpeechState.Type;
209
210
+const SupportedSpeechStatus = Schema.Struct({
211
+ supported: Schema.Literal(true),
212
+ state: EnvironmentSpeechState,
213
+ modelId: SpeechModelId,
214
+ model: Schema.String,
215
+ size: Schema.Finite,
216
+ supportsStreaming: Schema.Boolean,
217
+ supportsBufferedStreaming: Schema.optionalKey(Schema.Boolean),
218
+ supportsTranslation: Schema.Boolean,
219
+ languages: Schema.optionalKey(Schema.Array(Schema.String)),
220
+ supportsLanguageDetection: Schema.optionalKey(Schema.Boolean),
221
+ acceleration: SpeechAcceleration,
222
+ benchmarkTier: SpeechBenchmarkTier,
223
+ modelUnloadTimeout: SpeechModelUnloadTimeout,
224
+ gpuDevices: Schema.Array(SpeechGpuDevice),
225
+});
226
+const SupportedSpeechStatusWire = SupportedSpeechStatus.mapFields((fields) => ({
227
+ ...fields,
228
+ benchmarkTier: Schema.optionalKey(SpeechBenchmarkTier),
229
+}));
230
+
231
export const EnvironmentSpeechStatus = Schema.Union([
232
Schema.Struct({
233
supported: Schema.Literal(false),
234
reason: Schema.String,
235
}),
173
- Schema.Struct({
174
- supported: Schema.Literal(true),
175
- state: EnvironmentSpeechState,
176
- modelId: SpeechModelId,
177
- model: Schema.String,
178
- size: Schema.Finite,
179
- supportsStreaming: Schema.Boolean,
180
- supportsBufferedStreaming: Schema.optionalKey(Schema.Boolean),
181
- supportsTranslation: Schema.Boolean,
182
- languages: Schema.optionalKey(Schema.Array(Schema.String)),
183
- supportsLanguageDetection: Schema.optionalKey(Schema.Boolean),
184
- acceleration: SpeechAcceleration,
185
- modelUnloadTimeout: SpeechModelUnloadTimeout,
186
- gpuDevices: Schema.Array(SpeechGpuDevice),
187
- }),
236
+ SupportedSpeechStatusWire.pipe(
237
+ Schema.decodeTo(
238
+ Schema.toType(SupportedSpeechStatus),
239
+ SchemaTransformation.transform<
240
+ typeof SupportedSpeechStatus.Type,
241
+ typeof SupportedSpeechStatusWire.Type
242
+ >({
243
+ decode: (status) => ({
244
+ ...status,
245
+ benchmarkTier:
246
+ status.benchmarkTier ??
247
+ (status.acceleration === "cpu" ||
248
+ (status.acceleration === "auto" && status.gpuDevices.length === 0)
249
+ ? "cpu"
250
+ : "accelerated"),
251
+ }),
252
+ encode: (status) => status,
253
+ }),
254
+ ),
255
+ ),
256
]);
257
export type EnvironmentSpeechStatus = typeof EnvironmentSpeechStatus.Type;
258