The best way to control Claude Code, Codex and more from any computer or phone
feat(speech): recommend local models by language, speed and accuracy (#33336)
Utkarsh Patil committed
Nov 17, 2026 at 23:22 UTC
bfd37b2b5341645c5da172e6e2acae9b82c946f8
16 files changed
+715
-213
apps/mobile/src/features/settings/VoiceModelSection.tsx
+1
-1
@@ -148,7 +148,7 @@ function ModelStats(props: { readonly model: EnvironmentSpeechModel; readonly la
148
"speed",
149
<>
150
<Text className="font-t3-medium text-foreground">
151
- {data.secondsPer30s.accelerated} s
151
+ {Number(data.secondsPer30s.accelerated.toFixed(2))} s
152
</Text>{" "}
153
/ 30 s
154
</>,
apps/server/scripts/speech-catalog.test.ts
+29
-3
@@ -2,6 +2,8 @@
2
import * as NodeFS from "node:fs";
3
4
import { catalog } from "@handy-computer/models";
5
+import { recommendSpeechModels } from "@t3tools/shared/speech";
6
+import policy from "../../../packages/shared/src/speechRecommendationPolicy.json" with { type: "json" };
7
import { expect, it } from "vite-plus/test";
8
9
import manifest from "../package.json" with { type: "json" };
@@ -15,6 +17,8 @@ const generate = (models = catalog.models) =>
17
);
18
19
it("reproduces the bundled registry byte for byte without upstream requests", async () => {
20
+ const { fastCpuMaxWaitSeconds, ...methodology } = catalog.methodology;
21
+ expect(policy).toEqual({ ...methodology, fastMaxWaitSeconds: fastCpuMaxWaitSeconds });
22
expect(NodeFS.readFileSync(new URL("../src/speech/catalog.json", import.meta.url), "utf8")).toBe(
23
await renderSpeechCatalog(
24
catalog.data,
@@ -53,18 +57,40 @@ it("reproduces the bundled registry byte for byte without upstream requests", as
57
{
58
...catalog.data,
59
models: [parakeet],
56
- results: { [parakeet.id]: { accuracy: {}, performance } },
60
+ results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance } },
61
},
62
"0.2.3",
63
"0.4.1",
64
);
65
expect(result.models[0]?.benchmark.secondsPer30s).toEqual({
62
- cpu: performance.cpu ? 15 : null,
63
- accelerated: performance.accelerated ? 3 : null,
66
+ cpu: performance.cpu ? 30 / performance.cpu : null,
67
+ accelerated: performance.accelerated ? 30 / performance.accelerated : null,
68
});
69
}
70
});
71
72
+it("preserves Handy's accelerated balanced pick after filtering T3's offered candidates", () => {
73
+ const offered = generate().models;
74
+ const sources = catalog.models.filter((model) =>
75
+ offered.some(({ id }) => id === model.repository),
76
+ );
77
+ for (const languages of [["en"], ["de"], ["ja"], ["en", "de"], ["en", "fr"], ["hi", "mr"]]) {
78
+ const expected = catalog
79
+ .recommendModels(sources, languages)
80
+ .find((pick) => pick.status !== "unbenchmarked" && pick.roles.includes("best"));
81
+ expect(
82
+ recommendSpeechModels(offered, languages, "accelerated").keys().next().value,
83
+ languages.join(","),
84
+ ).toBe(expected?.model.repository);
85
+ }
86
+ const excluded = { ...catalog.models[0]!, license: "cc-by-nc-4.0" };
87
+ const allowed = { ...catalog.models[1]!, languages: ["en"] };
88
+ const filtered = generate([excluded, allowed]).models;
89
+ expect([...recommendSpeechModels(filtered, ["en"], "accelerated").keys()]).toEqual([
90
+ allowed.repository,
91
+ ]);
92
+});
93
+
94
it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {
95
const model = catalog.models[0]!;
96
for (const override of [
apps/server/scripts/speech-catalog.ts
+5
-6
@@ -31,7 +31,7 @@ const ASR_FAMILIES = new Set([
31
"medasr",
32
]);
33
34
-// Preserve Handy's onboarding badges; curation alone also includes specialist models.
34
+// Older clients rank and pick setup models by this flag; current clients use language recommendations.
35
const RECOMMENDED = new Set([
36
"parakeet-unified-en-0.6b",
37
"nemotron-3.5-asr-streaming-0.6b",
@@ -81,9 +81,8 @@ export function generateSpeechCatalog(
81
ids.add(model.repository);
82
filenames.add(file.filename);
83
const languages = [...new Set(model.languages.map(canonicalLanguage))];
84
- const performance = data.results[model.id]?.performance;
85
- const secondsPer30s = (realtimeFactor: number | undefined) =>
86
- realtimeFactor ? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2)) : null;
84
+ const realtimeFactor = data.results[model.id]?.performance.accelerated;
85
+ const cpuFactor = data.results[model.id]?.performance.cpu;
86
const accuracy = Object.fromEntries(
87
languages.flatMap((language) => {
88
const cell = data.results[model.id]?.accuracy[languageIdentity(language)];
@@ -112,8 +111,8 @@ export function generateSpeechCatalog(
111
benchmark: {
112
accuracy,
113
secondsPer30s: {
115
- cpu: secondsPer30s(performance?.cpu),
116
- accelerated: secondsPer30s(performance?.accelerated),
114
+ cpu: cpuFactor ? BENCHMARK_DICTATION_SECONDS / cpuFactor : null,
115
+ accelerated: realtimeFactor ? BENCHMARK_DICTATION_SECONDS / realtimeFactor : null,
116
},
117
},
118
recommended: RECOMMENDED.has(model.id),
apps/server/scripts/sync-speech-catalog.ts
+16
-1
@@ -5,6 +5,7 @@ import * as NodeModule from "node:module";
5
import * as NodePath from "node:path";
6
7
import { catalog } from "@handy-computer/models";
8
+import { format } from "vite-plus/fmt";
9
import * as Schema from "effect/Schema";
10
11
import manifest from "../package.json" with { type: "json" };
@@ -24,12 +25,26 @@ const expected = await renderSpeechCatalog(
25
sourceManifest.version,
26
manifest.dependencies["transcribe-cpp"],
27
);
28
+const policyTarget = new URL(
29
+ "../../../packages/shared/src/speechRecommendationPolicy.json",
30
+ import.meta.url,
31
+);
32
+const { fastCpuMaxWaitSeconds, ...methodology } = catalog.methodology;
33
+const policy = await format(
34
+ "speechRecommendationPolicy.json",
35
+ JSON.stringify({ ...methodology, fastMaxWaitSeconds: fastCpuMaxWaitSeconds }),
36
+);
37
+if (policy.errors.length > 0) throw new Error("Could not format speech recommendation policy.");
38
if (process.argv.includes("--check")) {
28
- if (NodeFS.readFileSync(target, "utf8") !== expected) {
39
+ if (
40
+ NodeFS.readFileSync(target, "utf8") !== expected ||
41
+ NodeFS.readFileSync(policyTarget, "utf8") !== policy.code
42
+ ) {
43
throw new Error("Speech catalog differs; run vp run speech:catalog:sync in apps/server.");
44
}
45
console.log("Bundled speech catalog matches the pinned source.");
46
} else {
47
NodeFS.writeFileSync(target, expected);
48
+ NodeFS.writeFileSync(policyTarget, policy.code);
49
console.log(`Generated speech catalog from @handy-computer/models@${sourceManifest.version}.`);
50
}
apps/server/src/speech/catalog.json
+58
-58
@@ -15,7 +15,7 @@
15
"languages": ["en"],
16
"benchmark": {
17
"accuracy": { "en": { "grade": "A-", "errorPercent": 3.99 } },
18
- "secondsPer30s": { "cpu": 2.15, "accelerated": 1.19 }
18
+ "secondsPer30s": { "cpu": 2.149767108563239, "accelerated": 1.1867088607594936 }
19
},
20
"recommended": true,
21
"supportsStreaming": true,
@@ -92,7 +92,7 @@
92
"ro": { "grade": "D", "errorPercent": 28.28 },
93
"et": { "grade": "F", "errorPercent": 31.84 }
94
},
95
- "secondsPer30s": { "cpu": 2.83, "accelerated": 1.75 }
95
+ "secondsPer30s": { "cpu": 2.8328611898016995, "accelerated": 1.7508024511234315 }
96
},
97
"recommended": true,
98
"supportsStreaming": true,
@@ -116,7 +116,7 @@
116
"es": { "grade": "B", "errorPercent": 6.54 },
117
"fr": { "grade": "B-", "errorPercent": 8.53 }
118
},
119
- "secondsPer30s": { "cpu": 1.38, "accelerated": 0.91 }
119
+ "secondsPer30s": { "cpu": 1.3805798435342844, "accelerated": 0.9067553271875473 }
120
},
121
"recommended": true,
122
"supportsStreaming": false,
@@ -165,7 +165,7 @@
165
"vi": { "grade": "B", "errorPercent": 7.39 },
166
"ko": { "grade": "B", "errorPercent": 6.57 }
167
},
168
- "secondsPer30s": { "cpu": 6.96, "accelerated": 3.71 }
168
+ "secondsPer30s": { "cpu": 6.960556844547565, "accelerated": 3.7059913526868433 }
169
},
170
"recommended": true,
171
"supportsStreaming": false,
@@ -367,7 +367,7 @@
367
"yo": { "grade": "F", "errorPercent": 109 },
368
"zh": { "grade": "C", "errorPercent": 13.13 }
369
},
370
- "secondsPer30s": { "cpu": 15.58, "accelerated": 7.34 }
370
+ "secondsPer30s": { "cpu": 15.584415584415584, "accelerated": 7.343941248470013 }
371
},
372
"recommended": true,
373
"supportsStreaming": false,
@@ -400,7 +400,7 @@
400
"hi": { "grade": "C", "errorPercent": 14.44 },
401
"ko": { "grade": "B+", "errorPercent": 5.27 }
402
},
403
- "secondsPer30s": { "cpu": 51.28, "accelerated": 39.47 }
403
+ "secondsPer30s": { "cpu": 51.282051282051285, "accelerated": 39.473684210526315 }
404
},
405
"recommended": false,
406
"supportsStreaming": true,
@@ -471,7 +471,7 @@
471
"el": { "grade": "F", "errorPercent": 32.25 },
472
"mt": { "grade": "C-", "errorPercent": 17.01 }
473
},
474
- "secondsPer30s": { "cpu": 2.26, "accelerated": 1.3 }
474
+ "secondsPer30s": { "cpu": 2.2573363431151243, "accelerated": 1.3043478260869565 }
475
},
476
"recommended": false,
477
"supportsStreaming": false,
@@ -542,7 +542,7 @@
542
"sv": { "grade": "C", "errorPercent": 15.25 },
543
"uk": { "grade": "B", "errorPercent": 6.84 }
544
},
545
- "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
545
+ "secondsPer30s": { "cpu": 2.1969974368363236, "accelerated": 1.196888090963495 }
546
},
547
"recommended": false,
548
"supportsStreaming": false,
@@ -561,7 +561,7 @@
561
"languages": ["en"],
562
"benchmark": {
563
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.11 } },
564
- "secondsPer30s": { "cpu": 2.1, "accelerated": 1.09 }
564
+ "secondsPer30s": { "cpu": 2.102312543798178, "accelerated": 1.0934937124111537 }
565
},
566
"recommended": false,
567
"supportsStreaming": false,
@@ -642,7 +642,7 @@
642
"hu": { "grade": "F", "errorPercent": 56.24 },
643
"mk": { "grade": "F", "errorPercent": 35.09 }
644
},
645
- "secondsPer30s": { "cpu": 5.67, "accelerated": 3.09 }
645
+ "secondsPer30s": { "cpu": 5.665722379603399, "accelerated": 3.091190108191654 }
646
},
647
"recommended": false,
648
"supportsStreaming": false,
@@ -725,7 +725,7 @@
725
"sl": { "grade": "F", "errorPercent": 75.22 },
726
"sv": { "grade": "F", "errorPercent": 75.36 }
727
},
728
- "secondsPer30s": { "cpu": 5.48, "accelerated": 3.4 }
728
+ "secondsPer30s": { "cpu": 5.484460694698355, "accelerated": 3.3955857385398978 }
729
},
730
"recommended": false,
731
"supportsStreaming": false,
@@ -796,7 +796,7 @@
796
"ru": { "grade": "B", "errorPercent": 7.83 },
797
"uk": { "grade": "C+", "errorPercent": 10.58 }
798
},
799
- "secondsPer30s": { "cpu": 3.6, "accelerated": 2.12 }
799
+ "secondsPer30s": { "cpu": 3.603603603603604, "accelerated": 2.1231422505307855 }
800
},
801
"recommended": false,
802
"supportsStreaming": false,
@@ -815,7 +815,7 @@
815
"languages": ["en"],
816
"benchmark": {
817
"accuracy": { "en": { "grade": "A", "errorPercent": 3.58 } },
818
- "secondsPer30s": { "cpu": 11.95, "accelerated": 7.55 }
818
+ "secondsPer30s": { "cpu": 11.952191235059763, "accelerated": 7.547169811320755 }
819
},
820
"recommended": false,
821
"supportsStreaming": false,
@@ -837,7 +837,7 @@
837
"en": { "grade": "A-", "errorPercent": 4.88 },
838
"ar": { "grade": "C+", "errorPercent": 11.06 }
839
},
840
- "secondsPer30s": { "cpu": 6.96, "accelerated": 3.61 }
840
+ "secondsPer30s": { "cpu": 6.960556844547565, "accelerated": 3.6122817579771223 }
841
},
842
"recommended": false,
843
"supportsStreaming": false,
@@ -860,7 +860,7 @@
860
"en": { "grade": "B+", "errorPercent": 5.49 },
861
"ja": { "grade": "B-", "errorPercent": 8.5 }
862
},
863
- "secondsPer30s": { "cpu": 3.78, "accelerated": 2.8 }
863
+ "secondsPer30s": { "cpu": 3.7759597230962867, "accelerated": 2.7958993476234855 }
864
},
865
"recommended": false,
866
"supportsStreaming": false,
@@ -879,7 +879,7 @@
879
"languages": ["ru"],
880
"benchmark": {
881
"accuracy": { "ru": { "grade": "B+", "errorPercent": 5.53 } },
882
- "secondsPer30s": { "cpu": 2.62, "accelerated": 0.85 }
882
+ "secondsPer30s": { "cpu": 2.618943692710607, "accelerated": 0.8480565371024735 }
883
},
884
"recommended": false,
885
"supportsStreaming": false,
@@ -898,7 +898,7 @@
898
"languages": ["ru"],
899
"benchmark": {
900
"accuracy": { "ru": { "grade": "B+", "errorPercent": 5.35 } },
901
- "secondsPer30s": { "cpu": 3.81, "accelerated": 2 }
901
+ "secondsPer30s": { "cpu": 3.8095238095238093, "accelerated": 1.9973368841544608 }
902
},
903
"recommended": false,
904
"supportsStreaming": false,
@@ -917,7 +917,7 @@
917
"languages": ["en"],
918
"benchmark": {
919
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.61 } },
920
- "secondsPer30s": { "cpu": 1.74, "accelerated": 1.31 }
920
+ "secondsPer30s": { "cpu": 1.7401392111368912, "accelerated": 1.312048983162038 }
921
},
922
"recommended": false,
923
"supportsStreaming": false,
@@ -943,7 +943,7 @@
943
"pt": { "grade": "B-", "errorPercent": 9.8 },
944
"ja": { "grade": "B", "errorPercent": 6.3 }
945
},
946
- "secondsPer30s": { "cpu": 10.81, "accelerated": 6.5 }
946
+ "secondsPer30s": { "cpu": 10.81081081081081, "accelerated": 6.500541711809317 }
947
},
948
"recommended": false,
949
"supportsStreaming": false,
@@ -962,7 +962,7 @@
962
"languages": ["en"],
963
"benchmark": {
964
"accuracy": { "en": { "grade": "F", "errorPercent": 37.48 } },
965
- "secondsPer30s": { "cpu": 1.16, "accelerated": 0.42 }
965
+ "secondsPer30s": { "cpu": 1.1598685482312006, "accelerated": 0.4209935447656469 }
966
},
967
"recommended": false,
968
"supportsStreaming": false,
@@ -981,7 +981,7 @@
981
"languages": ["en"],
982
"benchmark": {
983
"accuracy": { "en": { "grade": "B+", "errorPercent": 6.11 } },
984
- "secondsPer30s": { "cpu": 0.6, "accelerated": 0.39 }
984
+ "secondsPer30s": { "cpu": 0.5996402158704777, "accelerated": 0.39238767902687854 }
985
},
986
"recommended": false,
987
"supportsStreaming": false,
@@ -1000,7 +1000,7 @@
1000
"languages": ["en"],
1001
"benchmark": {
1002
"accuracy": { "en": { "grade": "B", "errorPercent": 6.52 } },
1003
- "secondsPer30s": { "cpu": 2.24, "accelerated": 1.14 }
1003
+ "secondsPer30s": { "cpu": 2.236302646291465, "accelerated": 1.1387360030366294 }
1004
},
1005
"recommended": false,
1006
"supportsStreaming": true,
@@ -1081,7 +1081,7 @@
1081
"hu": { "grade": "F", "errorPercent": 32.84 },
1082
"mk": { "grade": "C-", "errorPercent": 18.22 }
1083
},
1084
- "secondsPer30s": { "cpu": 12.15, "accelerated": 7.34 }
1084
+ "secondsPer30s": { "cpu": 12.14574898785425, "accelerated": 7.343941248470013 }
1085
},
1086
"recommended": false,
1087
"supportsStreaming": false,
@@ -1106,7 +1106,7 @@
1106
"ja": { "grade": "B", "errorPercent": 7.63 },
1107
"ko": { "grade": "B-", "errorPercent": 8.27 }
1108
},
1109
- "secondsPer30s": { "cpu": 1.52, "accelerated": 1.08 }
1109
+ "secondsPer30s": { "cpu": 1.5201418799087916, "accelerated": 1.0834236186348862 }
1110
},
1111
"recommended": false,
1112
"supportsStreaming": false,
@@ -1134,7 +1134,7 @@
1134
"nl": { "grade": "B", "errorPercent": 6.57 },
1135
"hi": { "grade": "B-", "errorPercent": 8.93 }
1136
},
1137
- "secondsPer30s": { "cpu": 53.57, "accelerated": 27.65 }
1137
+ "secondsPer30s": { "cpu": 53.57142857142857, "accelerated": 27.649769585253456 }
1138
},
1139
"recommended": false,
1140
"supportsStreaming": false,
@@ -1336,7 +1336,7 @@
1336
"yo": { "grade": "F", "errorPercent": 118.5 },
1337
"zh": { "grade": "D", "errorPercent": 23.06 }
1338
},
1339
- "secondsPer30s": { "cpu": 5.12, "accelerated": 2.57 }
1339
+ "secondsPer30s": { "cpu": 5.1194539249146755, "accelerated": 2.5651988029072252 }
1340
},
1341
"recommended": false,
1342
"supportsStreaming": false,
@@ -1540,7 +1540,7 @@
1540
"yue": { "grade": "F", "errorPercent": 34.62 },
1541
"zh": { "grade": "B-", "errorPercent": 8.5 }
1542
},
1543
- "secondsPer30s": { "cpu": 20.98, "accelerated": 9.69 }
1543
+ "secondsPer30s": { "cpu": 20.97902097902098, "accelerated": 9.693053311793214 }
1544
},
1545
"recommended": false,
1546
"supportsStreaming": false,
@@ -1562,7 +1562,7 @@
1562
"zh": { "grade": "B-", "errorPercent": 8.1 },
1563
"en": { "grade": "A-", "errorPercent": 4.14 }
1564
},
1565
- "secondsPer30s": { "cpu": 31.25, "accelerated": 14.85 }
1565
+ "secondsPer30s": { "cpu": 31.25, "accelerated": 14.851485148514852 }
1566
},
1567
"recommended": false,
1568
"supportsStreaming": false,
@@ -1766,7 +1766,7 @@
1766
"yue": { "grade": "D", "errorPercent": 22.06 },
1767
"zh": { "grade": "B-", "errorPercent": 7.98 }
1768
},
1769
- "secondsPer30s": { "cpu": 30.93, "accelerated": 15.04 }
1769
+ "secondsPer30s": { "cpu": 30.927835051546392, "accelerated": 15.037593984962406 }
1770
},
1771
"recommended": false,
1772
"supportsStreaming": false,
@@ -1790,7 +1790,7 @@
1790
"es": { "grade": "B", "errorPercent": 6.73 },
1791
"fr": { "grade": "B", "errorPercent": 7.22 }
1792
},
1793
- "secondsPer30s": { "cpu": 3.39, "accelerated": 1.94 }
1793
+ "secondsPer30s": { "cpu": 3.3898305084745766, "accelerated": 1.937984496124031 }
1794
},
1795
"recommended": false,
1796
"supportsStreaming": false,
@@ -1809,7 +1809,7 @@
1809
"languages": ["ru"],
1810
"benchmark": {
1811
"accuracy": { "ru": { "grade": "B-", "errorPercent": 8.42 } },
1812
- "secondsPer30s": { "cpu": 2.54, "accelerated": 0.75 }
1812
+ "secondsPer30s": { "cpu": 2.5369978858350954, "accelerated": 0.7543374402816192 }
1813
},
1814
"recommended": false,
1815
"supportsStreaming": false,
@@ -1828,7 +1828,7 @@
1828
"languages": ["ru"],
1829
"benchmark": {
1830
"accuracy": { "ru": { "grade": "B-", "errorPercent": 8.07 } },
1831
- "secondsPer30s": { "cpu": 2.9, "accelerated": 1.12 }
1831
+ "secondsPer30s": { "cpu": 2.9013539651837523, "accelerated": 1.1219147344801796 }
1832
},
1833
"recommended": false,
1834
"supportsStreaming": false,
@@ -1854,7 +1854,7 @@
1854
"pt": { "grade": "B-", "errorPercent": 9.6 },
1855
"ja": { "grade": "B", "errorPercent": 6.44 }
1856
},
1857
- "secondsPer30s": { "cpu": 10.89, "accelerated": 6.47 }
1857
+ "secondsPer30s": { "cpu": 10.88929219600726, "accelerated": 6.472491909385114 }
1858
},
1859
"recommended": false,
1860
"supportsStreaming": false,
@@ -1879,7 +1879,7 @@
1879
"es": { "grade": "A-", "errorPercent": 4.08 },
1880
"pt": { "grade": "B+", "errorPercent": 5.57 }
1881
},
1882
- "secondsPer30s": { "cpu": 13.99, "accelerated": 7.51 }
1882
+ "secondsPer30s": { "cpu": 13.986013986013987, "accelerated": 7.509386733416771 }
1883
},
1884
"recommended": false,
1885
"supportsStreaming": false,
@@ -1904,7 +1904,7 @@
1904
"es": { "grade": "B", "errorPercent": 6.53 },
1905
"pt": { "grade": "C+", "errorPercent": 10.61 }
1906
},
1907
- "secondsPer30s": { "cpu": 11.54, "accelerated": 6.46 }
1907
+ "secondsPer30s": { "cpu": 11.538461538461538, "accelerated": 6.458557588805167 }
1908
},
1909
"recommended": false,
1910
"supportsStreaming": false,
@@ -1923,7 +1923,7 @@
1923
"languages": ["en"],
1924
"benchmark": {
1925
"accuracy": { "en": { "grade": "C+", "errorPercent": 12.25 } },
1926
- "secondsPer30s": { "cpu": 1.28, "accelerated": 0.84 }
1926
+ "secondsPer30s": { "cpu": 1.277139208173691, "accelerated": 0.8429334082607474 }
1927
},
1928
"recommended": false,
1929
"supportsStreaming": false,
@@ -1942,7 +1942,7 @@
1942
"languages": ["en"],
1943
"benchmark": {
1944
"accuracy": { "en": { "grade": "B", "errorPercent": 7.87 } },
1945
- "secondsPer30s": { "cpu": 3.1, "accelerated": 2.1 }
1945
+ "secondsPer30s": { "cpu": 3.103983445421625, "accelerated": 2.100840336134454 }
1946
},
1947
"recommended": false,
1948
"supportsStreaming": true,
@@ -1961,7 +1961,7 @@
1961
"languages": ["en"],
1962
"benchmark": {
1963
"accuracy": { "en": { "grade": "B-", "errorPercent": 8.55 } },
1964
- "secondsPer30s": { "cpu": 2.22, "accelerated": 1.38 }
1964
+ "secondsPer30s": { "cpu": 2.2164758034724787, "accelerated": 1.3780431786862655 }
1965
},
1966
"recommended": false,
1967
"supportsStreaming": true,
@@ -1980,7 +1980,7 @@
1980
"languages": ["en"],
1981
"benchmark": {
1982
"accuracy": { "en": { "grade": "C-", "errorPercent": 18.18 } },
1983
- "secondsPer30s": { "cpu": 0.53, "accelerated": 0.51 }
1983
+ "secondsPer30s": { "cpu": 0.5331437711036076, "accelerated": 0.5071851225697379 }
1984
},
1985
"recommended": false,
1986
"supportsStreaming": true,
@@ -1999,7 +1999,7 @@
1999
"languages": ["en"],
2000
"benchmark": {
2001
"accuracy": { "en": { "grade": "C", "errorPercent": 14.13 } },
2002
- "secondsPer30s": { "cpu": 0.73, "accelerated": 0.5 }
2002
+ "secondsPer30s": { "cpu": 0.7260406582768635, "accelerated": 0.4964421644878372 }
2003
},
2004
"recommended": false,
2005
"supportsStreaming": false,
@@ -2021,7 +2021,7 @@
2021
"en": { "grade": "B+", "errorPercent": 5.13 },
2022
"zh": { "grade": "B-", "errorPercent": 9.23 }
2023
},
2024
- "secondsPer30s": { "cpu": 19.11, "accelerated": 9.79 }
2024
+ "secondsPer30s": { "cpu": 19.108280254777068, "accelerated": 9.787928221859707 }
2025
},
2026
"recommended": false,
2027
"supportsStreaming": false,
@@ -2040,7 +2040,7 @@
2040
"languages": ["en"],
2041
"benchmark": {
2042
"accuracy": { "en": { "grade": "B", "errorPercent": 6.43 } },
2043
- "secondsPer30s": { "cpu": 2.21, "accelerated": 1.11 }
2043
+ "secondsPer30s": { "cpu": 2.210759027266028, "accelerated": 1.1051759071652238 }
2044
},
2045
"recommended": false,
2046
"supportsStreaming": true,
@@ -2111,7 +2111,7 @@
2111
"sv": { "grade": "C", "errorPercent": 13.26 },
2112
"uk": { "grade": "B", "errorPercent": 6.53 }
2113
},
2114
- "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
2114
+ "secondsPer30s": { "cpu": 2.2026431718061676, "accelerated": 1.1954572624028692 }
2115
},
2116
"recommended": false,
2117
"supportsStreaming": false,
@@ -2130,7 +2130,7 @@
2130
"languages": ["en"],
2131
"benchmark": {
2132
"accuracy": { "en": { "grade": "B+", "errorPercent": 5.53 } },
2133
- "secondsPer30s": { "cpu": 1.98, "accelerated": 1.05 }
2133
+ "secondsPer30s": { "cpu": 1.9788918205804749, "accelerated": 1.048951048951049 }
2134
},
2135
"recommended": false,
2136
"supportsStreaming": false,
@@ -2149,7 +2149,7 @@
2149
"languages": ["en"],
2150
"benchmark": {
2151
"accuracy": { "en": { "grade": "B+", "errorPercent": 5.61 } },
2152
- "secondsPer30s": { "cpu": 3.36, "accelerated": 1.79 }
2152
+ "secondsPer30s": { "cpu": 3.3613445378151257, "accelerated": 1.7873100983020553 }
2153
},
2154
"recommended": false,
2155
"supportsStreaming": false,
@@ -2220,7 +2220,7 @@
2220
"sv": { "grade": "C-", "errorPercent": 16.42 },
2221
"uk": { "grade": "B-", "errorPercent": 8.11 }
2222
},
2223
- "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
2223
+ "secondsPer30s": { "cpu": 2.2002200220022003, "accelerated": 1.2014417300760913 }
2224
},
2225
"recommended": false,
2226
"supportsStreaming": false,
@@ -2239,7 +2239,7 @@
2239
"languages": ["en"],
2240
"benchmark": {
2241
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.57 } },
2242
- "secondsPer30s": { "cpu": 2.02, "accelerated": 1.11 }
2242
+ "secondsPer30s": { "cpu": 2.0208824520040416, "accelerated": 1.107011070110701 }
2243
},
2244
"recommended": false,
2245
"supportsStreaming": false,
@@ -2258,7 +2258,7 @@
2258
"languages": ["en"],
2259
"benchmark": {
2260
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.45 } },
2261
- "secondsPer30s": { "cpu": 3.49, "accelerated": 1.87 }
2261
+ "secondsPer30s": { "cpu": 3.492433061699651, "accelerated": 1.8650917003419334 }
2262
},
2263
"recommended": false,
2264
"supportsStreaming": false,
@@ -2277,7 +2277,7 @@
2277
"languages": ["en"],
2278
"benchmark": {
2279
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.68 } },
2280
- "secondsPer30s": { "cpu": 3.22, "accelerated": 1.76 }
2280
+ "secondsPer30s": { "cpu": 3.2206119162640903, "accelerated": 1.7605633802816902 }
2281
},
2282
"recommended": false,
2283
"supportsStreaming": false,
@@ -2296,7 +2296,7 @@
2296
"languages": ["en"],
2297
"benchmark": {
2298
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.24 } },
2299
- "secondsPer30s": { "cpu": 3.41, "accelerated": 1.82 }
2299
+ "secondsPer30s": { "cpu": 3.4071550255536627, "accelerated": 1.8242626938279112 }
2300
},
2301
"recommended": false,
2302
"supportsStreaming": false,
@@ -2526,7 +2526,7 @@
2526
"yo": { "grade": "F", "errorPercent": 103.28 },
2527
"zh": { "grade": "F", "errorPercent": 36.21 }
2528
},
2529
- "secondsPer30s": { "cpu": 1.55, "accelerated": 1.03 }
2529
+ "secondsPer30s": { "cpu": 1.5519917227108124, "accelerated": 1.033947957952783 }
2530
},
2531
"recommended": false,
2532
"supportsStreaming": false,
@@ -2545,7 +2545,7 @@
2545
"languages": ["en"],
2546
"benchmark": {
2547
"accuracy": { "en": { "grade": "B", "errorPercent": 7.6 } },
2548
- "secondsPer30s": { "cpu": 1.46, "accelerated": 0.94 }
2548
+ "secondsPer30s": { "cpu": 1.4563106796116503, "accelerated": 0.9370607527721381 }
2549
},
2550
"recommended": false,
2551
"supportsStreaming": false,
@@ -2747,7 +2747,7 @@
2747
"yo": { "grade": "F", "errorPercent": 113.72 },
2748
"zh": { "grade": "C-", "errorPercent": 19.32 }
2749
},
2750
- "secondsPer30s": { "cpu": 31.25, "accelerated": 14.96 }
2750
+ "secondsPer30s": { "cpu": 31.25, "accelerated": 14.962593516209477 }
2751
},
2752
"recommended": false,
2753
"supportsStreaming": false,
@@ -2949,7 +2949,7 @@
2949
"yo": { "grade": "F", "errorPercent": 96 },
2950
"zh": { "grade": "C", "errorPercent": 15.39 }
2951
},
2952
- "secondsPer30s": { "cpu": 31.25, "accelerated": 14.93 }
2952
+ "secondsPer30s": { "cpu": 31.25, "accelerated": 14.92537313432836 }
2953
},
2954
"recommended": false,
2955
"supportsStreaming": false,
@@ -2968,7 +2968,7 @@
2968
"languages": ["en"],
2969
"benchmark": {
2970
"accuracy": { "en": { "grade": "A-", "errorPercent": 4.88 } },
2971
- "secondsPer30s": { "cpu": 14.53, "accelerated": 6.81 }
2971
+ "secondsPer30s": { "cpu": 14.527845036319613, "accelerated": 6.81044267877412 }
2972
},
2973
"recommended": false,
2974
"supportsStreaming": false,
@@ -2987,7 +2987,7 @@
2987
"languages": ["en"],
2988
"benchmark": {
2989
"accuracy": { "en": { "grade": "B+", "errorPercent": 6.14 } },
2990
- "secondsPer30s": { "cpu": 4.79, "accelerated": 2.4 }
2990
+ "secondsPer30s": { "cpu": 4.7923322683706076, "accelerated": 2.404809619238477 }
2991
},
2992
"recommended": false,
2993
"supportsStreaming": false,
@@ -3189,7 +3189,7 @@
3189
"yo": { "grade": "F", "errorPercent": 102.29 },
3190
"zh": { "grade": "F", "errorPercent": 41.91 }
3191
},
3192
- "secondsPer30s": { "cpu": 0.75, "accelerated": 0.7 }
3192
+ "secondsPer30s": { "cpu": 0.7546220601182241, "accelerated": 0.696217219772569 }
3193
},
3194
"recommended": false,
3195
"supportsStreaming": false,
@@ -3208,7 +3208,7 @@
3208
"languages": ["en"],
3209
"benchmark": {
3210
"accuracy": { "en": { "grade": "C+", "errorPercent": 10.72 } },
3211
- "secondsPer30s": { "cpu": 0.73, "accelerated": 0.63 }
3211
+ "secondsPer30s": { "cpu": 0.727802037845706, "accelerated": 0.6318449873631002 }
3212
},
3213
"recommended": false,
3214
"supportsStreaming": false,
apps/web/src/components/settings/SpeechModelComparison.tsx
+19
-14
@@ -51,12 +51,12 @@ export function ModelBenchmarks({
51
}: {
52
model: EnvironmentSpeechModel;
53
languages: readonly string[];
54
- tier: SpeechBenchmarkTier;
54
+ tier: SpeechBenchmarkTier | undefined;
55
/** Plain facts, such as language count and size, shown after the scores. */
56
facts: readonly string[];
57
}) {
58
const data = model.benchmark;
59
- const seconds = data?.secondsPer30s[tier];
59
+ const seconds = tier === undefined ? undefined : data?.secondsPer30s[tier];
60
const visible = !data
61
? []
62
: languages.length
@@ -82,18 +82,23 @@ export function ModelBenchmarks({
82
</span>
83
);
84
})}
85
- <span>
86
- Speed:{" "}
87
- {tier === "cpu" && data && data.secondsPer30s.cpu === undefined ? (
88
- "update server"
89
- ) : seconds == null ? (
90
- "not benchmarked"
91
- ) : (
92
- <>
93
- <strong className="font-medium text-foreground">{seconds} s</strong> / 30 s
94
- </>
95
- )}
96
- </span>
85
+ {data ? (
86
+ <span>
87
+ Speed:{" "}
88
+ {seconds === undefined ? (
89
+ "update server for this benchmark"
90
+ ) : seconds === null ? (
91
+ "not benchmarked"
92
+ ) : (
93
+ <>
94
+ <strong className="font-medium text-foreground">
95
+ {Number(seconds.toFixed(2))} s
96
+ </strong>{" "}
97
+ / 30 s
98
+ </>
99
+ )}
100
+ </span>
101
+ ) : null}
102
{facts.map((fact) => (
103
<span key={fact}>{fact}</span>
104
))}
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+125
-78
@@ -148,7 +148,7 @@ it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accel
148
secondsPer30s,
149
},
150
};
151
- const cpuBenchmark = `Speed: ${secondsPer30s.cpu === undefined ? "update server" : secondsPer30s.cpu === null ? "not benchmarked" : `${secondsPer30s.cpu} s / 30 s`}`;
151
+ const cpuBenchmark = `Speed: ${secondsPer30s.cpu === undefined ? "update server for this benchmark" : secondsPer30s.cpu === null ? "not benchmarked" : `${secondsPer30s.cpu} s / 30 s`}`;
152
const gpuBenchmark = `Speed: ${secondsPer30s.accelerated === null ? "not benchmarked" : `${secondsPer30s.accelerated} s / 30 s`}`;
153
const fetch = globalThis.fetch;
154
using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
@@ -256,9 +256,9 @@ it("tests the selected model after download and preserves its transcript across
256
supportsStreaming: false,
257
supportsTranslation: false,
258
acceleration: "cpu",
259
+ benchmarkTier: "cpu",
260
modelUnloadTimeout: "never",
261
gpuDevices: [],
261
- benchmarkTier: "cpu",
262
} satisfies EnvironmentSpeechStatus;
263
let installed = false;
264
const fetch = globalThis.fetch;
@@ -290,7 +290,7 @@ it("tests the selected model after download and preserves its transcript across
290
en: { grade: "A-", errorPercent: 4 },
291
fr: { grade: "B+", errorPercent: 5.5 },
292
},
293
- secondsPer30s: { cpu: 5, accelerated: 2.1 },
293
+ secondsPer30s: { accelerated: 2.1, cpu: 5 },
294
},
295
}
296
: {}),
@@ -387,81 +387,128 @@ it("tests the selected model after download and preserves its transcript across
387
);
388
});
389
390
-it("loads the models of a relay-connected destination", async () => {
391
- const target = new RelayConnectionTarget({
392
- environmentId: EnvironmentId.make("voice-relay"),
393
- label: "MacBook Pro",
394
- });
395
- const prepared = {
396
- environmentId: target.environmentId,
397
- label: target.label,
398
- httpBaseUrl: window.location.origin,
399
- socketUrl: "ws://localhost",
400
- httpAuthorization: {
401
- _tag: "Dpop" as const,
402
- accessToken: "stale-access-token",
403
- expiresAtEpochMs: 4_102_444_800_000,
404
- },
405
- target,
406
- };
407
- const model: EnvironmentSpeechModel = {
408
- id: "tiny",
409
- name: "Tiny",
410
- description: "",
411
- size: 100,
412
- languages: ["en"],
413
- accuracy: 1,
414
- speed: 1,
415
- recommended: false,
416
- supportsStreaming: false,
417
- supportsTranslation: false,
418
- supportsLanguageDetection: false,
419
- active: false,
420
- state: "downloadable",
421
- };
422
- const status = {
423
- supported: true,
424
- state: "missing-model",
425
- modelId: model.id,
426
- model: model.name,
427
- size: model.size,
428
- supportsStreaming: false,
429
- supportsTranslation: false,
430
- acceleration: "cpu",
431
- modelUnloadTimeout: "never",
432
- gpuDevices: [],
433
- benchmarkTier: "cpu",
434
- } satisfies EnvironmentSpeechStatus;
435
- const authorizations: Array<string | null> = [];
436
- const fetch = globalThis.fetch;
437
- using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
438
- const url = new URL(input instanceof Request ? input.url : input, window.location.href);
439
- if (url.pathname === "/api/voice/status") {
440
- authorizations.push(new Headers(init?.headers).get("authorization"));
441
- return Promise.resolve(Response.json(status));
390
+it.each([
391
+ { tier: "cpu", legacy: false },
392
+ { tier: "accelerated", legacy: false },
393
+ { tier: "cpu", legacy: true },
394
+] as const)(
395
+ "loads relay models using the server tier $tier (legacy: $legacy)",
396
+ async ({ tier, legacy }) => {
397
+ const target = new RelayConnectionTarget({
398
+ environmentId: EnvironmentId.make("voice-relay"),
399
+ label: "MacBook Pro",
400
+ });
401
+ const prepared = {
402
+ environmentId: target.environmentId,
403
+ label: target.label,
404
+ httpBaseUrl: window.location.origin,
405
+ socketUrl: "ws://localhost",
406
+ httpAuthorization: {
407
+ _tag: "Dpop" as const,
408
+ accessToken: "stale-access-token",
409
+ expiresAtEpochMs: 4_102_444_800_000,
410
+ },
411
+ target,
412
+ };
413
+ const model = {
414
+ id: "gpu",
415
+ name: "GPU pick",
416
+ description: "",
417
+ size: 100,
418
+ languages: ["en"],
419
+ benchmark: {
420
+ accuracy: { en: { grade: "A", errorPercent: 4 } },
421
+ secondsPer30s: { cpu: 20, accelerated: 1 },
422
+ },
423
+ accuracy: 1,
424
+ speed: 1,
425
+ recommended: false,
426
+ supportsStreaming: false,
427
+ supportsTranslation: false,
428
+ supportsLanguageDetection: false,
429
+ active: false,
430
+ state: "downloadable",
431
+ } satisfies EnvironmentSpeechModel;
432
+ const status = {
433
+ supported: true,
434
+ state: "missing-model",
435
+ modelId: model.id,
436
+ model: model.name,
437
+ size: model.size,
438
+ supportsStreaming: false,
439
+ supportsTranslation: false,
440
+ acceleration: "auto",
441
+ benchmarkTier: tier,
442
+ modelUnloadTimeout: "never",
443
+ gpuDevices: [{ id: "remote-gpu", name: "Remote GPU" }],
444
+ } satisfies EnvironmentSpeechStatus;
445
+ const cpuModel = {
446
+ ...model,
447
+ id: "cpu",
448
+ name: "CPU pick",
449
+ benchmark: {
450
+ accuracy: { en: { grade: "B+", errorPercent: 6 } },
451
+ secondsPer30s: { cpu: 1, accelerated: 8 },
452
+ },
453
+ };
454
+ const { benchmarkTier: _tier, ...legacyStatus } = status;
455
+ const models = [model, cpuModel];
456
+ const authorizations: Array<string | null> = [];
457
+ const fetch = globalThis.fetch;
458
+ using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
459
+ const url = new URL(input instanceof Request ? input.url : input, window.location.href);
460
+ if (url.pathname === "/api/voice/status") {
461
+ authorizations.push(new Headers(init?.headers).get("authorization"));
462
+ return Promise.resolve(
463
+ Response.json(legacy ? { ...legacyStatus, acceleration: "cpu", gpuDevices: [] } : status),
464
+ );
465
+ }
466
+ if (url.pathname === "/api/voice/models")
467
+ return Promise.resolve(
468
+ Response.json({
469
+ models: legacy
470
+ ? models.map(({ benchmark, ...entry }) => ({
471
+ ...entry,
472
+ benchmark: {
473
+ accuracy: benchmark.accuracy,
474
+ secondsPer30s: benchmark.secondsPer30s.accelerated,
475
+ },
476
+ }))
477
+ : models,
478
+ }),
479
+ );
480
+ return fetch(input, init);
481
+ });
482
+ using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
483
+ // Restored even when an assertion fails, unlike `withImplementation`.
484
+ using _destination = vi.mocked(useVoiceTranscriptionDestination).mockImplementation(() => ({
485
+ environmentId: target.environmentId,
486
+ label: target.label,
487
+ grantId: "test",
488
+ prepared,
489
+ }));
490
+ await using screen = await render(
491
+ <SettingsScopeProvider search={{}} onChange={() => {}}>
492
+ <VoiceSettingsPanel />
493
+ </SettingsScopeProvider>,
494
+ );
495
+ await vi.waitFor(() => expect(authorizations.length + notify.mock.calls.length).not.toBe(0));
496
+ expect(notify).not.toHaveBeenCalled();
497
+ expect(authorizations).toEqual(["DPoP relay-access-token"]);
498
+ const region = screen.getByRole("region", { name: "Transcription models" });
499
+ await vi.waitFor(() => expect(region.element().textContent).toContain("CPU pick"));
500
+ if (legacy) {
501
+ expect(region.element().textContent).toContain("update server for this benchmark");
502
+ expect(region.element().textContent).not.toContain("Recommended");
503
+ } else {
504
+ const winner = tier === "cpu" ? "CPU pick" : "GPU pick";
505
+ await vi.waitFor(() =>
506
+ expect(region.element().firstElementChild?.textContent).toContain(winner + "Recommended"),
507
+ );
508
+ expect(region.element().textContent).toContain("Speed: 1 s / 30 s");
509
}
443
- if (url.pathname === "/api/voice/models")
444
- return Promise.resolve(Response.json({ models: [model] }));
445
- return fetch(input, init);
446
- });
447
- using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
448
- // Restored even when an assertion fails, unlike `withImplementation`.
449
- using _destination = vi.mocked(useVoiceTranscriptionDestination).mockImplementation(() => ({
450
- environmentId: target.environmentId,
451
- label: target.label,
452
- grantId: "test",
453
- prepared,
454
- }));
455
- await using screen = await render(
456
- <SettingsScopeProvider search={{}} onChange={() => {}}>
457
- <VoiceSettingsPanel />
458
- </SettingsScopeProvider>,
459
- );
460
- await vi.waitFor(() => expect(authorizations.length + notify.mock.calls.length).not.toBe(0));
461
- expect(notify).not.toHaveBeenCalled();
462
- expect(authorizations).toEqual(["DPoP relay-access-token"]);
463
- await expect.element(screen.getByText("Tiny", { exact: true })).toBeVisible();
464
-});
510
+ },
511
+);
512
513
it("recovers from a failed download progress poll without notifying, but reports a failed initial load", async () => {
514
const target = new PrimaryConnectionTarget({
@@ -495,9 +542,9 @@ it("recovers from a failed download progress poll without notifying, but reports
542
supportsStreaming: false,
543
supportsTranslation: false,
544
acceleration: "cpu",
545
+ benchmarkTier: "cpu",
546
modelUnloadTimeout: "never",
547
gpuDevices: [],
500
- benchmarkTier: "cpu",
548
};
549
const unavailable = () => new Response(null, { status: 503 });
550
const readModels = vi
apps/web/src/components/settings/VoiceSettingsPanel.tsx
+38
-14
@@ -30,7 +30,11 @@ import {
30
} from "@t3tools/contracts";
31
import { createModelSelection, pickModelSelection } from "@t3tools/shared/model";
32
import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings";
33
-import { cloudSpeechLanguages } from "@t3tools/shared/speech";
33
+import {
34
+ cloudSpeechLanguages,
35
+ recommendSpeechModels,
36
+ type SpeechRecommendationRole,
37
+} from "@t3tools/shared/speech";
38
import {
39
ArrowDownUpIcon,
40
AudioLinesIcon,
@@ -176,7 +180,8 @@ const spellingsOf = (words: SpeechCustomWords) =>
180
type ModelCardProps = {
181
readonly model: EnvironmentSpeechModel;
182
readonly comparisonLanguages: readonly string[];
179
- readonly benchmarkTier: SpeechBenchmarkTier;
183
+ readonly recommendation: SpeechRecommendationRole | undefined;
184
+ readonly benchmarkTier: SpeechBenchmarkTier | undefined;
185
readonly busy: boolean;
186
readonly canCancel: boolean;
187
readonly onDownload: () => void;
@@ -217,11 +222,23 @@ function ModelCard(props: ModelCardProps) {
222
<div className="min-w-0 flex-1">
223
<div className="flex flex-wrap items-center gap-1.5">
224
<span className="text-sm font-medium leading-5">{model.name}</span>
220
- {model.recommended ? (
225
+ {props.recommendation ? (
226
<ModelBadge
227
icon={<StarIcon className="mr-1 size-3" />}
223
- label="Recommended"
224
- tooltip="A good default pick for the languages it covers."
228
+ label={
229
+ props.recommendation === "recommended"
230
+ ? "Recommended"
231
+ : props.recommendation === "faster"
232
+ ? "Faster"
233
+ : "More accurate"
234
+ }
235
+ tooltip={
236
+ props.recommendation === "recommended"
237
+ ? "Balanced accuracy and speed for your selected languages on Handy’s reference benchmarks."
238
+ : props.recommendation === "faster"
239
+ ? "Quicker than Recommended on this reference tier, with useful accuracy."
240
+ : "Higher accuracy with a longer reference processing budget for your selected languages."
241
+ }
242
/>
243
) : null}
244
{model.supportsStreaming ? (
@@ -871,14 +888,20 @@ function SpeechModelBrowser(props: {
888
selectedLanguages.length === 0
889
? "All languages"
890
: selectedLanguages.map(languageLabel).join(", ");
874
- const sortedModels = filterSpeechModels(models, {
875
- search,
876
- languages: selectedLanguages,
877
- downloadedOnly,
878
- sort,
879
- accuracyLanguage,
880
- benchmarkTier: status?.supported ? status.benchmarkTier : "accelerated",
881
- });
891
+ const benchmarkTier = status?.supported ? status.benchmarkTier : undefined;
892
+ const recommendations = recommendSpeechModels(models, selectedLanguages, benchmarkTier);
893
+ const sortedModels = filterSpeechModels(
894
+ models,
895
+ {
896
+ search,
897
+ languages: selectedLanguages,
898
+ downloadedOnly,
899
+ sort,
900
+ accuracyLanguage,
901
+ benchmarkTier,
902
+ },
903
+ recommendations,
904
+ );
905
const activeModel = models.find((model) => model.active);
906
const renderedModels =
907
activeModel && !sortedModels.includes(activeModel)
@@ -898,7 +921,7 @@ function SpeechModelBrowser(props: {
921
<SettingsRow
922
{...searchableSetting("speech-models")}
923
title="Models"
901
- description={`Grades are error rates and speed is seconds per 30 s of audio, from Handy’s benchmark ${status.benchmarkTier === "cpu" ? "CPU" : "GPU"}.`}
924
+ description={`Grades and seconds per 30 s use Handy’s reference ${status.benchmarkTier === "cpu" ? "CPU" : "GPU"}, not your hardware. Select languages for balanced recommendations.`}
925
className="@container/voice-models"
926
>
927
<div className="mt-3 flex min-w-0 gap-2">
@@ -1018,6 +1041,7 @@ function SpeechModelBrowser(props: {
1041
busy: controlsDisabled,
1042
canCancel,
1043
comparisonLanguages: selectedLanguages,
1044
+ recommendation: recommendations.get(model.id),
1045
benchmarkTier: status.benchmarkTier,
1046
onDownload: () =>
1047
void run(
apps/web/src/components/settings/speechModelBrowser.test.ts
+60
-4
@@ -1,5 +1,6 @@
1
import type { EnvironmentSpeechModel } from "@t3tools/contracts";
2
import { expect, it } from "vite-plus/test";
3
+import { recommendSpeechModels } from "@t3tools/shared/speech";
4
5
import { filterSpeechModels } from "./speechModelBrowser";
6
@@ -31,9 +32,9 @@ const models: readonly EnvironmentSpeechModel[] = [
32
benchmark: {
33
accuracy: {
34
en: { grade: "A", errorPercent: 3.8 },
34
- fr: { grade: "B", errorPercent: 8 },
35
+ fr: { grade: "B", errorPercent: 7 },
36
},
36
- secondsPer30s: { cpu: 2, accelerated: 2 },
37
+ secondsPer30s: { accelerated: 2, cpu: 1 },
38
},
39
},
40
{
@@ -47,7 +48,7 @@ const models: readonly EnvironmentSpeechModel[] = [
48
en: { grade: "A", errorPercent: 3.2 },
49
fr: { grade: "B", errorPercent: 9 },
50
},
50
- secondsPer30s: { cpu: 1, accelerated: 1 },
51
+ secondsPer30s: { accelerated: 1, cpu: 3 },
52
},
53
},
54
{
@@ -58,7 +59,7 @@ const models: readonly EnvironmentSpeechModel[] = [
59
active: false,
60
state: "verifying",
61
languages: ["en"],
61
- benchmark: { accuracy: {}, secondsPer30s: { cpu: null, accelerated: null } },
62
+ benchmark: { accuracy: {}, secondsPer30s: { accelerated: null, cpu: null } },
63
},
64
];
65
@@ -100,3 +101,58 @@ it("combines text, all selected languages, and completed downloads without retai
101
).toEqual(["fast"]);
102
expect(filterSpeechModels(models, { ...options, search: "small" })).toEqual([]);
103
});
104
+
105
+it("ranks language recommendations before the rest without changing explicit sorts or search badges", () => {
106
+ const catalog = models.map((model) => ({
107
+ ...model,
108
+ active: false,
109
+ state: "downloadable" as const,
110
+ }));
111
+ const options = {
112
+ search: "",
113
+ languages: ["en"],
114
+ downloadedOnly: false,
115
+ sort: "recommended",
116
+ accuracyLanguage: "en",
117
+ benchmarkTier: "accelerated",
118
+ } as const;
119
+ expect(filterSpeechModels(catalog, options).map(({ id }) => id)).toEqual([
120
+ "fast",
121
+ "active",
122
+ "small",
123
+ "pending",
124
+ ]);
125
+ expect(
126
+ filterSpeechModels(catalog, { ...options, languages: ["en", "fr"] }).map(({ id }) => id),
127
+ ).toEqual(["small", "fast", "active"]);
128
+ expect(filterSpeechModels(catalog, { ...options, languages: [] }).map(({ id }) => id)).toEqual(
129
+ catalog.map(({ id }) => id),
130
+ );
131
+ expect(filterSpeechModels(catalog, { ...options, sort: "size" }).map(({ id }) => id)).toEqual([
132
+ "small",
133
+ "fast",
134
+ "active",
135
+ "pending",
136
+ ]);
137
+ expect(
138
+ filterSpeechModels(catalog, { ...options, sort: "speed", benchmarkTier: "cpu" }).map(
139
+ ({ id }) => id,
140
+ ),
141
+ ).toEqual(["small", "fast", "active", "pending"]);
142
+ expect(
143
+ filterSpeechModels(catalog, { ...options, sort: "accuracy", languages: ["en", "fr"] }).map(
144
+ ({ id }) => id,
145
+ ),
146
+ ).toEqual(["fast", "small", "active"]);
147
+ expect(
148
+ filterSpeechModels(catalog, { ...options, sort: "speed", languages: ["en", "fr"] }).map(
149
+ ({ id }) => id,
150
+ ),
151
+ ).toEqual(["fast", "small", "active"]);
152
+ expect(filterSpeechModels(catalog, { ...options, search: "small" }).map(({ id }) => id)).toEqual([
153
+ "small",
154
+ ]);
155
+ expect(
156
+ recommendSpeechModels(catalog, options.languages, options.benchmarkTier).has("small"),
157
+ ).toBe(false);
158
+});
apps/web/src/components/settings/speechModelBrowser.ts
+9
-4
@@ -1,4 +1,5 @@
1
import type { EnvironmentSpeechModel, SpeechBenchmarkTier } from "@t3tools/contracts";
2
+import { recommendSpeechModels } from "@t3tools/shared/speech";
3
4
export type SpeechModelSort = "recommended" | "size" | "accuracy" | "speed";
5
@@ -6,7 +7,7 @@ const modelSortOrder = (model: EnvironmentSpeechModel) => {
7
if (model.active) return 0;
8
if (model.state === "installed") return 1;
9
if (model.state === "downloading" || model.state === "verifying") return 2;
9
- return model.recommended ? 3 : 4;
10
+ return 3;
11
};
12
13
export function filterSpeechModels(
@@ -17,10 +18,12 @@ export function filterSpeechModels(
18
readonly downloadedOnly: boolean;
19
readonly sort: SpeechModelSort;
20
readonly accuracyLanguage: string;
20
- readonly benchmarkTier: SpeechBenchmarkTier;
21
+ readonly benchmarkTier: SpeechBenchmarkTier | undefined;
22
},
23
+ recommendations = recommendSpeechModels(models, options.languages, options.benchmarkTier),
24
) {
25
const query = options.search.trim().toLocaleLowerCase();
26
+ const ranks = new Map([...recommendations.keys()].map((id, index) => [id, index]));
27
const score = (model: EnvironmentSpeechModel) => {
28
switch (options.sort) {
29
case "size":
@@ -28,9 +31,11 @@ export function filterSpeechModels(
31
case "accuracy":
32
return model.benchmark?.accuracy[options.accuracyLanguage]?.errorPercent ?? Infinity;
33
case "speed":
31
- return model.benchmark?.secondsPer30s[options.benchmarkTier] ?? Infinity;
34
+ return options.benchmarkTier === undefined
35
+ ? Infinity
36
+ : (model.benchmark?.secondsPer30s[options.benchmarkTier] ?? Infinity);
37
case "recommended":
33
- return modelSortOrder(model);
38
+ return modelSortOrder(model) * 4 + (ranks.get(model.id) ?? 3);
39
}
40
};
41
return models
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+37
-10
@@ -42,6 +42,7 @@ const mocks = vi.hoisted(() => ({
42
primaryEnvironmentId: "primary-env",
43
statusRequests: [] as string[],
44
statusUnavailable: false,
45
+ statusTier: "accelerated" as "cpu" | "accelerated",
46
modelState: "ready" as "ready" | "missing-model",
47
statusModelId: "test-model",
48
models: [] as EnvironmentSpeechModel[],
@@ -134,9 +135,9 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
135
supportsStreaming: false,
136
supportsTranslation: false,
137
acceleration: "auto",
138
+ benchmarkTier: mocks.statusTier,
139
modelUnloadTimeout: "min_15",
140
gpuDevices: [],
139
- benchmarkTier: "cpu",
141
};
142
}),
143
downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
@@ -151,10 +152,14 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
152
mocks.statusModelId = modelId;
153
return { models: mocks.models };
154
}),
154
- getEnvironmentSpeechModels: vi.fn(() =>
155
+ getEnvironmentSpeechModels: vi.fn((_prepared: PreparedConnection, includeBenchmarks = false) =>
156
Effect.sync(() => {
157
if (mocks.catalogOffline) throw new Error("Catalog offline");
157
- return { models: mocks.models };
158
+ return {
159
+ models: mocks.models.map(({ benchmark, ...model }) =>
160
+ includeBenchmarks ? { ...model, ...(benchmark ? { benchmark } : {}) } : model,
161
+ ),
162
+ };
163
}),
164
),
165
postProcessEnvironmentTranscript: (
@@ -225,6 +230,7 @@ function setup(options: { readonly destination?: string | null } = {}) {
230
mocks.statusRequests = [];
231
mocks.statusUnavailable = false;
232
mocks.modelState = "ready";
233
+ mocks.statusTier = "accelerated";
234
mocks.statusModelId = "test-model";
235
mocks.models = [];
236
mocks.catalogOffline = false;
@@ -491,6 +497,7 @@ function catalogModel(
497
id: string,
498
languages: readonly string[],
499
recommended: boolean,
500
+ errorPercent = 4,
501
): EnvironmentSpeechModel {
502
return {
503
id,
@@ -501,6 +508,12 @@ function catalogModel(
508
accuracy: 0,
509
speed: 0,
510
recommended,
511
+ benchmark: {
512
+ accuracy: Object.fromEntries(
513
+ languages.map((language) => [language, { grade: "A", errorPercent }]),
514
+ ),
515
+ secondsPer30s: { accelerated: 2, cpu: 3 },
516
+ },
517
supportsStreaming: false,
518
supportsLanguageDetection: languages.length > 1,
519
supportsTranslation: false,
@@ -510,26 +523,30 @@ function catalogModel(
523
}
524
525
it.each([
513
- ["the browser prefers German", ["de-DE"], undefined, "auto"],
514
- ["the desktop exposes no locale methods", ["de-DE"], {}, "auto"],
526
+ ["the browser prefers German", ["de-DE"], undefined, "auto", "accelerated"],
527
+ ["the remote environment uses CPU", ["de-DE"], undefined, "auto", "cpu"],
528
+ ["the desktop exposes no locale methods", ["de-DE"], {}, "auto", "accelerated"],
529
[
530
"an older preload exposes only the OS locale",
531
["en-US"],
532
{ getSystemLocale: () => "de-DE" },
533
"auto",
534
+ "accelerated",
535
],
536
[
537
"the desktop OS prefers German",
538
["en-US"],
539
{ getPreferredSystemLanguages: () => ["de-DE"] },
540
"auto",
541
+ "accelerated",
542
],
527
- ["German is the chosen Voice language", ["en-US"], undefined, "de"],
543
+ ["German is the chosen Voice language", ["en-US"], undefined, "de", "accelerated"],
544
])(
545
"offers and downloads a German-capable model when %s",
530
- async (_, languages, bridge, speechLanguage) => {
546
+ async (_, languages, bridge, speechLanguage, tier) => {
547
setup({ destination: "env-a" });
548
mocks.settings = { speechLanguage };
549
+ mocks.statusTier = tier as "cpu" | "accelerated";
550
using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(languages);
551
using _bridge = stubGlobal("desktopBridge", bridge);
552
const englishDefault = "handy-computer/parakeet-unified-en-0.6b-gguf";
@@ -538,16 +555,24 @@ it.each([
555
mocks.statusModelId = englishDefault;
556
mocks.models = [
557
{ ...catalogModel(englishDefault, ["en"], true), active: true },
541
- catalogModel(german, ["en", "de", "fr"], true),
558
catalogModel("handy-computer/canary-180m-flash-gguf", ["en", "de", "es", "fr"], true),
559
+ {
560
+ ...catalogModel(german, ["en", "de", "fr"], false, 2),
561
+ benchmark: {
562
+ accuracy: { de: { grade: "A", errorPercent: 2 } },
563
+ secondsPer30s: { cpu: 20, accelerated: 2 },
564
+ },
565
+ },
566
];
567
mocks.download.mockResolvedValue(undefined);
568
await using _screen = await render(<Probe />);
569
await act(() => mocks.voice.start());
570
expect(mocks.voice.setup.open).toBe(true);
571
await act(() => mocks.voice.setup.download());
549
- expect(mocks.download).toHaveBeenCalledExactlyOnceWith(german);
550
- expect(mocks.voice.setup.model?.id).toBe(german);
572
+ const expected = tier === "cpu" ? "handy-computer/canary-180m-flash-gguf" : german;
573
+ expect(mocks.download).toHaveBeenCalledExactlyOnceWith(expected);
574
+ expect(mocks.voice.setup.model?.id).toBe(expected);
575
+ expect(mocks.voice.setup.model?.recommended).toBe(true);
576
},
577
);
578
@@ -607,6 +632,7 @@ it.each([true, false])(
632
mocks.models.push(catalogModel("german", ["de"], true));
633
// Another client can finish setup while the catalog is offline.
634
mocks.modelState = "ready";
635
+ mocks.statusTier = "accelerated";
636
await userEvent.click(retry);
637
await expect.element(wizard.getByRole("heading", { name: "german" })).toBeVisible();
638
await expect.element(download).toBeEnabled();
@@ -641,6 +667,7 @@ it.each([
667
mocks.download.mockImplementation(async () => {
668
mocks.models = [mocks.models[0]!, { ...mocks.models[1]!, state: landedState }];
669
mocks.modelState = "ready";
670
+ mocks.statusTier = "accelerated";
671
});
672
await using _screen = await render(<Probe />);
673
await act(() => mocks.voice.start());
apps/web/src/speech/useEnvironmentSpeechInput.ts
+25
-8
@@ -14,6 +14,7 @@ import type {
14
EnvironmentSpeechModel,
15
EnvironmentSpeechStatus,
16
ProjectId,
17
+ SpeechBenchmarkTier,
18
} from "@t3tools/contracts";
19
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
20
import { chooseSetupSpeechModel } from "@t3tools/shared/speech";
@@ -253,7 +254,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
254
.then((result) => {
255
if (disposed) return;
256
const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null;
256
- if (model) setSetup((current) => ({ ...current, model }));
257
+ if (model)
258
+ setSetup((current) => ({
259
+ ...current,
260
+ model: { ...model, recommended: current.model?.recommended ?? false },
261
+ }));
262
})
263
.catch(() => undefined)
264
.finally(() => {
@@ -347,20 +352,26 @@ export function useEnvironmentSpeechInput(input: HookInput) {
352
session.adopt(createTarget());
353
}, [session, input.ownerKey, createTarget]);
354
const loadSetupCatalog = useStableCallback(
350
- async (connection: PreparedConnection, selectedModelId: string, request: number) => {
355
+ async (
356
+ connection: PreparedConnection,
357
+ selectedModelId: string,
358
+ benchmarkTier: SpeechBenchmarkTier,
359
+ request: number,
360
+ ) => {
361
setSetup({ ...INITIAL_SETUP, open: true });
352
- const catalog = await runEnvironmentSpeech(getEnvironmentSpeechModels(connection)).catch(
353
- () => null,
354
- );
362
+ const catalog = await runEnvironmentSpeech(
363
+ getEnvironmentSpeechModels(connection, true),
364
+ ).catch(() => null);
365
if (request !== startRequestRef.current) return;
366
const choice = chooseSetupSpeechModel(catalog?.models ?? [], {
367
selectedModelId,
368
intent: latestRef.current.clientSettings.speechLanguage,
369
preferred: preferredSpeechLanguages(),
370
+ benchmarkTier,
371
});
372
setSetup((current) => ({
373
...current,
363
- model: choice?.model ?? null,
374
+ model: choice ? { ...choice.model, recommended: choice.recommended } : null,
375
language: choice?.language ?? null,
376
catalogFailed: catalog === null,
377
error: choice
@@ -410,7 +421,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
421
setStatus({ prepared, value: latestStatus });
422
setQueuedStart(null);
423
if (latestStatus.supported && latestStatus.state === "missing-model") {
413
- return loadSetupCatalog(prepared, latestStatus.modelId, request);
424
+ return loadSetupCatalog(prepared, latestStatus.modelId, latestStatus.benchmarkTier, request);
425
}
426
if (!latestStatus.supported) {
427
toastManager.add({
@@ -505,7 +516,13 @@ export function useEnvironmentSpeechInput(input: HookInput) {
516
setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
517
retryCatalog:
518
setup.open && setup.catalogFailed && prepared && currentStatus?.supported
508
- ? () => loadSetupCatalog(prepared, currentStatus.modelId, ++startRequestRef.current)
519
+ ? () =>
520
+ loadSetupCatalog(
521
+ prepared,
522
+ currentStatus.modelId,
523
+ currentStatus.benchmarkTier,
524
+ ++startRequestRef.current,
525
+ )
526
: null,
527
download: downloadSetupModel,
528
cancelDownload: cancelSetupDownload,
docs/user/voice-input.md
+4
-2
@@ -66,8 +66,10 @@ format, such as Core ML, ONNX, or MLX builds of the same model, can’t be used.
66
67
Select the languages you dictate in to compare models that support all of them.
68
Accuracy shows how often the model got words or characters wrong in each
69
-language; speed is the time to transcribe 30 seconds on a reference GPU. Choose
70
-**All languages** to browse the full catalog. Filtering does not switch your model.
69
+language; speed is the time to transcribe 30 seconds on a reference CPU or GPU,
70
+matching the selected environment's transcription backend. Recommendations use
71
+the same reference tier for every selected language. Choose **All languages** to
72
+browse without a language preference. Filtering does not switch your model.
73
74
**Advanced** controls GPU or CPU use and when an idle model is unloaded. With hardware set to
75
**Auto**, a recording that fails on the GPU is retried once on that same
packages/shared/src/speech.test.ts
+180
-2
@@ -3,6 +3,7 @@ import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES } from "
3
4
import {
5
chooseSetupSpeechModel,
6
+ recommendSpeechModels,
7
decodeSpeechPcmRequest,
8
encodeSpeechPcmRequest,
9
resolveSpeechVocabulary,
@@ -79,7 +80,18 @@ describe("chooseSetupSpeechModel", () => {
80
languages: string[],
81
supportsLanguageDetection: boolean,
82
recommended: boolean,
82
- ) => ({ id, languages, supportsLanguageDetection, recommended });
83
+ ) => ({
84
+ id,
85
+ languages,
86
+ supportsLanguageDetection,
87
+ recommended,
88
+ benchmark: {
89
+ accuracy: Object.fromEntries(
90
+ languages.map((language) => [language, { errorPercent: recommended ? 4 : 10 }]),
91
+ ),
92
+ secondsPer30s: { accelerated: 2, cpu: 3 },
93
+ },
94
+ });
95
const catalog = [
96
model("unrecommended", ["en", "de", "es"], true, false),
97
model("english", ["en"], false, true),
@@ -146,7 +158,173 @@ describe("chooseSetupSpeechModel", () => {
158
undefined,
159
],
160
] as const)("%s", (_, selectedModelId, intent, preferred, expected) => {
149
- const choice = chooseSetupSpeechModel(catalog, { selectedModelId, intent, preferred });
161
+ const choice = chooseSetupSpeechModel(catalog, {
162
+ selectedModelId,
163
+ intent,
164
+ preferred,
165
+ benchmarkTier: "accelerated",
166
+ });
167
expect(choice && { id: choice.model.id, language: choice.language }).toEqual(expected);
168
});
169
+
170
+ it("keeps suitable selections and preference order, and falls back safely without measurements", () => {
171
+ const best = {
172
+ ...catalog[0]!,
173
+ benchmark: {
174
+ accuracy: { de: { errorPercent: 1 } },
175
+ secondsPer30s: { accelerated: 1, cpu: 2 },
176
+ },
177
+ };
178
+ expect(
179
+ chooseSetupSpeechModel([...catalog, { ...best, id: "best" }], {
180
+ benchmarkTier: "accelerated",
181
+ selectedModelId: "gone",
182
+ intent: "auto",
183
+ preferred: ["de", "en"],
184
+ }),
185
+ ).toMatchObject({ model: { id: "best" }, language: "de", recommended: true });
186
+ expect(
187
+ chooseSetupSpeechModel([...catalog, { ...best, id: "best" }], {
188
+ benchmarkTier: "accelerated",
189
+ selectedModelId: "detecting",
190
+ intent: "auto",
191
+ preferred: ["de", "en"],
192
+ }),
193
+ ).toMatchObject({ model: { id: "detecting" }, language: "de", recommended: false });
194
+ const unmeasured = catalog.map(({ benchmark: _benchmark, ...model }) => model);
195
+ expect(
196
+ chooseSetupSpeechModel(unmeasured, {
197
+ benchmarkTier: "accelerated",
198
+ selectedModelId: "gone",
199
+ intent: "auto",
200
+ preferred: ["de", "en"],
201
+ }),
202
+ ).toMatchObject({ model: { id: "unrecommended" }, language: "de", recommended: false });
203
+ expect(
204
+ chooseSetupSpeechModel([], {
205
+ benchmarkTier: "accelerated",
206
+ selectedModelId: "gone",
207
+ intent: "auto",
208
+ preferred: [],
209
+ }),
210
+ ).toBeUndefined();
211
+ });
212
+});
213
+
214
+describe("recommendSpeechModels", () => {
215
+ const model = (
216
+ id: string,
217
+ errors: Record<string, number>,
218
+ wait: number | null,
219
+ cpu: number | null,
220
+ ) => ({
221
+ id,
222
+ languages: Object.keys(errors),
223
+ benchmark: {
224
+ accuracy: Object.fromEntries(
225
+ Object.entries(errors).map(([language, errorPercent]) => [language, { errorPercent }]),
226
+ ),
227
+ secondsPer30s: { accelerated: wait, cpu },
228
+ },
229
+ });
230
+ const models = [
231
+ model("balanced", { en: 4, de: 16 }, 2, 7),
232
+ model("fast", { en: 7, de: 7 }, 1, 3),
233
+ model("accurate", { en: 2, de: 2 }, 8, 20),
234
+ model("incomplete", { en: 1 }, 1, 1),
235
+ ];
236
+
237
+ it("balances all requested languages and assigns each model its first role", () => {
238
+ expect([...recommendSpeechModels(models.slice(0, 3), ["en"], "accelerated")]).toEqual([
239
+ ["balanced", "recommended"],
240
+ ["fast", "faster"],
241
+ ["accurate", "accurate"],
242
+ ]);
243
+ expect([...recommendSpeechModels(models, ["en", "de", "de"], "accelerated")]).toEqual([
244
+ ["fast", "recommended"],
245
+ ["accurate", "accurate"],
246
+ ]);
247
+ expect([...recommendSpeechModels(models.slice(0, 3), ["en"], "cpu")]).toEqual([
248
+ ["fast", "recommended"],
249
+ ["balanced", "accurate"],
250
+ ]);
251
+ expect([...recommendSpeechModels(models, ["en", "de"], "cpu")]).toEqual([
252
+ ["fast", "recommended"],
253
+ ]);
254
+ });
255
+
256
+ it("offers alternatives only when they improve on the balanced pick on the same tier", () => {
257
+ expect([
258
+ ...recommendSpeechModels(
259
+ [model("cpu-fast-gpu-slow", { en: 3 }, 4, 1), model("balanced", { en: 3.2 }, 1, 8)],
260
+ ["en"],
261
+ "accelerated",
262
+ ),
263
+ ]).toEqual([
264
+ ["balanced", "recommended"],
265
+ ["cpu-fast-gpu-slow", "accurate"],
266
+ ]);
267
+ expect([
268
+ ...recommendSpeechModels(
269
+ [model("equal-accuracy", { en: 4 }, 8, 20), model("balanced", { en: 4 }, 2, 20)],
270
+ ["en"],
271
+ "accelerated",
272
+ ),
273
+ ]).toEqual([["balanced", "recommended"]]);
274
+ expect([
275
+ ...recommendSpeechModels(
276
+ [model("balanced", { en: 4 }, 12, 20), model("outside-budget", { en: 2 }, 30, 40)],
277
+ ["en"],
278
+ "accelerated",
279
+ ),
280
+ ]).toEqual([["balanced", "recommended"]]);
281
+ });
282
+
283
+ it("does not invent measurements, preferences, or a model covering missing languages", () => {
284
+ expect(recommendSpeechModels(models, [], "accelerated").size).toBe(0);
285
+ expect(recommendSpeechModels(models, ["ja"], "accelerated").size).toBe(0);
286
+ expect(
287
+ recommendSpeechModels(
288
+ [
289
+ model("missing-speed", { en: 1, de: 1 }, null, 1),
290
+ { ...model("missing-accuracy", { en: 1 }, 1, 1), languages: ["en", "de"] },
291
+ ],
292
+ ["en", "de"],
293
+ "accelerated",
294
+ ).size,
295
+ ).toBe(0);
296
+ expect([
297
+ ...recommendSpeechModels([model("no-cpu", { en: 1 }, 1, null)], ["en"], "accelerated"),
298
+ ]).toEqual([["no-cpu", "recommended"]]);
299
+ expect(recommendSpeechModels([model("no-cpu", { en: 1 }, 1, null)], ["en"], "cpu").size).toBe(
300
+ 0,
301
+ );
302
+ expect([
303
+ ...recommendSpeechModels([model("cpu-only", { en: 1 }, null, 2)], ["en"], "cpu"),
304
+ ]).toEqual([["cpu-only", "recommended"]]);
305
+ });
306
+
307
+ it("prefers useful accuracy before speed and relaxes the wait budget before quality", () => {
308
+ expect([
309
+ ...recommendSpeechModels(
310
+ [model("poor", { en: 25 }, 1, 1), model("slow", { en: 4 }, 12, 20)],
311
+ ["en"],
312
+ "accelerated",
313
+ ),
314
+ ]).toEqual([["slow", "recommended"]]);
315
+ expect([
316
+ ...recommendSpeechModels(
317
+ [model("worse", { en: 30, de: 40 }, 1, 1), model("least-poor", { en: 29, de: 30 }, 20, 40)],
318
+ ["en", "de"],
319
+ "accelerated",
320
+ ),
321
+ ]).toEqual([["least-poor", "recommended"]]);
322
+ expect([
323
+ ...recommendSpeechModels(
324
+ [model("zero", { en: 0, de: 0 }, 1, 1)],
325
+ ["en", "de"],
326
+ "accelerated",
327
+ ),
328
+ ]).toEqual([["zero", "recommended"]]);
329
+ });
330
});
packages/shared/src/speech.ts
+98
-8
@@ -3,13 +3,102 @@ import {
3
type CloudSpeechProvider,
4
SPEECH_MAX_OPTIONS_BYTES,
5
type SpeechCustomWords,
6
+ type SpeechBenchmarkTier,
7
SpeechTranscriptionOptions,
8
} from "@t3tools/contracts";
9
import * as Schema from "effect/Schema";
10
+import recommendationPolicy from "./speechRecommendationPolicy.json" with { type: "json" };
11
12
const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions));
13
const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions));
14
15
+export type SpeechRecommendationRole = "recommended" | "faster" | "accurate";
16
+type RecommendationModel = {
17
+ readonly id: string;
18
+ readonly languages: readonly string[];
19
+ readonly benchmark?: {
20
+ readonly accuracy: Readonly<Record<string, { readonly errorPercent: number }>>;
21
+ readonly secondsPer30s: {
22
+ readonly cpu?: number | null;
23
+ readonly accelerated: number | null;
24
+ };
25
+ };
26
+};
27
+
28
+/** Reference accuracy/speed picks; no preference or incomplete measurements get no badge. */
29
+export function recommendSpeechModels(
30
+ models: readonly RecommendationModel[],
31
+ languages: readonly string[],
32
+ tier: SpeechBenchmarkTier | undefined,
33
+) {
34
+ const wanted = [...new Set(languages)];
35
+ const roles = new Map<string, SpeechRecommendationRole>();
36
+ if (wanted.length === 0 || tier === undefined) return roles;
37
+ const policy = recommendationPolicy;
38
+ const scored = models.flatMap((model) => {
39
+ if (!wanted.every((language) => model.languages.includes(language))) return [];
40
+ const errors = wanted.map((language) => model.benchmark?.accuracy[language]?.errorPercent);
41
+ const wait = model.benchmark?.secondsPer30s[tier];
42
+ if (
43
+ wait == null ||
44
+ !Number.isFinite(wait) ||
45
+ wait < 0 ||
46
+ errors.some((error) => error === undefined || !Number.isFinite(error) || error < 0)
47
+ )
48
+ return [];
49
+ const measured = errors as number[];
50
+ const error = Math.exp(
51
+ measured.reduce((sum, value) => sum + Math.log(value), 0) / measured.length,
52
+ );
53
+ return [
54
+ {
55
+ id: model.id,
56
+ error,
57
+ worst: Math.max(...measured),
58
+ wait,
59
+ cost: error + policy.waitWeight * Math.max(0, wait - policy.freeWaitSeconds),
60
+ },
61
+ ];
62
+ });
63
+ if (scored.length === 0) return roles;
64
+ const useful = scored.filter((model) => model.worst < policy.maxLanguageErrorPercent);
65
+ const pool = useful.length ? useful : scored;
66
+ const lowest = (items: typeof scored, score: (item: (typeof scored)[number]) => number) =>
67
+ items.reduce<(typeof scored)[number] | undefined>(
68
+ (best, item) => (best === undefined || score(item) < score(best) ? item : best),
69
+ undefined,
70
+ );
71
+ const withinBudget = pool.filter((model) => model.wait <= policy.overallMaxWaitSeconds);
72
+ const best = (
73
+ useful.length
74
+ ? lowest(withinBudget.length ? withinBudget : pool, (model) => model.cost)
75
+ : lowest(pool, (model) => model.worst + model.error / 1000)
76
+ )!;
77
+ const quicker = pool.filter((model) => model.wait < best.wait);
78
+ const fast = useful.length
79
+ ? (lowest(
80
+ quicker.filter((model) => model.wait <= policy.fastMaxWaitSeconds),
81
+ (model) => model.error,
82
+ ) ?? lowest(quicker, (model) => model.wait + model.error / 1000))
83
+ : undefined;
84
+ const accurate = useful.length
85
+ ? lowest(
86
+ pool.filter(
87
+ (model) => model.error < best.error && model.wait <= policy.accurateMaxWaitSeconds,
88
+ ),
89
+ (model) => model.error,
90
+ )
91
+ : undefined;
92
+ for (const [model, role] of [
93
+ [best, "recommended"],
94
+ [fast, "faster"],
95
+ [accurate, "accurate"],
96
+ ] as const) {
97
+ if (model && !roles.has(model.id)) roles.set(model.id, role);
98
+ }
99
+ return roles;
100
+}
101
+
102
export function cloudSpeechDisclosure(provider: CloudSpeechProvider, environment: string) {
103
return provider === "openrouter"
104
? `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.`
@@ -37,16 +126,13 @@ export function effectiveSpeechLanguage(
126
}
127
128
/**
40
- * The model first-run setup offers: the selected one, else a recommended one, else any that the
129
+ * The model first-run setup offers: the selected one, else the benchmark pick, else any that the
130
* server would really transcribe the person's language with. Without a language hint,
131
* offer only the selected model, without a recommendation.
132
*/
133
export function chooseSetupSpeechModel<
45
- Model extends {
46
- readonly id: string;
47
- readonly languages: readonly string[];
134
+ Model extends RecommendationModel & {
135
readonly supportsLanguageDetection: boolean;
49
- readonly recommended: boolean;
136
},
137
>(
138
models: readonly Model[],
@@ -54,22 +140,26 @@ export function chooseSetupSpeechModel<
140
readonly selectedModelId: string;
141
readonly intent: string;
142
readonly preferred: readonly string[];
143
+ readonly benchmarkTier: SpeechBenchmarkTier | undefined;
144
},
145
) {
146
if (input.intent === "auto" && input.preferred.length === 0) {
147
const model = models.find(({ id }) => id === input.selectedModelId);
61
- return model && { model, language: null };
148
+ return model && { model, language: null, recommended: false };
149
}
150
for (const language of input.intent === "auto" ? input.preferred : [input.intent]) {
151
const capable = models.filter((model) => {
152
const effective = effectiveSpeechLanguage(model, input.intent);
153
return effective === language || (effective === "auto" && model.languages.includes(language));
154
});
155
+ const recommendedId = recommendSpeechModels(capable, [language], input.benchmarkTier)
156
+ .keys()
157
+ .next().value;
158
const model =
159
capable.find(({ id }) => id === input.selectedModelId) ??
70
- capable.find(({ recommended }) => recommended) ??
160
+ capable.find(({ id }) => id === recommendedId) ??
161
capable[0];
72
- if (model) return { model, language };
162
+ if (model) return { model, language, recommended: model.id === recommendedId };
163
}
164
return undefined;
165
}
packages/shared/src/speechRecommendationPolicy.json
new
+11
@@ -0,0 +1,11 @@
1
+{
2
+ "dictationSeconds": 30,
3
+ "freeWaitSeconds": 3,
4
+ "waitWeight": 0.5,
5
+ "overallMaxWaitSeconds": 5,
6
+ "maxLanguageErrorPercent": 20,
7
+ "noteMinErrorPercent": 15,
8
+ "experimentalMaxErrorPercent": 30,
9
+ "accurateMaxWaitSeconds": 10,
10
+ "fastMaxWaitSeconds": 4
11
+}