@devries-cameron20 / t3code Mirror · read-only / commits / bfd37b2b53
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 08:30 UTC

feat(speech): recommend local models by language, speed and accuracy (#33336)

Utkarsh Patil committed Nov 17, 2026 at 23:22 UTC bfd37b2b5341645c5da172e6e2acae9b82c946f8
16 files changed +715 -213
apps/mobile/src/features/settings/VoiceModelSection.tsx
+1 -1
@@ -148,7 +148,7 @@ function ModelStats(props: { readonly model: EnvironmentSpeechModel; readonly la
148 "speed",
149 <>
150 <Text className="font-t3-medium text-foreground">
151 - {data.secondsPer30s.accelerated} s
151 + {Number(data.secondsPer30s.accelerated.toFixed(2))} s
152 </Text>{" "}
153 / 30 s
154 </>,
apps/server/scripts/speech-catalog.test.ts
+29 -3
@@ -2,6 +2,8 @@
2 import * as NodeFS from "node:fs";
3
4 import { catalog } from "@handy-computer/models";
5 +import { recommendSpeechModels } from "@t3tools/shared/speech";
6 +import policy from "../../../packages/shared/src/speechRecommendationPolicy.json" with { type: "json" };
7 import { expect, it } from "vite-plus/test";
8
9 import manifest from "../package.json" with { type: "json" };
@@ -15,6 +17,8 @@ const generate = (models = catalog.models) =>
17 );
18
19 it("reproduces the bundled registry byte for byte without upstream requests", async () => {
20 + const { fastCpuMaxWaitSeconds, ...methodology } = catalog.methodology;
21 + expect(policy).toEqual({ ...methodology, fastMaxWaitSeconds: fastCpuMaxWaitSeconds });
22 expect(NodeFS.readFileSync(new URL("../src/speech/catalog.json", import.meta.url), "utf8")).toBe(
23 await renderSpeechCatalog(
24 catalog.data,
@@ -53,18 +57,40 @@ it("reproduces the bundled registry byte for byte without upstream requests", as
57 {
58 ...catalog.data,
59 models: [parakeet],
56 - results: { [parakeet.id]: { accuracy: {}, performance } },
60 + results: { [parakeet.id]: { accuracy: { en: { error: 3.99 } }, performance } },
61 },
62 "0.2.3",
63 "0.4.1",
64 );
65 expect(result.models[0]?.benchmark.secondsPer30s).toEqual({
62 - cpu: performance.cpu ? 15 : null,
63 - accelerated: performance.accelerated ? 3 : null,
66 + cpu: performance.cpu ? 30 / performance.cpu : null,
67 + accelerated: performance.accelerated ? 30 / performance.accelerated : null,
68 });
69 }
70 });
71
72 +it("preserves Handy's accelerated balanced pick after filtering T3's offered candidates", () => {
73 + const offered = generate().models;
74 + const sources = catalog.models.filter((model) =>
75 + offered.some(({ id }) => id === model.repository),
76 + );
77 + for (const languages of [["en"], ["de"], ["ja"], ["en", "de"], ["en", "fr"], ["hi", "mr"]]) {
78 + const expected = catalog
79 + .recommendModels(sources, languages)
80 + .find((pick) => pick.status !== "unbenchmarked" && pick.roles.includes("best"));
81 + expect(
82 + recommendSpeechModels(offered, languages, "accelerated").keys().next().value,
83 + languages.join(","),
84 + ).toBe(expected?.model.repository);
85 + }
86 + const excluded = { ...catalog.models[0]!, license: "cc-by-nc-4.0" };
87 + const allowed = { ...catalog.models[1]!, languages: ["en"] };
88 + const filtered = generate([excluded, allowed]).models;
89 + expect([...recommendSpeechModels(filtered, ["en"], "accelerated").keys()]).toEqual([
90 + allowed.repository,
91 + ]);
92 +});
93 +
94 it("filters restricted licenses, unsupported roles, engines, architectures and files", () => {
95 const model = catalog.models[0]!;
96 for (const override of [
apps/server/scripts/speech-catalog.ts
+5 -6
@@ -31,7 +31,7 @@ const ASR_FAMILIES = new Set([
31 "medasr",
32 ]);
33
34 -// Preserve Handy's onboarding badges; curation alone also includes specialist models.
34 +// Older clients rank and pick setup models by this flag; current clients use language recommendations.
35 const RECOMMENDED = new Set([
36 "parakeet-unified-en-0.6b",
37 "nemotron-3.5-asr-streaming-0.6b",
@@ -81,9 +81,8 @@ export function generateSpeechCatalog(
81 ids.add(model.repository);
82 filenames.add(file.filename);
83 const languages = [...new Set(model.languages.map(canonicalLanguage))];
84 - const performance = data.results[model.id]?.performance;
85 - const secondsPer30s = (realtimeFactor: number | undefined) =>
86 - realtimeFactor ? Number((BENCHMARK_DICTATION_SECONDS / realtimeFactor).toFixed(2)) : null;
84 + const realtimeFactor = data.results[model.id]?.performance.accelerated;
85 + const cpuFactor = data.results[model.id]?.performance.cpu;
86 const accuracy = Object.fromEntries(
87 languages.flatMap((language) => {
88 const cell = data.results[model.id]?.accuracy[languageIdentity(language)];
@@ -112,8 +111,8 @@ export function generateSpeechCatalog(
111 benchmark: {
112 accuracy,
113 secondsPer30s: {
115 - cpu: secondsPer30s(performance?.cpu),
116 - accelerated: secondsPer30s(performance?.accelerated),
114 + cpu: cpuFactor ? BENCHMARK_DICTATION_SECONDS / cpuFactor : null,
115 + accelerated: realtimeFactor ? BENCHMARK_DICTATION_SECONDS / realtimeFactor : null,
116 },
117 },
118 recommended: RECOMMENDED.has(model.id),
apps/server/scripts/sync-speech-catalog.ts
+16 -1
@@ -5,6 +5,7 @@ import * as NodeModule from "node:module";
5 import * as NodePath from "node:path";
6
7 import { catalog } from "@handy-computer/models";
8 +import { format } from "vite-plus/fmt";
9 import * as Schema from "effect/Schema";
10
11 import manifest from "../package.json" with { type: "json" };
@@ -24,12 +25,26 @@ const expected = await renderSpeechCatalog(
25 sourceManifest.version,
26 manifest.dependencies["transcribe-cpp"],
27 );
28 +const policyTarget = new URL(
29 + "../../../packages/shared/src/speechRecommendationPolicy.json",
30 + import.meta.url,
31 +);
32 +const { fastCpuMaxWaitSeconds, ...methodology } = catalog.methodology;
33 +const policy = await format(
34 + "speechRecommendationPolicy.json",
35 + JSON.stringify({ ...methodology, fastMaxWaitSeconds: fastCpuMaxWaitSeconds }),
36 +);
37 +if (policy.errors.length > 0) throw new Error("Could not format speech recommendation policy.");
38 if (process.argv.includes("--check")) {
28 - if (NodeFS.readFileSync(target, "utf8") !== expected) {
39 + if (
40 + NodeFS.readFileSync(target, "utf8") !== expected ||
41 + NodeFS.readFileSync(policyTarget, "utf8") !== policy.code
42 + ) {
43 throw new Error("Speech catalog differs; run vp run speech:catalog:sync in apps/server.");
44 }
45 console.log("Bundled speech catalog matches the pinned source.");
46 } else {
47 NodeFS.writeFileSync(target, expected);
48 + NodeFS.writeFileSync(policyTarget, policy.code);
49 console.log(`Generated speech catalog from @handy-computer/models@${sourceManifest.version}.`);
50 }
apps/server/src/speech/catalog.json
+58 -58
@@ -15,7 +15,7 @@
15 "languages": ["en"],
16 "benchmark": {
17 "accuracy": { "en": { "grade": "A-", "errorPercent": 3.99 } },
18 - "secondsPer30s": { "cpu": 2.15, "accelerated": 1.19 }
18 + "secondsPer30s": { "cpu": 2.149767108563239, "accelerated": 1.1867088607594936 }
19 },
20 "recommended": true,
21 "supportsStreaming": true,
@@ -92,7 +92,7 @@
92 "ro": { "grade": "D", "errorPercent": 28.28 },
93 "et": { "grade": "F", "errorPercent": 31.84 }
94 },
95 - "secondsPer30s": { "cpu": 2.83, "accelerated": 1.75 }
95 + "secondsPer30s": { "cpu": 2.8328611898016995, "accelerated": 1.7508024511234315 }
96 },
97 "recommended": true,
98 "supportsStreaming": true,
@@ -116,7 +116,7 @@
116 "es": { "grade": "B", "errorPercent": 6.54 },
117 "fr": { "grade": "B-", "errorPercent": 8.53 }
118 },
119 - "secondsPer30s": { "cpu": 1.38, "accelerated": 0.91 }
119 + "secondsPer30s": { "cpu": 1.3805798435342844, "accelerated": 0.9067553271875473 }
120 },
121 "recommended": true,
122 "supportsStreaming": false,
@@ -165,7 +165,7 @@
165 "vi": { "grade": "B", "errorPercent": 7.39 },
166 "ko": { "grade": "B", "errorPercent": 6.57 }
167 },
168 - "secondsPer30s": { "cpu": 6.96, "accelerated": 3.71 }
168 + "secondsPer30s": { "cpu": 6.960556844547565, "accelerated": 3.7059913526868433 }
169 },
170 "recommended": true,
171 "supportsStreaming": false,
@@ -367,7 +367,7 @@
367 "yo": { "grade": "F", "errorPercent": 109 },
368 "zh": { "grade": "C", "errorPercent": 13.13 }
369 },
370 - "secondsPer30s": { "cpu": 15.58, "accelerated": 7.34 }
370 + "secondsPer30s": { "cpu": 15.584415584415584, "accelerated": 7.343941248470013 }
371 },
372 "recommended": true,
373 "supportsStreaming": false,
@@ -400,7 +400,7 @@
400 "hi": { "grade": "C", "errorPercent": 14.44 },
401 "ko": { "grade": "B+", "errorPercent": 5.27 }
402 },
403 - "secondsPer30s": { "cpu": 51.28, "accelerated": 39.47 }
403 + "secondsPer30s": { "cpu": 51.282051282051285, "accelerated": 39.473684210526315 }
404 },
405 "recommended": false,
406 "supportsStreaming": true,
@@ -471,7 +471,7 @@
471 "el": { "grade": "F", "errorPercent": 32.25 },
472 "mt": { "grade": "C-", "errorPercent": 17.01 }
473 },
474 - "secondsPer30s": { "cpu": 2.26, "accelerated": 1.3 }
474 + "secondsPer30s": { "cpu": 2.2573363431151243, "accelerated": 1.3043478260869565 }
475 },
476 "recommended": false,
477 "supportsStreaming": false,
@@ -542,7 +542,7 @@
542 "sv": { "grade": "C", "errorPercent": 15.25 },
543 "uk": { "grade": "B", "errorPercent": 6.84 }
544 },
545 - "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
545 + "secondsPer30s": { "cpu": 2.1969974368363236, "accelerated": 1.196888090963495 }
546 },
547 "recommended": false,
548 "supportsStreaming": false,
@@ -561,7 +561,7 @@
561 "languages": ["en"],
562 "benchmark": {
563 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.11 } },
564 - "secondsPer30s": { "cpu": 2.1, "accelerated": 1.09 }
564 + "secondsPer30s": { "cpu": 2.102312543798178, "accelerated": 1.0934937124111537 }
565 },
566 "recommended": false,
567 "supportsStreaming": false,
@@ -642,7 +642,7 @@
642 "hu": { "grade": "F", "errorPercent": 56.24 },
643 "mk": { "grade": "F", "errorPercent": 35.09 }
644 },
645 - "secondsPer30s": { "cpu": 5.67, "accelerated": 3.09 }
645 + "secondsPer30s": { "cpu": 5.665722379603399, "accelerated": 3.091190108191654 }
646 },
647 "recommended": false,
648 "supportsStreaming": false,
@@ -725,7 +725,7 @@
725 "sl": { "grade": "F", "errorPercent": 75.22 },
726 "sv": { "grade": "F", "errorPercent": 75.36 }
727 },
728 - "secondsPer30s": { "cpu": 5.48, "accelerated": 3.4 }
728 + "secondsPer30s": { "cpu": 5.484460694698355, "accelerated": 3.3955857385398978 }
729 },
730 "recommended": false,
731 "supportsStreaming": false,
@@ -796,7 +796,7 @@
796 "ru": { "grade": "B", "errorPercent": 7.83 },
797 "uk": { "grade": "C+", "errorPercent": 10.58 }
798 },
799 - "secondsPer30s": { "cpu": 3.6, "accelerated": 2.12 }
799 + "secondsPer30s": { "cpu": 3.603603603603604, "accelerated": 2.1231422505307855 }
800 },
801 "recommended": false,
802 "supportsStreaming": false,
@@ -815,7 +815,7 @@
815 "languages": ["en"],
816 "benchmark": {
817 "accuracy": { "en": { "grade": "A", "errorPercent": 3.58 } },
818 - "secondsPer30s": { "cpu": 11.95, "accelerated": 7.55 }
818 + "secondsPer30s": { "cpu": 11.952191235059763, "accelerated": 7.547169811320755 }
819 },
820 "recommended": false,
821 "supportsStreaming": false,
@@ -837,7 +837,7 @@
837 "en": { "grade": "A-", "errorPercent": 4.88 },
838 "ar": { "grade": "C+", "errorPercent": 11.06 }
839 },
840 - "secondsPer30s": { "cpu": 6.96, "accelerated": 3.61 }
840 + "secondsPer30s": { "cpu": 6.960556844547565, "accelerated": 3.6122817579771223 }
841 },
842 "recommended": false,
843 "supportsStreaming": false,
@@ -860,7 +860,7 @@
860 "en": { "grade": "B+", "errorPercent": 5.49 },
861 "ja": { "grade": "B-", "errorPercent": 8.5 }
862 },
863 - "secondsPer30s": { "cpu": 3.78, "accelerated": 2.8 }
863 + "secondsPer30s": { "cpu": 3.7759597230962867, "accelerated": 2.7958993476234855 }
864 },
865 "recommended": false,
866 "supportsStreaming": false,
@@ -879,7 +879,7 @@
879 "languages": ["ru"],
880 "benchmark": {
881 "accuracy": { "ru": { "grade": "B+", "errorPercent": 5.53 } },
882 - "secondsPer30s": { "cpu": 2.62, "accelerated": 0.85 }
882 + "secondsPer30s": { "cpu": 2.618943692710607, "accelerated": 0.8480565371024735 }
883 },
884 "recommended": false,
885 "supportsStreaming": false,
@@ -898,7 +898,7 @@
898 "languages": ["ru"],
899 "benchmark": {
900 "accuracy": { "ru": { "grade": "B+", "errorPercent": 5.35 } },
901 - "secondsPer30s": { "cpu": 3.81, "accelerated": 2 }
901 + "secondsPer30s": { "cpu": 3.8095238095238093, "accelerated": 1.9973368841544608 }
902 },
903 "recommended": false,
904 "supportsStreaming": false,
@@ -917,7 +917,7 @@
917 "languages": ["en"],
918 "benchmark": {
919 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.61 } },
920 - "secondsPer30s": { "cpu": 1.74, "accelerated": 1.31 }
920 + "secondsPer30s": { "cpu": 1.7401392111368912, "accelerated": 1.312048983162038 }
921 },
922 "recommended": false,
923 "supportsStreaming": false,
@@ -943,7 +943,7 @@
943 "pt": { "grade": "B-", "errorPercent": 9.8 },
944 "ja": { "grade": "B", "errorPercent": 6.3 }
945 },
946 - "secondsPer30s": { "cpu": 10.81, "accelerated": 6.5 }
946 + "secondsPer30s": { "cpu": 10.81081081081081, "accelerated": 6.500541711809317 }
947 },
948 "recommended": false,
949 "supportsStreaming": false,
@@ -962,7 +962,7 @@
962 "languages": ["en"],
963 "benchmark": {
964 "accuracy": { "en": { "grade": "F", "errorPercent": 37.48 } },
965 - "secondsPer30s": { "cpu": 1.16, "accelerated": 0.42 }
965 + "secondsPer30s": { "cpu": 1.1598685482312006, "accelerated": 0.4209935447656469 }
966 },
967 "recommended": false,
968 "supportsStreaming": false,
@@ -981,7 +981,7 @@
981 "languages": ["en"],
982 "benchmark": {
983 "accuracy": { "en": { "grade": "B+", "errorPercent": 6.11 } },
984 - "secondsPer30s": { "cpu": 0.6, "accelerated": 0.39 }
984 + "secondsPer30s": { "cpu": 0.5996402158704777, "accelerated": 0.39238767902687854 }
985 },
986 "recommended": false,
987 "supportsStreaming": false,
@@ -1000,7 +1000,7 @@
1000 "languages": ["en"],
1001 "benchmark": {
1002 "accuracy": { "en": { "grade": "B", "errorPercent": 6.52 } },
1003 - "secondsPer30s": { "cpu": 2.24, "accelerated": 1.14 }
1003 + "secondsPer30s": { "cpu": 2.236302646291465, "accelerated": 1.1387360030366294 }
1004 },
1005 "recommended": false,
1006 "supportsStreaming": true,
@@ -1081,7 +1081,7 @@
1081 "hu": { "grade": "F", "errorPercent": 32.84 },
1082 "mk": { "grade": "C-", "errorPercent": 18.22 }
1083 },
1084 - "secondsPer30s": { "cpu": 12.15, "accelerated": 7.34 }
1084 + "secondsPer30s": { "cpu": 12.14574898785425, "accelerated": 7.343941248470013 }
1085 },
1086 "recommended": false,
1087 "supportsStreaming": false,
@@ -1106,7 +1106,7 @@
1106 "ja": { "grade": "B", "errorPercent": 7.63 },
1107 "ko": { "grade": "B-", "errorPercent": 8.27 }
1108 },
1109 - "secondsPer30s": { "cpu": 1.52, "accelerated": 1.08 }
1109 + "secondsPer30s": { "cpu": 1.5201418799087916, "accelerated": 1.0834236186348862 }
1110 },
1111 "recommended": false,
1112 "supportsStreaming": false,
@@ -1134,7 +1134,7 @@
1134 "nl": { "grade": "B", "errorPercent": 6.57 },
1135 "hi": { "grade": "B-", "errorPercent": 8.93 }
1136 },
1137 - "secondsPer30s": { "cpu": 53.57, "accelerated": 27.65 }
1137 + "secondsPer30s": { "cpu": 53.57142857142857, "accelerated": 27.649769585253456 }
1138 },
1139 "recommended": false,
1140 "supportsStreaming": false,
@@ -1336,7 +1336,7 @@
1336 "yo": { "grade": "F", "errorPercent": 118.5 },
1337 "zh": { "grade": "D", "errorPercent": 23.06 }
1338 },
1339 - "secondsPer30s": { "cpu": 5.12, "accelerated": 2.57 }
1339 + "secondsPer30s": { "cpu": 5.1194539249146755, "accelerated": 2.5651988029072252 }
1340 },
1341 "recommended": false,
1342 "supportsStreaming": false,
@@ -1540,7 +1540,7 @@
1540 "yue": { "grade": "F", "errorPercent": 34.62 },
1541 "zh": { "grade": "B-", "errorPercent": 8.5 }
1542 },
1543 - "secondsPer30s": { "cpu": 20.98, "accelerated": 9.69 }
1543 + "secondsPer30s": { "cpu": 20.97902097902098, "accelerated": 9.693053311793214 }
1544 },
1545 "recommended": false,
1546 "supportsStreaming": false,
@@ -1562,7 +1562,7 @@
1562 "zh": { "grade": "B-", "errorPercent": 8.1 },
1563 "en": { "grade": "A-", "errorPercent": 4.14 }
1564 },
1565 - "secondsPer30s": { "cpu": 31.25, "accelerated": 14.85 }
1565 + "secondsPer30s": { "cpu": 31.25, "accelerated": 14.851485148514852 }
1566 },
1567 "recommended": false,
1568 "supportsStreaming": false,
@@ -1766,7 +1766,7 @@
1766 "yue": { "grade": "D", "errorPercent": 22.06 },
1767 "zh": { "grade": "B-", "errorPercent": 7.98 }
1768 },
1769 - "secondsPer30s": { "cpu": 30.93, "accelerated": 15.04 }
1769 + "secondsPer30s": { "cpu": 30.927835051546392, "accelerated": 15.037593984962406 }
1770 },
1771 "recommended": false,
1772 "supportsStreaming": false,
@@ -1790,7 +1790,7 @@
1790 "es": { "grade": "B", "errorPercent": 6.73 },
1791 "fr": { "grade": "B", "errorPercent": 7.22 }
1792 },
1793 - "secondsPer30s": { "cpu": 3.39, "accelerated": 1.94 }
1793 + "secondsPer30s": { "cpu": 3.3898305084745766, "accelerated": 1.937984496124031 }
1794 },
1795 "recommended": false,
1796 "supportsStreaming": false,
@@ -1809,7 +1809,7 @@
1809 "languages": ["ru"],
1810 "benchmark": {
1811 "accuracy": { "ru": { "grade": "B-", "errorPercent": 8.42 } },
1812 - "secondsPer30s": { "cpu": 2.54, "accelerated": 0.75 }
1812 + "secondsPer30s": { "cpu": 2.5369978858350954, "accelerated": 0.7543374402816192 }
1813 },
1814 "recommended": false,
1815 "supportsStreaming": false,
@@ -1828,7 +1828,7 @@
1828 "languages": ["ru"],
1829 "benchmark": {
1830 "accuracy": { "ru": { "grade": "B-", "errorPercent": 8.07 } },
1831 - "secondsPer30s": { "cpu": 2.9, "accelerated": 1.12 }
1831 + "secondsPer30s": { "cpu": 2.9013539651837523, "accelerated": 1.1219147344801796 }
1832 },
1833 "recommended": false,
1834 "supportsStreaming": false,
@@ -1854,7 +1854,7 @@
1854 "pt": { "grade": "B-", "errorPercent": 9.6 },
1855 "ja": { "grade": "B", "errorPercent": 6.44 }
1856 },
1857 - "secondsPer30s": { "cpu": 10.89, "accelerated": 6.47 }
1857 + "secondsPer30s": { "cpu": 10.88929219600726, "accelerated": 6.472491909385114 }
1858 },
1859 "recommended": false,
1860 "supportsStreaming": false,
@@ -1879,7 +1879,7 @@
1879 "es": { "grade": "A-", "errorPercent": 4.08 },
1880 "pt": { "grade": "B+", "errorPercent": 5.57 }
1881 },
1882 - "secondsPer30s": { "cpu": 13.99, "accelerated": 7.51 }
1882 + "secondsPer30s": { "cpu": 13.986013986013987, "accelerated": 7.509386733416771 }
1883 },
1884 "recommended": false,
1885 "supportsStreaming": false,
@@ -1904,7 +1904,7 @@
1904 "es": { "grade": "B", "errorPercent": 6.53 },
1905 "pt": { "grade": "C+", "errorPercent": 10.61 }
1906 },
1907 - "secondsPer30s": { "cpu": 11.54, "accelerated": 6.46 }
1907 + "secondsPer30s": { "cpu": 11.538461538461538, "accelerated": 6.458557588805167 }
1908 },
1909 "recommended": false,
1910 "supportsStreaming": false,
@@ -1923,7 +1923,7 @@
1923 "languages": ["en"],
1924 "benchmark": {
1925 "accuracy": { "en": { "grade": "C+", "errorPercent": 12.25 } },
1926 - "secondsPer30s": { "cpu": 1.28, "accelerated": 0.84 }
1926 + "secondsPer30s": { "cpu": 1.277139208173691, "accelerated": 0.8429334082607474 }
1927 },
1928 "recommended": false,
1929 "supportsStreaming": false,
@@ -1942,7 +1942,7 @@
1942 "languages": ["en"],
1943 "benchmark": {
1944 "accuracy": { "en": { "grade": "B", "errorPercent": 7.87 } },
1945 - "secondsPer30s": { "cpu": 3.1, "accelerated": 2.1 }
1945 + "secondsPer30s": { "cpu": 3.103983445421625, "accelerated": 2.100840336134454 }
1946 },
1947 "recommended": false,
1948 "supportsStreaming": true,
@@ -1961,7 +1961,7 @@
1961 "languages": ["en"],
1962 "benchmark": {
1963 "accuracy": { "en": { "grade": "B-", "errorPercent": 8.55 } },
1964 - "secondsPer30s": { "cpu": 2.22, "accelerated": 1.38 }
1964 + "secondsPer30s": { "cpu": 2.2164758034724787, "accelerated": 1.3780431786862655 }
1965 },
1966 "recommended": false,
1967 "supportsStreaming": true,
@@ -1980,7 +1980,7 @@
1980 "languages": ["en"],
1981 "benchmark": {
1982 "accuracy": { "en": { "grade": "C-", "errorPercent": 18.18 } },
1983 - "secondsPer30s": { "cpu": 0.53, "accelerated": 0.51 }
1983 + "secondsPer30s": { "cpu": 0.5331437711036076, "accelerated": 0.5071851225697379 }
1984 },
1985 "recommended": false,
1986 "supportsStreaming": true,
@@ -1999,7 +1999,7 @@
1999 "languages": ["en"],
2000 "benchmark": {
2001 "accuracy": { "en": { "grade": "C", "errorPercent": 14.13 } },
2002 - "secondsPer30s": { "cpu": 0.73, "accelerated": 0.5 }
2002 + "secondsPer30s": { "cpu": 0.7260406582768635, "accelerated": 0.4964421644878372 }
2003 },
2004 "recommended": false,
2005 "supportsStreaming": false,
@@ -2021,7 +2021,7 @@
2021 "en": { "grade": "B+", "errorPercent": 5.13 },
2022 "zh": { "grade": "B-", "errorPercent": 9.23 }
2023 },
2024 - "secondsPer30s": { "cpu": 19.11, "accelerated": 9.79 }
2024 + "secondsPer30s": { "cpu": 19.108280254777068, "accelerated": 9.787928221859707 }
2025 },
2026 "recommended": false,
2027 "supportsStreaming": false,
@@ -2040,7 +2040,7 @@
2040 "languages": ["en"],
2041 "benchmark": {
2042 "accuracy": { "en": { "grade": "B", "errorPercent": 6.43 } },
2043 - "secondsPer30s": { "cpu": 2.21, "accelerated": 1.11 }
2043 + "secondsPer30s": { "cpu": 2.210759027266028, "accelerated": 1.1051759071652238 }
2044 },
2045 "recommended": false,
2046 "supportsStreaming": true,
@@ -2111,7 +2111,7 @@
2111 "sv": { "grade": "C", "errorPercent": 13.26 },
2112 "uk": { "grade": "B", "errorPercent": 6.53 }
2113 },
2114 - "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
2114 + "secondsPer30s": { "cpu": 2.2026431718061676, "accelerated": 1.1954572624028692 }
2115 },
2116 "recommended": false,
2117 "supportsStreaming": false,
@@ -2130,7 +2130,7 @@
2130 "languages": ["en"],
2131 "benchmark": {
2132 "accuracy": { "en": { "grade": "B+", "errorPercent": 5.53 } },
2133 - "secondsPer30s": { "cpu": 1.98, "accelerated": 1.05 }
2133 + "secondsPer30s": { "cpu": 1.9788918205804749, "accelerated": 1.048951048951049 }
2134 },
2135 "recommended": false,
2136 "supportsStreaming": false,
@@ -2149,7 +2149,7 @@
2149 "languages": ["en"],
2150 "benchmark": {
2151 "accuracy": { "en": { "grade": "B+", "errorPercent": 5.61 } },
2152 - "secondsPer30s": { "cpu": 3.36, "accelerated": 1.79 }
2152 + "secondsPer30s": { "cpu": 3.3613445378151257, "accelerated": 1.7873100983020553 }
2153 },
2154 "recommended": false,
2155 "supportsStreaming": false,
@@ -2220,7 +2220,7 @@
2220 "sv": { "grade": "C-", "errorPercent": 16.42 },
2221 "uk": { "grade": "B-", "errorPercent": 8.11 }
2222 },
2223 - "secondsPer30s": { "cpu": 2.2, "accelerated": 1.2 }
2223 + "secondsPer30s": { "cpu": 2.2002200220022003, "accelerated": 1.2014417300760913 }
2224 },
2225 "recommended": false,
2226 "supportsStreaming": false,
@@ -2239,7 +2239,7 @@
2239 "languages": ["en"],
2240 "benchmark": {
2241 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.57 } },
2242 - "secondsPer30s": { "cpu": 2.02, "accelerated": 1.11 }
2242 + "secondsPer30s": { "cpu": 2.0208824520040416, "accelerated": 1.107011070110701 }
2243 },
2244 "recommended": false,
2245 "supportsStreaming": false,
@@ -2258,7 +2258,7 @@
2258 "languages": ["en"],
2259 "benchmark": {
2260 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.45 } },
2261 - "secondsPer30s": { "cpu": 3.49, "accelerated": 1.87 }
2261 + "secondsPer30s": { "cpu": 3.492433061699651, "accelerated": 1.8650917003419334 }
2262 },
2263 "recommended": false,
2264 "supportsStreaming": false,
@@ -2277,7 +2277,7 @@
2277 "languages": ["en"],
2278 "benchmark": {
2279 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.68 } },
2280 - "secondsPer30s": { "cpu": 3.22, "accelerated": 1.76 }
2280 + "secondsPer30s": { "cpu": 3.2206119162640903, "accelerated": 1.7605633802816902 }
2281 },
2282 "recommended": false,
2283 "supportsStreaming": false,
@@ -2296,7 +2296,7 @@
2296 "languages": ["en"],
2297 "benchmark": {
2298 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.24 } },
2299 - "secondsPer30s": { "cpu": 3.41, "accelerated": 1.82 }
2299 + "secondsPer30s": { "cpu": 3.4071550255536627, "accelerated": 1.8242626938279112 }
2300 },
2301 "recommended": false,
2302 "supportsStreaming": false,
@@ -2526,7 +2526,7 @@
2526 "yo": { "grade": "F", "errorPercent": 103.28 },
2527 "zh": { "grade": "F", "errorPercent": 36.21 }
2528 },
2529 - "secondsPer30s": { "cpu": 1.55, "accelerated": 1.03 }
2529 + "secondsPer30s": { "cpu": 1.5519917227108124, "accelerated": 1.033947957952783 }
2530 },
2531 "recommended": false,
2532 "supportsStreaming": false,
@@ -2545,7 +2545,7 @@
2545 "languages": ["en"],
2546 "benchmark": {
2547 "accuracy": { "en": { "grade": "B", "errorPercent": 7.6 } },
2548 - "secondsPer30s": { "cpu": 1.46, "accelerated": 0.94 }
2548 + "secondsPer30s": { "cpu": 1.4563106796116503, "accelerated": 0.9370607527721381 }
2549 },
2550 "recommended": false,
2551 "supportsStreaming": false,
@@ -2747,7 +2747,7 @@
2747 "yo": { "grade": "F", "errorPercent": 113.72 },
2748 "zh": { "grade": "C-", "errorPercent": 19.32 }
2749 },
2750 - "secondsPer30s": { "cpu": 31.25, "accelerated": 14.96 }
2750 + "secondsPer30s": { "cpu": 31.25, "accelerated": 14.962593516209477 }
2751 },
2752 "recommended": false,
2753 "supportsStreaming": false,
@@ -2949,7 +2949,7 @@
2949 "yo": { "grade": "F", "errorPercent": 96 },
2950 "zh": { "grade": "C", "errorPercent": 15.39 }
2951 },
2952 - "secondsPer30s": { "cpu": 31.25, "accelerated": 14.93 }
2952 + "secondsPer30s": { "cpu": 31.25, "accelerated": 14.92537313432836 }
2953 },
2954 "recommended": false,
2955 "supportsStreaming": false,
@@ -2968,7 +2968,7 @@
2968 "languages": ["en"],
2969 "benchmark": {
2970 "accuracy": { "en": { "grade": "A-", "errorPercent": 4.88 } },
2971 - "secondsPer30s": { "cpu": 14.53, "accelerated": 6.81 }
2971 + "secondsPer30s": { "cpu": 14.527845036319613, "accelerated": 6.81044267877412 }
2972 },
2973 "recommended": false,
2974 "supportsStreaming": false,
@@ -2987,7 +2987,7 @@
2987 "languages": ["en"],
2988 "benchmark": {
2989 "accuracy": { "en": { "grade": "B+", "errorPercent": 6.14 } },
2990 - "secondsPer30s": { "cpu": 4.79, "accelerated": 2.4 }
2990 + "secondsPer30s": { "cpu": 4.7923322683706076, "accelerated": 2.404809619238477 }
2991 },
2992 "recommended": false,
2993 "supportsStreaming": false,
@@ -3189,7 +3189,7 @@
3189 "yo": { "grade": "F", "errorPercent": 102.29 },
3190 "zh": { "grade": "F", "errorPercent": 41.91 }
3191 },
3192 - "secondsPer30s": { "cpu": 0.75, "accelerated": 0.7 }
3192 + "secondsPer30s": { "cpu": 0.7546220601182241, "accelerated": 0.696217219772569 }
3193 },
3194 "recommended": false,
3195 "supportsStreaming": false,
@@ -3208,7 +3208,7 @@
3208 "languages": ["en"],
3209 "benchmark": {
3210 "accuracy": { "en": { "grade": "C+", "errorPercent": 10.72 } },
3211 - "secondsPer30s": { "cpu": 0.73, "accelerated": 0.63 }
3211 + "secondsPer30s": { "cpu": 0.727802037845706, "accelerated": 0.6318449873631002 }
3212 },
3213 "recommended": false,
3214 "supportsStreaming": false,
apps/web/src/components/settings/SpeechModelComparison.tsx
+19 -14
@@ -51,12 +51,12 @@ export function ModelBenchmarks({
51 }: {
52 model: EnvironmentSpeechModel;
53 languages: readonly string[];
54 - tier: SpeechBenchmarkTier;
54 + tier: SpeechBenchmarkTier | undefined;
55 /** Plain facts, such as language count and size, shown after the scores. */
56 facts: readonly string[];
57 }) {
58 const data = model.benchmark;
59 - const seconds = data?.secondsPer30s[tier];
59 + const seconds = tier === undefined ? undefined : data?.secondsPer30s[tier];
60 const visible = !data
61 ? []
62 : languages.length
@@ -82,18 +82,23 @@ export function ModelBenchmarks({
82 </span>
83 );
84 })}
85 - <span>
86 - Speed:{" "}
87 - {tier === "cpu" && data && data.secondsPer30s.cpu === undefined ? (
88 - "update server"
89 - ) : seconds == null ? (
90 - "not benchmarked"
91 - ) : (
92 - <>
93 - <strong className="font-medium text-foreground">{seconds} s</strong> / 30 s
94 - </>
95 - )}
96 - </span>
85 + {data ? (
86 + <span>
87 + Speed:{" "}
88 + {seconds === undefined ? (
89 + "update server for this benchmark"
90 + ) : seconds === null ? (
91 + "not benchmarked"
92 + ) : (
93 + <>
94 + <strong className="font-medium text-foreground">
95 + {Number(seconds.toFixed(2))} s
96 + </strong>{" "}
97 + / 30 s
98 + </>
99 + )}
100 + </span>
101 + ) : null}
102 {facts.map((fact) => (
103 <span key={fact}>{fact}</span>
104 ))}
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+125 -78
@@ -148,7 +148,7 @@ it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accel
148 secondsPer30s,
149 },
150 };
151 - const cpuBenchmark = `Speed: ${secondsPer30s.cpu === undefined ? "update server" : secondsPer30s.cpu === null ? "not benchmarked" : `${secondsPer30s.cpu} s / 30 s`}`;
151 + const cpuBenchmark = `Speed: ${secondsPer30s.cpu === undefined ? "update server for this benchmark" : secondsPer30s.cpu === null ? "not benchmarked" : `${secondsPer30s.cpu} s / 30 s`}`;
152 const gpuBenchmark = `Speed: ${secondsPer30s.accelerated === null ? "not benchmarked" : `${secondsPer30s.accelerated} s / 30 s`}`;
153 const fetch = globalThis.fetch;
154 using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
@@ -256,9 +256,9 @@ it("tests the selected model after download and preserves its transcript across
256 supportsStreaming: false,
257 supportsTranslation: false,
258 acceleration: "cpu",
259 + benchmarkTier: "cpu",
260 modelUnloadTimeout: "never",
261 gpuDevices: [],
261 - benchmarkTier: "cpu",
262 } satisfies EnvironmentSpeechStatus;
263 let installed = false;
264 const fetch = globalThis.fetch;
@@ -290,7 +290,7 @@ it("tests the selected model after download and preserves its transcript across
290 en: { grade: "A-", errorPercent: 4 },
291 fr: { grade: "B+", errorPercent: 5.5 },
292 },
293 - secondsPer30s: { cpu: 5, accelerated: 2.1 },
293 + secondsPer30s: { accelerated: 2.1, cpu: 5 },
294 },
295 }
296 : {}),
@@ -387,81 +387,128 @@ it("tests the selected model after download and preserves its transcript across
387 );
388 });
389
390 -it("loads the models of a relay-connected destination", async () => {
391 - const target = new RelayConnectionTarget({
392 - environmentId: EnvironmentId.make("voice-relay"),
393 - label: "MacBook Pro",
394 - });
395 - const prepared = {
396 - environmentId: target.environmentId,
397 - label: target.label,
398 - httpBaseUrl: window.location.origin,
399 - socketUrl: "ws://localhost",
400 - httpAuthorization: {
401 - _tag: "Dpop" as const,
402 - accessToken: "stale-access-token",
403 - expiresAtEpochMs: 4_102_444_800_000,
404 - },
405 - target,
406 - };
407 - const model: EnvironmentSpeechModel = {
408 - id: "tiny",
409 - name: "Tiny",
410 - description: "",
411 - size: 100,
412 - languages: ["en"],
413 - accuracy: 1,
414 - speed: 1,
415 - recommended: false,
416 - supportsStreaming: false,
417 - supportsTranslation: false,
418 - supportsLanguageDetection: false,
419 - active: false,
420 - state: "downloadable",
421 - };
422 - const status = {
423 - supported: true,
424 - state: "missing-model",
425 - modelId: model.id,
426 - model: model.name,
427 - size: model.size,
428 - supportsStreaming: false,
429 - supportsTranslation: false,
430 - acceleration: "cpu",
431 - modelUnloadTimeout: "never",
432 - gpuDevices: [],
433 - benchmarkTier: "cpu",
434 - } satisfies EnvironmentSpeechStatus;
435 - const authorizations: Array<string | null> = [];
436 - const fetch = globalThis.fetch;
437 - using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
438 - const url = new URL(input instanceof Request ? input.url : input, window.location.href);
439 - if (url.pathname === "/api/voice/status") {
440 - authorizations.push(new Headers(init?.headers).get("authorization"));
441 - return Promise.resolve(Response.json(status));
390 +it.each([
391 + { tier: "cpu", legacy: false },
392 + { tier: "accelerated", legacy: false },
393 + { tier: "cpu", legacy: true },
394 +] as const)(
395 + "loads relay models using the server tier $tier (legacy: $legacy)",
396 + async ({ tier, legacy }) => {
397 + const target = new RelayConnectionTarget({
398 + environmentId: EnvironmentId.make("voice-relay"),
399 + label: "MacBook Pro",
400 + });
401 + const prepared = {
402 + environmentId: target.environmentId,
403 + label: target.label,
404 + httpBaseUrl: window.location.origin,
405 + socketUrl: "ws://localhost",
406 + httpAuthorization: {
407 + _tag: "Dpop" as const,
408 + accessToken: "stale-access-token",
409 + expiresAtEpochMs: 4_102_444_800_000,
410 + },
411 + target,
412 + };
413 + const model = {
414 + id: "gpu",
415 + name: "GPU pick",
416 + description: "",
417 + size: 100,
418 + languages: ["en"],
419 + benchmark: {
420 + accuracy: { en: { grade: "A", errorPercent: 4 } },
421 + secondsPer30s: { cpu: 20, accelerated: 1 },
422 + },
423 + accuracy: 1,
424 + speed: 1,
425 + recommended: false,
426 + supportsStreaming: false,
427 + supportsTranslation: false,
428 + supportsLanguageDetection: false,
429 + active: false,
430 + state: "downloadable",
431 + } satisfies EnvironmentSpeechModel;
432 + const status = {
433 + supported: true,
434 + state: "missing-model",
435 + modelId: model.id,
436 + model: model.name,
437 + size: model.size,
438 + supportsStreaming: false,
439 + supportsTranslation: false,
440 + acceleration: "auto",
441 + benchmarkTier: tier,
442 + modelUnloadTimeout: "never",
443 + gpuDevices: [{ id: "remote-gpu", name: "Remote GPU" }],
444 + } satisfies EnvironmentSpeechStatus;
445 + const cpuModel = {
446 + ...model,
447 + id: "cpu",
448 + name: "CPU pick",
449 + benchmark: {
450 + accuracy: { en: { grade: "B+", errorPercent: 6 } },
451 + secondsPer30s: { cpu: 1, accelerated: 8 },
452 + },
453 + };
454 + const { benchmarkTier: _tier, ...legacyStatus } = status;
455 + const models = [model, cpuModel];
456 + const authorizations: Array<string | null> = [];
457 + const fetch = globalThis.fetch;
458 + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
459 + const url = new URL(input instanceof Request ? input.url : input, window.location.href);
460 + if (url.pathname === "/api/voice/status") {
461 + authorizations.push(new Headers(init?.headers).get("authorization"));
462 + return Promise.resolve(
463 + Response.json(legacy ? { ...legacyStatus, acceleration: "cpu", gpuDevices: [] } : status),
464 + );
465 + }
466 + if (url.pathname === "/api/voice/models")
467 + return Promise.resolve(
468 + Response.json({
469 + models: legacy
470 + ? models.map(({ benchmark, ...entry }) => ({
471 + ...entry,
472 + benchmark: {
473 + accuracy: benchmark.accuracy,
474 + secondsPer30s: benchmark.secondsPer30s.accelerated,
475 + },
476 + }))
477 + : models,
478 + }),
479 + );
480 + return fetch(input, init);
481 + });
482 + using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
483 + // Restored even when an assertion fails, unlike `withImplementation`.
484 + using _destination = vi.mocked(useVoiceTranscriptionDestination).mockImplementation(() => ({
485 + environmentId: target.environmentId,
486 + label: target.label,
487 + grantId: "test",
488 + prepared,
489 + }));
490 + await using screen = await render(
491 + <SettingsScopeProvider search={{}} onChange={() => {}}>
492 + <VoiceSettingsPanel />
493 + </SettingsScopeProvider>,
494 + );
495 + await vi.waitFor(() => expect(authorizations.length + notify.mock.calls.length).not.toBe(0));
496 + expect(notify).not.toHaveBeenCalled();
497 + expect(authorizations).toEqual(["DPoP relay-access-token"]);
498 + const region = screen.getByRole("region", { name: "Transcription models" });
499 + await vi.waitFor(() => expect(region.element().textContent).toContain("CPU pick"));
500 + if (legacy) {
501 + expect(region.element().textContent).toContain("update server for this benchmark");
502 + expect(region.element().textContent).not.toContain("Recommended");
503 + } else {
504 + const winner = tier === "cpu" ? "CPU pick" : "GPU pick";
505 + await vi.waitFor(() =>
506 + expect(region.element().firstElementChild?.textContent).toContain(winner + "Recommended"),
507 + );
508 + expect(region.element().textContent).toContain("Speed: 1 s / 30 s");
509 }
443 - if (url.pathname === "/api/voice/models")
444 - return Promise.resolve(Response.json({ models: [model] }));
445 - return fetch(input, init);
446 - });
447 - using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
448 - // Restored even when an assertion fails, unlike `withImplementation`.
449 - using _destination = vi.mocked(useVoiceTranscriptionDestination).mockImplementation(() => ({
450 - environmentId: target.environmentId,
451 - label: target.label,
452 - grantId: "test",
453 - prepared,
454 - }));
455 - await using screen = await render(
456 - <SettingsScopeProvider search={{}} onChange={() => {}}>
457 - <VoiceSettingsPanel />
458 - </SettingsScopeProvider>,
459 - );
460 - await vi.waitFor(() => expect(authorizations.length + notify.mock.calls.length).not.toBe(0));
461 - expect(notify).not.toHaveBeenCalled();
462 - expect(authorizations).toEqual(["DPoP relay-access-token"]);
463 - await expect.element(screen.getByText("Tiny", { exact: true })).toBeVisible();
464 -});
510 + },
511 +);
512
513 it("recovers from a failed download progress poll without notifying, but reports a failed initial load", async () => {
514 const target = new PrimaryConnectionTarget({
@@ -495,9 +542,9 @@ it("recovers from a failed download progress poll without notifying, but reports
542 supportsStreaming: false,
543 supportsTranslation: false,
544 acceleration: "cpu",
545 + benchmarkTier: "cpu",
546 modelUnloadTimeout: "never",
547 gpuDevices: [],
500 - benchmarkTier: "cpu",
548 };
549 const unavailable = () => new Response(null, { status: 503 });
550 const readModels = vi
apps/web/src/components/settings/VoiceSettingsPanel.tsx
+38 -14
@@ -30,7 +30,11 @@ import {
30 } from "@t3tools/contracts";
31 import { createModelSelection, pickModelSelection } from "@t3tools/shared/model";
32 import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings";
33 -import { cloudSpeechLanguages } from "@t3tools/shared/speech";
33 +import {
34 + cloudSpeechLanguages,
35 + recommendSpeechModels,
36 + type SpeechRecommendationRole,
37 +} from "@t3tools/shared/speech";
38 import {
39 ArrowDownUpIcon,
40 AudioLinesIcon,
@@ -176,7 +180,8 @@ const spellingsOf = (words: SpeechCustomWords) =>
180 type ModelCardProps = {
181 readonly model: EnvironmentSpeechModel;
182 readonly comparisonLanguages: readonly string[];
179 - readonly benchmarkTier: SpeechBenchmarkTier;
183 + readonly recommendation: SpeechRecommendationRole | undefined;
184 + readonly benchmarkTier: SpeechBenchmarkTier | undefined;
185 readonly busy: boolean;
186 readonly canCancel: boolean;
187 readonly onDownload: () => void;
@@ -217,11 +222,23 @@ function ModelCard(props: ModelCardProps) {
222 <div className="min-w-0 flex-1">
223 <div className="flex flex-wrap items-center gap-1.5">
224 <span className="text-sm font-medium leading-5">{model.name}</span>
220 - {model.recommended ? (
225 + {props.recommendation ? (
226 <ModelBadge
227 icon={<StarIcon className="mr-1 size-3" />}
223 - label="Recommended"
224 - tooltip="A good default pick for the languages it covers."
228 + label={
229 + props.recommendation === "recommended"
230 + ? "Recommended"
231 + : props.recommendation === "faster"
232 + ? "Faster"
233 + : "More accurate"
234 + }
235 + tooltip={
236 + props.recommendation === "recommended"
237 + ? "Balanced accuracy and speed for your selected languages on Handy’s reference benchmarks."
238 + : props.recommendation === "faster"
239 + ? "Quicker than Recommended on this reference tier, with useful accuracy."
240 + : "Higher accuracy with a longer reference processing budget for your selected languages."
241 + }
242 />
243 ) : null}
244 {model.supportsStreaming ? (
@@ -871,14 +888,20 @@ function SpeechModelBrowser(props: {
888 selectedLanguages.length === 0
889 ? "All languages"
890 : selectedLanguages.map(languageLabel).join(", ");
874 - const sortedModels = filterSpeechModels(models, {
875 - search,
876 - languages: selectedLanguages,
877 - downloadedOnly,
878 - sort,
879 - accuracyLanguage,
880 - benchmarkTier: status?.supported ? status.benchmarkTier : "accelerated",
881 - });
891 + const benchmarkTier = status?.supported ? status.benchmarkTier : undefined;
892 + const recommendations = recommendSpeechModels(models, selectedLanguages, benchmarkTier);
893 + const sortedModels = filterSpeechModels(
894 + models,
895 + {
896 + search,
897 + languages: selectedLanguages,
898 + downloadedOnly,
899 + sort,
900 + accuracyLanguage,
901 + benchmarkTier,
902 + },
903 + recommendations,
904 + );
905 const activeModel = models.find((model) => model.active);
906 const renderedModels =
907 activeModel && !sortedModels.includes(activeModel)
@@ -898,7 +921,7 @@ function SpeechModelBrowser(props: {
921 <SettingsRow
922 {...searchableSetting("speech-models")}
923 title="Models"
901 - description={`Grades are error rates and speed is seconds per 30 s of audio, from Handy’s benchmark ${status.benchmarkTier === "cpu" ? "CPU" : "GPU"}.`}
924 + description={`Grades and seconds per 30 s use Handy’s reference ${status.benchmarkTier === "cpu" ? "CPU" : "GPU"}, not your hardware. Select languages for balanced recommendations.`}
925 className="@container/voice-models"
926 >
927 <div className="mt-3 flex min-w-0 gap-2">
@@ -1018,6 +1041,7 @@ function SpeechModelBrowser(props: {
1041 busy: controlsDisabled,
1042 canCancel,
1043 comparisonLanguages: selectedLanguages,
1044 + recommendation: recommendations.get(model.id),
1045 benchmarkTier: status.benchmarkTier,
1046 onDownload: () =>
1047 void run(
apps/web/src/components/settings/speechModelBrowser.test.ts
+60 -4
@@ -1,5 +1,6 @@
1 import type { EnvironmentSpeechModel } from "@t3tools/contracts";
2 import { expect, it } from "vite-plus/test";
3 +import { recommendSpeechModels } from "@t3tools/shared/speech";
4
5 import { filterSpeechModels } from "./speechModelBrowser";
6
@@ -31,9 +32,9 @@ const models: readonly EnvironmentSpeechModel[] = [
32 benchmark: {
33 accuracy: {
34 en: { grade: "A", errorPercent: 3.8 },
34 - fr: { grade: "B", errorPercent: 8 },
35 + fr: { grade: "B", errorPercent: 7 },
36 },
36 - secondsPer30s: { cpu: 2, accelerated: 2 },
37 + secondsPer30s: { accelerated: 2, cpu: 1 },
38 },
39 },
40 {
@@ -47,7 +48,7 @@ const models: readonly EnvironmentSpeechModel[] = [
48 en: { grade: "A", errorPercent: 3.2 },
49 fr: { grade: "B", errorPercent: 9 },
50 },
50 - secondsPer30s: { cpu: 1, accelerated: 1 },
51 + secondsPer30s: { accelerated: 1, cpu: 3 },
52 },
53 },
54 {
@@ -58,7 +59,7 @@ const models: readonly EnvironmentSpeechModel[] = [
59 active: false,
60 state: "verifying",
61 languages: ["en"],
61 - benchmark: { accuracy: {}, secondsPer30s: { cpu: null, accelerated: null } },
62 + benchmark: { accuracy: {}, secondsPer30s: { accelerated: null, cpu: null } },
63 },
64 ];
65
@@ -100,3 +101,58 @@ it("combines text, all selected languages, and completed downloads without retai
101 ).toEqual(["fast"]);
102 expect(filterSpeechModels(models, { ...options, search: "small" })).toEqual([]);
103 });
104 +
105 +it("ranks language recommendations before the rest without changing explicit sorts or search badges", () => {
106 + const catalog = models.map((model) => ({
107 + ...model,
108 + active: false,
109 + state: "downloadable" as const,
110 + }));
111 + const options = {
112 + search: "",
113 + languages: ["en"],
114 + downloadedOnly: false,
115 + sort: "recommended",
116 + accuracyLanguage: "en",
117 + benchmarkTier: "accelerated",
118 + } as const;
119 + expect(filterSpeechModels(catalog, options).map(({ id }) => id)).toEqual([
120 + "fast",
121 + "active",
122 + "small",
123 + "pending",
124 + ]);
125 + expect(
126 + filterSpeechModels(catalog, { ...options, languages: ["en", "fr"] }).map(({ id }) => id),
127 + ).toEqual(["small", "fast", "active"]);
128 + expect(filterSpeechModels(catalog, { ...options, languages: [] }).map(({ id }) => id)).toEqual(
129 + catalog.map(({ id }) => id),
130 + );
131 + expect(filterSpeechModels(catalog, { ...options, sort: "size" }).map(({ id }) => id)).toEqual([
132 + "small",
133 + "fast",
134 + "active",
135 + "pending",
136 + ]);
137 + expect(
138 + filterSpeechModels(catalog, { ...options, sort: "speed", benchmarkTier: "cpu" }).map(
139 + ({ id }) => id,
140 + ),
141 + ).toEqual(["small", "fast", "active", "pending"]);
142 + expect(
143 + filterSpeechModels(catalog, { ...options, sort: "accuracy", languages: ["en", "fr"] }).map(
144 + ({ id }) => id,
145 + ),
146 + ).toEqual(["fast", "small", "active"]);
147 + expect(
148 + filterSpeechModels(catalog, { ...options, sort: "speed", languages: ["en", "fr"] }).map(
149 + ({ id }) => id,
150 + ),
151 + ).toEqual(["fast", "small", "active"]);
152 + expect(filterSpeechModels(catalog, { ...options, search: "small" }).map(({ id }) => id)).toEqual([
153 + "small",
154 + ]);
155 + expect(
156 + recommendSpeechModels(catalog, options.languages, options.benchmarkTier).has("small"),
157 + ).toBe(false);
158 +});
apps/web/src/components/settings/speechModelBrowser.ts
+9 -4
@@ -1,4 +1,5 @@
1 import type { EnvironmentSpeechModel, SpeechBenchmarkTier } from "@t3tools/contracts";
2 +import { recommendSpeechModels } from "@t3tools/shared/speech";
3
4 export type SpeechModelSort = "recommended" | "size" | "accuracy" | "speed";
5
@@ -6,7 +7,7 @@ const modelSortOrder = (model: EnvironmentSpeechModel) => {
7 if (model.active) return 0;
8 if (model.state === "installed") return 1;
9 if (model.state === "downloading" || model.state === "verifying") return 2;
9 - return model.recommended ? 3 : 4;
10 + return 3;
11 };
12
13 export function filterSpeechModels(
@@ -17,10 +18,12 @@ export function filterSpeechModels(
18 readonly downloadedOnly: boolean;
19 readonly sort: SpeechModelSort;
20 readonly accuracyLanguage: string;
20 - readonly benchmarkTier: SpeechBenchmarkTier;
21 + readonly benchmarkTier: SpeechBenchmarkTier | undefined;
22 },
23 + recommendations = recommendSpeechModels(models, options.languages, options.benchmarkTier),
24 ) {
25 const query = options.search.trim().toLocaleLowerCase();
26 + const ranks = new Map([...recommendations.keys()].map((id, index) => [id, index]));
27 const score = (model: EnvironmentSpeechModel) => {
28 switch (options.sort) {
29 case "size":
@@ -28,9 +31,11 @@ export function filterSpeechModels(
31 case "accuracy":
32 return model.benchmark?.accuracy[options.accuracyLanguage]?.errorPercent ?? Infinity;
33 case "speed":
31 - return model.benchmark?.secondsPer30s[options.benchmarkTier] ?? Infinity;
34 + return options.benchmarkTier === undefined
35 + ? Infinity
36 + : (model.benchmark?.secondsPer30s[options.benchmarkTier] ?? Infinity);
37 case "recommended":
33 - return modelSortOrder(model);
38 + return modelSortOrder(model) * 4 + (ranks.get(model.id) ?? 3);
39 }
40 };
41 return models
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+37 -10
@@ -42,6 +42,7 @@ const mocks = vi.hoisted(() => ({
42 primaryEnvironmentId: "primary-env",
43 statusRequests: [] as string[],
44 statusUnavailable: false,
45 + statusTier: "accelerated" as "cpu" | "accelerated",
46 modelState: "ready" as "ready" | "missing-model",
47 statusModelId: "test-model",
48 models: [] as EnvironmentSpeechModel[],
@@ -134,9 +135,9 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
135 supportsStreaming: false,
136 supportsTranslation: false,
137 acceleration: "auto",
138 + benchmarkTier: mocks.statusTier,
139 modelUnloadTimeout: "min_15",
140 gpuDevices: [],
139 - benchmarkTier: "cpu",
141 };
142 }),
143 downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
@@ -151,10 +152,14 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
152 mocks.statusModelId = modelId;
153 return { models: mocks.models };
154 }),
154 - getEnvironmentSpeechModels: vi.fn(() =>
155 + getEnvironmentSpeechModels: vi.fn((_prepared: PreparedConnection, includeBenchmarks = false) =>
156 Effect.sync(() => {
157 if (mocks.catalogOffline) throw new Error("Catalog offline");
157 - return { models: mocks.models };
158 + return {
159 + models: mocks.models.map(({ benchmark, ...model }) =>
160 + includeBenchmarks ? { ...model, ...(benchmark ? { benchmark } : {}) } : model,
161 + ),
162 + };
163 }),
164 ),
165 postProcessEnvironmentTranscript: (
@@ -225,6 +230,7 @@ function setup(options: { readonly destination?: string | null } = {}) {
230 mocks.statusRequests = [];
231 mocks.statusUnavailable = false;
232 mocks.modelState = "ready";
233 + mocks.statusTier = "accelerated";
234 mocks.statusModelId = "test-model";
235 mocks.models = [];
236 mocks.catalogOffline = false;
@@ -491,6 +497,7 @@ function catalogModel(
497 id: string,
498 languages: readonly string[],
499 recommended: boolean,
500 + errorPercent = 4,
501 ): EnvironmentSpeechModel {
502 return {
503 id,
@@ -501,6 +508,12 @@ function catalogModel(
508 accuracy: 0,
509 speed: 0,
510 recommended,
511 + benchmark: {
512 + accuracy: Object.fromEntries(
513 + languages.map((language) => [language, { grade: "A", errorPercent }]),
514 + ),
515 + secondsPer30s: { accelerated: 2, cpu: 3 },
516 + },
517 supportsStreaming: false,
518 supportsLanguageDetection: languages.length > 1,
519 supportsTranslation: false,
@@ -510,26 +523,30 @@ function catalogModel(
523 }
524
525 it.each([
513 - ["the browser prefers German", ["de-DE"], undefined, "auto"],
514 - ["the desktop exposes no locale methods", ["de-DE"], {}, "auto"],
526 + ["the browser prefers German", ["de-DE"], undefined, "auto", "accelerated"],
527 + ["the remote environment uses CPU", ["de-DE"], undefined, "auto", "cpu"],
528 + ["the desktop exposes no locale methods", ["de-DE"], {}, "auto", "accelerated"],
529 [
530 "an older preload exposes only the OS locale",
531 ["en-US"],
532 { getSystemLocale: () => "de-DE" },
533 "auto",
534 + "accelerated",
535 ],
536 [
537 "the desktop OS prefers German",
538 ["en-US"],
539 { getPreferredSystemLanguages: () => ["de-DE"] },
540 "auto",
541 + "accelerated",
542 ],
527 - ["German is the chosen Voice language", ["en-US"], undefined, "de"],
543 + ["German is the chosen Voice language", ["en-US"], undefined, "de", "accelerated"],
544 ])(
545 "offers and downloads a German-capable model when %s",
530 - async (_, languages, bridge, speechLanguage) => {
546 + async (_, languages, bridge, speechLanguage, tier) => {
547 setup({ destination: "env-a" });
548 mocks.settings = { speechLanguage };
549 + mocks.statusTier = tier as "cpu" | "accelerated";
550 using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(languages);
551 using _bridge = stubGlobal("desktopBridge", bridge);
552 const englishDefault = "handy-computer/parakeet-unified-en-0.6b-gguf";
@@ -538,16 +555,24 @@ it.each([
555 mocks.statusModelId = englishDefault;
556 mocks.models = [
557 { ...catalogModel(englishDefault, ["en"], true), active: true },
541 - catalogModel(german, ["en", "de", "fr"], true),
558 catalogModel("handy-computer/canary-180m-flash-gguf", ["en", "de", "es", "fr"], true),
559 + {
560 + ...catalogModel(german, ["en", "de", "fr"], false, 2),
561 + benchmark: {
562 + accuracy: { de: { grade: "A", errorPercent: 2 } },
563 + secondsPer30s: { cpu: 20, accelerated: 2 },
564 + },
565 + },
566 ];
567 mocks.download.mockResolvedValue(undefined);
568 await using _screen = await render(<Probe />);
569 await act(() => mocks.voice.start());
570 expect(mocks.voice.setup.open).toBe(true);
571 await act(() => mocks.voice.setup.download());
549 - expect(mocks.download).toHaveBeenCalledExactlyOnceWith(german);
550 - expect(mocks.voice.setup.model?.id).toBe(german);
572 + const expected = tier === "cpu" ? "handy-computer/canary-180m-flash-gguf" : german;
573 + expect(mocks.download).toHaveBeenCalledExactlyOnceWith(expected);
574 + expect(mocks.voice.setup.model?.id).toBe(expected);
575 + expect(mocks.voice.setup.model?.recommended).toBe(true);
576 },
577 );
578
@@ -607,6 +632,7 @@ it.each([true, false])(
632 mocks.models.push(catalogModel("german", ["de"], true));
633 // Another client can finish setup while the catalog is offline.
634 mocks.modelState = "ready";
635 + mocks.statusTier = "accelerated";
636 await userEvent.click(retry);
637 await expect.element(wizard.getByRole("heading", { name: "german" })).toBeVisible();
638 await expect.element(download).toBeEnabled();
@@ -641,6 +667,7 @@ it.each([
667 mocks.download.mockImplementation(async () => {
668 mocks.models = [mocks.models[0]!, { ...mocks.models[1]!, state: landedState }];
669 mocks.modelState = "ready";
670 + mocks.statusTier = "accelerated";
671 });
672 await using _screen = await render(<Probe />);
673 await act(() => mocks.voice.start());
apps/web/src/speech/useEnvironmentSpeechInput.ts
+25 -8
@@ -14,6 +14,7 @@ import type {
14 EnvironmentSpeechModel,
15 EnvironmentSpeechStatus,
16 ProjectId,
17 + SpeechBenchmarkTier,
18 } from "@t3tools/contracts";
19 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
20 import { chooseSetupSpeechModel } from "@t3tools/shared/speech";
@@ -253,7 +254,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
254 .then((result) => {
255 if (disposed) return;
256 const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null;
256 - if (model) setSetup((current) => ({ ...current, model }));
257 + if (model)
258 + setSetup((current) => ({
259 + ...current,
260 + model: { ...model, recommended: current.model?.recommended ?? false },
261 + }));
262 })
263 .catch(() => undefined)
264 .finally(() => {
@@ -347,20 +352,26 @@ export function useEnvironmentSpeechInput(input: HookInput) {
352 session.adopt(createTarget());
353 }, [session, input.ownerKey, createTarget]);
354 const loadSetupCatalog = useStableCallback(
350 - async (connection: PreparedConnection, selectedModelId: string, request: number) => {
355 + async (
356 + connection: PreparedConnection,
357 + selectedModelId: string,
358 + benchmarkTier: SpeechBenchmarkTier,
359 + request: number,
360 + ) => {
361 setSetup({ ...INITIAL_SETUP, open: true });
352 - const catalog = await runEnvironmentSpeech(getEnvironmentSpeechModels(connection)).catch(
353 - () => null,
354 - );
362 + const catalog = await runEnvironmentSpeech(
363 + getEnvironmentSpeechModels(connection, true),
364 + ).catch(() => null);
365 if (request !== startRequestRef.current) return;
366 const choice = chooseSetupSpeechModel(catalog?.models ?? [], {
367 selectedModelId,
368 intent: latestRef.current.clientSettings.speechLanguage,
369 preferred: preferredSpeechLanguages(),
370 + benchmarkTier,
371 });
372 setSetup((current) => ({
373 ...current,
363 - model: choice?.model ?? null,
374 + model: choice ? { ...choice.model, recommended: choice.recommended } : null,
375 language: choice?.language ?? null,
376 catalogFailed: catalog === null,
377 error: choice
@@ -410,7 +421,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
421 setStatus({ prepared, value: latestStatus });
422 setQueuedStart(null);
423 if (latestStatus.supported && latestStatus.state === "missing-model") {
413 - return loadSetupCatalog(prepared, latestStatus.modelId, request);
424 + return loadSetupCatalog(prepared, latestStatus.modelId, latestStatus.benchmarkTier, request);
425 }
426 if (!latestStatus.supported) {
427 toastManager.add({
@@ -505,7 +516,13 @@ export function useEnvironmentSpeechInput(input: HookInput) {
516 setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
517 retryCatalog:
518 setup.open && setup.catalogFailed && prepared && currentStatus?.supported
508 - ? () => loadSetupCatalog(prepared, currentStatus.modelId, ++startRequestRef.current)
519 + ? () =>
520 + loadSetupCatalog(
521 + prepared,
522 + currentStatus.modelId,
523 + currentStatus.benchmarkTier,
524 + ++startRequestRef.current,
525 + )
526 : null,
527 download: downloadSetupModel,
528 cancelDownload: cancelSetupDownload,
docs/user/voice-input.md
+4 -2
@@ -66,8 +66,10 @@ format, such as Core ML, ONNX, or MLX builds of the same model, can’t be used.
66
67 Select the languages you dictate in to compare models that support all of them.
68 Accuracy shows how often the model got words or characters wrong in each
69 -language; speed is the time to transcribe 30 seconds on a reference GPU. Choose
70 -**All languages** to browse the full catalog. Filtering does not switch your model.
69 +language; speed is the time to transcribe 30 seconds on a reference CPU or GPU,
70 +matching the selected environment's transcription backend. Recommendations use
71 +the same reference tier for every selected language. Choose **All languages** to
72 +browse without a language preference. Filtering does not switch your model.
73
74 **Advanced** controls GPU or CPU use and when an idle model is unloaded. With hardware set to
75 **Auto**, a recording that fails on the GPU is retried once on that same
packages/shared/src/speech.test.ts
+180 -2
@@ -3,6 +3,7 @@ import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES } from "
3
4 import {
5 chooseSetupSpeechModel,
6 + recommendSpeechModels,
7 decodeSpeechPcmRequest,
8 encodeSpeechPcmRequest,
9 resolveSpeechVocabulary,
@@ -79,7 +80,18 @@ describe("chooseSetupSpeechModel", () => {
80 languages: string[],
81 supportsLanguageDetection: boolean,
82 recommended: boolean,
82 - ) => ({ id, languages, supportsLanguageDetection, recommended });
83 + ) => ({
84 + id,
85 + languages,
86 + supportsLanguageDetection,
87 + recommended,
88 + benchmark: {
89 + accuracy: Object.fromEntries(
90 + languages.map((language) => [language, { errorPercent: recommended ? 4 : 10 }]),
91 + ),
92 + secondsPer30s: { accelerated: 2, cpu: 3 },
93 + },
94 + });
95 const catalog = [
96 model("unrecommended", ["en", "de", "es"], true, false),
97 model("english", ["en"], false, true),
@@ -146,7 +158,173 @@ describe("chooseSetupSpeechModel", () => {
158 undefined,
159 ],
160 ] as const)("%s", (_, selectedModelId, intent, preferred, expected) => {
149 - const choice = chooseSetupSpeechModel(catalog, { selectedModelId, intent, preferred });
161 + const choice = chooseSetupSpeechModel(catalog, {
162 + selectedModelId,
163 + intent,
164 + preferred,
165 + benchmarkTier: "accelerated",
166 + });
167 expect(choice && { id: choice.model.id, language: choice.language }).toEqual(expected);
168 });
169 +
170 + it("keeps suitable selections and preference order, and falls back safely without measurements", () => {
171 + const best = {
172 + ...catalog[0]!,
173 + benchmark: {
174 + accuracy: { de: { errorPercent: 1 } },
175 + secondsPer30s: { accelerated: 1, cpu: 2 },
176 + },
177 + };
178 + expect(
179 + chooseSetupSpeechModel([...catalog, { ...best, id: "best" }], {
180 + benchmarkTier: "accelerated",
181 + selectedModelId: "gone",
182 + intent: "auto",
183 + preferred: ["de", "en"],
184 + }),
185 + ).toMatchObject({ model: { id: "best" }, language: "de", recommended: true });
186 + expect(
187 + chooseSetupSpeechModel([...catalog, { ...best, id: "best" }], {
188 + benchmarkTier: "accelerated",
189 + selectedModelId: "detecting",
190 + intent: "auto",
191 + preferred: ["de", "en"],
192 + }),
193 + ).toMatchObject({ model: { id: "detecting" }, language: "de", recommended: false });
194 + const unmeasured = catalog.map(({ benchmark: _benchmark, ...model }) => model);
195 + expect(
196 + chooseSetupSpeechModel(unmeasured, {
197 + benchmarkTier: "accelerated",
198 + selectedModelId: "gone",
199 + intent: "auto",
200 + preferred: ["de", "en"],
201 + }),
202 + ).toMatchObject({ model: { id: "unrecommended" }, language: "de", recommended: false });
203 + expect(
204 + chooseSetupSpeechModel([], {
205 + benchmarkTier: "accelerated",
206 + selectedModelId: "gone",
207 + intent: "auto",
208 + preferred: [],
209 + }),
210 + ).toBeUndefined();
211 + });
212 +});
213 +
214 +describe("recommendSpeechModels", () => {
215 + const model = (
216 + id: string,
217 + errors: Record<string, number>,
218 + wait: number | null,
219 + cpu: number | null,
220 + ) => ({
221 + id,
222 + languages: Object.keys(errors),
223 + benchmark: {
224 + accuracy: Object.fromEntries(
225 + Object.entries(errors).map(([language, errorPercent]) => [language, { errorPercent }]),
226 + ),
227 + secondsPer30s: { accelerated: wait, cpu },
228 + },
229 + });
230 + const models = [
231 + model("balanced", { en: 4, de: 16 }, 2, 7),
232 + model("fast", { en: 7, de: 7 }, 1, 3),
233 + model("accurate", { en: 2, de: 2 }, 8, 20),
234 + model("incomplete", { en: 1 }, 1, 1),
235 + ];
236 +
237 + it("balances all requested languages and assigns each model its first role", () => {
238 + expect([...recommendSpeechModels(models.slice(0, 3), ["en"], "accelerated")]).toEqual([
239 + ["balanced", "recommended"],
240 + ["fast", "faster"],
241 + ["accurate", "accurate"],
242 + ]);
243 + expect([...recommendSpeechModels(models, ["en", "de", "de"], "accelerated")]).toEqual([
244 + ["fast", "recommended"],
245 + ["accurate", "accurate"],
246 + ]);
247 + expect([...recommendSpeechModels(models.slice(0, 3), ["en"], "cpu")]).toEqual([
248 + ["fast", "recommended"],
249 + ["balanced", "accurate"],
250 + ]);
251 + expect([...recommendSpeechModels(models, ["en", "de"], "cpu")]).toEqual([
252 + ["fast", "recommended"],
253 + ]);
254 + });
255 +
256 + it("offers alternatives only when they improve on the balanced pick on the same tier", () => {
257 + expect([
258 + ...recommendSpeechModels(
259 + [model("cpu-fast-gpu-slow", { en: 3 }, 4, 1), model("balanced", { en: 3.2 }, 1, 8)],
260 + ["en"],
261 + "accelerated",
262 + ),
263 + ]).toEqual([
264 + ["balanced", "recommended"],
265 + ["cpu-fast-gpu-slow", "accurate"],
266 + ]);
267 + expect([
268 + ...recommendSpeechModels(
269 + [model("equal-accuracy", { en: 4 }, 8, 20), model("balanced", { en: 4 }, 2, 20)],
270 + ["en"],
271 + "accelerated",
272 + ),
273 + ]).toEqual([["balanced", "recommended"]]);
274 + expect([
275 + ...recommendSpeechModels(
276 + [model("balanced", { en: 4 }, 12, 20), model("outside-budget", { en: 2 }, 30, 40)],
277 + ["en"],
278 + "accelerated",
279 + ),
280 + ]).toEqual([["balanced", "recommended"]]);
281 + });
282 +
283 + it("does not invent measurements, preferences, or a model covering missing languages", () => {
284 + expect(recommendSpeechModels(models, [], "accelerated").size).toBe(0);
285 + expect(recommendSpeechModels(models, ["ja"], "accelerated").size).toBe(0);
286 + expect(
287 + recommendSpeechModels(
288 + [
289 + model("missing-speed", { en: 1, de: 1 }, null, 1),
290 + { ...model("missing-accuracy", { en: 1 }, 1, 1), languages: ["en", "de"] },
291 + ],
292 + ["en", "de"],
293 + "accelerated",
294 + ).size,
295 + ).toBe(0);
296 + expect([
297 + ...recommendSpeechModels([model("no-cpu", { en: 1 }, 1, null)], ["en"], "accelerated"),
298 + ]).toEqual([["no-cpu", "recommended"]]);
299 + expect(recommendSpeechModels([model("no-cpu", { en: 1 }, 1, null)], ["en"], "cpu").size).toBe(
300 + 0,
301 + );
302 + expect([
303 + ...recommendSpeechModels([model("cpu-only", { en: 1 }, null, 2)], ["en"], "cpu"),
304 + ]).toEqual([["cpu-only", "recommended"]]);
305 + });
306 +
307 + it("prefers useful accuracy before speed and relaxes the wait budget before quality", () => {
308 + expect([
309 + ...recommendSpeechModels(
310 + [model("poor", { en: 25 }, 1, 1), model("slow", { en: 4 }, 12, 20)],
311 + ["en"],
312 + "accelerated",
313 + ),
314 + ]).toEqual([["slow", "recommended"]]);
315 + expect([
316 + ...recommendSpeechModels(
317 + [model("worse", { en: 30, de: 40 }, 1, 1), model("least-poor", { en: 29, de: 30 }, 20, 40)],
318 + ["en", "de"],
319 + "accelerated",
320 + ),
321 + ]).toEqual([["least-poor", "recommended"]]);
322 + expect([
323 + ...recommendSpeechModels(
324 + [model("zero", { en: 0, de: 0 }, 1, 1)],
325 + ["en", "de"],
326 + "accelerated",
327 + ),
328 + ]).toEqual([["zero", "recommended"]]);
329 + });
330 });
packages/shared/src/speech.ts
+98 -8
@@ -3,13 +3,102 @@ import {
3 type CloudSpeechProvider,
4 SPEECH_MAX_OPTIONS_BYTES,
5 type SpeechCustomWords,
6 + type SpeechBenchmarkTier,
7 SpeechTranscriptionOptions,
8 } from "@t3tools/contracts";
9 import * as Schema from "effect/Schema";
10 +import recommendationPolicy from "./speechRecommendationPolicy.json" with { type: "json" };
11
12 const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions));
13 const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions));
14
15 +export type SpeechRecommendationRole = "recommended" | "faster" | "accurate";
16 +type RecommendationModel = {
17 + readonly id: string;
18 + readonly languages: readonly string[];
19 + readonly benchmark?: {
20 + readonly accuracy: Readonly<Record<string, { readonly errorPercent: number }>>;
21 + readonly secondsPer30s: {
22 + readonly cpu?: number | null;
23 + readonly accelerated: number | null;
24 + };
25 + };
26 +};
27 +
28 +/** Reference accuracy/speed picks; no preference or incomplete measurements get no badge. */
29 +export function recommendSpeechModels(
30 + models: readonly RecommendationModel[],
31 + languages: readonly string[],
32 + tier: SpeechBenchmarkTier | undefined,
33 +) {
34 + const wanted = [...new Set(languages)];
35 + const roles = new Map<string, SpeechRecommendationRole>();
36 + if (wanted.length === 0 || tier === undefined) return roles;
37 + const policy = recommendationPolicy;
38 + const scored = models.flatMap((model) => {
39 + if (!wanted.every((language) => model.languages.includes(language))) return [];
40 + const errors = wanted.map((language) => model.benchmark?.accuracy[language]?.errorPercent);
41 + const wait = model.benchmark?.secondsPer30s[tier];
42 + if (
43 + wait == null ||
44 + !Number.isFinite(wait) ||
45 + wait < 0 ||
46 + errors.some((error) => error === undefined || !Number.isFinite(error) || error < 0)
47 + )
48 + return [];
49 + const measured = errors as number[];
50 + const error = Math.exp(
51 + measured.reduce((sum, value) => sum + Math.log(value), 0) / measured.length,
52 + );
53 + return [
54 + {
55 + id: model.id,
56 + error,
57 + worst: Math.max(...measured),
58 + wait,
59 + cost: error + policy.waitWeight * Math.max(0, wait - policy.freeWaitSeconds),
60 + },
61 + ];
62 + });
63 + if (scored.length === 0) return roles;
64 + const useful = scored.filter((model) => model.worst < policy.maxLanguageErrorPercent);
65 + const pool = useful.length ? useful : scored;
66 + const lowest = (items: typeof scored, score: (item: (typeof scored)[number]) => number) =>
67 + items.reduce<(typeof scored)[number] | undefined>(
68 + (best, item) => (best === undefined || score(item) < score(best) ? item : best),
69 + undefined,
70 + );
71 + const withinBudget = pool.filter((model) => model.wait <= policy.overallMaxWaitSeconds);
72 + const best = (
73 + useful.length
74 + ? lowest(withinBudget.length ? withinBudget : pool, (model) => model.cost)
75 + : lowest(pool, (model) => model.worst + model.error / 1000)
76 + )!;
77 + const quicker = pool.filter((model) => model.wait < best.wait);
78 + const fast = useful.length
79 + ? (lowest(
80 + quicker.filter((model) => model.wait <= policy.fastMaxWaitSeconds),
81 + (model) => model.error,
82 + ) ?? lowest(quicker, (model) => model.wait + model.error / 1000))
83 + : undefined;
84 + const accurate = useful.length
85 + ? lowest(
86 + pool.filter(
87 + (model) => model.error < best.error && model.wait <= policy.accurateMaxWaitSeconds,
88 + ),
89 + (model) => model.error,
90 + )
91 + : undefined;
92 + for (const [model, role] of [
93 + [best, "recommended"],
94 + [fast, "faster"],
95 + [accurate, "accurate"],
96 + ] as const) {
97 + if (model && !roles.has(model.id)) roles.set(model.id, role);
98 + }
99 + return roles;
100 +}
101 +
102 export function cloudSpeechDisclosure(provider: CloudSpeechProvider, environment: string) {
103 return provider === "openrouter"
104 ? `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.`
@@ -37,16 +126,13 @@ export function effectiveSpeechLanguage(
126 }
127
128 /**
40 - * The model first-run setup offers: the selected one, else a recommended one, else any that the
129 + * The model first-run setup offers: the selected one, else the benchmark pick, else any that the
130 * server would really transcribe the person's language with. Without a language hint,
131 * offer only the selected model, without a recommendation.
132 */
133 export function chooseSetupSpeechModel<
45 - Model extends {
46 - readonly id: string;
47 - readonly languages: readonly string[];
134 + Model extends RecommendationModel & {
135 readonly supportsLanguageDetection: boolean;
49 - readonly recommended: boolean;
136 },
137 >(
138 models: readonly Model[],
@@ -54,22 +140,26 @@ export function chooseSetupSpeechModel<
140 readonly selectedModelId: string;
141 readonly intent: string;
142 readonly preferred: readonly string[];
143 + readonly benchmarkTier: SpeechBenchmarkTier | undefined;
144 },
145 ) {
146 if (input.intent === "auto" && input.preferred.length === 0) {
147 const model = models.find(({ id }) => id === input.selectedModelId);
61 - return model && { model, language: null };
148 + return model && { model, language: null, recommended: false };
149 }
150 for (const language of input.intent === "auto" ? input.preferred : [input.intent]) {
151 const capable = models.filter((model) => {
152 const effective = effectiveSpeechLanguage(model, input.intent);
153 return effective === language || (effective === "auto" && model.languages.includes(language));
154 });
155 + const recommendedId = recommendSpeechModels(capable, [language], input.benchmarkTier)
156 + .keys()
157 + .next().value;
158 const model =
159 capable.find(({ id }) => id === input.selectedModelId) ??
70 - capable.find(({ recommended }) => recommended) ??
160 + capable.find(({ id }) => id === recommendedId) ??
161 capable[0];
72 - if (model) return { model, language };
162 + if (model) return { model, language, recommended: model.id === recommendedId };
163 }
164 return undefined;
165 }
packages/shared/src/speechRecommendationPolicy.json new
+11
@@ -0,0 +1,11 @@
1 +{
2 + "dictationSeconds": 30,
3 + "freeWaitSeconds": 3,
4 + "waitWeight": 0.5,
5 + "overallMaxWaitSeconds": 5,
6 + "maxLanguageErrorPercent": 20,
7 + "noteMinErrorPercent": 15,
8 + "experimentalMaxErrorPercent": 30,
9 + "accurateMaxWaitSeconds": 10,
10 + "fastMaxWaitSeconds": 4
11 +}