The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { describe, expect, it } from "vite-plus/test"; |
| 2 | import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES } from "@t3tools/contracts"; |
| 3 | |
| 4 | import { |
| 5 | chooseSetupSpeechModel, |
| 6 | recommendSpeechModels, |
| 7 | decodeSpeechPcmRequest, |
| 8 | encodeSpeechPcmRequest, |
| 9 | resolveSpeechVocabulary, |
| 10 | } from "./speech.ts"; |
| 11 | |
| 12 | describe("speech PCM request framing", () => { |
| 13 | it("round-trips options and unaligned PCM without changing audio bytes", () => { |
| 14 | const pcm = new Uint8Array(new Float32Array([0.25, -0.5]).buffer); |
| 15 | const options = { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "fr" }; |
| 16 | const body = encodeSpeechPcmRequest(pcm, options); |
| 17 | const pool = new Uint8Array(body.length + 3); |
| 18 | pool.set(body, 3); |
| 19 | expect(decodeSpeechPcmRequest(pool.subarray(3))).toEqual({ pcm, options }); |
| 20 | }); |
| 21 | |
| 22 | it("accepts recordings from clients that predate silence filtering", () => { |
| 23 | const metadata = new TextEncoder().encode( |
| 24 | JSON.stringify({ |
| 25 | speechLanguage: "fr", |
| 26 | speechTranslateToEnglish: false, |
| 27 | speechCustomWords: [], |
| 28 | speechRemoveFillerWords: false, |
| 29 | }), |
| 30 | ); |
| 31 | const body = new Uint8Array(4 + metadata.length + 4); |
| 32 | new DataView(body.buffer).setUint32(0, metadata.length, true); |
| 33 | body.set(metadata, 4); |
| 34 | expect(decodeSpeechPcmRequest(body).options).toEqual({ |
| 35 | ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, |
| 36 | speechLanguage: "fr", |
| 37 | speechRemoveFillerWords: false, |
| 38 | speechSilenceFiltering: true, |
| 39 | }); |
| 40 | }); |
| 41 | |
| 42 | it("rejects truncated frames, oversized option lengths and missing audio", () => { |
| 43 | expect(() => decodeSpeechPcmRequest(new Uint8Array(3))).toThrow(); |
| 44 | const oversized = new Uint8Array(5); |
| 45 | new DataView(oversized.buffer).setUint32(0, SPEECH_MAX_OPTIONS_BYTES + 1, true); |
| 46 | expect(() => decodeSpeechPcmRequest(oversized)).toThrow(); |
| 47 | expect(() => |
| 48 | decodeSpeechPcmRequest( |
| 49 | encodeSpeechPcmRequest(new Uint8Array(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS), |
| 50 | ), |
| 51 | ).toThrow(); |
| 52 | }); |
| 53 | }); |
| 54 | |
| 55 | describe("resolveSpeechVocabulary", () => { |
| 56 | it("prefers project spellings and keeps hints out of the dictionary", () => { |
| 57 | const personal = Array.from({ length: 100 }, (_, index) => ({ |
| 58 | term: `word${index}`, |
| 59 | aliases: index === 0 ? ["Effect"] : [], |
| 60 | })); |
| 61 | const resolved = resolveSpeechVocabulary({ |
| 62 | personal, |
| 63 | project: [{ term: "effect", aliases: ["a fact"] }], |
| 64 | hints: [" T3 Code ", "word1", "", "x".repeat(51), "t3 code"], |
| 65 | }); |
| 66 | expect(resolved.speechHints).toEqual(["T3 Code"]); |
| 67 | expect(resolved.speechCustomWords).toHaveLength(100); |
| 68 | expect(resolved.speechCustomWords.slice(0, 2)).toEqual([ |
| 69 | { term: "effect", aliases: ["a fact"] }, |
| 70 | { term: "word0", aliases: [] }, |
| 71 | ]); |
| 72 | expect(resolved.speechCustomWords.at(-1)?.term).toBe("word98"); |
| 73 | }); |
| 74 | }); |
| 75 | |
| 76 | describe("chooseSetupSpeechModel", () => { |
| 77 | const model = ( |
| 78 | id: string, |
| 79 | languages: string[], |
| 80 | supportsLanguageDetection: boolean, |
| 81 | recommended: boolean, |
| 82 | ) => ({ |
| 83 | id, |
| 84 | languages, |
| 85 | supportsLanguageDetection, |
| 86 | recommended, |
| 87 | benchmark: { |
| 88 | accuracy: Object.fromEntries( |
| 89 | languages.map((language) => [language, { errorPercent: recommended ? 4 : 10 }]), |
| 90 | ), |
| 91 | secondsPer30s: { accelerated: 2, cpu: 3 }, |
| 92 | }, |
| 93 | }); |
| 94 | const catalog = [ |
| 95 | model("unrecommended", ["en", "de", "es"], true, false), |
| 96 | model("english", ["en"], false, true), |
| 97 | model("fixed", ["en", "de"], false, true), |
| 98 | model("detecting", ["en", "de", "fr"], true, true), |
| 99 | ]; |
| 100 | |
| 101 | it.each([ |
| 102 | [ |
| 103 | "keeps the selected English model for an English speaker", |
| 104 | "english", |
| 105 | "auto", |
| 106 | ["en"], |
| 107 | { id: "english", language: "en" }, |
| 108 | ], |
| 109 | [ |
| 110 | "skips models that would transcribe English instead of German", |
| 111 | "english", |
| 112 | "auto", |
| 113 | ["de", "en"], |
| 114 | { id: "detecting", language: "de" }, |
| 115 | ], |
| 116 | [ |
| 117 | "accepts a model without detection for a chosen language", |
| 118 | "english", |
| 119 | "de", |
| 120 | ["en"], |
| 121 | { id: "fixed", language: "de" }, |
| 122 | ], |
| 123 | [ |
| 124 | "keeps a selected model that transcribes the language", |
| 125 | "unrecommended", |
| 126 | "auto", |
| 127 | ["de"], |
| 128 | { id: "unrecommended", language: "de" }, |
| 129 | ], |
| 130 | [ |
| 131 | "moves on to the next preferred language a model transcribes", |
| 132 | "english", |
| 133 | "auto", |
| 134 | ["ja", "fr"], |
| 135 | { id: "detecting", language: "fr" }, |
| 136 | ], |
| 137 | [ |
| 138 | "falls back to a model that is not recommended", |
| 139 | "english", |
| 140 | "auto", |
| 141 | ["es"], |
| 142 | { id: "unrecommended", language: "es" }, |
| 143 | ], |
| 144 | ["offers nothing when no model transcribes the language", "english", "auto", ["ja"], undefined], |
| 145 | [ |
| 146 | "offers the selected model without a language recommendation when Auto has no preferences", |
| 147 | "fixed", |
| 148 | "auto", |
| 149 | [], |
| 150 | { id: "fixed", language: null }, |
| 151 | ], |
| 152 | [ |
| 153 | "does not guess another model when Auto has no preferences or selection", |
| 154 | "gone", |
| 155 | "auto", |
| 156 | [], |
| 157 | undefined, |
| 158 | ], |
| 159 | ] as const)("%s", (_, selectedModelId, intent, preferred, expected) => { |
| 160 | const choice = chooseSetupSpeechModel(catalog, { |
| 161 | selectedModelId, |
| 162 | intent, |
| 163 | preferred, |
| 164 | benchmarkTier: "accelerated", |
| 165 | }); |
| 166 | expect(choice && { id: choice.model.id, language: choice.language }).toEqual(expected); |
| 167 | }); |
| 168 | |
| 169 | it("keeps suitable selections and preference order, and falls back safely without measurements", () => { |
| 170 | const best = { |
| 171 | ...catalog[0]!, |
| 172 | benchmark: { |
| 173 | accuracy: { de: { errorPercent: 1 } }, |
| 174 | secondsPer30s: { accelerated: 1, cpu: 2 }, |
| 175 | }, |
| 176 | }; |
| 177 | expect( |
| 178 | chooseSetupSpeechModel([...catalog, { ...best, id: "best" }], { |
| 179 | benchmarkTier: "accelerated", |
| 180 | selectedModelId: "gone", |
| 181 | intent: "auto", |
| 182 | preferred: ["de", "en"], |
| 183 | }), |
| 184 | ).toMatchObject({ model: { id: "best" }, language: "de", recommended: true }); |
| 185 | expect( |
| 186 | chooseSetupSpeechModel([...catalog, { ...best, id: "best" }], { |
| 187 | benchmarkTier: "accelerated", |
| 188 | selectedModelId: "detecting", |
| 189 | intent: "auto", |
| 190 | preferred: ["de", "en"], |
| 191 | }), |
| 192 | ).toMatchObject({ model: { id: "detecting" }, language: "de", recommended: false }); |
| 193 | const unmeasured = catalog.map(({ benchmark: _benchmark, ...model }) => model); |
| 194 | expect( |
| 195 | chooseSetupSpeechModel(unmeasured, { |
| 196 | benchmarkTier: "accelerated", |
| 197 | selectedModelId: "gone", |
| 198 | intent: "auto", |
| 199 | preferred: ["de", "en"], |
| 200 | }), |
| 201 | ).toMatchObject({ model: { id: "unrecommended" }, language: "de", recommended: false }); |
| 202 | expect( |
| 203 | chooseSetupSpeechModel([], { |
| 204 | benchmarkTier: "accelerated", |
| 205 | selectedModelId: "gone", |
| 206 | intent: "auto", |
| 207 | preferred: [], |
| 208 | }), |
| 209 | ).toBeUndefined(); |
| 210 | }); |
| 211 | }); |
| 212 | |
| 213 | describe("recommendSpeechModels", () => { |
| 214 | const model = ( |
| 215 | id: string, |
| 216 | errors: Record<string, number>, |
| 217 | wait: number | null, |
| 218 | cpu: number | null, |
| 219 | ) => ({ |
| 220 | id, |
| 221 | languages: Object.keys(errors), |
| 222 | benchmark: { |
| 223 | accuracy: Object.fromEntries( |
| 224 | Object.entries(errors).map(([language, errorPercent]) => [language, { errorPercent }]), |
| 225 | ), |
| 226 | secondsPer30s: { accelerated: wait, cpu }, |
| 227 | }, |
| 228 | }); |
| 229 | const models = [ |
| 230 | model("balanced", { en: 4, de: 16 }, 2, 7), |
| 231 | model("fast", { en: 7, de: 7 }, 1, 3), |
| 232 | model("accurate", { en: 2, de: 2 }, 8, 20), |
| 233 | model("incomplete", { en: 1 }, 1, 1), |
| 234 | ]; |
| 235 | |
| 236 | it("balances all requested languages and assigns each model its first role", () => { |
| 237 | expect([...recommendSpeechModels(models.slice(0, 3), ["en"], "accelerated")]).toEqual([ |
| 238 | ["balanced", "recommended"], |
| 239 | ["fast", "faster"], |
| 240 | ["accurate", "accurate"], |
| 241 | ]); |
| 242 | expect([...recommendSpeechModels(models, ["en", "de", "de"], "accelerated")]).toEqual([ |
| 243 | ["fast", "recommended"], |
| 244 | ["accurate", "accurate"], |
| 245 | ]); |
| 246 | expect([...recommendSpeechModels(models.slice(0, 3), ["en"], "cpu")]).toEqual([ |
| 247 | ["fast", "recommended"], |
| 248 | ["balanced", "accurate"], |
| 249 | ]); |
| 250 | expect([...recommendSpeechModels(models, ["en", "de"], "cpu")]).toEqual([ |
| 251 | ["fast", "recommended"], |
| 252 | ]); |
| 253 | }); |
| 254 | |
| 255 | it("offers alternatives only when they improve on the balanced pick on the same tier", () => { |
| 256 | expect([ |
| 257 | ...recommendSpeechModels( |
| 258 | [model("cpu-fast-gpu-slow", { en: 3 }, 4, 1), model("balanced", { en: 3.2 }, 1, 8)], |
| 259 | ["en"], |
| 260 | "accelerated", |
| 261 | ), |
| 262 | ]).toEqual([ |
| 263 | ["balanced", "recommended"], |
| 264 | ["cpu-fast-gpu-slow", "accurate"], |
| 265 | ]); |
| 266 | expect([ |
| 267 | ...recommendSpeechModels( |
| 268 | [model("equal-accuracy", { en: 4 }, 8, 20), model("balanced", { en: 4 }, 2, 20)], |
| 269 | ["en"], |
| 270 | "accelerated", |
| 271 | ), |
| 272 | ]).toEqual([["balanced", "recommended"]]); |
| 273 | expect([ |
| 274 | ...recommendSpeechModels( |
| 275 | [model("balanced", { en: 4 }, 12, 20), model("outside-budget", { en: 2 }, 30, 40)], |
| 276 | ["en"], |
| 277 | "accelerated", |
| 278 | ), |
| 279 | ]).toEqual([["balanced", "recommended"]]); |
| 280 | }); |
| 281 | |
| 282 | it("does not invent measurements, preferences, or a model covering missing languages", () => { |
| 283 | expect(recommendSpeechModels(models, [], "accelerated").size).toBe(0); |
| 284 | expect(recommendSpeechModels(models, ["ja"], "accelerated").size).toBe(0); |
| 285 | expect( |
| 286 | recommendSpeechModels( |
| 287 | [ |
| 288 | model("missing-speed", { en: 1, de: 1 }, null, 1), |
| 289 | { ...model("missing-accuracy", { en: 1 }, 1, 1), languages: ["en", "de"] }, |
| 290 | ], |
| 291 | ["en", "de"], |
| 292 | "accelerated", |
| 293 | ).size, |
| 294 | ).toBe(0); |
| 295 | expect([ |
| 296 | ...recommendSpeechModels([model("no-cpu", { en: 1 }, 1, null)], ["en"], "accelerated"), |
| 297 | ]).toEqual([["no-cpu", "recommended"]]); |
| 298 | expect(recommendSpeechModels([model("no-cpu", { en: 1 }, 1, null)], ["en"], "cpu").size).toBe( |
| 299 | 0, |
| 300 | ); |
| 301 | expect([ |
| 302 | ...recommendSpeechModels([model("cpu-only", { en: 1 }, null, 2)], ["en"], "cpu"), |
| 303 | ]).toEqual([["cpu-only", "recommended"]]); |
| 304 | }); |
| 305 | |
| 306 | it("prefers useful accuracy before speed and relaxes the wait budget before quality", () => { |
| 307 | expect([ |
| 308 | ...recommendSpeechModels( |
| 309 | [model("poor", { en: 25 }, 1, 1), model("slow", { en: 4 }, 12, 20)], |
| 310 | ["en"], |
| 311 | "accelerated", |
| 312 | ), |
| 313 | ]).toEqual([["slow", "recommended"]]); |
| 314 | expect([ |
| 315 | ...recommendSpeechModels( |
| 316 | [model("worse", { en: 30, de: 40 }, 1, 1), model("least-poor", { en: 29, de: 30 }, 20, 40)], |
| 317 | ["en", "de"], |
| 318 | "accelerated", |
| 319 | ), |
| 320 | ]).toEqual([["least-poor", "recommended"]]); |
| 321 | expect([ |
| 322 | ...recommendSpeechModels( |
| 323 | [model("zero", { en: 0, de: 0 }, 1, 1)], |
| 324 | ["en", "de"], |
| 325 | "accelerated", |
| 326 | ), |
| 327 | ]).toEqual([["zero", "recommended"]]); |
| 328 | }); |
| 329 | }); |