The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { |
| 2 | SPEECH_MAX_OPTIONS_BYTES, |
| 3 | type SpeechCustomWords, |
| 4 | type SpeechBenchmarkTier, |
| 5 | SpeechTranscriptionOptions, |
| 6 | } from "@t3tools/contracts"; |
| 7 | import * as Schema from "effect/Schema"; |
| 8 | import recommendationPolicy from "./speechRecommendationPolicy.json" with { type: "json" }; |
| 9 | |
| 10 | const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions)); |
| 11 | const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions)); |
| 12 | |
| 13 | export type SpeechRecommendationRole = "recommended" | "faster" | "accurate"; |
| 14 | type RecommendationModel = { |
| 15 | readonly id: string; |
| 16 | readonly languages: readonly string[]; |
| 17 | readonly benchmark?: { |
| 18 | readonly accuracy: Readonly<Record<string, { readonly errorPercent: number }>>; |
| 19 | readonly secondsPer30s: { |
| 20 | readonly cpu?: number | null; |
| 21 | readonly accelerated: number | null; |
| 22 | }; |
| 23 | }; |
| 24 | }; |
| 25 | |
| 26 | /** Reference accuracy/speed picks; no preference or incomplete measurements get no badge. */ |
| 27 | export function recommendSpeechModels( |
| 28 | models: readonly RecommendationModel[], |
| 29 | languages: readonly string[], |
| 30 | tier: SpeechBenchmarkTier | undefined, |
| 31 | ) { |
| 32 | const wanted = [...new Set(languages)]; |
| 33 | const roles = new Map<string, SpeechRecommendationRole>(); |
| 34 | if (wanted.length === 0 || tier === undefined) return roles; |
| 35 | const policy = recommendationPolicy; |
| 36 | const scored = models.flatMap((model) => { |
| 37 | if (!wanted.every((language) => model.languages.includes(language))) return []; |
| 38 | const errors = wanted.map((language) => model.benchmark?.accuracy[language]?.errorPercent); |
| 39 | const wait = model.benchmark?.secondsPer30s[tier]; |
| 40 | if ( |
| 41 | wait == null || |
| 42 | !Number.isFinite(wait) || |
| 43 | wait < 0 || |
| 44 | errors.some((error) => error === undefined || !Number.isFinite(error) || error < 0) |
| 45 | ) |
| 46 | return []; |
| 47 | const measured = errors as number[]; |
| 48 | const error = Math.exp( |
| 49 | measured.reduce((sum, value) => sum + Math.log(value), 0) / measured.length, |
| 50 | ); |
| 51 | return [ |
| 52 | { |
| 53 | id: model.id, |
| 54 | error, |
| 55 | worst: Math.max(...measured), |
| 56 | wait, |
| 57 | cost: error + policy.waitWeight * Math.max(0, wait - policy.freeWaitSeconds), |
| 58 | }, |
| 59 | ]; |
| 60 | }); |
| 61 | if (scored.length === 0) return roles; |
| 62 | const useful = scored.filter((model) => model.worst < policy.maxLanguageErrorPercent); |
| 63 | const pool = useful.length ? useful : scored; |
| 64 | const lowest = (items: typeof scored, score: (item: (typeof scored)[number]) => number) => |
| 65 | items.reduce<(typeof scored)[number] | undefined>( |
| 66 | (best, item) => (best === undefined || score(item) < score(best) ? item : best), |
| 67 | undefined, |
| 68 | ); |
| 69 | const withinBudget = pool.filter((model) => model.wait <= policy.overallMaxWaitSeconds); |
| 70 | const best = ( |
| 71 | useful.length |
| 72 | ? lowest(withinBudget.length ? withinBudget : pool, (model) => model.cost) |
| 73 | : lowest(pool, (model) => model.worst + model.error / 1000) |
| 74 | )!; |
| 75 | const quicker = pool.filter((model) => model.wait < best.wait); |
| 76 | const fast = useful.length |
| 77 | ? (lowest( |
| 78 | quicker.filter((model) => model.wait <= policy.fastMaxWaitSeconds), |
| 79 | (model) => model.error, |
| 80 | ) ?? lowest(quicker, (model) => model.wait + model.error / 1000)) |
| 81 | : undefined; |
| 82 | const accurate = useful.length |
| 83 | ? lowest( |
| 84 | pool.filter( |
| 85 | (model) => model.error < best.error && model.wait <= policy.accurateMaxWaitSeconds, |
| 86 | ), |
| 87 | (model) => model.error, |
| 88 | ) |
| 89 | : undefined; |
| 90 | for (const [model, role] of [ |
| 91 | [best, "recommended"], |
| 92 | [fast, "faster"], |
| 93 | [accurate, "accurate"], |
| 94 | ] as const) { |
| 95 | if (model && !roles.has(model.id)) roles.set(model.id, role); |
| 96 | } |
| 97 | return roles; |
| 98 | } |
| 99 | |
| 100 | // Only offer explicit hints understood by all cloud APIs; other languages can still be detected. |
| 101 | export const cloudSpeechLanguages = { |
| 102 | supportsLanguageDetection: true, |
| 103 | languages: |
| 104 | "ar bg ca cs da de el en es fa fi fr hi hr hu id it ja ko mk ms nl pl pt ro ru sk sv th tr uk ur vi zh".split( |
| 105 | " ", |
| 106 | ), |
| 107 | }; |
| 108 | |
| 109 | /** Resolve the language the model can produce from the requested language. */ |
| 110 | export function effectiveSpeechLanguage( |
| 111 | model: { readonly languages: readonly string[]; readonly supportsLanguageDetection: boolean }, |
| 112 | intent: string, |
| 113 | ): string { |
| 114 | if (model.languages.length === 1) return model.languages[0]!; |
| 115 | if (intent !== "auto" && model.languages.includes(intent)) return intent; |
| 116 | if (model.supportsLanguageDetection) return "auto"; |
| 117 | return model.languages.find((language) => language === "en") ?? model.languages[0] ?? "auto"; |
| 118 | } |
| 119 | |
| 120 | /** |
| 121 | * The model first-run setup offers: the selected one, else the benchmark pick, else any that the |
| 122 | * server would really transcribe the person's language with. Without a language hint, |
| 123 | * offer only the selected model, without a recommendation. |
| 124 | */ |
| 125 | export function chooseSetupSpeechModel< |
| 126 | Model extends RecommendationModel & { |
| 127 | readonly supportsLanguageDetection: boolean; |
| 128 | }, |
| 129 | >( |
| 130 | models: readonly Model[], |
| 131 | input: { |
| 132 | readonly selectedModelId: string; |
| 133 | readonly intent: string; |
| 134 | readonly preferred: readonly string[]; |
| 135 | readonly benchmarkTier: SpeechBenchmarkTier | undefined; |
| 136 | }, |
| 137 | ) { |
| 138 | if (input.intent === "auto" && input.preferred.length === 0) { |
| 139 | const model = models.find(({ id }) => id === input.selectedModelId); |
| 140 | return model && { model, language: null, recommended: false }; |
| 141 | } |
| 142 | for (const language of input.intent === "auto" ? input.preferred : [input.intent]) { |
| 143 | const capable = models.filter((model) => { |
| 144 | const effective = effectiveSpeechLanguage(model, input.intent); |
| 145 | return effective === language || (effective === "auto" && model.languages.includes(language)); |
| 146 | }); |
| 147 | const recommendedId = recommendSpeechModels(capable, [language], input.benchmarkTier) |
| 148 | .keys() |
| 149 | .next().value; |
| 150 | const model = |
| 151 | capable.find(({ id }) => id === input.selectedModelId) ?? |
| 152 | capable.find(({ id }) => id === recommendedId) ?? |
| 153 | capable[0]; |
| 154 | if (model) return { model, language, recommended: model.id === recommendedId }; |
| 155 | } |
| 156 | return undefined; |
| 157 | } |
| 158 | |
| 159 | const vocabularyKey = (value: string) => value.trim().normalize("NFC").toLocaleLowerCase(); |
| 160 | |
| 161 | /** |
| 162 | * Project spellings win over personal ones. Hint terms, such as the project name or the |
| 163 | * correction cue, only steer recognition, so they travel apart from the dictionary. |
| 164 | */ |
| 165 | export function resolveSpeechVocabulary(input: { |
| 166 | readonly personal: SpeechCustomWords; |
| 167 | readonly project: SpeechCustomWords; |
| 168 | readonly hints?: ReadonlyArray<string>; |
| 169 | }): { readonly speechCustomWords: SpeechCustomWords; readonly speechHints: string[] } { |
| 170 | const seen = new Set<string>(); |
| 171 | const speechCustomWords = [...input.project, ...input.personal] |
| 172 | .flatMap(({ term, aliases }) => { |
| 173 | if (seen.has(vocabularyKey(term))) return []; |
| 174 | seen.add(vocabularyKey(term)); |
| 175 | const uniqueAliases = aliases.filter((alias) => { |
| 176 | if (seen.has(vocabularyKey(alias))) return false; |
| 177 | seen.add(vocabularyKey(alias)); |
| 178 | return true; |
| 179 | }); |
| 180 | return [{ term, aliases: uniqueAliases }]; |
| 181 | }) |
| 182 | .slice(0, 100); |
| 183 | const known = new Set( |
| 184 | speechCustomWords.flatMap(({ term, aliases }) => [term, ...aliases].map(vocabularyKey)), |
| 185 | ); |
| 186 | // A hint that is already a dictionary entry keeps that entry and its aliases in place. |
| 187 | const speechHints = (input.hints ?? []).flatMap((hint) => { |
| 188 | const term = hint.replace(/\s+/g, " ").trim(); |
| 189 | if (term.length === 0 || term.length > 50 || known.has(vocabularyKey(term))) return []; |
| 190 | known.add(vocabularyKey(term)); |
| 191 | return [term]; |
| 192 | }); |
| 193 | return { speechCustomWords, speechHints }; |
| 194 | } |
| 195 | |
| 196 | /** Binary requests contain a four-byte JSON length, UTF-8 options, then unencoded PCM. */ |
| 197 | export function encodeSpeechPcmRequest( |
| 198 | pcm: Uint8Array, |
| 199 | options: SpeechTranscriptionOptions, |
| 200 | ): Uint8Array<ArrayBuffer> { |
| 201 | const metadata = new TextEncoder().encode(encodeOptions(options)); |
| 202 | if (metadata.byteLength > SPEECH_MAX_OPTIONS_BYTES) |
| 203 | throw new Error("Speech options are too large."); |
| 204 | const body = new Uint8Array(4 + metadata.byteLength + pcm.byteLength); |
| 205 | new DataView(body.buffer).setUint32(0, metadata.byteLength, true); |
| 206 | body.set(metadata, 4); |
| 207 | body.set(pcm, 4 + metadata.byteLength); |
| 208 | return body; |
| 209 | } |
| 210 | |
| 211 | export function decodeSpeechPcmRequest(body: Uint8Array) { |
| 212 | if (body.byteLength < 4) throw new Error("Missing speech options."); |
| 213 | const length = new DataView(body.buffer, body.byteOffset, body.byteLength).getUint32(0, true); |
| 214 | if (length > SPEECH_MAX_OPTIONS_BYTES || 4 + length >= body.byteLength) |
| 215 | throw new Error("Invalid speech options length."); |
| 216 | const options = decodeOptions( |
| 217 | new TextDecoder("utf-8", { fatal: true }).decode(body.subarray(4, 4 + length)), |
| 218 | ); |
| 219 | return { pcm: body.subarray(4 + length), options }; |
| 220 | } |