@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 220 lines 8.54 KB
Raw
1 import {
2 SPEECH_MAX_OPTIONS_BYTES,
3 type SpeechCustomWords,
4 type SpeechBenchmarkTier,
5 SpeechTranscriptionOptions,
6 } from "@t3tools/contracts";
7 import * as Schema from "effect/Schema";
8 import recommendationPolicy from "./speechRecommendationPolicy.json" with { type: "json" };
9
10 const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions));
11 const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions));
12
13 export type SpeechRecommendationRole = "recommended" | "faster" | "accurate";
14 type RecommendationModel = {
15 readonly id: string;
16 readonly languages: readonly string[];
17 readonly benchmark?: {
18 readonly accuracy: Readonly<Record<string, { readonly errorPercent: number }>>;
19 readonly secondsPer30s: {
20 readonly cpu?: number | null;
21 readonly accelerated: number | null;
22 };
23 };
24 };
25
26 /** Reference accuracy/speed picks; no preference or incomplete measurements get no badge. */
27 export function recommendSpeechModels(
28 models: readonly RecommendationModel[],
29 languages: readonly string[],
30 tier: SpeechBenchmarkTier | undefined,
31 ) {
32 const wanted = [...new Set(languages)];
33 const roles = new Map<string, SpeechRecommendationRole>();
34 if (wanted.length === 0 || tier === undefined) return roles;
35 const policy = recommendationPolicy;
36 const scored = models.flatMap((model) => {
37 if (!wanted.every((language) => model.languages.includes(language))) return [];
38 const errors = wanted.map((language) => model.benchmark?.accuracy[language]?.errorPercent);
39 const wait = model.benchmark?.secondsPer30s[tier];
40 if (
41 wait == null ||
42 !Number.isFinite(wait) ||
43 wait < 0 ||
44 errors.some((error) => error === undefined || !Number.isFinite(error) || error < 0)
45 )
46 return [];
47 const measured = errors as number[];
48 const error = Math.exp(
49 measured.reduce((sum, value) => sum + Math.log(value), 0) / measured.length,
50 );
51 return [
52 {
53 id: model.id,
54 error,
55 worst: Math.max(...measured),
56 wait,
57 cost: error + policy.waitWeight * Math.max(0, wait - policy.freeWaitSeconds),
58 },
59 ];
60 });
61 if (scored.length === 0) return roles;
62 const useful = scored.filter((model) => model.worst < policy.maxLanguageErrorPercent);
63 const pool = useful.length ? useful : scored;
64 const lowest = (items: typeof scored, score: (item: (typeof scored)[number]) => number) =>
65 items.reduce<(typeof scored)[number] | undefined>(
66 (best, item) => (best === undefined || score(item) < score(best) ? item : best),
67 undefined,
68 );
69 const withinBudget = pool.filter((model) => model.wait <= policy.overallMaxWaitSeconds);
70 const best = (
71 useful.length
72 ? lowest(withinBudget.length ? withinBudget : pool, (model) => model.cost)
73 : lowest(pool, (model) => model.worst + model.error / 1000)
74 )!;
75 const quicker = pool.filter((model) => model.wait < best.wait);
76 const fast = useful.length
77 ? (lowest(
78 quicker.filter((model) => model.wait <= policy.fastMaxWaitSeconds),
79 (model) => model.error,
80 ) ?? lowest(quicker, (model) => model.wait + model.error / 1000))
81 : undefined;
82 const accurate = useful.length
83 ? lowest(
84 pool.filter(
85 (model) => model.error < best.error && model.wait <= policy.accurateMaxWaitSeconds,
86 ),
87 (model) => model.error,
88 )
89 : undefined;
90 for (const [model, role] of [
91 [best, "recommended"],
92 [fast, "faster"],
93 [accurate, "accurate"],
94 ] as const) {
95 if (model && !roles.has(model.id)) roles.set(model.id, role);
96 }
97 return roles;
98 }
99
100 // Only offer explicit hints understood by all cloud APIs; other languages can still be detected.
101 export const cloudSpeechLanguages = {
102 supportsLanguageDetection: true,
103 languages:
104 "ar bg ca cs da de el en es fa fi fr hi hr hu id it ja ko mk ms nl pl pt ro ru sk sv th tr uk ur vi zh".split(
105 " ",
106 ),
107 };
108
109 /** Resolve the language the model can produce from the requested language. */
110 export function effectiveSpeechLanguage(
111 model: { readonly languages: readonly string[]; readonly supportsLanguageDetection: boolean },
112 intent: string,
113 ): string {
114 if (model.languages.length === 1) return model.languages[0]!;
115 if (intent !== "auto" && model.languages.includes(intent)) return intent;
116 if (model.supportsLanguageDetection) return "auto";
117 return model.languages.find((language) => language === "en") ?? model.languages[0] ?? "auto";
118 }
119
120 /**
121 * The model first-run setup offers: the selected one, else the benchmark pick, else any that the
122 * server would really transcribe the person's language with. Without a language hint,
123 * offer only the selected model, without a recommendation.
124 */
125 export function chooseSetupSpeechModel<
126 Model extends RecommendationModel & {
127 readonly supportsLanguageDetection: boolean;
128 },
129 >(
130 models: readonly Model[],
131 input: {
132 readonly selectedModelId: string;
133 readonly intent: string;
134 readonly preferred: readonly string[];
135 readonly benchmarkTier: SpeechBenchmarkTier | undefined;
136 },
137 ) {
138 if (input.intent === "auto" && input.preferred.length === 0) {
139 const model = models.find(({ id }) => id === input.selectedModelId);
140 return model && { model, language: null, recommended: false };
141 }
142 for (const language of input.intent === "auto" ? input.preferred : [input.intent]) {
143 const capable = models.filter((model) => {
144 const effective = effectiveSpeechLanguage(model, input.intent);
145 return effective === language || (effective === "auto" && model.languages.includes(language));
146 });
147 const recommendedId = recommendSpeechModels(capable, [language], input.benchmarkTier)
148 .keys()
149 .next().value;
150 const model =
151 capable.find(({ id }) => id === input.selectedModelId) ??
152 capable.find(({ id }) => id === recommendedId) ??
153 capable[0];
154 if (model) return { model, language, recommended: model.id === recommendedId };
155 }
156 return undefined;
157 }
158
159 const vocabularyKey = (value: string) => value.trim().normalize("NFC").toLocaleLowerCase();
160
161 /**
162 * Project spellings win over personal ones. Hint terms, such as the project name or the
163 * correction cue, only steer recognition, so they travel apart from the dictionary.
164 */
165 export function resolveSpeechVocabulary(input: {
166 readonly personal: SpeechCustomWords;
167 readonly project: SpeechCustomWords;
168 readonly hints?: ReadonlyArray<string>;
169 }): { readonly speechCustomWords: SpeechCustomWords; readonly speechHints: string[] } {
170 const seen = new Set<string>();
171 const speechCustomWords = [...input.project, ...input.personal]
172 .flatMap(({ term, aliases }) => {
173 if (seen.has(vocabularyKey(term))) return [];
174 seen.add(vocabularyKey(term));
175 const uniqueAliases = aliases.filter((alias) => {
176 if (seen.has(vocabularyKey(alias))) return false;
177 seen.add(vocabularyKey(alias));
178 return true;
179 });
180 return [{ term, aliases: uniqueAliases }];
181 })
182 .slice(0, 100);
183 const known = new Set(
184 speechCustomWords.flatMap(({ term, aliases }) => [term, ...aliases].map(vocabularyKey)),
185 );
186 // A hint that is already a dictionary entry keeps that entry and its aliases in place.
187 const speechHints = (input.hints ?? []).flatMap((hint) => {
188 const term = hint.replace(/\s+/g, " ").trim();
189 if (term.length === 0 || term.length > 50 || known.has(vocabularyKey(term))) return [];
190 known.add(vocabularyKey(term));
191 return [term];
192 });
193 return { speechCustomWords, speechHints };
194 }
195
196 /** Binary requests contain a four-byte JSON length, UTF-8 options, then unencoded PCM. */
197 export function encodeSpeechPcmRequest(
198 pcm: Uint8Array,
199 options: SpeechTranscriptionOptions,
200 ): Uint8Array<ArrayBuffer> {
201 const metadata = new TextEncoder().encode(encodeOptions(options));
202 if (metadata.byteLength > SPEECH_MAX_OPTIONS_BYTES)
203 throw new Error("Speech options are too large.");
204 const body = new Uint8Array(4 + metadata.byteLength + pcm.byteLength);
205 new DataView(body.buffer).setUint32(0, metadata.byteLength, true);
206 body.set(metadata, 4);
207 body.set(pcm, 4 + metadata.byteLength);
208 return body;
209 }
210
211 export function decodeSpeechPcmRequest(body: Uint8Array) {
212 if (body.byteLength < 4) throw new Error("Missing speech options.");
213 const length = new DataView(body.buffer, body.byteOffset, body.byteLength).getUint32(0, true);
214 if (length > SPEECH_MAX_OPTIONS_BYTES || 4 + length >= body.byteLength)
215 throw new Error("Invalid speech options length.");
216 const options = decodeOptions(
217 new TextDecoder("utf-8", { fatal: true }).decode(body.subarray(4, 4 + length)),
218 );
219 return { pcm: body.subarray(4 + length), options };
220 }