@devries-cameron20 / t3code-1 / commits / d20e6c294c
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(speech): project name hints no longer recase ordinary words (#39884)

Julius Marminge committed Nov 20, 2026 at 13:17 UTC d20e6c294c527273ac046f56ac96d87f65456e45
10 files changed +72 -47
apps/server/src/speech/CloudSpeech.test.ts
+9 -2
@@ -38,12 +38,15 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved
38 ),
39 ),
40 );
41 + // A full dictionary plus a hint still stays within xAI's 100-term limit.
42 + const words = Array.from({ length: 99 }, (_, index) => `word${index}`);
43 const transcribe = (pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer)) =>
44 service
45 .transcribe(provider, pcm, {
46 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
47 speechLanguage: "en",
46 - speechCustomWords: [{ term: "T3 Code", aliases: [] }],
48 + speechCustomWords: ["T3 Code", ...words].map((term) => ({ term, aliases: [] })),
49 + speechHints: ["Server"],
50 })
51 .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream));
52 yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other"));
@@ -66,7 +69,11 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved
69 expect(form.get("model")).toBe(
70 provider === "openai" ? "gpt-transcribe" : "grok-voice-transcribe-2.0",
71 );
69 - expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual(["T3 Code"]);
72 + expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual([
73 + "Server",
74 + "T3 Code",
75 + ...words.slice(0, 98),
76 + ]);
77 expect(form.get(provider === "openai" ? "languages[]" : "language")).toBe("en");
78 expect([...form.keys()].at(-1)).toBe("file");
79 const wav = new DataView(yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer()));
apps/server/src/speech/CloudSpeech.ts
+5 -2
@@ -23,7 +23,7 @@ import * as Schema from "effect/Schema";
23 import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
24 import * as ServerSettings from "../serverSettings.ts";
25 import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts";
26 -import { applySpeechAliases } from "./customWords.ts";
26 +import { applySpeechAliases, speechRecognitionWords } from "./customWords.ts";
27 import { removeSpeechFillerWords } from "./fillerWords.ts";
28 import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts";
29
@@ -75,7 +75,10 @@ function audioBody(
75 const form = new FormData();
76 form.set("model", endpoints[provider].model);
77 if (language !== "auto") form.set(provider === "openai" ? "languages[]" : "language", language);
78 - for (const { term } of options.speechCustomWords) {
78 + for (const term of speechRecognitionWords(
79 + options.speechHints,
80 + options.speechCustomWords.map(({ term }) => term),
81 + )) {
82 if (provider === "openai" && /[<>\r\n]/.test(term)) continue;
83 form.append(provider === "openai" ? "keywords[]" : "keyterm", term);
84 }
apps/server/src/speech/SpeechService.test.ts
+8 -5
@@ -534,12 +534,13 @@ it.effect("uses vocabulary only for supported transcription and always applies a
534 { term: "Bob", aliases: ["Robert"] },
535 { term: "T3 Code", aliases: [] },
536 ],
537 + speechHints: ["Server"],
538 };
539 const vocabulary =
539 - supportsVocabulary && !translate ? ["GitHub", "Bob", "T3 Code"] : undefined;
540 - native.transcribe.mockResolvedValueOnce({ text: "git hub and Robert" });
540 + supportsVocabulary && !translate ? ["Server", "GitHub", "Bob", "T3 Code"] : undefined;
541 + native.transcribe.mockResolvedValueOnce({ text: "git hub and Robert on the server" });
542 expect(yield* speech.transcribe(pcm(), options)).toBe(
542 - vocabulary ? "git hub and Bob" : "GitHub and Bob",
543 + vocabulary ? "git hub and Bob on the server" : "GitHub and Bob on the server",
544 );
545 const delivered = native.transcribe.mock.lastCall?.[1];
546 expect(delivered).toEqual({
@@ -553,13 +554,15 @@ it.effect("uses vocabulary only for supported transcription and always applies a
554 revision: 1,
555 text: { committed: "git hub", tentative: "Robert" },
556 });
556 - native.finish.mockResolvedValueOnce("git hub and Robert");
557 + native.finish.mockResolvedValueOnce("git hub and Robert on the server");
558 const stream = yield* speech.startStream(options);
559 expect(yield* stream.feed(pcm())).toEqual({
560 revision: 1,
561 text: { committed: vocabulary ? "git hub" : "GitHub", tentative: "Bob" },
562 });
562 - expect(yield* stream.finish).toBe(vocabulary ? "git hub and Bob" : "GitHub and Bob");
563 + expect(yield* stream.finish).toBe(
564 + vocabulary ? "git hub and Bob on the server" : "GitHub and Bob on the server",
565 + );
566 expect(native.begin.mock.lastCall).toEqual([
567 { timestamps: "none", language: "fr", ...(vocabulary ? { vocabulary } : {}) },
568 true,
apps/server/src/speech/SpeechService.ts
+13 -5
@@ -23,7 +23,11 @@ import * as Scope from "effect/Scope";
23 import * as Stream from "effect/Stream";
24
25 import * as ServerSettings from "../serverSettings.ts";
26 -import { makeSpeechTextCorrector, normalizeSpeechCustomWords } from "./customWords.ts";
26 +import {
27 + makeSpeechTextCorrector,
28 + normalizeSpeechCustomWords,
29 + speechRecognitionWords,
30 +} from "./customWords.ts";
31 import { removeSpeechFillerWords } from "./fillerWords.ts";
32 import {
33 DEFAULT_SPEECH_MODEL_ID,
@@ -626,9 +630,10 @@ const make = Effect.gen(function* () {
630 const vadPath = yield* silenceModelPath(loaded, options, false);
631 const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
632 const customWords = dictionary.map(({ term }) => term);
633 + const recognitionWords = speechRecognitionWords(options.speechHints, customWords);
634 // The vocabulary capability guarantees plain transcription, not translation.
635 const vocabularyFor = (inference: LoadedModel) =>
631 - inference.supportsVocabulary && !translateToEnglish ? customWords : [];
636 + inference.supportsVocabulary && !translateToEnglish ? recognitionWords : [];
637 const transcribeWith = (inference: LoadedModel) =>
638 Effect.tryPromise({
639 try: (signal) => {
@@ -643,7 +648,9 @@ const make = Effect.gen(function* () {
648 ...(translateToEnglish
649 ? { task: "translate" as const, targetLanguage: "en" as const }
650 : {}),
646 - ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}),
651 + ...(vocabularyFor(inference).length > 0
652 + ? { vocabulary: recognitionWords }
653 + : {}),
654 },
655 options.speechSilenceFiltering,
656 vadPath,
@@ -753,6 +760,7 @@ const make = Effect.gen(function* () {
760 const language = effectiveSpeechLanguage(definition, options.speechLanguage);
761 const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
762 const customWords = dictionary.map(({ term }) => term);
763 + const recognitionWords = speechRecognitionWords(options.speechHints, customWords);
764 let correct = makeSpeechTextCorrector(customWords, dictionary);
765 const modelPath = yield* installedModelPath(definition);
766 let streamModel = yield* attempt(operation, () =>
@@ -768,8 +776,8 @@ const make = Effect.gen(function* () {
776 streamModel.supportsTranslation &&
777 language !== "en";
778 const vocabulary =
771 - streamModel.supportsVocabulary && !translateToEnglish && customWords.length
772 - ? customWords
779 + streamModel.supportsVocabulary && !translateToEnglish && recognitionWords.length
780 + ? recognitionWords
781 : undefined;
782 const hint = nativeSpeechLanguage(streamModel.languages, language);
783 await streamModel.begin(
apps/server/src/speech/customWords.ts
+6
@@ -194,6 +194,12 @@ export function applySpeechAliases(text: string, words: SpeechCustomWords): stri
194 return makeSpeechTextCorrector([], words)(text);
195 }
196
197 +/** Hints first, so they survive the 100-term cap that cloud providers such as xAI enforce. */
198 +export const speechRecognitionWords = (
199 + hints: readonly string[] | undefined,
200 + terms: readonly string[],
201 +): string[] => [...(hints ?? []), ...terms].slice(0, 100);
202 +
203 const normalizeWord = (word: string) => word.replace(/\s+/g, " ").trim();
204
205 export function normalizeSpeechCustomWords(words: SpeechCustomWords): SpeechCustomWords {
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+7 -10
@@ -901,29 +901,26 @@ it("sends the project's vocabulary and name as hints, and the cue only with clea
901 await act(() => mocks.voice.start());
902 await act(() => mocks.voice.stop());
903 expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual([
904 - { term: "Acme App", aliases: [] },
904 { term: "effect", aliases: ["affect"] },
905 ]);
906 + expect(mocks.recordingOptions.at(-1)?.speechHints).toEqual(["Acme App"]);
907 expect(mocks.cleanupCalls).toEqual([]);
908
909 chooseTranscriptCleanup(true);
910 await screen.rerender(<Probe projectId="proj-1" projectName="Acme App" />);
911 await act(() => mocks.voice.start());
912 await act(() => mocks.voice.stop());
913 - expect(mocks.recordingOptions.at(-1)?.speechCustomWords.map(({ term }) => term)).toEqual([
914 - "scratch that",
915 - "Acme App",
916 - "effect",
917 - ]);
918 - expect(mocks.cleanupOptions.at(-1)).toMatchObject({ speechCorrectionWord: "scratch that" });
913 + expect(mocks.recordingOptions.at(-1)?.speechHints).toEqual(["scratch that", "Acme App"]);
914 + expect(mocks.cleanupOptions.at(-1)).toMatchObject({
915 + speechCorrectionWord: "scratch that",
916 + speechCustomWords: [{ term: "effect", aliases: ["affect"] }],
917 + });
918 expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBe("scratch that");
919 chooseTranscriptCleanup(false);
920 await act(() => mocks.voice.start());
921 await act(() => mocks.voice.stop());
922 expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined();
924 - expect(mocks.recordingOptions.at(-1)?.speechCustomWords.map(({ term }) => term)).not.toContain(
925 - "scratch that",
926 - );
923 + expect(mocks.recordingOptions.at(-1)?.speechHints).not.toContain("scratch that");
924 });
925
926 it("shares one status probe per destination across every mounted editor", async () => {
apps/web/src/speech/voiceInputSession.ts
+2 -1
@@ -134,7 +134,7 @@ export class VoiceInputSession {
134 const cleanupGrant = context.cleanup ? readCleanupGrant() : null;
135 const cleanupEnabled = cleanupGrant !== null;
136 const cue = cleanupEnabled && context.cleanupPrepared ? settings.speechCorrectionWord : "";
137 - const speechCustomWords = resolveSpeechVocabulary({
137 + const { speechCustomWords, speechHints } = resolveSpeechVocabulary({
138 personal: settings.speechCustomWords,
139 project: context.projectWords,
140 hints: [cue, context.projectName ?? ""],
@@ -154,6 +154,7 @@ export class VoiceInputSession {
154 speechLanguage: settings.speechLanguage,
155 speechTranslateToEnglish: settings.speechTranslateToEnglish,
156 speechCustomWords,
157 + ...(speechHints.length > 0 ? { speechHints } : {}),
158 speechRemoveFillerWords: settings.speechRemoveFillerWords,
159 ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
160 speechSilenceFiltering: settings.speechSilenceFiltering,
packages/contracts/src/speech.ts
+2
@@ -293,6 +293,8 @@ export const SpeechTranscriptionOptions = Schema.Struct({
293 speechLanguage: SpeechLanguage,
294 speechTranslateToEnglish: Schema.Boolean,
295 speechCustomWords: SpeechCustomWords,
296 + /** Recognition-only terms, such as the project name; never rewrite the transcript. */
297 + speechHints: Schema.optionalKey(Schema.Array(SpeechCustomWord).check(Schema.isMaxLength(8))),
298 speechRemoveFillerWords: Schema.Boolean,
299 /** Preserve this cue for separately consented transcript cleanup. */
300 speechCorrectionWord: Schema.optionalKey(SpeechCorrectionCue),
packages/shared/src/speech.test.ts
+6 -7
@@ -53,7 +53,7 @@ describe("speech PCM request framing", () => {
53 });
54
55 describe("resolveSpeechVocabulary", () => {
56 - it("prefers project spellings, keeps hints under the cap, and drops duplicates", () => {
56 + it("prefers project spellings and keeps hints out of the dictionary", () => {
57 const personal = Array.from({ length: 100 }, (_, index) => ({
58 term: `word${index}`,
59 aliases: index === 0 ? ["Effect"] : [],
@@ -61,16 +61,15 @@ describe("resolveSpeechVocabulary", () => {
61 const resolved = resolveSpeechVocabulary({
62 personal,
63 project: [{ term: "effect", aliases: ["a fact"] }],
64 - hints: [" T3 Code ", "word1", "", "x".repeat(51)],
64 + hints: [" T3 Code ", "word1", "", "x".repeat(51), "t3 code"],
65 });
66 - expect(resolved).toHaveLength(100);
67 - expect(resolved.slice(0, 3)).toEqual([
68 - { term: "T3 Code", aliases: [] },
66 + expect(resolved.speechHints).toEqual(["T3 Code"]);
67 + expect(resolved.speechCustomWords).toHaveLength(100);
68 + expect(resolved.speechCustomWords.slice(0, 2)).toEqual([
69 { term: "effect", aliases: ["a fact"] },
70 { term: "word0", aliases: [] },
71 ]);
72 - expect(resolved.filter(({ term }) => term === "word1")).toHaveLength(1);
73 - expect(resolved.at(-1)?.term).toBe("word97");
72 + expect(resolved.speechCustomWords.at(-1)?.term).toBe("word98");
73 });
74 });
75
packages/shared/src/speech.ts
+14 -15
@@ -160,27 +160,15 @@ const vocabularyKey = (value: string) => value.trim().normalize("NFC").toLocaleL
160
161 /**
162 * Project spellings win over personal ones. Hint terms, such as the project name or the
163 - * correction cue, come first so the 100-entry wire limit cannot drop them.
163 + * correction cue, only steer recognition, so they travel apart from the dictionary.
164 */
165 export function resolveSpeechVocabulary(input: {
166 readonly personal: SpeechCustomWords;
167 readonly project: SpeechCustomWords;
168 readonly hints?: ReadonlyArray<string>;
169 -}): SpeechCustomWords {
170 - const known = new Set(
171 - [...input.project, ...input.personal].flatMap(({ term, aliases }) =>
172 - [term, ...aliases].map(vocabularyKey),
173 - ),
174 - );
175 - // A hint that is already a dictionary entry keeps that entry and its aliases in place.
176 - const hints = (input.hints ?? []).flatMap((hint) => {
177 - const term = hint.replace(/\s+/g, " ").trim();
178 - return term.length > 0 && term.length <= 50 && !known.has(vocabularyKey(term))
179 - ? [{ term, aliases: [] }]
180 - : [];
181 - });
169 +}): { readonly speechCustomWords: SpeechCustomWords; readonly speechHints: string[] } {
170 const seen = new Set<string>();
183 - return [...hints, ...input.project, ...input.personal]
171 + const speechCustomWords = [...input.project, ...input.personal]
172 .flatMap(({ term, aliases }) => {
173 if (seen.has(vocabularyKey(term))) return [];
174 seen.add(vocabularyKey(term));
@@ -192,6 +180,17 @@ export function resolveSpeechVocabulary(input: {
180 return [{ term, aliases: uniqueAliases }];
181 })
182 .slice(0, 100);
183 + const known = new Set(
184 + speechCustomWords.flatMap(({ term, aliases }) => [term, ...aliases].map(vocabularyKey)),
185 + );
186 + // A hint that is already a dictionary entry keeps that entry and its aliases in place.
187 + const speechHints = (input.hints ?? []).flatMap((hint) => {
188 + const term = hint.replace(/\s+/g, " ").trim();
189 + if (term.length === 0 || term.length > 50 || known.has(vocabularyKey(term))) return [];
190 + known.add(vocabularyKey(term));
191 + return [term];
192 + });
193 + return { speechCustomWords, speechHints };
194 }
195
196 /** Binary requests contain a four-byte JSON length, UTF-8 options, then unencoded PCM. */