The best way to control Claude Code, Codex and more from any computer or phone
fix(speech): project name hints no longer recase ordinary words (#39884)
Julius Marminge committed
Nov 20, 2026 at 13:17 UTC
d20e6c294c527273ac046f56ac96d87f65456e45
10 files changed
+72
-47
apps/server/src/speech/CloudSpeech.test.ts
+9
-2
@@ -38,12 +38,15 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved
38
),
39
),
40
);
41
+ // A full dictionary plus a hint still stays within xAI's 100-term limit.
42
+ const words = Array.from({ length: 99 }, (_, index) => `word${index}`);
43
const transcribe = (pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer)) =>
44
service
45
.transcribe(provider, pcm, {
46
...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
47
speechLanguage: "en",
46
- speechCustomWords: [{ term: "T3 Code", aliases: [] }],
48
+ speechCustomWords: ["T3 Code", ...words].map((term) => ({ term, aliases: [] })),
49
+ speechHints: ["Server"],
50
})
51
.pipe(Effect.provideService(FetchHttpClient.Fetch, upstream));
52
yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other"));
@@ -66,7 +69,11 @@ it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved
69
expect(form.get("model")).toBe(
70
provider === "openai" ? "gpt-transcribe" : "grok-voice-transcribe-2.0",
71
);
69
- expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual(["T3 Code"]);
72
+ expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual([
73
+ "Server",
74
+ "T3 Code",
75
+ ...words.slice(0, 98),
76
+ ]);
77
expect(form.get(provider === "openai" ? "languages[]" : "language")).toBe("en");
78
expect([...form.keys()].at(-1)).toBe("file");
79
const wav = new DataView(yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer()));
apps/server/src/speech/CloudSpeech.ts
+5
-2
@@ -23,7 +23,7 @@ import * as Schema from "effect/Schema";
23
import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
24
import * as ServerSettings from "../serverSettings.ts";
25
import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts";
26
-import { applySpeechAliases } from "./customWords.ts";
26
+import { applySpeechAliases, speechRecognitionWords } from "./customWords.ts";
27
import { removeSpeechFillerWords } from "./fillerWords.ts";
28
import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts";
29
@@ -75,7 +75,10 @@ function audioBody(
75
const form = new FormData();
76
form.set("model", endpoints[provider].model);
77
if (language !== "auto") form.set(provider === "openai" ? "languages[]" : "language", language);
78
- for (const { term } of options.speechCustomWords) {
78
+ for (const term of speechRecognitionWords(
79
+ options.speechHints,
80
+ options.speechCustomWords.map(({ term }) => term),
81
+ )) {
82
if (provider === "openai" && /[<>\r\n]/.test(term)) continue;
83
form.append(provider === "openai" ? "keywords[]" : "keyterm", term);
84
}
apps/server/src/speech/SpeechService.test.ts
+8
-5
@@ -534,12 +534,13 @@ it.effect("uses vocabulary only for supported transcription and always applies a
534
{ term: "Bob", aliases: ["Robert"] },
535
{ term: "T3 Code", aliases: [] },
536
],
537
+ speechHints: ["Server"],
538
};
539
const vocabulary =
539
- supportsVocabulary && !translate ? ["GitHub", "Bob", "T3 Code"] : undefined;
540
- native.transcribe.mockResolvedValueOnce({ text: "git hub and Robert" });
540
+ supportsVocabulary && !translate ? ["Server", "GitHub", "Bob", "T3 Code"] : undefined;
541
+ native.transcribe.mockResolvedValueOnce({ text: "git hub and Robert on the server" });
542
expect(yield* speech.transcribe(pcm(), options)).toBe(
542
- vocabulary ? "git hub and Bob" : "GitHub and Bob",
543
+ vocabulary ? "git hub and Bob on the server" : "GitHub and Bob on the server",
544
);
545
const delivered = native.transcribe.mock.lastCall?.[1];
546
expect(delivered).toEqual({
@@ -553,13 +554,15 @@ it.effect("uses vocabulary only for supported transcription and always applies a
554
revision: 1,
555
text: { committed: "git hub", tentative: "Robert" },
556
});
556
- native.finish.mockResolvedValueOnce("git hub and Robert");
557
+ native.finish.mockResolvedValueOnce("git hub and Robert on the server");
558
const stream = yield* speech.startStream(options);
559
expect(yield* stream.feed(pcm())).toEqual({
560
revision: 1,
561
text: { committed: vocabulary ? "git hub" : "GitHub", tentative: "Bob" },
562
});
562
- expect(yield* stream.finish).toBe(vocabulary ? "git hub and Bob" : "GitHub and Bob");
563
+ expect(yield* stream.finish).toBe(
564
+ vocabulary ? "git hub and Bob on the server" : "GitHub and Bob on the server",
565
+ );
566
expect(native.begin.mock.lastCall).toEqual([
567
{ timestamps: "none", language: "fr", ...(vocabulary ? { vocabulary } : {}) },
568
true,
apps/server/src/speech/SpeechService.ts
+13
-5
@@ -23,7 +23,11 @@ import * as Scope from "effect/Scope";
23
import * as Stream from "effect/Stream";
24
25
import * as ServerSettings from "../serverSettings.ts";
26
-import { makeSpeechTextCorrector, normalizeSpeechCustomWords } from "./customWords.ts";
26
+import {
27
+ makeSpeechTextCorrector,
28
+ normalizeSpeechCustomWords,
29
+ speechRecognitionWords,
30
+} from "./customWords.ts";
31
import { removeSpeechFillerWords } from "./fillerWords.ts";
32
import {
33
DEFAULT_SPEECH_MODEL_ID,
@@ -626,9 +630,10 @@ const make = Effect.gen(function* () {
630
const vadPath = yield* silenceModelPath(loaded, options, false);
631
const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
632
const customWords = dictionary.map(({ term }) => term);
633
+ const recognitionWords = speechRecognitionWords(options.speechHints, customWords);
634
// The vocabulary capability guarantees plain transcription, not translation.
635
const vocabularyFor = (inference: LoadedModel) =>
631
- inference.supportsVocabulary && !translateToEnglish ? customWords : [];
636
+ inference.supportsVocabulary && !translateToEnglish ? recognitionWords : [];
637
const transcribeWith = (inference: LoadedModel) =>
638
Effect.tryPromise({
639
try: (signal) => {
@@ -643,7 +648,9 @@ const make = Effect.gen(function* () {
648
...(translateToEnglish
649
? { task: "translate" as const, targetLanguage: "en" as const }
650
: {}),
646
- ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}),
651
+ ...(vocabularyFor(inference).length > 0
652
+ ? { vocabulary: recognitionWords }
653
+ : {}),
654
},
655
options.speechSilenceFiltering,
656
vadPath,
@@ -753,6 +760,7 @@ const make = Effect.gen(function* () {
760
const language = effectiveSpeechLanguage(definition, options.speechLanguage);
761
const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
762
const customWords = dictionary.map(({ term }) => term);
763
+ const recognitionWords = speechRecognitionWords(options.speechHints, customWords);
764
let correct = makeSpeechTextCorrector(customWords, dictionary);
765
const modelPath = yield* installedModelPath(definition);
766
let streamModel = yield* attempt(operation, () =>
@@ -768,8 +776,8 @@ const make = Effect.gen(function* () {
776
streamModel.supportsTranslation &&
777
language !== "en";
778
const vocabulary =
771
- streamModel.supportsVocabulary && !translateToEnglish && customWords.length
772
- ? customWords
779
+ streamModel.supportsVocabulary && !translateToEnglish && recognitionWords.length
780
+ ? recognitionWords
781
: undefined;
782
const hint = nativeSpeechLanguage(streamModel.languages, language);
783
await streamModel.begin(
apps/server/src/speech/customWords.ts
+6
@@ -194,6 +194,12 @@ export function applySpeechAliases(text: string, words: SpeechCustomWords): stri
194
return makeSpeechTextCorrector([], words)(text);
195
}
196
197
+/** Hints first, so they survive the 100-term cap that cloud providers such as xAI enforce. */
198
+export const speechRecognitionWords = (
199
+ hints: readonly string[] | undefined,
200
+ terms: readonly string[],
201
+): string[] => [...(hints ?? []), ...terms].slice(0, 100);
202
+
203
const normalizeWord = (word: string) => word.replace(/\s+/g, " ").trim();
204
205
export function normalizeSpeechCustomWords(words: SpeechCustomWords): SpeechCustomWords {
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+7
-10
@@ -901,29 +901,26 @@ it("sends the project's vocabulary and name as hints, and the cue only with clea
901
await act(() => mocks.voice.start());
902
await act(() => mocks.voice.stop());
903
expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual([
904
- { term: "Acme App", aliases: [] },
904
{ term: "effect", aliases: ["affect"] },
905
]);
906
+ expect(mocks.recordingOptions.at(-1)?.speechHints).toEqual(["Acme App"]);
907
expect(mocks.cleanupCalls).toEqual([]);
908
909
chooseTranscriptCleanup(true);
910
await screen.rerender(<Probe projectId="proj-1" projectName="Acme App" />);
911
await act(() => mocks.voice.start());
912
await act(() => mocks.voice.stop());
913
- expect(mocks.recordingOptions.at(-1)?.speechCustomWords.map(({ term }) => term)).toEqual([
914
- "scratch that",
915
- "Acme App",
916
- "effect",
917
- ]);
918
- expect(mocks.cleanupOptions.at(-1)).toMatchObject({ speechCorrectionWord: "scratch that" });
913
+ expect(mocks.recordingOptions.at(-1)?.speechHints).toEqual(["scratch that", "Acme App"]);
914
+ expect(mocks.cleanupOptions.at(-1)).toMatchObject({
915
+ speechCorrectionWord: "scratch that",
916
+ speechCustomWords: [{ term: "effect", aliases: ["affect"] }],
917
+ });
918
expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBe("scratch that");
919
chooseTranscriptCleanup(false);
920
await act(() => mocks.voice.start());
921
await act(() => mocks.voice.stop());
922
expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined();
924
- expect(mocks.recordingOptions.at(-1)?.speechCustomWords.map(({ term }) => term)).not.toContain(
925
- "scratch that",
926
- );
923
+ expect(mocks.recordingOptions.at(-1)?.speechHints).not.toContain("scratch that");
924
});
925
926
it("shares one status probe per destination across every mounted editor", async () => {
apps/web/src/speech/voiceInputSession.ts
+2
-1
@@ -134,7 +134,7 @@ export class VoiceInputSession {
134
const cleanupGrant = context.cleanup ? readCleanupGrant() : null;
135
const cleanupEnabled = cleanupGrant !== null;
136
const cue = cleanupEnabled && context.cleanupPrepared ? settings.speechCorrectionWord : "";
137
- const speechCustomWords = resolveSpeechVocabulary({
137
+ const { speechCustomWords, speechHints } = resolveSpeechVocabulary({
138
personal: settings.speechCustomWords,
139
project: context.projectWords,
140
hints: [cue, context.projectName ?? ""],
@@ -154,6 +154,7 @@ export class VoiceInputSession {
154
speechLanguage: settings.speechLanguage,
155
speechTranslateToEnglish: settings.speechTranslateToEnglish,
156
speechCustomWords,
157
+ ...(speechHints.length > 0 ? { speechHints } : {}),
158
speechRemoveFillerWords: settings.speechRemoveFillerWords,
159
...(cue.trim() ? { speechCorrectionWord: cue } : {}),
160
speechSilenceFiltering: settings.speechSilenceFiltering,
packages/contracts/src/speech.ts
+2
@@ -293,6 +293,8 @@ export const SpeechTranscriptionOptions = Schema.Struct({
293
speechLanguage: SpeechLanguage,
294
speechTranslateToEnglish: Schema.Boolean,
295
speechCustomWords: SpeechCustomWords,
296
+ /** Recognition-only terms, such as the project name; never rewrite the transcript. */
297
+ speechHints: Schema.optionalKey(Schema.Array(SpeechCustomWord).check(Schema.isMaxLength(8))),
298
speechRemoveFillerWords: Schema.Boolean,
299
/** Preserve this cue for separately consented transcript cleanup. */
300
speechCorrectionWord: Schema.optionalKey(SpeechCorrectionCue),
packages/shared/src/speech.test.ts
+6
-7
@@ -53,7 +53,7 @@ describe("speech PCM request framing", () => {
53
});
54
55
describe("resolveSpeechVocabulary", () => {
56
- it("prefers project spellings, keeps hints under the cap, and drops duplicates", () => {
56
+ it("prefers project spellings and keeps hints out of the dictionary", () => {
57
const personal = Array.from({ length: 100 }, (_, index) => ({
58
term: `word${index}`,
59
aliases: index === 0 ? ["Effect"] : [],
@@ -61,16 +61,15 @@ describe("resolveSpeechVocabulary", () => {
61
const resolved = resolveSpeechVocabulary({
62
personal,
63
project: [{ term: "effect", aliases: ["a fact"] }],
64
- hints: [" T3 Code ", "word1", "", "x".repeat(51)],
64
+ hints: [" T3 Code ", "word1", "", "x".repeat(51), "t3 code"],
65
});
66
- expect(resolved).toHaveLength(100);
67
- expect(resolved.slice(0, 3)).toEqual([
68
- { term: "T3 Code", aliases: [] },
66
+ expect(resolved.speechHints).toEqual(["T3 Code"]);
67
+ expect(resolved.speechCustomWords).toHaveLength(100);
68
+ expect(resolved.speechCustomWords.slice(0, 2)).toEqual([
69
{ term: "effect", aliases: ["a fact"] },
70
{ term: "word0", aliases: [] },
71
]);
72
- expect(resolved.filter(({ term }) => term === "word1")).toHaveLength(1);
73
- expect(resolved.at(-1)?.term).toBe("word97");
72
+ expect(resolved.speechCustomWords.at(-1)?.term).toBe("word98");
73
});
74
});
75
packages/shared/src/speech.ts
+14
-15
@@ -160,27 +160,15 @@ const vocabularyKey = (value: string) => value.trim().normalize("NFC").toLocaleL
160
161
/**
162
* Project spellings win over personal ones. Hint terms, such as the project name or the
163
- * correction cue, come first so the 100-entry wire limit cannot drop them.
163
+ * correction cue, only steer recognition, so they travel apart from the dictionary.
164
*/
165
export function resolveSpeechVocabulary(input: {
166
readonly personal: SpeechCustomWords;
167
readonly project: SpeechCustomWords;
168
readonly hints?: ReadonlyArray<string>;
169
-}): SpeechCustomWords {
170
- const known = new Set(
171
- [...input.project, ...input.personal].flatMap(({ term, aliases }) =>
172
- [term, ...aliases].map(vocabularyKey),
173
- ),
174
- );
175
- // A hint that is already a dictionary entry keeps that entry and its aliases in place.
176
- const hints = (input.hints ?? []).flatMap((hint) => {
177
- const term = hint.replace(/\s+/g, " ").trim();
178
- return term.length > 0 && term.length <= 50 && !known.has(vocabularyKey(term))
179
- ? [{ term, aliases: [] }]
180
- : [];
181
- });
169
+}): { readonly speechCustomWords: SpeechCustomWords; readonly speechHints: string[] } {
170
const seen = new Set<string>();
183
- return [...hints, ...input.project, ...input.personal]
171
+ const speechCustomWords = [...input.project, ...input.personal]
172
.flatMap(({ term, aliases }) => {
173
if (seen.has(vocabularyKey(term))) return [];
174
seen.add(vocabularyKey(term));
@@ -192,6 +180,17 @@ export function resolveSpeechVocabulary(input: {
180
return [{ term, aliases: uniqueAliases }];
181
})
182
.slice(0, 100);
183
+ const known = new Set(
184
+ speechCustomWords.flatMap(({ term, aliases }) => [term, ...aliases].map(vocabularyKey)),
185
+ );
186
+ // A hint that is already a dictionary entry keeps that entry and its aliases in place.
187
+ const speechHints = (input.hints ?? []).flatMap((hint) => {
188
+ const term = hint.replace(/\s+/g, " ").trim();
189
+ if (term.length === 0 || term.length > 50 || known.has(vocabularyKey(term))) return [];
190
+ known.add(vocabularyKey(term));
191
+ return [term];
192
+ });
193
+ return { speechCustomWords, speechHints };
194
}
195
196
/** Binary requests contain a four-byte JSON length, UTF-8 options, then unencoded PCM. */