@devries-cameron20 / t3code-1 / commits / 15b7275595
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(web): first-run voice setup offers a model for the user's language (#32035)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

Julius Marminge committed Nov 6, 2026 at 21:51 UTC 15b72755956c3338dd98440a7ed35beffe46b0b7
28 files changed +633 -86
apps/desktop/src/app/DesktopAppIdentity.test.ts
+1
@@ -42,6 +42,7 @@ const layerElectronApp = (calls: ElectronAppCalls) =>
42 metadata: Effect.die("unexpected metadata read"),
43 name: Effect.succeed("T3 Code"),
44 systemLocale: Effect.succeed("en-US"),
45 + preferredSystemLanguages: Effect.succeed(["en-US"]),
46 whenReady: Effect.void,
47 requestSingleInstanceLock: Effect.succeed(true),
48 quit: Effect.void,
apps/desktop/src/app/DesktopLifecycle.test.ts
+1
@@ -39,6 +39,7 @@ function layerElectronApp(
39 metadata: Effect.die("unexpected metadata read"),
40 name: Effect.succeed("T3 Code"),
41 systemLocale: Effect.succeed("en-US"),
42 + preferredSystemLanguages: Effect.succeed(["en-US"]),
43 whenReady: Effect.void,
44 requestSingleInstanceLock: Effect.succeed(true),
45 quit,
apps/desktop/src/electron/ElectronApp.ts
+2
@@ -46,6 +46,7 @@ export class ElectronApp extends Context.Service<
46 readonly name: Effect.Effect<string>;
47 /** Packaged builds ship only en-US; app.getLocale() and Intl defaults do not reflect the OS locale. */
48 readonly systemLocale: Effect.Effect<string>;
49 + readonly preferredSystemLanguages: Effect.Effect<ReadonlyArray<string>>;
50 readonly whenReady: Effect.Effect<void, ElectronAppWhenReadyError>;
51 readonly requestSingleInstanceLock: Effect.Effect<boolean>;
52 readonly quit: Effect.Effect<void>;
@@ -155,6 +156,7 @@ export const make = ElectronApp.of({
156 }
157 return toSystemLocaleTag(identifier, hourCycle);
158 }),
159 + preferredSystemLanguages: Effect.sync(() => Electron.app.getPreferredSystemLanguages()),
160 whenReady: Effect.gen(function* () {
161 const isPackaged = Electron.app.isPackaged;
162 yield* Effect.tryPromise({
apps/desktop/src/ipc/DesktopIpcHandlers.ts
+2
@@ -38,6 +38,7 @@ import {
38 getLocalEnvironmentBearerToken,
39 installRelayTelemetryEnabled,
40 getSystemLocale,
41 + getPreferredSystemLanguages,
42 getWindowFullscreenState,
43 openExternal,
44 readMedia,
@@ -94,6 +95,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"
95 yield* ipc.handleSync(getAppBranding);
96 yield* installRelayTelemetryEnabled();
97 yield* ipc.handleSync(getSystemLocale);
98 + yield* ipc.handleSync(getPreferredSystemLanguages);
99 yield* ipc.handleSync(getWindowFullscreenState);
100 yield* ipc.handleSync(getLocalEnvironmentBootstraps);
101 yield* ipc.handleSync(getLocalEnvironmentEnabled);
apps/desktop/src/ipc/channels.ts
+1
@@ -34,6 +34,7 @@ export const CLI_COMMAND_UNINSTALL_CHANNEL = "desktop:cli-command-uninstall";
34 export const GET_APP_BRANDING_CHANNEL = "desktop:get-app-branding";
35 export const GET_RELAY_TELEMETRY_ENABLED_CHANNEL = "desktop:get-relay-telemetry-enabled";
36 export const GET_SYSTEM_LOCALE_CHANNEL = "desktop:get-system-locale";
37 +export const GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL = "desktop:get-preferred-system-languages";
38 export const GET_LOCAL_ENVIRONMENT_BOOTSTRAPS_CHANNEL = "desktop:get-local-environment-bootstraps";
39 export const GET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:get-local-environment-enabled";
40 export const SET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:set-local-environment-enabled";
apps/desktop/src/ipc/methods/window.ts
+9
@@ -106,6 +106,15 @@ export const getSystemLocale = DesktopIpc.makeSyncIpcMethod({
106 }),
107 });
108
109 +export const getPreferredSystemLanguages = DesktopIpc.makeSyncIpcMethod({
110 + channel: IpcChannels.GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL,
111 + result: Schema.Array(Schema.String),
112 + handler: Effect.fn("desktop.ipc.window.getPreferredSystemLanguages")(function* () {
113 + const electronApp = yield* ElectronApp.ElectronApp;
114 + return yield* electronApp.preferredSystemLanguages;
115 + }),
116 +});
117 +
118 export const getWindowFullscreenState = DesktopIpc.makeSyncIpcMethod({
119 channel: IpcChannels.GET_WINDOW_FULLSCREEN_STATE_CHANNEL,
120 result: Schema.Boolean,
apps/desktop/src/preload.ts
+6
@@ -107,6 +107,12 @@ contextBridge.exposeInMainWorld("desktopBridge", {
107 const result = ipcRenderer.sendSync(IpcChannels.GET_SYSTEM_LOCALE_CHANNEL);
108 return typeof result === "string" ? result : null;
109 },
110 + getPreferredSystemLanguages: () => {
111 + const result: unknown = ipcRenderer.sendSync(
112 + IpcChannels.GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL,
113 + );
114 + return Array.isArray(result) ? result.filter((tag) => typeof tag === "string") : [];
115 + },
116 getLocalEnvironmentBootstraps: () => {
117 const result = ipcRenderer.sendSync(IpcChannels.GET_LOCAL_ENVIRONMENT_BOOTSTRAPS_CHANNEL);
118 if (!Array.isArray(result)) {
apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.ts
+1
@@ -37,6 +37,7 @@ function layerElectronApp(
37 metadata: Effect.die("unexpected metadata read"),
38 name: Effect.succeed("T3 Code"),
39 systemLocale: Effect.succeed("en-US"),
40 + preferredSystemLanguages: Effect.succeed(["en-US"]),
41 whenReady: Effect.void,
42 requestSingleInstanceLock: Effect.succeed(true),
43 quit: Effect.void,
apps/desktop/src/window/DesktopApplicationMenu.test.ts
+1
@@ -31,6 +31,7 @@ const layerElectronApp = Layer.succeed(ElectronApp.ElectronApp, {
31 metadata: Effect.die("unexpected metadata read"),
32 name: Effect.succeed("T3 Code"),
33 systemLocale: Effect.succeed("en-US"),
34 + preferredSystemLanguages: Effect.succeed(["en-US"]),
35 whenReady: Effect.void,
36 requestSingleInstanceLock: Effect.succeed(true),
37 quit: Effect.void,
apps/server/src/speech/SpeechService.test.ts
+56
@@ -25,6 +25,7 @@ const native = vi.hoisted(() => ({
25 supportsLongForm: false,
26 supportsTranslation: true,
27 supportsVocabulary: false,
28 + languages: ["en", "fr"] as readonly string[],
29 begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}),
30 feed: vi.fn(async (_pcm: Float32Array) => ({
31 revision: 1,
@@ -119,6 +120,7 @@ beforeEach(() => {
120 native.backend = "Vulkan0";
121 native.supportsVocabulary = false;
122 native.supportsStreaming = true;
123 + native.languages = ["en", "fr"];
124 readyModels.clear();
125 readyModels.add("test-model");
126 readyModels.add("fallback-model");
@@ -171,6 +173,60 @@ it.effect("prepares a batch model before audio arrives and reuses it for transcr
173 }).pipe(Effect.provide(layer)),
174 );
175
176 +it.effect("sends the chosen language in the spelling the loaded model advertises", () =>
177 + Effect.gen(function* () {
178 + const speech = yield* SpeechService.SpeechService;
179 + // Nemotron Streaming 3.5 advertises regional tags while the catalog lists bare codes.
180 + native.languages = ["en-US", "en-GB", "fr-FR"];
181 + yield* Effect.gen(function* () {
182 + const stream = yield* speech.startStream({
183 + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
184 + speechLanguage: "fr",
185 + });
186 + expect(native.begin).toHaveBeenLastCalledWith(
187 + { timestamps: "none", language: "fr-FR" },
188 + true,
189 + "test.gguf",
190 + );
191 + expect(yield* stream.finish).toBe("hello");
192 + }).pipe(Effect.scoped);
193 + native.supportsStreaming = false;
194 + expect(
195 + yield* speech.transcribe(pcm(), {
196 + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
197 + speechLanguage: "fr",
198 + }),
199 + ).toBe("hello");
200 + expect(native.transcribe).toHaveBeenLastCalledWith(
201 + expect.any(Float32Array),
202 + { timestamps: "none", language: "fr-FR" },
203 + true,
204 + "test.gguf",
205 + );
206 + // A language the model does not advertise reaches it unchanged, so the model reports it.
207 + yield* speech.updateAcceleration("cpu");
208 + native.languages = ["en-US"];
209 + native.transcribe.mockRejectedValueOnce(new Error("transcribe_run: unsupported language"));
210 + const rejected = yield* Effect.result(
211 + speech.transcribe(pcm(), { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "fr" }),
212 + );
213 + expect(Result.isFailure(rejected) && rejected.failure).toMatchObject({
214 + _tag: "SpeechOperationError",
215 + operation: "inference",
216 + });
217 + expect(native.transcribe).toHaveBeenLastCalledWith(
218 + expect.any(Float32Array),
219 + { timestamps: "none", language: "fr" },
220 + true,
221 + "test.gguf",
222 + );
223 + expect(loadNative.mock.calls.map(([path, , , acceleration]) => [path, acceleration])).toEqual([
224 + ["test.gguf", "auto"],
225 + ["test.gguf", "cpu"],
226 + ]);
227 + }).pipe(Effect.provide(layer)),
228 +);
229 +
230 it.effect.each(["cancellation", "timeout"] as const)(
231 "releases a stalled model preparation on %s",
232 (reason) =>
apps/server/src/speech/SpeechService.ts
+5 -2
@@ -32,6 +32,7 @@ import {
32 type SpeechModel,
33 } from "./model.ts";
34 import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts";
35 +import { nativeSpeechLanguage } from "./nativeLanguage.ts";
36 import { SILERO_VAD } from "./silero.ts";
37 import { isSpeechPlatformSupported } from "./platform.ts";
38 import * as SpeechModelStore from "./SpeechModelStore.ts";
@@ -613,11 +614,12 @@ const make = Effect.gen(function* () {
614 try: (signal) => {
615 // Interruption kills the isolated process; the next request reloads the model.
616 signal.addEventListener("abort", () => dropModel(inference), { once: true });
617 + const hint = nativeSpeechLanguage(inference.languages, language);
618 return inference.transcribe(
619 pcm,
620 {
621 timestamps: "none",
620 - ...(language === "auto" ? {} : { language }),
622 + ...(hint ? { language: hint } : {}),
623 ...(translateToEnglish
624 ? { task: "translate" as const, targetLanguage: "en" as const }
625 : {}),
@@ -749,10 +751,11 @@ const make = Effect.gen(function* () {
751 streamModel.supportsVocabulary && !translateToEnglish && customWords.length
752 ? customWords
753 : undefined;
754 + const hint = nativeSpeechLanguage(streamModel.languages, language);
755 await streamModel.begin(
756 {
757 timestamps: "none",
755 - ...(language === "auto" ? {} : { language }),
758 + ...(hint ? { language: hint } : {}),
759 ...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}),
760 ...(vocabulary ? { vocabulary } : {}),
761 },
apps/server/src/speech/native.test.ts
+2 -1
@@ -4,7 +4,7 @@ import { expect, it } from "vite-plus/test";
4 import { loadNativeSpeechModel } from "./native.ts";
5
6 const fixture = (transcribe: string, maxAudioMs = 0, supportsVocabulary = false) =>
7 - `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs} }, supports: (feature) => feature === "vocabulary" && ${supportsVocabulary}, transcribe: ${transcribe} }) };`)}`;
7 + `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs}, languages: ["en-US", "de-DE"] }, supports: (feature) => feature === "vocabulary" && ${supportsVocabulary}, transcribe: ${transcribe} }) };`)}`;
8
9 const fixtureWithUnrelatedIpcMessage = () =>
10 `data:text/javascript,${encodeURIComponent(`
@@ -38,6 +38,7 @@ it("returns transcription from an isolated process and disposes it", async () =>
38 fixture("async (pcm) => ({ text: String(pcm[0]) })"),
39 );
40 try {
41 + expect(model.languages).toEqual(["en-US", "de-DE"]);
42 expect(
43 await model.transcribe(
44 new Float32Array([0.25]),
apps/server/src/speech/native.ts
+5
@@ -147,6 +147,7 @@ process.on("message", async (message) => {
147 throw new Error("The selected speech GPU is unavailable.");
148 model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration });
149 process.send({ type: "t3-speech-reply", ok: true, backend: model.backend,
150 + languages: model.capabilities.languages,
151 supportsStreaming: model.capabilities.supportsStreaming,
152 supportsLongForm: model.supports("long_form"),
153 supportsTranslation: model.capabilities.supportsTranslate,
@@ -194,6 +195,7 @@ type Reply = {
195 readonly supportsTranslation?: boolean;
196 readonly supportsVocabulary?: boolean;
197 readonly backend?: string;
198 + readonly languages?: unknown;
199 readonly revision?: number;
200 readonly preview?: SpeechStreamText | null;
201 };
@@ -391,6 +393,9 @@ async function loadNativeSpeechModelOnce(
393 }
394 return {
395 backend: loaded.backend ?? "unknown",
396 + languages: Array.isArray(loaded.languages)
397 + ? loaded.languages.filter((entry): entry is string => typeof entry === "string")
398 + : [],
399 supportsStreaming: loaded.supportsStreaming === true,
400 supportsLongForm: loaded.supportsLongForm === true,
401 supportsTranslation: loaded.supportsTranslation === true,
apps/server/src/speech/nativeLanguage.test.ts new
+18
@@ -0,0 +1,18 @@
1 +import { expect, it } from "vite-plus/test";
2 +
3 +import { nativeSpeechLanguage } from "./nativeLanguage.ts";
4 +
5 +const nemotron = ["en-US", "en-GB", "es-US", "es-ES", "de-DE"];
6 +
7 +it.each([
8 + ["auto omits the hint", nemotron, "auto", undefined],
9 + ["exact entry", ["en", "de"], "de", "de"],
10 + ["exact entry before a regional one", ["de-DE", "de"], "de", "de"],
11 + ["case and underscore normalize to the advertised spelling", nemotron, "DE_de", "de-DE"],
12 + ["bare code takes the first regional entry", nemotron, "en", "en-US"],
13 + ["regional request with another region passes through", nemotron, "de-AT", "de-AT"],
14 + ["unadvertised language passes through", nemotron, "fr", "fr"],
15 + ["model without an advertised list passes through", [], "de", "de"],
16 +])("%s", (_name, advertised, language, expected) => {
17 + expect(nativeSpeechLanguage(advertised, language)).toBe(expected);
18 +});
apps/server/src/speech/nativeLanguage.ts new
+21
@@ -0,0 +1,21 @@
1 +/**
2 + * transcribe.cpp accepts a language only when it equals an entry of the GGUF's
3 + * `general.languages` byte for byte. Nemotron Streaming 3.5 lists `de-DE` where the catalog
4 + * says `de`, so the request carries the loaded model's own spelling. "auto" omits the hint.
5 + */
6 +export function nativeSpeechLanguage(
7 + advertised: readonly string[],
8 + language: string,
9 +): string | undefined {
10 + if (language === "auto") return undefined;
11 + const tag = (value: string) => value.toLowerCase().replaceAll("_", "-");
12 + const wanted = tag(language);
13 + return (
14 + advertised.find((entry) => entry === language) ??
15 + advertised.find((entry) => tag(entry) === wanted) ??
16 + (wanted.includes("-")
17 + ? undefined
18 + : advertised.find((entry) => tag(entry).split("-", 1)[0] === wanted)) ??
19 + language
20 + );
21 +}
apps/web/src/components/chat/ChatComposer.tsx
+2
@@ -3709,9 +3709,11 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
3709 step={speechInput.setup.step}
3710 status={speechInput.status}
3711 model={speechInput.setup.model}
3712 + language={speechInput.setup.language}
3713 downloading={speechInput.setup.downloading}
3714 error={speechInput.setup.error}
3715 onOpenChange={speechInput.setup.setOpen}
3716 + onRetryCatalog={speechInput.setup.retryCatalog}
3717 onDownload={() => void speechInput.setup.download()}
3718 onCancelDownload={() => void speechInput.setup.cancelDownload()}
3719 onStartRecording={() => void speechInput.setup.startRecording()}
apps/web/src/components/chat/VoiceInputSetup.tsx
+68 -53
@@ -13,6 +13,7 @@ import { useClientSettings, updateClientSettings } from "../../hooks/useSettings
13 import { useEnvironment } from "../../state/environments";
14 import { AuthSettingsWriteScope } from "@t3tools/contracts";
15 import { useEnvironmentScope } from "../../state/session";
16 +import { languageLabel } from "../settings/SpeechLanguageSelect";
17 import { Button } from "../ui/button";
18 import { Dialog } from "../ui/dialog";
19 import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
@@ -24,9 +25,12 @@ export function VoiceInputSetup(props: {
25 step: number;
26 status: EnvironmentSpeechStatus | null;
27 model: EnvironmentSpeechModel | null;
28 + /** The language the offered model was chosen for. */
29 + language: string | null;
30 downloading: boolean;
31 error: string | null;
32 onOpenChange(open: boolean): void;
33 + onRetryCatalog: (() => void) | null;
34 onDownload(): void;
35 onCancelDownload(): void;
36 onStartRecording(): void;
@@ -85,49 +89,55 @@ export function VoiceInputSetup(props: {
89 <WizardPanel>
90 {props.step === 0 ? (
91 <section className="space-y-3 text-sm">
88 - <div className="rounded-lg border border-border/70 bg-card/60 p-4">
89 - <div className="flex items-center justify-between gap-2">
90 - <h3 className="font-medium">{status?.model ?? "English transcription model"}</h3>
91 - {status?.modelId === "handy-computer/parakeet-unified-en-0.6b-gguf" ? (
92 - <span className="text-xs font-medium text-primary">
93 - Recommended for English
94 - </span>
92 + {model ? (
93 + <div className="rounded-lg border border-border/70 bg-card/60 p-4">
94 + <div className="flex items-center justify-between gap-2">
95 + <h3 className="font-medium">{model.name}</h3>
96 + {model?.recommended && props.language ? (
97 + <span className="text-xs font-medium text-primary">
98 + Recommended for {languageLabel(props.language)}
99 + </span>
100 + ) : null}
101 + </div>
102 + <p className="mt-2 text-muted-foreground">
103 + Download the {Math.round(model.size / 1024 / 1024)} MB model to{" "}
104 + {environment?.label ?? "your transcription environment"}. Microphone audio is
105 + sent there for transcription, as chosen in Voice settings.
106 + </p>
107 + <p className="mt-2 text-muted-foreground">
108 + Silero (1.2 MB), used for silence filtering and pause detection, downloads
109 + alongside the speech model. An installed copy is reused.
110 + </p>
111 + {props.downloading ? (
112 + <div role="status" className="mt-4 space-y-1 text-muted-foreground">
113 + <div
114 + role="progressbar"
115 + aria-label="Speech model download progress"
116 + aria-valuemin={0}
117 + aria-valuemax={100}
118 + aria-valuenow={progress ?? undefined}
119 + className="h-1 overflow-hidden rounded-full bg-muted"
120 + >
121 + <div
122 + className="h-full bg-primary transition-[width] duration-200"
123 + style={{ width: `${progress ?? 0}%` }}
124 + />
125 + </div>
126 + <p className="text-xs">
127 + {model?.state === "verifying"
128 + ? "Verifying download…"
129 + : progress === null
130 + ? "Starting download…"
131 + : `${progress}% downloaded`}
132 + </p>
133 + </div>
134 ) : null}
135 </div>
97 - <p className="mt-2 text-muted-foreground">
98 - Download the {status ? Math.round(status.size / 1024 / 1024) : 697} MB model to{" "}
99 - {environment?.label ?? "your transcription environment"}. Microphone audio is sent
100 - there for transcription, as chosen in Voice settings.
136 + ) : !props.error ? (
137 + <p role="status" className="text-muted-foreground">
138 + Loading transcription models…
139 </p>
102 - <p className="mt-2 text-muted-foreground">
103 - Silero (1.2 MB), used for silence filtering and pause detection, downloads
104 - alongside the speech model. An installed copy is reused.
105 - </p>
106 - {props.downloading ? (
107 - <div role="status" className="mt-4 space-y-1 text-muted-foreground">
108 - <div
109 - role="progressbar"
110 - aria-label="Speech model download progress"
111 - aria-valuemin={0}
112 - aria-valuemax={100}
113 - aria-valuenow={progress ?? undefined}
114 - className="h-1 overflow-hidden rounded-full bg-muted"
115 - >
116 - <div
117 - className="h-full bg-primary transition-[width] duration-200"
118 - style={{ width: `${progress ?? 0}%` }}
119 - />
120 - </div>
121 - <p className="text-xs">
122 - {model?.state === "verifying"
123 - ? "Verifying download…"
124 - : progress === null
125 - ? "Starting download…"
126 - : `${progress}% downloaded`}
127 - </p>
128 - </div>
129 - ) : null}
130 - </div>
140 + ) : null}
141 {props.error ? (
142 <p role="alert" className="text-destructive">
143 {props.error}
@@ -208,25 +218,30 @@ export function VoiceInputSetup(props: {
218 {props.step === 0 ? (
219 <>
220 <Button variant="outline" onClick={() => openSettings("speech-models")}>
211 - Choose another model or language
221 + {model ? "Choose another model or language" : "Choose a model or language"}
222 </Button>
223 + {props.onRetryCatalog ? (
224 + <Button onClick={props.onRetryCatalog}>Retry loading models</Button>
225 + ) : null}
226 {props.downloading ? (
227 <Button variant="outline" disabled={!canManage} onClick={props.onCancelDownload}>
228 Cancel download
229 </Button>
230 ) : null}
218 - <Button
219 - disabled={!canManage || props.downloading || !status}
220 - onClick={props.onDownload}
221 - >
222 - {props.downloading
223 - ? "Downloading…"
224 - : props.error
225 - ? "Retry download"
226 - : canManage
227 - ? "Download model"
228 - : "Ask an environment admin to download"}
229 - </Button>
231 + {model ? (
232 + <Button
233 + disabled={!canManage || props.downloading || !status}
234 + onClick={props.onDownload}
235 + >
236 + {props.downloading
237 + ? "Downloading…"
238 + : props.error
239 + ? "Try again"
240 + : canManage
241 + ? "Download model"
242 + : "Ask an environment admin to download"}
243 + </Button>
244 + ) : null}
245 </>
246 ) : (
247 <>
apps/web/src/components/preview/PreviewAnnotationSpeech.tsx
+2
@@ -184,9 +184,11 @@ export function PreviewAnnotationSpeech({
184 step={speech.setup.step}
185 status={speech.status}
186 model={speech.setup.model}
187 + language={speech.setup.language}
188 downloading={speech.setup.downloading}
189 error={speech.setup.error}
190 onOpenChange={speech.setup.setOpen}
191 + onRetryCatalog={speech.setup.retryCatalog}
192 onDownload={() => void speech.setup.download()}
193 onCancelDownload={() => void speech.setup.cancelDownload()}
194 onStartRecording={() => void speech.setup.startRecording()}
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+3
@@ -161,7 +161,10 @@ it("tests the selected model after download and preserves its transcript across
161 open: false,
162 step: 0,
163 model: null,
164 + language: null,
165 downloading: false,
166 + catalogFailed: false,
167 + retryCatalog: null,
168 error: null,
169 setOpen: () => {},
170 download: async () => {},
apps/web/src/components/settings/VoiceSettingsPanel.tsx
+4 -1
@@ -47,6 +47,7 @@ import { useEffect, useRef, useState, type ReactNode } from "react";
47
48 import { useStableCallback } from "~/hooks/useStableCallback";
49 import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";
50 +import { preferredSpeechLanguages } from "../../speech/preferredSpeechLanguages";
51 import {
52 chooseAudioDestination,
53 chooseTranscriptCleanup,
@@ -760,7 +761,7 @@ function SpeechModelBrowser(props: {
761 const [downloadedOnly, setDownloadedOnly] = useState(false);
762 const [languageSearch, setLanguageSearch] = useState("");
763 const [selection, setSelection] = useState<string[] | null>(null);
763 - const preferred = navigator.languages.map((language) => language.toLowerCase().split("-")[0]!);
764 + const [preferred] = useState(preferredSpeechLanguages);
765 const supportedLanguages = [...new Set(models.flatMap((model) => model.languages))];
766 const defaultLanguage = preferred.find((code) => supportedLanguages.includes(code));
767 const selectedLanguages = selection ?? (defaultLanguage ? [defaultLanguage] : []);
@@ -1077,9 +1078,11 @@ function TranscriptionTest(props: {
1078 step={speech.setup.step}
1079 status={speech.status}
1080 model={speech.setup.model}
1081 + language={speech.setup.language}
1082 downloading={speech.setup.downloading}
1083 error={speech.setup.error}
1084 onOpenChange={speech.setup.setOpen}
1085 + onRetryCatalog={speech.setup.retryCatalog}
1086 onDownload={() => void speech.setup.download()}
1087 onCancelDownload={() => void speech.setup.cancelDownload()}
1088 onStartRecording={() => void speech.setup.startRecording()}
apps/web/src/speech/preferredSpeechLanguages.ts new
+10
@@ -0,0 +1,10 @@
1 +// Packaged desktop pins `navigator.languages` to en-US, so the OS's own list leads.
2 +export function preferredSpeechLanguages() {
3 + const bridge = window.desktopBridge;
4 + let system = bridge?.getPreferredSystemLanguages?.() ?? [];
5 + if (!system.length) {
6 + const locale = bridge?.getSystemLocale?.();
7 + if (locale) system = [locale];
8 + }
9 + return [...system, ...navigator.languages].map((tag) => tag.toLowerCase().split(/[-_]/)[0]!);
10 +}
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+217 -6
@@ -1,19 +1,24 @@
1 import { useLayoutEffect } from "react";
2 import { expect, it, vi } from "vite-plus/test";
3 +import { page, userEvent } from "vite-plus/test/browser";
4 +import * as Deferred from "effect/Deferred";
5 import * as Effect from "effect/Effect";
6 import * as Option from "effect/Option";
7 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
8 +import { getEnvironmentSpeechModels } from "@t3tools/client-runtime/voice-input";
9 import {
10 DEFAULT_CLIENT_SETTINGS,
11 DEFAULT_SERVER_SETTINGS,
12 EnvironmentId,
13 + type EnvironmentSpeechModel,
14 type ProjectId,
15 type SpeechCustomWords,
16 type SpeechPostProcessingOptions,
17 type SpeechTranscriptionOptions,
18 } from "@t3tools/contracts";
19
16 -import { act, render } from "../../test/render";
20 +import { act, render, stubGlobal } from "../../test/render";
21 +import { VoiceInputSetup } from "../components/chat/VoiceInputSetup";
22 import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
23 import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
24 import {
@@ -36,7 +41,12 @@ const mocks = vi.hoisted(() => ({
41 statusRequests: [] as string[],
42 statusUnavailable: false,
43 modelState: "ready" as "ready" | "missing-model",
39 - download: vi.fn<() => Promise<void>>(),
44 + statusModelId: "test-model",
45 + models: [] as EnvironmentSpeechModel[],
46 + catalogOffline: false,
47 + download: vi.fn<(modelId: string) => Promise<void>>(),
48 + selected: [] as string[],
49 + selectionFails: false,
50 platforms: [] as string[],
51 microphoneRequests: [] as string[],
52 transcribeCalls: [] as string[],
@@ -80,6 +90,7 @@ vi.mock("../state/environments", () => ({
90 : null,
91 }));
92 vi.mock("../hooks/useSettings", () => ({
93 + updateClientSettings: vi.fn(),
94 useClientSettingsHydrated: () => true,
95 useClientSettings: (selector?: (settings: Record<string, unknown>) => unknown) => {
96 const settings = { ...DEFAULT_CLIENT_SETTINGS, ...mocks.settings };
@@ -109,7 +120,7 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
120 return {
121 supported: true,
122 state: mocks.modelState,
112 - modelId: "test-model",
123 + modelId: mocks.statusModelId,
124 model: "Test",
125 size: 1,
126 supportsStreaming: false,
@@ -119,8 +130,24 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
130 gpuDevices: [],
131 };
132 }),
122 - downloadEnvironmentSpeechModel: () => Effect.promise(mocks.download),
123 - getEnvironmentSpeechModels: () => Effect.succeed({ models: [] }),
133 + downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
134 + Effect.promise(async () => {
135 + await mocks.download(modelId);
136 + return { models: mocks.models };
137 + }),
138 + selectEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
139 + Effect.sync(() => {
140 + mocks.selected.push(modelId);
141 + if (mocks.selectionFails) throw new Error("selection failed");
142 + mocks.statusModelId = modelId;
143 + return { models: mocks.models };
144 + }),
145 + getEnvironmentSpeechModels: vi.fn(() =>
146 + Effect.sync(() => {
147 + if (mocks.catalogOffline) throw new Error("Catalog offline");
148 + return { models: mocks.models };
149 + }),
150 + ),
151 postProcessEnvironmentTranscript: (
152 prepared: PreparedConnection,
153 transcript: string,
@@ -189,7 +216,12 @@ function setup(options: { readonly destination?: string | null } = {}) {
216 mocks.statusRequests = [];
217 mocks.statusUnavailable = false;
218 mocks.modelState = "ready";
219 + mocks.statusModelId = "test-model";
220 + mocks.models = [];
221 + mocks.catalogOffline = false;
222 mocks.download.mockReset();
223 + mocks.selected = [];
224 + mocks.selectionFails = false;
225 mocks.platforms = [];
226 mocks.microphoneRequests = [];
227 mocks.transcribeCalls = [];
@@ -213,6 +245,7 @@ function Probe(props: {
245 readonly projectId?: string;
246 readonly projectName?: string;
247 readonly cleanup?: boolean;
248 + readonly showWizard?: boolean;
249 }) {
250 const value = useEnvironmentSpeechInput({
251 environmentId: THREAD_ENV as EnvironmentId,
@@ -230,7 +263,18 @@ function Probe(props: {
263 useLayoutEffect(() => {
264 mocks.voice = value;
265 });
233 - return null;
266 + return props.showWizard ? (
267 + <VoiceInputSetup
268 + {...value.setup}
269 + environmentId={value.transcriptionEnvironmentId}
270 + status={value.status}
271 + onOpenChange={value.setup.setOpen}
272 + onRetryCatalog={value.setup.retryCatalog}
273 + onDownload={value.setup.download}
274 + onCancelDownload={value.setup.cancelDownload}
275 + onStartRecording={value.setup.startRecording}
276 + />
277 + ) : null;
278 }
279
280 function expectNoAudioSent() {
@@ -333,6 +377,8 @@ it("retries a failed transcription on the same chosen environment", async () =>
377 it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {
378 setup({ destination: "env-a" });
379 mocks.modelState = "missing-model";
380 + mocks.settings = { speechLanguage: "en" };
381 + mocks.models = [catalogModel(mocks.statusModelId, ["en"], true)];
382 let finishDownload!: () => void;
383 mocks.download.mockReturnValue(
384 new Promise<void>((resolve) => {
@@ -360,6 +406,171 @@ it("reopens setup after reconnecting during a download and ignores its stale com
406 expect(mocks.voice.setup.error).toBeNull();
407 });
408
409 +function catalogModel(
410 + id: string,
411 + languages: readonly string[],
412 + recommended: boolean,
413 +): EnvironmentSpeechModel {
414 + return {
415 + id,
416 + name: id,
417 + description: "",
418 + size: 1,
419 + languages,
420 + accuracy: 0,
421 + speed: 0,
422 + recommended,
423 + supportsStreaming: false,
424 + supportsLanguageDetection: languages.length > 1,
425 + supportsTranslation: false,
426 + active: false,
427 + state: "downloadable",
428 + };
429 +}
430 +
431 +it.each([
432 + ["the browser prefers German", ["de-DE"], undefined, "auto"],
433 + ["the desktop exposes no locale methods", ["de-DE"], {}, "auto"],
434 + [
435 + "an older preload exposes only the OS locale",
436 + ["en-US"],
437 + { getSystemLocale: () => "de-DE" },
438 + "auto",
439 + ],
440 + [
441 + "the desktop OS prefers German",
442 + ["en-US"],
443 + { getPreferredSystemLanguages: () => ["de-DE"] },
444 + "auto",
445 + ],
446 + ["German is the chosen Voice language", ["en-US"], undefined, "de"],
447 +])(
448 + "offers and downloads a German-capable model when %s",
449 + async (_, languages, bridge, speechLanguage) => {
450 + setup({ destination: "env-a" });
451 + mocks.settings = { speechLanguage };
452 + using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(languages);
453 + using _bridge = stubGlobal("desktopBridge", bridge);
454 + const englishDefault = "handy-computer/parakeet-unified-en-0.6b-gguf";
455 + const german = "handy-computer/nemotron-3.5-asr-streaming-0.6b-gguf";
456 + mocks.modelState = "missing-model";
457 + mocks.statusModelId = englishDefault;
458 + mocks.models = [
459 + { ...catalogModel(englishDefault, ["en"], true), active: true },
460 + catalogModel(german, ["en", "de", "fr"], true),
461 + catalogModel("handy-computer/canary-180m-flash-gguf", ["en", "de", "es", "fr"], true),
462 + ];
463 + mocks.download.mockResolvedValue(undefined);
464 + await using _screen = await render(<Probe />);
465 + await act(() => mocks.voice.start());
466 + expect(mocks.voice.setup.open).toBe(true);
467 + await act(() => mocks.voice.setup.download());
468 + expect(mocks.download).toHaveBeenCalledExactlyOnceWith(german);
469 + expect(mocks.voice.setup.model?.id).toBe(german);
470 + },
471 +);
472 +
473 +it.each(["cancel", "failed restart"] as const)(
474 + "closes pending catalog setup on %s",
475 + async (action) => {
476 + setup({ destination: "env-a" });
477 + mocks.modelState = "missing-model";
478 + mocks.settings = { speechLanguage: "en" };
479 + mocks.models = [catalogModel("current", ["en"], true)];
480 + const response = Deferred.makeUnsafe<{ models: EnvironmentSpeechModel[] }>();
481 + vi.mocked(getEnvironmentSpeechModels).mockReturnValueOnce(Deferred.await(response));
482 + await using _screen = await render(<Probe showWizard />);
483 + let firstStart!: Promise<void>;
484 + await act(() => {
485 + firstStart = mocks.voice.start();
486 + });
487 + await expect.element(page.getByText("Loading transcription models…")).toBeVisible();
488 + if (action === "cancel") {
489 + await act(() => mocks.voice.cancel());
490 + } else {
491 + mocks.statusUnavailable = true;
492 + await act(() => mocks.voice.start());
493 + mocks.statusUnavailable = false;
494 + }
495 + expect(mocks.voice.setup.open).toBe(false);
496 +
497 + await act(() => mocks.voice.start());
498 + Deferred.doneUnsafe(
499 + response,
500 + Effect.succeed({ models: [catalogModel("stale", ["en"], true)] }),
501 + );
502 + await act(() => firstStart);
503 + await expect.element(page.getByRole("heading", { name: "current" })).toBeVisible();
504 + expect(mocks.microphoneRequests).toEqual([]);
505 + },
506 +);
507 +
508 +it.each([true, false])(
509 + "never downloads the English default when no German recommendation is available (catalog offline: %s)",
510 + async (offline) => {
511 + setup({ destination: "env-a" });
512 + mocks.settings = { speechLanguage: "de" };
513 + mocks.modelState = "missing-model";
514 + mocks.statusModelId = "english";
515 + mocks.models = [catalogModel("english", ["en"], true)];
516 + mocks.catalogOffline = offline;
517 + await using _screen = await render(<Probe showWizard />);
518 + await act(() => mocks.voice.start());
519 + const wizard = page.getByRole("dialog", { name: "Set up voice input" });
520 + const retry = wizard.getByRole("button", { name: "Retry loading models" });
521 + const download = wizard.getByRole("button", { name: "Download model" });
522 + await expect.element(wizard.getByRole("alert")).toBeVisible();
523 + await expect.element(download).not.toBeInTheDocument();
524 + if (offline) {
525 + mocks.catalogOffline = false;
526 + mocks.models.push(catalogModel("german", ["de"], true));
527 + // Another client can finish setup while the catalog is offline.
528 + mocks.modelState = "ready";
529 + await userEvent.click(retry);
530 + await expect.element(wizard.getByRole("heading", { name: "german" })).toBeVisible();
531 + await expect.element(download).toBeEnabled();
532 + }
533 + await expect.element(retry).not.toBeInTheDocument();
534 + expect(mocks.download).not.toHaveBeenCalled();
535 + expect(mocks.microphoneRequests).toEqual([]);
536 + },
537 +);
538 +
539 +it.each([
540 + ["selects the offered model once its download installed it", "installed", ["german"], null],
541 + [
542 + "does not advance a cancelled download when another model is ready",
543 + "downloadable",
544 + [],
545 + "The model is not ready. Try downloading it again.",
546 + ],
547 + [
548 + "reports a failed selection apart from the download",
549 + "installed",
550 + ["german"],
551 + "The model downloaded, but this environment could not select it. Try again.",
552 + ],
553 +] as const)("%s", async (_, landedState, selected, error) => {
554 + setup({ destination: "env-a" });
555 + using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(["de-DE"]);
556 + mocks.selectionFails = error !== null && landedState === "installed";
557 + mocks.modelState = "missing-model";
558 + mocks.statusModelId = "english";
559 + mocks.models = [catalogModel("english", ["en"], true), catalogModel("german", ["de"], true)];
560 + mocks.download.mockImplementation(async () => {
561 + mocks.models = [mocks.models[0]!, { ...mocks.models[1]!, state: landedState }];
562 + mocks.modelState = "ready";
563 + });
564 + await using _screen = await render(<Probe />);
565 + await act(() => mocks.voice.start());
566 + expect(mocks.voice.setup.language).toBe("de");
567 + await act(() => mocks.voice.setup.download());
568 + expect(mocks.selected).toEqual(selected);
569 + expect(mocks.voice.setup.error).toBe(error);
570 + expect(mocks.voice.setup.downloading).toBe(false);
571 + expect(mocks.voice.setup.step).toBe(landedState === "installed" && !error ? 1 : 0);
572 +});
573 +
574 it("does not resend a failed recording after consent is revoked", async () => {
575 setup({ destination: "env-a" });
576 mocks.transcripts = [new Error("inference failed")];
apps/web/src/speech/useEnvironmentSpeechInput.ts
+74 -23
@@ -4,6 +4,7 @@ import {
4 getEnvironmentSpeechModels,
5 getEnvironmentSpeechStatus,
6 postProcessEnvironmentTranscript,
7 + selectEnvironmentSpeechModel,
8 VoiceInputController,
9 voiceInputBlocksSubmission,
10 type VoiceDraftSnapshot,
@@ -22,7 +23,7 @@ import type {
23 SpeechTranscriptionOptions,
24 } from "@t3tools/contracts";
25 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
25 -import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
26 +import { chooseSetupSpeechModel, resolveSpeechVocabulary } from "@t3tools/shared/speech";
27 import { useNavigate } from "@tanstack/react-router";
28 import * as Option from "effect/Option";
29 import { useEffect, useRef, useState } from "react";
@@ -37,6 +38,7 @@ import {
38 import { useEnvironment } from "../state/environments";
39 import { useEnvironmentScope, usePreparedConnection } from "../state/session";
40 import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
41 +import { preferredSpeechLanguages } from "./preferredSpeechLanguages";
42 import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
43 import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent";
44
@@ -46,7 +48,9 @@ const INITIAL_SETUP = {
48 open: false,
49 step: 0,
50 model: null as EnvironmentSpeechModel | null,
51 + language: null as string | null,
52 downloading: false,
53 + catalogFailed: false,
54 error: null as string | null,
55 };
56
@@ -387,11 +391,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
391 };
392 }, [prepared, cloudProvider]);
393 const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null;
394 + const setupModelId = setup.model?.id;
395
396 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download
397 useEffect(() => {
393 - if (!setup.open || !setup.downloading || !prepared || !currentStatus?.supported) return;
394 - const modelId = currentStatus.modelId;
398 + if (!setup.open || !setup.downloading || !prepared || !setupModelId) return;
399 let disposed = false;
400 let refreshing = false;
401 const timer = window.setInterval(() => {
@@ -400,8 +404,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {
404 void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared))
405 .then((result) => {
406 if (disposed) return;
403 - const model = result.models.find((candidate) => candidate.id === modelId) ?? null;
404 - setSetup((current) => ({ ...current, model }));
407 + const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null;
408 + if (model) setSetup((current) => ({ ...current, model }));
409 })
410 .catch(() => undefined)
411 .finally(() => {
@@ -412,7 +416,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
416 disposed = true;
417 window.clearInterval(timer);
418 };
415 - }, [currentStatus, prepared, setup.downloading, setup.open]);
419 + }, [prepared, setup.downloading, setup.open, setupModelId]);
420
421 const previousOwnerRef = useRef(input.ownerKey);
422 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes
@@ -445,8 +449,34 @@ export function useEnvironmentSpeechInput(input: HookInput) {
449 if (controller.currentState.phase === "recording") void controller.stop();
450 });
451
452 + const loadSetupCatalog = useStableCallback(
453 + async (connection: PreparedConnection, selectedModelId: string, request: number) => {
454 + setSetup({ ...INITIAL_SETUP, open: true });
455 + const catalog = await runEnvironmentSpeech(getEnvironmentSpeechModels(connection)).catch(
456 + () => null,
457 + );
458 + if (request !== startRequestRef.current) return;
459 + const choice = chooseSetupSpeechModel(catalog?.models ?? [], {
460 + selectedModelId,
461 + intent: latestRef.current.clientSettings.speechLanguage,
462 + preferred: preferredSpeechLanguages(),
463 + });
464 + setSetup((current) => ({
465 + ...current,
466 + model: choice?.model ?? null,
467 + language: choice?.language ?? null,
468 + catalogFailed: catalog === null,
469 + error: choice
470 + ? null
471 + : catalog
472 + ? "No model matches your language. Choose a model or change the language in Voice settings."
473 + : "Could not load transcription models. Check the connection and try again.",
474 + }));
475 + },
476 + );
477 +
478 const start = useStableCallback(async () => {
449 - if (latestRef.current.input.disabled) return;
479 + if (latestRef.current.input.disabled || (setup.open && setup.downloading)) return;
480 if (!grantId) {
481 if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false)
482 await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });
@@ -466,6 +496,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
496 controllerRef.current === controller &&
497 latestRef.current.input.ownerKey === owner &&
498 !latestRef.current.input.disabled;
499 + setSetup(INITIAL_SETUP);
500 setQueuedStart(prepared);
501 let latestStatus: EnvironmentSpeechStatus;
502 try {
@@ -481,8 +512,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
512 return;
513 }
514 if (!stillCurrent()) return;
484 - setQueuedStart(null);
515 setStatus({ prepared, value: latestStatus });
516 + setQueuedStart(null);
517 + if (latestStatus.supported && latestStatus.state === "missing-model") {
518 + return loadSetupCatalog(prepared, latestStatus.modelId, request);
519 + }
520 if (!latestStatus.supported) {
521 toastManager.add({
522 type: "error",
@@ -491,32 +525,46 @@ export function useEnvironmentSpeechInput(input: HookInput) {
525 });
526 return;
527 }
494 - if (latestStatus.state === "missing-model") {
495 - setSetup({ ...INITIAL_SETUP, open: true });
496 - return;
497 - }
528 setLevel(0);
529 await controller.start();
530 });
531
532 const downloadSetupModel = useStableCallback(async () => {
503 - if (!prepared || !currentStatus?.supported || setup.downloading) return;
533 + if (!prepared || !currentStatus?.supported || !setupModelId || setup.downloading) return;
534 + const modelId = setupModelId;
535 const request = ++startRequestRef.current;
536 setSetup((current) => ({ ...current, downloading: true, error: null }));
537 try {
507 - await runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));
538 + const { models } = await runEnvironmentSpeech(
539 + downloadEnvironmentSpeechModel(prepared, modelId),
540 + );
541 + if (request !== startRequestRef.current) return;
542 + // A download cancelled from elsewhere still resolves, so select the model only once it landed.
543 + const installed = models.some(({ id, state }) => id === modelId && state === "installed");
544 + if (installed && modelId !== currentStatus.modelId) {
545 + try {
546 + await runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, modelId));
547 + } catch {
548 + if (request !== startRequestRef.current) return;
549 + setSetup((current) => ({
550 + ...current,
551 + downloading: false,
552 + error: "The model downloaded, but this environment could not select it. Try again.",
553 + }));
554 + return;
555 + }
556 + }
557 if (request !== startRequestRef.current) return;
558 const next = await probeSpeechStatus(prepared, true);
559 if (request !== startRequestRef.current) return;
560 setStatus({ prepared, value: next });
561 + const ready =
562 + installed && next.supported && next.modelId === modelId && next.state !== "missing-model";
563 setSetup((current) => ({
564 ...current,
565 downloading: false,
515 - step: next.supported && next.state !== "missing-model" ? 1 : current.step,
516 - error:
517 - next.supported && next.state !== "missing-model"
518 - ? null
519 - : "The model is not ready. Try downloading it again.",
566 + step: ready ? 1 : current.step,
567 + error: ready ? null : "The model is not ready. Try downloading it again.",
568 }));
569 } catch {
570 if (request !== startRequestRef.current) return;
@@ -529,13 +577,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
577 });
578
579 const cancelSetupDownload = useStableCallback(async () => {
532 - if (!prepared || !currentStatus?.supported) return;
580 + if (!prepared || !setupModelId) return;
581 startRequestRef.current += 1;
582 setSetup((current) => ({ ...current, open: false, downloading: false }));
583 try {
536 - await runEnvironmentSpeech(
537 - cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId),
538 - );
584 + await runEnvironmentSpeech(cancelEnvironmentSpeechModelDownload(prepared, setupModelId));
585 } catch (error) {
586 setSetup((current) => ({
587 ...current,
@@ -562,6 +608,10 @@ export function useEnvironmentSpeechInput(input: HookInput) {
608 setup: {
609 ...setup,
610 setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
611 + retryCatalog:
612 + setup.open && setup.catalogFailed && prepared && currentStatus?.supported
613 + ? () => loadSetupCatalog(prepared, currentStatus.modelId, ++startRequestRef.current)
614 + : null,
615 download: downloadSetupModel,
616 cancelDownload: cancelSetupDownload,
617 startRecording: async () => {
@@ -582,6 +632,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
632 pendingSubmissionRef.current = null;
633 startRequestRef.current += 1;
634 setQueuedStart(null);
635 + setSetup(INITIAL_SETUP);
636 controllerRef.current?.cancel();
637 }),
638 skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()),
apps/web/src/speech/useTextFieldSpeech.tsx
+2
@@ -110,9 +110,11 @@ export function TextFieldSpeechControls({
110 step={speech.setup.step}
111 status={speech.status}
112 model={speech.setup.model}
113 + language={speech.setup.language}
114 downloading={speech.setup.downloading}
115 error={speech.setup.error}
116 onOpenChange={speech.setup.setOpen}
117 + onRetryCatalog={speech.setup.retryCatalog}
118 onDownload={() => void speech.setup.download()}
119 onCancelDownload={() => void speech.setup.cancelDownload()}
120 onStartRecording={() => void speech.setup.startRecording()}
docs/user/voice-input.md
+1
@@ -52,6 +52,7 @@ until you choose another.
52 In **Settings → Voice → Transcription**, download a model and choose **Use model** if it
53 isn’t already selected. **Test model** lets you try it without sending a message.
54 If you use the composer microphone before a model is installed, it offers to download one.
55 +The recommendation uses your Voice language, or your system’s preferred languages with **Auto-detect**.
56
57 Select the languages you dictate in to compare models that support all of them.
58 Accuracy shows how often the model got words or characters wrong in each
packages/contracts/src/ipc.ts
+2
@@ -1048,6 +1048,8 @@ export interface DesktopBridge {
1048 * regardless of OS settings.
1049 */
1050 getSystemLocale: () => string | null;
1051 + /** The OS's preferred languages, most preferred first; `navigator.languages` is pinned like the locale. */
1052 + getPreferredSystemLanguages: () => readonly string[];
1053 getLocalEnvironmentBootstraps: () => readonly DesktopEnvironmentBootstrap[];
1054 getLocalEnvironmentEnabled: () => boolean;
1055 setLocalEnvironmentEnabled: (enabled: boolean) => Promise<void>;
packages/shared/src/speech.test.ts
+79
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test";
2 import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES } from "@t3tools/contracts";
3
4 import {
5 + chooseSetupSpeechModel,
6 decodeSpeechPcmRequest,
7 encodeSpeechPcmRequest,
8 resolveSpeechVocabulary,
@@ -71,3 +72,81 @@ describe("resolveSpeechVocabulary", () => {
72 expect(resolved.at(-1)?.term).toBe("word97");
73 });
74 });
75 +
76 +describe("chooseSetupSpeechModel", () => {
77 + const model = (
78 + id: string,
79 + languages: string[],
80 + supportsLanguageDetection: boolean,
81 + recommended: boolean,
82 + ) => ({ id, languages, supportsLanguageDetection, recommended });
83 + const catalog = [
84 + model("unrecommended", ["en", "de", "es"], true, false),
85 + model("english", ["en"], false, true),
86 + model("fixed", ["en", "de"], false, true),
87 + model("detecting", ["en", "de", "fr"], true, true),
88 + ];
89 +
90 + it.each([
91 + [
92 + "keeps the selected English model for an English speaker",
93 + "english",
94 + "auto",
95 + ["en"],
96 + { id: "english", language: "en" },
97 + ],
98 + [
99 + "skips models that would transcribe English instead of German",
100 + "english",
101 + "auto",
102 + ["de", "en"],
103 + { id: "detecting", language: "de" },
104 + ],
105 + [
106 + "accepts a model without detection for a chosen language",
107 + "english",
108 + "de",
109 + ["en"],
110 + { id: "fixed", language: "de" },
111 + ],
112 + [
113 + "keeps a selected model that transcribes the language",
114 + "unrecommended",
115 + "auto",
116 + ["de"],
117 + { id: "unrecommended", language: "de" },
118 + ],
119 + [
120 + "moves on to the next preferred language a model transcribes",
121 + "english",
122 + "auto",
123 + ["ja", "fr"],
124 + { id: "detecting", language: "fr" },
125 + ],
126 + [
127 + "falls back to a model that is not recommended",
128 + "english",
129 + "auto",
130 + ["es"],
131 + { id: "unrecommended", language: "es" },
132 + ],
133 + ["offers nothing when no model transcribes the language", "english", "auto", ["ja"], undefined],
134 + [
135 + "offers the selected model without a language recommendation when Auto has no preferences",
136 + "fixed",
137 + "auto",
138 + [],
139 + { id: "fixed", language: null },
140 + ],
141 + [
142 + "does not guess another model when Auto has no preferences or selection",
143 + "gone",
144 + "auto",
145 + [],
146 + undefined,
147 + ],
148 + ] as const)("%s", (_, selectedModelId, intent, preferred, expected) => {
149 + const choice = chooseSetupSpeechModel(catalog, { selectedModelId, intent, preferred });
150 + expect(choice && { id: choice.model.id, language: choice.language }).toEqual(expected);
151 + });
152 +});
packages/shared/src/speech.ts
+38
@@ -36,6 +36,44 @@ export function effectiveSpeechLanguage(
36 return model.languages.find((language) => language === "en") ?? model.languages[0] ?? "auto";
37 }
38
39 +/**
40 + * The model first-run setup offers: the selected one, else a recommended one, else any that the
41 + * server would really transcribe the person's language with. Without a language hint,
42 + * offer only the selected model, without a recommendation.
43 + */
44 +export function chooseSetupSpeechModel<
45 + Model extends {
46 + readonly id: string;
47 + readonly languages: readonly string[];
48 + readonly supportsLanguageDetection: boolean;
49 + readonly recommended: boolean;
50 + },
51 +>(
52 + models: readonly Model[],
53 + input: {
54 + readonly selectedModelId: string;
55 + readonly intent: string;
56 + readonly preferred: readonly string[];
57 + },
58 +) {
59 + if (input.intent === "auto" && input.preferred.length === 0) {
60 + const model = models.find(({ id }) => id === input.selectedModelId);
61 + return model && { model, language: null };
62 + }
63 + for (const language of input.intent === "auto" ? input.preferred : [input.intent]) {
64 + const capable = models.filter((model) => {
65 + const effective = effectiveSpeechLanguage(model, input.intent);
66 + return effective === language || (effective === "auto" && model.languages.includes(language));
67 + });
68 + const model =
69 + capable.find(({ id }) => id === input.selectedModelId) ??
70 + capable.find(({ recommended }) => recommended) ??
71 + capable[0];
72 + if (model) return { model, language };
73 + }
74 + return undefined;
75 +}
76 +
77 const vocabularyKey = (value: string) => value.trim().normalize("NFC").toLocaleLowerCase();
78
79 /**