The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 02:00 UTC
fix(web): first-run voice setup offers a model for the user's language (#32035)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Julius Marminge committed
Nov 6, 2026 at 21:51 UTC
15b72755956c3338dd98440a7ed35beffe46b0b7
28 files changed
+633
-86
apps/desktop/src/app/DesktopAppIdentity.test.ts
+1
@@ -42,6 +42,7 @@ const layerElectronApp = (calls: ElectronAppCalls) =>
42
metadata: Effect.die("unexpected metadata read"),
43
name: Effect.succeed("T3 Code"),
44
systemLocale: Effect.succeed("en-US"),
45
+ preferredSystemLanguages: Effect.succeed(["en-US"]),
46
whenReady: Effect.void,
47
requestSingleInstanceLock: Effect.succeed(true),
48
quit: Effect.void,
apps/desktop/src/app/DesktopLifecycle.test.ts
+1
@@ -39,6 +39,7 @@ function layerElectronApp(
39
metadata: Effect.die("unexpected metadata read"),
40
name: Effect.succeed("T3 Code"),
41
systemLocale: Effect.succeed("en-US"),
42
+ preferredSystemLanguages: Effect.succeed(["en-US"]),
43
whenReady: Effect.void,
44
requestSingleInstanceLock: Effect.succeed(true),
45
quit,
apps/desktop/src/electron/ElectronApp.ts
+2
@@ -46,6 +46,7 @@ export class ElectronApp extends Context.Service<
46
readonly name: Effect.Effect<string>;
47
/** Packaged builds ship only en-US; app.getLocale() and Intl defaults do not reflect the OS locale. */
48
readonly systemLocale: Effect.Effect<string>;
49
+ readonly preferredSystemLanguages: Effect.Effect<ReadonlyArray<string>>;
50
readonly whenReady: Effect.Effect<void, ElectronAppWhenReadyError>;
51
readonly requestSingleInstanceLock: Effect.Effect<boolean>;
52
readonly quit: Effect.Effect<void>;
@@ -155,6 +156,7 @@ export const make = ElectronApp.of({
156
}
157
return toSystemLocaleTag(identifier, hourCycle);
158
}),
159
+ preferredSystemLanguages: Effect.sync(() => Electron.app.getPreferredSystemLanguages()),
160
whenReady: Effect.gen(function* () {
161
const isPackaged = Electron.app.isPackaged;
162
yield* Effect.tryPromise({
apps/desktop/src/ipc/DesktopIpcHandlers.ts
+2
@@ -38,6 +38,7 @@ import {
38
getLocalEnvironmentBearerToken,
39
installRelayTelemetryEnabled,
40
getSystemLocale,
41
+ getPreferredSystemLanguages,
42
getWindowFullscreenState,
43
openExternal,
44
readMedia,
@@ -94,6 +95,7 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"
95
yield* ipc.handleSync(getAppBranding);
96
yield* installRelayTelemetryEnabled();
97
yield* ipc.handleSync(getSystemLocale);
98
+ yield* ipc.handleSync(getPreferredSystemLanguages);
99
yield* ipc.handleSync(getWindowFullscreenState);
100
yield* ipc.handleSync(getLocalEnvironmentBootstraps);
101
yield* ipc.handleSync(getLocalEnvironmentEnabled);
apps/desktop/src/ipc/channels.ts
+1
@@ -34,6 +34,7 @@ export const CLI_COMMAND_UNINSTALL_CHANNEL = "desktop:cli-command-uninstall";
34
export const GET_APP_BRANDING_CHANNEL = "desktop:get-app-branding";
35
export const GET_RELAY_TELEMETRY_ENABLED_CHANNEL = "desktop:get-relay-telemetry-enabled";
36
export const GET_SYSTEM_LOCALE_CHANNEL = "desktop:get-system-locale";
37
+export const GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL = "desktop:get-preferred-system-languages";
38
export const GET_LOCAL_ENVIRONMENT_BOOTSTRAPS_CHANNEL = "desktop:get-local-environment-bootstraps";
39
export const GET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:get-local-environment-enabled";
40
export const SET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:set-local-environment-enabled";
apps/desktop/src/ipc/methods/window.ts
+9
@@ -106,6 +106,15 @@ export const getSystemLocale = DesktopIpc.makeSyncIpcMethod({
106
}),
107
});
108
109
+export const getPreferredSystemLanguages = DesktopIpc.makeSyncIpcMethod({
110
+ channel: IpcChannels.GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL,
111
+ result: Schema.Array(Schema.String),
112
+ handler: Effect.fn("desktop.ipc.window.getPreferredSystemLanguages")(function* () {
113
+ const electronApp = yield* ElectronApp.ElectronApp;
114
+ return yield* electronApp.preferredSystemLanguages;
115
+ }),
116
+});
117
+
118
export const getWindowFullscreenState = DesktopIpc.makeSyncIpcMethod({
119
channel: IpcChannels.GET_WINDOW_FULLSCREEN_STATE_CHANNEL,
120
result: Schema.Boolean,
apps/desktop/src/preload.ts
+6
@@ -107,6 +107,12 @@ contextBridge.exposeInMainWorld("desktopBridge", {
107
const result = ipcRenderer.sendSync(IpcChannels.GET_SYSTEM_LOCALE_CHANNEL);
108
return typeof result === "string" ? result : null;
109
},
110
+ getPreferredSystemLanguages: () => {
111
+ const result: unknown = ipcRenderer.sendSync(
112
+ IpcChannels.GET_PREFERRED_SYSTEM_LANGUAGES_CHANNEL,
113
+ );
114
+ return Array.isArray(result) ? result.filter((tag) => typeof tag === "string") : [];
115
+ },
116
getLocalEnvironmentBootstraps: () => {
117
const result = ipcRenderer.sendSync(IpcChannels.GET_LOCAL_ENVIRONMENT_BOOTSTRAPS_CHANNEL);
118
if (!Array.isArray(result)) {
apps/desktop/src/telemetry/DesktopTelemetryPublisher.test.ts
+1
@@ -37,6 +37,7 @@ function layerElectronApp(
37
metadata: Effect.die("unexpected metadata read"),
38
name: Effect.succeed("T3 Code"),
39
systemLocale: Effect.succeed("en-US"),
40
+ preferredSystemLanguages: Effect.succeed(["en-US"]),
41
whenReady: Effect.void,
42
requestSingleInstanceLock: Effect.succeed(true),
43
quit: Effect.void,
apps/desktop/src/window/DesktopApplicationMenu.test.ts
+1
@@ -31,6 +31,7 @@ const layerElectronApp = Layer.succeed(ElectronApp.ElectronApp, {
31
metadata: Effect.die("unexpected metadata read"),
32
name: Effect.succeed("T3 Code"),
33
systemLocale: Effect.succeed("en-US"),
34
+ preferredSystemLanguages: Effect.succeed(["en-US"]),
35
whenReady: Effect.void,
36
requestSingleInstanceLock: Effect.succeed(true),
37
quit: Effect.void,
apps/server/src/speech/SpeechService.test.ts
+56
@@ -25,6 +25,7 @@ const native = vi.hoisted(() => ({
25
supportsLongForm: false,
26
supportsTranslation: true,
27
supportsVocabulary: false,
28
+ languages: ["en", "fr"] as readonly string[],
29
begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}),
30
feed: vi.fn(async (_pcm: Float32Array) => ({
31
revision: 1,
@@ -119,6 +120,7 @@ beforeEach(() => {
120
native.backend = "Vulkan0";
121
native.supportsVocabulary = false;
122
native.supportsStreaming = true;
123
+ native.languages = ["en", "fr"];
124
readyModels.clear();
125
readyModels.add("test-model");
126
readyModels.add("fallback-model");
@@ -171,6 +173,60 @@ it.effect("prepares a batch model before audio arrives and reuses it for transcr
173
}).pipe(Effect.provide(layer)),
174
);
175
176
+it.effect("sends the chosen language in the spelling the loaded model advertises", () =>
177
+ Effect.gen(function* () {
178
+ const speech = yield* SpeechService.SpeechService;
179
+ // Nemotron Streaming 3.5 advertises regional tags while the catalog lists bare codes.
180
+ native.languages = ["en-US", "en-GB", "fr-FR"];
181
+ yield* Effect.gen(function* () {
182
+ const stream = yield* speech.startStream({
183
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
184
+ speechLanguage: "fr",
185
+ });
186
+ expect(native.begin).toHaveBeenLastCalledWith(
187
+ { timestamps: "none", language: "fr-FR" },
188
+ true,
189
+ "test.gguf",
190
+ );
191
+ expect(yield* stream.finish).toBe("hello");
192
+ }).pipe(Effect.scoped);
193
+ native.supportsStreaming = false;
194
+ expect(
195
+ yield* speech.transcribe(pcm(), {
196
+ ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
197
+ speechLanguage: "fr",
198
+ }),
199
+ ).toBe("hello");
200
+ expect(native.transcribe).toHaveBeenLastCalledWith(
201
+ expect.any(Float32Array),
202
+ { timestamps: "none", language: "fr-FR" },
203
+ true,
204
+ "test.gguf",
205
+ );
206
+ // A language the model does not advertise reaches it unchanged, so the model reports it.
207
+ yield* speech.updateAcceleration("cpu");
208
+ native.languages = ["en-US"];
209
+ native.transcribe.mockRejectedValueOnce(new Error("transcribe_run: unsupported language"));
210
+ const rejected = yield* Effect.result(
211
+ speech.transcribe(pcm(), { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "fr" }),
212
+ );
213
+ expect(Result.isFailure(rejected) && rejected.failure).toMatchObject({
214
+ _tag: "SpeechOperationError",
215
+ operation: "inference",
216
+ });
217
+ expect(native.transcribe).toHaveBeenLastCalledWith(
218
+ expect.any(Float32Array),
219
+ { timestamps: "none", language: "fr" },
220
+ true,
221
+ "test.gguf",
222
+ );
223
+ expect(loadNative.mock.calls.map(([path, , , acceleration]) => [path, acceleration])).toEqual([
224
+ ["test.gguf", "auto"],
225
+ ["test.gguf", "cpu"],
226
+ ]);
227
+ }).pipe(Effect.provide(layer)),
228
+);
229
+
230
it.effect.each(["cancellation", "timeout"] as const)(
231
"releases a stalled model preparation on %s",
232
(reason) =>
apps/server/src/speech/SpeechService.ts
+5
-2
@@ -32,6 +32,7 @@ import {
32
type SpeechModel,
33
} from "./model.ts";
34
import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts";
35
+import { nativeSpeechLanguage } from "./nativeLanguage.ts";
36
import { SILERO_VAD } from "./silero.ts";
37
import { isSpeechPlatformSupported } from "./platform.ts";
38
import * as SpeechModelStore from "./SpeechModelStore.ts";
@@ -613,11 +614,12 @@ const make = Effect.gen(function* () {
614
try: (signal) => {
615
// Interruption kills the isolated process; the next request reloads the model.
616
signal.addEventListener("abort", () => dropModel(inference), { once: true });
617
+ const hint = nativeSpeechLanguage(inference.languages, language);
618
return inference.transcribe(
619
pcm,
620
{
621
timestamps: "none",
620
- ...(language === "auto" ? {} : { language }),
622
+ ...(hint ? { language: hint } : {}),
623
...(translateToEnglish
624
? { task: "translate" as const, targetLanguage: "en" as const }
625
: {}),
@@ -749,10 +751,11 @@ const make = Effect.gen(function* () {
751
streamModel.supportsVocabulary && !translateToEnglish && customWords.length
752
? customWords
753
: undefined;
754
+ const hint = nativeSpeechLanguage(streamModel.languages, language);
755
await streamModel.begin(
756
{
757
timestamps: "none",
755
- ...(language === "auto" ? {} : { language }),
758
+ ...(hint ? { language: hint } : {}),
759
...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}),
760
...(vocabulary ? { vocabulary } : {}),
761
},
apps/server/src/speech/native.test.ts
+2
-1
@@ -4,7 +4,7 @@ import { expect, it } from "vite-plus/test";
4
import { loadNativeSpeechModel } from "./native.ts";
5
6
const fixture = (transcribe: string, maxAudioMs = 0, supportsVocabulary = false) =>
7
- `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs} }, supports: (feature) => feature === "vocabulary" && ${supportsVocabulary}, transcribe: ${transcribe} }) };`)}`;
7
+ `data:text/javascript,${encodeURIComponent(`export const TranscribeModel = { load: async () => ({ capabilities: { supportsStreaming: false, maxAudioMs: ${maxAudioMs}, languages: ["en-US", "de-DE"] }, supports: (feature) => feature === "vocabulary" && ${supportsVocabulary}, transcribe: ${transcribe} }) };`)}`;
8
9
const fixtureWithUnrelatedIpcMessage = () =>
10
`data:text/javascript,${encodeURIComponent(`
@@ -38,6 +38,7 @@ it("returns transcription from an isolated process and disposes it", async () =>
38
fixture("async (pcm) => ({ text: String(pcm[0]) })"),
39
);
40
try {
41
+ expect(model.languages).toEqual(["en-US", "de-DE"]);
42
expect(
43
await model.transcribe(
44
new Float32Array([0.25]),
apps/server/src/speech/native.ts
+5
@@ -147,6 +147,7 @@ process.on("message", async (message) => {
147
throw new Error("The selected speech GPU is unavailable.");
148
model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration });
149
process.send({ type: "t3-speech-reply", ok: true, backend: model.backend,
150
+ languages: model.capabilities.languages,
151
supportsStreaming: model.capabilities.supportsStreaming,
152
supportsLongForm: model.supports("long_form"),
153
supportsTranslation: model.capabilities.supportsTranslate,
@@ -194,6 +195,7 @@ type Reply = {
195
readonly supportsTranslation?: boolean;
196
readonly supportsVocabulary?: boolean;
197
readonly backend?: string;
198
+ readonly languages?: unknown;
199
readonly revision?: number;
200
readonly preview?: SpeechStreamText | null;
201
};
@@ -391,6 +393,9 @@ async function loadNativeSpeechModelOnce(
393
}
394
return {
395
backend: loaded.backend ?? "unknown",
396
+ languages: Array.isArray(loaded.languages)
397
+ ? loaded.languages.filter((entry): entry is string => typeof entry === "string")
398
+ : [],
399
supportsStreaming: loaded.supportsStreaming === true,
400
supportsLongForm: loaded.supportsLongForm === true,
401
supportsTranslation: loaded.supportsTranslation === true,
apps/server/src/speech/nativeLanguage.test.ts
new
+18
@@ -0,0 +1,18 @@
1
+import { expect, it } from "vite-plus/test";
2
+
3
+import { nativeSpeechLanguage } from "./nativeLanguage.ts";
4
+
5
+const nemotron = ["en-US", "en-GB", "es-US", "es-ES", "de-DE"];
6
+
7
+it.each([
8
+ ["auto omits the hint", nemotron, "auto", undefined],
9
+ ["exact entry", ["en", "de"], "de", "de"],
10
+ ["exact entry before a regional one", ["de-DE", "de"], "de", "de"],
11
+ ["case and underscore normalize to the advertised spelling", nemotron, "DE_de", "de-DE"],
12
+ ["bare code takes the first regional entry", nemotron, "en", "en-US"],
13
+ ["regional request with another region passes through", nemotron, "de-AT", "de-AT"],
14
+ ["unadvertised language passes through", nemotron, "fr", "fr"],
15
+ ["model without an advertised list passes through", [], "de", "de"],
16
+])("%s", (_name, advertised, language, expected) => {
17
+ expect(nativeSpeechLanguage(advertised, language)).toBe(expected);
18
+});
apps/server/src/speech/nativeLanguage.ts
new
+21
@@ -0,0 +1,21 @@
1
+/**
2
+ * transcribe.cpp accepts a language only when it equals an entry of the GGUF's
3
+ * `general.languages` byte for byte. Nemotron Streaming 3.5 lists `de-DE` where the catalog
4
+ * says `de`, so the request carries the loaded model's own spelling. "auto" omits the hint.
5
+ */
6
+export function nativeSpeechLanguage(
7
+ advertised: readonly string[],
8
+ language: string,
9
+): string | undefined {
10
+ if (language === "auto") return undefined;
11
+ const tag = (value: string) => value.toLowerCase().replaceAll("_", "-");
12
+ const wanted = tag(language);
13
+ return (
14
+ advertised.find((entry) => entry === language) ??
15
+ advertised.find((entry) => tag(entry) === wanted) ??
16
+ (wanted.includes("-")
17
+ ? undefined
18
+ : advertised.find((entry) => tag(entry).split("-", 1)[0] === wanted)) ??
19
+ language
20
+ );
21
+}
apps/web/src/components/chat/ChatComposer.tsx
+2
@@ -3709,9 +3709,11 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
3709
step={speechInput.setup.step}
3710
status={speechInput.status}
3711
model={speechInput.setup.model}
3712
+ language={speechInput.setup.language}
3713
downloading={speechInput.setup.downloading}
3714
error={speechInput.setup.error}
3715
onOpenChange={speechInput.setup.setOpen}
3716
+ onRetryCatalog={speechInput.setup.retryCatalog}
3717
onDownload={() => void speechInput.setup.download()}
3718
onCancelDownload={() => void speechInput.setup.cancelDownload()}
3719
onStartRecording={() => void speechInput.setup.startRecording()}
apps/web/src/components/chat/VoiceInputSetup.tsx
+68
-53
@@ -13,6 +13,7 @@ import { useClientSettings, updateClientSettings } from "../../hooks/useSettings
13
import { useEnvironment } from "../../state/environments";
14
import { AuthSettingsWriteScope } from "@t3tools/contracts";
15
import { useEnvironmentScope } from "../../state/session";
16
+import { languageLabel } from "../settings/SpeechLanguageSelect";
17
import { Button } from "../ui/button";
18
import { Dialog } from "../ui/dialog";
19
import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
@@ -24,9 +25,12 @@ export function VoiceInputSetup(props: {
25
step: number;
26
status: EnvironmentSpeechStatus | null;
27
model: EnvironmentSpeechModel | null;
28
+ /** The language the offered model was chosen for. */
29
+ language: string | null;
30
downloading: boolean;
31
error: string | null;
32
onOpenChange(open: boolean): void;
33
+ onRetryCatalog: (() => void) | null;
34
onDownload(): void;
35
onCancelDownload(): void;
36
onStartRecording(): void;
@@ -85,49 +89,55 @@ export function VoiceInputSetup(props: {
89
<WizardPanel>
90
{props.step === 0 ? (
91
<section className="space-y-3 text-sm">
88
- <div className="rounded-lg border border-border/70 bg-card/60 p-4">
89
- <div className="flex items-center justify-between gap-2">
90
- <h3 className="font-medium">{status?.model ?? "English transcription model"}</h3>
91
- {status?.modelId === "handy-computer/parakeet-unified-en-0.6b-gguf" ? (
92
- <span className="text-xs font-medium text-primary">
93
- Recommended for English
94
- </span>
92
+ {model ? (
93
+ <div className="rounded-lg border border-border/70 bg-card/60 p-4">
94
+ <div className="flex items-center justify-between gap-2">
95
+ <h3 className="font-medium">{model.name}</h3>
96
+ {model?.recommended && props.language ? (
97
+ <span className="text-xs font-medium text-primary">
98
+ Recommended for {languageLabel(props.language)}
99
+ </span>
100
+ ) : null}
101
+ </div>
102
+ <p className="mt-2 text-muted-foreground">
103
+ Download the {Math.round(model.size / 1024 / 1024)} MB model to{" "}
104
+ {environment?.label ?? "your transcription environment"}. Microphone audio is
105
+ sent there for transcription, as chosen in Voice settings.
106
+ </p>
107
+ <p className="mt-2 text-muted-foreground">
108
+ Silero (1.2 MB), used for silence filtering and pause detection, downloads
109
+ alongside the speech model. An installed copy is reused.
110
+ </p>
111
+ {props.downloading ? (
112
+ <div role="status" className="mt-4 space-y-1 text-muted-foreground">
113
+ <div
114
+ role="progressbar"
115
+ aria-label="Speech model download progress"
116
+ aria-valuemin={0}
117
+ aria-valuemax={100}
118
+ aria-valuenow={progress ?? undefined}
119
+ className="h-1 overflow-hidden rounded-full bg-muted"
120
+ >
121
+ <div
122
+ className="h-full bg-primary transition-[width] duration-200"
123
+ style={{ width: `${progress ?? 0}%` }}
124
+ />
125
+ </div>
126
+ <p className="text-xs">
127
+ {model?.state === "verifying"
128
+ ? "Verifying download…"
129
+ : progress === null
130
+ ? "Starting download…"
131
+ : `${progress}% downloaded`}
132
+ </p>
133
+ </div>
134
) : null}
135
</div>
97
- <p className="mt-2 text-muted-foreground">
98
- Download the {status ? Math.round(status.size / 1024 / 1024) : 697} MB model to{" "}
99
- {environment?.label ?? "your transcription environment"}. Microphone audio is sent
100
- there for transcription, as chosen in Voice settings.
136
+ ) : !props.error ? (
137
+ <p role="status" className="text-muted-foreground">
138
+ Loading transcription models…
139
</p>
102
- <p className="mt-2 text-muted-foreground">
103
- Silero (1.2 MB), used for silence filtering and pause detection, downloads
104
- alongside the speech model. An installed copy is reused.
105
- </p>
106
- {props.downloading ? (
107
- <div role="status" className="mt-4 space-y-1 text-muted-foreground">
108
- <div
109
- role="progressbar"
110
- aria-label="Speech model download progress"
111
- aria-valuemin={0}
112
- aria-valuemax={100}
113
- aria-valuenow={progress ?? undefined}
114
- className="h-1 overflow-hidden rounded-full bg-muted"
115
- >
116
- <div
117
- className="h-full bg-primary transition-[width] duration-200"
118
- style={{ width: `${progress ?? 0}%` }}
119
- />
120
- </div>
121
- <p className="text-xs">
122
- {model?.state === "verifying"
123
- ? "Verifying download…"
124
- : progress === null
125
- ? "Starting download…"
126
- : `${progress}% downloaded`}
127
- </p>
128
- </div>
129
- ) : null}
130
- </div>
140
+ ) : null}
141
{props.error ? (
142
<p role="alert" className="text-destructive">
143
{props.error}
@@ -208,25 +218,30 @@ export function VoiceInputSetup(props: {
218
{props.step === 0 ? (
219
<>
220
<Button variant="outline" onClick={() => openSettings("speech-models")}>
211
- Choose another model or language
221
+ {model ? "Choose another model or language" : "Choose a model or language"}
222
</Button>
223
+ {props.onRetryCatalog ? (
224
+ <Button onClick={props.onRetryCatalog}>Retry loading models</Button>
225
+ ) : null}
226
{props.downloading ? (
227
<Button variant="outline" disabled={!canManage} onClick={props.onCancelDownload}>
228
Cancel download
229
</Button>
230
) : null}
218
- <Button
219
- disabled={!canManage || props.downloading || !status}
220
- onClick={props.onDownload}
221
- >
222
- {props.downloading
223
- ? "Downloading…"
224
- : props.error
225
- ? "Retry download"
226
- : canManage
227
- ? "Download model"
228
- : "Ask an environment admin to download"}
229
- </Button>
231
+ {model ? (
232
+ <Button
233
+ disabled={!canManage || props.downloading || !status}
234
+ onClick={props.onDownload}
235
+ >
236
+ {props.downloading
237
+ ? "Downloading…"
238
+ : props.error
239
+ ? "Try again"
240
+ : canManage
241
+ ? "Download model"
242
+ : "Ask an environment admin to download"}
243
+ </Button>
244
+ ) : null}
245
</>
246
) : (
247
<>
apps/web/src/components/preview/PreviewAnnotationSpeech.tsx
+2
@@ -184,9 +184,11 @@ export function PreviewAnnotationSpeech({
184
step={speech.setup.step}
185
status={speech.status}
186
model={speech.setup.model}
187
+ language={speech.setup.language}
188
downloading={speech.setup.downloading}
189
error={speech.setup.error}
190
onOpenChange={speech.setup.setOpen}
191
+ onRetryCatalog={speech.setup.retryCatalog}
192
onDownload={() => void speech.setup.download()}
193
onCancelDownload={() => void speech.setup.cancelDownload()}
194
onStartRecording={() => void speech.setup.startRecording()}
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+3
@@ -161,7 +161,10 @@ it("tests the selected model after download and preserves its transcript across
161
open: false,
162
step: 0,
163
model: null,
164
+ language: null,
165
downloading: false,
166
+ catalogFailed: false,
167
+ retryCatalog: null,
168
error: null,
169
setOpen: () => {},
170
download: async () => {},
apps/web/src/components/settings/VoiceSettingsPanel.tsx
+4
-1
@@ -47,6 +47,7 @@ import { useEffect, useRef, useState, type ReactNode } from "react";
47
48
import { useStableCallback } from "~/hooks/useStableCallback";
49
import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";
50
+import { preferredSpeechLanguages } from "../../speech/preferredSpeechLanguages";
51
import {
52
chooseAudioDestination,
53
chooseTranscriptCleanup,
@@ -760,7 +761,7 @@ function SpeechModelBrowser(props: {
761
const [downloadedOnly, setDownloadedOnly] = useState(false);
762
const [languageSearch, setLanguageSearch] = useState("");
763
const [selection, setSelection] = useState<string[] | null>(null);
763
- const preferred = navigator.languages.map((language) => language.toLowerCase().split("-")[0]!);
764
+ const [preferred] = useState(preferredSpeechLanguages);
765
const supportedLanguages = [...new Set(models.flatMap((model) => model.languages))];
766
const defaultLanguage = preferred.find((code) => supportedLanguages.includes(code));
767
const selectedLanguages = selection ?? (defaultLanguage ? [defaultLanguage] : []);
@@ -1077,9 +1078,11 @@ function TranscriptionTest(props: {
1078
step={speech.setup.step}
1079
status={speech.status}
1080
model={speech.setup.model}
1081
+ language={speech.setup.language}
1082
downloading={speech.setup.downloading}
1083
error={speech.setup.error}
1084
onOpenChange={speech.setup.setOpen}
1085
+ onRetryCatalog={speech.setup.retryCatalog}
1086
onDownload={() => void speech.setup.download()}
1087
onCancelDownload={() => void speech.setup.cancelDownload()}
1088
onStartRecording={() => void speech.setup.startRecording()}
apps/web/src/speech/preferredSpeechLanguages.ts
new
+10
@@ -0,0 +1,10 @@
1
+// Packaged desktop pins `navigator.languages` to en-US, so the OS's own list leads.
2
+export function preferredSpeechLanguages() {
3
+ const bridge = window.desktopBridge;
4
+ let system = bridge?.getPreferredSystemLanguages?.() ?? [];
5
+ if (!system.length) {
6
+ const locale = bridge?.getSystemLocale?.();
7
+ if (locale) system = [locale];
8
+ }
9
+ return [...system, ...navigator.languages].map((tag) => tag.toLowerCase().split(/[-_]/)[0]!);
10
+}
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+217
-6
@@ -1,19 +1,24 @@
1
import { useLayoutEffect } from "react";
2
import { expect, it, vi } from "vite-plus/test";
3
+import { page, userEvent } from "vite-plus/test/browser";
4
+import * as Deferred from "effect/Deferred";
5
import * as Effect from "effect/Effect";
6
import * as Option from "effect/Option";
7
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
8
+import { getEnvironmentSpeechModels } from "@t3tools/client-runtime/voice-input";
9
import {
10
DEFAULT_CLIENT_SETTINGS,
11
DEFAULT_SERVER_SETTINGS,
12
EnvironmentId,
13
+ type EnvironmentSpeechModel,
14
type ProjectId,
15
type SpeechCustomWords,
16
type SpeechPostProcessingOptions,
17
type SpeechTranscriptionOptions,
18
} from "@t3tools/contracts";
19
16
-import { act, render } from "../../test/render";
20
+import { act, render, stubGlobal } from "../../test/render";
21
+import { VoiceInputSetup } from "../components/chat/VoiceInputSetup";
22
import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
23
import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
24
import {
@@ -36,7 +41,12 @@ const mocks = vi.hoisted(() => ({
41
statusRequests: [] as string[],
42
statusUnavailable: false,
43
modelState: "ready" as "ready" | "missing-model",
39
- download: vi.fn<() => Promise<void>>(),
44
+ statusModelId: "test-model",
45
+ models: [] as EnvironmentSpeechModel[],
46
+ catalogOffline: false,
47
+ download: vi.fn<(modelId: string) => Promise<void>>(),
48
+ selected: [] as string[],
49
+ selectionFails: false,
50
platforms: [] as string[],
51
microphoneRequests: [] as string[],
52
transcribeCalls: [] as string[],
@@ -80,6 +90,7 @@ vi.mock("../state/environments", () => ({
90
: null,
91
}));
92
vi.mock("../hooks/useSettings", () => ({
93
+ updateClientSettings: vi.fn(),
94
useClientSettingsHydrated: () => true,
95
useClientSettings: (selector?: (settings: Record<string, unknown>) => unknown) => {
96
const settings = { ...DEFAULT_CLIENT_SETTINGS, ...mocks.settings };
@@ -109,7 +120,7 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
120
return {
121
supported: true,
122
state: mocks.modelState,
112
- modelId: "test-model",
123
+ modelId: mocks.statusModelId,
124
model: "Test",
125
size: 1,
126
supportsStreaming: false,
@@ -119,8 +130,24 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
130
gpuDevices: [],
131
};
132
}),
122
- downloadEnvironmentSpeechModel: () => Effect.promise(mocks.download),
123
- getEnvironmentSpeechModels: () => Effect.succeed({ models: [] }),
133
+ downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
134
+ Effect.promise(async () => {
135
+ await mocks.download(modelId);
136
+ return { models: mocks.models };
137
+ }),
138
+ selectEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
139
+ Effect.sync(() => {
140
+ mocks.selected.push(modelId);
141
+ if (mocks.selectionFails) throw new Error("selection failed");
142
+ mocks.statusModelId = modelId;
143
+ return { models: mocks.models };
144
+ }),
145
+ getEnvironmentSpeechModels: vi.fn(() =>
146
+ Effect.sync(() => {
147
+ if (mocks.catalogOffline) throw new Error("Catalog offline");
148
+ return { models: mocks.models };
149
+ }),
150
+ ),
151
postProcessEnvironmentTranscript: (
152
prepared: PreparedConnection,
153
transcript: string,
@@ -189,7 +216,12 @@ function setup(options: { readonly destination?: string | null } = {}) {
216
mocks.statusRequests = [];
217
mocks.statusUnavailable = false;
218
mocks.modelState = "ready";
219
+ mocks.statusModelId = "test-model";
220
+ mocks.models = [];
221
+ mocks.catalogOffline = false;
222
mocks.download.mockReset();
223
+ mocks.selected = [];
224
+ mocks.selectionFails = false;
225
mocks.platforms = [];
226
mocks.microphoneRequests = [];
227
mocks.transcribeCalls = [];
@@ -213,6 +245,7 @@ function Probe(props: {
245
readonly projectId?: string;
246
readonly projectName?: string;
247
readonly cleanup?: boolean;
248
+ readonly showWizard?: boolean;
249
}) {
250
const value = useEnvironmentSpeechInput({
251
environmentId: THREAD_ENV as EnvironmentId,
@@ -230,7 +263,18 @@ function Probe(props: {
263
useLayoutEffect(() => {
264
mocks.voice = value;
265
});
233
- return null;
266
+ return props.showWizard ? (
267
+ <VoiceInputSetup
268
+ {...value.setup}
269
+ environmentId={value.transcriptionEnvironmentId}
270
+ status={value.status}
271
+ onOpenChange={value.setup.setOpen}
272
+ onRetryCatalog={value.setup.retryCatalog}
273
+ onDownload={value.setup.download}
274
+ onCancelDownload={value.setup.cancelDownload}
275
+ onStartRecording={value.setup.startRecording}
276
+ />
277
+ ) : null;
278
}
279
280
function expectNoAudioSent() {
@@ -333,6 +377,8 @@ it("retries a failed transcription on the same chosen environment", async () =>
377
it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {
378
setup({ destination: "env-a" });
379
mocks.modelState = "missing-model";
380
+ mocks.settings = { speechLanguage: "en" };
381
+ mocks.models = [catalogModel(mocks.statusModelId, ["en"], true)];
382
let finishDownload!: () => void;
383
mocks.download.mockReturnValue(
384
new Promise<void>((resolve) => {
@@ -360,6 +406,171 @@ it("reopens setup after reconnecting during a download and ignores its stale com
406
expect(mocks.voice.setup.error).toBeNull();
407
});
408
409
+function catalogModel(
410
+ id: string,
411
+ languages: readonly string[],
412
+ recommended: boolean,
413
+): EnvironmentSpeechModel {
414
+ return {
415
+ id,
416
+ name: id,
417
+ description: "",
418
+ size: 1,
419
+ languages,
420
+ accuracy: 0,
421
+ speed: 0,
422
+ recommended,
423
+ supportsStreaming: false,
424
+ supportsLanguageDetection: languages.length > 1,
425
+ supportsTranslation: false,
426
+ active: false,
427
+ state: "downloadable",
428
+ };
429
+}
430
+
431
+it.each([
432
+ ["the browser prefers German", ["de-DE"], undefined, "auto"],
433
+ ["the desktop exposes no locale methods", ["de-DE"], {}, "auto"],
434
+ [
435
+ "an older preload exposes only the OS locale",
436
+ ["en-US"],
437
+ { getSystemLocale: () => "de-DE" },
438
+ "auto",
439
+ ],
440
+ [
441
+ "the desktop OS prefers German",
442
+ ["en-US"],
443
+ { getPreferredSystemLanguages: () => ["de-DE"] },
444
+ "auto",
445
+ ],
446
+ ["German is the chosen Voice language", ["en-US"], undefined, "de"],
447
+])(
448
+ "offers and downloads a German-capable model when %s",
449
+ async (_, languages, bridge, speechLanguage) => {
450
+ setup({ destination: "env-a" });
451
+ mocks.settings = { speechLanguage };
452
+ using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(languages);
453
+ using _bridge = stubGlobal("desktopBridge", bridge);
454
+ const englishDefault = "handy-computer/parakeet-unified-en-0.6b-gguf";
455
+ const german = "handy-computer/nemotron-3.5-asr-streaming-0.6b-gguf";
456
+ mocks.modelState = "missing-model";
457
+ mocks.statusModelId = englishDefault;
458
+ mocks.models = [
459
+ { ...catalogModel(englishDefault, ["en"], true), active: true },
460
+ catalogModel(german, ["en", "de", "fr"], true),
461
+ catalogModel("handy-computer/canary-180m-flash-gguf", ["en", "de", "es", "fr"], true),
462
+ ];
463
+ mocks.download.mockResolvedValue(undefined);
464
+ await using _screen = await render(<Probe />);
465
+ await act(() => mocks.voice.start());
466
+ expect(mocks.voice.setup.open).toBe(true);
467
+ await act(() => mocks.voice.setup.download());
468
+ expect(mocks.download).toHaveBeenCalledExactlyOnceWith(german);
469
+ expect(mocks.voice.setup.model?.id).toBe(german);
470
+ },
471
+);
472
+
473
+it.each(["cancel", "failed restart"] as const)(
474
+ "closes pending catalog setup on %s",
475
+ async (action) => {
476
+ setup({ destination: "env-a" });
477
+ mocks.modelState = "missing-model";
478
+ mocks.settings = { speechLanguage: "en" };
479
+ mocks.models = [catalogModel("current", ["en"], true)];
480
+ const response = Deferred.makeUnsafe<{ models: EnvironmentSpeechModel[] }>();
481
+ vi.mocked(getEnvironmentSpeechModels).mockReturnValueOnce(Deferred.await(response));
482
+ await using _screen = await render(<Probe showWizard />);
483
+ let firstStart!: Promise<void>;
484
+ await act(() => {
485
+ firstStart = mocks.voice.start();
486
+ });
487
+ await expect.element(page.getByText("Loading transcription models…")).toBeVisible();
488
+ if (action === "cancel") {
489
+ await act(() => mocks.voice.cancel());
490
+ } else {
491
+ mocks.statusUnavailable = true;
492
+ await act(() => mocks.voice.start());
493
+ mocks.statusUnavailable = false;
494
+ }
495
+ expect(mocks.voice.setup.open).toBe(false);
496
+
497
+ await act(() => mocks.voice.start());
498
+ Deferred.doneUnsafe(
499
+ response,
500
+ Effect.succeed({ models: [catalogModel("stale", ["en"], true)] }),
501
+ );
502
+ await act(() => firstStart);
503
+ await expect.element(page.getByRole("heading", { name: "current" })).toBeVisible();
504
+ expect(mocks.microphoneRequests).toEqual([]);
505
+ },
506
+);
507
+
508
+it.each([true, false])(
509
+ "never downloads the English default when no German recommendation is available (catalog offline: %s)",
510
+ async (offline) => {
511
+ setup({ destination: "env-a" });
512
+ mocks.settings = { speechLanguage: "de" };
513
+ mocks.modelState = "missing-model";
514
+ mocks.statusModelId = "english";
515
+ mocks.models = [catalogModel("english", ["en"], true)];
516
+ mocks.catalogOffline = offline;
517
+ await using _screen = await render(<Probe showWizard />);
518
+ await act(() => mocks.voice.start());
519
+ const wizard = page.getByRole("dialog", { name: "Set up voice input" });
520
+ const retry = wizard.getByRole("button", { name: "Retry loading models" });
521
+ const download = wizard.getByRole("button", { name: "Download model" });
522
+ await expect.element(wizard.getByRole("alert")).toBeVisible();
523
+ await expect.element(download).not.toBeInTheDocument();
524
+ if (offline) {
525
+ mocks.catalogOffline = false;
526
+ mocks.models.push(catalogModel("german", ["de"], true));
527
+ // Another client can finish setup while the catalog is offline.
528
+ mocks.modelState = "ready";
529
+ await userEvent.click(retry);
530
+ await expect.element(wizard.getByRole("heading", { name: "german" })).toBeVisible();
531
+ await expect.element(download).toBeEnabled();
532
+ }
533
+ await expect.element(retry).not.toBeInTheDocument();
534
+ expect(mocks.download).not.toHaveBeenCalled();
535
+ expect(mocks.microphoneRequests).toEqual([]);
536
+ },
537
+);
538
+
539
+it.each([
540
+ ["selects the offered model once its download installed it", "installed", ["german"], null],
541
+ [
542
+ "does not advance a cancelled download when another model is ready",
543
+ "downloadable",
544
+ [],
545
+ "The model is not ready. Try downloading it again.",
546
+ ],
547
+ [
548
+ "reports a failed selection apart from the download",
549
+ "installed",
550
+ ["german"],
551
+ "The model downloaded, but this environment could not select it. Try again.",
552
+ ],
553
+] as const)("%s", async (_, landedState, selected, error) => {
554
+ setup({ destination: "env-a" });
555
+ using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(["de-DE"]);
556
+ mocks.selectionFails = error !== null && landedState === "installed";
557
+ mocks.modelState = "missing-model";
558
+ mocks.statusModelId = "english";
559
+ mocks.models = [catalogModel("english", ["en"], true), catalogModel("german", ["de"], true)];
560
+ mocks.download.mockImplementation(async () => {
561
+ mocks.models = [mocks.models[0]!, { ...mocks.models[1]!, state: landedState }];
562
+ mocks.modelState = "ready";
563
+ });
564
+ await using _screen = await render(<Probe />);
565
+ await act(() => mocks.voice.start());
566
+ expect(mocks.voice.setup.language).toBe("de");
567
+ await act(() => mocks.voice.setup.download());
568
+ expect(mocks.selected).toEqual(selected);
569
+ expect(mocks.voice.setup.error).toBe(error);
570
+ expect(mocks.voice.setup.downloading).toBe(false);
571
+ expect(mocks.voice.setup.step).toBe(landedState === "installed" && !error ? 1 : 0);
572
+});
573
+
574
it("does not resend a failed recording after consent is revoked", async () => {
575
setup({ destination: "env-a" });
576
mocks.transcripts = [new Error("inference failed")];
apps/web/src/speech/useEnvironmentSpeechInput.ts
+74
-23
@@ -4,6 +4,7 @@ import {
4
getEnvironmentSpeechModels,
5
getEnvironmentSpeechStatus,
6
postProcessEnvironmentTranscript,
7
+ selectEnvironmentSpeechModel,
8
VoiceInputController,
9
voiceInputBlocksSubmission,
10
type VoiceDraftSnapshot,
@@ -22,7 +23,7 @@ import type {
23
SpeechTranscriptionOptions,
24
} from "@t3tools/contracts";
25
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
25
-import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
26
+import { chooseSetupSpeechModel, resolveSpeechVocabulary } from "@t3tools/shared/speech";
27
import { useNavigate } from "@tanstack/react-router";
28
import * as Option from "effect/Option";
29
import { useEffect, useRef, useState } from "react";
@@ -37,6 +38,7 @@ import {
38
import { useEnvironment } from "../state/environments";
39
import { useEnvironmentScope, usePreparedConnection } from "../state/session";
40
import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
41
+import { preferredSpeechLanguages } from "./preferredSpeechLanguages";
42
import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
43
import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent";
44
@@ -46,7 +48,9 @@ const INITIAL_SETUP = {
48
open: false,
49
step: 0,
50
model: null as EnvironmentSpeechModel | null,
51
+ language: null as string | null,
52
downloading: false,
53
+ catalogFailed: false,
54
error: null as string | null,
55
};
56
@@ -387,11 +391,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
391
};
392
}, [prepared, cloudProvider]);
393
const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null;
394
+ const setupModelId = setup.model?.id;
395
396
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download
397
useEffect(() => {
393
- if (!setup.open || !setup.downloading || !prepared || !currentStatus?.supported) return;
394
- const modelId = currentStatus.modelId;
398
+ if (!setup.open || !setup.downloading || !prepared || !setupModelId) return;
399
let disposed = false;
400
let refreshing = false;
401
const timer = window.setInterval(() => {
@@ -400,8 +404,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {
404
void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared))
405
.then((result) => {
406
if (disposed) return;
403
- const model = result.models.find((candidate) => candidate.id === modelId) ?? null;
404
- setSetup((current) => ({ ...current, model }));
407
+ const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null;
408
+ if (model) setSetup((current) => ({ ...current, model }));
409
})
410
.catch(() => undefined)
411
.finally(() => {
@@ -412,7 +416,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
416
disposed = true;
417
window.clearInterval(timer);
418
};
415
- }, [currentStatus, prepared, setup.downloading, setup.open]);
419
+ }, [prepared, setup.downloading, setup.open, setupModelId]);
420
421
const previousOwnerRef = useRef(input.ownerKey);
422
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes
@@ -445,8 +449,34 @@ export function useEnvironmentSpeechInput(input: HookInput) {
449
if (controller.currentState.phase === "recording") void controller.stop();
450
});
451
452
+ const loadSetupCatalog = useStableCallback(
453
+ async (connection: PreparedConnection, selectedModelId: string, request: number) => {
454
+ setSetup({ ...INITIAL_SETUP, open: true });
455
+ const catalog = await runEnvironmentSpeech(getEnvironmentSpeechModels(connection)).catch(
456
+ () => null,
457
+ );
458
+ if (request !== startRequestRef.current) return;
459
+ const choice = chooseSetupSpeechModel(catalog?.models ?? [], {
460
+ selectedModelId,
461
+ intent: latestRef.current.clientSettings.speechLanguage,
462
+ preferred: preferredSpeechLanguages(),
463
+ });
464
+ setSetup((current) => ({
465
+ ...current,
466
+ model: choice?.model ?? null,
467
+ language: choice?.language ?? null,
468
+ catalogFailed: catalog === null,
469
+ error: choice
470
+ ? null
471
+ : catalog
472
+ ? "No model matches your language. Choose a model or change the language in Voice settings."
473
+ : "Could not load transcription models. Check the connection and try again.",
474
+ }));
475
+ },
476
+ );
477
+
478
const start = useStableCallback(async () => {
449
- if (latestRef.current.input.disabled) return;
479
+ if (latestRef.current.input.disabled || (setup.open && setup.downloading)) return;
480
if (!grantId) {
481
if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false)
482
await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });
@@ -466,6 +496,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
496
controllerRef.current === controller &&
497
latestRef.current.input.ownerKey === owner &&
498
!latestRef.current.input.disabled;
499
+ setSetup(INITIAL_SETUP);
500
setQueuedStart(prepared);
501
let latestStatus: EnvironmentSpeechStatus;
502
try {
@@ -481,8 +512,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
512
return;
513
}
514
if (!stillCurrent()) return;
484
- setQueuedStart(null);
515
setStatus({ prepared, value: latestStatus });
516
+ setQueuedStart(null);
517
+ if (latestStatus.supported && latestStatus.state === "missing-model") {
518
+ return loadSetupCatalog(prepared, latestStatus.modelId, request);
519
+ }
520
if (!latestStatus.supported) {
521
toastManager.add({
522
type: "error",
@@ -491,32 +525,46 @@ export function useEnvironmentSpeechInput(input: HookInput) {
525
});
526
return;
527
}
494
- if (latestStatus.state === "missing-model") {
495
- setSetup({ ...INITIAL_SETUP, open: true });
496
- return;
497
- }
528
setLevel(0);
529
await controller.start();
530
});
531
532
const downloadSetupModel = useStableCallback(async () => {
503
- if (!prepared || !currentStatus?.supported || setup.downloading) return;
533
+ if (!prepared || !currentStatus?.supported || !setupModelId || setup.downloading) return;
534
+ const modelId = setupModelId;
535
const request = ++startRequestRef.current;
536
setSetup((current) => ({ ...current, downloading: true, error: null }));
537
try {
507
- await runEnvironmentSpeech(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));
538
+ const { models } = await runEnvironmentSpeech(
539
+ downloadEnvironmentSpeechModel(prepared, modelId),
540
+ );
541
+ if (request !== startRequestRef.current) return;
542
+ // A download cancelled from elsewhere still resolves, so select the model only once it landed.
543
+ const installed = models.some(({ id, state }) => id === modelId && state === "installed");
544
+ if (installed && modelId !== currentStatus.modelId) {
545
+ try {
546
+ await runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, modelId));
547
+ } catch {
548
+ if (request !== startRequestRef.current) return;
549
+ setSetup((current) => ({
550
+ ...current,
551
+ downloading: false,
552
+ error: "The model downloaded, but this environment could not select it. Try again.",
553
+ }));
554
+ return;
555
+ }
556
+ }
557
if (request !== startRequestRef.current) return;
558
const next = await probeSpeechStatus(prepared, true);
559
if (request !== startRequestRef.current) return;
560
setStatus({ prepared, value: next });
561
+ const ready =
562
+ installed && next.supported && next.modelId === modelId && next.state !== "missing-model";
563
setSetup((current) => ({
564
...current,
565
downloading: false,
515
- step: next.supported && next.state !== "missing-model" ? 1 : current.step,
516
- error:
517
- next.supported && next.state !== "missing-model"
518
- ? null
519
- : "The model is not ready. Try downloading it again.",
566
+ step: ready ? 1 : current.step,
567
+ error: ready ? null : "The model is not ready. Try downloading it again.",
568
}));
569
} catch {
570
if (request !== startRequestRef.current) return;
@@ -529,13 +577,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
577
});
578
579
const cancelSetupDownload = useStableCallback(async () => {
532
- if (!prepared || !currentStatus?.supported) return;
580
+ if (!prepared || !setupModelId) return;
581
startRequestRef.current += 1;
582
setSetup((current) => ({ ...current, open: false, downloading: false }));
583
try {
536
- await runEnvironmentSpeech(
537
- cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId),
538
- );
584
+ await runEnvironmentSpeech(cancelEnvironmentSpeechModelDownload(prepared, setupModelId));
585
} catch (error) {
586
setSetup((current) => ({
587
...current,
@@ -562,6 +608,10 @@ export function useEnvironmentSpeechInput(input: HookInput) {
608
setup: {
609
...setup,
610
setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
611
+ retryCatalog:
612
+ setup.open && setup.catalogFailed && prepared && currentStatus?.supported
613
+ ? () => loadSetupCatalog(prepared, currentStatus.modelId, ++startRequestRef.current)
614
+ : null,
615
download: downloadSetupModel,
616
cancelDownload: cancelSetupDownload,
617
startRecording: async () => {
@@ -582,6 +632,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
632
pendingSubmissionRef.current = null;
633
startRequestRef.current += 1;
634
setQueuedStart(null);
635
+ setSetup(INITIAL_SETUP);
636
controllerRef.current?.cancel();
637
}),
638
skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()),
apps/web/src/speech/useTextFieldSpeech.tsx
+2
@@ -110,9 +110,11 @@ export function TextFieldSpeechControls({
110
step={speech.setup.step}
111
status={speech.status}
112
model={speech.setup.model}
113
+ language={speech.setup.language}
114
downloading={speech.setup.downloading}
115
error={speech.setup.error}
116
onOpenChange={speech.setup.setOpen}
117
+ onRetryCatalog={speech.setup.retryCatalog}
118
onDownload={() => void speech.setup.download()}
119
onCancelDownload={() => void speech.setup.cancelDownload()}
120
onStartRecording={() => void speech.setup.startRecording()}
docs/user/voice-input.md
+1
@@ -52,6 +52,7 @@ until you choose another.
52
In **Settings → Voice → Transcription**, download a model and choose **Use model** if it
53
isn’t already selected. **Test model** lets you try it without sending a message.
54
If you use the composer microphone before a model is installed, it offers to download one.
55
+The recommendation uses your Voice language, or your system’s preferred languages with **Auto-detect**.
56
57
Select the languages you dictate in to compare models that support all of them.
58
Accuracy shows how often the model got words or characters wrong in each
packages/contracts/src/ipc.ts
+2
@@ -1048,6 +1048,8 @@ export interface DesktopBridge {
1048
* regardless of OS settings.
1049
*/
1050
getSystemLocale: () => string | null;
1051
+ /** The OS's preferred languages, most preferred first; `navigator.languages` is pinned like the locale. */
1052
+ getPreferredSystemLanguages: () => readonly string[];
1053
getLocalEnvironmentBootstraps: () => readonly DesktopEnvironmentBootstrap[];
1054
getLocalEnvironmentEnabled: () => boolean;
1055
setLocalEnvironmentEnabled: (enabled: boolean) => Promise<void>;
packages/shared/src/speech.test.ts
+79
@@ -2,6 +2,7 @@ import { describe, expect, it } from "vite-plus/test";
2
import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SPEECH_MAX_OPTIONS_BYTES } from "@t3tools/contracts";
3
4
import {
5
+ chooseSetupSpeechModel,
6
decodeSpeechPcmRequest,
7
encodeSpeechPcmRequest,
8
resolveSpeechVocabulary,
@@ -71,3 +72,81 @@ describe("resolveSpeechVocabulary", () => {
72
expect(resolved.at(-1)?.term).toBe("word97");
73
});
74
});
75
+
76
+describe("chooseSetupSpeechModel", () => {
77
+ const model = (
78
+ id: string,
79
+ languages: string[],
80
+ supportsLanguageDetection: boolean,
81
+ recommended: boolean,
82
+ ) => ({ id, languages, supportsLanguageDetection, recommended });
83
+ const catalog = [
84
+ model("unrecommended", ["en", "de", "es"], true, false),
85
+ model("english", ["en"], false, true),
86
+ model("fixed", ["en", "de"], false, true),
87
+ model("detecting", ["en", "de", "fr"], true, true),
88
+ ];
89
+
90
+ it.each([
91
+ [
92
+ "keeps the selected English model for an English speaker",
93
+ "english",
94
+ "auto",
95
+ ["en"],
96
+ { id: "english", language: "en" },
97
+ ],
98
+ [
99
+ "skips models that would transcribe English instead of German",
100
+ "english",
101
+ "auto",
102
+ ["de", "en"],
103
+ { id: "detecting", language: "de" },
104
+ ],
105
+ [
106
+ "accepts a model without detection for a chosen language",
107
+ "english",
108
+ "de",
109
+ ["en"],
110
+ { id: "fixed", language: "de" },
111
+ ],
112
+ [
113
+ "keeps a selected model that transcribes the language",
114
+ "unrecommended",
115
+ "auto",
116
+ ["de"],
117
+ { id: "unrecommended", language: "de" },
118
+ ],
119
+ [
120
+ "moves on to the next preferred language a model transcribes",
121
+ "english",
122
+ "auto",
123
+ ["ja", "fr"],
124
+ { id: "detecting", language: "fr" },
125
+ ],
126
+ [
127
+ "falls back to a model that is not recommended",
128
+ "english",
129
+ "auto",
130
+ ["es"],
131
+ { id: "unrecommended", language: "es" },
132
+ ],
133
+ ["offers nothing when no model transcribes the language", "english", "auto", ["ja"], undefined],
134
+ [
135
+ "offers the selected model without a language recommendation when Auto has no preferences",
136
+ "fixed",
137
+ "auto",
138
+ [],
139
+ { id: "fixed", language: null },
140
+ ],
141
+ [
142
+ "does not guess another model when Auto has no preferences or selection",
143
+ "gone",
144
+ "auto",
145
+ [],
146
+ undefined,
147
+ ],
148
+ ] as const)("%s", (_, selectedModelId, intent, preferred, expected) => {
149
+ const choice = chooseSetupSpeechModel(catalog, { selectedModelId, intent, preferred });
150
+ expect(choice && { id: choice.model.id, language: choice.language }).toEqual(expected);
151
+ });
152
+});
packages/shared/src/speech.ts
+38
@@ -36,6 +36,44 @@ export function effectiveSpeechLanguage(
36
return model.languages.find((language) => language === "en") ?? model.languages[0] ?? "auto";
37
}
38
39
+/**
40
+ * The model first-run setup offers: the selected one, else a recommended one, else any that the
41
+ * server would really transcribe the person's language with. Without a language hint,
42
+ * offer only the selected model, without a recommendation.
43
+ */
44
+export function chooseSetupSpeechModel<
45
+ Model extends {
46
+ readonly id: string;
47
+ readonly languages: readonly string[];
48
+ readonly supportsLanguageDetection: boolean;
49
+ readonly recommended: boolean;
50
+ },
51
+>(
52
+ models: readonly Model[],
53
+ input: {
54
+ readonly selectedModelId: string;
55
+ readonly intent: string;
56
+ readonly preferred: readonly string[];
57
+ },
58
+) {
59
+ if (input.intent === "auto" && input.preferred.length === 0) {
60
+ const model = models.find(({ id }) => id === input.selectedModelId);
61
+ return model && { model, language: null };
62
+ }
63
+ for (const language of input.intent === "auto" ? input.preferred : [input.intent]) {
64
+ const capable = models.filter((model) => {
65
+ const effective = effectiveSpeechLanguage(model, input.intent);
66
+ return effective === language || (effective === "auto" && model.languages.includes(language));
67
+ });
68
+ const model =
69
+ capable.find(({ id }) => id === input.selectedModelId) ??
70
+ capable.find(({ recommended }) => recommended) ??
71
+ capable[0];
72
+ if (model) return { model, language };
73
+ }
74
+ return undefined;
75
+}
76
+
77
const vocabularyKey = (value: string) => value.trim().normalize("NFC").toLocaleLowerCase();
78
79
/**