@devries-cameron20 / t3code Mirror · read-only / commits / fe901ac7ba
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 02:00 UTC

feat(voice): opt into cloud dictation with OpenAI, Grok, or OpenRouter (#31769)

Co-authored-by: Aditya Vardhan Sharma <adityaradhika13@gmail.com>

Julius Marminge committed Nov 6, 2026 at 21:51 UTC fe901ac7ba95e142ddf77599bb5b5187e6c5e3d8
40 files changed +2971 -214
apps/mobile/app.config.ts
+3
@@ -458,6 +458,9 @@ const config: ExpoConfig = {
458 ],
459 extra: {
460 appVariant: APP_VARIANT,
461 + secretsFeatureEnabled: ["1", "true"].includes(
462 + process.env.T3CODE_FEATURE_SECRETS?.trim().toLowerCase() ?? "",
463 + ),
464 iosPersonalTeamBuild: isIosPersonalTeamBuild,
465 relay: {
466 url: repoEnv.T3CODE_RELAY_URL ?? null,
apps/mobile/src/components/ProviderIcon.tsx
+3 -2
@@ -11,7 +11,7 @@ import { resolveOfficialAcpRegistryIconUrl } from "@t3tools/contracts";
11 import { useAppearancePreferences } from "../features/settings/appearance/AppearancePreferencesProvider";
12 import { AppText as Text } from "./ui/AppText";
13 import { cn } from "../lib/cn";
14 -import { getProviderClient } from "@t3tools/client-runtime/provider-clients";
14 +import { getProviderClient, openRouterIcon } from "@t3tools/client-runtime/provider-clients";
15
16 type ProviderIconProps = {
17 readonly provider: string | null | undefined;
@@ -99,7 +99,8 @@ export function ProviderIcon(props: ProviderIconProps) {
99 );
100 }
101
102 - const packageIcon = getProviderClient(props.provider)?.icon;
102 + const packageIcon =
103 + props.provider === "openrouter" ? openRouterIcon : getProviderClient(props.provider)?.icon;
104 if (packageIcon) {
105 const themeFill = (fill: { readonly light: string; readonly dark: string }) =>
106 isDarkMode ? fill.dark : fill.light;
apps/mobile/src/features/settings/CloudVoiceSection.tsx new
+473
@@ -0,0 +1,473 @@
1 +import { useAtomRefresh, useAtomValue } from "@effect/atom-react";
2 +import { cloudSpeechDisclosure } from "@t3tools/shared/speech";
3 +import { setCloudSpeechKey } from "@t3tools/client-runtime/voice-input";
4 +import {
5 + configuredSecretSources,
6 + secretRequestReference,
7 +} from "@t3tools/client-runtime/secret-request";
8 +import {
9 + AuthSettingsWriteScope,
10 + CLOUD_SPEECH_LABELS,
11 + CLOUD_SPEECH_PROVIDER_NAMES,
12 + CloudSpeechProvider,
13 + CloudSpeechApiKey,
14 + CloudSpeechModelId,
15 + isSecretReference,
16 + type CloudSpeechModel,
17 + type SecretProviderId,
18 + type SettingsSecretValue,
19 + type VoiceTranscriptionDestination,
20 +} from "@t3tools/contracts";
21 +import * as Schema from "effect/Schema";
22 +import Constants from "expo-constants";
23 +import { AsyncResult } from "effect/reactivity";
24 +import { useState } from "react";
25 +import { ScrollView, TextInput, View } from "react-native";
26 +import { Sheet } from "../../components/ui/Sheet";
27 +import * as Dialog from "../../components/ui/Dialog";
28 +import { ProviderIcon } from "../../components/ProviderIcon";
29 +import { AppText as Text } from "../../components/ui/AppText";
30 +import { ListActionRow } from "../../components/ui/ListActionRow";
31 +import { ListChoiceRow } from "../../components/ui/ListChoiceRow";
32 +import { Chip } from "../../components/ui/Chip";
33 +import type { EnvironmentPresentation } from "../../state/environments";
34 +import { useEnvironmentScope } from "../../state/session";
35 +import {
36 + chooseVoiceDestination,
37 + pendingCloudVoiceChoice,
38 + cloudVoiceConnectionAtom,
39 + cloudVoiceKeysAtom,
40 + cloudVoiceModelsAtom,
41 + runSpeech,
42 +} from "../voice-input/voiceDestination";
43 +import { SettingsSection } from "./components/SettingsSection";
44 +
45 +type CloudDestination = Extract<VoiceTranscriptionDestination, { kind: "cloud" }>;
46 +type Props = { environment: EnvironmentPresentation; active: CloudDestination | null };
47 +const isApiKey = Schema.is(CloudSpeechApiKey);
48 +const isModelId = Schema.is(CloudSpeechModelId);
49 +const MODEL_ROWS = 6;
50 +
51 +export function CloudVoiceSection({ environment, active }: Props) {
52 + const [open, setOpen] = useState(false);
53 + return (
54 + <SettingsSection title={`Cloud via ${environment.label}`}>
55 + <ListActionRow
56 + label="Cloud providers"
57 + description={
58 + active
59 + ? `Using ${CLOUD_SPEECH_LABELS[active.provider]}`
60 + : "Optional · use your own API key"
61 + }
62 + onPress={() => setOpen(true)}
63 + />
64 + {open && (
65 + <CloudChoices environment={environment} active={active} onClose={() => setOpen(false)} />
66 + )}
67 + </SettingsSection>
68 + );
69 +}
70 +
71 +function ModelPicker({
72 + models,
73 + value,
74 + current,
75 + disabled,
76 + onChange,
77 +}: {
78 + models: ReadonlyArray<CloudSpeechModel>;
79 + value: string;
80 + /** The grant's model, flagged when the catalog no longer lists it. */
81 + current: string | undefined;
82 + disabled: boolean;
83 + onChange: (model: CloudSpeechModelId) => void;
84 +}) {
85 + const [query, setQuery] = useState("");
86 + const [showAll, setShowAll] = useState(false);
87 + const needle = query.trim().toLowerCase();
88 + const selected = models.find((model) => model.id === value);
89 + const matches = models.filter((model) =>
90 + `${model.name} ${model.id}`.toLowerCase().includes(needle),
91 + );
92 + const shown = showAll ? matches : matches.slice(0, MODEL_ROWS);
93 + const delisted = current !== undefined && !models.some((model) => model.id === current);
94 + return (
95 + <>
96 + <TextInput
97 + accessibilityLabel="Search OpenRouter models"
98 + placeholder="Search transcription models"
99 + autoCapitalize="none"
100 + autoCorrect={false}
101 + editable={!disabled}
102 + value={query}
103 + onChangeText={setQuery}
104 + className="mx-4 rounded-lg bg-background px-3 py-2 text-foreground"
105 + />
106 + {delisted ? (
107 + <Text className="mx-4 text-sm text-destructive">
108 + OpenRouter no longer lists {current}. Dictation stays off until you choose another model.
109 + </Text>
110 + ) : null}
111 + {selected && !shown.some((model) => model.id === selected.id) ? (
112 + <ListChoiceRow
113 + label={selected.name}
114 + description={selected.id}
115 + selected
116 + disabled={disabled}
117 + onPress={() => {}}
118 + />
119 + ) : null}
120 + {shown.map((model, index) => (
121 + <ListChoiceRow
122 + key={model.id}
123 + label={model.name}
124 + description={model.id}
125 + selected={model.id === value}
126 + separated={index > 0}
127 + disabled={disabled}
128 + onPress={() => onChange(model.id)}
129 + />
130 + ))}
131 + {matches.length === 0 ? (
132 + <Text className="mx-4 text-sm text-foreground-muted">No transcription model matches.</Text>
133 + ) : matches.length > shown.length ? (
134 + <ListActionRow
135 + label={`Show all ${matches.length} models`}
136 + onPress={() => setShowAll(true)}
137 + />
138 + ) : null}
139 + </>
140 + );
141 +}
142 +
143 +function CloudChoices({ environment, active, onClose }: Props & { onClose: () => void }) {
144 + const id = environment.environmentId;
145 + const connection = useAtomValue(cloudVoiceConnectionAtom(id));
146 + const keys = useAtomValue(cloudVoiceKeysAtom(id));
147 + const reload = useAtomRefresh(cloudVoiceKeysAtom(id));
148 + const canManage = useEnvironmentScope(id, AuthSettingsWriteScope);
149 + const [editing, setEditing] = useState<{
150 + provider: CloudSpeechProvider;
151 + enable: boolean;
152 + } | null>(null);
153 + const catalogAtom = cloudVoiceModelsAtom(editing?.provider === "openrouter" ? id : null);
154 + const models = useAtomValue(catalogAtom);
155 + const reloadModels = useAtomRefresh(catalogAtom);
156 + const [draft, setDraft] = useState("");
157 + const [model, setModel] = useState("");
158 + const [sourceId, setSourceId] = useState<SecretProviderId | null>(null);
159 + const sources =
160 + Constants.expoConfig?.extra?.secretsFeatureEnabled === true &&
161 + environment.serverConfig?.secretReferences
162 + ? configuredSecretSources(environment.serverConfig.settings.secretProviders)
163 + : [];
164 + const source = sources.find((candidate) => candidate.id === sourceId);
165 + const credential =
166 + sourceId !== null
167 + ? source && secretRequestReference(source, draft)
168 + : isApiKey(draft.trim())
169 + ? draft.trim()
170 + : undefined;
171 + const [saving, setSaving] = useState(false);
172 + const activeModel = active?.provider === "openrouter" ? active.model : undefined;
173 + const enable = (commit: () => void) => {
174 + try {
175 + commit();
176 + } catch {
177 + Dialog.alert(
178 + "Could not enable cloud dictation",
179 + "Check device storage and choose the provider again.",
180 + );
181 + }
182 + };
183 + const edit = (provider: CloudSpeechProvider, enable: boolean) => {
184 + const stored = environment.serverConfig?.settings.cloudSpeechCredentials?.[provider];
185 + const ref =
186 + isSecretReference(stored) && sources.some((candidate) => candidate.id === stored.provider)
187 + ? stored
188 + : null;
189 + setSourceId(ref?.provider ?? null);
190 + setDraft(ref?.reference ?? "");
191 + setModel(provider === "openrouter" ? (activeModel ?? "") : "");
192 + setEditing({ provider, enable });
193 + };
194 + const reset = () => {
195 + setDraft("");
196 + setModel("");
197 + setEditing(null);
198 + };
199 + const changeKey = async (
200 + provider: CloudSpeechProvider,
201 + value: SettingsSecretValue | null,
202 + chosen?: CloudSpeechModelId,
203 + ) => {
204 + if (!connection || !canManage || saving) return;
205 + const commit =
206 + value !== null && (editing?.enable || (chosen !== undefined && chosen !== activeModel))
207 + ? pendingCloudVoiceChoice(id, provider, chosen)
208 + : null;
209 + setSaving(true);
210 + try {
211 + await runSpeech(setCloudSpeechKey(connection, provider, value));
212 + } catch {
213 + setSaving(false);
214 + Dialog.alert(
215 + "Could not update the API key",
216 + "Check the environment connection and your settings permissions.",
217 + );
218 + return;
219 + }
220 + setSaving(false);
221 + if (commit) enable(commit);
222 + reload();
223 + onClose();
224 + };
225 + const catalog = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null;
226 + const content =
227 + !AsyncResult.isSuccess(keys) || !keys.value || !connection ? (
228 + <ListActionRow
229 + label={
230 + !connection
231 + ? "Environment not connected"
232 + : keys._tag === "Failure"
233 + ? "Could not load keys · Retry"
234 + : "Checking API keys…"
235 + }
236 + disabled={!connection || keys._tag !== "Failure"}
237 + onPress={reload}
238 + />
239 + ) : (
240 + CloudSpeechProvider.literals.map((provider) => {
241 + if (editing && provider !== editing.provider) return null;
242 + const label = CLOUD_SPEECH_LABELS[provider];
243 + const company = CLOUD_SPEECH_PROVIDER_NAMES[provider];
244 + const routed = provider === "openrouter";
245 + const disclosure = cloudSpeechDisclosure(provider, environment.label);
246 + const saved = keys.value![provider];
247 + const isActive = active?.provider === provider;
248 + const isEditing = editing?.provider === provider;
249 + const modelChosen =
250 + isModelId(model) && (catalog?.some((entry) => entry.id === model) ?? false);
251 + const modelChanged = model !== (activeModel ?? "");
252 + const original = environment.serverConfig?.settings.cloudSpeechCredentials?.[provider];
253 + const keyChanged =
254 + credential !== undefined &&
255 + !(
256 + isSecretReference(credential) &&
257 + isSecretReference(original) &&
258 + original.provider === credential.provider &&
259 + original.reference === credential.reference
260 + );
261 + const routedReady =
262 + modelChosen &&
263 + (draft.trim() ? credential !== undefined : Boolean(saved)) &&
264 + (!isActive || modelChanged || keyChanged);
265 + const saveRouted = () => {
266 + if (!routedReady || !isModelId(model)) return;
267 + const grantChanges = !isActive || modelChanged;
268 + if (credential !== undefined && keyChanged)
269 + return void changeKey(provider, credential, model);
270 + // Model-only changes never write to the environment.
271 + if (grantChanges) enable(() => chooseVoiceDestination(id, provider, model));
272 + onClose();
273 + };
274 + return (
275 + <View key={provider}>
276 + {!isEditing && (
277 + <ListChoiceRow
278 + label={label}
279 + leading={
280 + <ProviderIcon
281 + provider={
282 + provider === "openai" ? "codex" : provider === "xai" ? "grok" : provider
283 + }
284 + size={20}
285 + />
286 + }
287 + selected={isActive}
288 + disabled={saved === undefined || (!saved && !canManage) || saving}
289 + description={
290 + saved === undefined
291 + ? "Update the environment to use this provider"
292 + : isActive && routed
293 + ? `${activeModel} · Tap to change model${canManage ? " or key" : ""}`
294 + : isActive && saved && canManage
295 + ? "Tap to manage key"
296 + : saved
297 + ? routed
298 + ? "API key saved · choose a model to use it"
299 + : "API key saved on this environment"
300 + : canManage
301 + ? "Add an API key to use this provider"
302 + : "Ask an environment administrator to add an API key"
303 + }
304 + onPress={() => {
305 + if (routed) return edit(provider, !isActive);
306 + if (!saved) return edit(provider, true);
307 + if (isActive) {
308 + if (canManage) edit(provider, false);
309 + return;
310 + }
311 + Dialog.confirm({
312 + title: `Use ${label} for dictation?`,
313 + message: `${disclosure} Select This device to stop sharing.`,
314 + confirmText: `Use ${label}`,
315 + onConfirm: () => enable(pendingCloudVoiceChoice(id, provider)),
316 + });
317 + }}
318 + />
319 + )}
320 + {isEditing && routed && (
321 + <>
322 + <Text className="mx-4 text-sm text-foreground-muted">{disclosure}</Text>
323 + {catalog ? (
324 + <ModelPicker
325 + models={catalog}
326 + value={model}
327 + current={activeModel}
328 + disabled={saving}
329 + onChange={setModel}
330 + />
331 + ) : (
332 + <ListActionRow
333 + label={
334 + AsyncResult.isFailure(models)
335 + ? "Could not load models · Retry"
336 + : "Loading transcription models…"
337 + }
338 + disabled={!AsyncResult.isFailure(models)}
339 + onPress={reloadModels}
340 + />
341 + )}
342 + </>
343 + )}
344 + {isEditing && (canManage || routed) && (
345 + <>
346 + {canManage && sources.length > 0 && (
347 + <View className="mx-4 flex-row flex-wrap gap-2">
348 + <Chip
349 + label="Value"
350 + size="small"
351 + selected={sourceId === null}
352 + disabled={saving}
353 + onPress={() => {
354 + setDraft("");
355 + setSourceId(null);
356 + }}
357 + />
358 + {sources.map((candidate) => (
359 + <Chip
360 + key={candidate.id}
361 + label={candidate.label}
362 + size="small"
363 + selected={sourceId === candidate.id}
364 + disabled={saving}
365 + onPress={() => {
366 + setDraft("");
367 + setSourceId(candidate.id);
368 + }}
369 + />
370 + ))}
371 + </View>
372 + )}
373 + {canManage && (
374 + <TextInput
375 + key={sourceId ?? "literal"}
376 + accessibilityLabel={
377 + sourceId ? `${label} API key reference` : `${label} API key`
378 + }
379 + placeholder={
380 + source?.referencePlaceholder ??
381 + (routed && saved ? "Saved key · enter a new one to replace" : "API key")
382 + }
383 + secureTextEntry={sourceId === null}
384 + autoCapitalize="none"
385 + autoCorrect={false}
386 + autoComplete="off"
387 + importantForAutofill="no"
388 + maxLength={4096}
389 + editable={!saving}
390 + returnKeyType="done"
391 + onSubmitEditing={() => {
392 + if (routed) saveRouted();
393 + else if (credential) void changeKey(provider, credential);
394 + }}
395 + value={draft}
396 + onChangeText={setDraft}
397 + className="mx-4 rounded-lg bg-background px-3 py-2 text-foreground"
398 + />
399 + )}
400 + {routed && !saved && !canManage ? (
401 + <ListActionRow
402 + label="Ask an administrator to add an OpenRouter key"
403 + disabled
404 + onPress={() => {}}
405 + />
406 + ) : (
407 + <ListActionRow
408 + label={
409 + routed
410 + ? isActive
411 + ? "Save"
412 + : "Save and enable"
413 + : editing.enable
414 + ? "Save and enable"
415 + : "Save key"
416 + }
417 + description={
418 + draft && !credential
419 + ? (source?.invalidReferenceMessage ?? "Enter a valid API key.")
420 + : routed
421 + ? saved && !credential
422 + ? "Uses the API key saved on this environment. The model choice applies only to this device."
423 + : `Anyone with dictation access to ${environment.label} can bill this key.`
424 + : editing.enable
425 + ? `${disclosure} Anyone with dictation access to ${environment.label} can bill this key.`
426 + : `Updates the ${company} key on ${environment.label}. Your dictation destination stays the same.`
427 + }
428 + disabled={(routed ? !routedReady : !credential) || saving}
429 + onPress={() => {
430 + if (routed) saveRouted();
431 + else if (credential) void changeKey(provider, credential);
432 + }}
433 + />
434 + )}
435 + {saved && canManage && (
436 + <ListActionRow
437 + tone="danger"
438 + label="Remove key from environment"
439 + disabled={saving}
440 + onPress={() =>
441 + Dialog.confirm({
442 + title: `Remove the ${label} key from ${environment.label}?`,
443 + message: "Everyone using it for dictation will lose access.",
444 + confirmText: "Remove key",
445 + destructive: true,
446 + onConfirm: () => void changeKey(provider, null),
447 + })
448 + }
449 + />
450 + )}
451 + <ListActionRow label="Back to providers" disabled={saving} onPress={reset} />
452 + </>
453 + )}
454 + </View>
455 + );
456 + })
457 + );
458 + return (
459 + <Sheet
460 + title={editing ? CLOUD_SPEECH_LABELS[editing.provider] : "Cloud providers"}
461 + onClose={onClose}
462 + dismissDisabled={saving}
463 + >
464 + <ScrollView
465 + contentInsetAdjustmentBehavior="automatic"
466 + keyboardShouldPersistTaps="handled"
467 + contentContainerClassName="gap-2 pb-8"
468 + >
469 + {content}
470 + </ScrollView>
471 + </Sheet>
472 + );
473 +}
apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx
+26 -2
@@ -30,6 +30,7 @@ import {
30 hasEnvironmentVoiceCapture,
31 } from "../voice-input/voiceDestination";
32 import { VoiceModelSection } from "./VoiceModelSection";
33 +import { CloudVoiceSection } from "./CloudVoiceSection";
34 import type { EnvironmentId } from "@t3tools/contracts";
35 import { ThreadSwipeable } from "../home/thread-swipe-actions";
36 import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow";
@@ -241,7 +242,10 @@ export function SettingsVoiceRouteScreen() {
242 <EnvironmentChoice
243 key={environment.environmentId}
244 environment={environment}
244 - selected={grant?.destination.environmentId === environment.environmentId}
245 + selected={
246 + grant?.destination.kind === "environment" &&
247 + grant.destination.environmentId === environment.environmentId
248 + }
249 />
250 ))}
251 {isReady && grant && !grantEnvironment ? (
@@ -255,7 +259,27 @@ export function SettingsVoiceRouteScreen() {
259 />
260 ) : null}
261 </SettingsSection>
258 - {grantEnvironment ? (
262 + {hasEnvironmentVoiceCapture &&
263 + environments
264 + .filter(
265 + (env) =>
266 + env.serverConfig?.environment.capabilities.cloudVoiceTranscription ||
267 + (grant?.destination.kind === "cloud" &&
268 + grant.destination.environmentId === env.environmentId),
269 + )
270 + .map((environment) => (
271 + <CloudVoiceSection
272 + key={environment.environmentId}
273 + environment={environment}
274 + active={
275 + grant?.destination.kind === "cloud" &&
276 + grant.destination.environmentId === environment.environmentId
277 + ? grant.destination
278 + : null
279 + }
280 + />
281 + ))}
282 + {grantEnvironment && grant?.destination.kind === "environment" ? (
283 <VoiceModelSection
284 environmentId={grantEnvironment.environmentId}
285 label={grantEnvironment.label}
apps/mobile/src/features/voice-input/voiceDestination.test.ts
+87 -2
@@ -2,7 +2,12 @@ import { EnvironmentId } from "@t3tools/contracts";
2 import { Atom } from "effect/reactivity";
3 import { expect, it, vi } from "vite-plus/test";
4 import { appAtomRegistry } from "../../state/atom-registry";
5 -import { chooseVoiceDestination, voiceGrantAtom, voiceRouteAtom } from "./voiceDestination";
5 +import {
6 + chooseVoiceDestination,
7 + pendingCloudVoiceChoice,
8 + voiceGrantAtom,
9 + voiceRouteAtom,
10 +} from "./voiceDestination";
11 import { readVoiceGrant } from "./voiceGrantStorage";
12
13 const mocks = vi.hoisted(() => ({
@@ -46,7 +51,11 @@ vi.mock("../../state/presentation", () => ({
51 mocks.probe(id);
52 return Atom.make({
53 connection: { phase: id === "offline" ? "offline" : "connected" },
49 - serverConfig: { environment: { capabilities: { voiceTranscription: true } } },
54 + serverConfig: {
55 + environment: {
56 + capabilities: { voiceTranscription: true, cloudVoiceTranscription: true },
57 + },
58 + },
59 });
60 }),
61 },
@@ -159,6 +168,58 @@ it("keeps local as the default and never reuses consent for an offline or resele
168 ["https://my-server.test/api/voice/status", "error"],
169 ["https://my-server.test/api/voice/transcribe", "error"],
170 ]);
171 + chooseVoiceDestination(host, "openai");
172 + const cloud = appAtomRegistry.get(voiceRouteAtom)!.transcriber!;
173 + mocks.fetch.mockResolvedValueOnce(Response.json({ openai: false, xai: true }));
174 + await expect(cloud.prepare(options)).rejects.toMatchObject({
175 + code: "unavailable",
176 + message: expect.stringContaining("Add your OpenAI API key"),
177 + });
178 + mocks.fetch.mockResolvedValueOnce(Response.json({ openai: true, xai: true }));
179 + const recorded = await cloud.prepare(options);
180 + chooseVoiceDestination(host, "xai");
181 + await expect(recorded.transcribe("file:///voice.m4a", options)).rejects.toThrow(
182 + "Your transcription choice changed.",
183 + );
184 + mocks.fetch
185 + .mockResolvedValueOnce(Response.json({ openai: true, xai: true }))
186 + .mockResolvedValueOnce(Response.json({ text: "cloud" }));
187 + const next = await appAtomRegistry.get(voiceRouteAtom)!.transcriber!.prepare(options);
188 + await expect(next.transcribe("file:///voice.m4a", options)).resolves.toBe("cloud");
189 + mocks.fetch.mockResolvedValueOnce(
190 + Response.json(
191 + {
192 + _tag: "CloudSpeechError",
193 + provider: "xai",
194 + reason: "rejected-key",
195 + },
196 + { status: 422 },
197 + ),
198 + );
199 + await expect(next.transcribe("file:///voice.m4a", options)).rejects.toMatchObject({
200 + _tag: "VoiceTranscriptionError",
201 + code: "transcription-failed",
202 + message: expect.stringContaining("rejected this API key"),
203 + });
204 + expect(mocks.fetch.mock.calls.slice(2).map(([url]) => new URL(String(url)).pathname)).toEqual([
205 + "/api/voice/cloud/keys",
206 + "/api/voice/cloud/keys",
207 + "/api/voice/cloud/keys",
208 + "/api/voice/cloud/xai/transcribe",
209 + "/api/voice/cloud/xai/transcribe",
210 + ]);
211 + // OpenRouter without a model is no destination; with one, the exact id travels with the audio.
212 + chooseVoiceDestination(host, "openrouter");
213 + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
214 + chooseVoiceDestination(host, "openrouter", "acme/transcribe-1");
215 + mocks.fetch
216 + .mockResolvedValueOnce(Response.json({ openai: true, xai: true, openrouter: true }))
217 + .mockResolvedValueOnce(Response.json({ text: "routed" }));
218 + const routed = await appAtomRegistry.get(voiceRouteAtom)!.transcriber!.prepare(options);
219 + await expect(routed.transcribe("file:///voice.m4a", options)).resolves.toBe("routed");
220 + const upload = new URL(String(mocks.fetch.mock.lastCall![0]));
221 + expect(upload.pathname).toBe("/api/voice/cloud/openrouter/transcribe");
222 + expect(upload.searchParams.get("model")).toBe("acme/transcribe-1");
223 const failWrite = () => {
224 throw new Error("Device storage is unavailable");
225 };
@@ -172,3 +233,27 @@ it("keeps local as the default and never reuses consent for an offline or resele
233 appAtomRegistry.reset();
234 }
235 });
236 +
237 +it("a late cloud key save cannot override a newer destination choice", () => {
238 + const host = EnvironmentId.make("my-server");
239 + try {
240 + chooseVoiceDestination(null);
241 + const first = pendingCloudVoiceChoice(host, "openai");
242 + chooseVoiceDestination(null);
243 + first();
244 + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
245 + expect(JSON.parse(mocks.write.mock.lastCall![1])).toBeNull();
246 +
247 + const older = pendingCloudVoiceChoice(host, "openai");
248 + const newer = pendingCloudVoiceChoice(host, "xai");
249 + older();
250 + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
251 + newer();
252 + const grant = appAtomRegistry.get(voiceGrantAtom);
253 + older();
254 + expect(appAtomRegistry.get(voiceGrantAtom)).toBe(grant);
255 + expect(grant?.destination).toEqual({ kind: "cloud", environmentId: host, provider: "xai" });
256 + } finally {
257 + appAtomRegistry.reset();
258 + }
259 +});
apps/mobile/src/features/voice-input/voiceDestination.ts
+103 -23
@@ -1,10 +1,17 @@
1 import {
2 AuthOrchestrationOperateScope,
3 + type CloudSpeechModelId,
4 + type CloudSpeechProvider,
5 type EnvironmentId,
6 DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
7 + type VoiceTranscriptionDestination,
8 } from "@t3tools/contracts";
9 import {
10 getEnvironmentSpeechModels,
11 + getCloudSpeechKeys,
12 + getCloudSpeechModels,
13 + requireCloudSpeechKey,
14 + transcribeCloudPcm,
15 getEnvironmentSpeechStatus,
16 transcribeEnvironmentPcm,
17 VoiceTranscriptionError,
@@ -34,38 +41,74 @@ export const voiceGrantAtom = Atom.make((get) => {
41 return AsyncResult.getOrElse(get(storedGrant), () => null);
42 });
43
37 -export function chooseVoiceDestination(environmentId: EnvironmentId | null) {
44 +let choiceRevision = 0;
45 +
46 +type CloudDestination = Extract<VoiceTranscriptionDestination, { kind: "cloud" }>;
47 +/** OpenRouter has no default model, so a cloud choice without one is no choice at all. */
48 +function cloudVoiceDestination(
49 + environmentId: EnvironmentId,
50 + provider: CloudSpeechProvider,
51 + model?: CloudSpeechModelId,
52 +): CloudDestination | null {
53 + if (provider === "openrouter")
54 + return model ? { kind: "cloud", environmentId, provider, model } : null;
55 + return { kind: "cloud", environmentId, provider };
56 +}
57 +
58 +export function chooseVoiceDestination(
59 + environmentId: EnvironmentId | null,
60 + provider?: CloudSpeechProvider,
61 + model?: CloudSpeechModelId,
62 +) {
63 + choiceRevision++;
64 // A failed change must not leave the old destination armed in this session.
65 appAtomRegistry.set(choice, null);
40 - const grant: VoiceGrant | null =
66 + const destination: VoiceTranscriptionDestination | null =
67 environmentId === null
68 ? null
43 - : {
44 - id: uuidv4(),
45 - destination: { kind: "environment", environmentId },
46 - };
69 + : provider
70 + ? cloudVoiceDestination(environmentId, provider, model)
71 + : { kind: "environment", environmentId };
72 + const grant: VoiceGrant | null = destination && { id: uuidv4(), destination };
73 writeVoiceGrant(grant);
74 appAtomRegistry.set(choice, grant);
75 }
76
77 +export function pendingCloudVoiceChoice(
78 + id: EnvironmentId,
79 + provider: CloudSpeechProvider,
80 + model?: CloudSpeechModelId,
81 +) {
82 + const revision = ++choiceRevision;
83 + return () => {
84 + if (revision === choiceRevision) chooseVoiceDestination(id, provider, model);
85 + };
86 +}
87 +
88 const native = requireOptionalNativeModule<{
89 decodeRecording?: (uri: string) => Promise<Uint8Array>;
90 }>("T3VoiceTranscription");
91 export const hasEnvironmentVoiceCapture = typeof native?.decodeRecording === "function";
92
56 -export const voiceConnectionAtom = Atom.family((id: EnvironmentId) =>
57 - Atom.make((get) => {
58 - const environment = get(environmentPresentations.presentationAtom(id));
59 - if (
60 - !hasEnvironmentVoiceCapture ||
61 - environment?.connection.phase !== "connected" ||
62 - !environment.serverConfig?.environment.capabilities.voiceTranscription ||
63 - !sessionHasScope(get(environmentSession.sessionStateAtom(id)), AuthOrchestrationOperateScope)
64 - )
65 - return null;
66 - return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id)));
67 - }),
68 -);
93 +const voiceConnections = (capability: "voiceTranscription" | "cloudVoiceTranscription") =>
94 + Atom.family((id: EnvironmentId) =>
95 + Atom.make((get) => {
96 + const environment = get(environmentPresentations.presentationAtom(id));
97 + if (
98 + !hasEnvironmentVoiceCapture ||
99 + environment?.connection.phase !== "connected" ||
100 + !environment.serverConfig?.environment.capabilities[capability] ||
101 + !sessionHasScope(
102 + get(environmentSession.sessionStateAtom(id)),
103 + AuthOrchestrationOperateScope,
104 + )
105 + )
106 + return null;
107 + return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id)));
108 + }),
109 + );
110 +export const voiceConnectionAtom = voiceConnections("voiceTranscription");
111 +export const cloudVoiceConnectionAtom = voiceConnections("cloudVoiceTranscription");
112
113 // React Native's XMLHttpRequest fetch follows redirects. Expo's native transport honors redirect: "error".
114 export function runSpeech<A, E>(
@@ -92,10 +135,20 @@ export function runSpeech<A, E>(
135 { signal },
136 );
137 }
95 -function environmentTranscriber(prepared: PreparedConnection, canSend: () => boolean) {
138 +function environmentTranscriber(
139 + prepared: PreparedConnection,
140 + canSend: () => boolean,
141 + cloud?: CloudDestination,
142 +) {
143 + const provider = cloud?.provider;
144 + const model = cloud?.provider === "openrouter" ? cloud.model : undefined;
145 return createEnvironmentTranscriber({
146 canSend,
147 prepare: async (signal) => {
148 + if (provider) {
149 + await runSpeech(requireCloudSpeechKey(prepared, provider), signal, canSend);
150 + return "auto";
151 + }
152 const status = await runSpeech(getEnvironmentSpeechStatus(prepared), signal, canSend);
153 if (!status.supported)
154 throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
@@ -126,7 +179,21 @@ function environmentTranscriber(prepared: PreparedConnection, canSend: () => boo
179 upload: async (pcm, signal) =>
180 (
181 await runSpeech(
129 - transcribeEnvironmentPcm(prepared, pcm, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, canSend),
182 + provider
183 + ? transcribeCloudPcm(
184 + prepared,
185 + provider,
186 + pcm,
187 + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
188 + canSend,
189 + model,
190 + )
191 + : transcribeEnvironmentPcm(
192 + prepared,
193 + pcm,
194 + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
195 + canSend,
196 + ),
197 signal,
198 canSend,
199 )
@@ -137,9 +204,10 @@ function environmentTranscriber(prepared: PreparedConnection, canSend: () => boo
204 const speechAtom = <A, E>(
205 id: EnvironmentId,
206 read: (prepared: PreparedConnection) => Effect.Effect<A, E, HttpClient.HttpClient>,
207 + connectionAtom = voiceConnectionAtom,
208 ) =>
209 Atom.make((get) => {
142 - const prepared = get(voiceConnectionAtom(id));
210 + const prepared = get(connectionAtom(id));
211 return prepared
212 ? Effect.tryPromise({
213 try: (signal) => runSpeech(read(prepared), signal),
@@ -155,6 +223,14 @@ const speechAtom = <A, E>(
223 export const voiceSpeechStatusAtom = Atom.family((id: EnvironmentId) =>
224 speechAtom(id, getEnvironmentSpeechStatus),
225 );
226 +export const cloudVoiceKeysAtom = Atom.family((id: EnvironmentId) =>
227 + speechAtom(id, getCloudSpeechKeys, cloudVoiceConnectionAtom),
228 +);
229 +export const cloudVoiceModelsAtom = Atom.family((id: EnvironmentId | null) =>
230 + id === null
231 + ? Atom.make(Effect.succeed(null))
232 + : speechAtom(id, getCloudSpeechModels, cloudVoiceConnectionAtom),
233 +);
234 export const voiceSpeechModelsAtom = Atom.family((id: EnvironmentId) =>
235 speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared)),
236 );
@@ -167,7 +243,10 @@ export const voiceSpeechBenchmarksAtom = Atom.family((id: EnvironmentId) =>
243 export const voiceRouteAtom = Atom.make((get) => {
244 const grant = get(voiceGrantAtom);
245 if (!grant) return null;
170 - const connectionAtom = voiceConnectionAtom(grant.destination.environmentId);
246 + const cloud = grant.destination.kind === "cloud" ? grant.destination : undefined;
247 + const connectionAtom = (cloud ? cloudVoiceConnectionAtom : voiceConnectionAtom)(
248 + grant.destination.environmentId,
249 + );
250 const connection = get(connectionAtom);
251 const registry = get.registry;
252 return {
@@ -176,6 +255,7 @@ export const voiceRouteAtom = Atom.make((get) => {
255 environmentTranscriber(
256 connection,
257 () => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection,
258 + cloud,
259 ),
260 };
261 });
apps/server/src/environment/ServerEnvironment.ts
+1
@@ -274,6 +274,7 @@ export const make = Effect.gen(function* () {
274 scheduledTaskThreadModes: true,
275 npmReleaseLookup: true,
276 voiceTranscription: isSpeechPlatformSupported(hostPlatform, hostArchitecture, hostLibc),
277 + cloudVoiceTranscription: true,
278 ...(serverSelfUpdate === null ? {} : { serverSelfUpdate }),
279 ...(serverInstallation === null ? {} : { serverInstallation }),
280 ...(serverSelfUpdate === "boot-service" || desktopAppUpdate
apps/server/src/server.ts
+2
@@ -43,6 +43,7 @@ import * as NodePtyAdapter from "./terminal/NodePtyAdapter.ts";
43 import * as PullRequestHttp from "./pullRequest/http.ts";
44 import * as SpeechModelStore from "./speech/SpeechModelStore.ts";
45 import * as SpeechService from "./speech/SpeechService.ts";
46 +import * as CloudSpeech from "./speech/CloudSpeech.ts";
47 import { speechHttpApiLayer } from "./speech/http.ts";
48 import { speechStreamRouteLayer } from "./speech/stream.ts";
49 import * as PullRequestProviderRegistry from "./pullRequest/PullRequestProviderRegistry.ts";
@@ -823,6 +824,7 @@ const layerMakeRoutes = Layer.mergeAll(
824 Layer.provide(SpeechModelStore.layer),
825 ),
826 ),
827 + Layer.provide(CloudSpeech.layer.pipe(Layer.provide(SecretResolver.layer))),
828 Layer.provide(layerServerSettings),
829 Layer.provide(PreviewAutomationBroker.layer),
830 Layer.provide(ServerSelfUpdate.layer.pipe(Layer.provide(layerDesktopAppUpdate))),
apps/server/src/serverSettings.test.ts
+17
@@ -1642,6 +1642,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
1642 const openCode = ProviderInstanceId.make("opencode");
1643 const hub = UsageLimitSourceId.make("hub");
1644 const withUsage = yield* serverSettings.updateSettings({
1645 + cloudSpeechCredentials: { openai: usageReference, xai: "literal-fixture" },
1646 usageLimitSources: {
1647 [hub]: {
1648 kind: "cliproxy",
@@ -1687,6 +1688,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
1688 secretReferences: false,
1689 });
1690 assert.notInclude(JSON.stringify(legacy), "op://");
1691 + assert.deepEqual(legacy.cloudSpeechCredentials, {
1692 + openai: SECRET_REDACTED,
1693 + xai: SECRET_REDACTED,
1694 + openrouter: "",
1695 + });
1696 assert.equal(legacy.sourceControlHosts.bitbucket?.accessToken, SECRET_REDACTED);
1697 assert.equal(legacy.usageLimitSources[hub]?.managementKey, SECRET_REDACTED);
1698 assert.equal(legacy.easDeviceHosts[0]?.accessToken, SECRET_REDACTED);
@@ -1696,6 +1702,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
1702 assert.equal(configOf(legacy, antigravity)?.apiKey, SECRET_REDACTED);
1703 assert.equal(configOf(legacy, openCode)?.serverPassword, SECRET_REDACTED);
1704 const echoed = yield* serverSettings.updateSettings({
1705 + cloudSpeechCredentials: legacy.cloudSpeechCredentials,
1706 sourceControlHosts: legacy.sourceControlHosts,
1707 usageLimitSources: legacy.usageLimitSources,
1708 easDeviceHosts: legacy.easDeviceHosts,
@@ -1708,6 +1715,11 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
1715 "ghe.acme.test": "ghp_ghe",
1716 });
1717 assert.deepEqual(echoed.usageLimitSources[hub]?.managementKey, usageReference);
1718 + assert.deepEqual(echoed.cloudSpeechCredentials, {
1719 + openai: usageReference,
1720 + xai: "literal-fixture",
1721 + openrouter: "",
1722 + });
1723 assert.deepEqual(echoed.easDeviceHosts[0]?.accessToken, easReference);
1724 assert.deepEqual(echoed.simlockDeviceHosts[0]?.agentToken, simlockReference);
1725 assert.deepEqual(configOf(echoed, antigravity)?.apiKey, antigravityReference);
@@ -1747,6 +1759,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
1759 bitbucket: { accessToken: personal("Bitbucket/token") },
1760 github: { tokens: { "github.com": personal("GitHub/token"), "ghe.acme.test": work } },
1761 },
1762 + cloudSpeechCredentials: { openai: personal("Voice/key"), xai: work },
1763 usageLimitSources: {
1764 [hub]: {
1765 kind: "cliproxy",
@@ -1791,6 +1804,7 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
1804 "ghe.acme.test": work,
1805 });
1806 assert.equal(removed.usageLimitSources[hub]?.managementKey, "");
1807 + assert.deepEqual(removed.cloudSpeechCredentials, { openai: "", xai: work, openrouter: "" });
1808 assert.equal(removed.easDeviceHosts[0]?.accessToken, "");
1809 assert.deepEqual(Object.keys(removed.secretProviders), ["work"]);
1810 }).pipe(Effect.provide(layerServerSettingsWithSecrets())),
@@ -1800,6 +1814,9 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings"
1814 Effect.sync(() => {
1815 // Shipped clients decode these as strings. Each must be covered by the older-client case above.
1816 assert.deepEqual(strictSecretReferencePaths(ServerSettings.ast), [
1817 + "cloudSpeechCredentials.openai",
1818 + "cloudSpeechCredentials.xai",
1819 + "cloudSpeechCredentials.openrouter",
1820 "easDeviceHosts.[].accessToken",
1821 "simlockDeviceHosts.[].agentToken",
1822 "usageLimitSources.*.managementKey",
apps/server/src/settingsSecrets.ts
+7
@@ -1,6 +1,7 @@
1 import {
2 AntigravitySettings,
3 ClaudeSettings,
4 + CloudSpeechCredentials,
5 CodexSettings,
6 EasDeviceHostConfig,
7 type ProviderInstanceConfig,
@@ -45,6 +46,7 @@ const providerSecretFields = new Map([
46 ),
47 ]);
48 const usageLimitSecretFields = secretSettingsFields(UsageLimitSourceConfig);
49 +const voiceSecretFields = secretSettingsFields(CloudSpeechCredentials);
50 const easDeviceHostSecretFields = secretSettingsFields(EasDeviceHostConfig);
51 const simlockDeviceHostSecretFields = secretSettingsFields(SimlockDeviceHostConfig);
52 const encode = (value: string) => Buffer.from(value, "utf8").toString("base64url");
@@ -232,6 +234,11 @@ function mapSecretFieldGroups(
234
235 return {
236 ...settings,
237 + cloudSpeechCredentials: mapFields(
238 + settings.cloudSpeechCredentials,
239 + voiceSecretFields,
240 + (provider) => `voice-cloud-${provider}`,
241 + ) as ServerSettings["cloudSpeechCredentials"],
242 providerInstances,
243 sourceControlHosts,
244 usageLimitSources,
apps/server/src/sourceControl/sourceControlHostSecrets.test.ts
+3
@@ -164,6 +164,7 @@ it.effect("reads a collected reference again once it is invalidated", () =>
164 },
165 ];
166 cache.set("cliproxy#key", "stale");
167 + cache.set("voice#key", "stale");
168 cache.set("expo#token", "stale");
169 cache.set("simlock#token", "stale");
170 assert.deepEqual((yield* read()).bitbucket, {
@@ -175,12 +176,14 @@ it.effect("reads a collected reference again once it is invalidated", () =>
176 resolver,
177 Effect.succeed({
178 sourceControlHosts: hosts,
179 + cloudSpeechCredentials: { openai: source("voice#key"), xai: "", openrouter: "" },
180 usageLimitSources,
181 easDeviceHosts,
182 simlockDeviceHosts,
183 }),
184 );
185 assert.isFalse(cache.has("cliproxy#key"));
186 + assert.isFalse(cache.has("voice#key"));
187 assert.isFalse(cache.has("expo#token"));
188 assert.isFalse(cache.has("simlock#token"));
189 assert.deepEqual((yield* read()).bitbucket, {
apps/server/src/sourceControl/sourceControlHostSecrets.ts
+6 -1
@@ -118,7 +118,11 @@ export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate")
118 settings: Effect.Effect<
119 Pick<
120 ServerSettings,
121 - "sourceControlHosts" | "usageLimitSources" | "easDeviceHosts" | "simlockDeviceHosts"
121 + | "sourceControlHosts"
122 + | "usageLimitSources"
123 + | "easDeviceHosts"
124 + | "simlockDeviceHosts"
125 + | "cloudSpeechCredentials"
126 >,
127 E
128 >,
@@ -128,6 +132,7 @@ export const invalidateSettingsSecrets = Effect.fn("SettingsSecrets.invalidate")
132 ...collectSourceControlHostSecretReferences(current.sourceControlHosts),
133 ...collectUsageLimitSecretReferences(current.usageLimitSources),
134 ...collectDeviceHostSecretReferences(current),
135 + ...Object.values(current.cloudSpeechCredentials).filter(isSecretReference),
136 ]),
137 Effect.orElseSucceed(() => undefined),
138 );
apps/server/src/speech/CloudSpeech.test.ts new
+200
@@ -0,0 +1,200 @@
1 +import { it } from "@effect/vitest";
2 +import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, SecretProviderId } from "@t3tools/contracts";
3 +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
4 +import * as Effect from "effect/Effect";
5 +import * as Fiber from "effect/Fiber";
6 +import * as Layer from "effect/Layer";
7 +import * as Redacted from "effect/Redacted";
8 +import * as FetchHttpClient from "effect/http/FetchHttpClient";
9 +import { expect, vi } from "vite-plus/test";
10 +import * as ServerSettings from "../serverSettings.ts";
11 +import * as CloudSpeech from "./CloudSpeech.ts";
12 +
13 +it.effect.each(["openai", "xai"] as const)("sends only to %s using its own saved key", (provider) =>
14 + Effect.gen(function* () {
15 + const reference = {
16 + provider: SecretProviderId.make("work"),
17 + reference: "op://Work/Voice/key",
18 + };
19 + let resolved: string | undefined;
20 + let cached: string | undefined;
21 + const read = vi.fn(() => Effect.sync(() => [(cached ??= resolved)]));
22 + const upstream = vi.fn<typeof fetch>(async () => Response.json({ text: "hello" }));
23 + const service = yield* CloudSpeech.CloudSpeech.pipe(
24 + Effect.provide(
25 + CloudSpeech.layer.pipe(
26 + Layer.provide(FetchHttpClient.layer),
27 + Layer.provide(ServerSettings.ServerSettingsService.layerTest()),
28 + Layer.provide(
29 + Layer.mock(SecretResolver.SecretResolver)({
30 + read,
31 + label: () => Effect.succeed("Work"),
32 + invalidate: () =>
33 + Effect.sync(() => {
34 + cached = undefined;
35 + }),
36 + }),
37 + ),
38 + ),
39 + ),
40 + );
41 + const transcribe = (pcm = new Uint8Array(new Float32Array([0, 0.5]).buffer)) =>
42 + service
43 + .transcribe(provider, pcm, {
44 + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
45 + speechLanguage: "en",
46 + speechCustomWords: [{ term: "T3 Code", aliases: [] }],
47 + })
48 + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream));
49 + yield* service.setKey(provider === "openai" ? "xai" : "openai", Redacted.make("other"));
50 + expect((yield* Effect.flip(transcribe()))._tag).toBe("CloudSpeechError");
51 + yield* service.setKey(provider, Redacted.make("chosen"));
52 + expect(yield* service.configured).toEqual({ openai: true, xai: true, openrouter: false });
53 + yield* Effect.flip(transcribe(new Uint8Array(3)));
54 + expect(upstream).not.toHaveBeenCalled();
55 + expect(yield* transcribe()).toBe("hello");
56 + const [url, init] = upstream.mock.calls[0]!;
57 + expect(String(url)).toBe(
58 + {
59 + openai: "https://api.openai.com/v1/audio/transcriptions",
60 + xai: "https://api.x.ai/v1/stt",
61 + }[provider],
62 + );
63 + expect(init?.redirect).toBe("error");
64 + expect(new Headers(init?.headers).get("authorization")).toBe("Bearer chosen");
65 + const form = init?.body as FormData;
66 + expect(form.get("model")).toBe(
67 + provider === "openai" ? "gpt-transcribe" : "grok-voice-transcribe-2.0",
68 + );
69 + expect(form.getAll(provider === "openai" ? "keywords[]" : "keyterm")).toEqual(["T3 Code"]);
70 + expect(form.get(provider === "openai" ? "languages[]" : "language")).toBe("en");
71 + expect([...form.keys()].at(-1)).toBe("file");
72 + const wav = new DataView(yield* Effect.promise(() => (form.get("file") as Blob).arrayBuffer()));
73 + expect(wav.getUint32(24, true)).toBe(16_000);
74 + expect(wav.getInt16(46, true)).toBe(16384);
75 + upstream.mockResolvedValueOnce(new Response("sensitive upstream diagnostic", { status: 401 }));
76 + expect(yield* Effect.flip(transcribe())).toMatchObject({ provider, reason: "rejected-key" });
77 + yield* service.setKey(provider, null);
78 + expect((yield* service.configured)[provider]).toBe(false);
79 + yield* Effect.flip(transcribe());
80 + expect(upstream).toHaveBeenCalledTimes(2);
81 + yield* service.setKey(provider, reference);
82 + expect((yield* service.configured)[provider]).toBe(true);
83 + expect(read).not.toHaveBeenCalled();
84 + resolved = "resolved-fixture";
85 + expect(yield* transcribe()).toBe("hello");
86 + expect(read).toHaveBeenCalledWith([reference]);
87 + expect(new Headers(upstream.mock.calls[2]?.[1]?.headers).get("authorization")).toBe(
88 + "Bearer resolved-fixture",
89 + );
90 + resolved = undefined;
91 + expect(yield* Effect.flip(transcribe())).toMatchObject({
92 + provider,
93 + reason: "unreadable-key",
94 + });
95 + expect(upstream).toHaveBeenCalledTimes(3);
96 + }),
97 +);
98 +
99 +it.effect("uploads to OpenRouter only for a listed exact model, as zero-retention JSON WAV", () =>
100 + Effect.gen(function* () {
101 + const upstream = vi.fn<typeof fetch>(async (input) =>
102 + new URL(String(input)).pathname.endsWith("/models")
103 + ? Response.json({ data: [{ id: "acme/transcribe-1" }, { id: "acme/transcribe-2" }] })
104 + : Response.json({ text: "hello" }),
105 + );
106 + const service = yield* CloudSpeech.CloudSpeech.pipe(
107 + Effect.provide(
108 + CloudSpeech.layer.pipe(
109 + Layer.provide(FetchHttpClient.layer),
110 + Layer.provide(ServerSettings.ServerSettingsService.layerTest()),
111 + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})),
112 + ),
113 + ),
114 + );
115 + const uploads = () =>
116 + upstream.mock.calls.filter(([url]) => String(url).endsWith("/audio/transcriptions"));
117 + const transcribe = (model: string | undefined) =>
118 + service
119 + .transcribe(
120 + "openrouter",
121 + new Uint8Array(new Float32Array([0, 0.5]).buffer),
122 + { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "en" },
123 + model,
124 + )
125 + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream));
126 + yield* service.setKey("openrouter", Redacted.make("router-key"));
127 + expect(yield* Effect.flip(transcribe(undefined))).toMatchObject({ provider: "openrouter" });
128 + upstream.mockResolvedValueOnce(new Response(null, { status: 503 }));
129 + expect(yield* Effect.flip(transcribe("acme/transcribe-2"))).toMatchObject({
130 + reason: "unavailable",
131 + });
132 + expect(yield* Effect.flip(transcribe("acme/delisted"))).toMatchObject({
133 + provider: "openrouter",
134 + });
135 + expect(uploads()).toEqual([]);
136 + expect(yield* transcribe("acme/transcribe-2")).toBe("hello");
137 + const [[url, init]] = uploads() as [[URL, RequestInit]];
138 + expect(new URL(String(upstream.mock.calls[0]![0])).searchParams.get("zdr")).toBe("true");
139 + expect(String(url)).toBe("https://openrouter.ai/api/v1/audio/transcriptions");
140 + expect(init.redirect).toBe("error");
141 + const headers = new Headers(init.headers);
142 + expect(headers.get("authorization")).toBe("Bearer router-key");
143 + expect(headers.get("content-type")).toContain("application/json");
144 + const body = JSON.parse(yield* Effect.promise(() => new Response(init.body).text())) as {
145 + model: string;
146 + models?: unknown;
147 + provider: { zdr: boolean };
148 + input_audio: { format: string; data: string };
149 + };
150 + expect(body).toMatchObject({
151 + model: "acme/transcribe-2",
152 + provider: { zdr: true },
153 + input_audio: { format: "wav" },
154 + });
155 + expect(body.models).toBeUndefined();
156 + const wav = Buffer.from(body.input_audio.data, "base64");
157 + expect(wav.toString("ascii", 0, 4)).toBe("RIFF");
158 + expect(wav.readInt16LE(46)).toBe(16384);
159 + }),
160 +);
161 +
162 +it.effect("removing a key while a catalog read is pending prevents the upload", () =>
163 + Effect.gen(function* () {
164 + const service = yield* CloudSpeech.CloudSpeech.pipe(
165 + Effect.provide(
166 + CloudSpeech.layer.pipe(
167 + Layer.provide(FetchHttpClient.layer),
168 + Layer.provide(ServerSettings.ServerSettingsService.layerTest()),
169 + Layer.provide(Layer.mock(SecretResolver.SecretResolver)({})),
170 + ),
171 + ),
172 + );
173 + const catalogStarted = Promise.withResolvers<void>();
174 + const catalog = Promise.withResolvers<Response>();
175 + const uploads: string[] = [];
176 + const upstream: typeof fetch = async (input) => {
177 + if (new URL(String(input)).pathname.endsWith("/models")) {
178 + catalogStarted.resolve();
179 + return catalog.promise;
180 + }
181 + uploads.push(String(input));
182 + return Response.json({ text: "unexpected" });
183 + };
184 + yield* service.setKey("openrouter", Redacted.make("old-key"));
185 + const job = yield* service
186 + .transcribe(
187 + "openrouter",
188 + new Uint8Array(4),
189 + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
190 + "acme/voice",
191 + )
192 + .pipe(Effect.provideService(FetchHttpClient.Fetch, upstream), Effect.forkChild);
193 + yield* Effect.promise(() => catalogStarted.promise);
194 + yield* service.setKey("openrouter", null);
195 + catalog.resolve(Response.json({ data: [{ id: "acme/voice" }] }));
196 + const exit = yield* Fiber.await(job);
197 + expect(uploads).toEqual([]);
198 + expect(exit._tag).toBe("Failure");
199 + }),
200 +);
apps/server/src/speech/CloudSpeech.ts new
+258
@@ -0,0 +1,258 @@
1 +import {
2 + CloudSpeechApiKey,
3 + CloudSpeechError,
4 + CloudSpeechModel,
5 + CloudSpeechModelId,
6 + SettingsSecretValue,
7 + isSecretReference,
8 + type SecretReference,
9 + type ServerSettingsError,
10 + type CloudSpeechKeys,
11 + type CloudSpeechProvider,
12 + EnvironmentSpeechTranscriptionResult,
13 + type SpeechTranscriptionOptions,
14 +} from "@t3tools/contracts";
15 +import * as SecretResolver from "@t3tools/secret-provider-core/server/SecretResolver";
16 +import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech";
17 +import * as Context from "effect/Context";
18 +import * as Effect from "effect/Effect";
19 +import * as Exit from "effect/Exit";
20 +import * as Layer from "effect/Layer";
21 +import * as Redacted from "effect/Redacted";
22 +import * as Schema from "effect/Schema";
23 +import { FetchHttpClient, HttpClient, HttpClientRequest, HttpClientResponse } from "effect/http";
24 +import * as ServerSettings from "../serverSettings.ts";
25 +import { requireSettingsSecret } from "../secrets/settingsSecretValue.ts";
26 +import { applySpeechAliases } from "./customWords.ts";
27 +import { removeSpeechFillerWords } from "./fillerWords.ts";
28 +import { decodeSpeechPcm, SpeechInvalidAudioError } from "./SpeechService.ts";
29 +
30 +const endpoints = {
31 + openai: { url: "https://api.openai.com/v1/audio/transcriptions", model: "gpt-transcribe" },
32 + xai: { url: "https://api.x.ai/v1/stt", model: "grok-voice-transcribe-2.0" },
33 + openrouter: { url: "https://openrouter.ai/api/v1/audio/transcriptions" },
34 +};
35 +const OPENROUTER_CATALOG_URL =
36 + "https://openrouter.ai/api/v1/models?output_modalities=transcription&zdr=true";
37 +const isApiKey = Schema.is(CloudSpeechApiKey);
38 +const isModelId = Schema.is(CloudSpeechModelId);
39 +const sameCredential = Schema.toEquivalence(SettingsSecretValue);
40 +const OpenRouterCatalog = Schema.Struct({
41 + data: Schema.Array(Schema.Struct({ id: Schema.String, name: Schema.optionalKey(Schema.String) })),
42 +});
43 +
44 +function wave(samples: Float32Array) {
45 + const buffer = new ArrayBuffer(44 + samples.length * 2);
46 + const view = new DataView(buffer);
47 + const tag = (at: number, value: string) =>
48 + new Uint8Array(buffer, at, 4).set(new TextEncoder().encode(value));
49 + tag(0, "RIFF");
50 + view.setUint32(4, buffer.byteLength - 8, true);
51 + tag(8, "WAVE");
52 + tag(12, "fmt ");
53 + view.setUint32(16, 16, true);
54 + view.setUint16(20, 1, true);
55 + view.setUint16(22, 1, true);
56 + view.setUint32(24, 16_000, true);
57 + view.setUint32(28, 32_000, true);
58 + view.setUint16(32, 2, true);
59 + view.setUint16(34, 16, true);
60 + tag(36, "data");
61 + view.setUint32(40, buffer.byteLength - 44, true);
62 + for (let index = 0; index < samples.length; index++) {
63 + const sample = samples[index]!;
64 + view.setInt16(44 + index * 2, Math.round(sample * (sample < 0 ? 32768 : 32767)), true);
65 + }
66 + return buffer;
67 +}
68 +
69 +function audioBody(
70 + provider: "openai" | "xai",
71 + audio: ArrayBuffer,
72 + language: string,
73 + options: SpeechTranscriptionOptions,
74 +) {
75 + const form = new FormData();
76 + form.set("model", endpoints[provider].model);
77 + if (language !== "auto") form.set(provider === "openai" ? "languages[]" : "language", language);
78 + for (const { term } of options.speechCustomWords) {
79 + if (provider === "openai" && /[<>\r\n]/.test(term)) continue;
80 + form.append(provider === "openai" ? "keywords[]" : "keyterm", term);
81 + }
82 + if (provider === "xai") {
83 + if (language !== "auto") form.set("format", "true");
84 + form.set(
85 + "filler_words",
86 + String(!options.speechRemoveFillerWords || Boolean(options.speechCorrectionWord)),
87 + );
88 + if (!options.speechSilenceFiltering) form.set("vad_threshold", "0");
89 + }
90 + // xAI ignores options that follow the file part.
91 + form.set("file", new Blob([audio], { type: "audio/wav" }), "dictation.wav");
92 + return HttpClientRequest.bodyFormData(form);
93 +}
94 +
95 +function openRouterBody(model: string, audio: ArrayBuffer, language: string) {
96 + return HttpClientRequest.bodyJsonUnsafe({
97 + model,
98 + input_audio: { data: Buffer.from(audio).toString("base64"), format: "wav" },
99 + ...(language !== "auto" ? { language } : {}),
100 + provider: { zdr: true },
101 + });
102 +}
103 +
104 +export class CloudSpeech extends Context.Service<
105 + CloudSpeech,
106 + {
107 + readonly configured: Effect.Effect<CloudSpeechKeys, ServerSettingsError>;
108 + /** OpenRouter's public transcription catalog, cached briefly on this server. */
109 + readonly models: Effect.Effect<ReadonlyArray<CloudSpeechModel>, CloudSpeechError>;
110 + readonly setKey: (
111 + provider: CloudSpeechProvider,
112 + key: Redacted.Redacted<string> | SecretReference | null,
113 + ) => Effect.Effect<void, ServerSettingsError>;
114 + readonly transcribe: (
115 + provider: CloudSpeechProvider,
116 + pcm: Uint8Array,
117 + options: SpeechTranscriptionOptions,
118 + model?: string | undefined,
119 + ) => Effect.Effect<string, CloudSpeechError | SpeechInvalidAudioError | ServerSettingsError>;
120 + }
121 +>()("t3/speech/CloudSpeech") {}
122 +
123 +export const layer = Layer.effect(
124 + CloudSpeech,
125 + Effect.gen(function* () {
126 + const settings = yield* ServerSettings.ServerSettingsService;
127 + const resolver = yield* SecretResolver.SecretResolver;
128 + const http = yield* HttpClient.HttpClient;
129 + const credentials = settings.getSettings.pipe(
130 + Effect.map((value) => value.cloudSpeechCredentials),
131 + );
132 + const configured = credentials.pipe(
133 + Effect.map(({ openai, xai, openrouter }) => ({
134 + openai: Boolean(openai),
135 + xai: Boolean(xai),
136 + openrouter: Boolean(openrouter),
137 + })),
138 + );
139 + const execute = (request: HttpClientRequest.HttpClientRequest) =>
140 + http
141 + .execute(request)
142 + .pipe(
143 + Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
144 + Effect.provideService(HttpClient.TracerPropagationEnabled, false),
145 + );
146 + const fetchCatalog = Effect.fn("CloudSpeech.catalog")(function* () {
147 + const unavailable = () =>
148 + new CloudSpeechError({ provider: "openrouter", reason: "unavailable" });
149 + const response = yield* execute(HttpClientRequest.get(OPENROUTER_CATALOG_URL)).pipe(
150 + Effect.mapError(unavailable),
151 + );
152 + if (response.status < 200 || response.status >= 300) return yield* unavailable();
153 + const { data } = yield* HttpClientResponse.schemaBodyJson(OpenRouterCatalog)(response).pipe(
154 + Effect.mapError(unavailable),
155 + );
156 + return data.flatMap(({ id, name }) =>
157 + isModelId(id) ? [{ id, name: name?.trim() || id }] : [],
158 + );
159 + });
160 + // The catalog is public, so the cache holds no credential or consent state.
161 + const models = yield* Effect.cachedWithTTL(
162 + fetchCatalog().pipe(
163 + Effect.timeoutOrElse({
164 + duration: "10 seconds",
165 + orElse: () =>
166 + Effect.fail(new CloudSpeechError({ provider: "openrouter", reason: "unavailable" })),
167 + }),
168 + ),
169 + (exit) => (Exit.isSuccess(exit) ? "10 minutes" : 0),
170 + );
171 + const setKey = Effect.fn("CloudSpeech.setKey")(function* (
172 + provider: CloudSpeechProvider,
173 + key: Redacted.Redacted<string> | SecretReference | null,
174 + ) {
175 + yield* settings.updateSettings({
176 + cloudSpeechCredentials: {
177 + [provider]: Redacted.isRedacted(key) ? Redacted.value(key) : (key ?? ""),
178 + },
179 + });
180 + });
181 + const transcribe = Effect.fn("CloudSpeech.transcribe")(
182 + function* (
183 + provider: CloudSpeechProvider,
184 + pcm: Uint8Array,
185 + options: SpeechTranscriptionOptions,
186 + model?: string,
187 + ) {
188 + const source = (yield* credentials)[provider];
189 + if (!source) return yield* new CloudSpeechError({ provider, reason: "missing-key" });
190 + if (provider === "openrouter" && (!model || !isModelId(model)))
191 + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
192 + const samples = yield* Effect.try({
193 + try: () => decodeSpeechPcm(pcm),
194 + catch: () =>
195 + new SpeechInvalidAudioError({
196 + byteLength: pcm.byteLength,
197 + message: "Invalid PCM audio.",
198 + }),
199 + });
200 + const language = effectiveSpeechLanguage(cloudSpeechLanguages, options.speechLanguage);
201 + if (provider === "openrouter" && !(yield* models).some((entry) => entry.id === model))
202 + return yield* new CloudSpeechError({ provider, reason: "model-unavailable" });
203 + // A vault that locked since the last recording must not authorize a new upload.
204 + if (isSecretReference(source)) yield* resolver.invalidate([source]);
205 + const unreadable = () => new CloudSpeechError({ provider, reason: "unreadable-key" });
206 + const key = yield* requireSettingsSecret(resolver, source).pipe(
207 + Effect.mapError(unreadable),
208 + );
209 + if (!isApiKey(key)) return yield* unreadable();
210 + const audio = wave(samples);
211 + const request = HttpClientRequest.post(endpoints[provider].url).pipe(
212 + HttpClientRequest.bearerToken(Redacted.make(key)),
213 + provider === "openrouter"
214 + ? openRouterBody(model!, audio, language)
215 + : audioBody(provider, audio, language, options),
216 + );
217 + const unavailable = () => new CloudSpeechError({ provider, reason: "unavailable" });
218 + // A catalog fetch or vault lookup may have outlived the credential it started with.
219 + const current = (yield* credentials)[provider];
220 + if (!current || !sameCredential(source, current))
221 + return yield* new CloudSpeechError({
222 + provider,
223 + reason: current ? "key-changed" : "missing-key",
224 + });
225 + const response = yield* execute(request).pipe(Effect.mapError(unavailable));
226 + if (response.status < 200 || response.status >= 300)
227 + return yield* new CloudSpeechError({
228 + provider,
229 + reason:
230 + response.status === 429
231 + ? "rate-limit"
232 + : [401, 402, 403].includes(response.status)
233 + ? "rejected-key"
234 + : response.status === 404 && provider === "openrouter"
235 + ? "model-unavailable"
236 + : "unavailable",
237 + });
238 + const { text } = yield* HttpClientResponse.schemaBodyJson(
239 + EnvironmentSpeechTranscriptionResult,
240 + )(response).pipe(Effect.mapError(unavailable));
241 + const corrected = applySpeechAliases(text, options.speechCustomWords);
242 + return (
243 + options.speechRemoveFillerWords
244 + ? removeSpeechFillerWords(corrected, language, options.speechCorrectionWord)
245 + : corrected
246 + ).trim();
247 + },
248 + (effect, provider) =>
249 + effect.pipe(
250 + Effect.timeoutOrElse({
251 + duration: "2 minutes",
252 + orElse: () => Effect.fail(new CloudSpeechError({ provider, reason: "unavailable" })),
253 + }),
254 + ),
255 + );
256 + return CloudSpeech.of({ configured, models, setKey, transcribe });
257 + }),
258 +);
apps/server/src/speech/http.test.ts
+123 -22
@@ -7,6 +7,9 @@ import { expect, vi } from "vite-plus/test";
7 import {
8 AuthSessionId,
9 AuthOrchestrationOperateScope,
10 + AuthSettingsWriteScope,
11 + type AuthEnvironmentScope,
12 + CloudSpeechError,
13 DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
14 SPEECH_MAX_OPTIONS_BYTES,
15 DEFAULT_SERVER_SETTINGS,
@@ -16,6 +19,7 @@ import {
19 } from "@t3tools/contracts";
20 import * as Effect from "effect/Effect";
21 import * as Layer from "effect/Layer";
22 +import * as Redacted from "effect/Redacted";
23 import * as Schema from "effect/Schema";
24 import * as HttpRouter from "effect/http/HttpRouter";
25 import * as HttpServer from "effect/http/HttpServer";
@@ -23,6 +27,7 @@ import * as HttpApi from "effect/http-api/HttpApi";
27 import * as HttpApiBuilder from "effect/http-api/HttpApiBuilder";
28 import type * as NetAddress from "effect/net/NetAddress";
29 import * as SpeechService from "./SpeechService.ts";
30 +import * as CloudSpeech from "./CloudSpeech.ts";
31 import * as ServerSettings from "../serverSettings.ts";
32 import * as TextGeneration from "../textGeneration/TextGeneration.ts";
33 import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
@@ -30,7 +35,20 @@ import { speechHttpApiLayer } from "./http.ts";
35
36 const decodeSessionId = Schema.decodeUnknownSync(AuthSessionId);
37
38 +const principal = (...scopes: AuthEnvironmentScope[]) =>
39 + Layer.succeed(EnvironmentAuthenticatedAuth, (effect) =>
40 + effect.pipe(
41 + Effect.provideService(EnvironmentAuthenticatedPrincipal, {
42 + sessionId: decodeSessionId("voice-test"),
43 + subject: "test",
44 + method: "bearer-access-token",
45 + scopes: new Set(scopes),
46 + }),
47 + ),
48 + );
49 +
50 const dependencies = Layer.mergeAll(
51 + Layer.mock(CloudSpeech.CloudSpeech)({}),
52 Layer.mock(ProviderRegistry.ProviderRegistry)({ getProviders: Effect.succeed([]) }),
53 Layer.mock(ServerSettings.ServerSettingsService)({
54 getSettings: Effect.succeed(DEFAULT_SERVER_SETTINGS),
@@ -51,21 +69,11 @@ const dependencies = Layer.mergeAll(
69 effectIt.live("rejects oversized chunked audio without invoking transcription", () =>
70 Effect.gen(function* () {
71 const transcribe = vi.fn(() => Effect.succeed("unexpected"));
54 - const auth = Layer.succeed(EnvironmentAuthenticatedAuth, (effect) =>
55 - effect.pipe(
56 - Effect.provideService(EnvironmentAuthenticatedPrincipal, {
57 - sessionId: decodeSessionId("voice-test"),
58 - subject: "test",
59 - method: "bearer-access-token",
60 - scopes: new Set([AuthOrchestrationOperateScope]),
61 - }),
62 - ),
63 - );
72 const routes = HttpApiBuilder.layer(
73 HttpApi.make("environment").add(EnvironmentVoiceHttpApi),
74 ).pipe(
75 Layer.provide(speechHttpApiLayer),
68 - Layer.provide(auth),
76 + Layer.provide(principal(AuthOrchestrationOperateScope)),
77 Layer.provide(
78 Layer.succeed(SpeechService.SpeechService, {
79 status: Effect.succeed({ supported: false as const, reason: "test" }),
@@ -97,6 +105,14 @@ effectIt.live("rejects oversized chunked audio without invoking transcription",
105 }),
106 );
107 expect(select.status).toBe(403);
108 + const changeKey = yield* Effect.promise(() =>
109 + fetch(new URL("cloud/xai/key", url), {
110 + method: "POST",
111 + headers: { "content-type": "application/json" },
112 + body: JSON.stringify({ key: "fixture" }),
113 + }),
114 + );
115 + expect(changeKey.status).toBe(403);
116 yield* Effect.promise(() =>
117 expect(
118 fetch(url, {
@@ -120,6 +136,101 @@ effectIt.live("rejects oversized chunked audio without invoking transcription",
136 }),
137 );
138
139 +effectIt.effect(
140 + "keeps keys write-only and returns provider failures without an auth challenge",
141 + () =>
142 + Effect.gen(function* () {
143 + let stored = false;
144 + const reference = { provider: "work", reference: "op://Work/Voice/key" };
145 + const save = vi.fn();
146 + const transcribe = vi.fn();
147 + const cloud = Layer.mock(CloudSpeech.CloudSpeech)({
148 + configured: Effect.sync(() => ({ openai: stored, xai: false })),
149 + models: Effect.succeed([{ id: "acme/transcribe-1", name: "Acme Transcribe" }]),
150 + setKey: (provider, key) =>
151 + Effect.sync(() => {
152 + save(provider, key);
153 + stored =
154 + provider === "openai" &&
155 + Redacted.isRedacted(key) &&
156 + Redacted.value(key) === "fixture";
157 + }),
158 + transcribe: (provider, _pcm, _options, model) => {
159 + transcribe(provider, model);
160 + return Effect.fail(new CloudSpeechError({ provider, reason: "rejected-key" }));
161 + },
162 + });
163 + const { handler, dispose } = HttpRouter.toWebHandler(
164 + HttpApiBuilder.layer(HttpApi.make("environment").add(EnvironmentVoiceHttpApi)).pipe(
165 + Layer.provide(speechHttpApiLayer),
166 + Layer.provide(principal(AuthOrchestrationOperateScope, AuthSettingsWriteScope)),
167 + Layer.provide(Layer.mock(SpeechService.SpeechService)({})),
168 + Layer.provide(cloud),
169 + Layer.provide(dependencies),
170 + Layer.provide(HttpServer.layerServices),
171 + Layer.provide(NodeServices.layer),
172 + ),
173 + { disableLogger: true },
174 + );
175 + yield* Effect.addFinalizer(() => Effect.promise(dispose));
176 + const context = yield* Layer.build(NodeServices.layer);
177 + const call = (path: string, init?: RequestInit) =>
178 + Effect.promise(() =>
179 + handler(new Request(`http://localhost/api/voice/cloud/${path}`, init), context),
180 + );
181 + const saved = yield* call("openai/key", {
182 + method: "POST",
183 + headers: { "content-type": "application/json" },
184 + body: JSON.stringify({ key: "fixture" }),
185 + });
186 + expect(saved.status).toBe(204);
187 + expect(yield* Effect.promise(() => saved.text())).toBe("");
188 + const keys = yield* call("keys");
189 + expect(yield* Effect.promise(() => keys.json())).toEqual({
190 + openai: true,
191 + xai: false,
192 + });
193 + const setReference = (key: unknown) =>
194 + call("xai/key", {
195 + method: "POST",
196 + headers: { "content-type": "application/json" },
197 + body: JSON.stringify({ key }),
198 + });
199 + expect((yield* setReference(reference)).status).toBe(204);
200 + expect(save).toHaveBeenLastCalledWith("xai", reference);
201 + expect((yield* setReference(reference.reference)).status).toBe(400);
202 + expect(save).toHaveBeenCalledTimes(2);
203 + const failed = yield* call("openai/transcribe", {
204 + method: "POST",
205 + headers: { "content-type": "application/octet-stream" },
206 + body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS),
207 + });
208 + expect(failed.status).toBe(422);
209 + expect(yield* Effect.promise(() => failed.json())).toEqual({
210 + _tag: "CloudSpeechError",
211 + provider: "openai",
212 + reason: "rejected-key",
213 + });
214 + expect(transcribe).toHaveBeenLastCalledWith("openai", undefined);
215 + const catalog = yield* call("openrouter/models");
216 + expect(yield* Effect.promise(() => catalog.json())).toEqual({
217 + models: [{ id: "acme/transcribe-1", name: "Acme Transcribe" }],
218 + });
219 + const audio = {
220 + method: "POST",
221 + headers: { "content-type": "application/octet-stream" },
222 + body: encodeSpeechPcmRequest(new Uint8Array(4), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS),
223 + };
224 + expect((yield* call("openrouter/transcribe?model=openrouter%2Fauto", audio)).status).toBe(
225 + 400,
226 + );
227 + expect((yield* call("openrouter/transcribe?model=acme%2Ftranscribe-1", audio)).status).toBe(
228 + 422,
229 + );
230 + expect(transcribe).toHaveBeenLastCalledWith("openrouter", "acme/transcribe-1");
231 + }).pipe(Effect.scoped),
232 +);
233 +
234 effectIt.effect.each([
235 {
236 error: new SpeechService.SpeechInvalidAudioError({ byteLength: 3, message: "invalid PCM" }),
@@ -151,16 +262,6 @@ effectIt.effect.each([
262 ({ error, reason, ...options }) =>
263 Effect.gen(function* () {
264 // Only authentication and the native service are replaced; exercise the actual HTTP handlers and codecs.
154 - const auth = Layer.succeed(EnvironmentAuthenticatedAuth, (effect) =>
155 - effect.pipe(
156 - Effect.provideService(EnvironmentAuthenticatedPrincipal, {
157 - sessionId: decodeSessionId("voice-test"),
158 - subject: "test",
159 - method: "bearer-access-token",
160 - scopes: new Set([AuthOrchestrationOperateScope]),
161 - }),
162 - ),
163 - );
265 const transcribe = vi.fn(() => Effect.fail(error));
266 const service = Layer.succeed(SpeechService.SpeechService, {
267 status: Effect.succeed({ supported: false as const, reason: "test" }),
@@ -179,7 +280,7 @@ effectIt.effect.each([
280 const { handler, dispose } = HttpRouter.toWebHandler(
281 HttpApiBuilder.layer(api).pipe(
282 Layer.provide(speechHttpApiLayer),
182 - Layer.provide(auth),
283 + Layer.provide(principal(AuthOrchestrationOperateScope)),
284 Layer.provide(service),
285 Layer.provide(HttpServer.layerServices),
286 Layer.provide(dependencies),
apps/server/src/speech/http.ts
+49
@@ -21,6 +21,7 @@ import {
21 requireEnvironmentScope,
22 } from "../auth/http.ts";
23 import * as SpeechPostProcessing from "./postProcessing.ts";
24 +import * as CloudSpeech from "./CloudSpeech.ts";
25 import * as SpeechService from "./SpeechService.ts";
26
27 const MAX_TRANSCRIBE_BODY_BYTES = SpeechService.MAX_SPEECH_BYTES + SPEECH_MAX_OPTIONS_BYTES + 4;
@@ -57,6 +58,7 @@ export const speechHttpApiLayer = HttpApiBuilder.group(
58 "voice",
59 Effect.fnUntraced(function* (handlers) {
60 const speech = yield* SpeechService.SpeechService;
61 + const cloud = yield* CloudSpeech.CloudSpeech;
62 const postProcessing = yield* SpeechPostProcessing.SpeechPostProcessing;
63 const guard = Effect.fnUntraced(function* (
64 endpoint: string,
@@ -66,6 +68,53 @@ export const speechHttpApiLayer = HttpApiBuilder.group(
68 yield* requireEnvironmentScope(scope);
69 });
70 return handlers
71 + .handle(
72 + "cloudKeys",
73 + Effect.fn("environment.voice.cloudKeys")(function* (args) {
74 + yield* guard(args.endpoint.name);
75 + return yield* cloud.configured.pipe(
76 + Effect.catch((error) => failEnvironmentInternal("internal_error", error)),
77 + );
78 + }),
79 + )
80 + .handle(
81 + "setCloudKey",
82 + Effect.fn("environment.voice.setCloudKey")(function* (args) {
83 + yield* guard(args.endpoint.name, AuthSettingsWriteScope);
84 + yield* cloud
85 + .setKey(args.params.provider, args.payload.key)
86 + .pipe(Effect.catch((error) => failEnvironmentInternal("internal_error", error)));
87 + }),
88 + )
89 + .handle(
90 + "cloudModels",
91 + Effect.fn("environment.voice.cloudModels")(function* (args) {
92 + yield* guard(args.endpoint.name);
93 + return { models: yield* cloud.models };
94 + }),
95 + )
96 + .handle(
97 + "transcribeCloud",
98 + Effect.fn("environment.voice.transcribeCloud")(function* (args) {
99 + yield* guard(args.endpoint.name);
100 + const decoded = parseAudio(args.payload);
101 + if (Option.isNone(decoded)) return yield* failEnvironmentInvalidRequest("invalid_audio");
102 + const text = yield* cloud
103 + .transcribe(
104 + args.params.provider,
105 + decoded.value.pcm,
106 + decoded.value.options,
107 + args.query.model,
108 + )
109 + .pipe(
110 + Effect.catchTags({
111 + ServerSettingsError: (error) => failEnvironmentInternal("internal_error", error),
112 + SpeechInvalidAudioError: () => failEnvironmentInvalidRequest("invalid_audio"),
113 + }),
114 + );
115 + return { text };
116 + }),
117 + )
118 .handle(
119 "status",
120 Effect.fn("environment.voice.status")(function* (args) {
apps/web/src/components/settings/CloudVoiceConfiguration.browser.test.tsx new
+227
@@ -0,0 +1,227 @@
1 +import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection";
2 +import { EnvironmentId } from "@t3tools/contracts";
3 +import * as Option from "effect/Option";
4 +import { expect, it, vi } from "vite-plus/test";
5 +import { page, userEvent } from "vite-plus/test/browser";
6 +import { render } from "../../../test/render";
7 +import "~/index.css";
8 +import { chooseAudioDestination } from "../../speech/voiceConsent";
9 +import { useEnvironmentScope, usePreparedConnection } from "../../state/session";
10 +import { VoiceSettingsPanel } from "./VoiceSettingsPanel";
11 +import { SettingsScopeProvider } from "./SettingsScopeContext";
12 +
13 +vi.mock(import("../../state/session"), { spy: true });
14 +vi.mock("../../env", async (original) => ({
15 + ...(await original<typeof import("../../env")>()),
16 + isSecretsFeatureEnabled: true,
17 +}));
18 +vi.mock("../../state/environments", async (importOriginal) => ({
19 + ...(await importOriginal<typeof import("../../state/environments")>()),
20 + useEnvironments: () => ({
21 + environments: [
22 + {
23 + environmentId: "voice-host",
24 + label: "Laptop",
25 + connection: { phase: "available" },
26 + serverConfig: {
27 + secretReferences: true,
28 + settings: {
29 + secretProviders: {
30 + work: { kind: "one_password", label: "Work vault", config: { account: "work" } },
31 + },
32 + },
33 + environment: {
34 + platform: { os: "linux", arch: "x64" },
35 + capabilities: { voiceTranscription: true, cloudVoiceTranscription: true },
36 + },
37 + },
38 + },
39 + ],
40 + }),
41 +}));
42 +
43 +it("enables the chosen provider only after a successful setup and keeps its grant when editing the key", async () => {
44 + const id = EnvironmentId.make("voice-host");
45 + const target = new PrimaryConnectionTarget({
46 + environmentId: id,
47 + label: "Laptop",
48 + httpBaseUrl: window.location.origin,
49 + wsBaseUrl: "ws://localhost",
50 + });
51 + const prepared = Option.some({
52 + ...target,
53 + target,
54 + socketUrl: target.wsBaseUrl,
55 + httpAuthorization: null,
56 + });
57 + using cleanup = new DisposableStack();
58 + cleanup.defer(() => chooseAudioDestination(null));
59 + chooseAudioDestination({ kind: "environment", environmentId: id });
60 + const writes: unknown[] = [];
61 + let rejectNextWrite = true;
62 + const fetch = globalThis.fetch;
63 + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => {
64 + const url = new URL(request instanceof Request ? request.url : request, window.location.href);
65 + if (url.pathname === "/api/voice/cloud/keys")
66 + return Promise.resolve(Response.json({ openai: writes.length > 0, xai: true }));
67 + if (url.pathname === "/api/voice/cloud/openai/key") {
68 + if (rejectNextWrite) {
69 + rejectNextWrite = false;
70 + return Promise.resolve(new Response(null, { status: 403 }));
71 + }
72 + return new Request(request, init).json().then((body) => {
73 + writes.push(body);
74 + return new Response(null, { status: 204 });
75 + });
76 + }
77 + return fetch(request, init);
78 + });
79 + await vi.mocked(useEnvironmentScope).withImplementation(
80 + () => true,
81 + async () => {
82 + await vi.mocked(usePreparedConnection).withImplementation(
83 + () => prepared,
84 + async () => {
85 + await using screen = await render(
86 + <SettingsScopeProvider search={{}} onChange={() => {}}>
87 + <VoiceSettingsPanel />
88 + </SettingsScopeProvider>,
89 + );
90 + const destination = screen.getByRole("combobox", { name: "Transcribe with" });
91 + await expect.element(destination).toHaveTextContent("Laptop");
92 + await destination.click();
93 + await page.getByRole("option", { name: "OpenAI via Laptop" }).click();
94 + const dialog = page.getByRole("dialog");
95 + await dialog.getByRole("button", { name: "Cancel" }).click();
96 + await expect.element(destination).toHaveTextContent("Laptop");
97 + expect(writes).toEqual([]);
98 + await destination.click();
99 + await page.getByRole("option", { name: "OpenAI via Laptop" }).click();
100 + await dialog.getByRole("combobox", { name: "OpenAI API key source" }).click();
101 + await page.getByRole("option", { name: "Work vault" }).click();
102 + await dialog.getByLabelText("OpenAI API key reference").fill("op://Work/Voice/key");
103 + await userEvent.keyboard("{Enter}");
104 + await expect.element(dialog.getByRole("alert")).toBeVisible();
105 + expect(writes).toEqual([]);
106 + await expect
107 + .element(dialog.getByRole("button", { name: "Save and enable" }))
108 + .toBeEnabled();
109 + await dialog.getByLabelText("OpenAI API key reference").click();
110 + await userEvent.keyboard("{Enter}");
111 + await expect.element(dialog).not.toBeInTheDocument();
112 + await expect.element(destination).toHaveTextContent("OpenAI via Laptop");
113 + expect(writes).toEqual([{ key: { provider: "work", reference: "op://Work/Voice/key" } }]);
114 + const grant = window.localStorage.getItem("t3code:voice-audio-grant:v1");
115 + await screen.getByRole("button", { name: "Manage key" }).click();
116 + await dialog.getByLabelText("OpenAI API key", { exact: true }).fill("replacement");
117 + await userEvent.keyboard("{Enter}");
118 + await expect.element(dialog).not.toBeInTheDocument();
119 + expect(writes.at(-1)).toEqual({ key: "replacement" });
120 + expect(window.localStorage.getItem("t3code:voice-audio-grant:v1")).toBe(grant);
121 + await destination.click();
122 + await page.getByRole("option", { name: "Grok Voice Transcribe via Laptop" }).click();
123 + await dialog.getByRole("button", { name: "Use Grok Voice Transcribe" }).click();
124 + await expect.element(destination).toHaveTextContent("Grok Voice Transcribe via Laptop");
125 + await destination.click();
126 + await page.getByRole("option", { name: "Off", exact: true }).click();
127 + await expect.element(destination).toHaveTextContent("Off");
128 + },
129 + );
130 + },
131 + );
132 +});
133 +
134 +it("OpenRouter needs an exact model, and changing it writes no credential", async () => {
135 + const id = EnvironmentId.make("voice-host");
136 + const target = new PrimaryConnectionTarget({
137 + environmentId: id,
138 + label: "Laptop",
139 + httpBaseUrl: window.location.origin,
140 + wsBaseUrl: "ws://localhost",
141 + });
142 + const prepared = Option.some({
143 + ...target,
144 + target,
145 + socketUrl: target.wsBaseUrl,
146 + httpAuthorization: null,
147 + });
148 + using cleanup = new DisposableStack();
149 + cleanup.defer(() => chooseAudioDestination(null));
150 + chooseAudioDestination(null);
151 + const writes: unknown[] = [];
152 + const fetch = globalThis.fetch;
153 + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((request, init) => {
154 + const url = new URL(request instanceof Request ? request.url : request, window.location.href);
155 + if (url.pathname === "/api/voice/cloud/keys")
156 + return Promise.resolve(Response.json({ openai: false, xai: false, openrouter: true }));
157 + if (url.pathname === "/api/voice/cloud/openrouter/models")
158 + return Promise.resolve(
159 + Response.json({
160 + models: [
161 + { id: "acme/transcribe-1", name: "Acme: Transcribe 1" },
162 + { id: "other/listen-2", name: "Other: Listen 2" },
163 + ],
164 + }),
165 + );
166 + if (url.pathname === "/api/voice/cloud/openrouter/key")
167 + return new Request(request, init).json().then((body) => {
168 + writes.push(body);
169 + return new Response(null, { status: 204 });
170 + });
171 + return fetch(request, init);
172 + });
173 + const grantKey = "t3code:voice-audio-grant:v1";
174 + const grantModel = () => {
175 + const raw = window.localStorage.getItem(grantKey);
176 + const parsed = raw ? (JSON.parse(raw) as { destination?: { model?: string } }) : null;
177 + return parsed?.destination?.model ?? null;
178 + };
179 + await vi.mocked(useEnvironmentScope).withImplementation(
180 + () => true,
181 + async () => {
182 + await vi.mocked(usePreparedConnection).withImplementation(
183 + () => prepared,
184 + async () => {
185 + await using screen = await render(
186 + <SettingsScopeProvider search={{}} onChange={() => {}}>
187 + <VoiceSettingsPanel />
188 + </SettingsScopeProvider>,
189 + );
190 + const destination = screen.getByRole("combobox", { name: "Transcribe with" });
191 + await destination.click();
192 + await page.getByRole("option", { name: "OpenRouter via Laptop" }).click();
193 + const dialog = page.getByRole("dialog");
194 + const save = dialog.getByRole("button", { name: "Save and enable" });
195 + await expect.element(save).toBeDisabled();
196 + const picker = dialog.getByRole("combobox", { name: "OpenRouter model" });
197 + await expect.element(picker).toBeVisible();
198 + await picker.click();
199 + await userEvent.keyboard("listen");
200 + await expect.element(page.getByRole("option", { name: /Other: Listen 2/ })).toBeVisible();
201 + await page.getByRole("option", { name: /Other: Listen 2/ }).click();
202 + await expect.element(save).toBeEnabled();
203 + const key = dialog.getByLabelText("OpenRouter API key", { exact: true });
204 + await key.fill("op://not-a-literal-key");
205 + await expect.element(save).toBeDisabled();
206 + await key.fill("");
207 + await save.click();
208 + await expect.element(dialog).not.toBeInTheDocument();
209 + await expect.element(destination).toHaveTextContent("OpenRouter via Laptop");
210 + expect(grantModel()).toBe("other/listen-2");
211 + expect(writes).toEqual([]);
212 + await screen.getByRole("button", { name: "Manage OpenRouter" }).click();
213 + const current = dialog.getByRole("combobox", { name: "OpenRouter model" });
214 + await expect.element(current).toHaveValue("Other: Listen 2");
215 + await userEvent.clear(current);
216 + await userEvent.type(current, "acme");
217 + await page.getByRole("option", { name: /Acme: Transcribe 1/ }).click();
218 + await dialog.getByRole("button", { name: "Save" }).click();
219 + await expect.element(dialog).not.toBeInTheDocument();
220 + await expect.element(destination).toHaveTextContent("OpenRouter via Laptop");
221 + expect(grantModel()).toBe("acme/transcribe-1");
222 + expect(writes).toEqual([]);
223 + },
224 + );
225 + },
226 + );
227 +});
apps/web/src/components/settings/CloudVoiceConfiguration.tsx new
+471
@@ -0,0 +1,471 @@
1 +import { useAtomRefresh, useAtomValue } from "@effect/atom-react";
2 +import { cloudSpeechDisclosure } from "@t3tools/shared/speech";
3 +import type { PreparedConnection } from "@t3tools/client-runtime/connection";
4 +import {
5 + configuredSecretSources,
6 + secretRequestReference,
7 +} from "@t3tools/client-runtime/secret-request";
8 +import {
9 + getCloudSpeechKeys,
10 + getCloudSpeechModels,
11 + setCloudSpeechKey,
12 +} from "@t3tools/client-runtime/voice-input";
13 +import {
14 + AuthOrchestrationOperateScope,
15 + AuthSettingsWriteScope,
16 + CloudSpeechApiKey,
17 + CloudSpeechModelId,
18 + CLOUD_SPEECH_LABELS,
19 + CLOUD_SPEECH_PROVIDER_NAMES,
20 + isSecretReference,
21 + type CloudSpeechModel,
22 + type CloudSpeechProvider,
23 + type SecretProviderId,
24 + type SettingsSecretValue,
25 + type VoiceTranscriptionDestination,
26 +} from "@t3tools/contracts";
27 +import * as Effect from "effect/Effect";
28 +import * as Option from "effect/Option";
29 +import * as Schema from "effect/Schema";
30 +import type { HttpClient } from "effect/http";
31 +import { Atom, AsyncResult } from "effect/reactivity";
32 +import { useId, useState } from "react";
33 +import { isSecretsFeatureEnabled } from "../../env";
34 +import {
35 + inEnvironmentSpeechRuntime,
36 + runEnvironmentSpeech,
37 +} from "../../speech/runEnvironmentSpeech";
38 +import {
39 + chooseAudioDestination,
40 + pendingCloudAudioChoice,
41 + useAudioGrant,
42 +} from "../../speech/voiceConsent";
43 +import type { EnvironmentPresentation } from "../../state/environments";
44 +import { useEnvironmentScope, usePreparedConnection } from "../../state/session";
45 +import { Button } from "../ui/button";
46 +import {
47 + Combobox,
48 + ComboboxEmpty,
49 + ComboboxInput,
50 + ComboboxItem,
51 + ComboboxList,
52 + ComboboxPopup,
53 + useComboboxFilter,
54 +} from "../ui/combobox";
55 +import {
56 + Dialog,
57 + DialogDescription,
58 + DialogFooter,
59 + DialogHeader,
60 + DialogPanel,
61 + DialogPopup,
62 + DialogTitle,
63 +} from "../ui/dialog";
64 +import { Input } from "../ui/input";
65 +import { Label } from "../ui/label";
66 +import { SecretInput } from "./SecretInput";
67 +import { SecretSourceSelect } from "./SecretSourceInput";
68 +
69 +// Per dialog instance, not an Atom.family: key presence must be read fresh each time it opens.
70 +function useCloudRead<A, E>(
71 + prepared: PreparedConnection | null,
72 + read: (prepared: PreparedConnection) => Effect.Effect<A, E, HttpClient.HttpClient>,
73 +) {
74 + const make = (connection: PreparedConnection | null) => ({
75 + prepared: connection,
76 + atom: Atom.make(
77 + connection ? inEnvironmentSpeechRuntime(read(connection)) : Effect.succeed(null),
78 + ),
79 + });
80 + const [current, setCurrent] = useState(() => make(prepared));
81 + if (current.prepared !== prepared) setCurrent(make(prepared));
82 + return [useAtomValue(current.atom), useAtomRefresh(current.atom)] as const;
83 +}
84 +const isApiKey = Schema.is(CloudSpeechApiKey);
85 +const isModelId = Schema.is(CloudSpeechModelId);
86 +type CloudDestination = Extract<VoiceTranscriptionDestination, { kind: "cloud" }>;
87 +
88 +function ModelPicker({
89 + models,
90 + value,
91 + onChange,
92 + disabled,
93 + current,
94 +}: {
95 + models: ReadonlyArray<CloudSpeechModel>;
96 + value: string;
97 + onChange: (next: string) => void;
98 + disabled: boolean;
99 + /** The grant's model, flagged when the catalog no longer lists it. */
100 + current: string | undefined;
101 +}) {
102 + const { contains } = useComboboxFilter();
103 + const selected = models.find((model) => model.id === value) ?? null;
104 + const delisted = current !== undefined && !models.some((model) => model.id === current);
105 + return (
106 + <div className="grid gap-1">
107 + <Combobox
108 + items={models}
109 + itemToStringLabel={(model: CloudSpeechModel) => model.name}
110 + itemToStringValue={(model: CloudSpeechModel) => model.id}
111 + filter={(model: CloudSpeechModel, query) =>
112 + contains(model.name, query) || contains(model.id, query)
113 + }
114 + value={selected}
115 + onValueChange={(model) => onChange(model?.id ?? "")}
116 + autoHighlight
117 + disabled={disabled}
118 + >
119 + <ComboboxInput
120 + id="voice-cloud-model"
121 + aria-label="OpenRouter model"
122 + placeholder="Search transcription models"
123 + autoFocus
124 + size="sm"
125 + />
126 + <ComboboxPopup>
127 + <ComboboxEmpty>No transcription model matches.</ComboboxEmpty>
128 + <ComboboxList className="max-h-64">
129 + {(model: CloudSpeechModel) => (
130 + <ComboboxItem key={model.id} value={model}>
131 + <span className="flex min-w-0 flex-col">
132 + <span className="truncate">{model.name}</span>
133 + <span className="truncate text-xs text-muted-foreground">{model.id}</span>
134 + </span>
135 + </ComboboxItem>
136 + )}
137 + </ComboboxList>
138 + </ComboboxPopup>
139 + </Combobox>
140 + {delisted ? (
141 + <p className="text-sm text-destructive" role="alert">
142 + OpenRouter no longer lists {current}. Dictation stays off until you choose another model.
143 + </p>
144 + ) : selected ? (
145 + <p className="truncate text-xs text-muted-foreground">{selected.id}</p>
146 + ) : null}
147 + </div>
148 + );
149 +}
150 +
151 +export function CloudVoiceConfiguration({
152 + environment,
153 + provider,
154 + active,
155 + close,
156 +}: {
157 + environment: Pick<EnvironmentPresentation, "environmentId" | "label" | "serverConfig">;
158 + provider: CloudSpeechProvider;
159 + active: boolean;
160 + close: () => void;
161 +}) {
162 + const id = environment.environmentId;
163 + const name = CLOUD_SPEECH_LABELS[provider];
164 + const company = CLOUD_SPEECH_PROVIDER_NAMES[provider];
165 + const routed = provider === "openrouter";
166 + const formId = useId();
167 + const prepared = Option.getOrNull(usePreparedConnection(id));
168 + const canUse = useEnvironmentScope(id, AuthOrchestrationOperateScope);
169 + const canManage = useEnvironmentScope(id, AuthSettingsWriteScope);
170 + const [presence, refresh] = useCloudRead(canUse ? prepared : null, getCloudSpeechKeys);
171 + const [models, refreshModels] = useCloudRead(
172 + routed && canUse ? prepared : null,
173 + getCloudSpeechModels,
174 + );
175 + const grant = useAudioGrant();
176 + const currentModel =
177 + active && grant?.destination.kind === "cloud" && grant.destination.provider === "openrouter"
178 + ? grant.destination.model
179 + : undefined;
180 + const [remove, setRemove] = useState(false);
181 + const [busy, setBusy] = useState(false);
182 + const [error, setError] = useState("");
183 + const [model, setModel] = useState(currentModel ?? "");
184 + const saved = AsyncResult.isSuccess(presence) ? presence.value?.[provider] : undefined;
185 + const existing = environment.serverConfig?.settings?.cloudSpeechCredentials?.[provider];
186 + const sources =
187 + isSecretsFeatureEnabled && environment.serverConfig?.secretReferences
188 + ? configuredSecretSources(environment.serverConfig.settings?.secretProviders ?? {})
189 + : [];
190 + const reference =
191 + isSecretReference(existing) && sources.some((s) => s.id === existing.provider)
192 + ? existing
193 + : null;
194 + const [sourceId, setSourceId] = useState<SecretProviderId | null>(reference?.provider ?? null);
195 + const [draft, setDraft] = useState(reference?.reference ?? "");
196 + const source = sources.find((s) => s.id === sourceId);
197 + const credential =
198 + sourceId !== null
199 + ? source && secretRequestReference(source, draft)
200 + : isApiKey(draft.trim())
201 + ? draft.trim()
202 + : undefined;
203 + const keyChanged =
204 + credential !== undefined &&
205 + !(
206 + isSecretReference(credential) &&
207 + reference?.provider === credential.provider &&
208 + reference.reference === credential.reference
209 + );
210 + const connected = canUse && prepared !== null;
211 + // Fixed providers edit the key only when nothing is saved or the device already uses them.
212 + const editingKey = canManage && saved !== undefined && (routed || active || !saved);
213 + const listedModels = AsyncResult.isSuccess(models) ? (models.value?.models ?? null) : null;
214 + const modelChosen = isModelId(model) && (listedModels?.some((m) => m.id === model) ?? false);
215 + const keyReady = draft.trim() ? credential !== undefined : Boolean(saved);
216 + const modelChanged = model !== (currentModel ?? "");
217 + const destination = (): CloudDestination =>
218 + routed && isModelId(model)
219 + ? { kind: "cloud", provider: "openrouter", environmentId: id, model }
220 + : { kind: "cloud", provider: provider as "openai" | "xai", environmentId: id };
221 + const enable = () => chooseAudioDestination(destination());
222 + const canSubmit = routed
223 + ? modelChosen && keyReady && (!active || modelChanged || keyChanged)
224 + : credential !== undefined;
225 +
226 + const saveKey = async (value: SettingsSecretValue | null, thenEnable: boolean) => {
227 + if (!connected || !canManage || busy) return;
228 + setError("");
229 + setBusy(true);
230 + try {
231 + const commit = thenEnable ? pendingCloudAudioChoice(destination()) : null;
232 + await runEnvironmentSpeech(setCloudSpeechKey(prepared, provider, value));
233 + refresh();
234 + commit?.();
235 + close();
236 + } catch {
237 + setError("Couldn’t save this change. Check your connection and try again.");
238 + }
239 + setBusy(false);
240 + };
241 + const submit = () => {
242 + if (!canSubmit || busy) return;
243 + const grantChanges = !active || modelChanged;
244 + if (credential !== undefined && (!routed || keyChanged))
245 + return void saveKey(credential, grantChanges);
246 + // Model-only changes never write to the environment.
247 + if (grantChanges) enable();
248 + close();
249 + };
250 +
251 + const keyField = (
252 + <>
253 + <Label htmlFor="voice-cloud-key">{routed && saved ? "Replace API key" : "API key"}</Label>
254 + <div className="flex min-w-0 items-start gap-2">
255 + {sources.length > 0 && (
256 + <SecretSourceSelect
257 + value={sourceId}
258 + sources={sources}
259 + label={`${company} API key`}
260 + disabled={busy}
261 + onValueChange={(next) => {
262 + setSourceId(next);
263 + setDraft("");
264 + setError("");
265 + }}
266 + />
267 + )}
268 + <div className="min-w-0 flex-1">
269 + {sourceId !== null ? (
270 + <Input
271 + key={sourceId}
272 + id="voice-cloud-key"
273 + name="reference"
274 + aria-label={`${company} API key reference`}
275 + autoFocus={!routed}
276 + placeholder={source?.referencePlaceholder}
277 + value={draft}
278 + disabled={busy}
279 + autoComplete="off"
280 + spellCheck={false}
281 + onChange={(event) => setDraft(event.target.value)}
282 + />
283 + ) : (
284 + <SecretInput
285 + id="voice-cloud-key"
286 + name="key"
287 + aria-label={`${company} API key`}
288 + autoFocus={!routed}
289 + placeholder="Enter API key"
290 + value={draft}
291 + configured={saved === true}
292 + disabled={busy}
293 + maxLength={4096}
294 + onChange={(event) => setDraft(event.target.value)}
295 + />
296 + )}
297 + </div>
298 + </div>
299 + <p className="text-sm text-muted-foreground">
300 + {draft.trim() && !credential
301 + ? (source?.invalidReferenceMessage ?? "Enter an API key, not a URL or secret reference.")
302 + : active && !routed
303 + ? "Updating the key keeps dictation enabled."
304 + : routed && saved
305 + ? "Leave blank to keep the saved key."
306 + : `Anyone with dictation access to ${environment.label} can bill this key.`}
307 + </p>
308 + </>
309 + );
310 +
311 + return (
312 + <Dialog
313 + open
314 + onOpenChange={(open) => {
315 + if (!open && !busy) close();
316 + }}
317 + >
318 + <DialogPopup className="max-w-md" showCloseButton={!busy}>
319 + <DialogHeader>
320 + <DialogTitle>
321 + {remove
322 + ? `Remove the ${company} key?`
323 + : active
324 + ? routed
325 + ? "Manage OpenRouter dictation"
326 + : `Manage ${company} key`
327 + : `Use ${name} for dictation?`}
328 + </DialogTitle>
329 + <DialogDescription>
330 + {remove
331 + ? `Everyone using ${name} for dictation on ${environment.label} will lose access.`
332 + : active && !routed
333 + ? `Anyone with dictation access to ${environment.label} can use this API key.`
334 + : cloudSpeechDisclosure(provider, environment.label)}
335 + </DialogDescription>
336 + </DialogHeader>
337 + {!remove && (
338 + <DialogPanel>
339 + {!connected ? (
340 + <p>Connect to {environment.label} with dictation access to continue.</p>
341 + ) : AsyncResult.isFailure(presence) && saved === undefined ? (
342 + <Button size="sm" variant="outline" onClick={refresh}>
343 + Retry loading API key
344 + </Button>
345 + ) : routed &&
346 + AsyncResult.isSuccess(presence) &&
347 + presence.value &&
348 + saved === undefined ? (
349 + <p>Update {environment.label} to use OpenRouter dictation.</p>
350 + ) : saved === undefined ? (
351 + <p role="status">Checking for an API key…</p>
352 + ) : routed ? (
353 + <form
354 + id={formId}
355 + className="grid gap-3"
356 + onSubmit={(event) => {
357 + event.preventDefault();
358 + submit();
359 + }}
360 + >
361 + <div className="grid gap-2">
362 + <Label htmlFor="voice-cloud-model">Model</Label>
363 + {listedModels ? (
364 + <ModelPicker
365 + models={listedModels}
366 + value={model}
367 + onChange={setModel}
368 + disabled={busy}
369 + current={currentModel}
370 + />
371 + ) : AsyncResult.isFailure(models) ? (
372 + <Button size="sm" variant="outline" type="button" onClick={refreshModels}>
373 + Retry loading models
374 + </Button>
375 + ) : (
376 + <p role="status" className="text-sm text-muted-foreground">
377 + Loading transcription models…
378 + </p>
379 + )}
380 + </div>
381 + {editingKey ? (
382 + <div className="grid gap-2">{keyField}</div>
383 + ) : !saved ? (
384 + <p>Ask an administrator on {environment.label} to add an API key.</p>
385 + ) : (
386 + <p className="text-sm text-muted-foreground">
387 + Uses the API key saved on {environment.label}. This choice applies only to this
388 + device.
389 + </p>
390 + )}
391 + </form>
392 + ) : editingKey ? (
393 + <form
394 + id={formId}
395 + className="grid gap-2"
396 + onSubmit={(event) => {
397 + event.preventDefault();
398 + submit();
399 + }}
400 + >
401 + {keyField}
402 + </form>
403 + ) : !saved ? (
404 + <p>Ask an administrator on {environment.label} to add an API key.</p>
405 + ) : (
406 + <p className="text-sm text-muted-foreground">
407 + This choice applies only to this device. Choose Off to stop sharing.
408 + </p>
409 + )}
410 + </DialogPanel>
411 + )}
412 + {error && (
413 + <p className="px-6 text-sm text-destructive" role="alert">
414 + {error}
415 + </p>
416 + )}
417 + <DialogFooter>
418 + {connected && canManage && saved && (active || routed) && !remove && (
419 + <div className="sm:mr-auto">
420 + <Button
421 + type="button"
422 + size="sm"
423 + variant="ghost"
424 + disabled={busy}
425 + onClick={() => setRemove(true)}
426 + >
427 + Remove key…
428 + </Button>
429 + </div>
430 + )}
431 + <Button type="button" size="sm" variant="outline" disabled={busy} onClick={close}>
432 + Cancel
433 + </Button>
434 + {connected &&
435 + (remove ? (
436 + <Button
437 + type="button"
438 + size="sm"
439 + variant="destructive"
440 + disabled={busy}
441 + onClick={() => void saveKey(null, false)}
442 + >
443 + Remove key
444 + </Button>
445 + ) : routed ? (
446 + saved !== undefined && (saved || canManage) ? (
447 + <Button type="submit" form={formId} size="sm" disabled={busy || !canSubmit}>
448 + {active ? "Save" : "Save and enable"}
449 + </Button>
450 + ) : null
451 + ) : editingKey ? (
452 + <Button type="submit" form={formId} size="sm" disabled={busy || !credential}>
453 + {active ? "Save key" : "Save and enable"}
454 + </Button>
455 + ) : saved && !active ? (
456 + <Button
457 + type="button"
458 + size="sm"
459 + onClick={() => {
460 + enable();
461 + close();
462 + }}
463 + >
464 + Use {name}
465 + </Button>
466 + ) : null)}
467 + </DialogFooter>
468 + </DialogPopup>
469 + </Dialog>
470 + );
471 +}
apps/web/src/components/settings/SecretsSettings.logic.test.ts
+2
@@ -31,6 +31,7 @@ const settings = Schema.decodeSync(ServerSettings)({
31 easDeviceHosts: [
32 { kind: "eas", id: "eas", label: "EAS Simulator", accessToken: personal("op://expo") },
33 ],
34 + cloudSpeechCredentials: { openai: personal("op://voice"), xai: "" },
35 });
36
37 it.each([
@@ -42,6 +43,7 @@ it.each([
43 "GitHub · github.com",
44 "https://hub.test · Management Key",
45 "EAS Simulator · Access Token",
46 + "Voice · OpenAI",
47 ],
48 ],
49 ["work", ["Antigravity Work · OTHER_TOKEN", "GitHub · ghe.acme.test"]],
apps/web/src/components/settings/SecretsSettings.logic.ts
+13 -1
@@ -1,4 +1,6 @@
1 import {
2 + CLOUD_SPEECH_LABELS,
3 + CloudSpeechProvider,
4 EasDeviceHostConfig,
5 isSecretReference,
6 secretReferencesInSettings,
@@ -12,7 +14,11 @@ import { deriveProviderSettingsFields, type SettingsFormDefinition } from "./Pro
14
15 type SecretUseSettings = Pick<
16 ServerSettings,
15 - "providerInstances" | "sourceControlHosts" | "usageLimitSources" | "easDeviceHosts"
17 + | "providerInstances"
18 + | "sourceControlHosts"
19 + | "usageLimitSources"
20 + | "easDeviceHosts"
21 + | "cloudSpeechCredentials"
22 >;
23
24 interface SecretProviderUse {
@@ -86,5 +92,11 @@ export function secretProviderUses(
92 ...settings.easDeviceHosts.flatMap((host) =>
93 uses({ settingsSchema: EasDeviceHostConfig }, `eas:${host.id}`, host.label, host),
94 ),
95 + ...CloudSpeechProvider.literals.flatMap((provider) => {
96 + const credential = settings.cloudSpeechCredentials[provider];
97 + return isSecretReference(credential) && credential.provider === providerId
98 + ? [{ id: `voice:${provider}`, label: `Voice · ${CLOUD_SPEECH_LABELS[provider]}` }]
99 + : [];
100 + }),
101 ];
102 }
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+2 -2
@@ -375,7 +375,7 @@ it.each([
375 );
376 await vi.waitFor(() => expect(document.activeElement?.id).toBe(prerequisite));
377 await expect
378 - .element(screen.getByRole("combobox", { name: "Send microphone audio to" }))
378 + .element(screen.getByRole("combobox", { name: "Transcribe with" }))
379 .toHaveTextContent("Off");
380 await expect
381 .element(screen.getByRole("switch", { name: "Clean up transcripts with a provider" }))
@@ -394,7 +394,7 @@ it("keeps a missing destination selected without claiming it cannot transcribe",
394 </SettingsScopeProvider>,
395 );
396 await expect
397 - .element(screen.getByRole("combobox", { name: "Send microphone audio to" }))
397 + .element(screen.getByRole("combobox", { name: "Transcribe with" }))
398 .toHaveTextContent("Chosen environment (unavailable)");
399 await expect.element(screen.getByText(/can’t transcribe audio/)).not.toBeInTheDocument();
400 });
apps/web/src/components/settings/VoiceSettingsPanel.tsx
+243 -60
@@ -9,13 +9,17 @@ import {
9 updateEnvironmentSpeechModelUnloadTimeout,
10 } from "@t3tools/client-runtime/voice-input";
11 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
12 +import { openRouterIcon } from "@t3tools/client-runtime/provider-clients";
13 import {
14 AuthSettingsWriteScope,
15 + CLOUD_SPEECH_LABELS,
16 + CloudSpeechProvider,
17 DEFAULT_SPEECH_POST_PROCESSING_PROMPT,
18 type EnvironmentId,
19 type EnvironmentSpeechModel,
20 type EnvironmentSpeechStatus,
21 type ProjectId,
22 + ProviderDriverKind,
23 type SpeechAcceleration,
24 type SpeechCustomWords,
25 type SpeechModelUnloadTimeout,
@@ -23,11 +27,15 @@ import {
27 } from "@t3tools/contracts";
28 import { createModelSelection } from "@t3tools/shared/model";
29 import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings";
30 +import { cloudSpeechLanguages } from "@t3tools/shared/speech";
31 import {
32 AudioLinesIcon,
33 CheckIcon,
34 DownloadIcon,
35 LanguagesIcon,
36 + KeyRoundIcon,
37 + MonitorIcon,
38 + MicOffIcon,
39 MicIcon,
40 RefreshCwIcon,
41 StarIcon,
@@ -60,6 +68,8 @@ import {
68 } from "../../speech/useEnvironmentSpeechInput";
69 import { ComposerSpeechRecordingPill } from "../chat/ComposerSpeechButton";
70 import { ProviderModelPicker } from "../chat/ProviderModelPicker";
71 +import { ProviderInstanceIcon } from "../chat/ProviderInstanceIcon";
72 +import { ProviderPackageIcon } from "../chat/ProviderPackageIcon";
73 import { TraitsPicker } from "../chat/TraitsPicker";
74 import { VoiceInputSetup } from "../chat/VoiceInputSetup";
75 import { Badge } from "../ui/badge";
@@ -79,11 +89,20 @@ import {
89 MenuSeparator,
90 MenuTrigger,
91 } from "../ui/menu";
82 -import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
92 +import {
93 + Select,
94 + SelectGroup,
95 + SelectGroupLabel,
96 + SelectItem,
97 + SelectPopup,
98 + SelectTrigger,
99 + SelectValue,
100 +} from "../ui/select";
101 import { Switch } from "../ui/switch";
102 import { Textarea } from "../ui/textarea";
103 import { toastManager } from "../ui/toast";
104 import { MicrophoneTest } from "./MicrophoneTest";
105 +import { CloudVoiceConfiguration } from "./CloudVoiceConfiguration";
106 import { LanguageRow, ModelBenchmarks, orderSpeechLanguages } from "./SpeechModelComparison";
107 import { languageLabel, SpeechLanguageSelect } from "./SpeechLanguageSelect";
108 import { filterSpeechModels, type SpeechModelSort } from "./speechModelBrowser";
@@ -102,6 +121,17 @@ import { useScopedModelDisabledReason } from "./useScopedModelAvailability";
121 import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings";
122
123 const OFF = "off";
124 +function CloudSpeechLogo({ provider }: { provider: CloudSpeechProvider }) {
125 + if (provider === "openrouter")
126 + return <ProviderPackageIcon icon={openRouterIcon} aria-hidden className="size-4 shrink-0" />;
127 + return (
128 + <ProviderInstanceIcon
129 + driverKind={ProviderDriverKind.make(provider === "openai" ? "codex" : "grok")}
130 + displayName={CLOUD_SPEECH_LABELS[provider]}
131 + iconClassName="size-4"
132 + />
133 + );
134 +}
135 const SYSTEM_DEFAULT = "system-default";
136 const formatSize = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`;
137 const SHORTCUT_MODE_LABELS: Record<VoiceShortcutMode, string> = {
@@ -1063,9 +1093,14 @@ export function VoiceSettingsPanel() {
1093 const settings = useClientSettings();
1094 const settingsScope = useOptionalSettingsScope();
1095 const destination = useVoiceTranscriptionDestination();
1096 + const canManageKey = useEnvironmentScope(destination.environmentId, AuthSettingsWriteScope);
1097 + const [pendingCloud, setPendingCloud] = useState<{
1098 + provider: CloudSpeechProvider;
1099 + environmentId: EnvironmentId;
1100 + } | null>(null);
1101 const { environments } = useEnvironments();
1102 const [microphones, setMicrophones] = useState<BrowserMicrophone[]>([]);
1068 - const prepared = destination.prepared;
1103 + const prepared = destination.cloudProvider ? null : destination.prepared;
1104 const catalog = useVoiceModels(prepared);
1105
1106 const refreshMicrophones = useStableCallback(async () => {
@@ -1098,11 +1133,39 @@ export function VoiceSettingsPanel() {
1133 ? (microphones.find((device) => device.deviceId === selectedMicrophone)?.label ??
1134 "Selected microphone (unavailable)")
1135 : "System default";
1101 - const destinationValue = destination.environmentId ?? OFF;
1136 + const environmentChoices = environments.map((environment) => ({
1137 + environment,
1138 + value: JSON.stringify(["environment", environment.environmentId]),
1139 + }));
1140 + const cloudGroups = environments
1141 + .filter(
1142 + (environment) => environment.serverConfig?.environment.capabilities.cloudVoiceTranscription,
1143 + )
1144 + .map((environment) => ({
1145 + environment,
1146 + choices: CloudSpeechProvider.literals.map((provider) => ({
1147 + environment,
1148 + provider,
1149 + value: JSON.stringify(["cloud", environment.environmentId, provider]),
1150 + })),
1151 + }));
1152 + const cloudChoices = cloudGroups.flatMap((group) => group.choices);
1153 + const destinationValue = destination.cloudProvider
1154 + ? JSON.stringify(["cloud", destination.environmentId, destination.cloudProvider])
1155 + : destination.environmentId
1156 + ? JSON.stringify(["environment", destination.environmentId])
1157 + : OFF;
1158 + const cloudEnvironment = environments.find(
1159 + (env) => env.environmentId === pendingCloud?.environmentId,
1160 + );
1161 + // OpenRouter users without settings access still manage their own model choice.
1162 + const manageLabel =
1163 + destination.cloudProvider === "openrouter" ? "Manage OpenRouter" : "Manage key";
1164 const chosenEnvironment = environments.find(
1165 (environment) => environment.environmentId === destination.environmentId,
1166 );
1167 const chosenUnsupported =
1168 + !destination.cloudProvider &&
1169 chosenEnvironment?.serverConfig != null &&
1170 chosenEnvironment.serverConfig.environment.capabilities.voiceTranscription !== true;
1171 const model = prepared && modelStatus?.supported ? modelStatus : null;
@@ -1110,6 +1173,8 @@ export function VoiceSettingsPanel() {
1173 return (
1174 <SettingsPageContainer
1175 searchTargetFallbacks={{
1176 + "speech-language":
1177 + model || destination.cloudProvider ? undefined : "voice-transcription-destination",
1178 "speech-models": prepared ? undefined : "voice-transcription-destination",
1179 "speech-hardware": prepared && model ? undefined : "voice-transcription-destination",
1180 "transcription-test": !prepared
@@ -1196,68 +1261,181 @@ export function VoiceSettingsPanel() {
1261 </SettingsSection>
1262
1263 <SettingsSection id="voice-transcription" title="Transcription">
1199 - <SettingsRow
1264 + <SettingsSearchTarget
1265 {...searchableSetting("voice-transcription-destination")}
1201 - title="Send microphone audio to"
1202 - description={
1203 - <>
1204 - Microphone audio is sent to this environment to transcribe, then discarded. Choose Off
1205 - to turn voice input off on this device. This choice is saved on this device only.
1206 - </>
1207 - }
1208 - control={
1209 - <Select
1210 - value={destinationValue}
1211 - onValueChange={(value) => {
1212 - if (!value) return;
1213 - chooseAudioDestination(
1214 - value === OFF
1215 - ? null
1216 - : { kind: "environment", environmentId: value as EnvironmentId },
1217 - );
1218 - }}
1219 - >
1220 - <SelectTrigger size="sm" aria-label="Send microphone audio to" className="max-w-72">
1221 - <SelectValue>
1222 - {destinationValue === OFF
1223 - ? "Off"
1224 - : (chosenEnvironment?.label ?? "Chosen environment (unavailable)")}
1225 - </SelectValue>
1226 - </SelectTrigger>
1227 - <SelectPopup align="end" alignItemWithTrigger={false}>
1228 - <SelectItem value={OFF}>Off</SelectItem>
1229 - {environments.map((environment) => (
1230 - <SelectItem key={environment.environmentId} value={environment.environmentId}>
1231 - {environment.label}
1232 - </SelectItem>
1233 - ))}
1234 - {destination.environmentId && !chosenEnvironment ? (
1235 - <SelectItem value={destination.environmentId}>
1236 - Chosen environment (unavailable)
1237 - </SelectItem>
1238 - ) : null}
1239 - </SelectPopup>
1240 - </Select>
1241 - }
1266 + className="@container/destination px-3 py-3 sm:px-4"
1267 >
1268 + <div className="flex items-center justify-between gap-2">
1269 + <h3 className="shrink-0 text-sm font-medium">
1270 + <span className="@max-[18rem]/destination:hidden">Transcribe with</span>
1271 + <span className="@min-[18rem]/destination:hidden">Use</span>
1272 + </h3>
1273 + <div className="flex min-w-0 items-center justify-end gap-1">
1274 + <Select
1275 + value={destinationValue}
1276 + onValueChange={(value) => {
1277 + if (!value) return;
1278 + const cloud = cloudChoices.find((choice) => choice.value === value);
1279 + if (cloud) {
1280 + setPendingCloud({
1281 + environmentId: cloud.environment.environmentId,
1282 + provider: cloud.provider,
1283 + });
1284 + return;
1285 + }
1286 + setPendingCloud(null);
1287 + const chosen = environmentChoices.find((choice) => choice.value === value);
1288 + if (value === OFF) chooseAudioDestination(null);
1289 + else if (chosen)
1290 + chooseAudioDestination({
1291 + kind: "environment",
1292 + environmentId: chosen.environment.environmentId,
1293 + });
1294 + }}
1295 + >
1296 + <SelectTrigger
1297 + size="sm"
1298 + aria-label="Transcribe with"
1299 + className="w-auto min-w-0 max-w-72"
1300 + >
1301 + <SelectValue>
1302 + <span className="flex min-w-0 items-center gap-2">
1303 + {destination.cloudProvider ? (
1304 + <CloudSpeechLogo provider={destination.cloudProvider} />
1305 + ) : destination.environmentId ? (
1306 + <MonitorIcon aria-hidden className="size-4" />
1307 + ) : (
1308 + <MicOffIcon aria-hidden className="size-4" />
1309 + )}
1310 + <span className="truncate">
1311 + {destination.cloudProvider
1312 + ? CLOUD_SPEECH_LABELS[destination.cloudProvider]
1313 + : destination.environmentId
1314 + ? (chosenEnvironment?.label ?? "Chosen environment (unavailable)")
1315 + : "Off"}
1316 + {destination.cloudProvider && (
1317 + <span className="text-muted-foreground">
1318 + {" via "}
1319 + {chosenEnvironment?.label ?? "unavailable environment"}
1320 + </span>
1321 + )}
1322 + </span>
1323 + </span>
1324 + </SelectValue>
1325 + </SelectTrigger>
1326 + <SelectPopup align="end" alignItemWithTrigger={false}>
1327 + <SelectItem value={OFF}>
1328 + <span className="flex items-center gap-2">
1329 + <MicOffIcon aria-hidden className="size-4" />
1330 + Off
1331 + </span>
1332 + </SelectItem>
1333 + {environmentChoices.length > 0 && (
1334 + <SelectGroup>
1335 + <SelectGroupLabel>On an environment</SelectGroupLabel>
1336 + {environmentChoices.map(({ environment, value }) => (
1337 + <SelectItem key={value} value={value}>
1338 + <span className="flex items-center gap-2">
1339 + <MonitorIcon aria-hidden className="size-4" />
1340 + {environment.label}
1341 + </span>
1342 + </SelectItem>
1343 + ))}
1344 + </SelectGroup>
1345 + )}
1346 + {cloudGroups.map(({ environment, choices }) => (
1347 + <SelectGroup key={environment.environmentId}>
1348 + <SelectGroupLabel>Cloud via {environment.label}</SelectGroupLabel>
1349 + {choices.map(({ value, provider }) => (
1350 + <SelectItem
1351 + key={value}
1352 + value={value}
1353 + aria-label={`${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}`}
1354 + >
1355 + <span className="flex items-center gap-2">
1356 + <CloudSpeechLogo provider={provider} />
1357 + {CLOUD_SPEECH_LABELS[provider]}
1358 + </span>
1359 + </SelectItem>
1360 + ))}
1361 + </SelectGroup>
1362 + ))}
1363 + {destination.environmentId &&
1364 + (!chosenEnvironment ||
1365 + (destination.cloudProvider &&
1366 + !cloudChoices.some((option) => option.value === destinationValue))) ? (
1367 + <SelectItem value={destinationValue}>
1368 + Chosen environment (unavailable)
1369 + </SelectItem>
1370 + ) : null}
1371 + </SelectPopup>
1372 + </Select>
1373 + {destination.cloudProvider &&
1374 + chosenEnvironment &&
1375 + (canManageKey || destination.cloudProvider === "openrouter") && (
1376 + <Tooltip>
1377 + <TooltipTrigger
1378 + render={
1379 + <Button
1380 + size="icon-sm"
1381 + type="button"
1382 + variant="ghost"
1383 + aria-label={manageLabel}
1384 + onClick={() => {
1385 + if (destination.cloudProvider)
1386 + setPendingCloud({
1387 + provider: destination.cloudProvider,
1388 + environmentId: chosenEnvironment.environmentId,
1389 + });
1390 + }}
1391 + >
1392 + <KeyRoundIcon aria-hidden className="size-4" />
1393 + </Button>
1394 + }
1395 + />
1396 + <TooltipPopup>{manageLabel}</TooltipPopup>
1397 + </Tooltip>
1398 + )}
1399 + </div>
1400 + </div>
1401 {destination.environmentId && chosenUnsupported ? (
1402 <p className="mt-2 text-xs text-destructive">
1245 - {chosenEnvironment.label} can’t transcribe audio. Pick an environment that can to use
1246 - voice input.
1403 + {chosenEnvironment.label} can’t transcribe audio locally. Choose a different
1404 + destination.
1405 </p>
1406 ) : null}
1249 - </SettingsRow>
1407 + </SettingsSearchTarget>
1408
1251 - <SettingsRow
1252 - {...searchableSetting("speech-language")}
1253 - title="Language"
1254 - description={
1255 - model
1256 - ? `${model.model} ${model.supportsLanguageDetection ? "detects the language automatically" : "uses one fixed language"}.`
1257 - : "Choose an environment above to see the languages its transcription model supports."
1258 - }
1259 - control={<SpeechLanguageSelect model={model} />}
1260 - />
1409 + {pendingCloud && cloudEnvironment && (
1410 + <CloudVoiceConfiguration
1411 + key={JSON.stringify(pendingCloud)}
1412 + provider={pendingCloud.provider}
1413 + environment={cloudEnvironment}
1414 + active={
1415 + destination.environmentId === pendingCloud.environmentId &&
1416 + destination.cloudProvider === pendingCloud.provider
1417 + }
1418 + close={() => setPendingCloud(null)}
1419 + />
1420 + )}
1421 + {(model || destination.cloudProvider) && (
1422 + <SettingsRow
1423 + {...searchableSetting("speech-language")}
1424 + title="Language"
1425 + description={
1426 + destination.cloudProvider
1427 + ? "The cloud provider can detect your language automatically."
1428 + : model
1429 + ? `${model.model} ${model.supportsLanguageDetection ? "detects the language automatically" : "uses one fixed language"}.`
1430 + : "Choose an environment above to see the languages its transcription model supports."
1431 + }
1432 + control={
1433 + <SpeechLanguageSelect
1434 + model={destination.cloudProvider ? cloudSpeechLanguages : model}
1435 + />
1436 + }
1437 + />
1438 + )}
1439 {model?.supportsTranslation ? (
1440 <SettingsRow
1441 title="Translate to English"
@@ -1291,10 +1469,15 @@ export function VoiceSettingsPanel() {
1469 <SettingsRow
1470 {...searchableSetting("speech-silence-filtering")}
1471 title="Silence filtering"
1294 - description="Skip silence and other non-speech audio before transcription, keeping a short buffer so words aren’t cut off."
1472 + description={
1473 + destination.cloudProvider === "openai"
1474 + ? "OpenAI handles silence automatically."
1475 + : "Skip silence and other non-speech audio before transcription, keeping a short buffer so words aren’t cut off."
1476 + }
1477 control={
1478 <Switch
1479 aria-label="Silence filtering"
1480 + disabled={destination.cloudProvider === "openai"}
1481 checked={settings.speechSilenceFiltering}
1482 onCheckedChange={(checked) =>
1483 void updateClientSettings({ speechSilenceFiltering: checked })
@@ -1341,7 +1524,7 @@ export function VoiceSettingsPanel() {
1524 <SettingsSection id={searchableSetting("speech-post-processing").id} title="Cleanup">
1525 <SettingsRow
1526 title="Clean up transcripts with a provider"
1344 - description="Sends each transcript and the draft text around your cursor to the draft environment’s text generation provider to fix punctuation, numbers, and spoken corrections. Your audio is never sent."
1527 + description="Sends the transcript and nearby draft text to the text generation provider on the draft’s environment to fix punctuation, numbers, and spoken corrections. Sends text only; off by default."
1528 control={
1529 <Switch
1530 aria-label="Clean up transcripts with a provider"
apps/web/src/components/settings/settingsSearch.ts
+4 -2
@@ -499,9 +499,11 @@ export const SETTINGS_SEARCH_ITEMS = [
499 },
500 {
501 id: "voice-transcription-destination",
502 - title: "Send microphone audio to",
502 + title: "Transcribe with",
503 to: "/settings/voice",
504 - searchTerms: ["speech voice dictation stt transcription environment machine consent privacy"],
504 + searchTerms: [
505 + "send microphone audio dictation destination speech voice stt transcription environment machine consent privacy cloud OpenAI Grok OpenRouter",
506 + ],
507 },
508 {
509 id: "voice-microphone",
apps/web/src/speech/browserVoiceInput.ts
+64 -26
@@ -1,5 +1,7 @@
1 -import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
1 +import { cloudSpeechLanguages, effectiveSpeechLanguage } from "@t3tools/shared/speech";
2 import {
3 + requireCloudSpeechKey,
4 + transcribeCloudPcm,
5 getEnvironmentSpeechStatus,
6 getEnvironmentSpeechStreamUrl,
7 openSpeechStream,
@@ -10,12 +12,19 @@ import {
12 } from "@t3tools/client-runtime/voice-input";
13 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
14 import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";
15 +import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts";
16 +import * as Schema from "effect/Schema";
17
14 -import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
18 import { createBrowserPcmCapture } from "./browserPcmCapture";
19 +import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
20 +
21 +const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
22
23 export function createBrowserVoiceInputPlatform(input: {
24 readonly prepared: PreparedConnection;
25 + readonly cloudProvider?: CloudSpeechProvider | undefined;
26 + /** The exact OpenRouter model the grant names; required with that provider. */
27 + readonly cloudModel?: CloudSpeechModelId | undefined;
28 readonly hasAudioConsent: () => boolean;
29 readonly getTranscriptionOptions: () => SpeechTranscriptionOptions;
30 readonly getMicrophoneId: () => string;
@@ -26,25 +35,6 @@ export function createBrowserVoiceInputPlatform(input: {
35 }) {
36 return createBrowserPcmCapture(input, async (options) => {
37 const recordingOptions = input.getTranscriptionOptions();
29 - const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
30 - throwIfVoiceTranscriptionAborted(options.signal);
31 - if (!status.supported)
32 - throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
33 - const language = status.languages
34 - ? effectiveSpeechLanguage(
35 - {
36 - languages: status.languages,
37 - supportsLanguageDetection: status.supportsLanguageDetection ?? false,
38 - },
39 - recordingOptions.speechLanguage,
40 - )
41 - : "auto";
42 - const locale =
43 - !status.supportsStreaming &&
44 - status.supportsTranslation &&
45 - recordingOptions.speechTranslateToEnglish
46 - ? "en"
47 - : language;
38 const transcribeRecording = async (
39 uri: string,
40 { signal: transcriptionSignal }: VoiceTranscriptionOptions,
@@ -53,22 +43,70 @@ export function createBrowserVoiceInputPlatform(input: {
43 const response = await fetch(uri, { signal: transcriptionSignal });
44 const pcm = new Uint8Array(await response.arrayBuffer());
45 const result = await runEnvironmentSpeech(
56 - transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent),
57 - {
58 - signal: transcriptionSignal,
59 - },
46 + input.cloudProvider
47 + ? transcribeCloudPcm(
48 + input.prepared,
49 + input.cloudProvider,
50 + pcm,
51 + recordingOptions,
52 + input.hasAudioConsent,
53 + input.cloudModel,
54 + )
55 + : transcribeEnvironmentPcm(
56 + input.prepared,
57 + pcm,
58 + recordingOptions,
59 + input.hasAudioConsent,
60 + ),
61 + { signal: transcriptionSignal },
62 );
63 throwIfVoiceTranscriptionAborted(transcriptionSignal);
64 return result.text;
65 } catch (cause) {
66 throwIfVoiceTranscriptionAborted(transcriptionSignal);
67 + if (isVoiceTranscriptionError(cause)) throw cause;
68 throw new VoiceTranscriptionError({
69 code: "transcription-failed",
67 - message: "Voice transcription on this environment failed.",
70 + message: "Voice transcription failed. Check the destination in Settings → Voice.",
71 cause,
72 });
73 }
74 };
75 + if (input.cloudProvider) {
76 + if (input.cloudProvider === "openrouter" && !input.cloudModel)
77 + throw new VoiceTranscriptionError({
78 + code: "unavailable",
79 + message: "Choose an OpenRouter model in Settings → Voice.",
80 + });
81 + await runEnvironmentSpeech(
82 + requireCloudSpeechKey(input.prepared, input.cloudProvider),
83 + options,
84 + );
85 + throwIfVoiceTranscriptionAborted(options.signal);
86 + return {
87 + locale: effectiveSpeechLanguage(cloudSpeechLanguages, recordingOptions.speechLanguage),
88 + transcribe: transcribeRecording,
89 + };
90 + }
91 + const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
92 + throwIfVoiceTranscriptionAborted(options.signal);
93 + if (!status.supported)
94 + throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
95 + const language = status.languages
96 + ? effectiveSpeechLanguage(
97 + {
98 + languages: status.languages,
99 + supportsLanguageDetection: status.supportsLanguageDetection ?? false,
100 + },
101 + recordingOptions.speechLanguage,
102 + )
103 + : "auto";
104 + const locale =
105 + !status.supportsStreaming &&
106 + status.supportsTranslation &&
107 + recordingOptions.speechTranslateToEnglish
108 + ? "en"
109 + : language;
110 if (!status.supportsStreaming && !status.supportsBufferedStreaming) {
111 return { locale, transcribe: transcribeRecording };
112 }
apps/web/src/speech/runEnvironmentSpeech.ts
+9 -6
@@ -6,14 +6,17 @@ import { connectionAtomRuntime } from "../connection/runtime";
6 import { appAtomRegistry } from "../rpc/atomRegistry";
7
8 // Relay (DPoP) environments authorize through services only the connection runtime provides.
9 +export function inEnvironmentSpeechRuntime<A, E>(
10 + effect: Effect.Effect<A, E, HttpClient.HttpClient>,
11 +) {
12 + return AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
13 + Effect.flatMap((context) => Effect.provide(effect, context)),
14 + );
15 +}
16 +
17 export function runEnvironmentSpeech<A, E>(
18 effect: Effect.Effect<A, E, HttpClient.HttpClient>,
19 options?: Effect.RunOptions,
20 ): Promise<A> {
13 - return Effect.runPromise(
14 - AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
15 - Effect.flatMap((context) => Effect.provide(effect, context)),
16 - ),
17 - options,
18 - );
21 + return Effect.runPromise(inEnvironmentSpeechRuntime(effect), options);
22 }
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+108 -13
@@ -6,7 +6,7 @@ import type { PreparedConnection } from "@t3tools/client-runtime/connection";
6 import {
7 DEFAULT_CLIENT_SETTINGS,
8 DEFAULT_SERVER_SETTINGS,
9 - type EnvironmentId,
9 + EnvironmentId,
10 type ProjectId,
11 type SpeechCustomWords,
12 type SpeechPostProcessingOptions,
@@ -14,8 +14,14 @@ import {
14 } from "@t3tools/contracts";
15
16 import { act, render } from "../../test/render";
17 +import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
18 import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
18 -import { chooseAudioDestination, chooseTranscriptCleanup, bindAudioGrant } from "./voiceConsent";
19 +import {
20 + chooseAudioDestination,
21 + chooseTranscriptCleanup,
22 + bindAudioGrant,
23 + pendingCloudAudioChoice,
24 +} from "./voiceConsent";
25 import { writeBrowserClientSettings } from "../clientPersistenceStorage";
26
27 const mocks = vi.hoisted(() => ({
@@ -23,6 +29,7 @@ const mocks = vi.hoisted(() => ({
29 settings: {} as Record<string, unknown>,
30 connected: new Set<string>(),
31 voiceCapable: new Set<string>(),
32 + cloudVoiceCapable: new Set<string>(),
33 canDictate: new Set<string>(),
34 prepared: new Map<string, unknown>(),
35 primaryEnvironmentId: "primary-env",
@@ -63,7 +70,10 @@ vi.mock("../state/environments", () => ({
70 environmentId,
71 serverConfig: {
72 environment: {
66 - capabilities: { voiceTranscription: mocks.voiceCapable.has(environmentId) },
73 + capabilities: {
74 + voiceTranscription: mocks.voiceCapable.has(environmentId),
75 + cloudVoiceTranscription: mocks.cloudVoiceCapable.has(environmentId),
76 + },
77 },
78 },
79 }
@@ -124,12 +134,10 @@ vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
134 return { text: `Cleaned ${transcript}` };
135 }),
136 }));
127 -// Capture and the wire live behind the platform; its `prepared` is where audio would go.
137 vi.mock("./browserVoiceInput", () => ({
129 - createBrowserVoiceInputPlatform: (input: {
130 - prepared: PreparedConnection;
131 - getTranscriptionOptions?: () => SpeechTranscriptionOptions;
132 - }) => {
138 + createBrowserVoiceInputPlatform: (
139 + input: Parameters<typeof createBrowserVoiceInputPlatform>[0],
140 + ) => {
141 const environmentId = input.prepared.environmentId;
142 mocks.platforms.push(environmentId);
143 return {
@@ -143,12 +151,15 @@ vi.mock("./browserVoiceInput", () => ({
151 },
152 transcriber: {
153 prepare: async () => {
146 - const options = input.getTranscriptionOptions?.();
147 - if (options) mocks.recordingOptions.push(options);
154 + mocks.recordingOptions.push(input.getTranscriptionOptions());
155 return {
156 locale: "en",
157 transcribe: async () => {
151 - mocks.transcribeCalls.push(environmentId);
158 + mocks.transcribeCalls.push(
159 + input.cloudProvider
160 + ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}`
161 + : environmentId,
162 + );
163 const next = mocks.transcripts.shift() ?? "dictated";
164 if (next instanceof Error) throw next;
165 return next;
@@ -171,6 +182,7 @@ function setup(options: { readonly destination?: string | null } = {}) {
182 mocks.primaryEnvironmentId = "primary-env";
183 mocks.connected = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
184 mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
185 + mocks.cloudVoiceCapable.clear();
186 mocks.canDictate = new Set(mocks.voiceCapable);
187 mocks.prepared.clear();
188 mocks.statusRequests = [];
@@ -367,7 +379,15 @@ it("revokes across windows even before the storage event and survives stale pref
379 const grant = JSON.parse(storage.getItem(key)!) as { id: string };
380 const canSend = bindAudioGrant(grant.id);
381 expect(canSend()).toBe(true);
382 + const saved = pendingCloudAudioChoice({
383 + kind: "cloud",
384 + provider: "openai",
385 + environmentId: EnvironmentId.make("env-a"),
386 + });
387 + expect(canSend()).toBe(true);
388 storage.removeItem(key);
389 + saved();
390 + expect(storage.getItem(key)).toBeNull();
391 expect(canSend()).toBe(false);
392 writeBrowserClientSettings({ ...DEFAULT_CLIENT_SETTINGS, wordWrap: false });
393 await vi.waitFor(() => expect(mocks.voice.available).toBe(false));
@@ -381,15 +401,89 @@ it("revokes across windows even before the storage event and survives stale pref
401 }
402 });
403
384 -it("cleans the transcript only after cleanup is separately enabled", async () => {
404 +it("recovers from a corrupt choice without enabling cloud dictation before the save completes", async () => {
405 + setup();
406 + mocks.cloudVoiceCapable.add("env-b");
407 + mocks.modelState = "missing-model";
408 + window.localStorage.setItem("t3code:voice-audio-grant:v1", "not-json");
409 + const save = pendingCloudAudioChoice({
410 + kind: "cloud",
411 + environmentId: EnvironmentId.make("env-b"),
412 + provider: "xai",
413 + });
414 + await using _screen = await render(<Probe />);
415 + expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
416 + await act(() => mocks.voice.start());
417 + expectNoAudioSent();
418 + await act(save);
419 + await act(() => mocks.voice.start());
420 + await act(() => mocks.voice.stop());
421 + expect(mocks.transcribeCalls).toEqual(["xai via env-b"]);
422 + expect(mocks.statusRequests).toEqual([]);
423 +});
424 +
425 +it("sends audio to OpenRouter only under a device grant naming the exact model", async () => {
426 + setup();
427 + mocks.cloudVoiceCapable.add("env-b");
428 + const key = "t3code:voice-audio-grant:v1";
429 + const destination = { kind: "cloud", environmentId: "env-b", provider: "openrouter" };
430 + window.localStorage.setItem(key, JSON.stringify({ id: "no-model", destination }));
431 + {
432 + await using _screen = await render(<Probe />);
433 + await act(() => mocks.voice.start());
434 + expectNoAudioSent();
435 + }
436 + const model = "acme/transcribe-1";
437 + window.localStorage.setItem(
438 + key,
439 + JSON.stringify({ id: "model", destination: { ...destination, model } }),
440 + );
441 + await using _screen = await render(<Probe />);
442 + await act(() => mocks.voice.start());
443 + await act(() => mocks.voice.stop());
444 + expect(mocks.transcribeCalls).toEqual([`openrouter:${model} via env-b`]);
445 + expect(mocks.statusRequests).toEqual([]);
446 +});
447 +
448 +it("uses neither destination when the cloud save marker cannot be read", async () => {
449 + setup({ destination: "env-a" });
450 + const key = "t3code:voice-audio-grant:v1";
451 + const previous = JSON.parse(window.localStorage.getItem(key)!) as { id: string };
452 + const canSendToPrevious = bindAudioGrant(previous.id);
453 + const save = pendingCloudAudioChoice({
454 + kind: "cloud",
455 + environmentId: EnvironmentId.make("env-b"),
456 + provider: "openai",
457 + });
458 + save();
459 + const getItem = Storage.prototype.getItem;
460 + using _blocked = vi.spyOn(Storage.prototype, "getItem").mockImplementation(function (
461 + this: Storage,
462 + k,
463 + ) {
464 + if (k.startsWith(`${key}:saved:`))
465 + throw new DOMException("Storage is blocked", "SecurityError");
466 + return getItem.call(this, k);
467 + });
468 + await using _screen = await render(<Probe />);
469 + expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
470 + expect(canSendToPrevious()).toBe(false);
471 + await act(() => mocks.voice.start());
472 + expectNoAudioSent();
473 +});
474 +
475 +it("cleans text on a cloud-only draft environment only while cleanup is separately enabled", async () => {
476 setup({ destination: "env-a" });
477 + mocks.voiceCapable.delete(THREAD_ENV);
478 + mocks.cloudVoiceCapable.add(THREAD_ENV);
479 chooseTranscriptCleanup(true);
480 const nearby = "x".repeat(1_024);
481 mocks.draft = "unrelated private paragraph " + nearby;
482 await using screen = await render(<Probe />);
483 await act(() => mocks.voice.start());
484 await act(() => mocks.voice.stop());
392 - expect(mocks.cleanupCalls).toHaveLength(1);
485 + expect(mocks.transcribeCalls).toEqual(["env-a"]);
486 + expect(mocks.cleanupCalls).toEqual([THREAD_ENV]);
487 expect(mocks.cleanupDraft).toHaveBeenCalledExactlyOnceWith({
488 text: nearby,
489 selection: { start: nearby.length, end: nearby.length },
@@ -414,6 +508,7 @@ it.each([
508 "keeps a recording without authorized cleanup off the provider: $cleanup / $authorized",
509 async ({ cleanup, authorized }) => {
510 setup({ destination: "env-a" });
511 + mocks.cloudVoiceCapable.add(THREAD_ENV);
512 chooseTranscriptCleanup(true);
513 mocks.settings.speechCorrectionWord = "sorry";
514 if (!authorized) mocks.canDictate.delete(THREAD_ENV);
apps/web/src/speech/useEnvironmentSpeechInput.ts
+32 -11
@@ -11,7 +11,7 @@ import {
11 type VoiceInputState,
12 } from "@t3tools/client-runtime/voice-input";
13 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
14 -import { AuthOrchestrationOperateScope } from "@t3tools/contracts";
14 +import { AuthOrchestrationOperateScope, CLOUD_SPEECH_LABELS } from "@t3tools/contracts";
15 import type {
16 EnvironmentId,
17 EnvironmentSpeechModel,
@@ -109,11 +109,22 @@ export function useVoiceTranscriptionDestination() {
109 const environment = useEnvironment(environmentId);
110 const connection = Option.getOrNull(usePreparedConnection(environmentId));
111 const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope);
112 - const supported = environment?.serverConfig?.environment.capabilities.voiceTranscription === true;
112 + const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined;
113 + const provider = cloud?.provider;
114 + const model = cloud?.provider === "openrouter" ? cloud.model : undefined;
115 + const supported =
116 + environment?.serverConfig?.environment.capabilities[
117 + provider ? "cloudVoiceTranscription" : "voiceTranscription"
118 + ] === true;
119 return {
120 environmentId,
121 + ...(provider ? { cloudProvider: provider } : {}),
122 + ...(model ? { cloudModel: model } : {}),
123 grantId: grant?.id ?? null,
116 - label: environment?.label ?? null,
124 + label:
125 + provider && environment
126 + ? `${CLOUD_SPEECH_LABELS[provider]} via ${environment.label}`
127 + : (environment?.label ?? null),
128 prepared: supported && canDictate ? connection : null,
129 };
130 }
@@ -122,12 +133,15 @@ export function useEnvironmentSpeechInput(input: HookInput) {
133 const destination = useVoiceTranscriptionDestination();
134 const prepared = destination.prepared;
135 const grantId = destination.grantId;
125 - const originatingEnvironment = useEnvironment(input.environmentId);
136 + const cloudProvider = destination.cloudProvider;
137 + const cloudModel = destination.cloudModel;
138 + const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment
139 + .capabilities;
140 const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId));
141 const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope);
142 const cleanupPrepared =
143 canCleanUp &&
130 - originatingEnvironment?.serverConfig?.environment.capabilities.voiceTranscription === true
144 + (draftCapabilities?.voiceTranscription || draftCapabilities?.cloudVoiceTranscription)
145 ? originatingConnection
146 : null;
147 const clientSettings = useClientSettings();
@@ -231,6 +245,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {
245 };
246 const platform = createBrowserVoiceInputPlatform({
247 prepared,
248 + cloudProvider,
249 + cloudModel,
250 hasAudioConsent,
251 getTranscriptionOptions: (): SpeechTranscriptionOptions => {
252 const {
@@ -344,11 +360,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
360 controller.dispose();
361 if (controllerRef.current === controller) controllerRef.current = null;
362 };
347 - }, [prepared, grantId]);
363 + }, [prepared, grantId, cloudProvider, cloudModel]);
364
365 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available
366 useEffect(() => {
351 - if (!prepared) return;
367 + if (!prepared || cloudProvider) return;
368 let disposed = false;
369 void probeSpeechStatus(prepared)
370 .then((value) => {
@@ -360,8 +376,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {
376 return () => {
377 disposed = true;
378 };
363 - }, [prepared]);
364 - const currentStatus = status?.prepared === prepared ? status.value : null;
379 + }, [prepared, cloudProvider]);
380 + const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null;
381
382 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download
383 useEffect(() => {
@@ -425,6 +441,11 @@ export function useEnvironmentSpeechInput(input: HookInput) {
441 const controller = controllerRef.current;
442 const owner = latestRef.current.input.ownerKey;
443 if (latestRef.current.input.disabled || !controller || !prepared) return;
444 + if (cloudProvider) {
445 + setLevel(0);
446 + await controller.start();
447 + return;
448 + }
449 const stillCurrent = () =>
450 request === startRequestRef.current &&
451 controllerRef.current === controller &&
@@ -511,8 +532,8 @@ export function useEnvironmentSpeechInput(input: HookInput) {
532 return {
533 available:
534 prepared !== null &&
514 - status?.prepared === prepared &&
515 - status.value?.supported !== false &&
535 + (cloudProvider !== undefined ||
536 + (status?.prepared === prepared && status.value?.supported !== false)) &&
537 typeof navigator !== "undefined" &&
538 Boolean(navigator.mediaDevices?.getUserMedia) &&
539 typeof AudioWorkletNode !== "undefined",
apps/web/src/speech/voiceConsent.ts
+73 -12
@@ -1,8 +1,11 @@
1 import { VoiceTranscriptionDestination } from "@t3tools/contracts";
2 import * as Schema from "effect/Schema";
3 +import { useSyncExternalStore } from "react";
4 import { randomUUID } from "../lib/utils";
5 import {
6 + getLocalStorageItem,
7 readLocalStorageValue,
8 + subscribeLocalStorageKey,
9 useLocalStorage,
10 writeLocalStorageValue,
11 } from "../hooks/useLocalStorage";
@@ -10,33 +13,91 @@ import {
13 // Separate keys: a stale window saving ordinary preferences must not restore either grant.
14 const AUDIO_KEY = "t3code:voice-audio-grant:v1";
15 const CLEANUP_KEY = "t3code:voice-cleanup-grant:v1";
13 -const AudioGrant = Schema.NullOr(
14 - Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }),
16 +const Grant = Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination });
17 +const AudioGrant = Schema.NullOr(Grant);
18 +const sameGrant = Schema.toEquivalence(AudioGrant);
19 +const AudioChoice = Schema.NullOr(
20 + Schema.Union([Grant, Schema.Struct({ pending: Grant, previous: AudioGrant })]),
21 );
22 +type AudioChoice = typeof AudioChoice.Type;
23 +const savedKey = (id: string) => `${AUDIO_KEY}:saved:${id}`;
24 +const pendingGrant = (choice: AudioChoice) =>
25 + choice && "pending" in choice ? choice.pending : null;
26 const CleanupGrant = Schema.NullOr(Schema.String);
17 -const decodeAudioGrant = Schema.decodeSync(Schema.fromJsonString(AudioGrant));
27 +const decodeChoice = Schema.decodeSync(Schema.fromJsonString(AudioChoice));
28 +const readChoice = () => readLocalStorageValue(AUDIO_KEY, null, AudioChoice);
29 +
30 +function readSaveMarker(id: string | undefined) {
31 + try {
32 + return id ? window.localStorage.getItem(savedKey(id)) : null;
33 + } catch {
34 + return undefined;
35 + }
36 +}
37 +
38 +function activeGrant(choice: AudioChoice, marker: string | null | undefined) {
39 + if (!choice || !("pending" in choice)) return choice;
40 + // Only a confirmed missing marker means the previous grant still applies.
41 + if (marker === null) return choice.previous;
42 + return marker === "true" ? choice.pending : null;
43 +}
44
45 export function useAudioGrant() {
20 - return useLocalStorage(AUDIO_KEY, null, AudioGrant)[0];
46 + const [choice] = useLocalStorage(AUDIO_KEY, null, AudioChoice);
47 + const id = pendingGrant(choice)?.id;
48 + const marker = useSyncExternalStore(
49 + (notify) => (id ? subscribeLocalStorageKey(savedKey(id), notify) : () => {}),
50 + () => readSaveMarker(id),
51 + () => undefined,
52 + );
53 + return activeGrant(choice, marker);
54 +}
55 +
56 +function replaceChoice(choice: AudioChoice, previous = readChoice()) {
57 + const old = pendingGrant(previous);
58 + writeLocalStorageValue(AUDIO_KEY, choice, null, AudioChoice);
59 + if (old && pendingGrant(getLocalStorageItem(AUDIO_KEY, AudioChoice))?.id !== old.id)
60 + window.localStorage.removeItem(savedKey(old.id));
61 }
62
63 export function chooseAudioDestination(destination: VoiceTranscriptionDestination | null) {
24 - writeLocalStorageValue(
25 - AUDIO_KEY,
26 - destination && { id: randomUUID(), destination },
27 - null,
28 - AudioGrant,
64 + replaceChoice(destination && { id: randomUUID(), destination });
65 +}
66 +
67 +export function pendingCloudAudioChoice(
68 + destination: Extract<VoiceTranscriptionDestination, { kind: "cloud" }>,
69 +) {
70 + const choice = readChoice();
71 + const pending = { id: randomUUID(), destination };
72 + replaceChoice(
73 + { pending, previous: activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id)) },
74 + choice,
75 );
76 + // A late response can acknowledge only its own choice, never write the destination another tab picked.
77 + return () => {
78 + if (pendingGrant(readChoice())?.id === pending.id)
79 + writeLocalStorageValue(savedKey(pending.id), true, false, Schema.Boolean);
80 + };
81 }
82
83 // Bind this recording to this act of consent, even if the user later reselects the same destination.
84 export function bindAudioGrant(id: string) {
85 try {
35 - const raw = window.localStorage.getItem(AUDIO_KEY);
36 - if (!raw || decodeAudioGrant(raw)?.id !== id) return () => false;
86 + let cachedRaw: string | null = null;
87 + let choice: AudioChoice = null;
88 + const current = () => {
89 + const raw = window.localStorage.getItem(AUDIO_KEY);
90 + if (raw !== cachedRaw) {
91 + choice = raw ? decodeChoice(raw) : null;
92 + cachedRaw = raw;
93 + }
94 + return activeGrant(choice, readSaveMarker(pendingGrant(choice)?.id));
95 + };
96 + const bound = current();
97 + if (bound?.id !== id) return () => false;
98 return () => {
99 try {
39 - return window.localStorage.getItem(AUDIO_KEY) === raw;
100 + return sameGrant(current(), bound);
101 } catch {
102 return false;
103 }
docs/internals/voice-input.md
+33 -18
@@ -2,8 +2,9 @@
2
3 Transcription edits a composer draft. It does not submit an agent turn. Audio is
4 temporary client input, and only normal message submission sends the resulting
5 -text. Mobile defaults to on-device transcription; any client that sends audio off
6 -the device needs a grant for one environment.
5 +text to the agent. Mobile defaults to on-device transcription. Sending audio off
6 +the device requires an explicit grant naming one environment and, for cloud
7 +transcription, one provider.
8
9 The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts)
10 owns the operation while the client supplies capture and transcription. Preparation
@@ -15,28 +16,42 @@ capture and `finish` replaces the file-based `transcribe`.
16
17 ## Where audio may go
18
18 -Only an explicit audio grant authorizes off-device transcription. On mobile the
19 -grant is stored in device-bound secure storage, outside preference backups. The
20 -recorder is the same for both destinations; only a finished recording can be
21 -uploaded, and its grant is checked again after native decoding and before dispatch.
19 +On mobile, on-device transcription needs no audio grant. Environment and cloud
20 +transcription share a recorder and upload only finished recordings. The client
21 +keeps the grant in device-bound secure storage, outside preference backups, and
22 +rechecks it after native decoding and before upload.
23 +
24 The [web grant](../../apps/web/src/speech/voiceConsent.ts) lives outside the preferences snapshot so another
23 -window saving unrelated settings cannot resurrect a revoked grant. Each new
24 -choice gets a distinct grant, even if the destination was previously selected. The
25 -[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds a
26 -recording controller to the prepared connection of that destination and to
27 -nothing else. When the destination changes, disconnects, or stops advertising the
28 -`voiceTranscription` capability, the controller is disposed: a recording in
29 -progress is discarded, a failed recording kept for retry is deleted, and nothing
30 -falls back to another environment. Every sender in the web app, including the
31 -settings page's transcription test, goes through that hook. The microphone loopback
25 +window saving unrelated settings cannot resurrect a revoked grant. Each new choice
26 +gets a distinct grant, even if the destination was previously selected. The
27 +[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds the
28 +controller to that grant and the selected environment's connection. It requires
29 +`voiceTranscription` to use the environment's model or `cloudVoiceTranscription`
30 +to forward audio to the selected provider. Changing the grant or losing the
31 +connection, permission, or required capability discards both the active recording
32 +and any recording retained for retry. It never chooses a fallback destination.
33 +The settings page's transcription test uses the same hook. The microphone loopback
34 test captures separately for local playback and sends nothing. The transport
35 refuses redirects so credentials and audio cannot follow one to another host.
36
37 Transcript cleanup requires its own grant, off by default.
38 It sends text, never audio, to the text generation provider of the draft's
37 -own environment, because that is where the provider credentials live. A later
38 -backend adds its own destination kind, disclosure, and transcriber; capture and
39 -the controller stay as they are.
39 +own environment, because that is where the provider credentials live.
40 +
41 +A cloud destination binds both the forwarding environment and the named provider.
42 +It uses a separate, batch-only endpoint: changing a server key cannot upgrade
43 +an environment-local grant, and a cloud recording never uses the streaming
44 +socket or falls back to another provider or the environment's native model.
45 +
46 +OpenRouter routes to many upstreams, so its grant also names the exact model and
47 +the model travels with every upload as a query parameter. The server accepts only
48 +ids present in OpenRouter's public ZDR-eligible transcription catalog,
49 +which it caches for ten minutes after a successful read;
50 +router aliases such as `openrouter/auto` or `:nitro` fail schema validation before
51 +that. Requests set `provider.zdr` and carry no `models` fallback list. OpenRouter
52 +applies the account's safety-retention policy when choosing an endpoint. The catalog
53 +holds no credential or consent state, which is why it may be cached while key
54 +presence is read fresh on every request.
55
56 ## Server side
57
docs/user/composer.md
+2
@@ -203,6 +203,8 @@ choose the environment, and confirm. Finished recordings go only to that environ
203 even when a draft belongs to another one. This also works on phones without local
204 speech recognition. If the chosen environment disconnects, dictation stops; it never
205 switches hosts. Choose **This device** to revoke access and discard pending audio.
206 +For [cloud dictation](./voice-input.md#choose-where-audio-goes), open **Cloud providers**
207 +under the environment in **Settings → Voice** and confirm which provider may receive recordings.
208 On Android, keep T3 Code open while recording and transcribing.
209
210 On-device dictation may need a network connection to download a speech model the
docs/user/voice-input.md
+27 -4
@@ -12,9 +12,9 @@ to a named environment on that phone, open **Settings → Voice → Transcribe o
12
13 ## Choose where audio goes
14
15 -On web and desktop, transcription runs on one of your T3 Code environments.
15 +On web and desktop, you choose where transcription runs.
16 Nothing is sent until you choose one: open **Settings → Voice** and set
17 -**Send microphone audio to** the environment that should transcribe for you.
17 +**Transcribe with** to the environment that should transcribe for you.
18 Microphone audio recorded on this device is sent to that environment over your
19 authenticated connection, transcribed there, and discarded. The choice is saved
20 on this device only, so every computer and browser you use asks for it once.
@@ -23,10 +23,33 @@ Set it back to **Off** at any time; a recording in progress is discarded.
23 If the chosen environment goes offline, voice input waits for it. T3 Code never
24 substitutes another environment.
25
26 +For cloud dictation, choose **OpenAI** or **Grok Voice Transcribe** via the environment
27 +you want to use. If it needs a key, enter one and choose **Save and enable**.
28 +You can also choose a [secret provider](secrets.md) instead of **Value** and enter a
29 +reference. Unlock the vault on that environment before dictating. Anyone with
30 +dictation access to the environment can use the key, but only this device opts in.
31 +If a key is already saved, just confirm the provider. After
32 +you stop, the environment forwards your recording and any vocabulary hints to
33 +the selected provider, which bills its API account. No local model download or chat
34 +subscription is needed. Switch back to **This device** on mobile or **Off** on
35 +web and desktop to stop sharing. Removing the key stops that provider for everyone
36 +on the environment.
37 +
38 +**OpenRouter** works the same way but has no default model: pick one from its live
39 +speech-to-text catalog when you enable it, or later under **Manage OpenRouter**.
40 +The model choice is yours alone and needs no settings access, so anyone with
41 +dictation access can pick a different model without touching the saved key.
42 +OpenRouter chooses who runs the model. T3 requests zero data retention; your
43 +OpenRouter account can also allow safety retention. If no endpoint qualifies,
44 +dictation fails instead of switching models.
45 +Vocabulary hints are not sent to OpenRouter; dictionary spellings are applied
46 +after transcription. If OpenRouter stops listing your model, dictation stops
47 +until you choose another.
48 +
49 ## Set up a model
50
28 -The first time you dictate, T3 Code offers to download the recommended English
29 -model to the chosen environment. In **Settings → Voice → Transcription**, select
51 +If you choose environment transcription, T3 Code offers to download the recommended
52 +English model to that environment. In **Settings → Voice → Transcription**, select
53 the languages you dictate in to compare models that support all of them.
54 Accuracy shows how often the model got words or characters wrong in each
55 language; speed is the time to transcribe 30 seconds on a reference GPU. Choose
packages/client-runtime/src/providerClients.ts
+12 -1
@@ -6,7 +6,7 @@ import {
6 ProviderDriverKind,
7 } from "@t3tools/contracts";
8 import { acpRegistryClient } from "@t3tools/provider-acp-registry/client";
9 -import { makeProviderClientRegistry } from "@t3tools/provider-core/client";
9 +import { makeProviderClientRegistry, type ProviderIcon } from "@t3tools/provider-core/client";
10 import { modelShortName } from "@t3tools/provider-core/modelShortName";
11 import { cursorClient } from "@t3tools/provider-cursor/client";
12 import { fxClient } from "@t3tools/provider-fx/client";
@@ -16,6 +16,17 @@ import { museClient } from "@t3tools/provider-muse/client";
16 import { openCodeClient } from "@t3tools/provider-opencode/client";
17 import { piClient } from "@t3tools/provider-pi/client";
18
19 +// OpenRouter's Ink and Cloud glyphs from https://openrouter.ai/brand.
20 +export const openRouterIcon: ProviderIcon = {
21 + viewBox: "0 0 1024 730",
22 + fill: { light: "#03080A", dark: "#FCFCFE" },
23 + paths: [
24 + {
25 + d: "M795.893 0C915.776 0 1012.95 97.9963 1012.95 218.88C1012.95 339.764 915.776 437.76 795.893 437.76L1011.2 654.869C1038.55 682.447 1019.18 729.6 980.504 729.6H361.77C161.97 729.6 0 566.273 0 364.8C0 163.327 161.97 0 361.77 0L795.893 0ZM361.77 145.92C241.89 145.92 144.708 243.916 144.708 364.8C144.708 485.684 241.89 583.68 361.77 583.68C481.649 583.68 578.831 485.684 578.831 364.8C578.831 243.916 481.649 145.92 361.77 145.92Z",
26 + },
27 + ],
28 +};
29 +
30 export const providerClients = makeProviderClientRegistry([
31 {
32 driverKind: ProviderDriverKind.make("codex"),
packages/client-runtime/src/secretRequest.ts
+6
@@ -95,6 +95,12 @@ export function secretRequestSources(
95 providers: ServerSettings["secretProviders"],
96 ): ReadonlyArray<SecretRequestSource> {
97 if (config?.secretRequestReferences !== true) return NO_SOURCES;
98 + return configuredSecretSources(providers);
99 +}
100 +
101 +export function configuredSecretSources(
102 + providers: ServerSettings["secretProviders"],
103 +): ReadonlyArray<SecretRequestSource> {
104 return (Object.keys(providers) as Array<SecretProviderId>).flatMap((id) => {
105 const provider = providers[id];
106 const definition =
packages/client-runtime/src/voice-input/environment.ts
+95 -1
@@ -1,8 +1,12 @@
1 import { encodeSpeechPcmRequest } from "@t3tools/shared/speech";
2 import * as Effect from "effect/Effect";
3 import * as Redacted from "effect/Redacted";
4 -import { SPEECH_STREAM_PATH } from "@t3tools/contracts";
4 +import * as Schema from "effect/Schema";
5 +import { CloudSpeechError, SPEECH_STREAM_PATH } from "@t3tools/contracts";
6 import type {
7 + CloudSpeechModelId,
8 + CloudSpeechProvider,
9 + SettingsSecretValue,
10 SpeechAcceleration,
11 SpeechModelUnloadTimeout,
12 SpeechTranscriptionOptions,
@@ -17,8 +21,10 @@ import {
21 makeEnvironmentHttpApiUrlBuilder,
22 } from "../rpc/http.ts";
23 import { executeAuthenticatedEnvironmentHttpRequest } from "../state/environmentHttpAuth.ts";
24 +import { VoiceTranscriptionError } from "./transcription.ts";
25
26 const INFERENCE_OR_DOWNLOAD_TIMEOUT_MS = 30 * 60_000;
27 +const isCloudSpeechError = Schema.is(CloudSpeechError);
28 type EnvironmentApiClient<G extends "voice" | "auth"> = Effect.Success<
29 ReturnType<typeof makeEnvironmentHttpApiGroupClient<G>>
30 >;
@@ -65,6 +71,94 @@ export const getEnvironmentSpeechStatus = (prepared: PreparedConnection) =>
71 run: ({ client, headers }) => client.status({ headers }),
72 });
73
74 +export const getCloudSpeechKeys = (prepared: PreparedConnection) =>
75 + request({
76 + group: "voice",
77 + prepared,
78 + method: "GET",
79 + path: (urls) => urls.cloudKeys(),
80 + run: ({ client, headers }) => client.cloudKeys({ headers }),
81 + });
82 +
83 +export const getCloudSpeechModels = (prepared: PreparedConnection) =>
84 + request({
85 + group: "voice",
86 + prepared,
87 + method: "GET",
88 + path: (urls) => urls.cloudModels(),
89 + run: ({ client, headers }) => client.cloudModels({ headers }),
90 + });
91 +
92 +export const requireCloudSpeechKey = (
93 + prepared: PreparedConnection,
94 + provider: CloudSpeechProvider,
95 +) =>
96 + getCloudSpeechKeys(prepared).pipe(
97 + Effect.flatMap((keys) =>
98 + keys[provider]
99 + ? Effect.void
100 + : Effect.fail(
101 + new VoiceTranscriptionError({
102 + code: "unavailable",
103 + message: new CloudSpeechError({ provider, reason: "missing-key" }).message,
104 + }),
105 + ),
106 + ),
107 + );
108 +
109 +export const setCloudSpeechKey = (
110 + prepared: PreparedConnection,
111 + provider: CloudSpeechProvider,
112 + key: SettingsSecretValue | null,
113 +) =>
114 + request({
115 + group: "voice",
116 + prepared,
117 + method: "POST",
118 + path: (urls) => urls.setCloudKey({ params: { provider } }),
119 + run: ({ client, headers }) =>
120 + client.setCloudKey({
121 + headers,
122 + params: { provider },
123 + payload: { key: typeof key === "string" ? Redacted.make(key) : key },
124 + }),
125 + });
126 +
127 +export const transcribeCloudPcm = (
128 + prepared: PreparedConnection,
129 + provider: CloudSpeechProvider,
130 + pcm: Uint8Array,
131 + options: SpeechTranscriptionOptions,
132 + canSend: () => boolean,
133 + model?: CloudSpeechModelId,
134 +) => {
135 + const query = model === undefined ? {} : { model };
136 + return request({
137 + group: "voice",
138 + prepared,
139 + method: "POST",
140 + canSend,
141 + timeoutMs: 130_000,
142 + path: (urls) => urls.transcribeCloud({ params: { provider }, query }),
143 + run: ({ client, headers }) =>
144 + client.transcribeCloud({
145 + headers,
146 + params: { provider },
147 + query,
148 + payload: encodeSpeechPcmRequest(pcm, options),
149 + }),
150 + }).pipe(
151 + Effect.mapError((error) =>
152 + error._tag === "RemoteEnvironmentAuthFetchError" && isCloudSpeechError(error.cause)
153 + ? new VoiceTranscriptionError({
154 + code: "transcription-failed",
155 + message: error.cause.message,
156 + })
157 + : error,
158 + ),
159 + );
160 +};
161 +
162 export const getEnvironmentSpeechStreamUrl = (prepared: PreparedConnection) => {
163 let endpoint = prepared.httpBaseUrl;
164 return request({
packages/client-runtime/src/voice-input/index.ts
+5
@@ -14,6 +14,11 @@ export {
14 cancelEnvironmentSpeechModelDownload,
15 downloadEnvironmentSpeechModel,
16 getEnvironmentSpeechModels,
17 + getCloudSpeechKeys,
18 + getCloudSpeechModels,
19 + requireCloudSpeechKey,
20 + setCloudSpeechKey,
21 + transcribeCloudPcm,
22 getEnvironmentSpeechStatus,
23 getEnvironmentSpeechStreamUrl,
24 postProcessEnvironmentTranscript,
packages/contracts/src/environment.ts
+1
@@ -174,6 +174,7 @@ export const ExecutionEnvironmentCapabilities = Schema.Struct({
174 scheduledTaskThreadModes: Schema.optionalKey(Schema.Boolean),
175 /** Server can transcribe 16 kHz mono PCM audio on the environment machine. */
176 voiceTranscription: Schema.optionalKey(Schema.Boolean),
177 + cloudVoiceTranscription: Schema.optionalKey(Schema.Boolean),
178 });
179 export type ExecutionEnvironmentCapabilities = typeof ExecutionEnvironmentCapabilities.Type;
180
packages/contracts/src/environmentHttp.ts
+51
@@ -49,6 +49,12 @@ import {
49 TrimmedNonEmptyString,
50 } from "./baseSchemas.ts";
51 import {
52 + CloudSpeechApiKey,
53 + CloudSpeechError,
54 + CloudSpeechKeys,
55 + CloudSpeechModelId,
56 + CloudSpeechModels,
57 + CloudSpeechProvider,
58 EnvironmentSpeechAccelerationRequest,
59 EnvironmentSpeechModelRequest,
60 EnvironmentSpeechModels,
@@ -58,6 +64,7 @@ import {
64 EnvironmentSpeechStatus,
65 EnvironmentSpeechTranscriptionResult,
66 } from "./speech.ts";
67 +import { SecretReference } from "./secretReference.ts";
68 import {
69 OrchestrationV2ShellSnapshot,
70 OrchestrationV2ThreadBoundedSnapshot,
@@ -775,6 +782,50 @@ export class EnvironmentVoiceBodyLimit extends HttpApiMiddleware.Service<Environ
782 ) {}
783
784 export class EnvironmentVoiceHttpApi extends HttpApiGroup.make("voice")
785 + .add(
786 + HttpApiEndpoint.get("cloudKeys", "/api/voice/cloud/keys", {
787 + headers: OptionalBearerHeaders,
788 + success: CloudSpeechKeys,
789 + error: EnvironmentScopedOperationErrors,
790 + }).middleware(EnvironmentAuthenticatedAuth),
791 + )
792 + .add(
793 + HttpApiEndpoint.post("setCloudKey", "/api/voice/cloud/:provider/key", {
794 + headers: OptionalBearerHeaders,
795 + params: Schema.Struct({ provider: CloudSpeechProvider }),
796 + payload: Schema.Struct({
797 + key: Schema.NullOr(
798 + Schema.Union([Schema.RedactedFromValue(CloudSpeechApiKey), SecretReference]),
799 + ),
800 + }),
801 + success: HttpApiSchema.NoContent,
802 + error: EnvironmentScopedOperationErrors,
803 + }).middleware(EnvironmentAuthenticatedAuth),
804 + )
805 + .add(
806 + // Public catalog; carries no credential and no audio.
807 + HttpApiEndpoint.get("cloudModels", "/api/voice/cloud/openrouter/models", {
808 + headers: OptionalBearerHeaders,
809 + success: CloudSpeechModels,
810 + error: [...EnvironmentScopedOperationErrors, CloudSpeechError],
811 + }).middleware(EnvironmentAuthenticatedAuth),
812 + )
813 + .add(
814 + HttpApiEndpoint.post("transcribeCloud", "/api/voice/cloud/:provider/transcribe", {
815 + headers: OptionalBearerHeaders,
816 + params: Schema.Struct({ provider: CloudSpeechProvider }),
817 + query: Schema.Struct({ model: Schema.optionalKey(CloudSpeechModelId) }),
818 + payload: Schema.Uint8Array.pipe(HttpApiSchema.asUint8Array()),
819 + success: EnvironmentSpeechTranscriptionResult,
820 + error: [
821 + ...EnvironmentScopedOperationErrors,
822 + EnvironmentRequestInvalidError,
823 + CloudSpeechError,
824 + ],
825 + })
826 + .middleware(EnvironmentVoiceBodyLimit)
827 + .middleware(EnvironmentAuthenticatedAuth),
828 + )
829 .add(
830 HttpApiEndpoint.get("status", "/api/voice/status", {
831 headers: OptionalBearerHeaders,
packages/contracts/src/settings.ts
+38 -5
@@ -53,6 +53,7 @@ import {
53 } from "./providerInstance.ts";
54 import { PullRequestMergeMethod } from "./pullRequest.ts";
55 import {
56 + CloudSpeechModelId,
57 SpeechAcceleration,
58 SpeechCorrectionCue,
59 SpeechCustomWords,
@@ -365,11 +366,24 @@ export type ModelPreferences = {
366 export const VoiceShortcutMode = Schema.Literals(["auto", "hold", "toggle"]);
367 export type VoiceShortcutMode = typeof VoiceShortcutMode.Type;
368
368 -/** Where this device sends microphone audio. A later backend adds its own kind here. */
369 -export const VoiceTranscriptionDestination = Schema.Struct({
370 - kind: Schema.Literal("environment"),
371 - environmentId: EnvironmentId,
372 -});
369 +/**
370 + * Where this device sends microphone audio. A cloud grant names the forwarding environment and
371 + * the provider; an OpenRouter grant also names the exact model, since OpenRouter has no default.
372 + */
373 +export const VoiceTranscriptionDestination = Schema.Union([
374 + Schema.Struct({ kind: Schema.Literal("environment"), environmentId: EnvironmentId }),
375 + Schema.Struct({
376 + kind: Schema.Literal("cloud"),
377 + environmentId: EnvironmentId,
378 + provider: Schema.Literals(["openai", "xai"]),
379 + }),
380 + Schema.Struct({
381 + kind: Schema.Literal("cloud"),
382 + environmentId: EnvironmentId,
383 + provider: Schema.Literal("openrouter"),
384 + model: CloudSpeechModelId,
385 + }),
386 +]);
387 export type VoiceTranscriptionDestination = typeof VoiceTranscriptionDestination.Type;
388
389 export const ClientSettingsSchema = Schema.Struct({
@@ -1224,6 +1238,15 @@ export const StorageCleanupSettings = Schema.Struct({
1238 });
1239 export type StorageCleanupSettings = typeof StorageCleanupSettings.Type;
1240
1241 +const CloudSpeechCredential = SettingsSecretValue.pipe(
1242 + Schema.withDecodingDefault(Effect.succeed("")),
1243 +);
1244 +export const CloudSpeechCredentials = Schema.Struct({
1245 + openai: CloudSpeechCredential,
1246 + xai: CloudSpeechCredential,
1247 + openrouter: CloudSpeechCredential,
1248 +});
1249 +
1250 export const ServerSettings = Schema.Struct({
1251 // Null means unmigrated; empty is authoritative and must not resurrect cleared preferences.
1252 favorites: Schema.NullOr(ModelFavorites).pipe(Schema.withDecodingDefault(Effect.succeed(null))),
@@ -1249,6 +1272,9 @@ export const ServerSettings = Schema.Struct({
1272 ),
1273
1274 enableAgentBrowserAccess: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
1275 + cloudSpeechCredentials: CloudSpeechCredentials.pipe(
1276 + Schema.withDecodingDefault(Effect.succeed({ openai: "", xai: "", openrouter: "" })),
1277 + ),
1278 speechModelId: Schema.String.pipe(
1279 Schema.withDecodingDefault(Effect.succeed("handy-computer/parakeet-unified-en-0.6b-gguf")),
1280 ),
@@ -1668,6 +1694,13 @@ export const ServerSettingsPatch = Schema.Struct({
1694 enableProviderUpdateChecks: Schema.optionalKey(Schema.Boolean),
1695 continueThreadsAfterServerUpdate: Schema.optionalKey(Schema.Boolean),
1696 enableAgentBrowserAccess: Schema.optionalKey(Schema.Boolean),
1697 + cloudSpeechCredentials: Schema.optionalKey(
1698 + Schema.Struct({
1699 + openai: Schema.optionalKey(SettingsSecretValue),
1700 + xai: Schema.optionalKey(SettingsSecretValue),
1701 + openrouter: Schema.optionalKey(SettingsSecretValue),
1702 + }),
1703 + ),
1704 speechModelId: Schema.optionalKey(Schema.String),
1705 speechAcceleration: Schema.optionalKey(SpeechAcceleration),
1706 speechModelUnloadTimeout: Schema.optionalKey(SpeechModelUnloadTimeout),
packages/contracts/src/speech.ts
+75
@@ -1,5 +1,80 @@
1 import * as Effect from "effect/Effect";
2 import * as Schema from "effect/Schema";
3 +import { TrimmedNonEmptyString } from "./baseSchemas.ts";
4 +
5 +export const CloudSpeechApiKey = TrimmedNonEmptyString.check(
6 + Schema.isMaxLength(4096),
7 + Schema.isPattern(/^(?![a-z][a-z\d+.-]*:\/\/)[!-~]+$/i),
8 +);
9 +
10 +export const CloudSpeechProvider = Schema.Literals(["openai", "xai", "openrouter"]);
11 +export type CloudSpeechProvider = typeof CloudSpeechProvider.Type;
12 +export const CLOUD_SPEECH_LABELS = {
13 + openai: "OpenAI",
14 + xai: "Grok Voice Transcribe",
15 + openrouter: "OpenRouter",
16 +} as const;
17 +export const CLOUD_SPEECH_PROVIDER_NAMES = {
18 + openai: "OpenAI",
19 + xai: "xAI",
20 + openrouter: "OpenRouter",
21 +} as const;
22 +/** Servers that predate OpenRouter report only the first two providers. */
23 +export const CloudSpeechKeys = Schema.Struct({
24 + openai: Schema.Boolean,
25 + xai: Schema.Boolean,
26 + openrouter: Schema.optionalKey(Schema.Boolean),
27 +});
28 +export type CloudSpeechKeys = typeof CloudSpeechKeys.Type;
29 +
30 +/** An exact `author/model` slug. Router shortcuts such as `:nitro` or `openrouter/auto` are not models. */
31 +export const CloudSpeechModelId = Schema.String.check(
32 + Schema.isMinLength(3),
33 + Schema.isMaxLength(160),
34 + Schema.isPattern(/^(?!openrouter\/)[a-z\d][a-z\d.-]*\/[a-z\d][a-z\d._-]*$/i),
35 +);
36 +export type CloudSpeechModelId = typeof CloudSpeechModelId.Type;
37 +export const CloudSpeechModel = Schema.Struct({ id: CloudSpeechModelId, name: Schema.String });
38 +export type CloudSpeechModel = typeof CloudSpeechModel.Type;
39 +export const CloudSpeechModels = Schema.Struct({ models: Schema.Array(CloudSpeechModel) });
40 +export type CloudSpeechModels = typeof CloudSpeechModels.Type;
41 +
42 +export class CloudSpeechError extends Schema.TaggedError<CloudSpeechError>()(
43 + "CloudSpeechError",
44 + {
45 + provider: CloudSpeechProvider,
46 + reason: Schema.Literals([
47 + "missing-key",
48 + "key-changed",
49 + "unreadable-key",
50 + "rejected-key",
51 + "rate-limit",
52 + "model-unavailable",
53 + "unavailable",
54 + ]),
55 + },
56 + { httpApiStatus: 422 },
57 +) {
58 + override get message() {
59 + const name = CLOUD_SPEECH_PROVIDER_NAMES[this.provider];
60 + switch (this.reason) {
61 + case "missing-key":
62 + return `Add your ${name} API key in Settings → Voice.`;
63 + case "key-changed":
64 + return `The ${name} key changed before your recording was sent. Try again.`;
65 + case "unreadable-key":
66 + return `Couldn’t read the ${name} API key. Unlock the vault on the selected environment or update the key in Settings → Voice. No audio was sent to ${name}.`;
67 + case "rejected-key":
68 + return `${name} rejected this API key. Check its permissions and billing in Settings → Voice.`;
69 + case "rate-limit":
70 + return `${name} reached a usage limit. Check the account's quota or try again later.`;
71 + case "model-unavailable":
72 + return `The chosen model isn’t available on ${name}. Choose another model in Settings → Voice.`;
73 + case "unavailable":
74 + return `${name} couldn’t transcribe this recording. Try again later.`;
75 + }
76 + }
77 +}
78
79 export const SpeechModelId = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(160));
80 export type SpeechModelId = typeof SpeechModelId.Type;
packages/shared/src/speech.ts
+17
@@ -1,4 +1,6 @@
1 import {
2 + CLOUD_SPEECH_PROVIDER_NAMES,
3 + type CloudSpeechProvider,
4 SPEECH_MAX_OPTIONS_BYTES,
5 type SpeechCustomWords,
6 SpeechTranscriptionOptions,
@@ -8,6 +10,21 @@ import * as Schema from "effect/Schema";
10 const decodeOptions = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechTranscriptionOptions));
11 const encodeOptions = Schema.encodeSync(Schema.fromJsonString(SpeechTranscriptionOptions));
12
13 +export function cloudSpeechDisclosure(provider: CloudSpeechProvider, environment: string) {
14 + return provider === "openrouter"
15 + ? `After you stop, ${environment} sends the recording to OpenRouter. OpenRouter chooses who runs your model. T3 requests zero data retention; your OpenRouter account may allow safety retention.`
16 + : `After you stop, ${environment} sends your recording and vocabulary hints, including project names, to ${CLOUD_SPEECH_PROVIDER_NAMES[provider]}.`;
17 +}
18 +
19 +// Only offer explicit hints understood by all cloud APIs; other languages can still be detected.
20 +export const cloudSpeechLanguages = {
21 + supportsLanguageDetection: true,
22 + languages:
23 + "ar bg ca cs da de el en es fa fi fr hi hr hu id it ja ko mk ms nl pl pt ro ru sk sv th tr uk ur vi zh".split(
24 + " ",
25 + ),
26 +};
27 +
28 /** Resolve the language the model can produce from the requested language. */
29 export function effectiveSpeechLanguage(
30 model: { readonly languages: readonly string[]; readonly supportsLanguageDetection: boolean },