@devries-cameron20 / t3code-1 / commits / 3027c2c840
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(voice): start dictation capture sooner on iOS and desktop (#40011)

Julius Marminge committed Nov 20, 2026 at 16:27 UTC 3027c2c8400438193cbf9903236e2f3fe644eaa6
14 files changed +294 -90
apps/mobile/src/features/voice-input/environmentTranscriber.test.ts
+13 -2
@@ -4,17 +4,22 @@ import { createEnvironmentTranscriber } from "./environmentTranscriber";
4
5 it("never uploads a recording whose grant changed while the phone decoded it", async () => {
6 let grant = 1;
7 + const decoding = Promise.withResolvers<void>();
8 const decoded = Promise.withResolvers<Uint8Array>();
9 const upload = vi.fn(async () => "hello");
10 const transcriber = createEnvironmentTranscriber({
11 canSend: () => grant === 1,
12 prepare: async () => "en",
12 - decode: () => decoded.promise,
13 + decode: () => {
14 + decoding.resolve();
15 + return decoded.promise;
16 + },
17 upload,
18 });
19 const options = { signal: new AbortController().signal };
20 const recording = await transcriber.prepare(options);
21 const transcription = recording.transcribe("file:///draft.m4a", options);
22 + await decoding.promise;
23 grant++;
24 decoded.resolve(new Uint8Array([0, 128, 0, 64]));
25 await expect(transcription).rejects.toMatchObject({ code: "cancelled" });
@@ -26,14 +31,20 @@ it("uploads normalized mono samples only after the recording is finished", async
31 expect(Array.from(new Float32Array(Uint8Array.from(bytes).buffer))).toEqual([-1, 0.5]);
32 return "hello";
33 });
34 + const ready = Promise.withResolvers<string>();
35 const transcriber = createEnvironmentTranscriber({
36 canSend: () => true,
31 - prepare: async () => "en",
37 + prepare: () => ready.promise,
38 decode: async () => new Uint8Array([255, 0, 128, 0, 64, 255]).subarray(1, 5),
39 upload,
40 });
41 const options = { signal: new AbortController().signal };
42 const recording = await transcriber.prepare(options);
43 + const starting = recording.start?.("file:///draft.m4a", options);
44 + expect(upload).not.toHaveBeenCalled();
45 + ready.resolve("en");
46 + await starting;
47 + expect(recording.locale).toBe("en");
48 expect(upload).not.toHaveBeenCalled();
49 await expect(recording.transcribe("file:///draft.m4a", options)).resolves.toBe("hello");
50 });
apps/mobile/src/features/voice-input/environmentTranscriber.ts
+15 -3
@@ -29,11 +29,23 @@ export function createEnvironmentTranscriber(input: {
29 return {
30 async prepare({ signal }) {
31 check(signal);
32 - const locale = await input.prepare(signal);
33 - check(signal);
32 + let locale = "auto";
33 + const ready = input.prepare(signal).then((resolved) => {
34 + check(signal);
35 + locale = resolved;
36 + });
37 + void ready.catch(() => {});
38 return {
35 - locale,
39 + get locale() {
40 + return locale;
41 + },
42 + async start(_uri, { signal }) {
43 + await ready;
44 + check(signal);
45 + return null;
46 + },
47 async transcribe(uri, { signal }) {
48 + await ready;
49 check(signal);
50 const decoded = await input.decode(uri);
51 check(signal);
apps/mobile/src/features/voice-input/voiceDestination.test.ts
+2 -1
@@ -172,7 +172,8 @@ it("keeps local as the default and never reuses consent for an offline or resele
172 chooseVoiceDestination(host, "openai");
173 const cloud = appAtomRegistry.get(voiceRouteAtom)!.transcriber!;
174 mocks.fetch.mockResolvedValueOnce(Response.json({ openai: false, xai: true }));
175 - await expect(cloud.prepare(options)).rejects.toMatchObject({
175 + const missingKey = await cloud.prepare(options);
176 + await expect(missingKey.start?.("file:///voice.m4a", options)).rejects.toMatchObject({
177 code: "unavailable",
178 message: expect.stringContaining("Add your OpenAI API key"),
179 });
apps/web/src/components/chat/ComposerSpeechButton.tsx
+28 -13
@@ -55,29 +55,37 @@ function formatElapsed(seconds: number) {
55
56 function RecordingStatus(props: {
57 level: number;
58 + preparing: boolean;
59 recordingStartedAt?: number | null | undefined;
60 recordingLimitSeconds?: number | undefined;
61 }) {
61 - const { recordingStartedAt } = props;
62 + const { preparing, recordingStartedAt } = props;
63 const [elapsedSeconds, setElapsedSeconds] = useState(() =>
64 recordingStartedAt ? Math.max(0, Math.floor((Date.now() - recordingStartedAt) / 1_000)) : 0,
65 );
66
67 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the timer displaying the native recording duration
68 useEffect(() => {
69 + if (preparing) return;
70 const startedAt = recordingStartedAt ?? Date.now();
71 const timer = window.setInterval(
72 () => setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1_000)),
73 250,
74 );
75 return () => window.clearInterval(timer);
74 - }, [recordingStartedAt]);
76 + }, [preparing, recordingStartedAt]);
77
78 return (
79 <>
78 - <span aria-hidden className="size-2 shrink-0 rounded-full bg-destructive" />
80 + <span
81 + aria-hidden
82 + className={cn(
83 + "size-2 shrink-0 rounded-full",
84 + preparing ? "bg-muted-foreground/40" : "bg-destructive",
85 + )}
86 + />
87 <span className="shrink-0 text-xs text-muted-foreground tabular-nums">
80 - {formatElapsed(elapsedSeconds)}
88 + {formatElapsed(preparing ? 0 : elapsedSeconds)}
89 {props.recordingLimitSeconds !== undefined
90 ? ` / ${formatElapsed(props.recordingLimitSeconds)}`
91 : ""}
@@ -174,7 +182,9 @@ export function ComposerSpeechRecordingPill(props: {
182 const presentation = resolveSpeechPresentation(props.state, props.progress, m);
183 if (!presentation.active) return null;
184 const recording = phase === "recording";
177 - const trailing = recording || phase === "post-processing";
185 + const starting = phase === "preparing" && props.progress === null;
186 + const showRecordingLayout = starting || recording;
187 + const trailing = showRecordingLayout || phase === "post-processing";
188
189 return (
190 <VoiceInputPill size={props.size}>
@@ -200,12 +210,16 @@ export function ComposerSpeechRecordingPill(props: {
210 aria-live={recording ? "off" : "polite"}
211 aria-label={presentation.status ?? undefined}
212 >
203 - {recording ? (
204 - <RecordingStatus
205 - level={props.level}
206 - recordingStartedAt={props.recordingStartedAt}
207 - recordingLimitSeconds={props.recordingLimitSeconds}
208 - />
213 + {showRecordingLayout ? (
214 + <>
215 + {starting && <span className="sr-only">{presentation.status}</span>}
216 + <RecordingStatus
217 + preparing={starting}
218 + level={starting ? 0 : props.level}
219 + recordingStartedAt={props.recordingStartedAt}
220 + recordingLimitSeconds={props.recordingLimitSeconds}
221 + />
222 + </>
223 ) : (
224 <>
225 <Spinner aria-hidden size="sm" tone="muted" className="shrink-0" />
@@ -226,17 +240,18 @@ export function ComposerSpeechRecordingPill(props: {
240 >
241 {gt("Skip", { $context: "verb, skips cleaning up the transcript" })}
242 </Button>
229 - ) : recording ? (
243 + ) : showRecordingLayout ? (
244 <ShortcutTooltip label={gt("Finish voice input")} shortcut={props.finishShortcutLabel}>
245 <Button
246 type="button"
247 size={compact ? "icon-xs" : "icon-sm-pill"}
248 aria-label={gt("Finish voice input")}
249 + disabled={starting}
250 onPointerDown={keepFocus}
251 onClick={props.onStop}
252 className="shrink-0"
253 >
239 - <CheckIcon />
254 + {starting ? <Spinner aria-hidden /> : <CheckIcon />}
255 </Button>
256 </ShortcutTooltip>
257 ) : null}
apps/web/src/speech/browserPcmCapture.ts
+61 -12
@@ -1,15 +1,19 @@
1 -import type {
2 - PreparedVoiceTranscription,
3 - VoiceRecorder,
4 - VoiceStreamingSession,
5 - VoiceInputErrorCode,
6 - VoiceTranscriptionOptions,
1 +import {
2 + VoiceTranscriptionError,
3 + type PreparedVoiceTranscription,
4 + type VoiceRecorder,
5 + type VoiceStreamingSession,
6 + type VoiceInputErrorCode,
7 + type VoiceTranscriptionOptions,
8 } from "@t3tools/client-runtime/voice-input";
9 +import * as Schema from "effect/Schema";
10 import { requestMicrophone } from "./requestMicrophone";
11 import workletUrl from "./pcmWorklet.ts?worker&url";
12 import { msg } from "gt-react";
13 import { m } from "../i18n/messages";
14
15 +const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
16 +
17 type PreparedBrowserTranscription = PreparedVoiceTranscription & {
18 readonly streaming?: VoiceStreamingSession & { readonly feed: (pcm: Float32Array) => void };
19 };
@@ -21,7 +25,10 @@ export function createBrowserPcmCapture(
25 readonly onDurationLimit: () => void;
26 readonly onError: (message: string, code?: VoiceInputErrorCode) => void;
27 },
24 - prepare: (options: VoiceTranscriptionOptions) => Promise<PreparedBrowserTranscription>,
28 + prepare: (
29 + options: VoiceTranscriptionOptions,
30 + canStream: () => boolean,
31 + ) => Promise<PreparedBrowserTranscription>,
32 ) {
33 let stream: MediaStream | undefined;
34 let recordingUri: string | null = null;
@@ -30,6 +37,7 @@ export function createBrowserPcmCapture(
37 let worklet: AudioWorkletNode | undefined;
38 let durationTimer: ReturnType<typeof setTimeout> | undefined;
39 let signal: AbortSignal | undefined;
40 + let stopAcceptingAudio: (() => void) | undefined;
41 let feed: ((pcm: Float32Array) => void) | undefined;
42 let stopped:
43 | { readonly resolve: () => void; readonly reject: (error: Error) => void }
@@ -48,6 +56,8 @@ export function createBrowserPcmCapture(
56 stopped?.resolve();
57 stopped = undefined;
58 chunks = [];
59 + stopAcceptingAudio?.();
60 + stopAcceptingAudio = undefined;
61 feed = undefined;
62 input.onLevel(0);
63 };
@@ -175,12 +185,51 @@ export function createBrowserPcmCapture(
185 if (recordingUri === uri) recordingUri = null;
186 },
187 transcriber: {
178 - prepare: async (options: VoiceTranscriptionOptions) => {
179 - const prepared = await prepare(options);
180 - options.signal.throwIfAborted();
188 + prepare: async (options: VoiceTranscriptionOptions): Promise<PreparedVoiceTranscription> => {
189 signal = options.signal;
182 - feed = prepared.streaming?.feed;
183 - return prepared;
190 + let acceptingAudio = true;
191 + stopAcceptingAudio = () => {
192 + acceptingAudio = false;
193 + };
194 + const canStream = () => acceptingAudio && !options.signal.aborted;
195 + let streaming: VoiceStreamingSession | undefined;
196 + let destination: PreparedBrowserTranscription | undefined;
197 + const pending = prepare(options, canStream)
198 + .then((prepared) => {
199 + options.signal.throwIfAborted();
200 + destination = prepared;
201 + if (canStream() && prepared.streaming) {
202 + streaming = prepared.streaming;
203 + feed = prepared.streaming.feed;
204 + for (const chunk of chunks) feed(chunk);
205 + chunks = [];
206 + }
207 + return prepared;
208 + })
209 + .catch((cause: unknown) => {
210 + const error = new VoiceTranscriptionError({
211 + code: "preparation-failed",
212 + message: isVoiceTranscriptionError(cause)
213 + ? cause.message
214 + : m(msg("Could not prepare voice transcription.")),
215 + cause,
216 + });
217 + if (canStream()) input.onError(error.message);
218 + throw error;
219 + });
220 + void pending.catch(() => {});
221 + return {
222 + get locale() {
223 + return destination?.locale ?? "auto";
224 + },
225 + get streaming() {
226 + return streaming;
227 + },
228 + async transcribe(uri, transcriptionOptions) {
229 + const prepared = await pending;
230 + return prepared.transcribe(uri, transcriptionOptions);
231 + },
232 + };
233 },
234 },
235 };
apps/web/src/speech/browserVoiceInput.browser.test.ts
+46 -3
@@ -9,6 +9,7 @@ import { decodeSpeechPcmRequest } from "@t3tools/shared/speech";
9 import { expect, it, vi } from "vite-plus/test";
10
11 import { stubGlobal } from "../../test/render";
12 +import { createBrowserPcmCapture } from "./browserPcmCapture";
13 import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
14
15 vi.mock("../connection/runtime", async () => {
@@ -158,13 +159,15 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be
159 await vi.waitFor(() => expect(_network).toHaveBeenCalled());
160 streaming = true;
161 options = { ...options, speechLanguage: "en" };
161 - resolveStatus(Response.json(status));
162 await start;
163 expect(controller.currentState.phase).toBe("recording");
164 microphone.feed(new Float32Array([0.1, 0.2]));
165 microphone.feed(new Float32Array([0.3]));
166 expect(requests).toHaveLength(0);
167 - await controller.stop();
167 + const stop = controller.stop();
168 + await vi.waitFor(() => expect(microphone.stopTrack).toHaveBeenCalledOnce());
169 + resolveStatus(Response.json(status));
170 + await stop;
171 expect(controller.currentState.phase).toBe("error");
172 expect(platform.recorder.uri).not.toBeNull();
173 await controller.start();
@@ -191,6 +194,46 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be
194 expect(platform.recorder.uri).toBeNull();
195 });
196
197 +it("streams the opening audio in order when the destination becomes ready during capture", async () => {
198 + using microphone = capture();
199 + const feed = vi.fn();
200 + const finish = vi.fn(async () => "hello");
201 + const ready = Promise.withResolvers<void>();
202 + const platform = createBrowserPcmCapture(
203 + {
204 + getMicrophoneId: () => "",
205 + onLevel() {},
206 + onDurationLimit() {},
207 + onError() {},
208 + },
209 + async () => {
210 + await ready.promise;
211 + return { locale: "en", streaming: { feed, finish }, transcribe: async () => "" };
212 + },
213 + );
214 + const abort = new AbortController();
215 + using _abort = {
216 + [Symbol.dispose]: () => {
217 + abort.abort();
218 + platform.cancelRecording();
219 + },
220 + };
221 + const transcription = await platform.transcriber.prepare({ signal: abort.signal });
222 + await platform.recorder.prepareToRecordAsync(abort.signal);
223 + platform.recorder.record({ forDuration: 1 });
224 + const opening = new Float32Array([0.1]);
225 + const continuing = new Float32Array([0.2]);
226 + microphone.feed(opening);
227 + expect(feed).not.toHaveBeenCalled();
228 + ready.resolve();
229 + await vi.waitFor(() => expect(feed).toHaveBeenCalledExactlyOnceWith(opening));
230 + microphone.feed(continuing);
231 + await platform.recorder.stop();
232 + expect(feed.mock.calls).toEqual([[opening], [continuing]]);
233 + expect(platform.recorder.uri).toBeNull();
234 + await expect(transcription.streaming?.finish({ signal: abort.signal })).resolves.toBe("hello");
235 +});
236 +
237 it.each([
238 { native: true, enabled: true, buffered: true, live: true },
239 { native: true, enabled: false, buffered: true, live: false },
@@ -241,8 +284,8 @@ it.each([
284 onError() {},
285 });
286 const transcription = await platform.transcriber.prepare({ signal: abort.signal });
287 + await vi.waitFor(() => expect(transcription.locale).toBe(native && enabled ? "fr" : "en"));
288 expect(Boolean(transcription.streaming)).toBe(live);
245 - expect(transcription.locale).toBe(native && enabled ? "fr" : "en");
289 expect(sockets).toEqual(
290 live ? ["wss://transcription.example.test/ws/voice?wsTicket=stream-ticket"] : [],
291 );
apps/web/src/speech/browserVoiceInput.ts
+25 -9
@@ -12,7 +12,11 @@ import {
12 type VoiceTranscriptionOptions,
13 } from "@t3tools/client-runtime/voice-input";
14 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
15 -import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";
15 +import type {
16 + SpeechTranscriptionOptions,
17 + SpeechStreamText,
18 + EnvironmentSpeechStatus,
19 +} from "@t3tools/contracts";
20 import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts";
21 import * as Schema from "effect/Schema";
22
@@ -30,6 +34,7 @@ export function createBrowserVoiceInputPlatform(input: {
34 readonly cloudModel?: CloudSpeechModelId | undefined;
35 readonly hasAudioConsent: () => boolean;
36 readonly getTranscriptionOptions: () => SpeechTranscriptionOptions;
37 + readonly getStatusListener?: () => ((status: EnvironmentSpeechStatus) => void) | undefined;
38 readonly getStreamingEnabled: () => boolean;
39 readonly getMicrophoneId: () => string | Promise<string>;
40 readonly onLevel: (level: number) => void;
@@ -37,7 +42,8 @@ export function createBrowserVoiceInputPlatform(input: {
42 readonly onText: (text: SpeechStreamText) => void;
43 readonly onError: (message: string, code?: VoiceInputErrorCode) => void;
44 }) {
40 - return createBrowserPcmCapture(input, async (options) => {
45 + return createBrowserPcmCapture(input, async (options, canStream) => {
46 + const onStatus = input.getStatusListener?.();
47 const recordingOptions = input.getTranscriptionOptions();
48 const streamingEnabled = input.getStreamingEnabled();
49 const transcribeRecording = async (
@@ -95,8 +101,15 @@ export function createBrowserVoiceInputPlatform(input: {
101 }
102 const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
103 throwIfVoiceTranscriptionAborted(options.signal);
104 + onStatus?.(status);
105 + throwIfVoiceTranscriptionAborted(options.signal);
106 if (!status.supported)
107 throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
108 + if (status.state === "missing-model")
109 + throw new VoiceTranscriptionError({
110 + code: "unavailable",
111 + message: m(msg("Could not start voice input")),
112 + });
113 const language = status.languages
114 ? effectiveSpeechLanguage(
115 {
@@ -107,16 +120,19 @@ export function createBrowserVoiceInputPlatform(input: {
120 )
121 : "auto";
122 const nativeStreaming = status.supportsStreaming && streamingEnabled;
110 - const locale =
111 - !nativeStreaming && status.supportsTranslation && recordingOptions.speechTranslateToEnglish
112 - ? "en"
113 - : language;
114 - if (status.supportsStreaming ? !nativeStreaming : !status.supportsBufferedStreaming) {
115 - return { locale, transcribe: transcribeRecording };
123 + const batchLocale =
124 + status.supportsTranslation && recordingOptions.speechTranslateToEnglish ? "en" : language;
125 + const batch = { locale: batchLocale, transcribe: transcribeRecording };
126 + if (
127 + !canStream() ||
128 + (status.supportsStreaming ? !nativeStreaming : !status.supportsBufferedStreaming)
129 + ) {
130 + return batch;
131 }
132 const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options);
133 + if (!canStream()) return batch;
134 return {
119 - locale,
135 + locale: nativeStreaming ? language : batchLocale,
136 transcribe: transcribeRecording,
137 streaming: openSpeechStream({
138 url,
apps/web/src/speech/useEnvironmentSpeechInput.ts
+35 -36
@@ -6,7 +6,6 @@ import {
6 getSpeechModelDownloadOutcome,
7 selectEnvironmentSpeechModel,
8 voiceInputBlocksSubmission,
9 - type VoiceInputState,
9 } from "@t3tools/client-runtime/voice-input";
10 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
11 import { AuthOrchestrationOperateScope, CLOUD_SPEECH_LABELS } from "@t3tools/contracts";
@@ -46,7 +45,6 @@ import {
45 type VoiceInputTarget,
46 } from "./voiceInputSession";
47
49 -const WAITING_STATE: VoiceInputState = { phase: "preparing", error: null, errorAction: null };
48 const INITIAL_SETUP = {
49 open: false,
50 step: 0,
@@ -176,7 +174,6 @@ export function useEnvironmentSpeechInput(input: HookInput) {
174 readonly prepared: PreparedConnection;
175 readonly value: EnvironmentSpeechStatus | null;
176 } | null>(null);
179 - const [queuedStart, setQueuedStart] = useState<PreparedConnection | null>(null);
177 const [setup, setSetup] = useState(INITIAL_SETUP);
178 const [previousDestination, setPreviousDestination] = useState({ prepared, grantId });
179 const [previousOwner, setPreviousOwner] = useState({
@@ -186,11 +183,9 @@ export function useEnvironmentSpeechInput(input: HookInput) {
183 if (previousDestination.prepared !== prepared || previousDestination.grantId !== grantId) {
184 setPreviousDestination({ prepared, grantId });
185 setSetup(INITIAL_SETUP);
189 - setQueuedStart(null);
186 }
187 if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) {
188 setPreviousOwner({ key: input.ownerKey, disabled: input.disabled });
193 - setQueuedStart(null);
189 setSetup(INITIAL_SETUP);
190 }
191
@@ -198,8 +193,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
193 const snapshot = useVoiceInputSessionSnapshot(session);
194 const owns = snapshot.target?.ownerKey === input.ownerKey;
195 const activity = useVoiceInputSessionActivity(session, input.ownerKey);
201 - const state: VoiceInputState =
202 - queuedStart === prepared && prepared ? WAITING_STATE : owns ? snapshot.state : IDLE_VOICE_STATE;
196 + const state = owns ? snapshot.state : IDLE_VOICE_STATE;
197 // One microphone: another draft's recording leaves this editor without voice input until it ends.
198 const busyElsewhere = !owns && voiceInputBlocksSubmission(snapshot.state);
199
@@ -401,48 +395,54 @@ export function useEnvironmentSpeechInput(input: HookInput) {
395 await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });
396 return;
397 }
398 + if (!session || !prepared || voiceInputBlocksSubmission(session.state)) return;
399 const request = ++startRequestRef.current;
400 const owner = latestRef.current.input.ownerKey;
406 - if (!session || !prepared || busyElsewhere) return;
401 if (cloudProvider) {
402 await session.start(createTarget());
403 return;
404 }
405 + setSetup(INITIAL_SETUP);
406 const stillCurrent = () =>
407 request === startRequestRef.current &&
408 latestRef.current.session === session &&
409 latestRef.current.input.ownerKey === owner &&
410 !latestRef.current.input.disabled;
416 - setSetup(INITIAL_SETUP);
417 - setQueuedStart(prepared);
418 - let latestStatus: EnvironmentSpeechStatus;
419 - try {
420 - latestStatus = await probeSpeechStatus(prepared, true);
421 - } catch (error) {
411 + const handleStatus = async (latestStatus: EnvironmentSpeechStatus) => {
412 if (!stillCurrent()) return;
423 - setQueuedStart(null);
424 - toastManager.add({
425 - type: "error",
426 - title: m(msg("Could not start voice input")),
427 - description: error instanceof Error ? error.message : String(error),
428 - });
429 - return;
430 - }
431 - if (!stillCurrent()) return;
432 - setStatus({ prepared, value: latestStatus });
433 - setQueuedStart(null);
434 - if (latestStatus.supported && latestStatus.state === "missing-model") {
435 - return loadSetupCatalog(prepared, latestStatus.modelId, latestStatus.benchmarkTier, request);
436 - }
437 - if (!latestStatus.supported) {
438 - toastManager.add({
439 - type: "error",
440 - title: m(msg("Voice input is not available")),
441 - description: latestStatus.reason,
413 + setStatus({ prepared, value: latestStatus });
414 + if (!latestStatus.supported) {
415 + session.cancel();
416 + toastManager.add({
417 + type: "error",
418 + title: m(msg("Voice input is not available")),
419 + description: latestStatus.reason,
420 + });
421 + } else if (latestStatus.state === "missing-model") {
422 + session.cancel();
423 + await loadSetupCatalog(prepared, latestStatus.modelId, latestStatus.benchmarkTier, request);
424 + }
425 + };
426 +
427 + if (currentStatus?.supported && currentStatus.state === "missing-model") {
428 + // Setup must remain usable without opening the microphone.
429 + const latestStatus = await probeSpeechStatus(prepared, true).catch((error: unknown) => {
430 + if (stillCurrent()) {
431 + toastManager.add({
432 + type: "error",
433 + title: m(msg("Could not start voice input")),
434 + description: error instanceof Error ? error.message : String(error),
435 + });
436 + }
437 + return null;
438 });
443 - return;
439 + if (!stillCurrent() || !latestStatus) return;
440 + await handleStatus(latestStatus);
441 + if (!latestStatus.supported || latestStatus.state === "missing-model" || !stillCurrent())
442 + return;
443 }
445 - await session.start(createTarget());
444 +
445 + await session.start(createTarget(), (latestStatus) => void handleStatus(latestStatus));
446 });
447
448 const downloadSetupModel = useStableCallback(async () => {
@@ -562,7 +562,6 @@ export function useEnvironmentSpeechInput(input: HookInput) {
562 cancel: useStableCallback(() => {
563 pendingSubmissionRef.current = null;
564 startRequestRef.current += 1;
565 - setQueuedStart(null);
565 setSetup(INITIAL_SETUP);
566 if (owns) session?.cancel();
567 }),
apps/web/src/speech/voiceInputSession.ts
+8 -1
@@ -12,6 +12,7 @@ import type {
12 CloudSpeechProvider,
13 ClientSettings,
14 EnvironmentId,
15 + EnvironmentSpeechStatus,
16 SpeechCustomWords,
17 SpeechPostProcessingOptions,
18 SpeechStreamText,
@@ -108,6 +109,7 @@ export class VoiceInputSession {
109 private readonly listeners = new Set<() => void>();
110 private readonly activityListeners = new Set<() => void>();
111 private disposed = false;
112 + private onSpeechStatus: ((status: EnvironmentSpeechStatus) => void) | undefined;
113
114 constructor(key: SessionKey) {
115 this.key = key;
@@ -124,6 +126,7 @@ export class VoiceInputSession {
126 cloudProvider: key.cloudProvider,
127 cloudModel: key.cloudModel,
128 hasAudioConsent,
129 + getStatusListener: () => this.onSpeechStatus,
130 getStreamingEnabled: () =>
131 this.target?.readContext().clientSettings.speechStreaming ??
132 DEFAULT_CLIENT_SETTINGS.speechStreaming,
@@ -253,13 +256,17 @@ export class VoiceInputSession {
256 readonly getActivity = (): VoiceInputSessionActivity => this.activity;
257
258 /** Starts recording into `target`, or does nothing while another draft holds the microphone. */
256 - start(target: VoiceInputTarget): Promise<void> {
259 + start(
260 + target: VoiceInputTarget,
261 + onSpeechStatus?: (status: EnvironmentSpeechStatus) => void,
262 + ): Promise<void> {
263 if (this.disposed) return Promise.resolve();
264 const busyElsewhere =
265 voiceInputBlocksSubmission(this.controller.currentState) &&
266 this.target?.ownerKey !== target.ownerKey;
267 if (busyElsewhere) return Promise.resolve();
268 this.target = target;
269 + this.onSpeechStatus = onSpeechStatus;
270 this.publishActivity(SILENT_ACTIVITY);
271 this.publish({ ...this.snapshot, target });
272 return this.controller.start();
docs/internals/voice-input.md
+3 -2
@@ -7,8 +7,9 @@ the device requires an explicit grant naming one environment and, for cloud
7 transcription, one provider.
8
9 The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts)
10 -owns the operation while the client supplies capture and transcription. Preparation
11 -binds the transcriber and resolved locale for the whole recording. Draft ownership,
10 +owns the operation while the client supplies capture and transcription. The
11 +transcriber is bound to the recording when it starts. Capture can begin while
12 +the locale and destination are still being resolved. Draft ownership,
13 text, and revision are captured before recording and checked before insertion, so
14 a late transcript cannot overwrite a draft that was edited or replaced. A
15 transcriber may expose a streaming session, in which case audio leaves during
packages/client-runtime/src/voice-input/controller.ts
+1
@@ -612,6 +612,7 @@ export class VoiceInputController {
612 if (this.isCurrent(operationToken)) {
613 if (
614 this.dependencies.retryFailedRecording &&
615 + errorCode(error) !== "preparation-failed" &&
616 !this.liveTranscription &&
617 !transcription.streaming &&
618 recordingUri
packages/client-runtime/src/voice-input/transcription.ts
+1 -1
@@ -21,7 +21,7 @@ export type PreparedVoiceTranscription = {
21 readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string | null>;
22 readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
23 /** Present when audio streams during capture; `finish` then replaces `transcribe`. */
24 - readonly streaming?: VoiceStreamingSession;
24 + readonly streaming?: VoiceStreamingSession | undefined;
25 };
26
27 export type VoiceTranscriber = {
patches/expo-audio@58.0.11.patch
+53 -4
@@ -388,7 +388,6 @@ index 0d63c9b949cfae7a101902fc3bb70fcf46094f68..a312b8fcdaf728220db3d6c04d92fd78
388 * Replaces the current audio source with a new one.
389 */
390 diff --git a/ios/AudioModule.swift b/ios/AudioModule.swift
391 -index 42ce9e8f213ebc243db25ef836456757b0daeb09..b422c3c10349b14803d5e9d7afeed617e76e09af 100644
391 --- a/ios/AudioModule.swift
392 +++ b/ios/AudioModule.swift
393 @@ -6,6 +6,7 @@ public class AudioModule: Module {
@@ -428,7 +427,22 @@ index 42ce9e8f213ebc243db25ef836456757b0daeb09..b422c3c10349b14803d5e9d7afeed617
427 self.deactivateSession()
428 }
429 }
431 -@@ -957,6 +968,7 @@ public class AudioModule: Module {
430 +@@ -486,11 +497,12 @@ public class AudioModule: Module {
431 + AsyncFunction("prepareToRecordAsync") { (recorder, options: RecordingOptions?) in
432 + try AudioRecordingRequester.requireUsageDescription()
433 +- let deactivateSessionOnFailure = sessionQueue.sync {
434 +- !sessionIsActive && audioSessionActivityKeepers.isEmpty
435 ++ let (activateSession, deactivateSessionOnFailure) = sessionQueue.sync {
436 ++ (!hasAudioSessionOwner || !sessionIsActive, !sessionIsActive && audioSessionActivityKeepers.isEmpty)
437 + }
438 + try recorder.prepare(
439 + options: options,
440 + sessionOptions: sessionOptions,
441 ++ activateSession: activateSession,
442 + deactivateSessionOnFailure: deactivateSessionOnFailure
443 + )
444 + recordSessionActive(true)
445 +@@ -957,6 +969,7 @@ public class AudioModule: Module {
446 sessionQueue.asyncAfter(deadline: .now() + .milliseconds(100)) { [weak self] in
447 guard let self,
448 self.sessionIsActive,
@@ -437,10 +451,32 @@ index 42ce9e8f213ebc243db25ef836456757b0daeb09..b422c3c10349b14803d5e9d7afeed617
451 !self.isSessionInUse else {
452 return
453 diff --git a/ios/AudioRecorder.swift b/ios/AudioRecorder.swift
440 -index 198fe6dc95b564ff3916657f14ea67fc44d2e424..5b396114bbeb5e1c99da54f4e6d2376ad978dfa0 100644
454 --- a/ios/AudioRecorder.swift
455 +++ b/ios/AudioRecorder.swift
443 -@@ -285,16 +285,16 @@ class AudioRecorder: SharedRef<AVAudioRecorder>, RecordingResultHandler {
456 +@@ -81,6 +81,7 @@ class AudioRecorder: SharedRef<AVAudioRecorder>, RecordingResultHandler {
457 + func prepare(
458 + options: RecordingOptions?,
459 + sessionOptions: AVAudioSession.CategoryOptions = [],
460 ++ activateSession: Bool = true,
461 + deactivateSessionOnFailure: Bool = true
462 + ) throws {
463 + if currentState == .recording {
464 +@@ -93,7 +94,12 @@ class AudioRecorder: SharedRef<AVAudioRecorder>, RecordingResultHandler {
465 + let session = AVAudioSession.sharedInstance()
466 + do {
467 +- try session.setCategory(.playAndRecord, mode: .default, options: sessionOptions)
468 +- try session.setActive(true)
469 ++ let categoryMatches = session.category == .playAndRecord && session.mode == .default && session.categoryOptions == sessionOptions
470 ++ if !categoryMatches {
471 ++ try session.setCategory(.playAndRecord, mode: .default, options: sessionOptions)
472 ++ }
473 ++ if activateSession || !categoryMatches {
474 ++ try session.setActive(true)
475 ++ }
476 + } catch {
477 + currentState = .error
478 + throw AudioRecordingException("Failed to configure audio session: \(error.localizedDescription)")
479 +@@ -285,16 +291,16 @@ class AudioRecorder: SharedRef<AVAudioRecorder>, RecordingResultHandler {
480 }
481
482 func didFinish(_ recorder: AVAudioRecorder, successfully flag: Bool) {
@@ -461,6 +497,19 @@ index 198fe6dc95b564ff3916657f14ea67fc44d2e424..5b396114bbeb5e1c99da54f4e6d2376a
497 "url": recorder.url.absoluteString
498 ])
499 }
500 +diff --git a/ios/AudioUtils.swift b/ios/AudioUtils.swift
501 +--- a/ios/AudioUtils.swift
502 ++++ b/ios/AudioUtils.swift
503 +@@ -23,4 +23,9 @@ struct RecordingUtils {
504 + static func setInput(_ input: String) throws {
505 ++ let session = AVAudioSession.sharedInstance()
506 ++ if session.currentRoute.inputs.first?.uid == input && session.preferredInput?.uid == input {
507 ++ return
508 ++ }
509 ++
510 + var preferredInput: AVAudioSessionPortDescription?
511 + if let currentInputs = AVAudioSession.sharedInstance().availableInputs {
512 + for desc in currentInputs where desc.uid == input {
513 diff --git a/src/AudioModule.types.ts b/src/AudioModule.types.ts
514 index 3194a8ebd1b91793d03a1692c563b5d2e83c2738..d8822a6c2ad8a581275e2ca428f4b5d0ed961be2 100644
515 --- a/src/AudioModule.types.ts
pnpm-lock.yaml
+3 -3
@@ -379,7 +379,7 @@ patchedDependencies:
379 app-builder-lib@26.16.1: 0fbe9ae51783f5768ebe197d12d1fdb7d477a08c92e5db2f6356bcce62613c53
380 dbus-next@0.10.2: cfff57561b0ee59b5addb3b2e6c6f20906e967507a530ab67e8db8108e520ba4
381 electron-updater@6.8.9: 91b5648a7eed42761523ef542ebb7e548b037eda41d66533c8152ac65ac72155
382 - expo-audio@58.0.11: 93691f7f59edb30a3bb2c23b489e605e7ea35506b1212d4a2ebbd1fd79079483
382 + expo-audio@58.0.11: a483900b44be8042b1eeaa9893fc2c9c7f53184c7a0b349e9d9d61262b306954
383 expo-modules-core@58.0.30: f0b0fd100550ed76e5c25b4d13eb7c9ff62825f8f197c53edf7bf447c50c9fb7
384 expo-network@58.0.4: 159c2d760d1cd5102e95df51c20600111e7a72096ef0daea8215d50cae48bd98
385 expo-sharing@58.0.22: 321a86e1471e34c53f1f2937fbb4304ea3f7b7a78ad9e4d35febc2b9e73e9688
@@ -644,7 +644,7 @@ importers:
644 version: 58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)
645 expo-audio:
646 specifier: 58.0.11
647 - version: 58.0.11(patch_hash=93691f7f59edb30a3bb2c23b489e605e7ea35506b1212d4a2ebbd1fd79079483)(expo-asset@58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
647 + version: 58.0.11(patch_hash=a483900b44be8042b1eeaa9893fc2c9c7f53184c7a0b349e9d9d61262b306954)(expo-asset@58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
648 expo-auth-session:
649 specifier: ~58.0.17
650 version: 58.0.17(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)
@@ -19312,7 +19312,7 @@ snapshots:
19312 - supports-color
19313 - typescript
19314
19315 - expo-audio@58.0.11(patch_hash=93691f7f59edb30a3bb2c23b489e605e7ea35506b1212d4a2ebbd1fd79079483)(expo-asset@58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
19315 + expo-audio@58.0.11(patch_hash=a483900b44be8042b1eeaa9893fc2c9c7f53184c7a0b349e9d9d61262b306954)(expo-asset@58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
19316 dependencies:
19317 expo: 58.0.23(d226faf1ca90e630d8c56a2108dee32d)
19318 expo-asset: 58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)