The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
fix(voice): start dictation capture sooner on iOS and desktop (#40011)
Julius Marminge committed
Nov 20, 2026 at 16:27 UTC
3027c2c8400438193cbf9903236e2f3fe644eaa6
14 files changed
+294
-90
apps/mobile/src/features/voice-input/environmentTranscriber.test.ts
+13
-2
@@ -4,17 +4,22 @@ import { createEnvironmentTranscriber } from "./environmentTranscriber";
4
5
it("never uploads a recording whose grant changed while the phone decoded it", async () => {
6
let grant = 1;
7
+ const decoding = Promise.withResolvers<void>();
8
const decoded = Promise.withResolvers<Uint8Array>();
9
const upload = vi.fn(async () => "hello");
10
const transcriber = createEnvironmentTranscriber({
11
canSend: () => grant === 1,
12
prepare: async () => "en",
12
- decode: () => decoded.promise,
13
+ decode: () => {
14
+ decoding.resolve();
15
+ return decoded.promise;
16
+ },
17
upload,
18
});
19
const options = { signal: new AbortController().signal };
20
const recording = await transcriber.prepare(options);
21
const transcription = recording.transcribe("file:///draft.m4a", options);
22
+ await decoding.promise;
23
grant++;
24
decoded.resolve(new Uint8Array([0, 128, 0, 64]));
25
await expect(transcription).rejects.toMatchObject({ code: "cancelled" });
@@ -26,14 +31,20 @@ it("uploads normalized mono samples only after the recording is finished", async
31
expect(Array.from(new Float32Array(Uint8Array.from(bytes).buffer))).toEqual([-1, 0.5]);
32
return "hello";
33
});
34
+ const ready = Promise.withResolvers<string>();
35
const transcriber = createEnvironmentTranscriber({
36
canSend: () => true,
31
- prepare: async () => "en",
37
+ prepare: () => ready.promise,
38
decode: async () => new Uint8Array([255, 0, 128, 0, 64, 255]).subarray(1, 5),
39
upload,
40
});
41
const options = { signal: new AbortController().signal };
42
const recording = await transcriber.prepare(options);
43
+ const starting = recording.start?.("file:///draft.m4a", options);
44
+ expect(upload).not.toHaveBeenCalled();
45
+ ready.resolve("en");
46
+ await starting;
47
+ expect(recording.locale).toBe("en");
48
expect(upload).not.toHaveBeenCalled();
49
await expect(recording.transcribe("file:///draft.m4a", options)).resolves.toBe("hello");
50
});
apps/mobile/src/features/voice-input/environmentTranscriber.ts
+15
-3
@@ -29,11 +29,23 @@ export function createEnvironmentTranscriber(input: {
29
return {
30
async prepare({ signal }) {
31
check(signal);
32
- const locale = await input.prepare(signal);
33
- check(signal);
32
+ let locale = "auto";
33
+ const ready = input.prepare(signal).then((resolved) => {
34
+ check(signal);
35
+ locale = resolved;
36
+ });
37
+ void ready.catch(() => {});
38
return {
35
- locale,
39
+ get locale() {
40
+ return locale;
41
+ },
42
+ async start(_uri, { signal }) {
43
+ await ready;
44
+ check(signal);
45
+ return null;
46
+ },
47
async transcribe(uri, { signal }) {
48
+ await ready;
49
check(signal);
50
const decoded = await input.decode(uri);
51
check(signal);
apps/mobile/src/features/voice-input/voiceDestination.test.ts
+2
-1
@@ -172,7 +172,8 @@ it("keeps local as the default and never reuses consent for an offline or resele
172
chooseVoiceDestination(host, "openai");
173
const cloud = appAtomRegistry.get(voiceRouteAtom)!.transcriber!;
174
mocks.fetch.mockResolvedValueOnce(Response.json({ openai: false, xai: true }));
175
- await expect(cloud.prepare(options)).rejects.toMatchObject({
175
+ const missingKey = await cloud.prepare(options);
176
+ await expect(missingKey.start?.("file:///voice.m4a", options)).rejects.toMatchObject({
177
code: "unavailable",
178
message: expect.stringContaining("Add your OpenAI API key"),
179
});
apps/web/src/components/chat/ComposerSpeechButton.tsx
+28
-13
@@ -55,29 +55,37 @@ function formatElapsed(seconds: number) {
55
56
function RecordingStatus(props: {
57
level: number;
58
+ preparing: boolean;
59
recordingStartedAt?: number | null | undefined;
60
recordingLimitSeconds?: number | undefined;
61
}) {
61
- const { recordingStartedAt } = props;
62
+ const { preparing, recordingStartedAt } = props;
63
const [elapsedSeconds, setElapsedSeconds] = useState(() =>
64
recordingStartedAt ? Math.max(0, Math.floor((Date.now() - recordingStartedAt) / 1_000)) : 0,
65
);
66
67
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the timer displaying the native recording duration
68
useEffect(() => {
69
+ if (preparing) return;
70
const startedAt = recordingStartedAt ?? Date.now();
71
const timer = window.setInterval(
72
() => setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1_000)),
73
250,
74
);
75
return () => window.clearInterval(timer);
74
- }, [recordingStartedAt]);
76
+ }, [preparing, recordingStartedAt]);
77
78
return (
79
<>
78
- <span aria-hidden className="size-2 shrink-0 rounded-full bg-destructive" />
80
+ <span
81
+ aria-hidden
82
+ className={cn(
83
+ "size-2 shrink-0 rounded-full",
84
+ preparing ? "bg-muted-foreground/40" : "bg-destructive",
85
+ )}
86
+ />
87
<span className="shrink-0 text-xs text-muted-foreground tabular-nums">
80
- {formatElapsed(elapsedSeconds)}
88
+ {formatElapsed(preparing ? 0 : elapsedSeconds)}
89
{props.recordingLimitSeconds !== undefined
90
? ` / ${formatElapsed(props.recordingLimitSeconds)}`
91
: ""}
@@ -174,7 +182,9 @@ export function ComposerSpeechRecordingPill(props: {
182
const presentation = resolveSpeechPresentation(props.state, props.progress, m);
183
if (!presentation.active) return null;
184
const recording = phase === "recording";
177
- const trailing = recording || phase === "post-processing";
185
+ const starting = phase === "preparing" && props.progress === null;
186
+ const showRecordingLayout = starting || recording;
187
+ const trailing = showRecordingLayout || phase === "post-processing";
188
189
return (
190
<VoiceInputPill size={props.size}>
@@ -200,12 +210,16 @@ export function ComposerSpeechRecordingPill(props: {
210
aria-live={recording ? "off" : "polite"}
211
aria-label={presentation.status ?? undefined}
212
>
203
- {recording ? (
204
- <RecordingStatus
205
- level={props.level}
206
- recordingStartedAt={props.recordingStartedAt}
207
- recordingLimitSeconds={props.recordingLimitSeconds}
208
- />
213
+ {showRecordingLayout ? (
214
+ <>
215
+ {starting && <span className="sr-only">{presentation.status}</span>}
216
+ <RecordingStatus
217
+ preparing={starting}
218
+ level={starting ? 0 : props.level}
219
+ recordingStartedAt={props.recordingStartedAt}
220
+ recordingLimitSeconds={props.recordingLimitSeconds}
221
+ />
222
+ </>
223
) : (
224
<>
225
<Spinner aria-hidden size="sm" tone="muted" className="shrink-0" />
@@ -226,17 +240,18 @@ export function ComposerSpeechRecordingPill(props: {
240
>
241
{gt("Skip", { $context: "verb, skips cleaning up the transcript" })}
242
</Button>
229
- ) : recording ? (
243
+ ) : showRecordingLayout ? (
244
<ShortcutTooltip label={gt("Finish voice input")} shortcut={props.finishShortcutLabel}>
245
<Button
246
type="button"
247
size={compact ? "icon-xs" : "icon-sm-pill"}
248
aria-label={gt("Finish voice input")}
249
+ disabled={starting}
250
onPointerDown={keepFocus}
251
onClick={props.onStop}
252
className="shrink-0"
253
>
239
- <CheckIcon />
254
+ {starting ? <Spinner aria-hidden /> : <CheckIcon />}
255
</Button>
256
</ShortcutTooltip>
257
) : null}
apps/web/src/speech/browserPcmCapture.ts
+61
-12
@@ -1,15 +1,19 @@
1
-import type {
2
- PreparedVoiceTranscription,
3
- VoiceRecorder,
4
- VoiceStreamingSession,
5
- VoiceInputErrorCode,
6
- VoiceTranscriptionOptions,
1
+import {
2
+ VoiceTranscriptionError,
3
+ type PreparedVoiceTranscription,
4
+ type VoiceRecorder,
5
+ type VoiceStreamingSession,
6
+ type VoiceInputErrorCode,
7
+ type VoiceTranscriptionOptions,
8
} from "@t3tools/client-runtime/voice-input";
9
+import * as Schema from "effect/Schema";
10
import { requestMicrophone } from "./requestMicrophone";
11
import workletUrl from "./pcmWorklet.ts?worker&url";
12
import { msg } from "gt-react";
13
import { m } from "../i18n/messages";
14
15
+const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
16
+
17
type PreparedBrowserTranscription = PreparedVoiceTranscription & {
18
readonly streaming?: VoiceStreamingSession & { readonly feed: (pcm: Float32Array) => void };
19
};
@@ -21,7 +25,10 @@ export function createBrowserPcmCapture(
25
readonly onDurationLimit: () => void;
26
readonly onError: (message: string, code?: VoiceInputErrorCode) => void;
27
},
24
- prepare: (options: VoiceTranscriptionOptions) => Promise<PreparedBrowserTranscription>,
28
+ prepare: (
29
+ options: VoiceTranscriptionOptions,
30
+ canStream: () => boolean,
31
+ ) => Promise<PreparedBrowserTranscription>,
32
) {
33
let stream: MediaStream | undefined;
34
let recordingUri: string | null = null;
@@ -30,6 +37,7 @@ export function createBrowserPcmCapture(
37
let worklet: AudioWorkletNode | undefined;
38
let durationTimer: ReturnType<typeof setTimeout> | undefined;
39
let signal: AbortSignal | undefined;
40
+ let stopAcceptingAudio: (() => void) | undefined;
41
let feed: ((pcm: Float32Array) => void) | undefined;
42
let stopped:
43
| { readonly resolve: () => void; readonly reject: (error: Error) => void }
@@ -48,6 +56,8 @@ export function createBrowserPcmCapture(
56
stopped?.resolve();
57
stopped = undefined;
58
chunks = [];
59
+ stopAcceptingAudio?.();
60
+ stopAcceptingAudio = undefined;
61
feed = undefined;
62
input.onLevel(0);
63
};
@@ -175,12 +185,51 @@ export function createBrowserPcmCapture(
185
if (recordingUri === uri) recordingUri = null;
186
},
187
transcriber: {
178
- prepare: async (options: VoiceTranscriptionOptions) => {
179
- const prepared = await prepare(options);
180
- options.signal.throwIfAborted();
188
+ prepare: async (options: VoiceTranscriptionOptions): Promise<PreparedVoiceTranscription> => {
189
signal = options.signal;
182
- feed = prepared.streaming?.feed;
183
- return prepared;
190
+ let acceptingAudio = true;
191
+ stopAcceptingAudio = () => {
192
+ acceptingAudio = false;
193
+ };
194
+ const canStream = () => acceptingAudio && !options.signal.aborted;
195
+ let streaming: VoiceStreamingSession | undefined;
196
+ let destination: PreparedBrowserTranscription | undefined;
197
+ const pending = prepare(options, canStream)
198
+ .then((prepared) => {
199
+ options.signal.throwIfAborted();
200
+ destination = prepared;
201
+ if (canStream() && prepared.streaming) {
202
+ streaming = prepared.streaming;
203
+ feed = prepared.streaming.feed;
204
+ for (const chunk of chunks) feed(chunk);
205
+ chunks = [];
206
+ }
207
+ return prepared;
208
+ })
209
+ .catch((cause: unknown) => {
210
+ const error = new VoiceTranscriptionError({
211
+ code: "preparation-failed",
212
+ message: isVoiceTranscriptionError(cause)
213
+ ? cause.message
214
+ : m(msg("Could not prepare voice transcription.")),
215
+ cause,
216
+ });
217
+ if (canStream()) input.onError(error.message);
218
+ throw error;
219
+ });
220
+ void pending.catch(() => {});
221
+ return {
222
+ get locale() {
223
+ return destination?.locale ?? "auto";
224
+ },
225
+ get streaming() {
226
+ return streaming;
227
+ },
228
+ async transcribe(uri, transcriptionOptions) {
229
+ const prepared = await pending;
230
+ return prepared.transcribe(uri, transcriptionOptions);
231
+ },
232
+ };
233
},
234
},
235
};
apps/web/src/speech/browserVoiceInput.browser.test.ts
+46
-3
@@ -9,6 +9,7 @@ import { decodeSpeechPcmRequest } from "@t3tools/shared/speech";
9
import { expect, it, vi } from "vite-plus/test";
10
11
import { stubGlobal } from "../../test/render";
12
+import { createBrowserPcmCapture } from "./browserPcmCapture";
13
import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
14
15
vi.mock("../connection/runtime", async () => {
@@ -158,13 +159,15 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be
159
await vi.waitFor(() => expect(_network).toHaveBeenCalled());
160
streaming = true;
161
options = { ...options, speechLanguage: "en" };
161
- resolveStatus(Response.json(status));
162
await start;
163
expect(controller.currentState.phase).toBe("recording");
164
microphone.feed(new Float32Array([0.1, 0.2]));
165
microphone.feed(new Float32Array([0.3]));
166
expect(requests).toHaveLength(0);
167
- await controller.stop();
167
+ const stop = controller.stop();
168
+ await vi.waitFor(() => expect(microphone.stopTrack).toHaveBeenCalledOnce());
169
+ resolveStatus(Response.json(status));
170
+ await stop;
171
expect(controller.currentState.phase).toBe("error");
172
expect(platform.recorder.uri).not.toBeNull();
173
await controller.start();
@@ -191,6 +194,46 @@ it("retains all PCM until stop, retries batch audio, and captures preferences be
194
expect(platform.recorder.uri).toBeNull();
195
});
196
197
+it("streams the opening audio in order when the destination becomes ready during capture", async () => {
198
+ using microphone = capture();
199
+ const feed = vi.fn();
200
+ const finish = vi.fn(async () => "hello");
201
+ const ready = Promise.withResolvers<void>();
202
+ const platform = createBrowserPcmCapture(
203
+ {
204
+ getMicrophoneId: () => "",
205
+ onLevel() {},
206
+ onDurationLimit() {},
207
+ onError() {},
208
+ },
209
+ async () => {
210
+ await ready.promise;
211
+ return { locale: "en", streaming: { feed, finish }, transcribe: async () => "" };
212
+ },
213
+ );
214
+ const abort = new AbortController();
215
+ using _abort = {
216
+ [Symbol.dispose]: () => {
217
+ abort.abort();
218
+ platform.cancelRecording();
219
+ },
220
+ };
221
+ const transcription = await platform.transcriber.prepare({ signal: abort.signal });
222
+ await platform.recorder.prepareToRecordAsync(abort.signal);
223
+ platform.recorder.record({ forDuration: 1 });
224
+ const opening = new Float32Array([0.1]);
225
+ const continuing = new Float32Array([0.2]);
226
+ microphone.feed(opening);
227
+ expect(feed).not.toHaveBeenCalled();
228
+ ready.resolve();
229
+ await vi.waitFor(() => expect(feed).toHaveBeenCalledExactlyOnceWith(opening));
230
+ microphone.feed(continuing);
231
+ await platform.recorder.stop();
232
+ expect(feed.mock.calls).toEqual([[opening], [continuing]]);
233
+ expect(platform.recorder.uri).toBeNull();
234
+ await expect(transcription.streaming?.finish({ signal: abort.signal })).resolves.toBe("hello");
235
+});
236
+
237
it.each([
238
{ native: true, enabled: true, buffered: true, live: true },
239
{ native: true, enabled: false, buffered: true, live: false },
@@ -241,8 +284,8 @@ it.each([
284
onError() {},
285
});
286
const transcription = await platform.transcriber.prepare({ signal: abort.signal });
287
+ await vi.waitFor(() => expect(transcription.locale).toBe(native && enabled ? "fr" : "en"));
288
expect(Boolean(transcription.streaming)).toBe(live);
245
- expect(transcription.locale).toBe(native && enabled ? "fr" : "en");
289
expect(sockets).toEqual(
290
live ? ["wss://transcription.example.test/ws/voice?wsTicket=stream-ticket"] : [],
291
);
apps/web/src/speech/browserVoiceInput.ts
+25
-9
@@ -12,7 +12,11 @@ import {
12
type VoiceTranscriptionOptions,
13
} from "@t3tools/client-runtime/voice-input";
14
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
15
-import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";
15
+import type {
16
+ SpeechTranscriptionOptions,
17
+ SpeechStreamText,
18
+ EnvironmentSpeechStatus,
19
+} from "@t3tools/contracts";
20
import type { CloudSpeechModelId, CloudSpeechProvider } from "@t3tools/contracts";
21
import * as Schema from "effect/Schema";
22
@@ -30,6 +34,7 @@ export function createBrowserVoiceInputPlatform(input: {
34
readonly cloudModel?: CloudSpeechModelId | undefined;
35
readonly hasAudioConsent: () => boolean;
36
readonly getTranscriptionOptions: () => SpeechTranscriptionOptions;
37
+ readonly getStatusListener?: () => ((status: EnvironmentSpeechStatus) => void) | undefined;
38
readonly getStreamingEnabled: () => boolean;
39
readonly getMicrophoneId: () => string | Promise<string>;
40
readonly onLevel: (level: number) => void;
@@ -37,7 +42,8 @@ export function createBrowserVoiceInputPlatform(input: {
42
readonly onText: (text: SpeechStreamText) => void;
43
readonly onError: (message: string, code?: VoiceInputErrorCode) => void;
44
}) {
40
- return createBrowserPcmCapture(input, async (options) => {
45
+ return createBrowserPcmCapture(input, async (options, canStream) => {
46
+ const onStatus = input.getStatusListener?.();
47
const recordingOptions = input.getTranscriptionOptions();
48
const streamingEnabled = input.getStreamingEnabled();
49
const transcribeRecording = async (
@@ -95,8 +101,15 @@ export function createBrowserVoiceInputPlatform(input: {
101
}
102
const status = await runEnvironmentSpeech(getEnvironmentSpeechStatus(input.prepared), options);
103
throwIfVoiceTranscriptionAborted(options.signal);
104
+ onStatus?.(status);
105
+ throwIfVoiceTranscriptionAborted(options.signal);
106
if (!status.supported)
107
throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
108
+ if (status.state === "missing-model")
109
+ throw new VoiceTranscriptionError({
110
+ code: "unavailable",
111
+ message: m(msg("Could not start voice input")),
112
+ });
113
const language = status.languages
114
? effectiveSpeechLanguage(
115
{
@@ -107,16 +120,19 @@ export function createBrowserVoiceInputPlatform(input: {
120
)
121
: "auto";
122
const nativeStreaming = status.supportsStreaming && streamingEnabled;
110
- const locale =
111
- !nativeStreaming && status.supportsTranslation && recordingOptions.speechTranslateToEnglish
112
- ? "en"
113
- : language;
114
- if (status.supportsStreaming ? !nativeStreaming : !status.supportsBufferedStreaming) {
115
- return { locale, transcribe: transcribeRecording };
123
+ const batchLocale =
124
+ status.supportsTranslation && recordingOptions.speechTranslateToEnglish ? "en" : language;
125
+ const batch = { locale: batchLocale, transcribe: transcribeRecording };
126
+ if (
127
+ !canStream() ||
128
+ (status.supportsStreaming ? !nativeStreaming : !status.supportsBufferedStreaming)
129
+ ) {
130
+ return batch;
131
}
132
const url = await runEnvironmentSpeech(getEnvironmentSpeechStreamUrl(input.prepared), options);
133
+ if (!canStream()) return batch;
134
return {
119
- locale,
135
+ locale: nativeStreaming ? language : batchLocale,
136
transcribe: transcribeRecording,
137
streaming: openSpeechStream({
138
url,
apps/web/src/speech/useEnvironmentSpeechInput.ts
+35
-36
@@ -6,7 +6,6 @@ import {
6
getSpeechModelDownloadOutcome,
7
selectEnvironmentSpeechModel,
8
voiceInputBlocksSubmission,
9
- type VoiceInputState,
9
} from "@t3tools/client-runtime/voice-input";
10
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
11
import { AuthOrchestrationOperateScope, CLOUD_SPEECH_LABELS } from "@t3tools/contracts";
@@ -46,7 +45,6 @@ import {
45
type VoiceInputTarget,
46
} from "./voiceInputSession";
47
49
-const WAITING_STATE: VoiceInputState = { phase: "preparing", error: null, errorAction: null };
48
const INITIAL_SETUP = {
49
open: false,
50
step: 0,
@@ -176,7 +174,6 @@ export function useEnvironmentSpeechInput(input: HookInput) {
174
readonly prepared: PreparedConnection;
175
readonly value: EnvironmentSpeechStatus | null;
176
} | null>(null);
179
- const [queuedStart, setQueuedStart] = useState<PreparedConnection | null>(null);
177
const [setup, setSetup] = useState(INITIAL_SETUP);
178
const [previousDestination, setPreviousDestination] = useState({ prepared, grantId });
179
const [previousOwner, setPreviousOwner] = useState({
@@ -186,11 +183,9 @@ export function useEnvironmentSpeechInput(input: HookInput) {
183
if (previousDestination.prepared !== prepared || previousDestination.grantId !== grantId) {
184
setPreviousDestination({ prepared, grantId });
185
setSetup(INITIAL_SETUP);
189
- setQueuedStart(null);
186
}
187
if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) {
188
setPreviousOwner({ key: input.ownerKey, disabled: input.disabled });
193
- setQueuedStart(null);
189
setSetup(INITIAL_SETUP);
190
}
191
@@ -198,8 +193,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
193
const snapshot = useVoiceInputSessionSnapshot(session);
194
const owns = snapshot.target?.ownerKey === input.ownerKey;
195
const activity = useVoiceInputSessionActivity(session, input.ownerKey);
201
- const state: VoiceInputState =
202
- queuedStart === prepared && prepared ? WAITING_STATE : owns ? snapshot.state : IDLE_VOICE_STATE;
196
+ const state = owns ? snapshot.state : IDLE_VOICE_STATE;
197
// One microphone: another draft's recording leaves this editor without voice input until it ends.
198
const busyElsewhere = !owns && voiceInputBlocksSubmission(snapshot.state);
199
@@ -401,48 +395,54 @@ export function useEnvironmentSpeechInput(input: HookInput) {
395
await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });
396
return;
397
}
398
+ if (!session || !prepared || voiceInputBlocksSubmission(session.state)) return;
399
const request = ++startRequestRef.current;
400
const owner = latestRef.current.input.ownerKey;
406
- if (!session || !prepared || busyElsewhere) return;
401
if (cloudProvider) {
402
await session.start(createTarget());
403
return;
404
}
405
+ setSetup(INITIAL_SETUP);
406
const stillCurrent = () =>
407
request === startRequestRef.current &&
408
latestRef.current.session === session &&
409
latestRef.current.input.ownerKey === owner &&
410
!latestRef.current.input.disabled;
416
- setSetup(INITIAL_SETUP);
417
- setQueuedStart(prepared);
418
- let latestStatus: EnvironmentSpeechStatus;
419
- try {
420
- latestStatus = await probeSpeechStatus(prepared, true);
421
- } catch (error) {
411
+ const handleStatus = async (latestStatus: EnvironmentSpeechStatus) => {
412
if (!stillCurrent()) return;
423
- setQueuedStart(null);
424
- toastManager.add({
425
- type: "error",
426
- title: m(msg("Could not start voice input")),
427
- description: error instanceof Error ? error.message : String(error),
428
- });
429
- return;
430
- }
431
- if (!stillCurrent()) return;
432
- setStatus({ prepared, value: latestStatus });
433
- setQueuedStart(null);
434
- if (latestStatus.supported && latestStatus.state === "missing-model") {
435
- return loadSetupCatalog(prepared, latestStatus.modelId, latestStatus.benchmarkTier, request);
436
- }
437
- if (!latestStatus.supported) {
438
- toastManager.add({
439
- type: "error",
440
- title: m(msg("Voice input is not available")),
441
- description: latestStatus.reason,
413
+ setStatus({ prepared, value: latestStatus });
414
+ if (!latestStatus.supported) {
415
+ session.cancel();
416
+ toastManager.add({
417
+ type: "error",
418
+ title: m(msg("Voice input is not available")),
419
+ description: latestStatus.reason,
420
+ });
421
+ } else if (latestStatus.state === "missing-model") {
422
+ session.cancel();
423
+ await loadSetupCatalog(prepared, latestStatus.modelId, latestStatus.benchmarkTier, request);
424
+ }
425
+ };
426
+
427
+ if (currentStatus?.supported && currentStatus.state === "missing-model") {
428
+ // Setup must remain usable without opening the microphone.
429
+ const latestStatus = await probeSpeechStatus(prepared, true).catch((error: unknown) => {
430
+ if (stillCurrent()) {
431
+ toastManager.add({
432
+ type: "error",
433
+ title: m(msg("Could not start voice input")),
434
+ description: error instanceof Error ? error.message : String(error),
435
+ });
436
+ }
437
+ return null;
438
});
443
- return;
439
+ if (!stillCurrent() || !latestStatus) return;
440
+ await handleStatus(latestStatus);
441
+ if (!latestStatus.supported || latestStatus.state === "missing-model" || !stillCurrent())
442
+ return;
443
}
445
- await session.start(createTarget());
444
+
445
+ await session.start(createTarget(), (latestStatus) => void handleStatus(latestStatus));
446
});
447
448
const downloadSetupModel = useStableCallback(async () => {
@@ -562,7 +562,6 @@ export function useEnvironmentSpeechInput(input: HookInput) {
562
cancel: useStableCallback(() => {
563
pendingSubmissionRef.current = null;
564
startRequestRef.current += 1;
565
- setQueuedStart(null);
565
setSetup(INITIAL_SETUP);
566
if (owns) session?.cancel();
567
}),
apps/web/src/speech/voiceInputSession.ts
+8
-1
@@ -12,6 +12,7 @@ import type {
12
CloudSpeechProvider,
13
ClientSettings,
14
EnvironmentId,
15
+ EnvironmentSpeechStatus,
16
SpeechCustomWords,
17
SpeechPostProcessingOptions,
18
SpeechStreamText,
@@ -108,6 +109,7 @@ export class VoiceInputSession {
109
private readonly listeners = new Set<() => void>();
110
private readonly activityListeners = new Set<() => void>();
111
private disposed = false;
112
+ private onSpeechStatus: ((status: EnvironmentSpeechStatus) => void) | undefined;
113
114
constructor(key: SessionKey) {
115
this.key = key;
@@ -124,6 +126,7 @@ export class VoiceInputSession {
126
cloudProvider: key.cloudProvider,
127
cloudModel: key.cloudModel,
128
hasAudioConsent,
129
+ getStatusListener: () => this.onSpeechStatus,
130
getStreamingEnabled: () =>
131
this.target?.readContext().clientSettings.speechStreaming ??
132
DEFAULT_CLIENT_SETTINGS.speechStreaming,
@@ -253,13 +256,17 @@ export class VoiceInputSession {
256
readonly getActivity = (): VoiceInputSessionActivity => this.activity;
257
258
/** Starts recording into `target`, or does nothing while another draft holds the microphone. */
256
- start(target: VoiceInputTarget): Promise<void> {
259
+ start(
260
+ target: VoiceInputTarget,
261
+ onSpeechStatus?: (status: EnvironmentSpeechStatus) => void,
262
+ ): Promise<void> {
263
if (this.disposed) return Promise.resolve();
264
const busyElsewhere =
265
voiceInputBlocksSubmission(this.controller.currentState) &&
266
this.target?.ownerKey !== target.ownerKey;
267
if (busyElsewhere) return Promise.resolve();
268
this.target = target;
269
+ this.onSpeechStatus = onSpeechStatus;
270
this.publishActivity(SILENT_ACTIVITY);
271
this.publish({ ...this.snapshot, target });
272
return this.controller.start();
docs/internals/voice-input.md
+3
-2
@@ -7,8 +7,9 @@ the device requires an explicit grant naming one environment and, for cloud
7
transcription, one provider.
8
9
The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts)
10
-owns the operation while the client supplies capture and transcription. Preparation
11
-binds the transcriber and resolved locale for the whole recording. Draft ownership,
10
+owns the operation while the client supplies capture and transcription. The
11
+transcriber is bound to the recording when it starts. Capture can begin while
12
+the locale and destination are still being resolved. Draft ownership,
13
text, and revision are captured before recording and checked before insertion, so
14
a late transcript cannot overwrite a draft that was edited or replaced. A
15
transcriber may expose a streaming session, in which case audio leaves during
packages/client-runtime/src/voice-input/controller.ts
+1
@@ -612,6 +612,7 @@ export class VoiceInputController {
612
if (this.isCurrent(operationToken)) {
613
if (
614
this.dependencies.retryFailedRecording &&
615
+ errorCode(error) !== "preparation-failed" &&
616
!this.liveTranscription &&
617
!transcription.streaming &&
618
recordingUri
packages/client-runtime/src/voice-input/transcription.ts
+1
-1
@@ -21,7 +21,7 @@ export type PreparedVoiceTranscription = {
21
readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string | null>;
22
readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
23
/** Present when audio streams during capture; `finish` then replaces `transcribe`. */
24
- readonly streaming?: VoiceStreamingSession;
24
+ readonly streaming?: VoiceStreamingSession | undefined;
25
};
26
27
export type VoiceTranscriber = {
patches/expo-audio@58.0.11.patch
+53
-4
@@ -388,7 +388,6 @@ index 0d63c9b949cfae7a101902fc3bb70fcf46094f68..a312b8fcdaf728220db3d6c04d92fd78
388
* Replaces the current audio source with a new one.
389
*/
390
diff --git a/ios/AudioModule.swift b/ios/AudioModule.swift
391
-index 42ce9e8f213ebc243db25ef836456757b0daeb09..b422c3c10349b14803d5e9d7afeed617e76e09af 100644
391
--- a/ios/AudioModule.swift
392
+++ b/ios/AudioModule.swift
393
@@ -6,6 +6,7 @@ public class AudioModule: Module {
@@ -428,7 +427,22 @@ index 42ce9e8f213ebc243db25ef836456757b0daeb09..b422c3c10349b14803d5e9d7afeed617
427
self.deactivateSession()
428
}
429
}
431
-@@ -957,6 +968,7 @@ public class AudioModule: Module {
430
+@@ -486,11 +497,12 @@ public class AudioModule: Module {
431
+ AsyncFunction("prepareToRecordAsync") { (recorder, options: RecordingOptions?) in
432
+ try AudioRecordingRequester.requireUsageDescription()
433
+- let deactivateSessionOnFailure = sessionQueue.sync {
434
+- !sessionIsActive && audioSessionActivityKeepers.isEmpty
435
++ let (activateSession, deactivateSessionOnFailure) = sessionQueue.sync {
436
++ (!hasAudioSessionOwner || !sessionIsActive, !sessionIsActive && audioSessionActivityKeepers.isEmpty)
437
+ }
438
+ try recorder.prepare(
439
+ options: options,
440
+ sessionOptions: sessionOptions,
441
++ activateSession: activateSession,
442
+ deactivateSessionOnFailure: deactivateSessionOnFailure
443
+ )
444
+ recordSessionActive(true)
445
+@@ -957,6 +969,7 @@ public class AudioModule: Module {
446
sessionQueue.asyncAfter(deadline: .now() + .milliseconds(100)) { [weak self] in
447
guard let self,
448
self.sessionIsActive,
@@ -437,10 +451,32 @@ index 42ce9e8f213ebc243db25ef836456757b0daeb09..b422c3c10349b14803d5e9d7afeed617
451
!self.isSessionInUse else {
452
return
453
diff --git a/ios/AudioRecorder.swift b/ios/AudioRecorder.swift
440
-index 198fe6dc95b564ff3916657f14ea67fc44d2e424..5b396114bbeb5e1c99da54f4e6d2376ad978dfa0 100644
454
--- a/ios/AudioRecorder.swift
455
+++ b/ios/AudioRecorder.swift
443
-@@ -285,16 +285,16 @@ class AudioRecorder: SharedRef<AVAudioRecorder>, RecordingResultHandler {
456
+@@ -81,6 +81,7 @@ class AudioRecorder: SharedRef<AVAudioRecorder>, RecordingResultHandler {
457
+ func prepare(
458
+ options: RecordingOptions?,
459
+ sessionOptions: AVAudioSession.CategoryOptions = [],
460
++ activateSession: Bool = true,
461
+ deactivateSessionOnFailure: Bool = true
462
+ ) throws {
463
+ if currentState == .recording {
464
+@@ -93,7 +94,12 @@ class AudioRecorder: SharedRef<AVAudioRecorder>, RecordingResultHandler {
465
+ let session = AVAudioSession.sharedInstance()
466
+ do {
467
+- try session.setCategory(.playAndRecord, mode: .default, options: sessionOptions)
468
+- try session.setActive(true)
469
++ let categoryMatches = session.category == .playAndRecord && session.mode == .default && session.categoryOptions == sessionOptions
470
++ if !categoryMatches {
471
++ try session.setCategory(.playAndRecord, mode: .default, options: sessionOptions)
472
++ }
473
++ if activateSession || !categoryMatches {
474
++ try session.setActive(true)
475
++ }
476
+ } catch {
477
+ currentState = .error
478
+ throw AudioRecordingException("Failed to configure audio session: \(error.localizedDescription)")
479
+@@ -285,16 +291,16 @@ class AudioRecorder: SharedRef<AVAudioRecorder>, RecordingResultHandler {
480
}
481
482
func didFinish(_ recorder: AVAudioRecorder, successfully flag: Bool) {
@@ -461,6 +497,19 @@ index 198fe6dc95b564ff3916657f14ea67fc44d2e424..5b396114bbeb5e1c99da54f4e6d2376a
497
"url": recorder.url.absoluteString
498
])
499
}
500
+diff --git a/ios/AudioUtils.swift b/ios/AudioUtils.swift
501
+--- a/ios/AudioUtils.swift
502
++++ b/ios/AudioUtils.swift
503
+@@ -23,4 +23,9 @@ struct RecordingUtils {
504
+ static func setInput(_ input: String) throws {
505
++ let session = AVAudioSession.sharedInstance()
506
++ if session.currentRoute.inputs.first?.uid == input && session.preferredInput?.uid == input {
507
++ return
508
++ }
509
++
510
+ var preferredInput: AVAudioSessionPortDescription?
511
+ if let currentInputs = AVAudioSession.sharedInstance().availableInputs {
512
+ for desc in currentInputs where desc.uid == input {
513
diff --git a/src/AudioModule.types.ts b/src/AudioModule.types.ts
514
index 3194a8ebd1b91793d03a1692c563b5d2e83c2738..d8822a6c2ad8a581275e2ca428f4b5d0ed961be2 100644
515
--- a/src/AudioModule.types.ts
pnpm-lock.yaml
+3
-3
@@ -379,7 +379,7 @@ patchedDependencies:
379
app-builder-lib@26.16.1: 0fbe9ae51783f5768ebe197d12d1fdb7d477a08c92e5db2f6356bcce62613c53
380
dbus-next@0.10.2: cfff57561b0ee59b5addb3b2e6c6f20906e967507a530ab67e8db8108e520ba4
381
electron-updater@6.8.9: 91b5648a7eed42761523ef542ebb7e548b037eda41d66533c8152ac65ac72155
382
- expo-audio@58.0.11: 93691f7f59edb30a3bb2c23b489e605e7ea35506b1212d4a2ebbd1fd79079483
382
+ expo-audio@58.0.11: a483900b44be8042b1eeaa9893fc2c9c7f53184c7a0b349e9d9d61262b306954
383
expo-modules-core@58.0.30: f0b0fd100550ed76e5c25b4d13eb7c9ff62825f8f197c53edf7bf447c50c9fb7
384
expo-network@58.0.4: 159c2d760d1cd5102e95df51c20600111e7a72096ef0daea8215d50cae48bd98
385
expo-sharing@58.0.22: 321a86e1471e34c53f1f2937fbb4304ea3f7b7a78ad9e4d35febc2b9e73e9688
@@ -644,7 +644,7 @@ importers:
644
version: 58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)
645
expo-audio:
646
specifier: 58.0.11
647
- version: 58.0.11(patch_hash=93691f7f59edb30a3bb2c23b489e605e7ea35506b1212d4a2ebbd1fd79079483)(expo-asset@58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
647
+ version: 58.0.11(patch_hash=a483900b44be8042b1eeaa9893fc2c9c7f53184c7a0b349e9d9d61262b306954)(expo-asset@58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
648
expo-auth-session:
649
specifier: ~58.0.17
650
version: 58.0.17(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)
@@ -19312,7 +19312,7 @@ snapshots:
19312
- supports-color
19313
- typescript
19314
19315
- expo-audio@58.0.11(patch_hash=93691f7f59edb30a3bb2c23b489e605e7ea35506b1212d4a2ebbd1fd79079483)(expo-asset@58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
19315
+ expo-audio@58.0.11(patch_hash=a483900b44be8042b1eeaa9893fc2c9c7f53184c7a0b349e9d9d61262b306954)(expo-asset@58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
19316
dependencies:
19317
expo: 58.0.23(d226faf1ca90e630d8c56a2108dee32d)
19318
expo-asset: 58.0.20(expo@58.0.23)(react-native@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.1(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)