The best way to control Claude Code, Codex and more from any computer or phone
fix(mobile): keep the opening words of local iPhone dictation (#31801)
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
Julius Marminge committed
Nov 5, 2026 at 22:08 UTC
98b930f932a8754de5a9ca87fceb4496bcfb99f9
6 files changed
+151
-20
apps/mobile/src/native/voiceTranscription.android.test.ts
+3
-1
@@ -78,10 +78,11 @@ it.each([
78
selection: { start: 16, end: 16 },
79
revision: 1,
80
};
81
+ const prepareToRecordAsync = vi.fn(async () => {});
82
const controller = new VoiceInputController({
83
recorder: {
84
uri: "file:///acceptance.m4a",
84
- prepareToRecordAsync: async () => {},
85
+ prepareToRecordAsync,
86
record: () => {},
87
stop: async () => {},
88
},
@@ -101,4 +102,5 @@ it.each([
102
expect(commit).not.toHaveBeenCalled();
103
expect(draft.text).toBe("Preserved draft.");
104
expect(controller.currentState.error).toMatch(expected);
105
+ expect(prepareToRecordAsync).toHaveBeenCalledTimes(phase === "preparation" ? 0 : 1);
106
});
apps/mobile/src/native/voiceTranscription.ios.test.ts
+94
-1
@@ -1,7 +1,8 @@
1
import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription";
2
import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
3
4
-import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
4
+import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
5
+import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller";
6
7
const mocks = vi.hoisted(() => ({
8
isAvailable: vi.fn<(locale: string) => boolean>(),
@@ -65,8 +66,43 @@ async function transcribeOnce(signal = new AbortController().signal) {
66
.catch((error: unknown) => error);
67
}
68
69
+/** Drives one on-device dictation through the controller the composer uses. */
70
+function createDictation() {
71
+ const recorder = {
72
+ uri: "file:///voice.m4a",
73
+ prepareToRecordAsync: async () => {},
74
+ record: vi.fn(),
75
+ stop: vi.fn(async () => {}),
76
+ };
77
+ const commits: string[] = [];
78
+ const deleted: string[] = [];
79
+ const ended = Promise.withResolvers<void>();
80
+ const controller = new VoiceInputController({
81
+ recorder,
82
+ getTranscriber: () => ({
83
+ prepare: async (options) => (await getLocalVoiceTranscriber())!.prepare(options),
84
+ }),
85
+ requestPermission: async () => ({ granted: true, canAskAgain: true }),
86
+ configureRecording: async () => {},
87
+ releaseRecording: async () => {},
88
+ deleteRecording: (uri) => deleted.push(uri),
89
+ readDraft: () => ({
90
+ ownerKey: "environment:thread",
91
+ text: "",
92
+ selection: { start: 0, end: 0 },
93
+ revision: 1,
94
+ }),
95
+ commitDraft: (text) => commits.push(text),
96
+ onStateChange: (state) => {
97
+ if (state.phase === "idle" || state.phase === "error") ended.resolve();
98
+ },
99
+ });
100
+ return { controller, recorder, commits, deleted, ended: ended.promise };
101
+}
102
+
103
beforeEach(() => {
104
vi.resetAllMocks();
105
+ resetVoiceInputGlobalsForTests();
106
useLanguages([], "sv-SE");
107
mocks.isAvailable.mockReturnValue(true);
108
mocks.prepare.mockResolvedValue("sv-SE");
@@ -147,6 +183,63 @@ describe("getLocalVoiceTranscriber", () => {
183
},
184
);
185
186
+ it("records while Apple prepares its speech model and transcribes once it is ready", async () => {
187
+ const nativePrepare = Promise.withResolvers<string>();
188
+ mocks.prepare.mockReturnValue(nativePrepare.promise);
189
+ const dictation = createDictation();
190
+
191
+ await dictation.controller.start();
192
+ expect(dictation.recorder.record).toHaveBeenCalledOnce();
193
+ expect(dictation.controller.currentState.phase).toBe("recording");
194
+
195
+ const stopping = dictation.controller.stop();
196
+ nativePrepare.resolve("sv-SE");
197
+ await stopping;
198
+
199
+ expect(mocks.transcribe).toHaveBeenCalledOnce();
200
+ expect(dictation.commits).toEqual(["Hej världen."]);
201
+ expect(dictation.deleted).toEqual(["file:///voice.m4a"]);
202
+ });
203
+
204
+ it.each(["cancel", "cancel after stop", "model failure", "model failure after stop"] as const)(
205
+ "discards early audio without a late commit after %s",
206
+ async (ending) => {
207
+ const nativePrepare = Promise.withResolvers<string>();
208
+ mocks.prepare.mockReturnValue(nativePrepare.promise);
209
+ const dictation = createDictation();
210
+ await dictation.controller.start();
211
+
212
+ let finishing: Promise<void> | undefined;
213
+ const cancelled = ending === "cancel" || ending === "cancel after stop";
214
+ if (ending.endsWith("after stop")) finishing = dictation.controller.stop();
215
+ if (cancelled) {
216
+ dictation.controller.cancel();
217
+ } else {
218
+ nativePrepare.reject(Object.assign(new Error("download failed"), { code: "Network" }));
219
+ }
220
+ await finishing;
221
+ await dictation.ended;
222
+
223
+ const next = createDictation();
224
+ mocks.prepare.mockResolvedValue("sv-SE");
225
+ await next.controller.start();
226
+ expect(next.controller.currentState.phase).toBe("recording");
227
+ expect(dictation.deleted).toEqual(["file:///voice.m4a"]);
228
+ expect(mocks.transcribe).not.toHaveBeenCalled();
229
+ await next.controller.stop();
230
+ expect(next.commits).toEqual(["Hej världen."]);
231
+ nativePrepare.resolve("sv-SE");
232
+
233
+ expect(dictation.commits).toEqual([]);
234
+ expect(dictation.recorder.stop).toHaveBeenCalled();
235
+ expect(dictation.controller.currentState).toMatchObject(
236
+ cancelled
237
+ ? { phase: "idle" }
238
+ : { phase: "error", error: "Voice transcription could not prepare this language." },
239
+ );
240
+ },
241
+ );
242
+
243
it("does not start native transcription after cancellation during a file read", async () => {
244
const enteredRead = Promise.withResolvers<void>();
245
const readResult = Promise.withResolvers<ArrayBuffer>();
apps/mobile/src/native/voiceTranscription.ios.ts
+37
-8
@@ -63,6 +63,41 @@ async function prepareVoiceTranscription(
63
});
64
}
65
66
+ let locale = locales[0]!;
67
+ const model = prepareLocales(locales, signal).then((supported) => {
68
+ locale = supported;
69
+ });
70
+ // Recorder setup can fail before either consumer observes the model.
71
+ void model.catch(() => {});
72
+
73
+ async function ready(signal: AbortSignal) {
74
+ throwIfVoiceTranscriptionAborted(signal);
75
+ const cancelled = Promise.withResolvers<void>();
76
+ const cancel = () => cancelled.resolve();
77
+ signal.addEventListener("abort", cancel, { once: true });
78
+ try {
79
+ await Promise.race([model, cancelled.promise]);
80
+ throwIfVoiceTranscriptionAborted(signal);
81
+ return locale;
82
+ } finally {
83
+ signal.removeEventListener("abort", cancel);
84
+ }
85
+ }
86
+ return {
87
+ get locale() {
88
+ return locale;
89
+ },
90
+ start: async (_uri, { signal }) => {
91
+ await ready(signal);
92
+ return null;
93
+ },
94
+ transcribe: async (uri, options) =>
95
+ transcribeVoiceRecording(uri, await ready(options.signal), options),
96
+ };
97
+}
98
+
99
+async function prepareLocales(locales: readonly string[], signal: AbortSignal): Promise<string> {
100
+ throwIfVoiceTranscriptionAborted(signal);
101
let unsupported: VoiceTranscriptionError | undefined;
102
for (const locale of locales) {
103
try {
@@ -75,17 +110,11 @@ async function prepareVoiceTranscription(
110
throw unsupported!;
111
}
112
78
-async function prepareLocale(
79
- locale: string,
80
- signal: AbortSignal,
81
-): Promise<PreparedVoiceTranscription> {
113
+async function prepareLocale(locale: string, signal: AbortSignal): Promise<string> {
114
try {
115
const supportedLocale = await AppleTranscription.prepare(locale);
116
throwIfVoiceTranscriptionAborted(signal);
85
- return {
86
- locale: supportedLocale,
87
- transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options),
88
- };
117
+ return supportedLocale;
118
} catch (error) {
119
throwIfVoiceTranscriptionAborted(signal);
120
if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") {
packages/client-runtime/src/voice-input/controller.test.ts
+4
@@ -309,7 +309,9 @@ describe("VoiceInputController", () => {
309
code: "unavailable",
310
message: "This device requires iOS 26 or later.",
311
});
312
+ const configureRecording = vi.fn(async () => undefined);
313
const unavailable = createHarness({
314
+ configureRecording,
315
getTranscriber: () => ({
316
prepare: async () => {
317
throw reason;
@@ -318,6 +320,8 @@ describe("VoiceInputController", () => {
320
});
321
await unavailable.controller.start();
322
expect(unavailable.controller.currentState.error).toBe(reason.message);
323
+ // Environment and web transcribers gate capture on consent checked during preparation.
324
+ expect(configureRecording).not.toHaveBeenCalled();
325
expect(unavailable.recorder.record).not.toHaveBeenCalled();
326
327
const denied = createHarness({
packages/client-runtime/src/voice-input/controller.ts
+7
-9
@@ -236,7 +236,7 @@ export class VoiceInputController {
236
private operationToken = 0;
237
private sessionToken: VoiceInputSession | null = null;
238
private transcription: PreparedVoiceTranscription | null = null;
239
- private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null;
239
+ private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null;
240
private transcriptionAbortController: AbortController | null = null;
241
private postProcessingAbortController: AbortController | null = null;
242
private skipPostProcessingRequested = false;
@@ -504,17 +504,15 @@ export class VoiceInputController {
504
const capturedDraft = this.capturedDraft;
505
let transcript: string;
506
try {
507
- if (this.liveTranscription) {
508
- const result = await this.liveTranscription;
509
- if ("error" in result) throw result.error;
510
- transcript = result.text;
511
- } else {
512
- transcript = await runTranscriptionOperation(() =>
507
+ const live = this.liveTranscription ? await this.liveTranscription : null;
508
+ if (live && "error" in live) throw live.error;
509
+ transcript =
510
+ live?.text ??
511
+ (await runTranscriptionOperation(() =>
512
transcription.streaming
513
? transcription.streaming.finish({ signal })
514
: transcription.transcribe(recordingUri!, { signal }),
516
- );
517
- }
515
+ ));
516
} catch (error) {
517
if (this.isCurrent(operationToken)) {
518
if (
packages/client-runtime/src/voice-input/transcription.ts
+6
-1
@@ -11,7 +11,12 @@ export type VoiceStreamingSession = {
11
12
export type PreparedVoiceTranscription = {
13
readonly locale: string;
14
- readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
14
+ /**
15
+ * Runs alongside capture once recording starts. Resolves with the transcript when it
16
+ * transcribed live, or null when `transcribe` still runs after the recording stops.
17
+ * A rejection stops the recording.
18
+ */
19
+ readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string | null>;
20
readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
21
/** Present when audio streams during capture; `finish` then replaces `transcribe`. */
22
readonly streaming?: VoiceStreamingSession;