@devries-cameron20 / t3code-1 / commits / 98b930f932
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(mobile): keep the opening words of local iPhone dictation (#31801)

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

Julius Marminge committed Nov 5, 2026 at 22:08 UTC 98b930f932a8754de5a9ca87fceb4496bcfb99f9
6 files changed +151 -20
apps/mobile/src/native/voiceTranscription.android.test.ts
+3 -1
@@ -78,10 +78,11 @@ it.each([
78 selection: { start: 16, end: 16 },
79 revision: 1,
80 };
81 + const prepareToRecordAsync = vi.fn(async () => {});
82 const controller = new VoiceInputController({
83 recorder: {
84 uri: "file:///acceptance.m4a",
84 - prepareToRecordAsync: async () => {},
85 + prepareToRecordAsync,
86 record: () => {},
87 stop: async () => {},
88 },
@@ -101,4 +102,5 @@ it.each([
102 expect(commit).not.toHaveBeenCalled();
103 expect(draft.text).toBe("Preserved draft.");
104 expect(controller.currentState.error).toMatch(expected);
105 + expect(prepareToRecordAsync).toHaveBeenCalledTimes(phase === "preparation" ? 0 : 1);
106 });
apps/mobile/src/native/voiceTranscription.ios.test.ts
+94 -1
@@ -1,7 +1,8 @@
1 import type { TranscriptionResult } from "@react-native-ai/apple/src/NativeAppleTranscription";
2 import { afterEach, beforeEach, describe, expect, it, vi } from "vite-plus/test";
3
4 -import { VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
4 +import { VoiceInputController, VoiceTranscriptionError } from "@t3tools/client-runtime/voice-input";
5 +import { resetVoiceInputGlobalsForTests } from "../../../../packages/client-runtime/src/voice-input/controller";
6
7 const mocks = vi.hoisted(() => ({
8 isAvailable: vi.fn<(locale: string) => boolean>(),
@@ -65,8 +66,43 @@ async function transcribeOnce(signal = new AbortController().signal) {
66 .catch((error: unknown) => error);
67 }
68
69 +/** Drives one on-device dictation through the controller the composer uses. */
70 +function createDictation() {
71 + const recorder = {
72 + uri: "file:///voice.m4a",
73 + prepareToRecordAsync: async () => {},
74 + record: vi.fn(),
75 + stop: vi.fn(async () => {}),
76 + };
77 + const commits: string[] = [];
78 + const deleted: string[] = [];
79 + const ended = Promise.withResolvers<void>();
80 + const controller = new VoiceInputController({
81 + recorder,
82 + getTranscriber: () => ({
83 + prepare: async (options) => (await getLocalVoiceTranscriber())!.prepare(options),
84 + }),
85 + requestPermission: async () => ({ granted: true, canAskAgain: true }),
86 + configureRecording: async () => {},
87 + releaseRecording: async () => {},
88 + deleteRecording: (uri) => deleted.push(uri),
89 + readDraft: () => ({
90 + ownerKey: "environment:thread",
91 + text: "",
92 + selection: { start: 0, end: 0 },
93 + revision: 1,
94 + }),
95 + commitDraft: (text) => commits.push(text),
96 + onStateChange: (state) => {
97 + if (state.phase === "idle" || state.phase === "error") ended.resolve();
98 + },
99 + });
100 + return { controller, recorder, commits, deleted, ended: ended.promise };
101 +}
102 +
103 beforeEach(() => {
104 vi.resetAllMocks();
105 + resetVoiceInputGlobalsForTests();
106 useLanguages([], "sv-SE");
107 mocks.isAvailable.mockReturnValue(true);
108 mocks.prepare.mockResolvedValue("sv-SE");
@@ -147,6 +183,63 @@ describe("getLocalVoiceTranscriber", () => {
183 },
184 );
185
186 + it("records while Apple prepares its speech model and transcribes once it is ready", async () => {
187 + const nativePrepare = Promise.withResolvers<string>();
188 + mocks.prepare.mockReturnValue(nativePrepare.promise);
189 + const dictation = createDictation();
190 +
191 + await dictation.controller.start();
192 + expect(dictation.recorder.record).toHaveBeenCalledOnce();
193 + expect(dictation.controller.currentState.phase).toBe("recording");
194 +
195 + const stopping = dictation.controller.stop();
196 + nativePrepare.resolve("sv-SE");
197 + await stopping;
198 +
199 + expect(mocks.transcribe).toHaveBeenCalledOnce();
200 + expect(dictation.commits).toEqual(["Hej världen."]);
201 + expect(dictation.deleted).toEqual(["file:///voice.m4a"]);
202 + });
203 +
204 + it.each(["cancel", "cancel after stop", "model failure", "model failure after stop"] as const)(
205 + "discards early audio without a late commit after %s",
206 + async (ending) => {
207 + const nativePrepare = Promise.withResolvers<string>();
208 + mocks.prepare.mockReturnValue(nativePrepare.promise);
209 + const dictation = createDictation();
210 + await dictation.controller.start();
211 +
212 + let finishing: Promise<void> | undefined;
213 + const cancelled = ending === "cancel" || ending === "cancel after stop";
214 + if (ending.endsWith("after stop")) finishing = dictation.controller.stop();
215 + if (cancelled) {
216 + dictation.controller.cancel();
217 + } else {
218 + nativePrepare.reject(Object.assign(new Error("download failed"), { code: "Network" }));
219 + }
220 + await finishing;
221 + await dictation.ended;
222 +
223 + const next = createDictation();
224 + mocks.prepare.mockResolvedValue("sv-SE");
225 + await next.controller.start();
226 + expect(next.controller.currentState.phase).toBe("recording");
227 + expect(dictation.deleted).toEqual(["file:///voice.m4a"]);
228 + expect(mocks.transcribe).not.toHaveBeenCalled();
229 + await next.controller.stop();
230 + expect(next.commits).toEqual(["Hej världen."]);
231 + nativePrepare.resolve("sv-SE");
232 +
233 + expect(dictation.commits).toEqual([]);
234 + expect(dictation.recorder.stop).toHaveBeenCalled();
235 + expect(dictation.controller.currentState).toMatchObject(
236 + cancelled
237 + ? { phase: "idle" }
238 + : { phase: "error", error: "Voice transcription could not prepare this language." },
239 + );
240 + },
241 + );
242 +
243 it("does not start native transcription after cancellation during a file read", async () => {
244 const enteredRead = Promise.withResolvers<void>();
245 const readResult = Promise.withResolvers<ArrayBuffer>();
apps/mobile/src/native/voiceTranscription.ios.ts
+37 -8
@@ -63,6 +63,41 @@ async function prepareVoiceTranscription(
63 });
64 }
65
66 + let locale = locales[0]!;
67 + const model = prepareLocales(locales, signal).then((supported) => {
68 + locale = supported;
69 + });
70 + // Recorder setup can fail before either consumer observes the model.
71 + void model.catch(() => {});
72 +
73 + async function ready(signal: AbortSignal) {
74 + throwIfVoiceTranscriptionAborted(signal);
75 + const cancelled = Promise.withResolvers<void>();
76 + const cancel = () => cancelled.resolve();
77 + signal.addEventListener("abort", cancel, { once: true });
78 + try {
79 + await Promise.race([model, cancelled.promise]);
80 + throwIfVoiceTranscriptionAborted(signal);
81 + return locale;
82 + } finally {
83 + signal.removeEventListener("abort", cancel);
84 + }
85 + }
86 + return {
87 + get locale() {
88 + return locale;
89 + },
90 + start: async (_uri, { signal }) => {
91 + await ready(signal);
92 + return null;
93 + },
94 + transcribe: async (uri, options) =>
95 + transcribeVoiceRecording(uri, await ready(options.signal), options),
96 + };
97 +}
98 +
99 +async function prepareLocales(locales: readonly string[], signal: AbortSignal): Promise<string> {
100 + throwIfVoiceTranscriptionAborted(signal);
101 let unsupported: VoiceTranscriptionError | undefined;
102 for (const locale of locales) {
103 try {
@@ -75,17 +110,11 @@ async function prepareVoiceTranscription(
110 throw unsupported!;
111 }
112
78 -async function prepareLocale(
79 - locale: string,
80 - signal: AbortSignal,
81 -): Promise<PreparedVoiceTranscription> {
113 +async function prepareLocale(locale: string, signal: AbortSignal): Promise<string> {
114 try {
115 const supportedLocale = await AppleTranscription.prepare(locale);
116 throwIfVoiceTranscriptionAborted(signal);
85 - return {
86 - locale: supportedLocale,
87 - transcribe: (uri, options) => transcribeVoiceRecording(uri, supportedLocale, options),
88 - };
117 + return supportedLocale;
118 } catch (error) {
119 throwIfVoiceTranscriptionAborted(signal);
120 if (getNativeErrorCode(error) === "AppleTranscriptionUnsupportedLocale") {
packages/client-runtime/src/voice-input/controller.test.ts
+4
@@ -309,7 +309,9 @@ describe("VoiceInputController", () => {
309 code: "unavailable",
310 message: "This device requires iOS 26 or later.",
311 });
312 + const configureRecording = vi.fn(async () => undefined);
313 const unavailable = createHarness({
314 + configureRecording,
315 getTranscriber: () => ({
316 prepare: async () => {
317 throw reason;
@@ -318,6 +320,8 @@ describe("VoiceInputController", () => {
320 });
321 await unavailable.controller.start();
322 expect(unavailable.controller.currentState.error).toBe(reason.message);
323 + // Environment and web transcribers gate capture on consent checked during preparation.
324 + expect(configureRecording).not.toHaveBeenCalled();
325 expect(unavailable.recorder.record).not.toHaveBeenCalled();
326
327 const denied = createHarness({
packages/client-runtime/src/voice-input/controller.ts
+7 -9
@@ -236,7 +236,7 @@ export class VoiceInputController {
236 private operationToken = 0;
237 private sessionToken: VoiceInputSession | null = null;
238 private transcription: PreparedVoiceTranscription | null = null;
239 - private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null;
239 + private liveTranscription: Promise<{ text: string | null } | { error: unknown }> | null = null;
240 private transcriptionAbortController: AbortController | null = null;
241 private postProcessingAbortController: AbortController | null = null;
242 private skipPostProcessingRequested = false;
@@ -504,17 +504,15 @@ export class VoiceInputController {
504 const capturedDraft = this.capturedDraft;
505 let transcript: string;
506 try {
507 - if (this.liveTranscription) {
508 - const result = await this.liveTranscription;
509 - if ("error" in result) throw result.error;
510 - transcript = result.text;
511 - } else {
512 - transcript = await runTranscriptionOperation(() =>
507 + const live = this.liveTranscription ? await this.liveTranscription : null;
508 + if (live && "error" in live) throw live.error;
509 + transcript =
510 + live?.text ??
511 + (await runTranscriptionOperation(() =>
512 transcription.streaming
513 ? transcription.streaming.finish({ signal })
514 : transcription.transcribe(recordingUri!, { signal }),
516 - );
517 - }
515 + ));
516 } catch (error) {
517 if (this.isCurrent(operationToken)) {
518 if (
packages/client-runtime/src/voice-input/transcription.ts
+6 -1
@@ -11,7 +11,12 @@ export type VoiceStreamingSession = {
11
12 export type PreparedVoiceTranscription = {
13 readonly locale: string;
14 - readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
14 + /**
15 + * Runs alongside capture once recording starts. Resolves with the transcript when it
16 + * transcribed live, or null when `transcribe` still runs after the recording stops.
17 + * A rejection stops the recording.
18 + */
19 + readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string | null>;
20 readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
21 /** Present when audio streams during capture; `finish` then replaces `transcribe`. */
22 readonly streaming?: VoiceStreamingSession;