@devries-cameron20 / t3code-1 / commits / 3de3eee0d1
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

refactor(mobile): voice input session owns recording, metering and keep-awake (#25631)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Oct 26, 2026 at 15:13 UTC 3de3eee0d1b72365bee293e9e5742a0503979417
8 files changed +361 -264
apps/mobile/src/App.tsx
+19 -22
@@ -40,7 +40,6 @@ import {
40 import { useMobileNavigationTheme } from "./lib/useMobileNavigationTheme";
41 import { useUiRuntimeMemoryWarningGc } from "./lib/useUiRuntimeMemoryWarningGc";
42 import { SubscriptionUsageCoordinator } from "./widgets/SubscriptionUsageCoordinator";
43 -import { VoiceInputProvider } from "./features/voice-input/VoiceInputProvider";
43 import { GlobalVoiceInputControl } from "./features/voice-input/GlobalVoiceInputControl";
44 import { restoreSettingsLinkState } from "./features/settings/settings-navigation";
45 import {
@@ -138,27 +137,25 @@ function AppContent() {
137 <GestureHandlerRootView className="flex-1">
138 <KeyboardProvider statusBarTranslucent>
139 <SafeAreaProvider>
141 - <VoiceInputProvider>
142 - <StatusBar barStyle={themeAppearance === "dark" ? "light-content" : "dark-content"} />
143 - {/* native-stack needs the theme's dark flag to select dark native headers. */}
144 - <GlobalVoiceInputControl>
145 - <NativeLayoutMetricsProvider>
146 - <IncomingShareProvider>
147 - <Navigation
148 - ref={navigationRef}
149 - linking={appLinking}
150 - onReady={() => recordWorkspaceSelections(navigationRef.getRootState())}
151 - theme={navigationTheme}
152 - onStateChange={recordWorkspaceSelections}
153 - />
154 - </IncomingShareProvider>
155 - <Dialog.Host />
156 - <ThreadArrangementHost />
157 - <ProjectChipIconRasterizer />
158 - </NativeLayoutMetricsProvider>
159 - </GlobalVoiceInputControl>
160 - <OverlayPortalHost />
161 - </VoiceInputProvider>
140 + <StatusBar barStyle={themeAppearance === "dark" ? "light-content" : "dark-content"} />
141 + {/* native-stack needs the theme's dark flag to select dark native headers. */}
142 + <GlobalVoiceInputControl>
143 + <NativeLayoutMetricsProvider>
144 + <IncomingShareProvider>
145 + <Navigation
146 + ref={navigationRef}
147 + linking={appLinking}
148 + onReady={() => recordWorkspaceSelections(navigationRef.getRootState())}
149 + theme={navigationTheme}
150 + onStateChange={recordWorkspaceSelections}
151 + />
152 + </IncomingShareProvider>
153 + <Dialog.Host />
154 + <ThreadArrangementHost />
155 + <ProjectChipIconRasterizer />
156 + </NativeLayoutMetricsProvider>
157 + </GlobalVoiceInputControl>
158 + <OverlayPortalHost />
159 </SafeAreaProvider>
160 </KeyboardProvider>
161 </GestureHandlerRootView>
apps/mobile/src/features/voice-input/ComposerDictationControl.tsx
+4 -4
@@ -144,7 +144,7 @@ export function ComposerDictationToolbar(props: {
144 }
145
146 const WaveformBar = memo(function WaveformBar(props: {
147 - readonly audioLevels: SharedValue<number[]>;
147 + readonly audioLevels: SharedValue<ReadonlyArray<number>>;
148 readonly sampleIndex: number;
149 readonly height?: number;
150 }) {
@@ -177,7 +177,7 @@ const COMPACT_WAVEFORM_BAR_COUNT = 8;
177 const COMPACT_WAVEFORM_BAR_HEIGHT = 14;
178
179 export const CompactVoiceWaveform = memo(function CompactVoiceWaveform(props: {
180 - readonly audioLevels: SharedValue<number[]>;
180 + readonly audioLevels: SharedValue<ReadonlyArray<number>>;
181 }) {
182 return (
183 <View
@@ -199,7 +199,7 @@ export const CompactVoiceWaveform = memo(function CompactVoiceWaveform(props: {
199 });
200
201 const VoiceWaveform = memo(function VoiceWaveform(props: {
202 - readonly audioLevels: SharedValue<number[]>;
202 + readonly audioLevels: SharedValue<ReadonlyArray<number>>;
203 }) {
204 const [barCount, setBarCount] = useState(0);
205 const handleLayout = useCallback((event: LayoutChangeEvent) => {
@@ -311,7 +311,7 @@ function VoiceActionButton(props: {
311 }
312
313 export function ComposerDictationStatus(props: {
314 - readonly audioLevels: SharedValue<number[]>;
314 + readonly audioLevels: SharedValue<ReadonlyArray<number>>;
315 readonly elapsedSeconds: number;
316 readonly phase: VoiceInputPhase;
317 readonly presentation: VoiceComposerPresentation;
apps/mobile/src/features/voice-input/GlobalVoiceInputControl.tsx
+15 -19
@@ -1,5 +1,5 @@
1 import { AppActivityIndicator } from "../../components/ui/AppActivityIndicator";
2 -import { useEffect, useRef, useState, type ReactNode, type RefObject } from "react";
2 +import { useState, type ReactNode } from "react";
3 import { Linking, Platform, Pressable, useWindowDimensions, View } from "react-native";
4 import Animated, {
5 Easing,
@@ -8,7 +8,7 @@ import Animated, {
8 interpolate,
9 ReduceMotion,
10 useAnimatedStyle,
11 - useSharedValue,
11 + useDerivedValue,
12 withTiming,
13 } from "react-native-reanimated";
14 import { useSafeAreaInsets } from "react-native-safe-area-context";
@@ -35,18 +35,22 @@ const ENTERING = FadeIn.duration(180).reduceMotion(ReduceMotion.System);
35
36 export function GlobalVoiceInputControl(props: { readonly children: ReactNode }) {
37 const voice = useGlobalVoiceInput();
38 - const collapseRef = useRef<(() => void) | null>(null);
38 + const [expanded, setExpanded] = useState(false);
39 const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds);
40 const visible =
41 presentation.statusLabel !== null &&
42 !(voice.ownerKey && voice.focusedOwners.has(voice.ownerKey));
43 - const pill = visible ? <EdgeDictationPill collapseRef={collapseRef} /> : null;
43 + // A pill that reappears starts collapsed.
44 + if (!visible && expanded) setExpanded(false);
45 + const pill = visible ? (
46 + <EdgeDictationPill expanded={expanded} onExpandedChange={setExpanded} />
47 + ) : null;
48 return (
49 <>
50 <View
51 className="flex-1"
52 onStartShouldSetResponderCapture={() => {
49 - collapseRef.current?.();
53 + if (expanded) setExpanded(false);
54 return false;
55 }}
56 >
@@ -57,24 +61,16 @@ export function GlobalVoiceInputControl(props: { readonly children: ReactNode })
61 );
62 }
63
60 -function EdgeDictationPill(props: { readonly collapseRef: RefObject<(() => void) | null> }) {
64 +function EdgeDictationPill(props: {
65 + readonly expanded: boolean;
66 + readonly onExpandedChange: (expanded: boolean) => void;
67 +}) {
68 const voice = useGlobalVoiceInput();
69 const insets = useSafeAreaInsets();
70 const { width } = useWindowDimensions();
71 const { fabSize } = useAndroidControlSizing();
65 - const [expanded, setExpanded] = useState(false);
66 - const progress = useSharedValue(0);
67 - useEffect(() => {
68 - progress.set(withTiming(expanded ? 1 : 0, MORPH_TIMING));
69 - }, [expanded, progress]);
70 - const { collapseRef } = props;
71 - useEffect(() => {
72 - if (!expanded) return;
73 - collapseRef.current = () => setExpanded(false);
74 - return () => {
75 - collapseRef.current = null;
76 - };
77 - }, [collapseRef, expanded]);
72 + const { expanded, onExpandedChange: setExpanded } = props;
73 + const progress = useDerivedValue(() => withTiming(expanded ? 1 : 0, MORPH_TIMING));
74
75 const expandedWidth = Math.min(width - insets.left - insets.right - 32, EXPANDED_MAX_WIDTH);
76 const pillStyle = useAnimatedStyle(() => ({
apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
+98 -199
@@ -1,4 +1,4 @@
1 -import { RegistryContext, useAtomMount } from "@effect/atom-react";
1 +import { useAtomValue } from "@effect/atom-react";
2 import {
3 AudioModule,
4 RecordingPresets,
@@ -7,19 +7,10 @@ import {
7 } from "expo-audio";
8 import { File } from "expo-file-system";
9 import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
10 -import {
11 - createContext,
12 - use,
13 - useCallback,
14 - useEffect,
15 - useId,
16 - useRef,
17 - useState,
18 - type ReactNode,
19 -} from "react";
10 +import { useSyncExternalStore } from "react";
11 import { AppState, Platform } from "react-native";
21 -import { AsyncResult } from "effect/reactivity";
22 -import { useSharedValue } from "react-native-reanimated";
12 +import { Atom, AsyncResult } from "effect/reactivity";
13 +import { makeMutable } from "react-native-reanimated";
14
15 import {
16 microphoneOrder,
@@ -29,13 +20,9 @@ import {
20 import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
21 import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
22 import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene";
32 -import {
33 - VOICE_RECORDING_LIMIT_SECONDS,
34 - voiceInputBlocksSubmission,
35 - type VoiceInputState,
36 -} from "@t3tools/client-runtime/voice-input";
23 +import { voiceInputBlocksSubmission } from "@t3tools/client-runtime/voice-input";
24 import { createLazyVoiceRecorder } from "./lazyVoiceRecorder";
38 -import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
25 +import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
26 import { VoiceInputSession } from "./voiceInputSession";
27 import {
28 acquireAudioSession,
@@ -43,8 +30,6 @@ import {
30 releaseAudioSession,
31 } from "../../lib/audioSession";
32
46 -const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
47 -const VOICE_METERING_INTERVAL_MS = 80;
33 // The native constructor expects platform-flattened options, as built by useAudioRecorder.
34 const { ios: IOS_RECORDING_OPTIONS, android: ANDROID_RECORDING_OPTIONS } =
35 RecordingPresets.HIGH_QUALITY;
@@ -57,194 +42,108 @@ const VOICE_RECORDING_OPTIONS = {
42 ...(Platform.OS === "ios" ? IOS_RECORDING_OPTIONS : ANDROID_RECORDING_OPTIONS),
43 };
44
60 -const VoiceInputContext = createContext<ReturnType<typeof useVoiceInputRuntime> | null>(null);
61 -
62 -export function VoiceInputProvider({ children }: { readonly children: ReactNode }) {
63 - const runtime = useVoiceInputRuntime();
64 - return <VoiceInputContext value={runtime}>{children}</VoiceInputContext>;
45 +function keepScreenAwake(tag: string): () => void {
46 + const activation = activateKeepAwakeAsync(tag);
47 + void activation.catch(() => {});
48 + // Release after activation settles, even if the recording ends immediately.
49 + return () => void activation.then(() => deactivateKeepAwake(tag)).catch(() => {});
50 }
51
67 -export function useGlobalVoiceInput() {
68 - const context = use(VoiceInputContext);
69 - if (!context) throw new Error("Voice input provider is missing.");
70 - return context;
71 -}
72 -
73 -function useVoiceInputRuntime() {
74 - const registry = use(RegistryContext);
75 - useAtomMount(mobilePreferencesAtom);
76 - useAtomMount(updateMobilePreferencesAtom);
77 - const [{ state, ownerKey, label }, setState] = useState({
78 - state: INITIAL_STATE,
79 - ownerKey: null as string | null,
80 - label: null as string | null,
81 - });
82 - const [focusedOwners, setFocusedOwners] = useState<ReadonlySet<string>>(new Set());
83 - const setOwnerFocused = useCallback((key: string, focused: boolean) => {
84 - setFocusedOwners((current) => {
85 - const next = new Set(current);
86 - if (focused) next.add(key);
87 - else next.delete(key);
88 - return next;
89 - });
90 - }, []);
91 - const [elapsedSeconds, setElapsedSeconds] = useState(0);
92 - const keepAwakeId = useId();
93 - const keepAwakeSessionRef = useRef(0);
94 - const elapsedSecondsRef = useRef(0);
95 - const [initialLevels] = useState(() => Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0));
96 - const audioLevelsRef = useRef(initialLevels);
97 - const audioLevels = useSharedValue(initialLevels);
98 - const audioToken = useRef<ReturnType<typeof acquireAudioSession> | null>(null);
99 -
100 - /* oxlint-disable react/refs -- The initializer captures the audio-session ref for later recording callbacks; it does not read it. */
101 - const [{ session, recorder }] = useState(() => {
102 - let session: VoiceInputSession | null = null;
103 - const recorder = createLazyVoiceRecorder({
104 - create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
105 - onStatus: (status: RecordingStatus) => {
106 - session?.controller.handleRecorderStatus({
107 - isFinished: status.isFinished,
108 - hasError: status.hasError || status.mediaServicesDidReset === true,
109 - error: status.error,
110 - url: status.url,
52 +/**
53 + * The app has one microphone and one dictation at a time, so the session
54 + * lives for the registry's lifetime rather than any screen's.
55 + */
56 +const voiceInputAtom = Atom.make((get) => {
57 + const registry = get.registry;
58 + // Mounted so the recorder can read and remember microphones synchronously.
59 + get.mount(mobilePreferencesAtom);
60 + get.mount(updateMobilePreferencesAtom);
61 + const audioLevels = makeMutable<ReadonlyArray<number>>(
62 + Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0),
63 + );
64 + let keepAwakeSequence = 0;
65 + let audioToken: ReturnType<typeof acquireAudioSession> | null = null;
66 + const recorder = createLazyVoiceRecorder({
67 + create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
68 + onStatus: (status: RecordingStatus) => {
69 + session.controller.handleRecorderStatus({
70 + isFinished: status.isFinished,
71 + hasError: status.hasError || status.mediaServicesDidReset === true,
72 + error: status.error,
73 + url: status.url,
74 + });
75 + },
76 + // Read when recording starts so preference changes do not re-render every composer.
77 + selectInput: (inputs) => {
78 + const preferences = registry.get(mobilePreferencesAtom);
79 + if (!AsyncResult.isSuccess(preferences)) return null;
80 + const known = microphoneOrder(preferences.value.microphones);
81 + const order = rememberMicrophones(known, inputs);
82 + if (order !== known) {
83 + registry.set(updateMobilePreferencesAtom, {
84 + transform: (current) => ({
85 + microphones: rememberMicrophones(microphoneOrder(current.microphones), inputs),
86 + }),
87 });
112 - },
113 - // Read when recording starts so preference changes do not re-render every composer.
114 - selectInput: (inputs) => {
115 - const preferences = registry.get(mobilePreferencesAtom);
116 - if (!AsyncResult.isSuccess(preferences)) return null;
117 - const known = microphoneOrder(preferences.value.microphones);
118 - const order = rememberMicrophones(known, inputs);
119 - if (order !== known) {
120 - registry.set(updateMobilePreferencesAtom, {
121 - transform: (current) => ({
122 - microphones: rememberMicrophones(microphoneOrder(current.microphones), inputs),
123 - }),
124 - });
125 - }
126 - return preferredMicrophoneInput(inputs, order);
127 - },
128 - });
129 - session = new VoiceInputSession({
130 - recorder,
131 - getTranscriber: getLocalVoiceTranscriber,
132 - requestPermission: async () => {
133 - const permission = await requestRecordingPermissionsAsync();
134 - return { granted: permission.granted, canAskAgain: permission.canAskAgain };
135 - },
136 - configureRecording: async () => {
137 - const token = acquireAudioSession("recording");
138 - audioToken.current = token;
139 - try {
140 - await activateAudioSession(token);
141 - } catch (error) {
142 - await releaseAudioSession(token).catch(() => {});
143 - throw error;
144 - }
145 - },
146 - releaseRecording: async () => {
147 - if (audioToken.current) await releaseAudioSession(audioToken.current);
148 - audioToken.current = null;
149 - },
150 - deleteRecording: (uri) => new File(uri).delete(),
151 - onStateChange: (nextState) =>
152 - setState({
153 - state: nextState,
154 - ownerKey: session?.ownerKey ?? null,
155 - label: session?.label ?? null,
156 - }),
157 - });
158 - return { session, recorder };
88 + }
89 + return preferredMicrophoneInput(inputs, order);
90 + },
91 });
160 - /* oxlint-enable react/refs */
161 -
162 - const controller = session.controller;
163 -
164 - useEffect(() => {
165 - const subscription = AppState.addEventListener("change", (nextState) => {
166 - // iOS permission dialogs report inactive; cancel preparation only on background. Recorder status handles capture interruptions.
167 - if (nextState === "background") controller.appMovedToBackground();
168 - });
169 - return () => subscription.remove();
170 - }, [controller]);
171 -
172 - useEffect(
173 - () => () => {
174 - controller.dispose();
175 - recorder.release();
92 + const session: VoiceInputSession = new VoiceInputSession({
93 + recorder,
94 + getTranscriber: getLocalVoiceTranscriber,
95 + requestPermission: async () => {
96 + const permission = await requestRecordingPermissionsAsync();
97 + return { granted: permission.granted, canAskAgain: permission.canAskAgain };
98 },
177 - [controller, recorder],
178 - );
179 -
180 - useEffect(() => {
181 - if (state.phase !== "recording") return;
182 -
183 - const tag = `voice-input:${keepAwakeId}:${++keepAwakeSessionRef.current}`;
184 - const activation = activateKeepAwakeAsync(tag);
185 - void activation.catch(() => {});
186 - return () => {
187 - // Release after activation settles, even if the recording ends immediately.
188 - void activation.then(() => deactivateKeepAwake(tag)).catch(() => {});
189 - };
190 - }, [keepAwakeId, state.phase]);
191 -
192 - useEffect(() => {
193 - if (state.phase !== "preparing" && state.phase !== "recording") return;
194 -
195 - if (audioLevelsRef.current.some((level) => level !== 0)) {
196 - audioLevelsRef.current = Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0);
197 - audioLevels.set(audioLevelsRef.current);
198 - }
199 - if (elapsedSecondsRef.current !== 0) {
200 - elapsedSecondsRef.current = 0;
201 - setElapsedSeconds(0);
202 - }
203 - if (state.phase !== "recording") return;
204 -
205 - const sampleRecording = () => {
206 - if (controller.currentState.phase !== "recording") return;
207 - const status = recorder.getStatus();
208 - if (!status?.isRecording) return;
209 -
210 - const level = normalizeVoiceInputDecibels(status.metering);
211 - const history = audioLevelsRef.current;
212 - if (level !== 0 || history.some((sample) => sample !== 0)) {
213 - const nextLevels = [...history.slice(1), level];
214 - audioLevelsRef.current = nextLevels;
215 - audioLevels.set(nextLevels);
99 + configureRecording: async () => {
100 + const token = acquireAudioSession("recording");
101 + audioToken = token;
102 + try {
103 + await activateAudioSession(token);
104 + } catch (error) {
105 + await releaseAudioSession(token).catch(() => {});
106 + throw error;
107 }
108 + },
109 + releaseRecording: async () => {
110 + if (audioToken) await releaseAudioSession(audioToken);
111 + audioToken = null;
112 + },
113 + deleteRecording: (uri) => new File(uri).delete(),
114 + sampleRecording: () => {
115 + const status = recorder.getStatus();
116 + return status?.isRecording
117 + ? { durationMillis: status.durationMillis, metering: status.metering }
118 + : null;
119 + },
120 + publishAudioLevels: (levels) => audioLevels.set(levels),
121 + keepAwake: () => keepScreenAwake(`voice-input:${++keepAwakeSequence}`),
122 + });
123 + // iOS reports `inactive` while its permission dialog is open. Only the real
124 + // background state cancels preparation; recorder status handles calls and
125 + // route interruptions during capture.
126 + const appStateSubscription = AppState.addEventListener("change", (nextState) => {
127 + if (nextState === "background") void session.controller.appMovedToBackground();
128 + });
129 + get.addFinalizer(() => {
130 + appStateSubscription.remove();
131 + session.dispose();
132 + recorder.release();
133 + });
134 + return { session, audioLevels };
135 +}).pipe(Atom.keepAlive, Atom.withLabel("mobile:voice-input"));
136
218 - const nextElapsedSeconds = Math.min(
219 - VOICE_RECORDING_LIMIT_SECONDS,
220 - Math.max(0, Math.floor(status.durationMillis / 1_000)),
221 - );
222 - if (nextElapsedSeconds !== elapsedSecondsRef.current) {
223 - elapsedSecondsRef.current = nextElapsedSeconds;
224 - setElapsedSeconds(nextElapsedSeconds);
225 - }
226 - };
227 -
228 - sampleRecording();
229 - const intervalId = setInterval(sampleRecording, VOICE_METERING_INTERVAL_MS);
230 - return () => clearInterval(intervalId);
231 - }, [audioLevels, controller, recorder, state.phase]);
232 -
233 - const stop = useCallback(() => controller.stop(), [controller]);
234 - const cancel = useCallback(() => controller.cancel(), [controller]);
235 -
137 +export function useGlobalVoiceInput() {
138 + const { session, audioLevels } = useAtomValue(voiceInputAtom);
139 + const snapshot = useSyncExternalStore(session.subscribe, session.getSnapshot);
140 return {
141 + ...snapshot,
142 isAvailable: getLocalVoiceTranscriber() !== null || getNativeShowcaseScene() !== null,
238 - state,
143 audioLevels,
240 - elapsedSeconds,
241 - isBusy: voiceInputBlocksSubmission(state),
242 - ownerKey,
243 - label,
244 - focusedOwners,
245 - setOwnerFocused,
144 + isBusy: voiceInputBlocksSubmission(snapshot.state),
145 session,
247 - stop,
248 - cancel,
146 + stop: () => session.controller.stop(),
147 + cancel: () => session.controller.cancel(),
148 };
149 }
apps/mobile/src/features/voice-input/useVoiceInputController.ts
+4 -4
@@ -22,7 +22,7 @@ export function useVoiceInputController(input: {
22 readonly onChangeSelection: (selection: ComposerEditorSelection) => void;
23 }) {
24 const global = useGlobalVoiceInput();
25 - const { setOwnerFocused, session } = global;
25 + const { session } = global;
26 const latestInput = useRef(input);
27 // oxlint-disable-next-line react/refs -- Transcription finishes asynchronously and must reach the composer's current selection setter, not the one captured at start.
28 latestInput.current = input;
@@ -38,9 +38,9 @@ export function useVoiceInputController(input: {
38 useCallback(() => {
39 const ownerKey = input.ownerKey;
40 if (!ownerKey) return;
41 - setOwnerFocused(ownerKey, true);
42 - return () => setOwnerFocused(ownerKey, false);
43 - }, [input.ownerKey, setOwnerFocused]),
41 + session.setOwnerFocused(ownerKey, true);
42 + return () => session.setOwnerFocused(ownerKey, false);
43 + }, [input.ownerKey, session]),
44 );
45
46 const start = useCallback(() => {
apps/mobile/src/features/voice-input/voiceInputSession.test.ts
+73 -7
@@ -22,7 +22,9 @@ vi.mock("expo-audio", () => ({
22 import {
23 createVoiceInputTarget,
24 VoiceInputSession,
25 + type VoiceInputSessionDependencies,
26 type VoiceInputTarget,
27 + type VoiceRecordingSample,
28 } from "./voiceInputSession";
29
30 function createTarget(
@@ -34,10 +36,17 @@ function createTarget(
36 return createVoiceInputTarget(ownerKey, readText, commit, selection, () => () => {});
37 }
38
37 -function createSession(recordingAudio?: {
38 - configureRecording: () => Promise<void>;
39 - releaseRecording: () => Promise<void>;
40 -}) {
39 +function createSession(
40 + overrides?: Partial<
41 + Pick<
42 + VoiceInputSessionDependencies,
43 + "configureRecording" | "releaseRecording" | "sampleRecording"
44 + >
45 + >,
46 +) {
47 + const releaseKeepAwake = vi.fn();
48 + const keepAwake = vi.fn(() => releaseKeepAwake);
49 + const publishAudioLevels = vi.fn();
50 const recorder = {
51 uri: "file:///voice.m4a",
52 prepareToRecordAsync: vi.fn(async () => {}),
@@ -54,11 +63,13 @@ function createSession(recordingAudio?: {
63 requestPermission: async () => ({ granted: true, canAskAgain: true }),
64 configureRecording: async () => {},
65 releaseRecording: vi.fn(async () => {}),
57 - ...recordingAudio,
66 deleteRecording: vi.fn(),
59 - onStateChange: vi.fn(),
67 + sampleRecording: () => null,
68 + publishAudioLevels,
69 + keepAwake,
70 + ...overrides,
71 });
61 - return { session, recorder, prepare };
72 + return { session, recorder, prepare, keepAwake, releaseKeepAwake, publishAudioLevels };
73 }
74
75 describe("global voice input", () => {
@@ -344,4 +355,59 @@ describe("global voice input", () => {
355 expect(oldCommit).not.toHaveBeenCalled();
356 expect(nextCommit).toHaveBeenCalledWith("second spoken text", { start: 18, end: 18 });
357 });
358 +
359 + it("holds the screen awake and meters only while recording", async () => {
360 + vi.useFakeTimers();
361 + try {
362 + let sample: VoiceRecordingSample = { durationMillis: 0, metering: -100 };
363 + const { session, keepAwake, releaseKeepAwake, publishAudioLevels } = createSession({
364 + sampleRecording: () => sample,
365 + });
366 + const snapshots: number[] = [];
367 + session.subscribe(() => snapshots.push(session.getSnapshot().elapsedSeconds));
368 +
369 + await session.start(createTarget("first", () => "first", vi.fn(), { start: 5, end: 5 }));
370 + expect(session.getSnapshot().state.phase).toBe("recording");
371 + expect(keepAwake).toHaveBeenCalledTimes(1);
372 + // Silence on a silent history publishes nothing.
373 + expect(publishAudioLevels).not.toHaveBeenCalled();
374 +
375 + sample = { durationMillis: 2_400, metering: 0 };
376 + vi.advanceTimersByTime(80);
377 + expect(session.getSnapshot().elapsedSeconds).toBe(2);
378 + expect(publishAudioLevels).toHaveBeenLastCalledWith(expect.arrayContaining([1]));
379 +
380 + await session.controller.stop();
381 + expect(releaseKeepAwake).toHaveBeenCalledTimes(1);
382 + const publishedLevels = publishAudioLevels.mock.calls.length;
383 + const publishedSnapshots = snapshots.length;
384 + vi.advanceTimersByTime(800);
385 + expect(publishAudioLevels).toHaveBeenCalledTimes(publishedLevels);
386 + expect(snapshots).toHaveLength(publishedSnapshots);
387 + // The finished recording's time stays readable until the next one starts.
388 + expect(session.getSnapshot().elapsedSeconds).toBe(2);
389 +
390 + sample = { durationMillis: 0, metering: -100 };
391 + await session.start(createTarget("first", () => "first", vi.fn(), { start: 5, end: 5 }));
392 + expect(session.getSnapshot().elapsedSeconds).toBe(0);
393 + expect(publishAudioLevels).toHaveBeenLastCalledWith(expect.not.arrayContaining([1]));
394 + expect(keepAwake).toHaveBeenCalledTimes(2);
395 + session.dispose();
396 + expect(releaseKeepAwake).toHaveBeenCalledTimes(2);
397 + } finally {
398 + vi.useRealTimers();
399 + }
400 + });
401 +
402 + it("publishes which owners are focused", () => {
403 + const { session } = createSession();
404 + const listener = vi.fn();
405 + session.subscribe(listener);
406 + session.setOwnerFocused("first", true);
407 + session.setOwnerFocused("first", true);
408 + expect(session.getSnapshot().focusedOwners).toEqual(new Set(["first"]));
409 + expect(listener).toHaveBeenCalledTimes(1);
410 + session.setOwnerFocused("first", false);
411 + expect(session.getSnapshot().focusedOwners.size).toBe(0);
412 + });
413 });
apps/mobile/src/features/voice-input/voiceInputSession.ts
+148 -8
@@ -1,10 +1,19 @@
1 import {
2 + VOICE_RECORDING_LIMIT_SECONDS,
3 VoiceInputController,
4 voiceInputBlocksSubmission,
5 type VoiceDraftSnapshot,
6 type VoiceInputControllerDependencies,
7 + type VoiceInputPhase,
8 + type VoiceInputState,
9 } from "@t3tools/client-runtime/voice-input";
10
11 +import { normalizeVoiceInputDecibels, VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
12 +
13 +const VOICE_METERING_INTERVAL_MS = 80;
14 +const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
15 +const SILENT_LEVELS: ReadonlyArray<number> = Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0);
16 +
17 export type VoiceInputTarget = {
18 readonly ownerKey: string;
19 readonly label?: string;
@@ -13,6 +22,31 @@ export type VoiceInputTarget = {
22 readonly subscribe: () => () => void;
23 };
24
25 +export type VoiceInputSnapshot = {
26 + readonly state: VoiceInputState;
27 + readonly ownerKey: string | null;
28 + readonly label: string | null;
29 + readonly elapsedSeconds: number;
30 + readonly focusedOwners: ReadonlySet<string>;
31 +};
32 +
33 +export type VoiceRecordingSample = {
34 + readonly durationMillis: number;
35 + readonly metering: number | undefined;
36 +};
37 +
38 +export type VoiceInputSessionDependencies = Omit<
39 + VoiceInputControllerDependencies,
40 + "readDraft" | "commitDraft" | "onStateChange"
41 +> & {
42 + /** Reads the live recorder, or null when it is not capturing. */
43 + readonly sampleRecording: () => VoiceRecordingSample | null;
44 + /** Levels change every metering tick, so they bypass React state. */
45 + readonly publishAudioLevels: (levels: ReadonlyArray<number>) => void;
46 + /** Holds the screen awake while recording; returns the release. */
47 + readonly keepAwake: () => () => void;
48 +};
49 +
50 export function createVoiceInputTarget(
51 ownerKey: string,
52 readText: () => string | null,
@@ -42,21 +76,32 @@ export function createVoiceInputTarget(
76 };
77 }
78
79 +/**
80 + * The app's single voice input: it owns the controller, the recording-only
81 + * side effects (keep-awake, metering) and publishes what the UI renders.
82 + */
83 export class VoiceInputSession {
84 readonly controller: VoiceInputController;
85 + private readonly dependencies: VoiceInputSessionDependencies;
86 private target: VoiceInputTarget | null = null;
87 private unsubscribe: (() => void) | null = null;
88 + private snapshot: VoiceInputSnapshot = {
89 + state: IDLE_STATE,
90 + ownerKey: null,
91 + label: null,
92 + elapsedSeconds: 0,
93 + focusedOwners: new Set(),
94 + };
95 + private readonly listeners = new Set<() => void>();
96 + private audioLevels = SILENT_LEVELS;
97 + private releaseKeepAwake: (() => void) | null = null;
98 + private meteringInterval: ReturnType<typeof setInterval> | null = null;
99
50 - constructor(dependencies: Omit<VoiceInputControllerDependencies, "readDraft" | "commitDraft">) {
100 + constructor(dependencies: VoiceInputSessionDependencies) {
101 + this.dependencies = dependencies;
102 this.controller = new VoiceInputController({
103 ...dependencies,
53 - onStateChange: (state) => {
54 - if (!voiceInputBlocksSubmission(state)) {
55 - this.unsubscribe?.();
56 - this.unsubscribe = null;
57 - }
58 - dependencies.onStateChange(state);
59 - },
104 + onStateChange: (state) => this.handleStateChange(state),
105 readDraft: () => this.target?.readDraft() ?? null,
106 commitDraft: (text, selection) => this.target?.commitDraft(text, selection),
107 });
@@ -70,6 +115,21 @@ export class VoiceInputSession {
115 return this.target?.label ?? null;
116 }
117
118 + readonly subscribe = (listener: () => void): (() => void) => {
119 + this.listeners.add(listener);
120 + return () => this.listeners.delete(listener);
121 + };
122 +
123 + readonly getSnapshot = (): VoiceInputSnapshot => this.snapshot;
124 +
125 + setOwnerFocused(ownerKey: string, focused: boolean): void {
126 + if (this.snapshot.focusedOwners.has(ownerKey) === focused) return;
127 + const focusedOwners = new Set(this.snapshot.focusedOwners);
128 + if (focused) focusedOwners.add(ownerKey);
129 + else focusedOwners.delete(ownerKey);
130 + this.publish({ ...this.snapshot, focusedOwners });
131 + }
132 +
133 cancel(ownerKey: string | null): void {
134 if (ownerKey !== null && this.ownerKey === ownerKey) this.controller.cancel();
135 }
@@ -84,4 +144,84 @@ export class VoiceInputSession {
144 this.unsubscribe = target.subscribe();
145 return this.controller.start();
146 }
147 +
148 + dispose(): void {
149 + this.controller.dispose();
150 + this.stopRecordingActivity();
151 + this.unsubscribe?.();
152 + this.unsubscribe = null;
153 + this.listeners.clear();
154 + }
155 +
156 + private handleStateChange(state: VoiceInputState): void {
157 + if (!voiceInputBlocksSubmission(state)) {
158 + this.unsubscribe?.();
159 + this.unsubscribe = null;
160 + }
161 + const previousPhase = this.snapshot.state.phase;
162 + let elapsedSeconds = this.snapshot.elapsedSeconds;
163 + if (state.phase !== previousPhase) {
164 + if (previousPhase === "recording") this.stopRecordingActivity();
165 + if (capturesAudio(state.phase)) {
166 + if (this.audioLevels.some((level) => level !== 0)) this.setAudioLevels(SILENT_LEVELS);
167 + elapsedSeconds = 0;
168 + }
169 + }
170 + this.publish({
171 + ...this.snapshot,
172 + state,
173 + ownerKey: this.ownerKey,
174 + label: this.label,
175 + elapsedSeconds,
176 + });
177 + if (state.phase === "recording" && previousPhase !== "recording") {
178 + this.startRecordingActivity();
179 + }
180 + }
181 +
182 + private startRecordingActivity(): void {
183 + this.releaseKeepAwake = this.dependencies.keepAwake();
184 + this.sampleRecording();
185 + this.meteringInterval = setInterval(() => this.sampleRecording(), VOICE_METERING_INTERVAL_MS);
186 + }
187 +
188 + private stopRecordingActivity(): void {
189 + if (this.meteringInterval !== null) clearInterval(this.meteringInterval);
190 + this.meteringInterval = null;
191 + this.releaseKeepAwake?.();
192 + this.releaseKeepAwake = null;
193 + }
194 +
195 + private sampleRecording(): void {
196 + if (this.controller.currentState.phase !== "recording") return;
197 + const sample = this.dependencies.sampleRecording();
198 + if (!sample) return;
199 +
200 + const level = normalizeVoiceInputDecibels(sample.metering);
201 + if (level !== 0 || this.audioLevels.some((sample) => sample !== 0)) {
202 + this.setAudioLevels([...this.audioLevels.slice(1), level]);
203 + }
204 +
205 + const elapsedSeconds = Math.min(
206 + VOICE_RECORDING_LIMIT_SECONDS,
207 + Math.max(0, Math.floor(sample.durationMillis / 1_000)),
208 + );
209 + if (elapsedSeconds !== this.snapshot.elapsedSeconds) {
210 + this.publish({ ...this.snapshot, elapsedSeconds });
211 + }
212 + }
213 +
214 + private setAudioLevels(levels: ReadonlyArray<number>): void {
215 + this.audioLevels = levels;
216 + this.dependencies.publishAudioLevels(levels);
217 + }
218 +
219 + private publish(snapshot: VoiceInputSnapshot): void {
220 + this.snapshot = snapshot;
221 + for (const listener of this.listeners) listener();
222 + }
223 +}
224 +
225 +function capturesAudio(phase: VoiceInputPhase): boolean {
226 + return phase === "preparing" || phase === "recording";
227 }
apps/mobile/src/reactCompilerCoverage.test.ts
-1
@@ -18,7 +18,6 @@ const KNOWN_BAILOUTS: Record<string, string> = {
18 "src/features/diffs/nativeReviewDiffSurface.ts": "ref read during render",
19 "src/features/home/thread-swipe-actions.tsx": "ref read during render",
20 "src/features/threads/ThreadArrangementSheet.tsx": "ref read during render",
21 - "src/features/voice-input/VoiceInputProvider.tsx": "ref read during render",
21 "src/features/voice-input/useVoiceInputController.ts": "ref read during render",
22 "src/native/T3ComposerEditor.ios.tsx": "ref read during render",
23 "src/native/T3ComposerEditor.native.tsx": "ref read during render",