@devries-cameron20 / t3code Mirror · read-only / commits / 79ee39fdcb
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 03:15 UTC

feat(mobile): opt in to dictation on a paired environment (#31059)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Nov 5, 2026 at 15:59 UTC 79ee39fdcbb8638e625a05cd26f4f92e5d6a9319
33 files changed +1308 -96
apps/mobile/.swiftlint.yml
+1
@@ -5,6 +5,7 @@ included:
5 - modules/t3-native-controls/ios
6 - modules/t3-terminal/ios
7 - modules/t3-review-diff/ios
8 + - modules/t3-voice-transcription/ios
9
10 excluded:
11 - ios/Pods
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt
+17
@@ -29,6 +29,8 @@ class T3VoiceTranscriptionModule : Module() {
29 override fun definition() = ModuleDefinition {
30 Name("T3VoiceTranscription")
31
32 + defineDecodeRecording()
33 +
34 AsyncFunction("checkAvailability").Coroutine<Boolean> {
35 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
36 false
@@ -104,6 +106,21 @@ class T3VoiceTranscriptionModule : Module() {
106 }
107 }
108
109 + private fun ModuleDefinitionBuilder.defineDecodeRecording() {
110 + AsyncFunction("decodeRecording").Coroutine { uri: String ->
111 + val context = requireNotNull(appContext.reactContext)
112 + withContext(Dispatchers.IO) {
113 + require(android.net.Uri.parse(uri).scheme == "file") { "Expected a local recording." }
114 + withRecordedPcm(context, uri) { pcm ->
115 + requireCompleteAudio(pcm.length() <= 16000L * 2 * 301) {
116 + "Recording exceeds five minutes."
117 + }
118 + pcm.readBytes()
119 + }
120 + }
121 + }
122 + }
123 +
124 private fun ModuleDefinitionBuilder.defineLiveRecorder() {
125 Class(LiveVoiceRecorder::class) {
126 Constructor {
apps/mobile/modules/t3-voice-transcription/expo-module.config.json
+5 -1
@@ -1,5 +1,9 @@
1 {
2 - "platforms": ["android"],
2 + "platforms": ["apple", "android"],
3 + "apple": {
4 + "modules": ["T3VoiceTranscriptionModule"],
5 + "podspecPath": "ios/T3VoiceTranscription.podspec"
6 + },
7 "android": {
8 "modules": ["expo.modules.t3voicetranscription.T3VoiceTranscriptionModule"]
9 }
apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscription.podspec new
+13
@@ -0,0 +1,13 @@
1 +Pod::Spec.new do |s|
2 + s.name = 'T3VoiceTranscription'
3 + s.version = '1.0.0'
4 + s.summary = 'Local recording conversion for T3 Code dictation.'
5 + s.author = 'T3 Tools'
6 + s.homepage = 'https://t3tools.com'
7 + s.platform = :ios, '18.0'
8 + s.source = { :path => '.' }
9 + s.static_framework = true
10 + s.dependency 'ExpoModulesCore'
11 + s.source_files = '*.swift'
12 + s.frameworks = 'AudioToolbox'
13 +end
apps/mobile/modules/t3-voice-transcription/ios/T3VoiceTranscriptionModule.swift new
+44
@@ -0,0 +1,44 @@
1 +import AudioToolbox
2 +import ExpoModulesCore
3 +import Foundation
4 +
5 +public class T3VoiceTranscriptionModule: Module {
6 + public func definition() -> ModuleDefinition {
7 + Name("T3VoiceTranscription")
8 +
9 + AsyncFunction("decodeRecording") { (uri: URL) throws -> Data in
10 + guard uri.isFileURL else { throw CocoaError(.fileReadUnsupportedScheme) }
11 + var opened: ExtAudioFileRef?
12 + try check(ExtAudioFileOpenURL(uri as CFURL, &opened))
13 + guard let file = opened else { throw CocoaError(.fileReadCorruptFile) }
14 + defer { ExtAudioFileDispose(file) }
15 + var format = AudioStreamBasicDescription(
16 + mSampleRate: 16_000, mFormatID: kAudioFormatLinearPCM,
17 + mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked,
18 + mBytesPerPacket: 2, mFramesPerPacket: 1, mBytesPerFrame: 2,
19 + mChannelsPerFrame: 1, mBitsPerChannel: 16, mReserved: 0)
20 + let formatSize = UInt32(MemoryLayout<AudioStreamBasicDescription>.size)
21 + try check(ExtAudioFileSetProperty(file, kExtAudioFileProperty_ClientDataFormat, formatSize, &format))
22 + var pcm = Data()
23 + var chunk = [Int16](repeating: 0, count: 8192)
24 + try chunk.withUnsafeMutableBytes { bytes in
25 + while true {
26 + var frames = UInt32(bytes.count / 2)
27 + var buffers = AudioBufferList(mNumberBuffers: 1, mBuffers: AudioBuffer(
28 + mNumberChannels: 1, mDataByteSize: UInt32(bytes.count), mData: bytes.baseAddress))
29 + try check(ExtAudioFileRead(file, &frames, &buffers))
30 + if frames == 0 { break }
31 + let count = Int(frames) * 2
32 + guard pcm.count + count <= 16_000 * 2 * 301 else { throw CocoaError(.fileReadTooLarge) }
33 + pcm.append(contentsOf: bytes.prefix(count))
34 + }
35 + }
36 + guard !pcm.isEmpty else { throw CocoaError(.fileReadCorruptFile) }
37 + return pcm
38 + }
39 + }
40 +}
41 +
42 +private func check(_ status: OSStatus) throws {
43 + guard status == noErr else { throw NSError(domain: NSOSStatusErrorDomain, code: Int(status)) }
44 +}
apps/mobile/src/Stack.tsx
+10 -5
@@ -113,7 +113,8 @@ import {
113 } from "./features/settings/ScheduledTaskPickerScreens";
114 import { SettingsKeyboardRouteScreen } from "./features/settings/SettingsKeyboardRouteScreen";
115 import { SettingsLegalRouteScreen } from "./features/settings/SettingsLegalRouteScreen";
116 -import { SettingsMicrophoneRouteScreen } from "./features/settings/SettingsMicrophoneRouteScreen";
116 +import { SettingsVoiceRouteScreen } from "./features/settings/SettingsVoiceRouteScreen";
117 +import { SettingsVoiceModelsRouteScreen } from "./features/settings/SettingsVoiceModelsRouteScreen";
118 import {
119 SettingsOpenSourceLicenseRouteScreen,
120 SettingsOpenSourceLicensesRouteScreen,
@@ -265,10 +266,14 @@ const SETTINGS_SCREENS = {
266 linking: "settings/keyboard",
267 options: settingsPage("Keyboard"),
268 }),
268 - SettingsMicrophone: createNativeStackScreen({
269 - screen: SettingsMicrophoneRouteScreen,
270 - linking: "settings/microphone",
271 - options: settingsPage("Microphone"),
269 + SettingsVoice: createNativeStackScreen({
270 + screen: SettingsVoiceRouteScreen,
271 + linking: { path: "settings/voice", alias: ["settings/microphone"] },
272 + options: settingsPage("Voice"),
273 + }),
274 + SettingsVoiceModels: createNativeStackScreen({
275 + screen: SettingsVoiceModelsRouteScreen,
276 + options: SEARCH_SHEET_ROUTE,
277 }),
278 SettingsFollowUp: createNativeStackScreen({
279 screen: SettingsFollowUpRouteScreen,
apps/mobile/src/components/ui/AppSymbol.tsx
+2
@@ -123,6 +123,7 @@ import IconTextIncrease from "@tabler/icons-react-native/IconTextIncrease";
123 import IconTool from "@tabler/icons-react-native/IconTool";
124 import IconTrash from "@tabler/icons-react-native/IconTrash";
125 import IconTypography from "@tabler/icons-react-native/IconTypography";
126 +import IconDownload from "@tabler/icons-react-native/IconDownload";
127 import IconUpload from "@tabler/icons-react-native/IconUpload";
128 import IconUserCircle from "@tabler/icons-react-native/IconUserCircle";
129 import IconUserCheck from "@tabler/icons-react-native/IconUserCheck";
@@ -277,6 +278,7 @@ const ANDROID_ICON_BY_SF_SYMBOL = {
278 "textformat.size": IconTypography,
279 "textformat.size.larger": IconTextIncrease,
280 "textformat.size.smaller": IconTextDecrease,
281 + "tray.and.arrow.down": IconDownload,
282 "tray.and.arrow.up": IconUpload,
283 trash: IconTrash,
284 wifi: IconWifi,
apps/mobile/src/components/ui/ListChoiceRow.tsx
+5
@@ -1,3 +1,4 @@
1 +import type { ReactNode } from "react";
2 import { Platform, View } from "react-native";
3 import { useResolveClassNames } from "uniwind";
4
@@ -9,7 +10,9 @@ import { RowPressable } from "./RowPressable";
10
11 export function ListChoiceRow(props: {
12 readonly label: string;
13 + readonly leading?: ReactNode;
14 readonly description?: string;
15 + readonly meta?: ReactNode;
16 readonly selected: boolean;
17 readonly separated?: boolean;
18 readonly disabled?: boolean;
@@ -31,7 +34,9 @@ export function ListChoiceRow(props: {
34 >
35 <ListRowLayout
36 label={props.label}
37 + leading={props.leading}
38 description={props.description}
39 + meta={props.meta}
40 disabled={props.disabled}
41 accessory={
42 <View
apps/mobile/src/components/ui/ListRowLayout.tsx
+9
@@ -7,17 +7,23 @@ import { SymbolView, type AppSymbolName } from "./AppSymbol";
7
8 export interface ListRowContentProps {
9 readonly icon?: AppSymbolName;
10 + /** Drawn in the icon column for marks a single symbol can't express. */
11 + readonly leading?: ReactNode;
12 readonly label: string;
13 readonly description?: string;
14 readonly detail?: string;
15 + /** Rich secondary content under the description, such as scores with emphasized values. */
16 + readonly meta?: ReactNode;
17 readonly disabled?: boolean;
18 }
19
20 export function ListRowLayout({
21 icon,
22 + leading,
23 label,
24 description,
25 detail,
26 + meta,
27 disabled,
28 accessory,
29 tone = "default",
@@ -42,6 +48,8 @@ export function ListRowLayout({
48 weight="regular"
49 />
50 </View>
51 + ) : leading ? (
52 + <View className="size-6 items-center justify-center">{leading}</View>
53 ) : null}
54 <View className="min-w-0 flex-1 gap-1">
55 <AppText
@@ -55,6 +63,7 @@ export function ListRowLayout({
63 {description}
64 </AppText>
65 ) : null}
66 + {meta}
67 {detail && stackDetail ? (
68 <AppText textRole="footnote" className="text-foreground-muted">
69 {detail}
apps/mobile/src/features/layout/workspace/workspace-tab-navigation.test.ts
+5
@@ -101,6 +101,11 @@ describe("opening a workspace tab", () => {
101 "Settings",
102 "SettingsEnvironments",
103 ]);
104 + expect(
105 + names(
106 + workspaceTabNavigationState(state, "settings", { settings: "SettingsMicrophone" }, columns),
107 + ),
108 + ).toEqual(["Home", "Settings", "SettingsVoice"]);
109 });
110
111 it("keeps the route that is already open, so re-selecting a tab doesn't rebuild it", () => {
apps/mobile/src/features/layout/workspace/workspace-tab-navigation.ts
+1
@@ -32,6 +32,7 @@ const REMEMBERED_SETTINGS_PAGES: ReadonlySet<string> = new Set(
32 );
33
34 export function rememberedSettingsPage(selections: WorkspaceSelections): SettingsPageTarget {
35 + if (selections.settings === "SettingsMicrophone") return "SettingsVoice";
36 return selections.settings && REMEMBERED_SETTINGS_PAGES.has(selections.settings)
37 ? (selections.settings as SettingsPageTarget)
38 : DEFAULT_SETTINGS_PAGE;
apps/mobile/src/features/pull-requests/PullRequestSearchSheet.tsx
+10 -2
@@ -1,6 +1,7 @@
1 import { useNavigation } from "@react-navigation/native";
2 import type { ReactNode } from "react";
3 -import { View } from "react-native";
3 +import { Platform, View } from "react-native";
4 +import { useSafeAreaInsets } from "react-native-safe-area-context";
5
6 import { AppBar } from "../../components/ui/AppBar";
7 import { useNativeMailSearchToolbar } from "../../native/use-native-mail-search-toolbar";
@@ -20,6 +21,7 @@ export function PullRequestSearchSheet(props: {
21 const navigation = useNavigation();
22 // The compact search toolbar floats over the end of the list.
23 const bottomSearch = useNativeMailSearchToolbar();
24 + const insets = useSafeAreaInsets();
25 return (
26 <View className="flex-1 bg-sheet" collapsable={false}>
27 <AppBar
@@ -37,7 +39,13 @@ export function PullRequestSearchSheet(props: {
39 }}
40 />
41 {props.children({
40 - paddingBottom: bottomSearch ? NATIVE_MAIL_SEARCH_TOOLBAR_CONTENT_INSET + 16 : 16,
42 + // Android's bottom inset also covers the floating workspace tabs while Settings is open.
43 + paddingBottom:
44 + Platform.OS === "android"
45 + ? insets.bottom + 16
46 + : bottomSearch
47 + ? NATIVE_MAIL_SEARCH_TOOLBAR_CONTENT_INSET + 16
48 + : 16,
49 })}
50 </View>
51 );
apps/mobile/src/features/settings/SettingsClientStorageRouteScreen.tsx
+1 -6
@@ -21,6 +21,7 @@ import { useSavedRemoteConnections } from "../../state/use-remote-environment-re
21 import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow";
22 import { SettingsSection } from "./components/SettingsSection";
23 import { SettingsScreen } from "./components/SettingsScreen";
24 +import { formatBytes } from "../../lib/formatBytes";
25
26 export function SettingsClientStorageRouteScreen() {
27 const insets = useSafeAreaInsets();
@@ -179,9 +180,3 @@ function CacheEnvironmentRow(props: {
180 </View>
181 );
182 }
182 -
183 -function formatBytes(bytes: number): string {
184 - if (bytes < 1024) return `${bytes} B`;
185 - if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(bytes < 10 * 1024 ? 1 : 0)} KB`;
186 - return `${(bytes / (1024 * 1024)).toFixed(bytes < 10 * 1024 * 1024 ? 1 : 0)} MB`;
187 -}
apps/mobile/src/features/settings/SettingsRouteScreen.tsx
+1 -1
@@ -247,7 +247,7 @@ function useSettingsIndexSections(): SettingsListSection[] {
247 ? [page("SettingsNavigation", "line.3.horizontal", "Navigation tabs")]
248 : []),
249 ...(Platform.OS === "ios" ? [page("SettingsKeyboard", "keyboard", "Keyboard")] : []),
250 - page("SettingsMicrophone", "mic", "Microphone"),
250 + page("SettingsVoice", "mic", "Voice"),
251 ],
252 },
253 {
apps/mobile/src/features/settings/SettingsVoiceModelsRouteScreen.tsx new
+102
@@ -0,0 +1,102 @@
1 +import { useAtomValue } from "@effect/atom-react";
2 +import type { StaticScreenProps } from "@react-navigation/native";
3 +import type { EnvironmentId } from "@t3tools/contracts";
4 +import { AsyncResult } from "effect/reactivity";
5 +import { useState } from "react";
6 +import { FlatList, Platform, View } from "react-native";
7 +
8 +import { AppActivityIndicator } from "../../components/ui/AppActivityIndicator";
9 +import { AppText as Text } from "../../components/ui/AppText";
10 +import { EmptyState } from "../../components/ui/EmptyState";
11 +import { cn } from "../../lib/cn";
12 +import { useEnvironments } from "../../state/environments";
13 +import { voiceSpeechBenchmarksAtom } from "../voice-input/voiceDestination";
14 +import { PullRequestSearchSheet } from "../pull-requests/PullRequestSearchSheet";
15 +import { useVoiceModels, VoiceModelRow } from "./VoiceModelSection";
16 +
17 +export function SettingsVoiceModelsRouteScreen({
18 + route,
19 +}: StaticScreenProps<{ readonly environmentId: EnvironmentId }>) {
20 + const { environments } = useEnvironments();
21 + const label =
22 + environments.find((environment) => environment.environmentId === route.params.environmentId)
23 + ?.label ?? "This environment";
24 + const voice = useVoiceModels(route.params.environmentId, label);
25 + const benchmarks = new Map(
26 + (
27 + AsyncResult.getOrElse(
28 + useAtomValue(voiceSpeechBenchmarksAtom(route.params.environmentId)),
29 + () => null,
30 + )?.models ?? []
31 + ).map((model) => [model.id, model.benchmark]),
32 + );
33 + const language = new Intl.DateTimeFormat().resolvedOptions().locale.split("-")[0]!.toLowerCase();
34 + const [query, setQuery] = useState("");
35 + const needle = query.trim().toLowerCase();
36 + // Catalog order, not state order: re-sorting when a download finishes would move rows under a tap.
37 + const models = voice.models
38 + .map((model) => {
39 + const benchmark = benchmarks.get(model.id);
40 + return benchmark ? { ...model, benchmark } : model;
41 + })
42 + .filter((model) =>
43 + `${model.name} ${model.description} ${model.languages.join(" ")}`
44 + .toLowerCase()
45 + .includes(needle),
46 + );
47 + return (
48 + <PullRequestSearchSheet
49 + title="Transcription models"
50 + placeholder="Search models or languages"
51 + closeAccessibilityLabel="Close transcription models"
52 + clearAccessibilityLabel="Clear model search"
53 + query={query}
54 + onQueryChange={setQuery}
55 + >
56 + {({ paddingBottom }) => (
57 + <FlatList
58 + data={models}
59 + keyExtractor={(model) => model.id}
60 + className="flex-1"
61 + contentInsetAdjustmentBehavior="automatic"
62 + automaticallyAdjustKeyboardInsets={Platform.OS === "ios"}
63 + keyboardDismissMode={Platform.OS === "ios" ? "interactive" : "on-drag"}
64 + keyboardShouldPersistTaps="handled"
65 + showsVerticalScrollIndicator={false}
66 + contentContainerStyle={{ paddingHorizontal: 16, paddingTop: 16, paddingBottom }}
67 + ListHeaderComponent={
68 + voice.canManage ? undefined : (
69 + <Text className="pb-3 text-xs text-foreground-muted">
70 + This connection can’t change settings on {label}.
71 + </Text>
72 + )
73 + }
74 + renderItem={({ item, index }) => (
75 + <View
76 + className={cn(
77 + "overflow-hidden",
78 + index === 0 && "rounded-t-2xl",
79 + index === models.length - 1 && "rounded-b-2xl",
80 + )}
81 + >
82 + <VoiceModelRow model={item} voice={voice} separated={index > 0} language={language} />
83 + </View>
84 + )}
85 + ListEmptyComponent={
86 + voice.loading ? (
87 + <AppActivityIndicator className="my-8" />
88 + ) : (
89 + <EmptyState
90 + title={needle ? "No models match" : "No transcription models"}
91 + detail={
92 + needle ? "Try another name or language." : `${label} has no models to list.`
93 + }
94 + variant="plain"
95 + />
96 + )
97 + }
98 + />
99 + )}
100 + </PullRequestSearchSheet>
101 + );
102 +}
apps/mobile/src/features/settings/SettingsVoiceRouteScreen.tsx renamed
+115 -34
@@ -19,6 +19,18 @@ import {
19 } from "../../lib/microphonePriority";
20 import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
21 import { ListRowLayout } from "../../components/ui/ListRowLayout";
22 +import { ListChoiceRow } from "../../components/ui/ListChoiceRow";
23 +import * as Dialog from "../../components/ui/Dialog";
24 +import { useEnvironments, type EnvironmentPresentation } from "../../state/environments";
25 +import {
26 + chooseVoiceDestination,
27 + voiceConnectionAtom,
28 + voiceGrantAtom,
29 + voiceSpeechStatusAtom,
30 + hasEnvironmentVoiceCapture,
31 +} from "../voice-input/voiceDestination";
32 +import { VoiceModelSection } from "./VoiceModelSection";
33 +import type { EnvironmentId } from "@t3tools/contracts";
34 import { ThreadSwipeable } from "../home/thread-swipe-actions";
35 import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow";
36 import { SettingsDragHandle } from "./components/SettingsDragHandle";
@@ -69,7 +81,7 @@ const KINDS: Record<
81 wired: {
82 label: "Wired",
83 row: "Other wired",
72 - rowDescription: "Headsets and USB microphones not listed",
84 + rowDescription: "Headsets and USB microphones not listed here",
85 icon: "cable.connector",
86 },
87 bluetooth: {
@@ -77,8 +89,8 @@ const KINDS: Record<
89 row: "Other Bluetooth",
90 rowDescription:
91 Platform.OS === "android"
80 - ? "Earbuds, headsets, and cars, including Android Auto, not listed"
81 - : "AirPods, headsets, and cars not listed",
92 + ? "Earbuds, headsets, and cars, including Android Auto, not listed here"
93 + : "AirPods, headsets, and cars not listed here",
94 icon: "headphones",
95 },
96 builtIn: {
@@ -97,37 +109,71 @@ const KINDS: Record<
109
110 const UNAVAILABLE_EXPLANATION =
111 Platform.OS === "android"
100 - ? "T3 Code's voice input uses Google's on-device speech recognition, which needs Android 12 or later with compatible Google speech services for the device language. Without it, the composer has no microphone button, and allowing microphone access won't add one. Keyboard voice typing is separate and isn't affected."
101 - : "T3 Code's voice input uses Apple's on-device speech transcription, which needs iOS 26 or later on a device that supports it. Without it, the composer has no microphone button, and allowing microphone access won't add one. Apple keyboard dictation is separate and isn't affected.";
112 + ? "Dictation on this device needs Android 12 or later and Google speech services that support your language. Your keyboard’s voice typing is separate."
113 + : "Dictation on this device needs iOS 26 or later, a supported device, and a supported language. Your keyboard’s dictation is separate.";
114
103 -export function SettingsMicrophoneRouteScreen() {
104 - const transcriber = useAtomValue(localVoiceTranscriberAtom);
105 - // Android's probe is asynchronous; an empty sheet avoids flashing "unavailable" first.
106 - if (transcriber === "checking") return <SettingsScreen title="Microphone">{null}</SettingsScreen>;
107 - return transcriber === null ? <VoiceInputUnavailable /> : <MicrophoneSettings />;
115 +function saveDestination(id: EnvironmentId | null) {
116 + try {
117 + chooseVoiceDestination(id);
118 + } catch {
119 + Dialog.alert(
120 + "Could not save transcription choice",
121 + "Dictation on an environment is off for this session. Save again before closing the app, or your previous choice may return when it restarts.",
122 + );
123 + }
124 }
125
110 -function VoiceInputUnavailable() {
111 - const insets = useSafeAreaInsets();
126 +function EnvironmentChoice({
127 + environment,
128 + selected,
129 +}: {
130 + environment: EnvironmentPresentation;
131 + selected: boolean;
132 +}) {
133 + const usable = useAtomValue(voiceConnectionAtom(environment.environmentId)) !== null;
134 + const status = AsyncResult.getOrElse(
135 + useAtomValue(voiceSpeechStatusAtom(environment.environmentId)),
136 + () => null,
137 + );
138 return (
113 - <SettingsScreen title="Microphone">
114 - <ScrollView
115 - contentInsetAdjustmentBehavior="automatic"
116 - showsVerticalScrollIndicator={false}
117 - className="flex-1"
118 - contentContainerClassName="gap-3 px-5 pt-4"
119 - contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }}
120 - >
121 - <SettingsSection title="Voice input">
122 - <ListRowLayout icon="mic" label="Not available on this device" />
123 - </SettingsSection>
124 - <Text className="px-2 text-sm text-foreground-muted">{UNAVAILABLE_EXPLANATION}</Text>
125 - </ScrollView>
126 - </SettingsScreen>
139 + <ListChoiceRow
140 + label={environment.label}
141 + selected={selected}
142 + disabled={!usable}
143 + description={
144 + !usable
145 + ? environment.connection.phase !== "connected"
146 + ? "Not connected"
147 + : "Transcription unavailable"
148 + : status?.supported === false
149 + ? status.reason
150 + : status?.state === "missing-model"
151 + ? "No transcription model installed"
152 + : (environment.displayUrl ?? "Send finished recordings to this environment.")
153 + }
154 + separated
155 + onPress={() => {
156 + if (!selected)
157 + Dialog.confirm({
158 + title: `Send microphone audio to ${environment.label}?`,
159 + message: `Finished recordings will be sent to ${environment.displayUrl ?? environment.label} for transcription. You can switch back to This device at any time.`,
160 + confirmText: "Use environment",
161 + onConfirm: () => saveDestination(environment.environmentId),
162 + });
163 + }}
164 + />
165 );
166 }
167
130 -function MicrophoneSettings() {
168 +export function SettingsVoiceRouteScreen() {
169 + const transcriber = useAtomValue(localVoiceTranscriberAtom);
170 + const grant = useAtomValue(voiceGrantAtom);
171 + const { environments, isReady } = useEnvironments();
172 + const grantEnvironment = grant
173 + ? environments.find(
174 + (environment) => environment.environmentId === grant.destination.environmentId,
175 + )
176 + : undefined;
177 const insets = useSafeAreaInsets();
178 const preferencesResult = useAtomValue(mobilePreferencesAtom);
179 const savePreferences = useAtomSet(updateMobilePreferencesAtom);
@@ -175,7 +221,7 @@ function MicrophoneSettings() {
221 const dragTo = drag === null ? -1 : dragFrom + dropSteps(dragFrom, drag.translation);
222
223 return (
178 - <SettingsScreen title="Microphone">
224 + <SettingsScreen title="Voice">
225 <ScrollView
226 contentInsetAdjustmentBehavior="automatic"
227 showsVerticalScrollIndicator={false}
@@ -183,6 +229,41 @@ function MicrophoneSettings() {
229 contentContainerClassName="gap-3 px-5 pt-4"
230 contentContainerStyle={{ paddingBottom: Math.max(insets.bottom, 18) + 18 }}
231 >
232 + <SettingsSection title="Transcribe on">
233 + <ListChoiceRow
234 + label="This device"
235 + description="Audio stays on this device and is transcribed here."
236 + selected={!grant}
237 + onPress={() => saveDestination(null)}
238 + />
239 + {hasEnvironmentVoiceCapture &&
240 + environments.map((environment) => (
241 + <EnvironmentChoice
242 + key={environment.environmentId}
243 + environment={environment}
244 + selected={grant?.destination.environmentId === environment.environmentId}
245 + />
246 + ))}
247 + {isReady && grant && !grantEnvironment ? (
248 + <ListChoiceRow
249 + label="Previously selected environment"
250 + description="No longer paired"
251 + separated
252 + selected
253 + disabled
254 + onPress={() => {}}
255 + />
256 + ) : null}
257 + </SettingsSection>
258 + {grantEnvironment ? (
259 + <VoiceModelSection
260 + environmentId={grantEnvironment.environmentId}
261 + label={grantEnvironment.label}
262 + />
263 + ) : null}
264 + {!grant && transcriber === null ? (
265 + <Text className="px-2 text-sm text-foreground-muted">{UNAVAILABLE_EXPLANATION}</Text>
266 + ) : null}
267 {permission === "ask" || permission === "denied" ? (
268 <>
269 <SettingsSection title="Microphone access">
@@ -196,8 +277,8 @@ function MicrophoneSettings() {
277 </SettingsSection>
278 <Text className="px-2 text-sm text-foreground-muted">
279 {permission === "ask"
199 - ? "Voice input needs microphone access before it can record or list your microphones."
200 - : "Microphone access is off for T3 Code. Turn it on in Settings to use voice input."}
280 + ? "Allow microphone access to record and to see your microphones."
281 + : "Turn on microphone access for T3 Code in Settings to use voice input."}
282 </Text>
283 </>
284 ) : null}
@@ -236,11 +317,11 @@ function MicrophoneSettings() {
317 })}
318 </SettingsSection>
319 <Text className="px-2 text-sm text-foreground-muted">
239 - Voice input prefers the first connected microphone in this list. Wired and Bluetooth
240 - devices are added by name after you use voice input with them connected; until then, the
241 - Other row ranks them. Swipe a device left to forget it.
320 + Voice input uses the first connected microphone in this list. A wired or Bluetooth
321 + microphone is added by name the first time you dictate with it connected; until then it
322 + falls under Other. Swipe left on a named microphone to forget it.
323 {Platform.OS === "android"
243 - ? " Android may use another input when the preferred microphone cannot be routed."
324 + ? " Android may still use a different microphone if the preferred one can’t be routed."
325 : null}
326 </Text>
327 </ScrollView>
apps/mobile/src/features/settings/VoiceModelSection.tsx new
+319
@@ -0,0 +1,319 @@
1 +import { useAtomRefresh, useAtomValue } from "@effect/atom-react";
2 +import { useNavigation } from "@react-navigation/native";
3 +import {
4 + cancelEnvironmentSpeechModelDownload,
5 + downloadEnvironmentSpeechModel,
6 + selectEnvironmentSpeechModel,
7 +} from "@t3tools/client-runtime/voice-input";
8 +import {
9 + AuthSettingsWriteScope,
10 + type EnvironmentId,
11 + type EnvironmentSpeechModel,
12 +} from "@t3tools/contracts";
13 +import { AsyncResult } from "effect/reactivity";
14 +import { type ReactNode, useEffect, useState } from "react";
15 +import { Platform, Text as NativeText, View } from "react-native";
16 +
17 +import { SymbolView } from "../../components/ui/AppSymbol";
18 +import { AppText as Text } from "../../components/ui/AppText";
19 +import * as Dialog from "../../components/ui/Dialog";
20 +import { ListActionRow } from "../../components/ui/ListActionRow";
21 +import { ListChoiceRow } from "../../components/ui/ListChoiceRow";
22 +import { ListRow } from "../../components/ui/ListRow";
23 +import { Progress } from "../../components/ui/Progress";
24 +import { formatBytes } from "../../lib/formatBytes";
25 +import { useEnvironmentScope } from "../../state/session";
26 +import {
27 + runSpeech,
28 + voiceConnectionAtom,
29 + voiceSpeechModelsAtom,
30 + voiceSpeechStatusAtom,
31 +} from "../voice-input/voiceDestination";
32 +import { SettingsSection } from "./components/SettingsSection";
33 +
34 +const isDownloading = (model: EnvironmentSpeechModel) =>
35 + model.state === "downloading" || model.state === "verifying";
36 +
37 +export function useVoiceModels(environmentId: EnvironmentId, label: string) {
38 + const connection = useAtomValue(voiceConnectionAtom(environmentId));
39 + const statusAtom = voiceSpeechStatusAtom(environmentId);
40 + const modelsAtom = voiceSpeechModelsAtom(environmentId);
41 + const status = AsyncResult.getOrElse(useAtomValue(statusAtom), () => null);
42 + const modelsResult = useAtomValue(modelsAtom);
43 + const refreshStatus = useAtomRefresh(statusAtom);
44 + const refreshModels = useAtomRefresh(modelsAtom);
45 + const canManage = useEnvironmentScope(environmentId, AuthSettingsWriteScope);
46 + const [requested, setRequested] = useState<string | null>(null);
47 + const [busy, setBusy] = useState(false);
48 +
49 + const models = AsyncResult.getOrElse(modelsResult, () => null)?.models ?? [];
50 + const downloading = requested !== null || models.some(isDownloading);
51 +
52 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls the environment's download progress while one runs
53 + useEffect(() => {
54 + if (!downloading) return;
55 + const timer = setInterval(refreshModels, 1_000);
56 + return () => clearInterval(timer);
57 + }, [downloading, refreshModels]);
58 +
59 + const run = async (title: string, work: () => Promise<unknown>, downloadId?: string) => {
60 + if (downloadId) setRequested(downloadId);
61 + else setBusy(true);
62 + try {
63 + await work();
64 + } catch (error) {
65 + Dialog.alert(title, error instanceof Error ? error.message : String(error));
66 + }
67 + if (downloadId) setRequested(null);
68 + else setBusy(false);
69 + refreshModels();
70 + refreshStatus();
71 + };
72 + const ready = connection !== null && status?.supported === true;
73 + return {
74 + connection,
75 + status,
76 + models,
77 + loading: modelsResult._tag === "Initial",
78 + canManage,
79 + requested,
80 + busy,
81 + downloading,
82 + label,
83 + download: (model: EnvironmentSpeechModel) => {
84 + if (!ready) return;
85 + Dialog.confirm({
86 + title: `Download ${model.name}?`,
87 + message: `${label} will download ${formatBytes(model.size)} and use it as its transcription model.`,
88 + confirmText: "Download",
89 + onConfirm: () =>
90 + void run(
91 + "Could not download the model",
92 + async () => {
93 + await runSpeech(downloadEnvironmentSpeechModel(connection, model.id));
94 + if (model.id !== status.modelId)
95 + await runSpeech(selectEnvironmentSpeechModel(connection, model.id));
96 + },
97 + model.id,
98 + ),
99 + });
100 + },
101 + select: (model: EnvironmentSpeechModel) => {
102 + if (!ready || model.active) return;
103 + void run("Could not select the model", () =>
104 + runSpeech(selectEnvironmentSpeechModel(connection, model.id)),
105 + );
106 + },
107 + cancel: (model: EnvironmentSpeechModel) => {
108 + if (!ready) return;
109 + void run("Could not cancel the download", () =>
110 + runSpeech(cancelEnvironmentSpeechModelDownload(connection, model.id)),
111 + );
112 + },
113 + };
114 +}
115 +
116 +export type VoiceModels = ReturnType<typeof useVoiceModels>;
117 +
118 +const languageNames =
119 + typeof Intl.DisplayNames === "function"
120 + ? new Intl.DisplayNames(undefined, { type: "language" })
121 + : null;
122 +const languageLabel = (code: string) => languageNames?.of(code) ?? code.toUpperCase();
123 +
124 +function ModelStats(props: { readonly model: EnvironmentSpeechModel; readonly language: string }) {
125 + const { model, language } = props;
126 + const data = model.benchmark;
127 + const code = !data
128 + ? undefined
129 + : data.accuracy[language]
130 + ? language
131 + : (model.languages.find((candidate) => data.accuracy[candidate]) ?? model.languages[0]);
132 + const score = code ? data?.accuracy[code] : undefined;
133 + const parts: ReadonlyArray<readonly [string, ReactNode]> = [
134 + ...(code && score
135 + ? ([
136 + [
137 + "score",
138 + <>
139 + {languageLabel(code)}{" "}
140 + <Text className="font-t3-medium text-foreground">{score.grade}</Text> (
141 + {score.errorPercent}% error)
142 + </>,
143 + ],
144 + ] as const)
145 + : []),
146 + ...(data && data.secondsPer30s !== null
147 + ? ([
148 + [
149 + "speed",
150 + <>
151 + <Text className="font-t3-medium text-foreground">{data.secondsPer30s} s</Text> / 30 s
152 + </>,
153 + ],
154 + ] as const)
155 + : []),
156 + // A single-language model's score already names its language.
157 + ...(model.languages.length > 1
158 + ? ([
159 + [
160 + "languages",
161 + <>
162 + <SymbolView
163 + name="globe"
164 + size={13}
165 + tintColorClassName="accent-icon-muted"
166 + type="monochrome"
167 + weight="regular"
168 + />{" "}
169 + {model.languages.length}
170 + </>,
171 + ],
172 + ] as const)
173 + : score
174 + ? []
175 + : ([["languages", languageLabel(model.languages[0] ?? "")]] as const)),
176 + ["size", formatBytes(model.size)],
177 + ];
178 + return (
179 + <Text textRole="footnote" className="text-foreground-muted tabular-nums">
180 + {parts.map(([key, part], index) => (
181 + // Plain Text inherits the muted footnote style; AppText would reset it.
182 + <NativeText key={key}>
183 + {index > 0 ? " · " : null}
184 + {part}
185 + </NativeText>
186 + ))}
187 + </Text>
188 + );
189 +}
190 +
191 +function DownloadedGlyph() {
192 + return (
193 + <View className="size-6">
194 + <SymbolView
195 + name="tray.and.arrow.down"
196 + size={Platform.OS === "android" ? 24 : 22}
197 + tintColorClassName="accent-icon"
198 + type="monochrome"
199 + weight="regular"
200 + />
201 + <View className="absolute -right-1.5 -bottom-1 size-3.5 items-center justify-center rounded-full bg-primary ring-2 ring-grouped-card">
202 + <SymbolView
203 + name="checkmark"
204 + size={9}
205 + tintColorClassName="accent-primary-foreground"
206 + type="monochrome"
207 + weight="bold"
208 + />
209 + </View>
210 + </View>
211 + );
212 +}
213 +
214 +export function VoiceModelRow(props: {
215 + readonly model: EnvironmentSpeechModel;
216 + readonly voice: VoiceModels;
217 + readonly separated: boolean;
218 + /** The language whose benchmark score to show, when the model has one. */
219 + readonly language: string;
220 +}) {
221 + const { model, voice, separated } = props;
222 + const meta = <ModelStats model={model} language={props.language} />;
223 + if (model.state === "installed")
224 + return (
225 + <ListChoiceRow
226 + leading={<DownloadedGlyph />}
227 + label={model.name}
228 + description={model.description}
229 + meta={meta}
230 + selected={model.active}
231 + separated={separated}
232 + disabled={!voice.canManage || voice.busy || voice.downloading}
233 + onPress={() => voice.select(model)}
234 + />
235 + );
236 + const progress = model.downloaded === undefined ? undefined : model.downloaded / model.size;
237 + if (isDownloading(model) || voice.requested === model.id)
238 + return (
239 + <View className={separated ? "border-t border-border-subtle" : ""}>
240 + <ListActionRow
241 + icon="xmark"
242 + label={model.name}
243 + description={
244 + model.state === "verifying"
245 + ? "Verifying download…"
246 + : `${Math.round((progress ?? 0) * 100)}% downloaded · Tap to cancel`
247 + }
248 + disabled={!voice.canManage || voice.busy}
249 + onPress={() => voice.cancel(model)}
250 + />
251 + <Progress
252 + className="bg-grouped-card px-4 pb-3"
253 + value={model.state === "verifying" ? undefined : progress}
254 + label={`${model.name} download progress`}
255 + />
256 + </View>
257 + );
258 + return (
259 + <View className={separated ? "border-t border-border-subtle" : ""}>
260 + <ListActionRow
261 + icon="tray.and.arrow.down"
262 + label={model.name}
263 + description={model.description}
264 + meta={meta}
265 + disabled={!voice.canManage || voice.busy || voice.downloading}
266 + onPress={() => voice.download(model)}
267 + />
268 + </View>
269 + );
270 +}
271 +
272 +export function VoiceModelSection(props: {
273 + readonly environmentId: EnvironmentId;
274 + readonly label: string;
275 +}) {
276 + const navigation = useNavigation();
277 + const voice = useVoiceModels(props.environmentId, props.label);
278 + const { status } = voice;
279 + if (!voice.connection || !status) return null;
280 + const title = `Transcription model on ${props.label}`;
281 + if (!status.supported) {
282 + return (
283 + <SettingsSection title={title} footer={status.reason}>
284 + {null}
285 + </SettingsSection>
286 + );
287 + }
288 +
289 + const active = status.state === "missing-model" ? null : status;
290 + const pending = voice.models.find(isDownloading);
291 + const progress =
292 + pending?.downloaded === undefined ? undefined : pending.downloaded / pending.size;
293 + return (
294 + <SettingsSection
295 + title={title}
296 + footer={
297 + voice.canManage
298 + ? undefined
299 + : `This connection can’t change settings on ${props.label}. Add a transcription model using T3 Code on that machine.`
300 + }
301 + >
302 + <ListRow
303 + label={active?.model ?? "No model selected"}
304 + description={
305 + pending
306 + ? pending.state === "verifying"
307 + ? `Verifying ${pending.name}…`
308 + : `Downloading ${pending.name} · ${Math.round((progress ?? 0) * 100)}%`
309 + : active
310 + ? formatBytes(active.size)
311 + : "Choose a transcription model to dictate on this environment."
312 + }
313 + onPress={() =>
314 + navigation.navigate("SettingsVoiceModels", { environmentId: props.environmentId })
315 + }
316 + />
317 + </SettingsSection>
318 + );
319 +}
apps/mobile/src/features/settings/components/settings-page-targets.ts
+1 -1
@@ -15,7 +15,7 @@ export const SETTINGS_PAGE_TARGETS = [
15 "SettingsEnvironmentMaintenance",
16 "SettingsProviderAccounts",
17 "SettingsKeyboard",
18 - "SettingsMicrophone",
18 + "SettingsVoice",
19 "SettingsFollowUp",
20 "SettingsScheduledTasks",
21 "SettingsProjectGrouping",
apps/mobile/src/features/settings/settings-search.test.ts
+2
@@ -16,6 +16,7 @@ const SECTIONS = [
16 rows: [
17 { id: "SettingsAppearance", label: "Appearance" },
18 { id: "SettingsKeyboard", label: "Keyboard" },
19 + { id: "SettingsVoice", label: "Voice" },
20 ],
21 },
22 {
@@ -47,6 +48,7 @@ describe("filterSettingsSections", () => {
48 { query: "appear", expected: [["Interface", ["Appearance"]]] },
49 { query: "KEYBOARD", expected: [["Interface", ["Keyboard"]]] },
50 { query: "dark mode", expected: [["Interface", ["Appearance"]]] },
51 + { query: "microphone", expected: [["Interface", ["Voice"]]] },
52 { query: "agent credits", expected: [["Server settings", ["Source control"]]] },
53 {
54 query: "server settings",
apps/mobile/src/features/settings/settings-search.ts
+14 -1
@@ -29,7 +29,20 @@ const SETTINGS_SEARCH_TERMS = {
29 ],
30 SettingsNavigation: ["tabs", "tab order", "reorder", "hide tabs"],
31 SettingsKeyboard: ["return key", "send message", "new line", "hardware keyboard"],
32 - SettingsMicrophone: ["voice", "dictation", "audio input", "bluetooth"],
32 + SettingsVoice: [
33 + "microphone",
34 + "dictation",
35 + "transcription",
36 + "transcribe on",
37 + "transcription model",
38 + "this device",
39 + "environment",
40 + "on-device",
41 + "audio input",
42 + "preferred microphone",
43 + "microphone order",
44 + "bluetooth",
45 + ],
46 SettingsScheduledTasks: ["automations", "schedule", "cron", "recurring", "webhook"],
47 SettingsProjectOverview: ["project", "overview", "checkouts", "delete project"],
48 SettingsOrganization: [
apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
+14 -6
@@ -1,5 +1,5 @@
1 import { useAtomValue } from "@effect/atom-react";
2 -import { RecordingPresets, type RecordingStatus } from "expo-audio";
2 +import { AudioModule, RecordingPresets, type RecordingStatus } from "expo-audio";
3 import { File } from "expo-file-system";
4 import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
5 import { useSyncExternalStore } from "react";
@@ -25,6 +25,7 @@ import { createLazyVoiceRecorder } from "./lazyVoiceRecorder";
25 import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
26 import { VoiceInputSession } from "./voiceInputSession";
27 import { createVoiceRecordingPermission } from "./voiceRecordingPermission";
28 +import { voiceRouteAtom } from "./voiceDestination";
29 import { makePermissionStatusAtom } from "../../lib/permissionStatusAtom";
30 import {
31 acquireAudioSession,
@@ -63,8 +64,9 @@ export const localVoiceTranscriberAtom = makePermissionStatusAtom<
64 read: () => getLocalVoiceTranscriber().catch(() => null),
65 }).pipe(Atom.keepAlive);
66
66 -// Each foreground re-probe yields a new transcriber; composers re-render only when availability flips.
67 -const localVoiceInputAvailableAtom = Atom.make((get) => {
67 +const voiceInputAvailableAtom = Atom.make((get) => {
68 + const route = get(voiceRouteAtom);
69 + if (route) return route.transcriber !== null;
70 const transcriber = get(localVoiceTranscriberAtom);
71 return transcriber !== null && transcriber !== "checking";
72 });
@@ -78,7 +80,6 @@ const voiceInputAtom = Atom.make((get) => {
80 // Mounted so the recorder can read and remember microphones synchronously.
81 get.mount(mobilePreferencesAtom);
82 get.mount(updateMobilePreferencesAtom);
81 - get.mount(localVoiceTranscriberAtom);
83 const audioLevels = makeMutable<ReadonlyArray<number>>(
84 Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0),
85 );
@@ -88,7 +89,11 @@ const voiceInputAtom = Atom.make((get) => {
89 void session.controller.appMovedToBackground();
90 });
91 const recorder = createLazyVoiceRecorder({
91 - create: () => createVoiceRecorder(VOICE_RECORDING_OPTIONS),
92 + getDestination: () => (registry.get(voiceRouteAtom) === null ? "local" : "environment"),
93 + create: (destination) =>
94 + destination === "local"
95 + ? createVoiceRecorder(VOICE_RECORDING_OPTIONS)
96 + : new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
97 onStatus: (status: RecordingStatus) => {
98 session.controller.handleRecorderStatus({
99 isFinished: status.isFinished,
@@ -116,6 +121,8 @@ const voiceInputAtom = Atom.make((get) => {
121 const session: VoiceInputSession = new VoiceInputSession({
122 recorder,
123 getTranscriber: () => {
124 + const route = registry.get(voiceRouteAtom);
125 + if (route) return route.transcriber;
126 const transcriber = registry.get(localVoiceTranscriberAtom);
127 return transcriber === "checking" ? null : transcriber;
128 },
@@ -144,6 +151,7 @@ const voiceInputAtom = Atom.make((get) => {
151 publishAudioLevels: (levels) => audioLevels.set(levels),
152 keepAwake: () => keepScreenAwake(`voice-input:${++keepAwakeSequence}`),
153 });
154 + get.subscribe(voiceRouteAtom, () => session.controller.cancel());
155 get.addFinalizer(() => {
156 permission.dispose();
157 session.dispose();
@@ -154,7 +162,7 @@ const voiceInputAtom = Atom.make((get) => {
162
163 export function useGlobalVoiceInput() {
164 const { session, audioLevels } = useAtomValue(voiceInputAtom);
157 - const isTranscriberAvailable = useAtomValue(localVoiceInputAvailableAtom);
165 + const isTranscriberAvailable = useAtomValue(voiceInputAvailableAtom);
166 const snapshot = useSyncExternalStore(session.subscribe, session.getSnapshot);
167 return {
168 ...snapshot,
apps/mobile/src/features/voice-input/environmentTranscriber.test.ts new
+69
@@ -0,0 +1,69 @@
1 +import { VOICE_RECORDING_LIMIT_SECONDS } from "@t3tools/client-runtime/voice-input";
2 +import { expect, it, vi } from "vite-plus/test";
3 +import { createEnvironmentTranscriber } from "./environmentTranscriber";
4 +
5 +it("never uploads a recording whose grant changed while the phone decoded it", async () => {
6 + let grant = 1;
7 + const decoded = Promise.withResolvers<Uint8Array>();
8 + const upload = vi.fn(async () => "hello");
9 + const transcriber = createEnvironmentTranscriber({
10 + canSend: () => grant === 1,
11 + prepare: async () => "en",
12 + decode: () => decoded.promise,
13 + upload,
14 + });
15 + const options = { signal: new AbortController().signal };
16 + const recording = await transcriber.prepare(options);
17 + const transcription = recording.transcribe("file:///draft.m4a", options);
18 + grant++;
19 + decoded.resolve(new Uint8Array([0, 128, 0, 64]));
20 + await expect(transcription).rejects.toMatchObject({ code: "cancelled" });
21 + expect(upload).not.toHaveBeenCalled();
22 +});
23 +
24 +it("uploads normalized mono samples only after the recording is finished", async () => {
25 + const upload = vi.fn(async (bytes: Uint8Array) => {
26 + expect(Array.from(new Float32Array(Uint8Array.from(bytes).buffer))).toEqual([-1, 0.5]);
27 + return "hello";
28 + });
29 + const transcriber = createEnvironmentTranscriber({
30 + canSend: () => true,
31 + prepare: async () => "en",
32 + decode: async () => new Uint8Array([255, 0, 128, 0, 64, 255]).subarray(1, 5),
33 + upload,
34 + });
35 + const options = { signal: new AbortController().signal };
36 + const recording = await transcriber.prepare(options);
37 + expect(upload).not.toHaveBeenCalled();
38 + await expect(recording.transcribe("file:///draft.m4a", options)).resolves.toBe("hello");
39 +});
40 +
41 +it("trims at most one second of recorder padding and rejects longer recordings", async () => {
42 + const limit = 16_000 * VOICE_RECORDING_LIMIT_SECONDS;
43 + const padded = new Uint8Array((limit + 16_000 + 1) * 2);
44 + padded[(limit - 1) * 2 + 1] = 64;
45 + let bytes = padded.subarray(0, -2);
46 + const upload = vi.fn(async (body: Uint8Array) => {
47 + expect(body.byteLength).toBe(limit * 4);
48 + expect(
49 + new DataView(body.buffer, body.byteOffset, body.byteLength).getFloat32(
50 + body.byteLength - 4,
51 + true,
52 + ),
53 + ).toBe(0.5);
54 + return "hello";
55 + });
56 + const options = { signal: new AbortController().signal };
57 + const recording = await createEnvironmentTranscriber({
58 + canSend: () => true,
59 + prepare: async () => "en",
60 + decode: async () => bytes,
61 + upload,
62 + }).prepare(options);
63 + await expect(recording.transcribe("file:///padded.m4a", options)).resolves.toBe("hello");
64 + bytes = padded;
65 + await expect(recording.transcribe("file:///too-long.m4a", options)).rejects.toThrow(
66 + "This recording couldn’t be read. Try recording again.",
67 + );
68 + expect(upload).toHaveBeenCalledOnce();
69 +});
apps/mobile/src/features/voice-input/environmentTranscriber.ts new
+62
@@ -0,0 +1,62 @@
1 +import {
2 + VoiceTranscriptionError,
3 + VOICE_RECORDING_LIMIT_SECONDS,
4 + throwIfVoiceTranscriptionAborted,
5 + type VoiceTranscriber,
6 +} from "@t3tools/client-runtime/voice-input";
7 +
8 +const pcmBytesPerSecond = 16_000 * 2;
9 +const maxPcmBytes = pcmBytesPerSecond * VOICE_RECORDING_LIMIT_SECONDS;
10 +
11 +export function createEnvironmentTranscriber(input: {
12 + readonly canSend: () => boolean;
13 + readonly prepare: (signal: AbortSignal) => Promise<string>;
14 + readonly decode: (uri: string) => Promise<Uint8Array>;
15 + readonly upload: (pcm: Uint8Array, signal: AbortSignal) => Promise<string>;
16 +}): VoiceTranscriber {
17 + function check(signal: AbortSignal) {
18 + throwIfVoiceTranscriptionAborted(signal);
19 + if (!input.canSend()) {
20 + throw new VoiceTranscriptionError({
21 + code: "cancelled",
22 + message: "Your transcription choice changed.",
23 + });
24 + }
25 + }
26 + return {
27 + async prepare({ signal }) {
28 + check(signal);
29 + const locale = await input.prepare(signal);
30 + check(signal);
31 + return {
32 + locale,
33 + async transcribe(uri, { signal }) {
34 + check(signal);
35 + const decoded = await input.decode(uri);
36 + check(signal);
37 + if (
38 + !decoded.length ||
39 + decoded.length % 2 ||
40 + decoded.length > maxPcmBytes + pcmBytesPerSecond
41 + ) {
42 + throw new VoiceTranscriptionError({
43 + code: "transcription-failed",
44 + message: "This recording couldn’t be read. Try recording again.",
45 + });
46 + }
47 + // A timed native stop can include a final encoder packet beyond the recording limit.
48 + const bytes = decoded.subarray(0, maxPcmBytes);
49 + const samples =
50 + bytes.byteOffset % 2 === 0
51 + ? new Int16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2)
52 + : new Int16Array(Uint8Array.from(bytes).buffer);
53 + const pcm = new Float32Array(samples.length);
54 + for (let i = 0; i < samples.length; i++) pcm[i] = samples[i]! / 32768;
55 + const text = await input.upload(new Uint8Array(pcm.buffer), signal);
56 + check(signal);
57 + return text;
58 + },
59 + };
60 + },
61 + };
62 +}
apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts
+27
@@ -46,6 +46,7 @@ function createHarness(
46 const created: Array<ReturnType<typeof createNativeRecorder>> = [];
47 const statuses: Status[] = [];
48 const recorder = createLazyVoiceRecorder({
49 + getDestination: () => "local",
50 create: () => {
51 const native = createNativeRecorder(`file:///voice-${created.length}.m4a`);
52 configure?.(native);
@@ -59,6 +60,32 @@ function createHarness(
60 }
61
62 describe("createLazyVoiceRecorder", () => {
63 + it("switches between local live capture and an encoded file only for the next recording", async () => {
64 + let destination: "local" | "environment" = "local";
65 + const local = createNativeRecorder("file:///local.pcm");
66 + const nextLocal = createNativeRecorder("file:///next-local.pcm");
67 + const encoded = createNativeRecorder("file:///environment.m4a");
68 + const localRecorders = [local, nextLocal];
69 + const recorder = createLazyVoiceRecorder({
70 + getDestination: () => destination,
71 + create: (mode) => (mode === "local" ? localRecorders.shift()! : encoded),
72 + onStatus: () => {},
73 + });
74 + await recorder.prepareToRecordAsync();
75 + destination = "environment";
76 + expect(recorder.uri).toBe(local.prepared);
77 + await recorder.stop();
78 + await recorder.prepareToRecordAsync();
79 + expect(recorder.uri).toBe(encoded.prepared);
80 + expect(local.release).toHaveBeenCalledOnce();
81 + destination = "local";
82 + await recorder.stop();
83 + await recorder.prepareToRecordAsync();
84 + expect(recorder.uri).toBe(nextLocal.prepared);
85 + expect(encoded.release).toHaveBeenCalledOnce();
86 + recorder.release();
87 + });
88 +
89 it("keeps the system input when none is preferred or the preferred one disconnects", async () => {
90 const unpreferred = createHarness(() => null);
91 await unpreferred.recorder.prepareToRecordAsync();
apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts
+29 -19
@@ -4,6 +4,7 @@ import {
4 } from "@t3tools/client-runtime/voice-input";
5
6 type Subscription = { remove(): void };
7 +type RecordingDestination = "local" | "environment";
8
9 export interface VoiceRecordingInput {
10 readonly uid: string;
@@ -34,11 +35,15 @@ export interface LazyVoiceRecorder<TRecorderState> extends VoiceRecorder {
35
36 /** Construct lazily: cold startup blocks JS for tens of milliseconds and most composers never record. */
37 export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
37 - readonly create: () => NativeVoiceRecorder<TStatus, TRecorderState>;
38 + readonly getDestination: () => RecordingDestination;
39 + readonly create: (
40 + destination: RecordingDestination,
41 + ) => NativeVoiceRecorder<TStatus, TRecorderState>;
42 readonly onStatus: (status: TStatus) => void;
43 readonly selectInput?: (inputs: ReadonlyArray<VoiceRecordingInput>) => VoiceRecordingInput | null;
44 }): LazyVoiceRecorder<TRecorderState> {
45 let recorder: NativeVoiceRecorder<TStatus, TRecorderState> | null = null;
46 + let destination: RecordingDestination | null = null;
47 let subscription: Subscription | null = null;
48 let retired: NativeVoiceRecorder<TStatus, TRecorderState> | null = null;
49 let retiredUri: string | null = null;
@@ -50,14 +55,34 @@ export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
55 return promise;
56 };
57
58 + const release = () => {
59 + const released = recorder;
60 + if (!released) return;
61 + abandonInputSelection?.();
62 + recorder = null;
63 + subscription?.remove();
64 + subscription = null;
65 + retired = released;
66 + void pending.then(() => {
67 + if (retired === released) {
68 + retiredUri = released.uri;
69 + retired = null;
70 + }
71 + released.release();
72 + });
73 + };
74 +
75 return {
76 get uri() {
77 if (recorder) return recorder.uri;
78 return retired ? retired.uri : retiredUri;
79 },
80 prepareToRecordAsync(signal) {
81 + const next = input.getDestination();
82 + if (next !== destination) release();
83 if (!recorder) {
60 - recorder = input.create();
84 + recorder = input.create(next);
85 + destination = next;
86 subscription = recorder.addListener("recordingStatusUpdate", input.onStatus);
87 }
88 const prepared = recorder;
@@ -95,7 +120,7 @@ export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
120 );
121 },
122 record(options) {
98 - if (!recorder) throw new Error("Voice recorder is not prepared.");
123 + if (!recorder) throw new Error("The voice recorder isn’t ready. Try again.");
124 recorder.record(options);
125 },
126 stop() {
@@ -104,21 +129,6 @@ export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
129 getStatus() {
130 return recorder ? recorder.getStatus() : null;
131 },
107 - release() {
108 - const released = recorder;
109 - if (!released) return;
110 - abandonInputSelection?.();
111 - recorder = null;
112 - subscription?.remove();
113 - subscription = null;
114 - retired = released;
115 - void pending.then(() => {
116 - if (retired === released) {
117 - retiredUri = released.uri;
118 - retired = null;
119 - }
120 - released.release();
121 - });
122 - },
132 + release,
133 };
134 }
apps/mobile/src/features/voice-input/voiceDestination.test.ts new
+174
@@ -0,0 +1,174 @@
1 +import { EnvironmentId } from "@t3tools/contracts";
2 +import { Atom } from "effect/reactivity";
3 +import { expect, it, vi } from "vite-plus/test";
4 +import { appAtomRegistry } from "../../state/atom-registry";
5 +import { chooseVoiceDestination, voiceGrantAtom, voiceRouteAtom } from "./voiceDestination";
6 +import { readVoiceGrant } from "./voiceGrantStorage";
7 +
8 +const mocks = vi.hoisted(() => ({
9 + read: vi.fn<() => Promise<string | null>>(async () => null),
10 + write: vi.fn(),
11 + markerRead: vi.fn<() => string>(),
12 + markerWrite: vi.fn<(id: string) => void>(),
13 + probe: vi.fn(),
14 + fetch: vi.fn<typeof fetch>(),
15 +}));
16 +vi.mock("expo", () => ({
17 + requireOptionalNativeModule: () => ({ decodeRecording: async () => new Uint8Array([0, 0]) }),
18 +}));
19 +vi.mock("expo/fetch", () => ({ fetch: mocks.fetch }));
20 +vi.mock("expo-file-system", () => ({
21 + Paths: { document: "file:///app/" },
22 + File: class {
23 + textSync = mocks.markerRead;
24 + writeSync = mocks.markerWrite;
25 + write(id: string) {
26 + return Promise.resolve().then(() => mocks.markerWrite(id));
27 + }
28 + },
29 +}));
30 +vi.mock("expo-secure-store", () => ({
31 + getItemAsync: mocks.read,
32 + setItem: mocks.write,
33 + WHEN_UNLOCKED_THIS_DEVICE_ONLY: 6,
34 +}));
35 +vi.mock("../../lib/uuid", () => {
36 + let nextGrant = 0;
37 + return { uuidv4: () => `test-grant-${++nextGrant}` };
38 +});
39 +vi.mock("../../lib/protectedData", () => ({ whenProtectedDataAvailable: async () => {} }));
40 +vi.mock("../../connection/runtime", async () => ({
41 + connectionAtomRuntime: Atom.runtime((await import("effect/http")).FetchHttpClient.layer),
42 +}));
43 +vi.mock("../../state/presentation", () => ({
44 + environmentPresentations: {
45 + presentationAtom: Atom.family((id: string) => {
46 + mocks.probe(id);
47 + return Atom.make({
48 + connection: { phase: id === "offline" ? "offline" : "connected" },
49 + serverConfig: { environment: { capabilities: { voiceTranscription: true } } },
50 + });
51 + }),
52 + },
53 +}));
54 +vi.mock("../../state/session", async () => {
55 + const Option = await import("effect/Option");
56 + return {
57 + environmentSession: {
58 + preparedConnectionValueAtom: Atom.family((environmentId: string) =>
59 + Atom.make(
60 + Option.some({
61 + environmentId,
62 + httpBaseUrl: `https://${environmentId}.test`,
63 + httpAuthorization: { _tag: "Bearer", token: "fixture" },
64 + }),
65 + ),
66 + ),
67 + sessionStateAtom: Atom.family((_id: string) => Atom.make(true)),
68 + },
69 + sessionHasScope: () => true,
70 + };
71 +});
72 +
73 +it.each(["secure store", "marker"])(
74 + "revokes across restarts when writing the %s fails",
75 + async (failedStore) => {
76 + let saved: string | null = null;
77 + let marker = "";
78 + mocks.read.mockImplementation(async () => saved);
79 + mocks.write.mockImplementation((_key: string, value: string) => {
80 + saved = value;
81 + });
82 + mocks.markerRead.mockImplementation(() => marker);
83 + mocks.markerWrite.mockImplementation((id) => {
84 + marker = id;
85 + });
86 + const host = EnvironmentId.make("my-server");
87 + try {
88 + chooseVoiceDestination(host);
89 + expect(marker).toBe(appAtomRegistry.get(voiceGrantAtom)?.id);
90 + expect((await readVoiceGrant())?.destination.environmentId).toBe(host);
91 + mocks.markerRead.mockReturnValueOnce("");
92 + expect(await readVoiceGrant()).toBeNull();
93 + const old = { saved, marker };
94 + const failedWrite = failedStore === "secure store" ? mocks.write : mocks.markerWrite;
95 + failedWrite.mockImplementationOnce(() => {
96 + throw new Error("storage unavailable");
97 + });
98 + chooseVoiceDestination(null);
99 + expect(failedStore === "secure store" ? saved : marker).toBe(
100 + failedStore === "secure store" ? old.saved : old.marker,
101 + );
102 + expect(await readVoiceGrant()).toBeNull();
103 + chooseVoiceDestination(host);
104 + failedWrite.mockImplementationOnce(() => {
105 + throw new Error("storage unavailable");
106 + });
107 + expect(() => chooseVoiceDestination(EnvironmentId.make("other-host"))).toThrow();
108 + expect(appAtomRegistry.get(voiceGrantAtom)).toBeNull();
109 + expect(await readVoiceGrant()).toBeNull();
110 + } finally {
111 + mocks.read.mockReset();
112 + mocks.write.mockReset();
113 + mocks.markerRead.mockReset();
114 + mocks.markerWrite.mockReset();
115 + appAtomRegistry.reset();
116 + }
117 + },
118 +);
119 +
120 +it("keeps local as the default and never reuses consent for an offline or reselected host", async () => {
121 + const unmount = appAtomRegistry.mount(voiceRouteAtom);
122 + try {
123 + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
124 + expect(mocks.probe).not.toHaveBeenCalled();
125 + chooseVoiceDestination(EnvironmentId.make("offline"));
126 + expect(appAtomRegistry.get(voiceRouteAtom)).toEqual({ transcriber: null });
127 + const host = EnvironmentId.make("my-server");
128 + chooseVoiceDestination(host);
129 + const first = appAtomRegistry.get(voiceRouteAtom)?.transcriber;
130 + expect(first).toBeTruthy();
131 + chooseVoiceDestination(null);
132 + expect(appAtomRegistry.get(voiceRouteAtom)).toBeNull();
133 + chooseVoiceDestination(host);
134 + const current = appAtomRegistry.get(voiceRouteAtom)?.transcriber;
135 + await expect(first!.prepare({ signal: new AbortController().signal })).rejects.toThrow(
136 + "Your transcription choice changed.",
137 + );
138 + expect(mocks.fetch).not.toHaveBeenCalled();
139 + mocks.fetch
140 + .mockResolvedValueOnce(
141 + Response.json({
142 + supported: true,
143 + state: "ready",
144 + modelId: "fixture",
145 + model: "Fixture",
146 + size: 1,
147 + supportsStreaming: false,
148 + supportsTranslation: false,
149 + acceleration: "cpu",
150 + modelUnloadTimeout: "never",
151 + gpuDevices: [],
152 + }),
153 + )
154 + .mockResolvedValueOnce(Response.json({ text: "hello" }));
155 + const options = { signal: new AbortController().signal };
156 + const recording = await current!.prepare(options);
157 + await expect(recording.transcribe("file:///voice.m4a", options)).resolves.toBe("hello");
158 + expect(mocks.fetch.mock.calls.map(([url, init]) => [String(url), init?.redirect])).toEqual([
159 + ["https://my-server.test/api/voice/status", "error"],
160 + ["https://my-server.test/api/voice/transcribe", "error"],
161 + ]);
162 + const failWrite = () => {
163 + throw new Error("Device storage is unavailable");
164 + };
165 + mocks.markerWrite.mockImplementationOnce(failWrite);
166 + mocks.write.mockImplementationOnce(failWrite);
167 + expect(() => chooseVoiceDestination(null)).toThrow("Device storage is unavailable");
168 + expect(appAtomRegistry.get(voiceGrantAtom)).toBeNull();
169 + await expect(current!.prepare(options)).rejects.toThrow("Your transcription choice changed.");
170 + } finally {
171 + unmount();
172 + appAtomRegistry.reset();
173 + }
174 +});
apps/mobile/src/features/voice-input/voiceDestination.ts new
+181
@@ -0,0 +1,181 @@
1 +import {
2 + AuthOrchestrationOperateScope,
3 + type EnvironmentId,
4 + DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
5 +} from "@t3tools/contracts";
6 +import {
7 + getEnvironmentSpeechModels,
8 + getEnvironmentSpeechStatus,
9 + transcribeEnvironmentPcm,
10 + VoiceTranscriptionError,
11 +} from "@t3tools/client-runtime/voice-input";
12 +import { FetchHttpClient, type HttpClient } from "effect/http";
13 +import type { PreparedConnection } from "@t3tools/client-runtime/connection";
14 +import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
15 +import * as Effect from "effect/Effect";
16 +import * as Option from "effect/Option";
17 +import { Atom, AsyncResult, AtomRegistry } from "effect/reactivity";
18 +import { requireOptionalNativeModule } from "expo";
19 +import { fetch as expoFetch } from "expo/fetch";
20 +
21 +import { appAtomRegistry } from "../../state/atom-registry";
22 +import { environmentPresentations } from "../../state/presentation";
23 +import { environmentSession, sessionHasScope } from "../../state/session";
24 +import { connectionAtomRuntime } from "../../connection/runtime";
25 +import { uuidv4 } from "../../lib/uuid";
26 +import { createEnvironmentTranscriber } from "./environmentTranscriber";
27 +import { readVoiceGrant, writeVoiceGrant, type VoiceGrant } from "./voiceGrantStorage";
28 +
29 +const storedGrant = Atom.make(Effect.promise(readVoiceGrant)).pipe(Atom.keepAlive);
30 +const choice = Atom.make<VoiceGrant | null | undefined>(undefined).pipe(Atom.keepAlive);
31 +export const voiceGrantAtom = Atom.make((get) => {
32 + const chosen = get(choice);
33 + if (chosen !== undefined) return chosen;
34 + return AsyncResult.getOrElse(get(storedGrant), () => null);
35 +});
36 +
37 +export function chooseVoiceDestination(environmentId: EnvironmentId | null) {
38 + // A failed change must not leave the old destination armed in this session.
39 + appAtomRegistry.set(choice, null);
40 + const grant: VoiceGrant | null =
41 + environmentId === null
42 + ? null
43 + : {
44 + id: uuidv4(),
45 + destination: { kind: "environment", environmentId },
46 + };
47 + writeVoiceGrant(grant);
48 + appAtomRegistry.set(choice, grant);
49 +}
50 +
51 +const native = requireOptionalNativeModule<{
52 + decodeRecording?: (uri: string) => Promise<Uint8Array>;
53 +}>("T3VoiceTranscription");
54 +export const hasEnvironmentVoiceCapture = typeof native?.decodeRecording === "function";
55 +
56 +export const voiceConnectionAtom = Atom.family((id: EnvironmentId) =>
57 + Atom.make((get) => {
58 + const environment = get(environmentPresentations.presentationAtom(id));
59 + if (
60 + !hasEnvironmentVoiceCapture ||
61 + environment?.connection.phase !== "connected" ||
62 + !environment.serverConfig?.environment.capabilities.voiceTranscription ||
63 + !sessionHasScope(get(environmentSession.sessionStateAtom(id)), AuthOrchestrationOperateScope)
64 + )
65 + return null;
66 + return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id)));
67 + }),
68 +);
69 +
70 +// React Native's XMLHttpRequest fetch follows redirects. Expo's native transport honors redirect: "error".
71 +export function runSpeech<A, E>(
72 + effect: Effect.Effect<A, E, HttpClient.HttpClient>,
73 + signal?: AbortSignal,
74 + canSend: () => boolean = () => true,
75 +) {
76 + return Effect.runPromise(
77 + AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
78 + Effect.flatMap((context) =>
79 + effect.pipe(
80 + Effect.provideService(FetchHttpClient.Fetch, async (url, options) => {
81 + if (signal?.aborted || !canSend())
82 + throw new VoiceTranscriptionError({
83 + code: "cancelled",
84 + message: "Your transcription choice changed.",
85 + });
86 + return expoFetch(url, { ...options, redirect: "error" });
87 + }),
88 + Effect.provide(context),
89 + ),
90 + ),
91 + ),
92 + { signal },
93 + );
94 +}
95 +function environmentTranscriber(prepared: PreparedConnection, canSend: () => boolean) {
96 + return createEnvironmentTranscriber({
97 + canSend,
98 + prepare: async (signal) => {
99 + const status = await runSpeech(getEnvironmentSpeechStatus(prepared), signal, canSend);
100 + if (!status.supported)
101 + throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
102 + if (status.state === "missing-model")
103 + throw new VoiceTranscriptionError({
104 + code: "unavailable",
105 + message:
106 + "Install a transcription model for this environment in Settings → Voice, then try again.",
107 + });
108 + return status.languages
109 + ? effectiveSpeechLanguage(
110 + {
111 + languages: status.languages,
112 + supportsLanguageDetection: status.supportsLanguageDetection ?? false,
113 + },
114 + "auto",
115 + )
116 + : "auto";
117 + },
118 + decode: (uri) => {
119 + if (!native?.decodeRecording)
120 + throw new VoiceTranscriptionError({
121 + code: "unavailable",
122 + message: "Update T3 Code to transcribe on an environment.",
123 + });
124 + return native.decodeRecording(uri);
125 + },
126 + upload: async (pcm, signal) =>
127 + (
128 + await runSpeech(
129 + transcribeEnvironmentPcm(prepared, pcm, DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, canSend),
130 + signal,
131 + canSend,
132 + )
133 + ).text,
134 + });
135 +}
136 +
137 +const speechAtom = <A, E>(
138 + id: EnvironmentId,
139 + read: (prepared: PreparedConnection) => Effect.Effect<A, E, HttpClient.HttpClient>,
140 +) =>
141 + Atom.make((get) => {
142 + const prepared = get(voiceConnectionAtom(id));
143 + return prepared
144 + ? Effect.tryPromise({
145 + try: (signal) => runSpeech(read(prepared), signal),
146 + catch: (cause) =>
147 + new VoiceTranscriptionError({
148 + code: "unavailable",
149 + message: cause instanceof Error ? cause.message : String(cause),
150 + }),
151 + })
152 + : Effect.succeed(null);
153 + });
154 +
155 +export const voiceSpeechStatusAtom = Atom.family((id: EnvironmentId) =>
156 + speechAtom(id, getEnvironmentSpeechStatus),
157 +);
158 +export const voiceSpeechModelsAtom = Atom.family((id: EnvironmentId) =>
159 + speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared)),
160 +);
161 +// Fetched once for the model sheet; download progress polls the lighter list above.
162 +export const voiceSpeechBenchmarksAtom = Atom.family((id: EnvironmentId) =>
163 + speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared, true)),
164 +);
165 +
166 +// null selects on-device; a grant with no usable transcriber must never fall back.
167 +export const voiceRouteAtom = Atom.make((get) => {
168 + const grant = get(voiceGrantAtom);
169 + if (!grant) return null;
170 + const connectionAtom = voiceConnectionAtom(grant.destination.environmentId);
171 + const connection = get(connectionAtom);
172 + const registry = get.registry;
173 + return {
174 + transcriber:
175 + connection &&
176 + environmentTranscriber(
177 + connection,
178 + () => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection,
179 + ),
180 + };
181 +});
apps/mobile/src/features/voice-input/voiceGrantStorage.ts new
+42
@@ -0,0 +1,42 @@
1 +import { VoiceTranscriptionDestination } from "@t3tools/contracts";
2 +import * as Schema from "effect/Schema";
3 +import { File, Paths } from "expo-file-system";
4 +import * as SecureStore from "expo-secure-store";
5 +import { whenProtectedDataAvailable } from "../../lib/protectedData";
6 +
7 +const key = "t3code.voice.audio-grant.v1";
8 +const deviceOnly = { keychainAccessible: SecureStore.WHEN_UNLOCKED_THIS_DEVICE_ONLY };
9 +// Both stores must name the same grant; clearing either one revokes it on the next launch.
10 +const markerFile = () => new File(Paths.document, "voice-audio-grant-id");
11 +const Grant = Schema.Struct({
12 + id: Schema.NonEmptyString,
13 + destination: VoiceTranscriptionDestination,
14 +});
15 +export type VoiceGrant = typeof Grant.Type;
16 +const decodeGrant = Schema.decodeUnknownSync(Schema.fromJsonString(Schema.NullOr(Grant)));
17 +
18 +export async function readVoiceGrant() {
19 + await whenProtectedDataAvailable();
20 + try {
21 + const raw = await SecureStore.getItemAsync(key, deviceOnly);
22 + const grant = raw ? decodeGrant(raw) : null;
23 + return grant && markerFile().textSync() === grant.id ? grant : null;
24 + } catch {
25 + return null;
26 + }
27 +}
28 +
29 +export function writeVoiceGrant(grant: VoiceGrant | null) {
30 + try {
31 + markerFile().writeSync(grant?.id ?? "");
32 + } catch (cause) {
33 + SecureStore.setItem(key, "null", deviceOnly);
34 + if (grant) throw cause;
35 + return;
36 + }
37 + try {
38 + SecureStore.setItem(key, JSON.stringify(grant), deviceOnly);
39 + } catch (cause) {
40 + if (grant) throw cause;
41 + }
42 +}
apps/mobile/src/lib/formatBytes.ts new
+7
@@ -0,0 +1,7 @@
1 +export function formatBytes(bytes: number): string {
2 + if (bytes < 1024) return `${bytes} B`;
3 + if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(bytes < 10 * 1024 ? 1 : 0)} KB`;
4 + if (bytes < 1024 ** 3)
5 + return `${(bytes / (1024 * 1024)).toFixed(bytes < 10 * 1024 * 1024 ? 1 : 0)} MB`;
6 + return `${(bytes / 1024 ** 3).toFixed(1)} GB`;
7 +}
docs/internals/voice-input.md
+7 -4
@@ -2,8 +2,8 @@
2
3 Transcription edits a composer draft. It does not submit an agent turn. Audio is
4 temporary client input, and only normal message submission sends the resulting
5 -text. Mobile transcribes on the device; web and desktop send audio to one
6 -environment the person chose.
5 +text. Mobile defaults to on-device transcription; any client that sends audio off
6 +the device needs a grant for one environment.
7
8 The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts)
9 owns the operation while the client supplies capture and transcription. Preparation
@@ -15,8 +15,11 @@ capture and `finish` replaces the file-based `transcribe`.
15
16 ## Where audio may go
17
18 -Only an explicit [audio grant](../../apps/web/src/speech/voiceConsent.ts) authorizes
19 -off-device transcription. It lives outside the preferences snapshot so another
18 +Only an explicit audio grant authorizes off-device transcription. On mobile the
19 +grant is stored in device-bound secure storage, outside preference backups. The
20 +recorder is the same for both destinations; only a finished recording can be
21 +uploaded, and its grant is checked again after native decoding and before dispatch.
22 +The [web grant](../../apps/web/src/speech/voiceConsent.ts) lives outside the preferences snapshot so another
23 window saving unrelated settings cannot resurrect a revoked grant. Each new
24 choice gets a distinct grant, even if the destination was previously selected. The
25 [web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds a
docs/user/composer.md
+16 -14
@@ -192,26 +192,28 @@ sent until you choose one. See [Voice input](./voice-input.md).
192
193 ## Voice input on mobile
194
195 -On supported iPhones with iOS 26 or later and Android devices with Android 12 or
196 -later, use the composer's microphone to record, then confirm to transcribe. Text is
195 +Use the composer's microphone to record, then confirm to transcribe. Text is
196 inserted where your selection was when recording started, ready for you to review
197 and edit before sending.
198
200 -Voice input uses on-device speech transcription: Apple's on iOS, and compatible
201 -Google speech services on Android. When it is unavailable for the device or its
202 -language, the composer has no microphone button and **Settings → Microphone** says
203 -voice input isn't available. On Android, keep T3 Code open while recording and
204 -transcribing.
205 -
206 -The first use may download a speech model and needs a network connection.
207 -Later transcription works offline for that language. Android processes audio while
208 -you record, then finishes the transcript after you stop. Recordings can be up to five
199 +By default, audio stays on the phone. Local transcription needs a supported iPhone
200 +with iOS 26 or later, or Android 12 or later with compatible Google speech services.
201 +To use a paired environment instead, open **Settings → Voice → Transcribe on**,
202 +choose the environment, and confirm. Finished recordings go only to that environment,
203 +even when a draft belongs to another one. This also works on phones without local
204 +speech recognition. If the chosen environment disconnects, dictation stops; it never
205 +switches hosts. Choose **This device** to revoke access and discard pending audio.
206 +On Android, keep T3 Code open while recording and transcribing.
207 +
208 +On-device dictation may need a network connection to download a speech model the
209 +first time. After that it works offline for that language. Android transcribes on
210 +the device while you record and finishes after you stop. Recordings can be up to five
211 minutes long. Canceling or an audio interruption discards the recording and
212 preserves your existing draft. Navigating to another screen keeps dictation active;
213 use the floating controls to finish or cancel. While recording, the screen stays
214 awake; it can sleep normally once recording stops.
215
214 -Choose a preferred order in **Settings → Microphone**. Voice input requests the first
216 +Choose a preferred order in **Settings → Voice**. Voice input requests the first
217 connected microphone in the list. The list mixes kinds (built-in, other wired, other
218 Bluetooth, and CarPlay on iOS) with wired and Bluetooth devices you have dictated
219 with, listed by name. Drag to rank them together:
@@ -221,8 +223,8 @@ microphone on iOS. Swipe a device left to forget it. Android may use another inp
223 when the preferred microphone cannot be routed. Devices with the same name may share
224 a saved preference, but simultaneous same-name inputs remain separately selectable.
225
224 -Transcription runs on your device. T3 Code deletes the temporary audio after
225 -transcription or cancellation; only the message text is sent when you submit.
226 +T3 Code deletes the temporary recording from your phone after transcription or
227 +cancellation. The transcript stays in your draft until you send the message to the agent.
228
229 ## Queued messages
230
docs/user/faq.md
+1 -1
@@ -77,7 +77,7 @@ Open **Settings > Storage** and turn on automatic worktree cleanup. You can also
77
78 ## Is there voice input?
79
80 -On iPhone, yes. You need iOS 26 or later. Tap the mic in the composer, talk, check the text, then send. Web, desktop, and Android do not have it yet. [Voice input on iPhone](./composer.md#voice-input-on-iphone)
80 +Yes. On iPhone and Android, dictation uses on-device speech by default; you can opt in to a paired environment instead. Web and desktop require you to choose an environment first. The transcript goes into your draft for review. See [Voice input](./voice-input.md).
81
82 ## Does it work on Windows, Mac, and Linux?
83
docs/user/voice-input.md
+2 -1
@@ -6,7 +6,8 @@ also works in citation comments, diff and file annotations, pull request
6 comments, replies, and review summaries, and in the desktop app's browser
7 annotation editor.
8
9 -On iPhone and Android, voice input transcribes on the device. See
9 +On iPhone and Android, voice input transcribes on the device by default. To opt in
10 +to a named environment on that phone, open **Settings → Voice → Transcribe on**. See
11 [Voice input on mobile](./composer.md#voice-input-on-mobile).
12
13 ## Choose where audio goes