@devries-cameron20 / t3code Mirror · read-only / commits / d1ec71e6ff
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 05:45 UTC

feat(mobile): add on-device Android dictation (#26734)

Co-authored-by: Julius Marminge <julius0216@outlook.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Alex Jankowiak <alexjankowiak@nebius.com>

Alex committed Nov 3, 2026 at 12:48 UTC d1ec71e6ff8b46bacfbe103af72034eba3654e63
38 files changed +2119 -155
apps/mobile/app.config.ts
+9 -2
@@ -367,7 +367,7 @@ const config: ExpoConfig = {
367 "expo-audio",
368 {
369 microphonePermission: "Allow T3 Code to use your microphone for voice input.",
370 - recordAudioAndroid: false,
370 + recordAudioAndroid: true,
371 enableBackgroundPlayback: false,
372 enableBackgroundRecording: false,
373 },
@@ -381,7 +381,14 @@ const config: ExpoConfig = {
381 recordAudioAndroid: false,
382 },
383 ],
384 - ["expo-image-picker", { photosPermission: false, microphonePermission: false }],
384 + [
385 + "expo-image-picker",
386 + {
387 + photosPermission: false,
388 + // `false` makes this plugin add RECORD_AUDIO to blockedPermissions, breaking Android dictation.
389 + microphonePermission: "Allow T3 Code to use your microphone for voice input.",
390 + },
391 + ],
392 [
393 "expo-splash-screen",
394 {
apps/mobile/modules/t3-voice-transcription/android/build.gradle new
+31
@@ -0,0 +1,31 @@
1 +apply plugin: 'com.android.library'
2 +apply plugin: 'org.jetbrains.kotlin.android'
3 +
4 +group = 'com.t3tools.voicetranscription'
5 +version = '0.0.0'
6 +
7 +android {
8 + namespace 'expo.modules.t3voicetranscription'
9 + compileSdk rootProject.ext.compileSdkVersion
10 +
11 + defaultConfig {
12 + minSdkVersion rootProject.ext.minSdkVersion
13 + targetSdkVersion rootProject.ext.targetSdkVersion
14 + }
15 +}
16 +
17 +dependencies {
18 + implementation project(':expo-modules-core')
19 + implementation 'androidx.media3:media3-common:1.9.1'
20 + implementation 'com.google.mlkit:genai-speech-recognition:1.0.0-alpha2'
21 + testImplementation 'junit:junit:4.13.2'
22 + testImplementation 'org.robolectric:robolectric:4.16.1'
23 + testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2'
24 +}
25 +
26 +tasks.withType(Test).configureEach {
27 + systemProperty 'robolectric.dependency.repo.url', 'https://repo.maven.apache.org/maven2'
28 + javaLauncher = javaToolchains.launcherFor {
29 + languageVersion = JavaLanguageVersion.of(21)
30 + }
31 +}
apps/mobile/modules/t3-voice-transcription/android/src/main/AndroidManifest.xml new
+5
@@ -0,0 +1,5 @@
1 +<manifest xmlns:android="http://schemas.android.com/apk/res/android"
2 + xmlns:tools="http://schemas.android.com/tools">
3 + <!-- Recognition is gated to API 31; older devices retain the rest of the app. -->
4 + <uses-sdk tools:overrideLibrary="com.google.mlkit.genai.speechrecognition,com.google.mlkit.genai.common" />
5 +</manifest>
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt new
+162
@@ -0,0 +1,162 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import android.content.Context
4 +import com.google.mlkit.genai.common.DownloadStatus
5 +import com.google.mlkit.genai.common.FeatureStatus
6 +import com.google.mlkit.genai.common.GenAiException
7 +import com.google.mlkit.genai.speechrecognition.SpeechRecognition
8 +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions
9 +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
10 +import com.google.mlkit.genai.speechrecognition.speechRecognizerOptions
11 +import com.google.mlkit.genai.speechrecognition.speechRecognizerRequest
12 +import java.io.Closeable
13 +import java.util.Locale
14 +import kotlinx.coroutines.NonCancellable
15 +import kotlinx.coroutines.flow.Flow
16 +import kotlinx.coroutines.flow.first
17 +import kotlinx.coroutines.withContext
18 +
19 +internal class VoiceTranscriptionException(
20 + val code: String,
21 + message: String,
22 + cause: Throwable? = null
23 +) : Exception(message, cause)
24 +
25 +private val recognitionModes = listOf(
26 + SpeechRecognizerOptions.Mode.MODE_ADVANCED,
27 + SpeechRecognizerOptions.Mode.MODE_BASIC
28 +)
29 +
30 +internal suspend fun isGoogleVoiceTranscriptionSupported(
31 + createClient: (Int) -> GoogleSpeechClient
32 +): Boolean {
33 + for (mode in recognitionModes) {
34 + createClient(mode).use { client ->
35 + when (checkModelSupport(client)) {
36 + FeatureStatus.AVAILABLE,
37 + FeatureStatus.DOWNLOADABLE,
38 + FeatureStatus.DOWNLOADING -> return true
39 + }
40 + }
41 + }
42 + return false
43 +}
44 +
45 +internal suspend fun prepareGoogleVoiceTranscription(
46 + createClient: (Int) -> GoogleSpeechClient
47 +): Int {
48 + for (mode in recognitionModes) {
49 + createClient(mode).use { client ->
50 + if (prepareModel(client)) return mode
51 + }
52 + }
53 + throw VoiceTranscriptionException(
54 + "unavailable",
55 + "On-device voice transcription is not available for this device language."
56 + )
57 +}
58 +
59 +private suspend fun prepareModel(client: GoogleSpeechClient): Boolean = try {
60 + val status = client.checkStatus()
61 + if (status == FeatureStatus.UNAVAILABLE) {
62 + false
63 + } else {
64 + if (status == FeatureStatus.DOWNLOADABLE ||
65 + status == FeatureStatus.DOWNLOADING
66 + ) {
67 + client.download()
68 + }
69 + when (client.checkStatus()) {
70 + FeatureStatus.UNAVAILABLE -> false
71 + FeatureStatus.AVAILABLE -> true
72 + else -> throw VoiceTranscriptionException(
73 + "preparation-failed",
74 + "Voice transcription could not prepare this language."
75 + )
76 + }
77 + }
78 +} catch (error: GenAiException) {
79 + if (!isModelUnavailable(error)) throw error
80 + false
81 +}
82 +
83 +// Any probe failure for one mode means "try the next mode"; prepare surfaces the real error.
84 +private suspend fun checkModelSupport(client: GoogleSpeechClient): Int = try {
85 + client.checkStatus()
86 +} catch (_: GenAiException) {
87 + FeatureStatus.UNAVAILABLE
88 +}
89 +
90 +private fun isModelUnavailable(error: GenAiException): Boolean = when (error.errorCode) {
91 + GenAiException.ErrorCode.NOT_AVAILABLE,
92 + GenAiException.ErrorCode.NOT_SUPPORTED,
93 + GenAiException.ErrorCode.AICORE_INCOMPATIBLE,
94 + GenAiException.ErrorCode.NEEDS_SYSTEM_UPDATE -> true
95 + else -> false
96 +}
97 +
98 +internal suspend fun collectGoogleVoiceTranscript(
99 + responses: Flow<SpeechRecognizerResponse>
100 +): String {
101 + val text = StringBuilder()
102 + var partial = ""
103 + var completed = false
104 + responses.collect { response ->
105 + when (response) {
106 + // Separators between final utterances are supplied by the SDK.
107 + is SpeechRecognizerResponse.FinalTextResponse -> {
108 + text.append(response.text)
109 + partial = ""
110 + }
111 + is SpeechRecognizerResponse.PartialTextResponse -> partial = response.text
112 + is SpeechRecognizerResponse.ErrorResponse -> throw response.e
113 + is SpeechRecognizerResponse.CompletedResponse -> completed = true
114 + else -> Unit
115 + }
116 + }
117 + requireCompleteAudio(completed) { "Speech recognition did not complete the recording." }
118 + // Some sessions complete with a trailing partial instead of another final event.
119 + return text.append(partial).toString().trim()
120 +}
121 +
122 +internal interface GoogleSpeechClient : Closeable {
123 + suspend fun checkStatus(): Int
124 + suspend fun download()
125 +}
126 +
127 +internal class MlKitSpeechClient(context: Context, locale: Locale, mode: Int) : GoogleSpeechClient {
128 + private val context = context.applicationContext
129 + private val recognizer = SpeechRecognition.getClient(
130 + speechRecognizerOptions {
131 + this.locale = locale
132 + preferredMode = mode
133 + }
134 + )
135 +
136 + override suspend fun checkStatus() = recognizer.checkStatus()
137 +
138 + override suspend fun download() {
139 + val result = recognizer.download().first {
140 + it is DownloadStatus.DownloadCompleted || it is DownloadStatus.DownloadFailed
141 + }
142 + if (result is DownloadStatus.DownloadFailed) throw result.e
143 + }
144 +
145 + suspend fun transcribe(uri: String): String = withRecordedPcm(context, uri) { pcm ->
146 + PcmPipe.open().use { pipe ->
147 + val request = speechRecognizerRequest { audioSource = pipe.source }
148 + try {
149 + collectGoogleVoiceTranscript(
150 + pacedRecordedResponses(pcm, pipe) {
151 + recognizer.startRecognition(request)
152 + }
153 + )
154 + } finally {
155 + // Stop inference before closing the SDK-owned descriptor, including cancellation.
156 + withContext(NonCancellable) { runCatching { recognizer.stopRecognition() } }
157 + }
158 + }
159 + }
160 +
161 + override fun close() = recognizer.close()
162 +}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt new
+85
@@ -0,0 +1,85 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
4 +import kotlinx.coroutines.NonCancellable
5 +import kotlinx.coroutines.TimeoutCancellationException
6 +import kotlinx.coroutines.async
7 +import kotlinx.coroutines.coroutineScope
8 +import kotlinx.coroutines.delay
9 +import kotlinx.coroutines.flow.Flow
10 +import kotlinx.coroutines.flow.flow
11 +import kotlinx.coroutines.withContext
12 +import kotlinx.coroutines.withTimeout
13 +import java.io.File
14 +import java.util.concurrent.atomic.AtomicBoolean
15 +
16 +internal interface PcmOutput : AutoCloseable {
17 + suspend fun write(buffer: ByteArray, count: Int)
18 + fun finish()
19 + fun verifyConsumed()
20 +}
21 +
22 +private suspend fun replayPcm(file: File, output: PcmOutput) {
23 + val expected = file.length()
24 + var sent = 0L
25 + file.inputStream().use { input ->
26 + val buffer = ByteArray(3200)
27 + while (true) {
28 + val count = input.read(buffer)
29 + if (count == -1) break
30 + requireCompleteAudio(count > 0 && count % 2 == 0) {
31 + "Recorded PCM contains an incomplete audio frame."
32 + }
33 + // STREAMING requires real-time delivery, including the final short buffer.
34 + delay((count * 1000L + 31999) / 32000)
35 + output.write(buffer, count)
36 + sent += count
37 + }
38 + }
39 + requireCompleteAudio(sent == expected && file.length() == expected) {
40 + "The recording changed while speech recognition was reading it. No transcript was added."
41 + }
42 +}
43 +
44 +internal fun pacedRecordedResponses(
45 + file: File,
46 + output: PcmOutput,
47 + responses: () -> Flow<SpeechRecognizerResponse>
48 +): Flow<SpeechRecognizerResponse> = flow {
49 + coroutineScope {
50 + val exhausted = AtomicBoolean()
51 + val sender = async {
52 + replayPcm(file, output)
53 + exhausted.set(true)
54 + output.finish()
55 + }
56 + try {
57 + withTimeout(file.length() / 32 + 30000) {
58 + responses().collect { emit(it) }
59 + requireCompleteAudio(exhausted.get()) {
60 + "Speech recognition ended before receiving the complete recording. " +
61 + "No transcript was added. Retry dictation."
62 + }
63 + sender.await()
64 + output.verifyConsumed()
65 + }
66 + } catch (e: TimeoutCancellationException) {
67 + throw VoiceTranscriptionException(
68 + "incomplete-audio",
69 + "Speech recognition did not finish the complete recording. No transcript was added. Retry dictation.",
70 + e
71 + )
72 + } finally {
73 + sender.cancel()
74 + try {
75 + output.finish()
76 + } finally {
77 + withContext(NonCancellable) { sender.join() }
78 + }
79 + }
80 + }
81 +}
82 +
83 +internal inline fun requireCompleteAudio(condition: Boolean, message: () -> String) {
84 + if (!condition) throw VoiceTranscriptionException("incomplete-audio", message())
85 +}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmPipe.kt new
+105
@@ -0,0 +1,105 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import android.os.ParcelFileDescriptor
4 +import android.system.ErrnoException
5 +import android.system.Os
6 +import android.system.OsConstants
7 +import com.google.mlkit.genai.common.audio.AudioSource
8 +import kotlinx.coroutines.currentCoroutineContext
9 +import kotlinx.coroutines.delay
10 +import kotlinx.coroutines.ensureActive
11 +import kotlinx.coroutines.withTimeout
12 +import kotlinx.coroutines.TimeoutCancellationException
13 +import java.util.concurrent.atomic.AtomicBoolean
14 +
15 +internal class PcmPipe private constructor(
16 + val source: AudioSource,
17 + private val writer: ParcelFileDescriptor,
18 + verificationReader: ParcelFileDescriptor
19 +) : PcmOutput {
20 + private val closed = AtomicBoolean()
21 + private val reader = ParcelFileDescriptor.AutoCloseInputStream(verificationReader)
22 + override suspend fun write(buffer: ByteArray, count: Int) {
23 + try {
24 + withTimeout(5000) {
25 + var offset = 0
26 + while (offset < count) {
27 + currentCoroutineContext().ensureActive()
28 + try {
29 + val written = Os.write(writer.fileDescriptor, buffer, offset, count - offset)
30 + requireCompleteAudio(written > 0) { "On-device speech stopped reading the recording." }
31 + offset += written
32 + } catch (e: ErrnoException) {
33 + if (e.errno != OsConstants.EAGAIN) {
34 + throw VoiceTranscriptionException(
35 + "incomplete-audio",
36 + "On-device speech stopped reading the complete recording. No transcript was added. Retry dictation.",
37 + e
38 + )
39 + }
40 + delay(10)
41 + }
42 + }
43 + }
44 + } catch (e: TimeoutCancellationException) {
45 + throw VoiceTranscriptionException(
46 + "incomplete-audio",
47 + "On-device speech could not read the complete recording. No transcript was added. Retry dictation.",
48 + e
49 + )
50 + }
51 + }
52 + override fun finish() = writer.close()
53 + override fun verifyConsumed() {
54 + requireCompleteAudio(reader.available() == 0) {
55 + "Speech recognition ended with unread recorded audio. No transcript was added. Retry dictation."
56 + }
57 + }
58 + override fun close() {
59 + if (!closed.compareAndSet(false, true)) return
60 + try {
61 + writer.close()
62 + } finally {
63 + try {
64 + reader.close()
65 + } finally {
66 + source.close()
67 + }
68 + }
69 + }
70 + companion object {
71 + fun open(): PcmPipe {
72 + val ends = ParcelFileDescriptor.createPipe()
73 + var sdkReader: ParcelFileDescriptor? = null
74 + var retained = false
75 + try {
76 + Os.fcntlInt(ends[1].fileDescriptor, OsConstants.F_SETFL, OsConstants.O_NONBLOCK)
77 + // The SDK closes its descriptor; verification must retain a separate read end.
78 + sdkReader = ends[0].dup()
79 + val pipe = PcmPipe(
80 + AudioSource.fromPfd(sdkReader, AudioSource.Mode.STREAMING),
81 + ends[1],
82 + ends[0]
83 + )
84 + retained = true
85 + return pipe
86 + } finally {
87 + if (!retained) closeFailedPipe(sdkReader, ends)
88 + }
89 + }
90 + private fun closeFailedPipe(
91 + sdkReader: ParcelFileDescriptor?,
92 + ends: Array<ParcelFileDescriptor>
93 + ) {
94 + try {
95 + sdkReader?.close()
96 + } finally {
97 + try {
98 + ends[0].close()
99 + } finally {
100 + ends[1].close()
101 + }
102 + }
103 + }
104 + }
105 +}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/RecordedPcm.kt new
+172
@@ -0,0 +1,172 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import android.content.Context
4 +import android.media.AudioFormat
5 +import android.media.MediaCodec
6 +import android.media.MediaExtractor
7 +import android.media.MediaFormat
8 +import android.net.Uri
9 +import java.io.Closeable
10 +import java.io.File
11 +import java.io.OutputStream
12 +import java.nio.ByteOrder
13 +import kotlinx.coroutines.currentCoroutineContext
14 +import kotlinx.coroutines.ensureActive
15 +import kotlinx.coroutines.TimeoutCancellationException
16 +import kotlinx.coroutines.withTimeout
17 +
18 +internal suspend fun <T> withRecordedPcm(
19 + context: Context,
20 + uri: String,
21 + transcribe: suspend (File) -> T
22 +): T {
23 + val pcm = File.createTempFile("dictation-", ".pcm", context.cacheDir)
24 + try {
25 + try {
26 + withTimeout(30000) { decodeRecording(context, Uri.parse(uri), pcm) }
27 + } catch (_: TimeoutCancellationException) {
28 + throw VoiceTranscriptionException(
29 + "incomplete-audio",
30 + "Audio decoding did not finish the recording. Retry dictation."
31 + )
32 + }
33 + requireCompleteAudio(pcm.length() > 0 && pcm.length() % 2 == 0L) {
34 + "The recording did not contain complete audio samples."
35 + }
36 + return transcribe(pcm)
37 + } finally {
38 + pcm.delete()
39 + }
40 +}
41 +
42 +private suspend fun decodeRecording(context: Context, uri: Uri, output: File) {
43 + val extractor = MediaExtractor()
44 + Closeable { extractor.release() }.use {
45 + extractor.setDataSource(context, uri, null)
46 + val track = (0 until extractor.trackCount).firstOrNull {
47 + extractor.getTrackFormat(it).getString(MediaFormat.KEY_MIME)?.startsWith("audio/") == true
48 + } ?: throw VoiceTranscriptionException("incomplete-audio", "The recording has no audio track.")
49 + extractor.selectTrack(track)
50 + decodeTrack(extractor, extractor.getTrackFormat(track), output)
51 + }
52 +}
53 +
54 +private suspend fun decodeTrack(extractor: MediaExtractor, format: MediaFormat, output: File) {
55 + format.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT)
56 + val decoder = MediaCodec.createDecoderByType(
57 + requireNotNull(format.getString(MediaFormat.KEY_MIME))
58 + )
59 + Closeable { decoder.release() }.use {
60 + decoder.configure(format, null, null, 0)
61 + decoder.start()
62 + Closeable { decoder.stop() }.use {
63 + output.outputStream().buffered().use { stream -> decodeBuffers(decoder, extractor, stream) }
64 + }
65 + }
66 +}
67 +
68 +private suspend fun decodeBuffers(
69 + decoder: MediaCodec,
70 + extractor: MediaExtractor,
71 + stream: OutputStream
72 +) {
73 + var converter: RecordingPcmConverter? = null
74 + var decodedFormat: Triple<Int, Int, Int>? = null
75 + try {
76 + var inputEnded = false
77 + var outputEnded = false
78 + val info = MediaCodec.BufferInfo()
79 + while (!outputEnded) {
80 + currentCoroutineContext().ensureActive()
81 + if (!inputEnded) inputEnded = queueDecoderInput(decoder, extractor)
82 + val index = decoder.dequeueOutputBuffer(info, 10000)
83 + when {
84 + index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
85 + val next = pcmFormat(decoder.outputFormat, decodedFormat)
86 + decodedFormat = next
87 + converter = converter ?: createConverter(next, stream)
88 + }
89 + index >= 0 -> {
90 + consumeDecodedBuffer(decoder, index, info, requireNotNull(converter))
91 + outputEnded = info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0
92 + }
93 + }
94 + }
95 + requireCompleteAudio(inputEnded) {
96 + "Audio decoding ended before reading the complete recording."
97 + }
98 + converter?.finish()
99 + } finally {
100 + converter?.close()
101 + }
102 +}
103 +
104 +private fun pcmFormat(
105 + format: MediaFormat,
106 + previous: Triple<Int, Int, Int>?
107 +): Triple<Int, Int, Int> {
108 + val encoding = if (format.containsKey(MediaFormat.KEY_PCM_ENCODING)) {
109 + format.getInteger(MediaFormat.KEY_PCM_ENCODING)
110 + } else {
111 + AudioFormat.ENCODING_PCM_16BIT
112 + }
113 + requireCompleteAudio(encoding == AudioFormat.ENCODING_PCM_16BIT) {
114 + "The decoder did not provide 16-bit audio."
115 + }
116 + val next = Triple(
117 + format.getInteger(MediaFormat.KEY_SAMPLE_RATE),
118 + format.getInteger(MediaFormat.KEY_CHANNEL_COUNT),
119 + encoding
120 + )
121 + requireCompleteAudio(previous == null || previous == next) {
122 + "The recording changed audio format while decoding."
123 + }
124 + return next
125 +}
126 +
127 +private fun createConverter(
128 + format: Triple<Int, Int, Int>,
129 + stream: OutputStream
130 +): RecordingPcmConverter {
131 + val bytes = ByteArray(8192)
132 + return RecordingPcmConverter(format.first, format.second) { buffer ->
133 + while (buffer.hasRemaining()) {
134 + val count = minOf(buffer.remaining(), bytes.size)
135 + buffer.get(bytes, 0, count)
136 + stream.write(bytes, 0, count)
137 + }
138 + }
139 +}
140 +
141 +private fun queueDecoderInput(decoder: MediaCodec, extractor: MediaExtractor): Boolean {
142 + val index = decoder.dequeueInputBuffer(10000)
143 + if (index < 0) return false
144 + val buffer = requireNotNull(decoder.getInputBuffer(index))
145 + val count = extractor.readSampleData(buffer, 0)
146 + return if (count < 0) {
147 + decoder.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM)
148 + true
149 + } else {
150 + decoder.queueInputBuffer(index, 0, count, extractor.sampleTime, 0)
151 + extractor.advance()
152 + false
153 + }
154 +}
155 +
156 +private fun consumeDecodedBuffer(
157 + decoder: MediaCodec,
158 + index: Int,
159 + info: MediaCodec.BufferInfo,
160 + converter: RecordingPcmConverter
161 +) {
162 + try {
163 + if (info.size <= 0 || info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) return
164 + val buffer = requireNotNull(decoder.getOutputBuffer(index))
165 + buffer.position(info.offset)
166 + buffer.limit(info.offset + info.size)
167 + buffer.order(ByteOrder.nativeOrder())
168 + converter.write(buffer)
169 + } finally {
170 + decoder.releaseOutputBuffer(index, false)
171 + }
172 +}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/RecordingPcmConverter.kt new
+68
@@ -0,0 +1,68 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import androidx.annotation.OptIn
4 +import androidx.media3.common.C
5 +import androidx.media3.common.audio.AudioProcessingPipeline
6 +import androidx.media3.common.audio.AudioProcessor
7 +import androidx.media3.common.audio.ChannelMixingAudioProcessor
8 +import androidx.media3.common.audio.ChannelMixingMatrix
9 +import androidx.media3.common.audio.SonicAudioProcessor
10 +import androidx.media3.common.util.UnstableApi
11 +import com.google.common.collect.ImmutableList
12 +import java.nio.ByteBuffer
13 +
14 +@OptIn(UnstableApi::class)
15 +internal class RecordingPcmConverter(
16 + sampleRate: Int,
17 + channels: Int,
18 + private val consume: (ByteBuffer) -> Unit
19 +) : AutoCloseable {
20 + private val frameSize = channels * 2
21 + private val pipeline = AudioProcessingPipeline(
22 + ImmutableList.of(
23 + ChannelMixingAudioProcessor().apply {
24 + putChannelMixingMatrix(ChannelMixingMatrix.createForConstantGain(channels, 1))
25 + },
26 + SonicAudioProcessor().apply { setOutputSampleRateHz(16000) }
27 + )
28 + )
29 +
30 + init {
31 + pipeline.configure(AudioProcessor.AudioFormat(sampleRate, channels, C.ENCODING_PCM_16BIT))
32 + pipeline.flush()
33 + }
34 +
35 + fun write(input: ByteBuffer) {
36 + requireCompleteAudio(input.remaining() % frameSize == 0) {
37 + "Decoded audio contains an incomplete sample."
38 + }
39 + if (!pipeline.isOperational) {
40 + consume(input)
41 + return
42 + }
43 + while (input.hasRemaining()) {
44 + val before = input.remaining()
45 + pipeline.queueInput(input)
46 + val output = pipeline.output
47 + val produced = output.remaining()
48 + consume(output)
49 + requireCompleteAudio(input.remaining() < before || produced > 0) {
50 + "Audio conversion stopped before the recording ended."
51 + }
52 + }
53 + }
54 +
55 + fun finish() {
56 + if (!pipeline.isOperational) return
57 + pipeline.queueEndOfStream()
58 + while (!pipeline.isEnded) {
59 + val output = pipeline.output
60 + requireCompleteAudio(output.hasRemaining() || pipeline.isEnded) {
61 + "Audio conversion did not finish the recording."
62 + }
63 + consume(output)
64 + }
65 + }
66 +
67 + override fun close() = pipeline.reset()
68 +}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt new
+112
@@ -0,0 +1,112 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import android.content.res.Resources
4 +import android.os.Build
5 +import expo.modules.kotlin.Promise
6 +import expo.modules.kotlin.functions.Coroutine
7 +import expo.modules.kotlin.modules.Module
8 +import expo.modules.kotlin.modules.ModuleDefinition
9 +import java.util.Locale
10 +import kotlinx.coroutines.CancellationException
11 +import kotlinx.coroutines.CoroutineScope
12 +import kotlinx.coroutines.CoroutineStart
13 +import kotlinx.coroutines.Dispatchers
14 +import kotlinx.coroutines.Job
15 +import kotlinx.coroutines.SupervisorJob
16 +import kotlinx.coroutines.cancel
17 +import kotlinx.coroutines.cancelAndJoin
18 +import kotlinx.coroutines.launch
19 +import kotlinx.coroutines.withContext
20 +
21 +class T3VoiceTranscriptionModule : Module() {
22 + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
23 + private val lock = Any()
24 + private var operation: Pair<Int, Job>? = null
25 +
26 + override fun definition() = ModuleDefinition {
27 + Name("T3VoiceTranscription")
28 +
29 + AsyncFunction("checkAvailability").Coroutine<Boolean> {
30 + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
31 + false
32 + } else {
33 + withContext(Dispatchers.IO) {
34 + val context = requireNotNull(appContext.reactContext)
35 + val locale = Resources.getSystem().configuration.locales[0]
36 + isGoogleVoiceTranscriptionSupported { MlKitSpeechClient(context, locale, it) }
37 + }
38 + }
39 + }
40 +
41 + AsyncFunction("prepare") { id: Int, promise: Promise ->
42 + runOperation(id, promise, "preparation-failed") {
43 + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
44 + throw VoiceTranscriptionException(
45 + "unavailable",
46 + "Voice transcription requires Android 12 or later."
47 + )
48 + }
49 + val context = requireNotNull(appContext.reactContext)
50 + val locale = Resources.getSystem().configuration.locales[0]
51 + val mode = prepareGoogleVoiceTranscription { MlKitSpeechClient(context, locale, it) }
52 + mapOf("locale" to locale.toLanguageTag(), "mode" to mode)
53 + }
54 + }
55 +
56 + AsyncFunction("transcribe") {
57 + id: Int,
58 + uri: String,
59 + language: String,
60 + mode: Int,
61 + promise: Promise
62 + ->
63 + runOperation(id, promise, "transcription-failed") {
64 + val context = requireNotNull(appContext.reactContext)
65 + MlKitSpeechClient(context, Locale.forLanguageTag(language), mode).use { it.transcribe(uri) }
66 + }
67 + }
68 +
69 + AsyncFunction("cancel") { id: Int, promise: Promise ->
70 + scope.launch {
71 + val job = synchronized(lock) { operation?.takeIf { it.first == id }?.second }
72 + job?.cancelAndJoin()
73 + promise.resolve(null)
74 + }
75 + }
76 +
77 + OnDestroy { scope.cancel() }
78 + }
79 +
80 + private fun runOperation(
81 + id: Int,
82 + promise: Promise,
83 + errorCode: String,
84 + block: suspend () -> Any?
85 + ) {
86 + var result: Result<Any?>? = null
87 + val job = scope.launch(start = CoroutineStart.LAZY) {
88 + result = runCatching { block() }
89 + }
90 + synchronized(lock) {
91 + if (operation != null) {
92 + job.cancel()
93 + promise.reject("busy", "Another voice transcription is already active.", null)
94 + return
95 + }
96 + operation = id to job
97 + }
98 + job.invokeOnCompletion { cause ->
99 + synchronized(lock) { if (operation?.first == id) operation = null }
100 + val outcome = cause?.let { Result.failure<Any?>(it) } ?: result
101 + ?: Result.failure(CancellationException("Voice transcription was cancelled."))
102 + outcome.fold(
103 + onSuccess = { promise.resolve(it) },
104 + onFailure = { error ->
105 + val code = voiceTranscriptionErrorCode(error, errorCode)
106 + promise.reject(code, error.message, error)
107 + }
108 + )
109 + }
110 + job.start()
111 + }
112 +}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/VoiceTranscriptionFailure.kt new
+24
@@ -0,0 +1,24 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import com.google.mlkit.genai.common.GenAiException
4 +import kotlinx.coroutines.CancellationException
5 +
6 +internal fun voiceTranscriptionErrorCode(
7 + error: Throwable,
8 + fallback: String
9 +): String = when (error) {
10 + is CancellationException -> "cancelled"
11 + is VoiceTranscriptionException -> error.code
12 + is GenAiException -> when {
13 + error.message.orEmpty().contains("NO_SPEECH_DETECTED") -> "no-speech"
14 + error.errorCode == GenAiException.ErrorCode.NOT_ENOUGH_DISK_SPACE -> "storage"
15 + error.errorCode == GenAiException.ErrorCode.PER_APP_BATTERY_USE_QUOTA_EXCEEDED -> "quota"
16 + error.errorCode == GenAiException.ErrorCode.BUSY -> "busy"
17 + error.errorCode == GenAiException.ErrorCode.AUDIO_BUFFER_OVERFLOW -> "audio-overflow"
18 + error.errorCode == GenAiException.ErrorCode.NEEDS_SYSTEM_UPDATE ||
19 + error.errorCode == GenAiException.ErrorCode.AICORE_INCOMPATIBLE -> "needs-update"
20 + error.message.orEmpty().contains("DOWNLOAD_ERROR") -> "download-failed"
21 + else -> fallback
22 + }
23 + else -> fallback
24 +}
apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriberTest.kt new
+211
@@ -0,0 +1,211 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import com.google.mlkit.genai.common.FeatureStatus
4 +import com.google.mlkit.genai.common.GenAiException
5 +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerOptions
6 +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
7 +import kotlinx.coroutines.CompletableDeferred
8 +import kotlinx.coroutines.cancelAndJoin
9 +import kotlinx.coroutines.flow.flow
10 +import kotlinx.coroutines.launch
11 +import kotlinx.coroutines.runBlocking
12 +import org.junit.Assert.assertEquals
13 +import org.junit.Assert.assertTrue
14 +import org.junit.Test
15 +
16 +class GoogleVoiceTranscriberTest {
17 + @Test
18 + fun preservesUtterancesAndTheirSpacingWithoutDuplicatingHypotheses() = runBlocking {
19 + for ((finals, trailing, expected) in listOf(
20 + Triple(
21 + listOf("The beginning.", " The middle.", " Repeat this.", " Repeat this."),
22 + emptyList(),
23 + "The beginning. The middle. Repeat this. Repeat this."
24 + ),
25 + Triple(listOf("最初の文。", "次の文。"), emptyList(), "最初の文。次の文。"),
26 + Triple(
27 + listOf("The beginning."),
28 + listOf(" An unfinished ending", " The final sentence."),
29 + "The beginning. The final sentence."
30 + )
31 + )) {
32 + val responses = flow {
33 + for (text in finals) {
34 + emit(SpeechRecognizerResponse.PartialTextResponse("Unfinished hypothesis"))
35 + emit(SpeechRecognizerResponse.FinalTextResponse(text, emptyList()))
36 + }
37 + for (text in trailing) emit(SpeechRecognizerResponse.PartialTextResponse(text))
38 + emit(SpeechRecognizerResponse.CompletedResponse)
39 + }
40 + assertEquals(expected, collectGoogleVoiceTranscript(responses))
41 + }
42 + val incomplete = runCatching {
43 + collectGoogleVoiceTranscript(
44 + flow {
45 + emit(SpeechRecognizerResponse.FinalTextResponse("Incomplete recording.", emptyList()))
46 + }
47 + )
48 + }
49 + assertEquals(
50 + "incomplete-audio",
51 + (incomplete.exceptionOrNull() as VoiceTranscriptionException).code
52 + )
53 + }
54 +
55 + private class Client(
56 + var status: Int,
57 + val onStatus: () -> Unit = {},
58 + val downloadedStatus: Int = FeatureStatus.AVAILABLE,
59 + val onDownload: suspend () -> Unit = {}
60 + ) : GoogleSpeechClient {
61 + var closed = false
62 + var downloads = 0
63 +
64 + override suspend fun checkStatus(): Int {
65 + onStatus()
66 + return status
67 + }
68 +
69 + override suspend fun download() {
70 + downloads += 1
71 + onDownload()
72 + status = downloadedStatus
73 + }
74 +
75 + override fun close() {
76 + closed = true
77 + }
78 + }
79 +
80 + @Test
81 + fun checksSupportWithoutDownloadingAndClosesProbedClients() = runBlocking {
82 + for ((advancedStatus, basicStatus, supported) in listOf(
83 + Triple(FeatureStatus.AVAILABLE, FeatureStatus.UNAVAILABLE, true),
84 + Triple(FeatureStatus.DOWNLOADABLE, FeatureStatus.UNAVAILABLE, true),
85 + Triple(FeatureStatus.DOWNLOADING, FeatureStatus.UNAVAILABLE, true),
86 + Triple(FeatureStatus.UNAVAILABLE, FeatureStatus.AVAILABLE, true),
87 + Triple(FeatureStatus.UNAVAILABLE, FeatureStatus.DOWNLOADABLE, true),
88 + Triple(FeatureStatus.UNAVAILABLE, FeatureStatus.DOWNLOADING, true),
89 + Triple(FeatureStatus.UNAVAILABLE, FeatureStatus.UNAVAILABLE, false)
90 + )) {
91 + val clients = mutableListOf<Client>()
92 + assertEquals(
93 + supported,
94 + isGoogleVoiceTranscriptionSupported { mode ->
95 + Client(
96 + if (mode == SpeechRecognizerOptions.Mode.MODE_ADVANCED) advancedStatus else basicStatus
97 + ).also { clients.add(it) }
98 + }
99 + )
100 + assertTrue(clients.all { it.closed && it.downloads == 0 })
101 + assertEquals(if (advancedStatus == FeatureStatus.UNAVAILABLE) 2 else 1, clients.size)
102 + }
103 + }
104 +
105 + @Test
106 + fun preparesAdvancedBeforeConsideringBasic() = runBlocking {
107 + for (status in listOf(
108 + FeatureStatus.AVAILABLE,
109 + FeatureStatus.DOWNLOADABLE,
110 + FeatureStatus.DOWNLOADING
111 + )) {
112 + val client = Client(status)
113 + val mode = prepareGoogleVoiceTranscription {
114 + assertEquals(SpeechRecognizerOptions.Mode.MODE_ADVANCED, it)
115 + client
116 + }
117 + assertEquals(SpeechRecognizerOptions.Mode.MODE_ADVANCED, mode)
118 + assertEquals(if (status == FeatureStatus.AVAILABLE) 0 else 1, client.downloads)
119 + assertTrue(client.closed)
120 + }
121 + }
122 +
123 + @Test
124 + fun fallsBackToBasicOnlyForUnavailableAdvancedModels() = runBlocking {
125 + for ((duringDownload, errorCode) in listOf(
126 + false to null,
127 + false to GenAiException.ErrorCode.NOT_AVAILABLE,
128 + false to GenAiException.ErrorCode.NOT_SUPPORTED,
129 + true to GenAiException.ErrorCode.NOT_AVAILABLE,
130 + true to GenAiException.ErrorCode.NOT_SUPPORTED,
131 + true to null
132 + )) {
133 + val advanced = Client(
134 + if (duringDownload) FeatureStatus.DOWNLOADABLE else FeatureStatus.UNAVAILABLE,
135 + downloadedStatus = if (duringDownload &&
136 + errorCode == null
137 + ) {
138 + FeatureStatus.UNAVAILABLE
139 + } else {
140 + FeatureStatus.AVAILABLE
141 + },
142 + onStatus = {
143 + if (!duringDownload && errorCode != null) {
144 + throw GenAiException(Exception("Unavailable"), errorCode)
145 + }
146 + },
147 + onDownload = {
148 + if (duringDownload && errorCode != null) {
149 + throw GenAiException(Exception("Unavailable"), errorCode)
150 + }
151 + }
152 + )
153 + val basic = Client(FeatureStatus.DOWNLOADABLE)
154 + val mode = prepareGoogleVoiceTranscription {
155 + if (it == SpeechRecognizerOptions.Mode.MODE_ADVANCED) advanced else basic
156 + }
157 + assertEquals(SpeechRecognizerOptions.Mode.MODE_BASIC, mode)
158 + assertTrue(advanced.closed && basic.closed)
159 + assertEquals(1, basic.downloads)
160 + }
161 + val clients = mutableListOf<Client>()
162 + val result = runCatching {
163 + prepareGoogleVoiceTranscription { Client(FeatureStatus.UNAVAILABLE).also { clients.add(it) } }
164 + }
165 + assertEquals("unavailable", (result.exceptionOrNull() as VoiceTranscriptionException).code)
166 + assertTrue(clients.all { it.closed })
167 + }
168 +
169 + @Test
170 + fun preservesDownloadAndQuotaFailuresInsteadOfSilentlyFallingBack() = runBlocking {
171 + for (errorCode in listOf(
172 + GenAiException.ErrorCode.NOT_ENOUGH_DISK_SPACE,
173 + GenAiException.ErrorCode.PER_APP_BATTERY_USE_QUOTA_EXCEEDED
174 + )) {
175 + val error = GenAiException(Exception("Preparation failed"), errorCode)
176 + val client = Client(FeatureStatus.DOWNLOADABLE, onDownload = { throw error })
177 + val result = runCatching {
178 + prepareGoogleVoiceTranscription {
179 + assertEquals(SpeechRecognizerOptions.Mode.MODE_ADVANCED, it)
180 + client
181 + }
182 + }
183 + assertEquals(error, result.exceptionOrNull())
184 + assertEquals(
185 + if (errorCode == GenAiException.ErrorCode.NOT_ENOUGH_DISK_SPACE) "storage" else "quota",
186 + voiceTranscriptionErrorCode(error, "preparation-failed")
187 + )
188 + assertTrue(client.closed)
189 + }
190 + }
191 +
192 + @Test
193 + fun cancellationClosesTheDownloadingClientWithoutStartingBasic() = runBlocking {
194 + val enteredDownload = CompletableDeferred<Unit>()
195 + val download = CompletableDeferred<Unit>()
196 + val client = Client(FeatureStatus.DOWNLOADABLE, onDownload = {
197 + enteredDownload.complete(Unit)
198 + download.await()
199 + })
200 + val job = launch {
201 + prepareGoogleVoiceTranscription {
202 + assertEquals(SpeechRecognizerOptions.Mode.MODE_ADVANCED, it)
203 + client
204 + }
205 + }
206 + enteredDownload.await()
207 + job.cancelAndJoin()
208 + assertTrue(client.closed)
209 + assertEquals(FeatureStatus.DOWNLOADABLE, client.status)
210 + }
211 +}
apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/PcmReplayTest.kt new
+126
@@ -0,0 +1,126 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
4 +import java.io.ByteArrayOutputStream
5 +import java.io.File
6 +import kotlinx.coroutines.CompletableDeferred
7 +import kotlinx.coroutines.cancelAndJoin
8 +import kotlinx.coroutines.flow.flow
9 +import kotlinx.coroutines.launch
10 +import kotlinx.coroutines.test.currentTime
11 +import kotlinx.coroutines.test.runTest
12 +import org.junit.Assert.assertArrayEquals
13 +import org.junit.Assert.assertEquals
14 +import org.junit.Assert.assertTrue
15 +import org.junit.Test
16 +
17 +@OptIn(kotlinx.coroutines.ExperimentalCoroutinesApi::class)
18 +class PcmReplayTest {
19 + private class Output : PcmOutput {
20 + val bytes = ByteArrayOutputStream()
21 + val ended = CompletableDeferred<Unit>()
22 + var unread = false
23 + var closed = false
24 +
25 + override suspend fun write(buffer: ByteArray, count: Int) {
26 + bytes.write(buffer, 0, count)
27 + }
28 + override fun finish() {
29 + ended.complete(Unit)
30 + }
31 + override fun verifyConsumed() {
32 + requireCompleteAudio(!unread) { "Unread audio." }
33 + }
34 + override fun close() {
35 + closed = true
36 + }
37 + }
38 +
39 + @Test fun replaysTheEntireLongFileIncludingTheLastShortBuffer() = runTest {
40 + val pcm = ByteArray(1994140) { (it % 251).toByte() }
41 + val file = File.createTempFile("dictation-replay-", ".pcm").apply { writeBytes(pcm) }
42 + try {
43 + Output().use { output ->
44 + val responses = pacedRecordedResponses(file, output) {
45 + flow {
46 + output.ended.await()
47 + emit(SpeechRecognizerResponse.FinalTextResponse("Opening. ", emptyList()))
48 + emit(SpeechRecognizerResponse.FinalTextResponse("Repeat. Repeat. 中間。", emptyList()))
49 + emit(SpeechRecognizerResponse.PartialTextResponse(" A longer unfinished ending"))
50 + emit(SpeechRecognizerResponse.PartialTextResponse(" End."))
51 + emit(SpeechRecognizerResponse.CompletedResponse)
52 + }
53 + }
54 + assertEquals("Opening. Repeat. Repeat. 中間。 End.", collectGoogleVoiceTranscript(responses))
55 + assertArrayEquals(pcm, output.bytes.toByteArray())
56 + assertEquals(62317L, currentTime)
57 + }
58 + } finally {
59 + file.delete()
60 + }
61 + }
62 +
63 + @Test fun refusesSuccessfulSdkCompletionBeforeAllAudioIsConsumed() = runTest {
64 + val file = File.createTempFile("dictation-replay-", ".pcm").apply {
65 + writeBytes(ByteArray(6400))
66 + }
67 + try {
68 + for (early in listOf(true, false)) {
69 + Output().use { output ->
70 + output.unread = !early
71 + val result = runCatching {
72 + collectGoogleVoiceTranscript(
73 + pacedRecordedResponses(file, output) {
74 + flow {
75 + if (!early) output.ended.await()
76 + emit(
77 + SpeechRecognizerResponse.FinalTextResponse(
78 + "Plausible but incomplete.",
79 + emptyList()
80 + )
81 + )
82 + emit(SpeechRecognizerResponse.CompletedResponse)
83 + }
84 + }
85 + )
86 + }
87 + assertEquals(
88 + "incomplete-audio",
89 + (result.exceptionOrNull() as VoiceTranscriptionException).code
90 + )
91 + assertTrue(output.ended.isCompleted)
92 + }
93 + }
94 + } finally {
95 + file.delete()
96 + }
97 + }
98 +
99 + @Test fun cancellationSettlesTheSenderBeforeReleasingItsOutput() = runTest {
100 + val file = File.createTempFile("dictation-replay-", ".pcm").apply {
101 + writeBytes(ByteArray(6400))
102 + }
103 + val entered = CompletableDeferred<Unit>()
104 + val output = Output()
105 + try {
106 + val job = launch {
107 + output.use {
108 + collectGoogleVoiceTranscript(
109 + pacedRecordedResponses(file, it) {
110 + flow {
111 + entered.complete(Unit)
112 + CompletableDeferred<Unit>().await()
113 + }
114 + }
115 + )
116 + }
117 + }
118 + entered.await()
119 + job.cancelAndJoin()
120 + assertTrue(output.ended.isCompleted)
121 + assertTrue(output.closed)
122 + } finally {
123 + file.delete()
124 + }
125 + }
126 +}
apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/RecordingPcmConverterTest.kt new
+51
@@ -0,0 +1,51 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import java.io.ByteArrayOutputStream
4 +import java.nio.ByteBuffer
5 +import java.nio.ByteOrder
6 +import kotlin.math.abs
7 +import org.junit.Assert.assertArrayEquals
8 +import org.junit.Assert.assertEquals
9 +import org.junit.Assert.assertTrue
10 +import org.junit.Test
11 +import org.junit.runner.RunWith
12 +import org.robolectric.RobolectricTestRunner
13 +import org.robolectric.annotation.Config
14 +
15 +@RunWith(RobolectricTestRunner::class)
16 +@Config(manifest = Config.NONE, sdk = [31])
17 +class RecordingPcmConverterTest {
18 + @Test fun preservesAlreadyCompatiblePcmAndResamplesStereoThroughItsTail() {
19 + for ((rate, channels) in listOf(16000 to 1, 44100 to 2, 48000 to 2)) {
20 + val frames = rate * 2 + 37
21 + val input = ByteBuffer.allocateDirect(frames * channels * 2).order(ByteOrder.nativeOrder())
22 + repeat(frames) { frame ->
23 + val level = if (frame < rate) 6000 else 18000
24 + repeat(channels) { input.putShort(level.toShort()) }
25 + }
26 + input.flip()
27 + val original = ByteArray(input.remaining()).also { input.duplicate().get(it) }
28 + val result = ByteArrayOutputStream()
29 + RecordingPcmConverter(rate, channels) { buffer ->
30 + val bytes = ByteArray(buffer.remaining())
31 + buffer.get(bytes)
32 + result.write(bytes)
33 + }.use { converter ->
34 + while (input.hasRemaining()) {
35 + val end = minOf(input.limit(), input.position() + channels * 2 * 317)
36 + val block = input.slice().order(ByteOrder.nativeOrder())
37 + block.limit(end - input.position())
38 + converter.write(block)
39 + input.position(end)
40 + }
41 + converter.finish()
42 + }
43 + val bytes = result.toByteArray()
44 + if (rate == 16000) assertArrayEquals(original, bytes)
45 + val samples = ByteBuffer.wrap(bytes).order(ByteOrder.nativeOrder()).asShortBuffer()
46 + assertTrue(abs(samples.remaining() * rate.toLong() - frames * 16000L) <= rate)
47 + assertEquals(6000, samples[8000].toInt())
48 + assertEquals(18000, samples[samples.limit() - 10].toInt())
49 + }
50 + }
51 +}
apps/mobile/modules/t3-voice-transcription/expo-module.config.json new
+6
@@ -0,0 +1,6 @@
1 +{
2 + "platforms": ["android"],
3 + "android": {
4 + "modules": ["expo.modules.t3voicetranscription.T3VoiceTranscriptionModule"]
5 + }
6 +}
apps/mobile/src/features/settings/SettingsMicrophoneRouteScreen.tsx
+26 -15
@@ -3,7 +3,7 @@ import { useAtomSet, useAtomValue } from "@effect/atom-react";
3 import { AsyncResult } from "effect/reactivity";
4 import { getRecordingPermissionsAsync, requestRecordingPermissionsAsync } from "expo-audio";
5 import { useState } from "react";
6 -import { Linking, useWindowDimensions, View } from "react-native";
6 +import { Linking, Platform, useWindowDimensions, View } from "react-native";
7 import Reanimated, { ReduceMotion, useAnimatedStyle, withTiming } from "react-native-reanimated";
8 import { useSafeAreaInsets } from "react-native-safe-area-context";
9
@@ -16,15 +16,17 @@ import {
16 type MicrophoneEntry,
17 type MicrophoneKind,
18 microphoneEntryKey,
19 - microphoneOrder,
19 } from "../../lib/microphonePriority";
21 -import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
20 import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
21 import { ListRowLayout } from "../../components/ui/ListRowLayout";
22 import { ThreadSwipeable } from "../home/thread-swipe-actions";
23 import { ListActionRow as SettingsActionRow } from "../../components/ui/ListActionRow";
24 import { SettingsDragHandle } from "./components/SettingsDragHandle";
25 import { SettingsSection } from "./components/SettingsSection";
26 +import {
27 + deviceMicrophoneOrder,
28 + localVoiceTranscriberAtom,
29 +} from "../voice-input/VoiceInputProvider";
30
31 type MicrophonePermission = "checking" | "granted" | "ask" | "denied";
32
@@ -72,7 +74,10 @@ const KINDS: Record<
74 bluetooth: {
75 label: "Bluetooth",
76 row: "Other Bluetooth",
75 - rowDescription: "AirPods, headsets, and cars not listed",
77 + rowDescription:
78 + Platform.OS === "android"
79 + ? "Earbuds, headsets, and cars, including Android Auto, not listed"
80 + : "AirPods, headsets, and cars not listed",
81 icon: "headphones",
82 },
83 builtIn: {
@@ -89,8 +94,16 @@ const KINDS: Record<
94 },
95 };
96
97 +const UNAVAILABLE_EXPLANATION =
98 + Platform.OS === "android"
99 + ? "T3 Code's voice input uses Google's on-device speech recognition, which needs Android 12 or later with compatible Google speech services for the device language. Without it, the composer has no microphone button, and allowing microphone access won't add one. Keyboard voice typing is separate and isn't affected."
100 + : "T3 Code's voice input uses Apple's on-device speech transcription, which needs iOS 26 or later on a device that supports it. Without it, the composer has no microphone button, and allowing microphone access won't add one. Apple keyboard dictation is separate and isn't affected.";
101 +
102 export function SettingsMicrophoneRouteScreen() {
93 - return getLocalVoiceTranscriber() === null ? <VoiceInputUnavailable /> : <MicrophoneSettings />;
103 + const transcriber = useAtomValue(localVoiceTranscriberAtom);
104 + // Android's probe is asynchronous; an empty sheet avoids flashing "unavailable" first.
105 + if (transcriber === "checking") return <View collapsable={false} className="flex-1 bg-sheet" />;
106 + return transcriber === null ? <VoiceInputUnavailable /> : <MicrophoneSettings />;
107 }
108
109 function VoiceInputUnavailable() {
@@ -107,12 +120,7 @@ function VoiceInputUnavailable() {
120 <SettingsSection title="Voice input">
121 <ListRowLayout icon="mic" label="Not available on this device" />
122 </SettingsSection>
110 - <Text className="px-2 text-sm text-foreground-muted">
111 - T3 Code's voice input uses Apple's on-device speech transcription, which needs iOS 26 or
112 - later on a device that supports it. Without it, the composer has no microphone button, and
113 - allowing microphone access won't add one. Apple keyboard dictation is separate and isn't
114 - affected.
115 - </Text>
123 + <Text className="px-2 text-sm text-foreground-muted">{UNAVAILABLE_EXPLANATION}</Text>
124 </ScrollView>
125 </View>
126 );
@@ -123,7 +131,7 @@ function MicrophoneSettings() {
131 const preferencesResult = useAtomValue(mobilePreferencesAtom);
132 const savePreferences = useAtomSet(updateMobilePreferencesAtom);
133 const ready = AsyncResult.isSuccess(preferencesResult) && !preferencesResult.waiting;
126 - const entries = microphoneOrder(
134 + const entries = deviceMicrophoneOrder(
135 AsyncResult.isSuccess(preferencesResult) ? preferencesResult.value.microphones : undefined,
136 );
137 const { permission, request: requestPermission } = useMicrophonePermission();
@@ -138,7 +146,7 @@ function MicrophoneSettings() {
146 const move = (key: string, steps: number) =>
147 savePreferences({
148 transform: (current) => {
141 - const next = [...microphoneOrder(current.microphones)];
149 + const next = [...deviceMicrophoneOrder(current.microphones)];
150 const from = next.findIndex((entry) => microphoneEntryKey(entry) === key);
151 const to = Math.min(next.length - 1, Math.max(0, from + steps));
152 if (from === -1 || from === to) return {};
@@ -150,7 +158,7 @@ function MicrophoneSettings() {
158 const forget = (key: string) =>
159 savePreferences({
160 transform: (current) => ({
153 - microphones: microphoneOrder(current.microphones).filter(
161 + microphones: deviceMicrophoneOrder(current.microphones).filter(
162 (entry) => microphoneEntryKey(entry) !== key,
163 ),
164 }),
@@ -227,9 +235,12 @@ function MicrophoneSettings() {
235 })}
236 </SettingsSection>
237 <Text className="px-2 text-sm text-foreground-muted">
230 - Voice input records from the first connected microphone in this list. Wired and Bluetooth
238 + Voice input prefers the first connected microphone in this list. Wired and Bluetooth
239 devices are added by name after you use voice input with them connected; until then, the
240 Other row ranks them. Swipe a device left to forget it.
241 + {Platform.OS === "android"
242 + ? " Android may use another input when the preferred microphone cannot be routed."
243 + : null}
244 </Text>
245 </ScrollView>
246 </View>
apps/mobile/src/features/settings/SettingsRouteScreen.tsx
+2 -6
@@ -244,12 +244,8 @@ function useSettingsIndexSections(): SettingsListSection[] {
244 ...(navigationCustomizationAvailable()
245 ? [page("SettingsNavigation", "line.3.horizontal", "Navigation tabs")]
246 : []),
247 - ...(Platform.OS === "ios"
248 - ? [
249 - page("SettingsKeyboard", "keyboard", "Keyboard"),
250 - page("SettingsMicrophone", "mic", "Microphone"),
251 - ]
252 - : []),
247 + ...(Platform.OS === "ios" ? [page("SettingsKeyboard", "keyboard", "Keyboard")] : []),
248 + page("SettingsMicrophone", "mic", "Microphone"),
249 ],
250 },
251 {
apps/mobile/src/features/settings/components/ScheduledTaskPromptField.tsx
+6 -2
@@ -4,6 +4,7 @@ import { AppText as Text } from "../../../components/ui/AppText";
4 import type { ComposerEditorSelection } from "../../../components/ComposerEditor";
5 import {
6 ComposerDictationCancelAction,
7 + ComposerDictationError,
8 ComposerDictationPrimaryAction,
9 ComposerDictationStartAction,
10 ComposerDictationStatus,
@@ -22,7 +23,7 @@ export function ScheduledTaskPromptField(props: {
23 }) {
24 const voice = props.voiceInput;
25 const presentation = resolveVoiceComposerPresentation(voice.state, voice.elapsedSeconds);
25 - const showsDictation = presentation.statusLabel !== null;
26 + const showsDictation = presentation.statusKind === "active";
27 const showsIdleTrigger = voice.isAvailable && !showsDictation;
28
29 return (
@@ -55,6 +56,10 @@ export function ScheduledTaskPromptField(props: {
56 </View>
57 ) : null}
58 </View>
59 + <ComposerDictationError
60 + error={voice.state.phase === "error" ? voice.state.error : null}
61 + onDismiss={voice.cancel}
62 + />
63 {showsDictation ? (
64 <ComposerDictationToolbar showsDictation>
65 <View className="h-11 flex-row items-center">
@@ -64,7 +69,6 @@ export function ScheduledTaskPromptField(props: {
69 elapsedSeconds={voice.elapsedSeconds}
70 phase={voice.state.phase}
71 presentation={presentation}
67 - onDismissError={voice.cancel}
72 />
73 <ComposerDictationPrimaryAction
74 state={voice.state}
apps/mobile/src/features/threads/ThreadComposer.tsx
+6 -2
@@ -107,6 +107,7 @@ import { offersLinkedEnvironments } from "@t3tools/client-runtime/link-request";
107 import {
108 ComposerDictationCancelAction,
109 ComposerDictationDraftContent,
110 + ComposerDictationError,
111 ComposerDictationPrimaryAction,
112 ComposerDictationStartAction,
113 ComposerDictationStatus,
@@ -576,7 +577,7 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
577 voiceInput.state,
578 voiceInput.elapsedSeconds,
579 );
579 - const isVoiceInputPresented = voicePresentation.statusLabel !== null;
580 + const isVoiceInputPresented = voicePresentation.statusKind === "active";
581 const isExpanded = isFocused || settingsSheetPresentation.keepsComposerExpanded;
582 const { layout, setComposerEditing } = useAdaptiveWorkspaceLayout();
583 const floatingSidebarComposer = useAndroidFloatingTabs() && layout.usesSplitView;
@@ -881,6 +882,10 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
882 paddingVertical: surfacePaddingVertical,
883 }}
884 >
885 + <ComposerDictationError
886 + error={voiceInput.state.phase === "error" ? voiceInput.state.error : null}
887 + onDismiss={voiceInput.cancel}
888 + />
889 {isExpanded ? editingAttachments : null}
890 <View
891 onLayout={(event) => {
@@ -1145,7 +1150,6 @@ export const ThreadComposer = function ThreadComposer(props: ThreadComposerProps
1150 elapsedSeconds={voiceInput.elapsedSeconds}
1151 phase={voiceInput.state.phase}
1152 presentation={voicePresentation}
1148 - onDismissError={voiceInput.cancel}
1153 />
1154 <View className="shrink-0 flex-row items-center">
1155 <ComposerDictationPrimaryAction
apps/mobile/src/features/voice-input/ComposerDictationControl.tsx
+46 -45
@@ -319,12 +319,39 @@ function VoiceActionButton(props: {
319 );
320 }
321
322 +export function ComposerDictationError(props: {
323 + readonly error: string | null;
324 + readonly onDismiss: () => void;
325 +}) {
326 + if (!props.error) return null;
327 + return (
328 + <View className="min-w-0 flex-row items-center gap-2 px-4 py-3">
329 + <Text accessibilityRole="alert" className="min-w-0 flex-1 text-sm text-danger-foreground">
330 + {props.error}
331 + </Text>
332 + <Pressable
333 + accessibilityLabel="Dismiss voice input error"
334 + accessibilityRole="button"
335 + className="size-7 items-center justify-center active:opacity-70"
336 + hitSlop={8}
337 + onPress={props.onDismiss}
338 + >
339 + <SymbolView
340 + name="xmark"
341 + size={12}
342 + tintColorClassName="accent-icon-muted"
343 + type="monochrome"
344 + />
345 + </Pressable>
346 + </View>
347 + );
348 +}
349 +
350 export function ComposerDictationStatus(props: {
351 readonly audioLevels: SharedValue<ReadonlyArray<number>>;
352 readonly elapsedSeconds: number;
353 readonly phase: VoiceInputPhase;
354 readonly presentation: VoiceComposerPresentation;
327 - readonly onDismissError: () => void;
355 }) {
356 const recording = props.phase === "recording";
357 const waveformStyle = useAnimatedStyle(() => ({
@@ -334,54 +361,28 @@ export function ComposerDictationStatus(props: {
361 opacity: withTiming(recording ? 0 : 1, DICTATION_TIMING),
362 }));
363
337 - if (!props.presentation.statusLabel) return null;
338 - const isError = props.presentation.statusKind === "error";
364 + if (!props.presentation.statusLabel || props.presentation.statusKind === "error") return null;
365 return (
366 <View className="relative h-11 min-w-0 flex-1 justify-center">
341 - {isError ? (
342 - <View className="min-w-0 flex-row items-center gap-1.5 px-2">
343 - <Text className="min-w-0 flex-1 text-sm text-danger-foreground" numberOfLines={2}>
367 + <View
368 + accessible
369 + accessibilityLabel={props.presentation.statusLabel}
370 + accessibilityLiveRegion={props.phase === "recording" ? "none" : "polite"}
371 + className="h-11"
372 + >
373 + <Animated.View
374 + className="absolute inset-0 min-w-0 flex-row items-center gap-2 px-1"
375 + style={waveformStyle}
376 + >
377 + <VoiceWaveform audioLevels={props.audioLevels} />
378 + <DictationElapsedTime className="text-foreground-muted" seconds={props.elapsedSeconds} />
379 + </Animated.View>
380 + <Animated.View className="absolute inset-0 justify-center px-2" style={labelStyle}>
381 + <Text className="text-center text-sm text-foreground-muted" numberOfLines={1}>
382 {props.presentation.statusLabel}
383 </Text>
346 - <Pressable
347 - accessibilityLabel="Dismiss voice input error"
348 - accessibilityRole="button"
349 - className="size-7 items-center justify-center active:opacity-70"
350 - hitSlop={8}
351 - onPress={props.onDismissError}
352 - >
353 - <SymbolView
354 - name="xmark"
355 - size={12}
356 - tintColorClassName="accent-icon-muted"
357 - type="monochrome"
358 - />
359 - </Pressable>
360 - </View>
361 - ) : (
362 - <View
363 - accessible
364 - accessibilityLabel={props.presentation.statusLabel}
365 - accessibilityLiveRegion={props.phase === "recording" ? "none" : "polite"}
366 - className="h-11"
367 - >
368 - <Animated.View
369 - className="absolute inset-0 min-w-0 flex-row items-center gap-2 px-1"
370 - style={waveformStyle}
371 - >
372 - <VoiceWaveform audioLevels={props.audioLevels} />
373 - <DictationElapsedTime
374 - className="text-foreground-muted"
375 - seconds={props.elapsedSeconds}
376 - />
377 - </Animated.View>
378 - <Animated.View className="absolute inset-0 justify-center px-2" style={labelStyle}>
379 - <Text className="text-center text-sm text-foreground-muted" numberOfLines={1}>
380 - {props.presentation.statusLabel}
381 - </Text>
382 - </Animated.View>
383 - </View>
384 - )}
384 + </Animated.View>
385 + </View>
386 </View>
387 );
388 }
apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
+40 -22
@@ -1,18 +1,14 @@
1 import { useAtomValue } from "@effect/atom-react";
2 -import {
3 - AudioModule,
4 - RecordingPresets,
5 - requestRecordingPermissionsAsync,
6 - type RecordingStatus,
7 -} from "expo-audio";
2 +import { AudioModule, RecordingPresets, type RecordingStatus } from "expo-audio";
3 import { File } from "expo-file-system";
4 import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
5 import { useSyncExternalStore } from "react";
11 -import { AppState, Platform } from "react-native";
6 +import { Platform } from "react-native";
7 import { Atom, AsyncResult } from "effect/reactivity";
8 import { makeMutable } from "react-native-reanimated";
9
10 import {
11 + type MicrophoneEntry,
12 microphoneOrder,
13 preferredMicrophoneInput,
14 rememberMicrophones,
@@ -20,10 +16,15 @@ import {
16 import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
17 import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
18 import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene";
23 -import { voiceInputBlocksSubmission } from "@t3tools/client-runtime/voice-input";
19 +import {
20 + voiceInputBlocksSubmission,
21 + type VoiceTranscriber,
22 +} from "@t3tools/client-runtime/voice-input";
23 import { createLazyVoiceRecorder } from "./lazyVoiceRecorder";
24 import { VOICE_WAVEFORM_SAMPLE_COUNT } from "./voiceInputMetering";
25 import { VoiceInputSession } from "./voiceInputSession";
26 +import { createVoiceRecordingPermission } from "./voiceRecordingPermission";
27 +import { makePermissionStatusAtom } from "../../lib/permissionStatusAtom";
28 import {
29 acquireAudioSession,
30 activateAudioSession,
@@ -49,6 +50,24 @@ function keepScreenAwake(tag: string): () => void {
50 return () => void activation.then(() => deactivateKeepAwake(tag)).catch(() => {});
51 }
52
53 +export const deviceMicrophoneOrder = (saved?: ReadonlyArray<MicrophoneEntry>) =>
54 + microphoneOrder(saved, Platform.OS === "android" ? "android" : "ios");
55 +
56 +/** "checking" until the first availability probe settles; re-probed on foreground. */
57 +export const localVoiceTranscriberAtom = makePermissionStatusAtom<
58 + VoiceTranscriber | null | "checking"
59 +>({
60 + label: "mobile:voice-input:transcriber",
61 + initial: "checking",
62 + read: () => getLocalVoiceTranscriber().catch(() => null),
63 +}).pipe(Atom.keepAlive);
64 +
65 +// Each foreground re-probe yields a new transcriber; composers re-render only when availability flips.
66 +const localVoiceInputAvailableAtom = Atom.make((get) => {
67 + const transcriber = get(localVoiceTranscriberAtom);
68 + return transcriber !== null && transcriber !== "checking";
69 +});
70 +
71 /**
72 * The app has one microphone and one dictation at a time, so the session
73 * lives for the registry's lifetime rather than any screen's.
@@ -58,11 +77,15 @@ const voiceInputAtom = Atom.make((get) => {
77 // Mounted so the recorder can read and remember microphones synchronously.
78 get.mount(mobilePreferencesAtom);
79 get.mount(updateMobilePreferencesAtom);
80 + get.mount(localVoiceTranscriberAtom);
81 const audioLevels = makeMutable<ReadonlyArray<number>>(
82 Array<number>(VOICE_WAVEFORM_SAMPLE_COUNT).fill(0),
83 );
84 let keepAwakeSequence = 0;
85 let audioToken: ReturnType<typeof acquireAudioSession> | null = null;
86 + const permission = createVoiceRecordingPermission(() => {
87 + void session.controller.appMovedToBackground();
88 + });
89 const recorder = createLazyVoiceRecorder({
90 create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
91 onStatus: (status: RecordingStatus) => {
@@ -77,12 +100,12 @@ const voiceInputAtom = Atom.make((get) => {
100 selectInput: (inputs) => {
101 const preferences = registry.get(mobilePreferencesAtom);
102 if (!AsyncResult.isSuccess(preferences)) return null;
80 - const known = microphoneOrder(preferences.value.microphones);
103 + const known = deviceMicrophoneOrder(preferences.value.microphones);
104 const order = rememberMicrophones(known, inputs);
105 if (order !== known) {
106 registry.set(updateMobilePreferencesAtom, {
107 transform: (current) => ({
85 - microphones: rememberMicrophones(microphoneOrder(current.microphones), inputs),
108 + microphones: rememberMicrophones(deviceMicrophoneOrder(current.microphones), inputs),
109 }),
110 });
111 }
@@ -91,11 +114,11 @@ const voiceInputAtom = Atom.make((get) => {
114 });
115 const session: VoiceInputSession = new VoiceInputSession({
116 recorder,
94 - getTranscriber: getLocalVoiceTranscriber,
95 - requestPermission: async () => {
96 - const permission = await requestRecordingPermissionsAsync();
97 - return { granted: permission.granted, canAskAgain: permission.canAskAgain };
117 + getTranscriber: () => {
118 + const transcriber = registry.get(localVoiceTranscriberAtom);
119 + return transcriber === "checking" ? null : transcriber;
120 },
121 + requestPermission: permission.requestPermission,
122 configureRecording: async () => {
123 const token = acquireAudioSession("recording");
124 audioToken = token;
@@ -120,14 +143,8 @@ const voiceInputAtom = Atom.make((get) => {
143 publishAudioLevels: (levels) => audioLevels.set(levels),
144 keepAwake: () => keepScreenAwake(`voice-input:${++keepAwakeSequence}`),
145 });
123 - // iOS reports `inactive` while its permission dialog is open. Only the real
124 - // background state cancels preparation; recorder status handles calls and
125 - // route interruptions during capture.
126 - const appStateSubscription = AppState.addEventListener("change", (nextState) => {
127 - if (nextState === "background") void session.controller.appMovedToBackground();
128 - });
146 get.addFinalizer(() => {
130 - appStateSubscription.remove();
147 + permission.dispose();
148 session.dispose();
149 recorder.release();
150 });
@@ -136,10 +153,11 @@ const voiceInputAtom = Atom.make((get) => {
153
154 export function useGlobalVoiceInput() {
155 const { session, audioLevels } = useAtomValue(voiceInputAtom);
156 + const isTranscriberAvailable = useAtomValue(localVoiceInputAvailableAtom);
157 const snapshot = useSyncExternalStore(session.subscribe, session.getSnapshot);
158 return {
159 ...snapshot,
142 - isAvailable: getLocalVoiceTranscriber() !== null || getNativeShowcaseScene() !== null,
160 + isAvailable: isTranscriberAvailable || getNativeShowcaseScene() !== null,
161 audioLevels,
162 isBusy: voiceInputBlocksSubmission(snapshot.state),
163 session,
apps/mobile/src/features/voice-input/lazyVoiceRecorder.test.ts
+52 -4
@@ -9,14 +9,20 @@ type State = { readonly isRecording: boolean };
9
10 function createNativeRecorder(uri: string) {
11 const listeners = new Set<(status: Status) => void>();
12 + // Like expo-audio on Android, preparing again requires a stop in between.
13 + let isPrepared = false;
14 return {
15 uri: null as string | null,
16 prepared: uri,
17 prepareToRecordAsync: vi.fn(async function (this: { uri: string | null; prepared: string }) {
18 + if (isPrepared) throw new Error("Recorder is already prepared");
19 + isPrepared = true;
20 this.uri = this.prepared;
21 }),
22 record: vi.fn(),
19 - stop: vi.fn(async (): Promise<void> => undefined),
23 + stop: vi.fn(async (): Promise<void> => {
24 + isPrepared = false;
25 + }),
26 getStatus: vi.fn((): State => ({ isRecording: true })),
27 addListener: vi.fn((_event: "recordingStatusUpdate", listener: (status: Status) => void) => {
28 listeners.add(listener);
@@ -26,7 +32,8 @@ function createNativeRecorder(uri: string) {
32 { uid: "car", name: "CarPlay", type: "CarAudio" },
33 { uid: "built-in", name: "iPhone Microphone", type: "MicrophoneBuiltIn" },
34 ]),
29 - setInput: vi.fn((_uid: string) => undefined),
35 + setInput: vi.fn((_uid: string): void | Promise<void> => undefined),
36 + cancelInputSelection: vi.fn((): void => undefined),
37 release: vi.fn(),
38 emit: (status: Status) => listeners.forEach((listener) => listener(status)),
39 };
@@ -34,12 +41,14 @@ function createNativeRecorder(uri: string) {
41
42 function createHarness(
43 selectInput?: (inputs: ReadonlyArray<VoiceRecordingInput>) => VoiceRecordingInput | null,
44 + configure?: (native: ReturnType<typeof createNativeRecorder>) => void,
45 ) {
46 const created: Array<ReturnType<typeof createNativeRecorder>> = [];
47 const statuses: Status[] = [];
48 const recorder = createLazyVoiceRecorder({
49 create: () => {
50 const native = createNativeRecorder(`file:///voice-${created.length}.m4a`);
51 + configure?.(native);
52 created.push(native);
53 return native;
54 },
@@ -57,10 +66,21 @@ describe("createLazyVoiceRecorder", () => {
66
67 const disconnected = createHarness((inputs) => inputs[1] ?? null);
68 await disconnected.recorder.prepareToRecordAsync();
69 + await disconnected.recorder.stop();
70 disconnected.created[0]!.setInput.mockImplementationOnce(() => {
71 throw new Error("Preferred input not found");
72 });
73 await expect(disconnected.recorder.prepareToRecordAsync()).resolves.toBeUndefined();
74 +
75 + const timedOut = createHarness(
76 + (inputs) => inputs[1] ?? null,
77 + (native) => {
78 + native.setInput.mockImplementationOnce(() =>
79 + Promise.reject(new Error("Routing timed out")),
80 + );
81 + },
82 + );
83 + await expect(timedOut.recorder.prepareToRecordAsync()).resolves.toBeUndefined();
84 });
85
86 it("forwards status events until the recorder is released", async () => {
@@ -76,8 +96,12 @@ describe("createLazyVoiceRecorder", () => {
96 });
97
98 describe("with the voice input controller", () => {
79 - function createController(granted: boolean) {
80 - const harness = createHarness();
99 + function createController(
100 + granted: boolean,
101 + selectInput?: (inputs: ReadonlyArray<VoiceRecordingInput>) => VoiceRecordingInput | null,
102 + configure?: (native: ReturnType<typeof createNativeRecorder>) => void,
103 + ) {
104 + const harness = createHarness(selectInput, configure);
105 const deleted: string[] = [];
106 const controller = new VoiceInputController({
107 recorder: harness.recorder,
@@ -108,5 +132,29 @@ describe("createLazyVoiceRecorder", () => {
132 expect(controller.currentState.phase).toBe("error");
133 expect(created).toHaveLength(0);
134 });
135 +
136 + it("abandons a pending input route when dictation is canceled", async () => {
137 + const routing = Promise.withResolvers<void>();
138 + const { controller, created } = createController(
139 + true,
140 + (inputs) => inputs[1] ?? null,
141 + (native) => {
142 + // Expo drops the promise of a canceled coroutine function, so it never settles.
143 + native.setInput.mockImplementationOnce(() => {
144 + routing.resolve();
145 + return new Promise(() => undefined);
146 + });
147 + },
148 + );
149 + const starting = controller.start();
150 + await routing.promise;
151 + controller.cancel();
152 + await starting;
153 + expect(created[0]!.cancelInputSelection).toHaveBeenCalled();
154 + expect(controller.currentState.phase).toBe("idle");
155 +
156 + await controller.start();
157 + expect(controller.currentState.phase).toBe("recording");
158 + });
159 });
160 });
apps/mobile/src/features/voice-input/lazyVoiceRecorder.ts
+36 -7
@@ -1,4 +1,7 @@
1 -import type { VoiceRecorder } from "@t3tools/client-runtime/voice-input";
1 +import {
2 + throwIfVoiceTranscriptionAborted,
3 + type VoiceRecorder,
4 +} from "@t3tools/client-runtime/voice-input";
5
6 type Subscription = { remove(): void };
7
@@ -19,7 +22,8 @@ export interface NativeVoiceRecorder<TStatus, TRecorderState> {
22 listener: (status: TStatus) => void,
23 ): Subscription;
24 getAvailableInputs(): ReadonlyArray<VoiceRecordingInput>;
22 - setInput(uid: string): void;
25 + setInput(uid: string): void | Promise<void>;
26 + cancelInputSelection?(): void;
27 release(): void;
28 }
29
@@ -39,6 +43,7 @@ export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
43 let retired: NativeVoiceRecorder<TStatus, TRecorderState> | null = null;
44 let retiredUri: string | null = null;
45 let pending: Promise<unknown> = Promise.resolve();
46 + let abandonInputSelection: (() => void) | null = null;
47
48 const track = <T>(promise: Promise<T>): Promise<T> => {
49 pending = Promise.allSettled([pending, promise]).then(() => undefined);
@@ -50,19 +55,42 @@ export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
55 if (recorder) return recorder.uri;
56 return retired ? retired.uri : retiredUri;
57 },
53 - prepareToRecordAsync() {
58 + prepareToRecordAsync(signal) {
59 if (!recorder) {
60 recorder = input.create();
61 subscription = recorder.addListener("recordingStatusUpdate", input.onStatus);
62 }
63 const prepared = recorder;
64 return track(
60 - prepared.prepareToRecordAsync().then(() => {
65 + prepared.prepareToRecordAsync().then(async () => {
66 + if (signal) throwIfVoiceTranscriptionAborted(signal);
67 if (!input.selectInput) return;
68 + let selected: VoiceRecordingInput | null;
69 try {
63 - const selected = input.selectInput(prepared.getAvailableInputs());
64 - if (selected) prepared.setInput(selected.uid);
65 - } catch {}
70 + selected = input.selectInput(prepared.getAvailableInputs());
71 + } catch {
72 + return;
73 + }
74 + if (!selected) return;
75 + const uid = selected.uid;
76 + // Expo never settles a canceled coroutine function's promise, so stop waiting on abort.
77 + const abandoned = Promise.withResolvers<void>();
78 + const abandon = () => {
79 + prepared.cancelInputSelection?.();
80 + abandoned.resolve();
81 + };
82 + signal?.addEventListener("abort", abandon, { once: true });
83 + abandonInputSelection = abandon;
84 + try {
85 + await Promise.race([
86 + new Promise<void>((resolve) => resolve(prepared.setInput(uid))).catch(() => {}),
87 + abandoned.promise,
88 + ]);
89 + if (signal) throwIfVoiceTranscriptionAborted(signal);
90 + } finally {
91 + signal?.removeEventListener("abort", abandon);
92 + if (abandonInputSelection === abandon) abandonInputSelection = null;
93 + }
94 }),
95 );
96 },
@@ -79,6 +107,7 @@ export function createLazyVoiceRecorder<TStatus, TRecorderState>(input: {
107 release() {
108 const released = recorder;
109 if (!released) return;
110 + abandonInputSelection?.();
111 recorder = null;
112 subscription?.remove();
113 subscription = null;
apps/mobile/src/features/voice-input/voiceRecordingPermission.test.ts new
+52
@@ -0,0 +1,52 @@
1 +import { expect, it, vi } from "vite-plus/test";
2 +
3 +const mocks = vi.hoisted(() => ({
4 + state: "active",
5 + listeners: new Set<(state: string) => void>(),
6 + request: vi.fn<() => Promise<{ granted: boolean; canAskAgain: boolean }>>(),
7 +}));
8 +
9 +vi.mock("expo-audio", () => ({ requestRecordingPermissionsAsync: mocks.request }));
10 +vi.mock("react-native", () => ({
11 + Platform: { OS: "android" },
12 + AppState: {
13 + get currentState() {
14 + return mocks.state;
15 + },
16 + addEventListener: (_event: string, listener: (state: string) => void) => {
17 + mocks.listeners.add(listener);
18 + return { remove: () => mocks.listeners.delete(listener) };
19 + },
20 + },
21 +}));
22 +
23 +import { createVoiceRecordingPermission } from "./voiceRecordingPermission";
24 +
25 +function changeAppState(state: string) {
26 + mocks.state = state;
27 + mocks.listeners.forEach((listener) => listener(state));
28 +}
29 +
30 +it("keeps dictation alive while Android's permission activity backgrounds the app", async () => {
31 + const permission = Promise.withResolvers<{ granted: boolean; canAskAgain: boolean }>();
32 + mocks.request.mockReturnValue(permission.promise);
33 + const interrupted = vi.fn();
34 + const recording = createVoiceRecordingPermission(interrupted);
35 + let settled = false;
36 + const result = recording.requestPermission().then((value) => {
37 + settled = true;
38 + return value;
39 + });
40 +
41 + changeAppState("background");
42 + permission.resolve({ granted: true, canAskAgain: true });
43 + await Promise.resolve();
44 + expect(settled).toBe(false);
45 + changeAppState("active");
46 + expect(await result).toMatchObject({ granted: true });
47 + expect(interrupted).not.toHaveBeenCalled();
48 +
49 + changeAppState("background");
50 + expect(interrupted).toHaveBeenCalledOnce();
51 + recording.dispose();
52 +});
apps/mobile/src/features/voice-input/voiceRecordingPermission.ts new
+36
@@ -0,0 +1,36 @@
1 +import { requestRecordingPermissionsAsync } from "expo-audio";
2 +import { AppState, Platform } from "react-native";
3 +
4 +export function createVoiceRecordingPermission(onBackground: () => void) {
5 + let requesting = false;
6 + let disposed = false;
7 + let resume: (() => void) | null = null;
8 + const subscription = AppState.addEventListener("change", (state) => {
9 + if (state === "active") resume?.();
10 + if (state === "background" && !requesting) onBackground();
11 + });
12 + return {
13 + requestPermission: async () => {
14 + if (Platform.OS !== "android") return requestRecordingPermissionsAsync();
15 + // Android's permission activity backgrounds the app; resume before preparing Google's model.
16 + requesting = true;
17 + try {
18 + const permission = await requestRecordingPermissionsAsync();
19 + if (!disposed && AppState.currentState !== "active") {
20 + await new Promise<void>((resolve) => {
21 + resume = resolve;
22 + });
23 + }
24 + return permission;
25 + } finally {
26 + requesting = false;
27 + resume = null;
28 + }
29 + },
30 + dispose: () => {
31 + disposed = true;
32 + subscription.remove();
33 + resume?.();
34 + },
35 + };
36 +}
apps/mobile/src/lib/microphonePriority.test.ts
+10
@@ -31,6 +31,12 @@ describe("microphoneOrder", () => {
31 "kind:bluetooth",
32 "kind:carPlay",
33 ]);
34 + expect(keys(microphoneOrder([{ kind: "carPlay" }, ...saved], "android"))).toEqual([
35 + "kind:wired",
36 + "kind:builtIn",
37 + "airpods",
38 + "kind:bluetooth",
39 + ]);
40 });
41 });
42
@@ -75,5 +81,9 @@ describe("preferredMicrophoneInput", () => {
81 expect(preferredMicrophoneInput([airPods, usb], order)).toBe(usb);
82 expect(preferredMicrophoneInput([airPods, carPlay], order)).toBe(airPods);
83 expect(preferredMicrophoneInput([airPods, iPhone], order)).toBe(iPhone);
84 + const headset = { uid: "android:7:headset", name: "Headset", type: "BluetoothSCO" };
85 + expect(preferredMicrophoneInput([headset, iPhone], microphoneOrder([], "android"))).toBe(
86 + headset,
87 + );
88 });
89 });
apps/mobile/src/lib/microphonePriority.ts
+7 -5
@@ -1,11 +1,9 @@
1 -/** iOS lists connected inputs only after recording-session setup. */
1 const MICROPHONE_KINDS = ["wired", "bluetooth", "builtIn", "carPlay"] as const;
2
3 export type MicrophoneKind = (typeof MICROPHONE_KINDS)[number];
4
5 export interface MicrophoneDevice {
6 readonly kind: MicrophoneKind;
8 - /** iOS's port identifier, stable across reconnects of the same device. */
7 readonly uid: string;
8 readonly name: string;
9 }
@@ -18,12 +16,13 @@ interface MicrophoneInput {
16 readonly type: string;
17 }
18
21 -// A car without CarPlay connects as a Bluetooth headset, so it is a Bluetooth device.
19 +// A car without CarPlay, and Android Auto, routes its microphone over Bluetooth, so it is a Bluetooth device.
20 const KIND_BY_PORT_TYPE: Readonly<Record<string, MicrophoneKind>> = {
21 MicrophoneWired: "wired",
22 USBAudio: "wired",
23 LineIn: "wired",
24 BluetoothHFP: "bluetooth",
25 + BluetoothSCO: "bluetooth",
26 BluetoothLE: "bluetooth",
27 MicrophoneBuiltIn: "builtIn",
28 CarAudio: "carPlay",
@@ -47,10 +46,13 @@ export function microphoneEntryKey(entry: MicrophoneEntry): string {
46
47 export function microphoneOrder(
48 saved: ReadonlyArray<MicrophoneEntry> = [],
49 + platform: "ios" | "android" = "ios",
50 ): ReadonlyArray<MicrophoneEntry> {
51 + const kinds = MICROPHONE_KINDS.filter((kind) => platform === "ios" || kind !== "carPlay");
52 const order: MicrophoneEntry[] = [];
53 const keys = new Set<string>();
54 for (const entry of saved) {
55 + if (!kinds.includes(entry.kind)) continue;
56 const row =
57 isMicrophoneDevice(entry) && DEVICE_KINDS.has(entry.kind) ? entry : { kind: entry.kind };
58 const key = microphoneEntryKey(row);
@@ -58,7 +60,7 @@ export function microphoneOrder(
60 keys.add(key);
61 order.push(row);
62 }
61 - for (const kind of MICROPHONE_KINDS) {
63 + for (const kind of kinds) {
64 if (keys.has(`kind:${kind}`)) continue;
65 const lastDevice = order.map((entry) => entry.kind).lastIndexOf(kind);
66 const below = order.findIndex(
@@ -95,7 +97,7 @@ export function rememberMicrophones(
97 return next ?? order;
98 }
99
98 -/** The connected input of the first entry that has one, or null to leave the route to iOS. */
100 +/** The connected input of the first entry that has one, or null for the system default. */
101 export function preferredMicrophoneInput<Input extends MicrophoneInput>(
102 inputs: ReadonlyArray<Input>,
103 order: ReadonlyArray<MicrophoneEntry>,
apps/mobile/src/lib/permissionStatusAtom.test.ts
+19
@@ -26,6 +26,10 @@ function deferredReads() {
26 pending.shift()?.(status);
27 await Promise.resolve();
28 },
29 + resolveLast: async (status: string) => {
30 + pending.pop()?.(status);
31 + await Promise.resolve();
32 + },
33 get count() {
34 return pending.length;
35 },
@@ -68,6 +72,21 @@ describe("makePermissionStatusAtom", () => {
72 expect(native.listeners.size).toBe(0);
73 });
74
75 + it("keeps the newest read when an older one settles last", async () => {
76 + const reads = deferredReads();
77 + const atom = makePermissionStatusAtom({ label: "test", initial: "checking", read: reads.read });
78 + const { registry, flush } = makeRegistry();
79 + const unmount = registry.mount(atom);
80 +
81 + for (const listener of native.listeners) listener("active");
82 + await reads.resolveLast("available");
83 + await reads.resolveNext("unavailable");
84 + expect(registry.get(atom)).toBe("available");
85 +
86 + unmount();
87 + flush();
88 + });
89 +
90 it("stores a request's outcome", () => {
91 const reads = deferredReads();
92 const atom = makePermissionStatusAtom({ label: "test", initial: "checking", read: reads.read });
apps/mobile/src/lib/permissionStatusAtom.ts
+9 -4
@@ -3,9 +3,9 @@ import { Atom } from "effect/reactivity";
3 import { subscribeAppActive } from "./appState";
4
5 /**
6 - * A system permission as the app last saw it. Reads it when first mounted and
7 - * again each time the app returns to the foreground, where the user may have
8 - * changed it in system Settings. Writing stores the outcome of a request.
6 + * A system permission or device capability as the app last saw it. Reads it
7 + * when first mounted and again each time the app returns to the foreground,
8 + * where the user may have changed it in system Settings. Writing stores the outcome of a request.
9 */
10 export function makePermissionStatusAtom<S>(options: {
11 readonly label: string;
@@ -15,8 +15,13 @@ export function makePermissionStatusAtom<S>(options: {
15 return Atom.writable(
16 (get) => {
17 // A read that settles after unmount is dropped: setSelf on a disposed lifetime is a no-op.
18 + // Only the latest read lands, so a slow earlier read cannot overwrite a newer status.
19 + let latest = 0;
20 const refresh = () => {
19 - void options.read().then((status) => get.setSelf(status));
21 + const read = ++latest;
22 + void options.read().then((status) => {
23 + if (read === latest) get.setSelf(status);
24 + });
25 };
26 refresh();
27 get.addFinalizer(subscribeAppActive(refresh));
apps/mobile/src/native/voiceTranscription.android.test.ts new
+84
@@ -0,0 +1,84 @@
1 +import { expect, it, vi } from "vite-plus/test";
2 +
3 +const mocks = vi.hoisted(() => ({
4 + checkAvailability: vi.fn<() => Promise<boolean>>(),
5 + prepare: vi.fn<(id: number) => Promise<{ locale: string; mode: number }>>(),
6 + transcribe: vi.fn<(id: number, uri: string, locale: string, mode: number) => Promise<string>>(),
7 + cancel: vi.fn<(id: number) => Promise<void>>(),
8 +}));
9 +
10 +vi.mock("expo", () => ({
11 + requireOptionalNativeModule: () => ({ isAvailable: () => true, ...mocks }),
12 +}));
13 +
14 +import {
15 + VoiceInputController,
16 + resetVoiceInputGlobalsForTests,
17 +} from "../../../../packages/client-runtime/src/voice-input/controller";
18 +
19 +import { getLocalVoiceTranscriber } from "./voiceTranscription.android";
20 +
21 +it.each([false, true])(
22 + "waits for device support before offering a transcriber (%s)",
23 + async (supported) => {
24 + vi.resetAllMocks();
25 + const probe = Promise.withResolvers<boolean>();
26 + mocks.checkAvailability.mockReturnValue(probe.promise);
27 + let settled = false;
28 + const result = Promise.resolve(getLocalVoiceTranscriber()).then((transcriber) => {
29 + settled = true;
30 + return transcriber;
31 + });
32 + await Promise.resolve();
33 + expect(settled).toBe(false);
34 + probe.resolve(supported);
35 + expect((await result) !== null).toBe(supported);
36 + },
37 +);
38 +
39 +it.each([
40 + ["preparation", "storage", /disk|storage|space/i],
41 + ["preparation", "quota", /quota|limit/i],
42 + ["transcription", "storage", /disk|storage|space/i],
43 + ["transcription", "quota", /quota|limit/i],
44 +] as const)("preserves an actionable %s error: %s", async (phase, code, expected) => {
45 + vi.resetAllMocks();
46 + resetVoiceInputGlobalsForTests();
47 + mocks.checkAvailability.mockResolvedValue(true);
48 + mocks.prepare.mockResolvedValue({ locale: "en-US", mode: 0 });
49 + mocks.transcribe.mockResolvedValue("Complete transcript.");
50 + mocks.cancel.mockResolvedValue(undefined);
51 + const rejection = Object.assign(new Error("Native SDK failure"), { code });
52 + (phase === "preparation" ? mocks.prepare : mocks.transcribe).mockRejectedValue(rejection);
53 + const transcriber = await getLocalVoiceTranscriber();
54 + const commit = vi.fn();
55 + const draft = {
56 + ownerKey: "acceptance:dictation",
57 + text: "Preserved draft.",
58 + selection: { start: 16, end: 16 },
59 + revision: 1,
60 + };
61 + const controller = new VoiceInputController({
62 + recorder: {
63 + uri: "file:///acceptance.m4a",
64 + prepareToRecordAsync: async () => {},
65 + record: () => {},
66 + stop: async () => {},
67 + },
68 + getTranscriber: () => transcriber,
69 + requestPermission: async () => ({ granted: true, canAskAgain: true }),
70 + configureRecording: async () => {},
71 + releaseRecording: async () => {},
72 + deleteRecording: () => {},
73 + readDraft: () => draft,
74 + commitDraft: commit,
75 + onStateChange: () => {},
76 + });
77 + await controller.start();
78 + if (phase === "transcription") await controller.stop();
79 +
80 + expect(controller.currentState).toMatchObject({ phase: "error", errorAction: "retry" });
81 + expect(commit).not.toHaveBeenCalled();
82 + expect(draft.text).toBe("Preserved draft.");
83 + expect(controller.currentState.error).toMatch(expected);
84 +});
apps/mobile/src/native/voiceTranscription.android.ts new
+88
@@ -0,0 +1,88 @@
1 +import { requireOptionalNativeModule } from "expo";
2 +
3 +import {
4 + VoiceTranscriptionError,
5 + throwIfVoiceTranscriptionAborted,
6 + type VoiceTranscriber,
7 +} from "@t3tools/client-runtime/voice-input";
8 +
9 +type PreparedGoogleTranscription = { readonly locale: string; readonly mode: number };
10 +
11 +type GoogleTranscriptionModule = {
12 + checkAvailability(): Promise<boolean>;
13 + prepare(id: number): Promise<PreparedGoogleTranscription>;
14 + transcribe(id: number, uri: string, locale: string, mode: number): Promise<string>;
15 + cancel(id: number): Promise<void>;
16 +};
17 +
18 +const native = requireOptionalNativeModule<GoogleTranscriptionModule>("T3VoiceTranscription");
19 +
20 +const nativeErrorMessages: Readonly<Record<string, string>> = {
21 + unavailable: "On-device voice transcription is not available for this device language.",
22 + storage: "Not enough storage for on-device dictation. Free device storage and retry.",
23 + quota: "On-device speech quota reached. Let the device rest or charge it, then retry.",
24 + "no-speech": "No speech was detected. Speak again and retry dictation.",
25 + busy: "On-device speech is busy. Wait for it to finish, then retry dictation.",
26 + "needs-update": "Update Android and its on-device speech services, then retry dictation.",
27 + "download-failed": "Speech model download failed. Check your network connection and retry.",
28 + "audio-overflow":
29 + "The speech engine could not process the complete recording. No text was added. Retry dictation.",
30 + "incomplete-audio":
31 + "Speech recognition could not process the complete recording. No text was added. Retry dictation.",
32 +};
33 +
34 +let nextOperationId = 0;
35 +
36 +export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | null> {
37 + if (!native || !(await native.checkAvailability())) return null;
38 + const module = native;
39 + return {
40 + prepare: async ({ signal }) => {
41 + const prepared = await runOperation(module, signal, "preparation-failed", (id) =>
42 + module.prepare(id),
43 + );
44 + return {
45 + locale: prepared.locale,
46 + transcribe: (uri, { signal }) =>
47 + runOperation(module, signal, "transcription-failed", (id) =>
48 + module.transcribe(id, uri, prepared.locale, prepared.mode),
49 + ),
50 + };
51 + },
52 + };
53 +}
54 +
55 +async function runOperation<T>(
56 + module: GoogleTranscriptionModule,
57 + signal: AbortSignal,
58 + fallbackCode: "preparation-failed" | "transcription-failed",
59 + operation: (id: number) => Promise<T>,
60 +): Promise<T> {
61 + throwIfVoiceTranscriptionAborted(signal);
62 + const id = ++nextOperationId;
63 + let cancellation: Promise<void> | undefined;
64 + const abort = () => {
65 + cancellation = module.cancel(id).catch(() => {});
66 + };
67 + signal.addEventListener("abort", abort, { once: true });
68 + try {
69 + const result = await operation(id);
70 + throwIfVoiceTranscriptionAborted(signal);
71 + return result;
72 + } catch (error) {
73 + throwIfVoiceTranscriptionAborted(signal);
74 + const code =
75 + typeof error === "object" && error !== null && "code" in error ? error.code : undefined;
76 + throw new VoiceTranscriptionError(
77 + code === "unavailable" || code === "unsupported-locale" ? code : fallbackCode,
78 + (typeof code === "string" ? nativeErrorMessages[code] : undefined) ??
79 + (fallbackCode === "preparation-failed"
80 + ? "Voice transcription could not prepare this language. Check Google speech services and try again."
81 + : "Voice transcription failed."),
82 + { cause: error },
83 + );
84 + } finally {
85 + signal.removeEventListener("abort", abort);
86 + await cancellation;
87 + }
88 +}
apps/mobile/src/native/voiceTranscription.ios.test.ts
+1 -1
@@ -65,7 +65,7 @@ describe("getLocalVoiceTranscriber", () => {
65 });
66 const controller = new AbortController();
67 const options = { signal: controller.signal };
68 - const prepared = await getLocalVoiceTranscriber()!.prepare(options);
68 + const prepared = await (await getLocalVoiceTranscriber())!.prepare(options);
69 const result = prepared
70 .transcribe("file:///voice.m4a", options)
71 .catch((error: unknown) => error);
apps/mobile/src/native/voiceTranscription.ios.ts
+1 -1
@@ -33,7 +33,7 @@ function getNativeErrorCode(error: unknown): string | undefined {
33 return typeof error.code === "string" ? error.code : undefined;
34 }
35
36 -export function getLocalVoiceTranscriber(): VoiceTranscriber | null {
36 +export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | null> {
37 const locale = getDeviceLocale();
38 if (!AppleTranscription.isAvailable(locale)) return null;
39 return { prepare: (options) => prepareVoiceTranscription(locale, options) };
apps/mobile/src/native/voiceTranscription.ts
+1 -1
@@ -1,5 +1,5 @@
1 import type { VoiceTranscriber } from "@t3tools/client-runtime/voice-input";
2
3 -export function getLocalVoiceTranscriber(): VoiceTranscriber | null {
3 +export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | null> {
4 return null;
5 }
docs/user/composer.md
+26 -18
@@ -177,28 +177,36 @@ Stashes containing uploaded files must be restored in their original environment
177 Those files are retained for 24 hours. After an upload expires, restore the prompt
178 and use **Attach again** or remove the missing file before sending.
179
180 -## Voice input on iPhone
181 -
182 -On supported iPhones with iOS 26 or later, use the composer's microphone to record,
183 -then confirm to transcribe. Text is inserted where your selection was when
184 -recording started, ready for you to review and edit before sending.
185 -
186 -Voice input depends on Apple's on-device speech transcription. On a device without
187 -it, the composer has no microphone button and **Settings → Microphone** says voice
188 -input isn't available.
189 -
190 -The first use may download Apple's speech model and needs a network connection.
191 -Later transcription works offline for that language. Recordings can be up to five
192 -minutes long. Canceling, leaving the screen, or an audio interruption discards the
193 -recording and preserves your existing draft. While recording, the screen stays
180 +## Voice input on mobile
181 +
182 +On supported iPhones with iOS 26 or later and Android devices with Android 12 or
183 +later, use the composer's microphone to record, then confirm to transcribe. Text is
184 +inserted where your selection was when recording started, ready for you to review
185 +and edit before sending.
186 +
187 +Voice input uses on-device speech transcription: Apple's on iOS, and compatible
188 +Google speech services on Android. When it is unavailable for the device or its
189 +language, the composer has no microphone button and **Settings → Microphone** says
190 +voice input isn't available. On Android, keep T3 Code open while recording and
191 +transcribing.
192 +
193 +The first use may download a speech model and needs a network connection.
194 +Later transcription works offline for that language. On Android, transcription
195 +processes the full recording at roughly its original duration. Recordings can be up to five
196 +minutes long. Canceling or an audio interruption discards the recording and
197 +preserves your existing draft. Navigating to another screen keeps dictation active;
198 +use the floating controls to finish or cancel. While recording, the screen stays
199 awake; it can sleep normally once recording stops.
200
196 -Voice input records from the first connected microphone in **Settings → Microphone**.
197 -The list mixes kinds (built-in, other wired, other Bluetooth, CarPlay) with wired and
198 -Bluetooth devices you have dictated with, listed by name. Drag to rank them together:
201 +Choose a preferred order in **Settings → Microphone**. Voice input requests the first
202 +connected microphone in the list. The list mixes kinds (built-in, other wired, other
203 +Bluetooth, and CarPlay on iOS) with wired and Bluetooth devices you have dictated
204 +with, listed by name. Drag to rank them together:
205 for example, put **Other Bluetooth** above your AirPods so a new headset is preferred
206 while the AirPods microphone stays low. By default, CarPlay comes after the built-in
201 -microphone. Swipe a device left to forget it.
207 +microphone on iOS. Swipe a device left to forget it. Android may use another input
208 +when the preferred microphone cannot be routed. Devices with the same name may share
209 +a saved preference, but simultaneous same-name inputs remain separately selectable.
210
211 Transcription runs on your device. T3 Code deletes the temporary audio after
212 transcription or cancellation; only the message text is sent when you submit.
packages/client-runtime/src/voice-input/controller.test.ts
+20 -1
@@ -9,7 +9,11 @@ import {
9 type VoiceInputControllerDependencies,
10 type VoiceRecorder,
11 } from "./controller.ts";
12 -import type { PreparedVoiceTranscription, VoiceTranscriber } from "./transcription.ts";
12 +import {
13 + VoiceTranscriptionError,
14 + type PreparedVoiceTranscription,
15 + type VoiceTranscriber,
16 +} from "./transcription.ts";
17
18 function deferred<T>() {
19 let resolve!: (value: T) => void;
@@ -116,6 +120,21 @@ describe("VoiceInputController", () => {
120 expect(unsupported.controller.currentState.error).toContain("not available");
121 expect(unsupported.recorder.record).not.toHaveBeenCalled();
122
123 + const reason = new VoiceTranscriptionError(
124 + "unavailable",
125 + "This device requires iOS 26 or later.",
126 + );
127 + const unavailable = createHarness({
128 + getTranscriber: () => ({
129 + prepare: async () => {
130 + throw reason;
131 + },
132 + }),
133 + });
134 + await unavailable.controller.start();
135 + expect(unavailable.controller.currentState.error).toBe(reason.message);
136 + expect(unavailable.recorder.record).not.toHaveBeenCalled();
137 +
138 const denied = createHarness({
139 requestPermission: async () => ({ granted: false, canAskAgain: false }),
140 });
packages/client-runtime/src/voice-input/controller.ts
+19 -11
@@ -1,6 +1,10 @@
1 import { replaceTextRange } from "@t3tools/shared/composerTrigger";
2
3 -import type { PreparedVoiceTranscription, VoiceTranscriber } from "./transcription.ts";
3 +import {
4 + VoiceTranscriptionError,
5 + type PreparedVoiceTranscription,
6 + type VoiceTranscriber,
7 +} from "./transcription.ts";
8
9 export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60;
10
@@ -34,7 +38,7 @@ export type VoiceRecorderStatus = {
38
39 export interface VoiceRecorder {
40 readonly uri: string | null;
37 - prepareToRecordAsync(): Promise<void>;
41 + prepareToRecordAsync(signal?: AbortSignal): Promise<void>;
42 record(options: { readonly forDuration: number }): void;
43 stop(): Promise<void>;
44 }
@@ -167,14 +171,18 @@ function preparationErrorMessage(error: unknown): string {
171 if (errorCode(error) === "unsupported-locale") {
172 return "Voice transcription is not available for this language.";
173 }
170 - return "Could not prepare voice transcription.";
174 + return error instanceof VoiceTranscriptionError
175 + ? error.message
176 + : "Could not prepare voice transcription.";
177 }
178
179 function transcriptionErrorMessage(error: unknown): string {
180 if (error instanceof Error && error.message === "voice-operation-busy") {
181 return "Voice transcription is still finishing. Try again shortly.";
182 }
177 - return "Could not transcribe this recording.";
183 + return error instanceof VoiceTranscriptionError
184 + ? error.message
185 + : "Could not transcribe this recording.";
186 }
187
188 const IDLE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
@@ -258,7 +266,7 @@ export class VoiceInputController {
266 await this.dependencies.configureRecording();
267 this.recordingConfigured = true;
268 if (!this.isCurrent(operationToken)) return;
261 - await this.dependencies.recorder.prepareToRecordAsync();
269 + await this.dependencies.recorder.prepareToRecordAsync(abortController.signal);
270 if (!this.isCurrent(operationToken)) return;
271 this.recordingUri = this.dependencies.recorder.uri;
272 this.rememberRecordingUri(this.recordingUri);
@@ -441,12 +449,7 @@ export class VoiceInputController {
449 ? { phase: "error", error, errorAction: "retry" }
450 : { phase: "idle", error: null, errorAction: null },
451 );
444 - try {
445 - await this.dependencies.recorder.stop();
446 - } catch {
447 - } finally {
448 - await this.releaseResources();
449 - }
452 + await this.releaseResources();
453 }
454
455 private async releaseResources(): Promise<void> {
@@ -454,6 +457,11 @@ export class VoiceInputController {
457 try {
458 this.rememberRecordingUri(this.dependencies.recorder.uri);
459 } catch {}
460 + if (this.recordingConfigured) {
461 + try {
462 + await this.dependencies.recorder.stop();
463 + } catch {}
464 + }
465 this.recordingUri = null;
466 for (const uri of this.ownedRecordingUris) {
467 try {
patches/expo-audio@58.0.9.patch
+362 -5
@@ -72,9 +72,10 @@ diff --git a/ios/AudioModule.swift b/ios/AudioModule.swift
72 !self.isSessionInUse else {
73 return
74 diff --git a/src/AudioModule.types.ts b/src/AudioModule.types.ts
75 +index 3194a8ebd1b91793d03a1692c563b5d2e83c2738..e7e713199f776b60093b75da782197427cf2000d 100644
76 --- a/src/AudioModule.types.ts
77 +++ b/src/AudioModule.types.ts
77 -@@ -27,6 +27,8 @@
78 +@@ -24,6 +24,8 @@ import type { AudioStream } from './AudioStream.types';
79 */
80 export declare class NativeAudioModule extends NativeModule {
81 setIsAudioActiveAsync(active: boolean): Promise<void>;
@@ -83,7 +84,7 @@ diff --git a/src/AudioModule.types.ts b/src/AudioModule.types.ts
84 setAudioModeAsync(category: Partial<AudioMode>): Promise<void>;
85 requestRecordingPermissionsAsync(): Promise<PermissionResponse>;
86 requestNotificationPermissionsAsync(): Promise<PermissionResponse>;
86 -@@ -175,8 +175,9 @@
87 +@@ -175,8 +177,9 @@ export declare class AudioPlayer extends SharedObject<AudioEvents> {
88
89 /**
90 * Pauses the player.
@@ -95,9 +96,10 @@ diff --git a/src/AudioModule.types.ts b/src/AudioModule.types.ts
96 /**
97 * Replaces the current audio source with a new one.
98 diff --git a/build/AudioModule.types.d.ts b/build/AudioModule.types.d.ts
99 +index 0d63c9b949cfae7a101902fc3bb70fcf46094f68..b590f9654bbf74b5052b71e31b7ed6d3d0050aa4 100644
100 --- a/build/AudioModule.types.d.ts
101 +++ b/build/AudioModule.types.d.ts
100 -@@ -6,6 +6,8 @@
102 +@@ -8,6 +8,8 @@ import type { AudioStream } from './AudioStream.types';
103 */
104 export declare class NativeAudioModule extends NativeModule {
105 setIsAudioActiveAsync(active: boolean): Promise<void>;
@@ -106,7 +108,7 @@ diff --git a/build/AudioModule.types.d.ts b/build/AudioModule.types.d.ts
108 setAudioModeAsync(category: Partial<AudioMode>): Promise<void>;
109 requestRecordingPermissionsAsync(): Promise<PermissionResponse>;
110 requestNotificationPermissionsAsync(): Promise<PermissionResponse>;
109 -@@ -135,8 +135,9 @@
111 +@@ -135,8 +137,9 @@ export declare class AudioPlayer extends SharedObject<AudioEvents> {
112 play(): void;
113 /**
114 * Pauses the player.
@@ -118,12 +120,367 @@ diff --git a/build/AudioModule.types.d.ts b/build/AudioModule.types.d.ts
120 * Replaces the current audio source with a new one.
121 */
122 diff --git a/android/src/main/java/expo/modules/audio/AudioModule.kt b/android/src/main/java/expo/modules/audio/AudioModule.kt
123 +index 6c81fce3dd518d3b82e3280940a001c7be84dd0a..16125c679c870e2eae48f8db953dedd3fc842c3d 100644
124 --- a/android/src/main/java/expo/modules/audio/AudioModule.kt
125 +++ b/android/src/main/java/expo/modules/audio/AudioModule.kt
123 -@@ -678,5 +678,5 @@
126 +@@ -677,7 +677,7 @@ class AudioModule : Module() {
127 + }
128 }
129
130 - Function("pause") { player: AudioPlayer ->
131 + Function("pause") { player: AudioPlayer, /* deactivateSession - iOS only */ _: Boolean? ->
132 runOnMain {
133 player.isPaused = false
134 + player.ref.pause()
135 +@@ -824,10 +824,14 @@ class AudioModule : Module() {
136 + return@Function recorder.getAvailableInputs(audioManager)
137 + }
138 +
139 +- Function("setInput") { recorder: AudioRecorder, input: String ->
140 ++ AsyncFunction("setInput") Coroutine { recorder: AudioRecorder, input: String ->
141 + recorder.setInput(input, audioManager)
142 + }
143 +
144 ++ Function("cancelInputSelection") { recorder: AudioRecorder ->
145 ++ recorder.cancelInputSelection()
146 ++ }
147 ++
148 + Function("startRecordingAtTime") { recorder: AudioRecorder, seconds: Double ->
149 + checkRecordingPermission()
150 + if (recorder.isPrepared) {
151 +diff --git a/android/src/main/java/expo/modules/audio/AudioRecorder.kt b/android/src/main/java/expo/modules/audio/AudioRecorder.kt
152 +index 2ff4a474fdea3efa652436a8a3960c3669ad298b..1ced279991294d3d81cb20fa339ca73e5d6d6c01 100644
153 +--- a/android/src/main/java/expo/modules/audio/AudioRecorder.kt
154 ++++ b/android/src/main/java/expo/modules/audio/AudioRecorder.kt
155 +@@ -19,6 +19,9 @@ import expo.modules.kotlin.exception.CodedException
156 + import expo.modules.kotlin.exception.Exceptions
157 + import expo.modules.kotlin.sharedobjects.SharedObject
158 + import kotlinx.coroutines.Job
159 ++import kotlinx.coroutines.CancellationException
160 ++import kotlinx.coroutines.currentCoroutineContext
161 ++import kotlinx.coroutines.job
162 + import kotlinx.coroutines.delay
163 + import kotlinx.coroutines.launch
164 + import java.io.File
165 +@@ -42,12 +45,19 @@ class AudioRecorder(
166 + var isPrepared = false
167 +
168 + private var recorder: MediaRecorder? = null
169 ++ private val inputRoute = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
170 ++ RecordingInputRoute(context.getSystemService(Context.AUDIO_SERVICE) as AudioManager)
171 ++ } else {
172 ++ null
173 ++ }
174 + val id = UUID.randomUUID().toString()
175 + var startTime = 0L
176 + var isRecording = false
177 + var isPaused = false
178 + private var isPausedBySystem = false
179 + private var recordingTimerJob: Job? = null
180 ++ private var inputSelectionJob: Job? = null
181 ++ private var inputSelectionCanceled = false
182 + private var durationLimitMillis: Long? = null
183 + var useForegroundService = false
184 +
185 +@@ -90,6 +100,7 @@ class AudioRecorder(
186 + throw AudioRecorderAlreadyPreparedException()
187 + }
188 +
189 ++ synchronized(this) { inputSelectionCanceled = false }
190 + val recordingOptions = options ?: this.options
191 + val mediaRecorder = createRecorder(recordingOptions)
192 + recorder = mediaRecorder
193 +@@ -245,10 +256,15 @@ class AudioRecorder(
194 + }
195 +
196 + private fun reset() {
197 ++ cancelInputSelection()
198 + recordingTimerJob?.cancel()
199 + recordingTimerJob = null
200 +
201 +- recorder?.release()
202 ++ try {
203 ++ recorder?.release()
204 ++ } finally {
205 ++ inputRoute?.close()
206 ++ }
207 + recorder = null
208 + isRecording = false
209 + isPaused = false
210 +@@ -468,55 +484,68 @@ class AudioRecorder(
211 + throw DeviceInfoNotFoundException()
212 + }
213 +
214 +- return getMapFromDeviceInfo(deviceInfo)
215 ++ return getMapFromDeviceInfo(deviceInfo, recordingInputs(audioManager))
216 + }
217 +
218 + private fun hasRecordingPermissions() =
219 + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED
220 +
221 +- fun getAvailableInputs(audioManager: AudioManager) =
222 +- audioManager.getDevices(AudioManager.GET_DEVICES_INPUTS).mapNotNull { deviceInfo ->
223 +- val type = deviceInfo.type
224 +- if (type == AudioDeviceInfo.TYPE_BUILTIN_MIC || type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO || type == AudioDeviceInfo.TYPE_WIRED_HEADSET) {
225 +- getMapFromDeviceInfo(deviceInfo)
226 +- } else {
227 +- null
228 +- }
229 ++ fun getAvailableInputs(audioManager: AudioManager): List<Bundle> {
230 ++ val inputs = recordingInputs(audioManager)
231 ++ return inputs.map { getMapFromDeviceInfo(it, inputs) }
232 ++ }
233 ++
234 ++ suspend fun setInput(uid: String, audioManager: AudioManager) {
235 ++ val job = currentCoroutineContext().job
236 ++ synchronized(this) {
237 ++ if (inputSelectionCanceled) throw CancellationException("Input selection canceled")
238 ++ inputSelectionJob = job
239 ++ }
240 ++ try {
241 ++ setInputDevice(uid, audioManager)
242 ++ } finally {
243 ++ synchronized(this) { if (inputSelectionJob === job) inputSelectionJob = null }
244 + }
245 ++ }
246 ++
247 ++ fun cancelInputSelection() {
248 ++ val job = synchronized(this) {
249 ++ inputSelectionCanceled = true
250 ++ inputSelectionJob
251 ++ }
252 ++ job?.cancel()
253 ++ }
254 +
255 + @Suppress("DEPRECATION")
256 +- fun setInput(uid: String, audioManager: AudioManager) {
257 +- val deviceInfo: AudioDeviceInfo? = getDeviceInfoFromUid(uid, audioManager)
258 ++ private suspend fun setInputDevice(uid: String, audioManager: AudioManager) {
259 ++ val deviceInfo = getDeviceInfoFromUid(uid, audioManager) ?: throw PreferredInputNotFoundException()
260 +
261 + if (Build.VERSION.SDK_INT <= Build.VERSION_CODES.P) {
262 + throw SetAudioInputNotSupportedException()
263 + }
264 +
265 +- if (deviceInfo != null && deviceInfo.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO) {
266 +- if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
267 +- audioManager.setCommunicationDevice(deviceInfo)
268 +- } else {
269 ++ try {
270 ++ if (inputRoute != null) {
271 ++ inputRoute.select(deviceInfo)
272 ++ } else if (deviceInfo.type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO) {
273 + audioManager.startBluetoothSco()
274 +- }
275 +- } else {
276 +- if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
277 +- audioManager.clearCommunicationDevice()
278 + } else {
279 + audioManager.stopBluetoothSco()
280 + }
281 +- }
282 +-
283 +- val success = recorder?.setPreferredDevice(deviceInfo)
284 +- if (success == false) {
285 +- throw PreferredInputNotFoundException()
286 ++ val source = getDeviceInfoFromUid(uid, audioManager)?.takeIf { it.isSource }
287 ++ if (recorder?.setPreferredDevice(source) != true) {
288 ++ throw PreferredInputNotFoundException()
289 ++ }
290 ++ } catch (error: Exception) {
291 ++ inputRoute?.close()
292 ++ throw error
293 + }
294 + }
295 +
296 + private fun getDeviceInfoFromUid(uid: String, audioManager: AudioManager): AudioDeviceInfo? {
297 +- val id = uid.toInt()
298 +- val audioDevices: Array<AudioDeviceInfo> = audioManager.getDevices(AudioManager.GET_DEVICES_INPUTS)
299 ++ val audioDevices = recordingInputs(audioManager)
300 + for (device in audioDevices) {
301 +- if (device.id == id) {
302 ++ if (recordingInputUid(device, audioDevices) == uid) {
303 + return device
304 + }
305 + }
306 +diff --git a/android/src/main/java/expo/modules/audio/AudioUtils.kt b/android/src/main/java/expo/modules/audio/AudioUtils.kt
307 +index 81819c1d771bdeec093bb35a65a5e1e1af0cf9d7..cd0c7624b4cd8a4ebf7ed3b3390cfc76802e8530 100644
308 +--- a/android/src/main/java/expo/modules/audio/AudioUtils.kt
309 ++++ b/android/src/main/java/expo/modules/audio/AudioUtils.kt
310 +@@ -2,6 +2,8 @@ package expo.modules.audio
311 +
312 + import android.content.Context
313 + import android.media.AudioDeviceInfo
314 ++import android.media.AudioManager
315 ++import android.os.Build
316 + import android.os.Bundle
317 + import androidx.annotation.OptIn
318 + import androidx.media3.common.Player
319 +@@ -17,22 +19,57 @@ fun ensureDirExists(dir: File): File {
320 + return dir
321 + }
322 +
323 +-fun getMapFromDeviceInfo(deviceInfo: AudioDeviceInfo): Bundle {
324 ++fun getMapFromDeviceInfo(deviceInfo: AudioDeviceInfo, inputs: List<AudioDeviceInfo>): Bundle {
325 + val map = Bundle()
326 +- val type = when (deviceInfo.type) {
327 +- AudioDeviceInfo.TYPE_BUILTIN_MIC -> "MicrophoneBuiltIn"
328 +- AudioDeviceInfo.TYPE_BLUETOOTH_SCO -> "BluetoothSCO"
329 +- AudioDeviceInfo.TYPE_BLUETOOTH_A2DP -> "BluetoothA2DP"
330 +- AudioDeviceInfo.TYPE_TELEPHONY -> "Telephony"
331 +- AudioDeviceInfo.TYPE_WIRED_HEADSET -> "MicrophoneWired"
332 +- else -> "Unknown device type"
333 +- }
334 ++ val type = recordingInputType(deviceInfo.type) ?: "Unknown device type"
335 + map.putString("name", deviceInfo.productName.toString())
336 + map.putString("type", type)
337 +- map.putString("uid", deviceInfo.id.toString())
338 ++ map.putString("uid", recordingInputUid(deviceInfo, inputs))
339 + return map
340 + }
341 +
342 ++internal fun recordingInputs(manager: AudioManager): List<AudioDeviceInfo> {
343 ++ val inputs = manager.getDevices(AudioManager.GET_DEVICES_INPUTS).filter {
344 ++ recordingInputType(it.type) != null
345 ++ }
346 ++ val bluetooth = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
347 ++ manager.availableCommunicationDevices.filter { isBluetoothInput(it.type) }
348 ++ } else {
349 ++ emptyList()
350 ++ }
351 ++ // Some Bluetooth microphone ports appear only after the communication route is activated.
352 ++ return (inputs + bluetooth).distinctBy {
353 ++ if (isBluetoothInput(it.type)) recordingInputUid(it) else "port:${it.id}"
354 ++ }
355 ++}
356 ++
357 ++internal fun isBluetoothInput(type: Int) = type == AudioDeviceInfo.TYPE_BLUETOOTH_SCO ||
358 ++ type == AudioDeviceInfo.TYPE_BLE_HEADSET
359 ++
360 ++internal fun recordingInputType(type: Int): String? = when (type) {
361 ++ AudioDeviceInfo.TYPE_BUILTIN_MIC -> "MicrophoneBuiltIn"
362 ++ AudioDeviceInfo.TYPE_BLUETOOTH_SCO -> "BluetoothSCO"
363 ++ AudioDeviceInfo.TYPE_BLE_HEADSET -> "BluetoothLE"
364 ++ AudioDeviceInfo.TYPE_WIRED_HEADSET -> "MicrophoneWired"
365 ++ AudioDeviceInfo.TYPE_USB_DEVICE, AudioDeviceInfo.TYPE_USB_HEADSET -> "USBAudio"
366 ++ AudioDeviceInfo.TYPE_LINE_ANALOG, AudioDeviceInfo.TYPE_LINE_DIGITAL -> "LineIn"
367 ++ else -> null
368 ++}
369 ++
370 ++internal fun recordingInputUid(device: AudioDeviceInfo, inputs: List<AudioDeviceInfo> = emptyList()): String {
371 ++ val base = recordingInputBaseUid(device)
372 ++ val ambiguous = !isBluetoothInput(device.type) && inputs.any {
373 ++ it.id != device.id && recordingInputBaseUid(it) == base
374 ++ }
375 ++ return if (ambiguous) "$base:port:${device.id}" else base
376 ++}
377 ++
378 ++private fun recordingInputBaseUid(device: AudioDeviceInfo): String {
379 ++ // Port IDs and USB addresses (ALSA card numbers) change on reconnect; Bluetooth addresses don't.
380 ++ val stable = Build.VERSION.SDK_INT >= Build.VERSION_CODES.P && isBluetoothInput(device.type)
381 ++ return "android:${device.type}:${if (stable) device.address.ifBlank { device.productName } else device.productName}"
382 ++}
383 ++
384 + @OptIn(UnstableApi::class)
385 + fun buildBasicMediaSession(context: Context, player: Player, onPlay: () -> Unit): MediaSession {
386 + return MediaSession.Builder(context, PlayRequestingPlayer(player, onPlay))
387 +diff --git a/android/src/main/java/expo/modules/audio/RecordingInputRoute.kt b/android/src/main/java/expo/modules/audio/RecordingInputRoute.kt
388 +new file mode 100644
389 +index 0000000000000000000000000000000000000000..cdba9724318cf06afd21b4df6b895b381afa8e6f
390 +--- /dev/null
391 ++++ b/android/src/main/java/expo/modules/audio/RecordingInputRoute.kt
392 +@@ -0,0 +1,53 @@
393 ++package expo.modules.audio
394 ++
395 ++import android.media.AudioDeviceInfo
396 ++import android.media.AudioManager
397 ++import androidx.annotation.RequiresApi
398 ++import java.io.Closeable
399 ++import java.util.concurrent.Executor
400 ++import kotlinx.coroutines.CompletableDeferred
401 ++import kotlinx.coroutines.withTimeout
402 ++
403 ++@RequiresApi(31)
404 ++internal class RecordingInputRoute(private val manager: AudioManager) : Closeable {
405 ++ private var previousMode: Int? = null
406 ++
407 ++ suspend fun select(input: AudioDeviceInfo) {
408 ++ if (!isBluetoothInput(input.type)) {
409 ++ close()
410 ++ return
411 ++ }
412 ++ // Communication routing takes the headset's output; Android pairs its microphone.
413 ++ val output = manager.availableCommunicationDevices.firstOrNull {
414 ++ it.isSink && it.type == input.type &&
415 ++ if (input.address.isNotBlank()) it.address == input.address else it.productName == input.productName
416 ++ } ?: throw PreferredInputNotFoundException()
417 ++ val ready = CompletableDeferred<Unit>()
418 ++ val listener = AudioManager.OnCommunicationDeviceChangedListener {
419 ++ if (it?.id == output.id) ready.complete(Unit)
420 ++ }
421 ++ manager.addOnCommunicationDeviceChangedListener(Executor { it.run() }, listener)
422 ++ try {
423 ++ if (previousMode == null) previousMode = manager.mode
424 ++ manager.mode = AudioManager.MODE_IN_COMMUNICATION
425 ++ if (!manager.setCommunicationDevice(output)) throw PreferredInputNotFoundException()
426 ++ if (manager.communicationDevice?.id == output.id) ready.complete(Unit)
427 ++ withTimeout(5_000) { ready.await() }
428 ++ } catch (error: Exception) {
429 ++ close()
430 ++ throw error
431 ++ } finally {
432 ++ manager.removeOnCommunicationDeviceChangedListener(listener)
433 ++ }
434 ++ }
435 ++
436 ++ override fun close() {
437 ++ val mode = previousMode ?: return
438 ++ previousMode = null
439 ++ try {
440 ++ manager.clearCommunicationDevice()
441 ++ } finally {
442 ++ if (manager.mode == AudioManager.MODE_IN_COMMUNICATION) manager.mode = mode
443 ++ }
444 ++ }
445 ++}
446 +diff --git a/android/src/test/java/expo/modules/audio/RecordingInputIdentityTest.kt b/android/src/test/java/expo/modules/audio/RecordingInputIdentityTest.kt
447 +new file mode 100644
448 +index 0000000000000000000000000000000000000000..4dd0567e2170770e9b0f6a17dc4581bc769d5c6a
449 +--- /dev/null
450 ++++ b/android/src/test/java/expo/modules/audio/RecordingInputIdentityTest.kt
451 +@@ -0,0 +1,35 @@
452 ++package expo.modules.audio
453 ++
454 ++import android.media.AudioDeviceInfo
455 ++import android.media.AudioManager
456 ++import io.mockk.every
457 ++import io.mockk.mockk
458 ++import org.junit.Assert.*
459 ++import org.junit.Test
460 ++
461 ++class RecordingInputIdentityTest {
462 ++ private fun device(id: Int) = mockk<AudioDeviceInfo> {
463 ++ every { this@mockk.id } returns id
464 ++ every { type } returns AudioDeviceInfo.TYPE_USB_DEVICE
465 ++ every { productName } returns "USB Microphone"
466 ++ every { address } returns "card=$id;device=0;"
467 ++ }
468 ++
469 ++ @Test
470 ++ fun identicalUsbMicrophonesRemainSelectable() {
471 ++ val first = device(1)
472 ++ val second = device(2)
473 ++ val manager = mockk<AudioManager> {
474 ++ every { getDevices(AudioManager.GET_DEVICES_INPUTS) } returns arrayOf(first, second)
475 ++ }
476 ++ val inputs = recordingInputs(manager)
477 ++ assertEquals(2, inputs.size)
478 ++ val firstUid = recordingInputUid(first, inputs)
479 ++ val secondUid = recordingInputUid(second, inputs)
480 ++ assertNotEquals(firstUid, secondUid)
481 ++ assertSame(first, inputs.single { recordingInputUid(it, inputs) == firstUid })
482 ++ assertSame(second, inputs.single { recordingInputUid(it, inputs) == secondUid })
483 ++ val reconnected = device(99)
484 ++ assertEquals(recordingInputUid(first, listOf(first)), recordingInputUid(reconnected, listOf(reconnected)))
485 ++ }
486 ++}
pnpm-lock.yaml
+3 -3
@@ -378,7 +378,7 @@ patchedDependencies:
378 '@react-navigation/native-stack@7.21.0': 990d575444c4af2ce8218b3b7a8280fe244c03abbdb32b822a2e862f791dbf42
379 app-builder-lib@26.16.1: 0fbe9ae51783f5768ebe197d12d1fdb7d477a08c92e5db2f6356bcce62613c53
380 dbus-next@0.10.2: cfff57561b0ee59b5addb3b2e6c6f20906e967507a530ab67e8db8108e520ba4
381 - expo-audio@58.0.9: 530af071088a809fe1a97ce2b2269c78335fbb5dcd3893d6937ff5c304c85a0a
381 + expo-audio@58.0.9: b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba
382 expo-modules-core@58.0.22: f0b0fd100550ed76e5c25b4d13eb7c9ff62825f8f197c53edf7bf447c50c9fb7
383 expo-network@58.0.3: cd09b6d1744dde4efa081b5619ebcff38877d98f7502a51cab121629dece8237
384 expo-sharing@58.0.19: 321a86e1471e34c53f1f2937fbb4304ea3f7b7a78ad9e4d35febc2b9e73e9688
@@ -627,7 +627,7 @@ importers:
627 version: 58.0.16(@tsc-rs/typescript@0.8.0)(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)
628 expo-audio:
629 specifier: 58.0.9
630 - version: 58.0.9(patch_hash=530af071088a809fe1a97ce2b2269c78335fbb5dcd3893d6937ff5c304c85a0a)(expo-asset@58.0.16(@tsc-rs/typescript@0.8.0)(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
630 + version: 58.0.9(patch_hash=b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba)(expo-asset@58.0.16(@tsc-rs/typescript@0.8.0)(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
631 expo-auth-session:
632 specifier: ~58.0.13
633 version: 58.0.13(@tsc-rs/typescript@0.8.0)(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)
@@ -18508,7 +18508,7 @@ snapshots:
18508 - supports-color
18509 - typescript
18510
18511 - expo-audio@58.0.9(patch_hash=530af071088a809fe1a97ce2b2269c78335fbb5dcd3893d6937ff5c304c85a0a)(expo-asset@58.0.16(@tsc-rs/typescript@0.8.0)(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18511 + expo-audio@58.0.9(patch_hash=b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba)(expo-asset@58.0.16(@tsc-rs/typescript@0.8.0)(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18512 dependencies:
18513 expo: 58.0.14(277a83826bb26ba15256633fefeb0bc2)
18514 expo-asset: 58.0.16(@tsc-rs/typescript@0.8.0)(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)