The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 22, 04:30 UTC
fix(mobile): transcribe Android audio during recording (#31300)
Alex committed
Nov 5, 2026 at 12:18 UTC
66633fb6826c42ce2c08913fc99ab4adaae7648c
19 files changed
+1004
-15
apps/mobile/modules/t3-voice-transcription/android/build.gradle
+2
@@ -16,9 +16,11 @@ android {
16
17
dependencies {
18
implementation project(':expo-modules-core')
19
+ implementation project(':expo-audio')
20
implementation 'androidx.media3:media3-common:1.9.1'
21
implementation 'com.google.mlkit:genai-speech-recognition:1.0.0-alpha2'
22
testImplementation 'junit:junit:4.13.2'
23
+ testImplementation 'com.facebook.react:react-android'
24
testImplementation 'org.robolectric:robolectric:4.16.1'
25
testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2'
26
}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt
+11
@@ -158,5 +158,16 @@ internal class MlKitSpeechClient(context: Context, locale: Locale, mode: Int) :
158
}
159
}
160
161
+ suspend fun transcribeLive(capture: LivePcmCapture, pipe: PcmPipe): String = pipe.use {
162
+ val request = speechRecognizerRequest { audioSource = pipe.source }
163
+ try {
164
+ collectGoogleVoiceTranscript(
165
+ livePcmResponses(capture, pipe) { recognizer.startRecognition(request) }
166
+ )
167
+ } finally {
168
+ withContext(NonCancellable) { runCatching { recognizer.stopRecognition() } }
169
+ }
170
+ }
171
+
172
override fun close() = recognizer.close()
173
}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LivePcmCapture.kt
new
+216
@@ -0,0 +1,216 @@
1
+package expo.modules.t3voicetranscription
2
+
3
+import java.io.File
4
+import kotlin.math.abs
5
+import kotlin.math.log10
6
+import kotlin.time.TimeSource
7
+import kotlinx.coroutines.CancellationException
8
+import kotlinx.coroutines.CompletableDeferred
9
+import kotlinx.coroutines.CoroutineScope
10
+import kotlinx.coroutines.Job
11
+import kotlinx.coroutines.NonCancellable
12
+import kotlinx.coroutines.TimeoutCancellationException
13
+import kotlinx.coroutines.currentCoroutineContext
14
+import kotlinx.coroutines.coroutineScope
15
+import kotlinx.coroutines.channels.Channel
16
+import kotlinx.coroutines.delay
17
+import kotlinx.coroutines.ensureActive
18
+import kotlinx.coroutines.launch
19
+import kotlinx.coroutines.withContext
20
+import kotlinx.coroutines.withTimeout
21
+
22
+internal interface PcmCaptureSource {
23
+ val capacityBytes: Int
24
+ fun start()
25
+ suspend fun read(buffer: ByteArray, blocking: Boolean): Int
26
+ fun stop()
27
+ fun release()
28
+}
29
+
30
+internal class LivePcmCapture(
31
+ private val source: PcmCaptureSource,
32
+ val file: File,
33
+ private val output: PcmOutput,
34
+ private val clock: TimeSource = TimeSource.Monotonic
35
+) {
36
+ @Volatile private var worker: Job? = null
37
+ private val completion = CompletableDeferred<Result<Unit>>()
38
+ private val microphoneStopped = CompletableDeferred<Result<Unit>>()
39
+
40
+ @Volatile private var stopping = false
41
+ private var sourceStopped = false
42
+ private var sourceReleased = false
43
+
44
+ @Volatile var bytes = 0L
45
+ private set
46
+
47
+ @Volatile var metering = -160.0
48
+ private set
49
+ val isFinished get() = completion.isCompleted
50
+
51
+ @Volatile var inputExhausted = false
52
+ private set
53
+ val hasStarted get() = worker != null
54
+ val isRecording get() = hasStarted && !microphoneStopped.isCompleted
55
+
56
+ fun start(scope: CoroutineScope, seconds: Double, onFinished: (Throwable?) -> Unit) {
57
+ check(worker == null && !sourceReleased) { "The microphone is not prepared." }
58
+ worker = scope.launch {
59
+ var result = runCatching {
60
+ source.start()
61
+ writeCapture(seconds, onFinished)
62
+ requireCompleteAudio(bytes > 0 && file.length() == bytes) {
63
+ "The microphone did not finish its captured audio."
64
+ }
65
+ }
66
+ result = finishSource(result)
67
+ inputExhausted = result.isSuccess
68
+ result = runCatching { output.finish() }.fold(
69
+ onSuccess = { result },
70
+ onFailure = { failure -> mergeFailure(result, failure) }
71
+ )
72
+ if (microphoneStopped.complete(result)) onFinished(result.exceptionOrNull())
73
+ completion.complete(result)
74
+ }.also { job ->
75
+ job.invokeOnCompletion { error ->
76
+ if (!completion.isCompleted) {
77
+ val result =
78
+ finishSource(Result.failure(error ?: CancellationException("Capture stopped.")))
79
+ completion.complete(result)
80
+ microphoneStopped.complete(result)
81
+ runCatching { output.finish() }
82
+ }
83
+ }
84
+ }
85
+ }
86
+
87
+ private suspend fun writeCapture(
88
+ seconds: Double,
89
+ onFinished: (Throwable?) -> Unit
90
+ ) = coroutineScope {
91
+ val started = clock.markNow()
92
+ val chunks = Channel<ByteArray>(100)
93
+ var sent = 0L
94
+ val sender = launch {
95
+ var deadline = 0L
96
+ for (chunk in chunks) {
97
+ deadline = maxOf(
98
+ deadline + (chunk.size * 1000L + 31999) / 32000,
99
+ started.elapsedNow().inWholeMilliseconds
100
+ )
101
+ delay((deadline - started.elapsedNow().inWholeMilliseconds).coerceAtLeast(0))
102
+ output.write(chunk, chunk.size)
103
+ deadline = started.elapsedNow().inWholeMilliseconds
104
+ sent += chunk.size
105
+ }
106
+ }
107
+ val result = finishSource(runCatching { captureChunks(seconds, chunks) })
108
+ microphoneStopped.complete(result)
109
+ if (result.exceptionOrNull() !is CancellationException) onFinished(result.exceptionOrNull())
110
+ chunks.close()
111
+ result.getOrThrow()
112
+ sender.join()
113
+ requireCompleteAudio(sent == bytes) { "Speech recognition did not receive all captured audio." }
114
+ }
115
+
116
+ private suspend fun captureChunks(seconds: Double, chunks: Channel<ByteArray>) {
117
+ val buffer = ByteArray(3200)
118
+ var drained = 0
119
+ file.outputStream().use { stream ->
120
+ while (true) {
121
+ currentCoroutineContext().ensureActive()
122
+ val draining = stopping || bytes >= seconds * 32000
123
+ val count = source.read(buffer, !draining)
124
+ requireCompleteAudio(count in 0..buffer.size && count % 2 == 0) {
125
+ "The microphone returned incomplete audio."
126
+ }
127
+ if (count == 0) {
128
+ requireCompleteAudio(draining) { "The microphone stopped supplying audio." }
129
+ break
130
+ }
131
+ if (draining) {
132
+ drained += count
133
+ requireCompleteAudio(drained <= source.capacityBytes + buffer.size) {
134
+ "The microphone audio queue did not drain."
135
+ }
136
+ }
137
+ stream.write(buffer, 0, count)
138
+ // Keep draining the microphone while inference starts or applies backpressure.
139
+ if (!chunks.trySend(buffer.copyOf(count)).isSuccess) {
140
+ throw VoiceTranscriptionException(
141
+ "audio-overflow",
142
+ "Speech recognition fell behind capture."
143
+ )
144
+ }
145
+ bytes += count
146
+ var peak = 0
147
+ for (i in 0 until count step 2) {
148
+ val sample = ((buffer[i].toInt() and 255) or (buffer[i + 1].toInt() shl 8)).toShort()
149
+ peak = maxOf(peak, abs(sample.toInt()))
150
+ }
151
+ metering = if (peak == 0) -160.0 else 20 * log10(peak / 32768.0)
152
+ }
153
+ }
154
+ }
155
+
156
+ suspend fun stop() {
157
+ stopping = true
158
+ if (worker == null) {
159
+ finishSource(Result.success(Unit)).getOrThrow()
160
+ output.finish()
161
+ return
162
+ }
163
+ try {
164
+ withTimeout(2000) { microphoneStopped.await().getOrThrow() }
165
+ } catch (error: TimeoutCancellationException) {
166
+ cancel()
167
+ throw VoiceTranscriptionException("incomplete-audio", "The microphone did not stop.", error)
168
+ }
169
+ }
170
+
171
+ suspend fun awaitFinished() = completion.await().getOrThrow()
172
+
173
+ fun abort() {
174
+ stopping = true
175
+ worker?.cancel()
176
+ runCatching { stopSource() }
177
+ runCatching { output.finish() }
178
+ if (worker == null) {
179
+ val result = finishSource(Result.failure(CancellationException("Capture was cancelled.")))
180
+ completion.complete(result)
181
+ microphoneStopped.complete(result)
182
+ }
183
+ }
184
+
185
+ suspend fun cancel() = withContext(NonCancellable) {
186
+ abort()
187
+ worker?.join()
188
+ finishSource(Result.success(Unit))
189
+ }
190
+
191
+ @Synchronized private fun stopSource() {
192
+ if (sourceStopped) return
193
+ sourceStopped = true
194
+ source.stop()
195
+ }
196
+
197
+ @Synchronized private fun finishSource(result: Result<Unit>): Result<Unit> {
198
+ if (sourceReleased) return result
199
+ sourceReleased = true
200
+ var outcome = runCatching { stopSource() }.fold(
201
+ onSuccess = { result },
202
+ onFailure = { mergeFailure(result, it) }
203
+ )
204
+ outcome = runCatching { source.release() }.fold(
205
+ onSuccess = { outcome },
206
+ onFailure = { mergeFailure(outcome, it) }
207
+ )
208
+ return outcome
209
+ }
210
+}
211
+
212
+private fun mergeFailure(result: Result<Unit>, failure: Throwable): Result<Unit> {
213
+ val original = result.exceptionOrNull() ?: return Result.failure(failure)
214
+ if (original !== failure) original.addSuppressed(failure)
215
+ return result
216
+}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LiveVoiceRecorder.kt
new
+198
@@ -0,0 +1,198 @@
1
+package expo.modules.t3voicetranscription
2
+
3
+import android.content.Context
4
+import android.media.AudioFormat
5
+import android.media.AudioManager
6
+import android.media.AudioRecord
7
+import android.media.MediaRecorder
8
+import android.net.Uri
9
+import androidx.annotation.RequiresApi
10
+import expo.modules.audio.RecordingInputRoute
11
+import expo.modules.audio.getMapFromDeviceInfo
12
+import expo.modules.audio.recordingInputs
13
+import expo.modules.audio.recordingInputUid
14
+import expo.modules.kotlin.AppContext
15
+import expo.modules.kotlin.sharedobjects.SharedObject
16
+import java.io.File
17
+import java.util.UUID
18
+import kotlinx.coroutines.CoroutineScope
19
+import kotlinx.coroutines.Job
20
+import kotlinx.coroutines.currentCoroutineContext
21
+import kotlinx.coroutines.job
22
+
23
+@RequiresApi(31)
24
+internal class LiveVoiceRecorder(
25
+ private val context: Context,
26
+ appContext: AppContext,
27
+ private val scope: CoroutineScope,
28
+ private val onRelease: (String) -> Unit
29
+) : SharedObject(appContext) {
30
+ val id = UUID.randomUUID().toString()
31
+ private val manager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
32
+ private val route = RecordingInputRoute(manager)
33
+ private var audio: AudioRecord? = null
34
+ private var pipe: PcmPipe? = null
35
+ private var capture: LivePcmCapture? = null
36
+ private var inputSelection: Job? = null
37
+ private var inputCanceled = false
38
+ private var inferenceStarted = false
39
+ val uri: String? get() = capture?.file?.let { Uri.fromFile(it).toString() }
40
+
41
+ @Suppress("TooGenericExceptionCaught") // Release native handles for every setup failure.
42
+ fun prepare() {
43
+ check(capture?.isRecording != true) { "Another voice recording is active." }
44
+ capture?.abort()
45
+ pipe?.close()
46
+ inputCanceled = false
47
+ inferenceStarted = false
48
+ val capacity = maxOf(
49
+ 32000,
50
+ AudioRecord.getMinBufferSize(
51
+ 16000,
52
+ AudioFormat.CHANNEL_IN_MONO,
53
+ AudioFormat.ENCODING_PCM_16BIT
54
+ )
55
+ )
56
+ val recorder = AudioRecord.Builder()
57
+ .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION)
58
+ .setAudioFormat(
59
+ AudioFormat.Builder().setSampleRate(16000)
60
+ .setChannelMask(AudioFormat.CHANNEL_IN_MONO)
61
+ .setEncoding(AudioFormat.ENCODING_PCM_16BIT).build()
62
+ )
63
+ .setBufferSizeInBytes(capacity)
64
+ .build()
65
+ try {
66
+ check(recorder.state == AudioRecord.STATE_INITIALIZED) { "The microphone could not prepare." }
67
+ val output = PcmPipe.open()
68
+ try {
69
+ val file = File.createTempFile("dictation-live-", ".pcm", context.cacheDir)
70
+ capture = LivePcmCapture(
71
+ object : PcmCaptureSource {
72
+ override val capacityBytes = capacity
73
+ override fun start() {
74
+ recorder.startRecording()
75
+ check(recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
76
+ "The microphone could not start."
77
+ }
78
+ }
79
+ override suspend fun read(buffer: ByteArray, blocking: Boolean) = recorder.read(
80
+ buffer,
81
+ 0,
82
+ buffer.size,
83
+ if (blocking) AudioRecord.READ_BLOCKING else AudioRecord.READ_NON_BLOCKING
84
+ )
85
+ override fun stop() {
86
+ if (recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) recorder.stop()
87
+ }
88
+ override fun release() {
89
+ try {
90
+ recorder.release()
91
+ } finally {
92
+ route.close()
93
+ }
94
+ }
95
+ },
96
+ file,
97
+ output
98
+ )
99
+ audio = recorder
100
+ pipe = output
101
+ } catch (error: Throwable) {
102
+ output.close()
103
+ throw error
104
+ }
105
+ } catch (error: Throwable) {
106
+ recorder.release()
107
+ throw error
108
+ }
109
+ }
110
+
111
+ fun record(seconds: Double) {
112
+ val recording = requireNotNull(capture) { "The microphone is not prepared." }
113
+ recording.start(scope, seconds) { error ->
114
+ emit(
115
+ "recordingStatusUpdate",
116
+ mapOf(
117
+ "id" to id,
118
+ "isFinished" to true,
119
+ "hasError" to (error != null),
120
+ "error" to error?.message,
121
+ "url" to Uri.fromFile(recording.file).toString()
122
+ )
123
+ )
124
+ }
125
+ }
126
+
127
+ suspend fun stop() {
128
+ val recording = capture
129
+ val output = pipe
130
+ try {
131
+ val selection = synchronized(this) { inputSelection }
132
+ selection?.cancel()
133
+ selection?.join()
134
+ recording?.stop()
135
+ } finally {
136
+ // Recognition can start after Stop and still needs the captured PCM pipe.
137
+ if (recording?.hasStarted != true) output?.close()
138
+ }
139
+ }
140
+
141
+ fun status() = mapOf(
142
+ "canRecord" to (capture != null),
143
+ "isRecording" to (capture?.isRecording == true),
144
+ "durationMillis" to (capture?.bytes?.div(32) ?: 0),
145
+ "fileSize" to (capture?.file?.length() ?: 0),
146
+ "metering" to (capture?.metering ?: -160.0),
147
+ "mediaServicesDidReset" to false,
148
+ "url" to uri
149
+ )
150
+
151
+ fun inputs() = recordingInputs(manager).let { inputs ->
152
+ inputs.map { getMapFromDeviceInfo(it, inputs) }
153
+ }
154
+
155
+ // Release the preferred route on cancellation as well as selection errors.
156
+ @Suppress("TooGenericExceptionCaught")
157
+ suspend fun setInput(uid: String) {
158
+ val job = currentCoroutineContext().job
159
+ synchronized(this) {
160
+ check(!inputCanceled) { "Microphone selection was cancelled." }
161
+ inputSelection = job
162
+ }
163
+ try {
164
+ val inputs = recordingInputs(manager)
165
+ val selected = inputs.firstOrNull { recordingInputUid(it, inputs) == uid }
166
+ ?: error("The preferred microphone is disconnected.")
167
+ route.select(selected)
168
+ val routed = recordingInputs(manager)
169
+ val source = routed.firstOrNull { it.isSource && recordingInputUid(it, routed) == uid }
170
+ ?: error("The preferred microphone is disconnected.")
171
+ check(audio?.setPreferredDevice(source) == true) { "Could not select the microphone." }
172
+ } catch (error: Throwable) {
173
+ route.close()
174
+ throw error
175
+ } finally {
176
+ synchronized(this) { if (inputSelection === job) inputSelection = null }
177
+ }
178
+ }
179
+
180
+ @Synchronized fun cancelInputSelection() {
181
+ inputCanceled = true
182
+ inputSelection?.cancel()
183
+ }
184
+
185
+ suspend fun transcribe(uri: String, client: MlKitSpeechClient): String {
186
+ check(this.uri == uri && !inferenceStarted) { "The voice recording is no longer available." }
187
+ inferenceStarted = true
188
+ return client.transcribeLive(requireNotNull(capture), requireNotNull(pipe))
189
+ }
190
+
191
+ override fun sharedObjectDidRelease() {
192
+ super.sharedObjectDidRelease()
193
+ cancelInputSelection()
194
+ capture?.abort()
195
+ pipe?.close()
196
+ onRelease(id)
197
+ }
198
+}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt
+25
@@ -80,6 +80,31 @@ internal fun pacedRecordedResponses(
80
}
81
}
82
83
+internal fun livePcmResponses(
84
+ capture: LivePcmCapture,
85
+ output: PcmOutput,
86
+ responses: () -> Flow<SpeechRecognizerResponse>
87
+): Flow<SpeechRecognizerResponse> = flow {
88
+ try {
89
+ withTimeout(330000) {
90
+ responses().collect { emit(it) }
91
+ requireCompleteAudio(capture.inputExhausted) {
92
+ "Speech recognition ended before receiving the complete recording."
93
+ }
94
+ capture.awaitFinished()
95
+ output.verifyConsumed()
96
+ }
97
+ } catch (e: TimeoutCancellationException) {
98
+ throw VoiceTranscriptionException(
99
+ "incomplete-audio",
100
+ "Speech recognition did not finish the complete recording. No transcript was added. Retry dictation.",
101
+ e
102
+ )
103
+ } finally {
104
+ capture.cancel()
105
+ }
106
+}
107
+
108
internal inline fun requireCompleteAudio(condition: Boolean, message: () -> String) {
109
if (!condition) throw VoiceTranscriptionException("incomplete-audio", message())
110
}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt
+55
-1
@@ -6,7 +6,9 @@ import expo.modules.kotlin.Promise
6
import expo.modules.kotlin.functions.Coroutine
7
import expo.modules.kotlin.modules.Module
8
import expo.modules.kotlin.modules.ModuleDefinition
9
+import expo.modules.kotlin.modules.ModuleDefinitionBuilder
10
import java.util.Locale
11
+import java.util.concurrent.ConcurrentHashMap
12
import kotlinx.coroutines.CancellationException
13
import kotlinx.coroutines.CoroutineScope
14
import kotlinx.coroutines.CoroutineStart
@@ -22,6 +24,7 @@ class T3VoiceTranscriptionModule : Module() {
24
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
25
private val lock = Any()
26
private var operation: Pair<Int, Job>? = null
27
+ private val recorders = ConcurrentHashMap<String, LiveVoiceRecorder>()
28
29
override fun definition() = ModuleDefinition {
30
Name("T3VoiceTranscription")
@@ -66,6 +69,27 @@ class T3VoiceTranscriptionModule : Module() {
69
}
70
}
71
72
+ AsyncFunction("startLive") {
73
+ id: Int,
74
+ uri: String,
75
+ language: String,
76
+ mode: Int,
77
+ promise: Promise
78
+ ->
79
+ runOperation(id, promise, "transcription-failed") {
80
+ val recorder = recorders.values.firstOrNull { it.uri == uri }
81
+ ?: error("The voice recording is no longer available.")
82
+ val context = requireNotNull(appContext.reactContext)
83
+ MlKitSpeechClient(context, Locale.forLanguageTag(language), mode).use {
84
+ recorder.transcribe(uri, it)
85
+ }
86
+ }
87
+ }
88
+
89
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
90
+ defineLiveRecorder()
91
+ }
92
+
93
AsyncFunction("cancel") { id: Int, promise: Promise ->
94
scope.launch {
95
val job = synchronized(lock) { operation?.takeIf { it.first == id }?.second }
@@ -74,7 +98,37 @@ class T3VoiceTranscriptionModule : Module() {
98
}
99
}
100
77
- OnDestroy { scope.cancel() }
101
+ OnDestroy {
102
+ recorders.values.toList().forEach { it.sharedObjectDidRelease() }
103
+ scope.cancel()
104
+ }
105
+ }
106
+
107
+ private fun ModuleDefinitionBuilder.defineLiveRecorder() {
108
+ Class(LiveVoiceRecorder::class) {
109
+ Constructor {
110
+ check(Build.VERSION.SDK_INT >= Build.VERSION_CODES.S)
111
+ LiveVoiceRecorder(requireNotNull(appContext.reactContext), appContext, scope) {
112
+ recorders.remove(it)
113
+ }.also { recorders[it.id] = it }
114
+ }
115
+ Property("uri") { recorder -> recorder.uri }
116
+ AsyncFunction("prepareToRecordAsync") Coroutine { recorder: LiveVoiceRecorder ->
117
+ withContext(Dispatchers.IO) { recorder.prepare() }
118
+ }
119
+ Function("record") { recorder: LiveVoiceRecorder, options: Map<String, Double> ->
120
+ recorder.record(options["forDuration"] ?: 300.0)
121
+ }
122
+ AsyncFunction("stop") Coroutine { recorder: LiveVoiceRecorder -> recorder.stop() }
123
+ Function("getStatus") { recorder: LiveVoiceRecorder -> recorder.status() }
124
+ Function("getAvailableInputs") { recorder: LiveVoiceRecorder -> recorder.inputs() }
125
+ AsyncFunction("setInput") Coroutine { recorder: LiveVoiceRecorder, uid: String ->
126
+ recorder.setInput(uid)
127
+ }
128
+ Function("cancelInputSelection") { recorder: LiveVoiceRecorder ->
129
+ recorder.cancelInputSelection()
130
+ }
131
+ }
132
}
133
134
private fun runOperation(
apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LivePcmCaptureTest.kt
new
+184
@@ -0,0 +1,184 @@
1
+package expo.modules.t3voicetranscription
2
+
3
+import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
4
+import java.io.ByteArrayOutputStream
5
+import java.io.File
6
+import kotlinx.coroutines.CompletableDeferred
7
+import kotlinx.coroutines.CoroutineStart
8
+import kotlinx.coroutines.async
9
+import kotlinx.coroutines.awaitCancellation
10
+import kotlinx.coroutines.channels.Channel
11
+import kotlinx.coroutines.flow.flow
12
+import kotlinx.coroutines.test.runTest
13
+import org.junit.Assert.assertArrayEquals
14
+import org.junit.Assert.assertEquals
15
+import org.junit.Assert.assertFalse
16
+import org.junit.Assert.assertTrue
17
+import org.junit.Test
18
+
19
+class LivePcmCaptureTest {
20
+ private class Source : PcmCaptureSource {
21
+ override val capacityBytes = 32000
22
+ val chunks = Channel<ByteArray>(Channel.UNLIMITED)
23
+ val reading = Channel<Unit>(Channel.UNLIMITED)
24
+ var released = false
25
+ override fun start() = Unit
26
+ override suspend fun read(buffer: ByteArray, blocking: Boolean): Int {
27
+ reading.send(Unit)
28
+ val chunk = if (blocking) chunks.receive() else chunks.tryReceive().getOrNull() ?: return 0
29
+ chunk.copyInto(buffer)
30
+ return chunk.size
31
+ }
32
+ override fun stop() = Unit
33
+ override fun release() {
34
+ released = true
35
+ }
36
+ }
37
+
38
+ private class Output(private val gate: CompletableDeferred<Unit>? = null) : PcmOutput {
39
+ val bytes = ByteArrayOutputStream()
40
+ val first = CompletableDeferred<Unit>()
41
+ val writing = CompletableDeferred<Unit>()
42
+ val ended = CompletableDeferred<Unit>()
43
+ var unread = false
44
+ override suspend fun write(buffer: ByteArray, count: Int) {
45
+ writing.complete(Unit)
46
+ gate?.await()
47
+ bytes.write(buffer, 0, count)
48
+ first.complete(Unit)
49
+ }
50
+ override fun finish() {
51
+ ended.complete(Unit)
52
+ }
53
+ override fun verifyConsumed() = requireCompleteAudio(!unread) { "Unread captured audio." }
54
+ override fun close() = Unit
55
+ }
56
+
57
+ @Test fun transcribesDuringCaptureAndDrainsTheShortTailAfterStop() = runTest {
58
+ val source = Source()
59
+ val output = Output()
60
+ val file = File.createTempFile("live-capture-test-", ".pcm")
61
+ val first = ByteArray(3200) { (it % 251).toByte() }
62
+ val tail = byteArrayOf(1, 2, 3, 4, 5, 6)
63
+ val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
64
+ try {
65
+ capture.start(this, 300.0) {}
66
+ source.chunks.send(first)
67
+ output.first.await()
68
+ assertTrue(capture.isRecording)
69
+ assertFalse(output.ended.isCompleted)
70
+ assertArrayEquals(first, output.bytes.toByteArray())
71
+ source.reading.receive()
72
+ source.reading.receive()
73
+
74
+ val stopped = async(start = CoroutineStart.UNDISPATCHED) { capture.stop() }
75
+ source.chunks.send(tail)
76
+ stopped.await()
77
+ assertTrue(source.released)
78
+ assertFalse(capture.isRecording)
79
+ val text = collectGoogleVoiceTranscript(
80
+ livePcmResponses(capture, output) {
81
+ flow {
82
+ output.ended.await()
83
+ emit(
84
+ SpeechRecognizerResponse.FinalTextResponse("Opening. Repeat. Repeat.", emptyList())
85
+ )
86
+ emit(SpeechRecognizerResponse.FinalTextResponse(" 中間。", emptyList()))
87
+ emit(SpeechRecognizerResponse.PartialTextResponse(" An unfinished ending"))
88
+ emit(SpeechRecognizerResponse.PartialTextResponse(" End."))
89
+ emit(SpeechRecognizerResponse.CompletedResponse)
90
+ }
91
+ }
92
+ )
93
+ assertEquals("Opening. Repeat. Repeat. 中間。 End.", text)
94
+ assertArrayEquals(first + tail, output.bytes.toByteArray())
95
+ assertArrayEquals(first + tail, file.readBytes())
96
+ } finally {
97
+ capture.cancel()
98
+ file.delete()
99
+ }
100
+ }
101
+
102
+ @Test fun rejectsIncompleteRecognitionAndUnreadCapturedAudio() = runTest {
103
+ for (failure in listOf("early", "unread", "timeout")) {
104
+ val early = failure == "early"
105
+ val source = Source()
106
+ val output = Output().apply { unread = failure == "unread" }
107
+ val file = File.createTempFile("live-capture-test-", ".pcm")
108
+ val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
109
+ try {
110
+ capture.start(this, if (early) 300.0 else 0.1) {}
111
+ source.chunks.send(ByteArray(3200))
112
+ val result = runCatching {
113
+ collectGoogleVoiceTranscript(
114
+ livePcmResponses(capture, output) {
115
+ flow {
116
+ if (!early) output.ended.await()
117
+ if (failure == "timeout") awaitCancellation()
118
+ emit(SpeechRecognizerResponse.FinalTextResponse("Incomplete.", emptyList()))
119
+ emit(SpeechRecognizerResponse.CompletedResponse)
120
+ }
121
+ }
122
+ )
123
+ }
124
+ assertEquals(
125
+ "incomplete-audio",
126
+ (result.exceptionOrNull() as VoiceTranscriptionException).code
127
+ )
128
+ assertTrue(source.released)
129
+ } finally {
130
+ capture.cancel()
131
+ file.delete()
132
+ }
133
+ }
134
+ }
135
+
136
+ @Test fun durationLimitDrainsQueuedPcmBeforeReleasingTheMicrophone() = runTest {
137
+ val source = Source()
138
+ val output = Output()
139
+ val file = File.createTempFile("live-capture-test-", ".pcm")
140
+ val expected = ByteArray(6400) { (it % 251).toByte() } + byteArrayOf(42, 0)
141
+ val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
142
+ val stopped = CompletableDeferred<Unit>()
143
+ try {
144
+ source.chunks.send(expected.copyOfRange(0, 3200))
145
+ source.chunks.send(expected.copyOfRange(3200, 6400))
146
+ source.chunks.send(expected.copyOfRange(6400, expected.size))
147
+ capture.start(this, 0.1) { stopped.complete(Unit) }
148
+ stopped.await()
149
+ assertTrue(source.released)
150
+ assertEquals(expected.size.toLong(), file.length())
151
+ capture.awaitFinished()
152
+ assertArrayEquals(expected, output.bytes.toByteArray())
153
+ } finally {
154
+ capture.cancel()
155
+ file.delete()
156
+ }
157
+ }
158
+
159
+ @Test fun cancellationWaitsForCaptureAndFeederCleanup() = runTest {
160
+ for (phase in listOf("prepared", "reading", "writing")) {
161
+ val source = Source()
162
+ val output = Output(if (phase == "writing") CompletableDeferred() else null)
163
+ val file = File.createTempFile("live-capture-test-", ".pcm")
164
+ val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
165
+ try {
166
+ if (phase != "prepared") {
167
+ capture.start(this, 300.0) {}
168
+ source.reading.receive()
169
+ if (phase == "writing") {
170
+ source.chunks.send(ByteArray(3200))
171
+ output.writing.await()
172
+ }
173
+ }
174
+ capture.cancel()
175
+ assertTrue(source.released)
176
+ assertTrue(output.ended.isCompleted)
177
+ assertTrue(capture.isFinished)
178
+ assertTrue(runCatching { capture.awaitFinished() }.isFailure)
179
+ } finally {
180
+ file.delete()
181
+ }
182
+ }
183
+ }
184
+}
apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LiveVoiceRecorderTest.kt
new
+123
@@ -0,0 +1,123 @@
1
+package expo.modules.t3voicetranscription
2
+
3
+import android.content.Context
4
+import android.media.AudioRecord
5
+import android.os.ParcelFileDescriptor
6
+import com.facebook.react.bridge.BridgeReactContext
7
+import expo.modules.kotlin.AppContext
8
+import expo.modules.kotlin.ModulesProvider
9
+import expo.modules.kotlin.modules.Module
10
+import java.io.File
11
+import java.lang.ref.WeakReference
12
+import kotlinx.coroutines.CoroutineStart
13
+import kotlinx.coroutines.async
14
+import kotlinx.coroutines.channels.Channel
15
+import kotlinx.coroutines.test.runTest
16
+import org.junit.Assert.assertEquals
17
+import org.junit.Assert.assertFalse
18
+import org.junit.Assert.assertTrue
19
+import org.junit.Test
20
+import org.junit.runner.RunWith
21
+import org.robolectric.RobolectricTestRunner
22
+import org.robolectric.RuntimeEnvironment
23
+import org.robolectric.annotation.Config
24
+import org.robolectric.util.ReflectionHelpers
25
+
26
+@RunWith(RobolectricTestRunner::class)
27
+@Config(manifest = Config.NONE, sdk = [31])
28
+class LiveVoiceRecorderTest {
29
+ private class Source(private val failStop: Boolean = false) : PcmCaptureSource {
30
+ override val capacityBytes = 32000
31
+ val chunks = Channel<ByteArray>(Channel.UNLIMITED)
32
+ val reading = Channel<Unit>(Channel.UNLIMITED)
33
+ override fun start() = Unit
34
+ override suspend fun read(buffer: ByteArray, blocking: Boolean): Int {
35
+ reading.send(Unit)
36
+ val chunk = if (blocking) chunks.receive() else chunks.tryReceive().getOrNull() ?: return 0
37
+ chunk.copyInto(buffer)
38
+ return chunk.size
39
+ }
40
+ override fun stop() {
41
+ check(!failStop) { "The microphone could not stop." }
42
+ }
43
+ override fun release() = Unit
44
+ }
45
+
46
+ @Test fun preparingAgainReleasesTheUnusedMicrophone() = runTest {
47
+ withAppContext { context, appContext ->
48
+ val recorder = LiveVoiceRecorder(context, appContext, this) {}
49
+ try {
50
+ recorder.prepare()
51
+ val previous = ReflectionHelpers.getField<AudioRecord>(recorder, "audio")
52
+ assertEquals(AudioRecord.STATE_INITIALIZED, previous.state)
53
+ recorder.prepare()
54
+ assertEquals(AudioRecord.STATE_UNINITIALIZED, previous.state)
55
+ } finally {
56
+ recorder.sharedObjectDidRelease()
57
+ }
58
+ }
59
+ }
60
+
61
+ @Test fun stoppingBeforeInferencePreservesQueuedAudioAndClosesUnusedPipes() = runTest {
62
+ withAppContext { context, appContext ->
63
+ for (phase in listOf("prepared", "recording", "draining", "prepared-failure")) {
64
+ val started = phase == "recording" || phase == "draining"
65
+ val source = Source(phase == "prepared-failure")
66
+ val pipe = PcmPipe.open()
67
+ val reader = ReflectionHelpers.getField<ParcelFileDescriptor.AutoCloseInputStream>(
68
+ pipe,
69
+ "reader"
70
+ )
71
+ val file = File.createTempFile("recorder-stop-test-", ".pcm")
72
+ val capture = LivePcmCapture(source, file, pipe, testScheduler.timeSource)
73
+ val recorder = LiveVoiceRecorder(context, appContext, this) {}
74
+ ReflectionHelpers.setField(recorder, "capture", capture)
75
+ ReflectionHelpers.setField(recorder, "pipe", pipe)
76
+ try {
77
+ assertTrue(reader.fd.valid())
78
+ if (started) {
79
+ capture.start(this, 300.0) {}
80
+ source.reading.receive()
81
+ source.chunks.send(ByteArray(3200))
82
+ source.reading.receive()
83
+ val stopped = async(start = CoroutineStart.UNDISPATCHED) {
84
+ if (phase == "draining") capture.stop() else recorder.stop()
85
+ }
86
+ source.chunks.send(byteArrayOf(1, 2))
87
+ stopped.await()
88
+ if (phase == "draining") recorder.stop()
89
+ assertFalse(capture.isFinished)
90
+ } else {
91
+ assertEquals(phase == "prepared-failure", runCatching { recorder.stop() }.isFailure)
92
+ }
93
+ assertEquals(started, reader.fd.valid())
94
+ } finally {
95
+ capture.cancel()
96
+ recorder.sharedObjectDidRelease()
97
+ file.delete()
98
+ }
99
+ }
100
+ }
101
+ }
102
+
103
+ private suspend fun withAppContext(block: suspend (Context, AppContext) -> Unit) {
104
+ val context = RuntimeEnvironment.getApplication()
105
+ val reactContext = BridgeReactContext(context)
106
+ val appContext = AppContext(
107
+ object : ModulesProvider {
108
+ override fun getModulesMap() = emptyMap<Class<out Module>, String?>()
109
+ },
110
+ expo.modules.core.ModuleRegistry(emptyList(), emptyList()),
111
+ WeakReference(reactContext)
112
+ )
113
+ try {
114
+ block(context, appContext)
115
+ } finally {
116
+ appContext.javaClass.declaredMethods.first {
117
+ it.name.startsWith("onDestroy") && it.parameterCount == 0
118
+ }
119
+ .invoke(appContext)
120
+ reactContext.destroy()
121
+ }
122
+ }
123
+}
apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
+3
-2
@@ -1,5 +1,5 @@
1
import { useAtomValue } from "@effect/atom-react";
2
-import { AudioModule, RecordingPresets, type RecordingStatus } from "expo-audio";
2
+import { RecordingPresets, type RecordingStatus } from "expo-audio";
3
import { File } from "expo-file-system";
4
import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
5
import { useSyncExternalStore } from "react";
@@ -14,6 +14,7 @@ import {
14
rememberMicrophones,
15
} from "../../lib/microphonePriority";
16
import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
17
+import { createVoiceRecorder } from "../../native/voiceRecorder";
18
import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
19
import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene";
20
import {
@@ -87,7 +88,7 @@ const voiceInputAtom = Atom.make((get) => {
88
void session.controller.appMovedToBackground();
89
});
90
const recorder = createLazyVoiceRecorder({
90
- create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
91
+ create: () => createVoiceRecorder(VOICE_RECORDING_OPTIONS),
92
onStatus: (status: RecordingStatus) => {
93
session.controller.handleRecorderStatus({
94
isFinished: status.isFinished,
apps/mobile/src/native/voiceRecorder.android.ts
new
+8
@@ -0,0 +1,8 @@
1
+import { AudioModule } from "expo-audio";
2
+import { createLiveVoiceRecorder } from "./voiceTranscription.android";
3
+
4
+export function createVoiceRecorder(
5
+ options: ConstructorParameters<typeof AudioModule.AudioRecorder>[0],
6
+) {
7
+ return createLiveVoiceRecorder() ?? new AudioModule.AudioRecorder(options);
8
+}
apps/mobile/src/native/voiceRecorder.ts
new
+7
@@ -0,0 +1,7 @@
1
+import { AudioModule } from "expo-audio";
2
+
3
+export function createVoiceRecorder(
4
+ options: ConstructorParameters<typeof AudioModule.AudioRecorder>[0],
5
+) {
6
+ return new AudioModule.AudioRecorder(options);
7
+}
apps/mobile/src/native/voiceTranscription.android.test.ts
+21
-1
@@ -5,6 +5,17 @@ const mocks = vi.hoisted(() => ({
5
prepare: vi.fn<(id: number) => Promise<{ locale: string; mode: number }>>(),
6
transcribe: vi.fn<(id: number, uri: string, locale: string, mode: number) => Promise<string>>(),
7
cancel: vi.fn<(id: number) => Promise<void>>(),
8
+ LiveVoiceRecorder: class {
9
+ readonly uri = "file:///live.pcm";
10
+ },
11
+}));
12
+
13
+vi.mock("react-native", () => ({
14
+ Platform: {
15
+ get Version() {
16
+ return 31;
17
+ },
18
+ },
19
}));
20
21
vi.mock("expo", () => ({
@@ -15,8 +26,17 @@ import {
26
VoiceInputController,
27
resetVoiceInputGlobalsForTests,
28
} from "../../../../packages/client-runtime/src/voice-input/controller";
29
+import { Platform } from "react-native";
30
+
31
+import { createLiveVoiceRecorder, getLocalVoiceTranscriber } from "./voiceTranscription.android";
32
19
-import { getLocalVoiceTranscriber } from "./voiceTranscription.android";
33
+it.each([
34
+ [30, null],
35
+ [31, "file:///live.pcm"],
36
+] as const)("offers live recording only on supported Android versions (%s)", (version, uri) => {
37
+ using _version = vi.spyOn(Platform, "Version", "get").mockReturnValue(version);
38
+ expect(createLiveVoiceRecorder()?.uri ?? null).toBe(uri);
39
+});
40
41
it.each([false, true])(
42
"waits for device support before offering a transcriber (%s)",
apps/mobile/src/native/voiceTranscription.android.ts
+20
@@ -1,4 +1,7 @@
1
import { requireOptionalNativeModule } from "expo";
2
+import type { RecorderState, RecordingStatus } from "expo-audio";
3
+import { Platform } from "react-native";
4
+import type { NativeVoiceRecorder } from "../features/voice-input/lazyVoiceRecorder";
5
6
import {
7
VoiceTranscriptionError,
@@ -13,6 +16,8 @@ type GoogleTranscriptionModule = {
16
prepare(id: number): Promise<PreparedGoogleTranscription>;
17
transcribe(id: number, uri: string, locale: string, mode: number): Promise<string>;
18
cancel(id: number): Promise<void>;
19
+ readonly LiveVoiceRecorder?: new () => NativeVoiceRecorder<RecordingStatus, RecorderState>;
20
+ startLive?(id: number, uri: string, locale: string, mode: number): Promise<string>;
21
};
22
23
const native = requireOptionalNativeModule<GoogleTranscriptionModule>("T3VoiceTranscription");
@@ -33,6 +38,12 @@ const nativeErrorMessages: Readonly<Record<string, string>> = {
38
39
let nextOperationId = 0;
40
41
+export function createLiveVoiceRecorder() {
42
+ return Number(Platform.Version) >= 31 && native?.LiveVoiceRecorder
43
+ ? new native.LiveVoiceRecorder()
44
+ : null;
45
+}
46
+
47
export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | null> {
48
if (!native || !(await native.checkAvailability())) return null;
49
const module = native;
@@ -41,8 +52,17 @@ export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | nul
52
const prepared = await runOperation(module, signal, "preparation-failed", (id) =>
53
module.prepare(id),
54
);
55
+ const startLive = module.startLive?.bind(module);
56
return {
57
locale: prepared.locale,
58
+ ...(module.LiveVoiceRecorder && startLive
59
+ ? {
60
+ start: (uri: string, { signal }: { signal: AbortSignal }) =>
61
+ runOperation(module, signal, "transcription-failed", (id) =>
62
+ startLive(id, uri, prepared.locale, prepared.mode),
63
+ ),
64
+ }
65
+ : {}),
66
transcribe: (uri, { signal }) =>
67
runOperation(module, signal, "transcription-failed", (id) =>
68
module.transcribe(id, uri, prepared.locale, prepared.mode),
docs/user/composer.md
+2
-2
@@ -198,8 +198,8 @@ voice input isn't available. On Android, keep T3 Code open while recording and
198
transcribing.
199
200
The first use may download a speech model and needs a network connection.
201
-Later transcription works offline for that language. On Android, transcription
202
-processes the full recording at roughly its original duration. Recordings can be up to five
201
+Later transcription works offline for that language. Android processes audio while
202
+you record, then finishes the transcript after you stop. Recordings can be up to five
203
minutes long. Canceling or an audio interruption discards the recording and
204
preserves your existing draft. Navigating to another screen keeps dictation active;
205
use the floating controls to finish or cancel. While recording, the screen stays
packages/client-runtime/src/voice-input/controller.test.ts
+93
@@ -243,6 +243,13 @@ describe("VoiceInputController", () => {
243
expect(harness.recorder.record).toHaveBeenCalledWith({
244
forDuration: VOICE_RECORDING_LIMIT_SECONDS,
245
});
246
+ await harness.controller.handleRecorderStatus({
247
+ isFinished: true,
248
+ hasError: false,
249
+ error: null,
250
+ url: "file:///previous-recording.m4a",
251
+ });
252
+ expect(harness.controller.currentState.phase).toBe("recording");
253
254
const stopping = harness.controller.stop();
255
harness.controller.handleRecorderStatus({
@@ -342,6 +349,92 @@ describe("VoiceInputController", () => {
349
expect(events).toEqual(["released", "transcribed"]);
350
});
351
352
+ it("starts streaming transcription during recording and commits only after stopping", async () => {
353
+ const transcript = deferred<string>();
354
+ const start = vi.fn(() => transcript.promise);
355
+ const transcribe = vi.fn(async () => "replayed audio");
356
+ const harness = createHarness({
357
+ getTranscriber: () => ({
358
+ prepare: async () => ({ locale: "en-US", start, transcribe }),
359
+ }),
360
+ });
361
+ await harness.controller.start();
362
+ expect(start).toHaveBeenCalledWith("file:///voice.m4a", {
363
+ signal: expect.any(AbortSignal),
364
+ });
365
+ expect(harness.controller.currentState.phase).toBe("recording");
366
+ expect(harness.commits).toEqual([]);
367
+
368
+ const stopping = harness.controller.stop();
369
+ transcript.resolve("opening middle and final words");
370
+ await stopping;
371
+ expect(harness.commits[0]?.text).toBe("hello opening middle and final words");
372
+ expect(transcribe).not.toHaveBeenCalled();
373
+ expect(harness.deleted).toEqual(["file:///voice.m4a"]);
374
+ });
375
+
376
+ it("waits for canceled live inference before deleting its recording or starting again", async () => {
377
+ const transcript = deferred<string>();
378
+ const released = deferred<void>();
379
+ let signal: AbortSignal | undefined;
380
+ const harness = createHarness({
381
+ getTranscriber: () => ({
382
+ prepare: async () => ({
383
+ ...preparedTranscription(),
384
+ start: (_uri: string, options: { signal: AbortSignal }) => {
385
+ signal = options.signal;
386
+ return transcript.promise;
387
+ },
388
+ }),
389
+ }),
390
+ deleteRecording: () => released.resolve(undefined),
391
+ });
392
+ await harness.controller.start();
393
+ expect(signal).toBeDefined();
394
+ harness.controller.cancel();
395
+ expect(signal?.aborted).toBe(true);
396
+ const next = createHarness();
397
+ const nextStart = next.controller.start();
398
+ expect(next.recorder.record).not.toHaveBeenCalled();
399
+
400
+ transcript.resolve("discard this");
401
+ await released.promise;
402
+ await nextStart;
403
+ expect(harness.commits).toEqual([]);
404
+ expect(next.controller.currentState.phase).toBe("recording");
405
+ await next.controller.interruptRecording();
406
+ });
407
+
408
+ it("stops recording after a live inference failure and preserves the draft for retry", async () => {
409
+ const transcript = deferred<string>();
410
+ const released = deferred<void>();
411
+ const harness = createHarness({
412
+ getTranscriber: () => ({
413
+ prepare: async () => ({ ...preparedTranscription(), start: () => transcript.promise }),
414
+ }),
415
+ deleteRecording: () => released.resolve(undefined),
416
+ });
417
+ await harness.controller.start();
418
+ transcript.reject(
419
+ new VoiceTranscriptionError({
420
+ code: "transcription-failed",
421
+ message: "Speech quota reached.",
422
+ }),
423
+ );
424
+ await released.promise;
425
+ expect(harness.controller.currentState).toMatchObject({
426
+ phase: "error",
427
+ error: "Speech quota reached.",
428
+ errorAction: "retry",
429
+ });
430
+ expect(harness.recorder.stop).toHaveBeenCalled();
431
+ expect(harness.commits).toEqual([]);
432
+ const next = createHarness();
433
+ await next.controller.start();
434
+ expect(next.controller.currentState.phase).toBe("recording");
435
+ await next.controller.interruptRecording();
436
+ });
437
+
438
it("ignores a late transcript after the draft owner changes", async () => {
439
const transcription = deferred<string>();
440
const transcriptionEntered = deferred<void>();
packages/client-runtime/src/voice-input/controller.ts
+29
-3
@@ -194,6 +194,7 @@ export class VoiceInputController {
194
private operationToken = 0;
195
private sessionToken: VoiceInputSession | null = null;
196
private transcription: PreparedVoiceTranscription | null = null;
197
+ private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null;
198
private transcriptionAbortController: AbortController | null = null;
199
private capturedDraft: VoiceDraftSnapshot | null = null;
200
private recordingUri: string | null = null;
@@ -278,6 +279,21 @@ export class VoiceInputController {
279
return;
280
}
281
this.capturedDraft = capturedDraft;
282
+ if (this.transcription.start && this.recordingUri) {
283
+ const start = this.transcription.start;
284
+ const uri = this.recordingUri;
285
+ this.liveTranscription = runTranscriptionOperation(() =>
286
+ start(uri, { signal: abortController.signal }),
287
+ ).then(
288
+ (text) => ({ text }),
289
+ (error: unknown) => {
290
+ if (this.isCurrent(operationToken) && this.state.phase === "recording") {
291
+ void this.discardRecording(transcriptionErrorMessage(error));
292
+ }
293
+ return { error };
294
+ },
295
+ );
296
+ }
297
this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS });
298
this.setState({ phase: "recording", error: null, errorAction: null });
299
} catch {
@@ -341,6 +357,7 @@ export class VoiceInputController {
357
358
handleRecorderStatus(status: VoiceRecorderStatus): Promise<void> | void {
359
if (this.state.phase !== "recording") return;
360
+ if (status.url && this.recordingUri && status.url !== this.recordingUri) return;
361
if (status.hasError) {
362
return this.interruptRecording(
363
status.error ?? "Voice recording was interrupted.",
@@ -402,9 +419,15 @@ export class VoiceInputController {
419
const capturedDraft = this.capturedDraft;
420
let transcript: string;
421
try {
405
- transcript = await runTranscriptionOperation(() =>
406
- transcription.transcribe(recordingUri, { signal }),
407
- );
422
+ if (this.liveTranscription) {
423
+ const result = await this.liveTranscription;
424
+ if ("error" in result) throw result.error;
425
+ transcript = result.text;
426
+ } else {
427
+ transcript = await runTranscriptionOperation(() =>
428
+ transcription.transcribe(recordingUri, { signal }),
429
+ );
430
+ }
431
} catch (error) {
432
if (this.isCurrent(operationToken)) {
433
this.setError(transcriptionErrorMessage(error), "retry");
@@ -454,6 +477,7 @@ export class VoiceInputController {
477
}
478
479
private async releaseResources(): Promise<void> {
480
+ if (this.liveTranscription) this.transcriptionAbortController?.abort();
481
this.rememberRecordingUri(this.recordingUri);
482
try {
483
this.rememberRecordingUri(this.dependencies.recorder.uri);
@@ -463,6 +487,8 @@ export class VoiceInputController {
487
await this.dependencies.recorder.stop();
488
} catch {}
489
}
490
+ await this.liveTranscription;
491
+ this.liveTranscription = null;
492
this.recordingUri = null;
493
for (const uri of this.ownedRecordingUris) {
494
try {
packages/client-runtime/src/voice-input/transcription.ts
+1
@@ -7,6 +7,7 @@ export type VoiceTranscriptionOptions = {
7
8
export type PreparedVoiceTranscription = {
9
readonly locale: string;
10
+ readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
11
readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
12
};
13
patches/expo-audio@58.0.9.patch
+3
-3
@@ -339,7 +339,7 @@ index 81819c1d771bdeec093bb35a65a5e1e1af0cf9d7..cd0c7624b4cd8a4ebf7ed3b3390cfc76
339
return map
340
}
341
342
-+internal fun recordingInputs(manager: AudioManager): List<AudioDeviceInfo> {
342
++fun recordingInputs(manager: AudioManager): List<AudioDeviceInfo> {
343
+ val inputs = manager.getDevices(AudioManager.GET_DEVICES_INPUTS).filter {
344
+ recordingInputType(it.type) != null
345
+ }
@@ -367,7 +367,7 @@ index 81819c1d771bdeec093bb35a65a5e1e1af0cf9d7..cd0c7624b4cd8a4ebf7ed3b3390cfc76
367
+ else -> null
368
+}
369
+
370
-+internal fun recordingInputUid(device: AudioDeviceInfo, inputs: List<AudioDeviceInfo> = emptyList()): String {
370
++fun recordingInputUid(device: AudioDeviceInfo, inputs: List<AudioDeviceInfo> = emptyList()): String {
371
+ val base = recordingInputBaseUid(device)
372
+ val ambiguous = !isBluetoothInput(device.type) && inputs.any {
373
+ it.id != device.id && recordingInputBaseUid(it) == base
@@ -401,7 +401,7 @@ index 0000000000000000000000000000000000000000..cdba9724318cf06afd21b4df6b895b38
401
+import kotlinx.coroutines.withTimeout
402
+
403
+@RequiresApi(31)
404
-+internal class RecordingInputRoute(private val manager: AudioManager) : Closeable {
404
++class RecordingInputRoute(private val manager: AudioManager) : Closeable {
405
+ private var previousMode: Int? = null
406
+
407
+ suspend fun select(input: AudioDeviceInfo) {
pnpm-lock.yaml
+3
-3
@@ -370,7 +370,7 @@ patchedDependencies:
370
alchemy@2.0.0-beta.85: 07cc8c7e14b3b9af143ddd40a059f117c27e55139fbf20bd5dde2eb30509cedd
371
app-builder-lib@26.16.1: 0fbe9ae51783f5768ebe197d12d1fdb7d477a08c92e5db2f6356bcce62613c53
372
dbus-next@0.10.2: cfff57561b0ee59b5addb3b2e6c6f20906e967507a530ab67e8db8108e520ba4
373
- expo-audio@58.0.9: b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba
373
+ expo-audio@58.0.9: cb873a17fbd9aca52ddcecbde4c3d899566f7e265ab49a992963a6276b29bf52
374
expo-modules-core@58.0.22: f0b0fd100550ed76e5c25b4d13eb7c9ff62825f8f197c53edf7bf447c50c9fb7
375
expo-network@58.0.3: cd09b6d1744dde4efa081b5619ebcff38877d98f7502a51cab121629dece8237
376
expo-sharing@58.0.19: 321a86e1471e34c53f1f2937fbb4304ea3f7b7a78ad9e4d35febc2b9e73e9688
@@ -619,7 +619,7 @@ importers:
619
version: 58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)
620
expo-audio:
621
specifier: 58.0.9
622
- version: 58.0.9(patch_hash=b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba)(expo-asset@58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
622
+ version: 58.0.9(patch_hash=cb873a17fbd9aca52ddcecbde4c3d899566f7e265ab49a992963a6276b29bf52)(expo-asset@58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
623
expo-auth-session:
624
specifier: ~58.0.13
625
version: 58.0.13(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)
@@ -18132,7 +18132,7 @@ snapshots:
18132
- supports-color
18133
- typescript
18134
18135
- expo-audio@58.0.9(patch_hash=b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba)(expo-asset@58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18135
+ expo-audio@58.0.9(patch_hash=cb873a17fbd9aca52ddcecbde4c3d899566f7e265ab49a992963a6276b29bf52)(expo-asset@58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18136
dependencies:
18137
expo: 58.0.14(ecd3689b2bda1172f8915f190f5c7383)
18138
expo-asset: 58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)