@devries-cameron20 / t3code-1 / commits / 66633fb682
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

fix(mobile): transcribe Android audio during recording (#31300)

Alex committed Nov 5, 2026 at 12:18 UTC 66633fb6826c42ce2c08913fc99ab4adaae7648c
19 files changed +1004 -15
apps/mobile/modules/t3-voice-transcription/android/build.gradle
+2
@@ -16,9 +16,11 @@ android {
16
17 dependencies {
18 implementation project(':expo-modules-core')
19 + implementation project(':expo-audio')
20 implementation 'androidx.media3:media3-common:1.9.1'
21 implementation 'com.google.mlkit:genai-speech-recognition:1.0.0-alpha2'
22 testImplementation 'junit:junit:4.13.2'
23 + testImplementation 'com.facebook.react:react-android'
24 testImplementation 'org.robolectric:robolectric:4.16.1'
25 testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.10.2'
26 }
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/GoogleVoiceTranscriber.kt
+11
@@ -158,5 +158,16 @@ internal class MlKitSpeechClient(context: Context, locale: Locale, mode: Int) :
158 }
159 }
160
161 + suspend fun transcribeLive(capture: LivePcmCapture, pipe: PcmPipe): String = pipe.use {
162 + val request = speechRecognizerRequest { audioSource = pipe.source }
163 + try {
164 + collectGoogleVoiceTranscript(
165 + livePcmResponses(capture, pipe) { recognizer.startRecognition(request) }
166 + )
167 + } finally {
168 + withContext(NonCancellable) { runCatching { recognizer.stopRecognition() } }
169 + }
170 + }
171 +
172 override fun close() = recognizer.close()
173 }
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LivePcmCapture.kt new
+216
@@ -0,0 +1,216 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import java.io.File
4 +import kotlin.math.abs
5 +import kotlin.math.log10
6 +import kotlin.time.TimeSource
7 +import kotlinx.coroutines.CancellationException
8 +import kotlinx.coroutines.CompletableDeferred
9 +import kotlinx.coroutines.CoroutineScope
10 +import kotlinx.coroutines.Job
11 +import kotlinx.coroutines.NonCancellable
12 +import kotlinx.coroutines.TimeoutCancellationException
13 +import kotlinx.coroutines.currentCoroutineContext
14 +import kotlinx.coroutines.coroutineScope
15 +import kotlinx.coroutines.channels.Channel
16 +import kotlinx.coroutines.delay
17 +import kotlinx.coroutines.ensureActive
18 +import kotlinx.coroutines.launch
19 +import kotlinx.coroutines.withContext
20 +import kotlinx.coroutines.withTimeout
21 +
22 +internal interface PcmCaptureSource {
23 + val capacityBytes: Int
24 + fun start()
25 + suspend fun read(buffer: ByteArray, blocking: Boolean): Int
26 + fun stop()
27 + fun release()
28 +}
29 +
30 +internal class LivePcmCapture(
31 + private val source: PcmCaptureSource,
32 + val file: File,
33 + private val output: PcmOutput,
34 + private val clock: TimeSource = TimeSource.Monotonic
35 +) {
36 + @Volatile private var worker: Job? = null
37 + private val completion = CompletableDeferred<Result<Unit>>()
38 + private val microphoneStopped = CompletableDeferred<Result<Unit>>()
39 +
40 + @Volatile private var stopping = false
41 + private var sourceStopped = false
42 + private var sourceReleased = false
43 +
44 + @Volatile var bytes = 0L
45 + private set
46 +
47 + @Volatile var metering = -160.0
48 + private set
49 + val isFinished get() = completion.isCompleted
50 +
51 + @Volatile var inputExhausted = false
52 + private set
53 + val hasStarted get() = worker != null
54 + val isRecording get() = hasStarted && !microphoneStopped.isCompleted
55 +
56 + fun start(scope: CoroutineScope, seconds: Double, onFinished: (Throwable?) -> Unit) {
57 + check(worker == null && !sourceReleased) { "The microphone is not prepared." }
58 + worker = scope.launch {
59 + var result = runCatching {
60 + source.start()
61 + writeCapture(seconds, onFinished)
62 + requireCompleteAudio(bytes > 0 && file.length() == bytes) {
63 + "The microphone did not finish its captured audio."
64 + }
65 + }
66 + result = finishSource(result)
67 + inputExhausted = result.isSuccess
68 + result = runCatching { output.finish() }.fold(
69 + onSuccess = { result },
70 + onFailure = { failure -> mergeFailure(result, failure) }
71 + )
72 + if (microphoneStopped.complete(result)) onFinished(result.exceptionOrNull())
73 + completion.complete(result)
74 + }.also { job ->
75 + job.invokeOnCompletion { error ->
76 + if (!completion.isCompleted) {
77 + val result =
78 + finishSource(Result.failure(error ?: CancellationException("Capture stopped.")))
79 + completion.complete(result)
80 + microphoneStopped.complete(result)
81 + runCatching { output.finish() }
82 + }
83 + }
84 + }
85 + }
86 +
87 + private suspend fun writeCapture(
88 + seconds: Double,
89 + onFinished: (Throwable?) -> Unit
90 + ) = coroutineScope {
91 + val started = clock.markNow()
92 + val chunks = Channel<ByteArray>(100)
93 + var sent = 0L
94 + val sender = launch {
95 + var deadline = 0L
96 + for (chunk in chunks) {
97 + deadline = maxOf(
98 + deadline + (chunk.size * 1000L + 31999) / 32000,
99 + started.elapsedNow().inWholeMilliseconds
100 + )
101 + delay((deadline - started.elapsedNow().inWholeMilliseconds).coerceAtLeast(0))
102 + output.write(chunk, chunk.size)
103 + deadline = started.elapsedNow().inWholeMilliseconds
104 + sent += chunk.size
105 + }
106 + }
107 + val result = finishSource(runCatching { captureChunks(seconds, chunks) })
108 + microphoneStopped.complete(result)
109 + if (result.exceptionOrNull() !is CancellationException) onFinished(result.exceptionOrNull())
110 + chunks.close()
111 + result.getOrThrow()
112 + sender.join()
113 + requireCompleteAudio(sent == bytes) { "Speech recognition did not receive all captured audio." }
114 + }
115 +
116 + private suspend fun captureChunks(seconds: Double, chunks: Channel<ByteArray>) {
117 + val buffer = ByteArray(3200)
118 + var drained = 0
119 + file.outputStream().use { stream ->
120 + while (true) {
121 + currentCoroutineContext().ensureActive()
122 + val draining = stopping || bytes >= seconds * 32000
123 + val count = source.read(buffer, !draining)
124 + requireCompleteAudio(count in 0..buffer.size && count % 2 == 0) {
125 + "The microphone returned incomplete audio."
126 + }
127 + if (count == 0) {
128 + requireCompleteAudio(draining) { "The microphone stopped supplying audio." }
129 + break
130 + }
131 + if (draining) {
132 + drained += count
133 + requireCompleteAudio(drained <= source.capacityBytes + buffer.size) {
134 + "The microphone audio queue did not drain."
135 + }
136 + }
137 + stream.write(buffer, 0, count)
138 + // Keep draining the microphone while inference starts or applies backpressure.
139 + if (!chunks.trySend(buffer.copyOf(count)).isSuccess) {
140 + throw VoiceTranscriptionException(
141 + "audio-overflow",
142 + "Speech recognition fell behind capture."
143 + )
144 + }
145 + bytes += count
146 + var peak = 0
147 + for (i in 0 until count step 2) {
148 + val sample = ((buffer[i].toInt() and 255) or (buffer[i + 1].toInt() shl 8)).toShort()
149 + peak = maxOf(peak, abs(sample.toInt()))
150 + }
151 + metering = if (peak == 0) -160.0 else 20 * log10(peak / 32768.0)
152 + }
153 + }
154 + }
155 +
156 + suspend fun stop() {
157 + stopping = true
158 + if (worker == null) {
159 + finishSource(Result.success(Unit)).getOrThrow()
160 + output.finish()
161 + return
162 + }
163 + try {
164 + withTimeout(2000) { microphoneStopped.await().getOrThrow() }
165 + } catch (error: TimeoutCancellationException) {
166 + cancel()
167 + throw VoiceTranscriptionException("incomplete-audio", "The microphone did not stop.", error)
168 + }
169 + }
170 +
171 + suspend fun awaitFinished() = completion.await().getOrThrow()
172 +
173 + fun abort() {
174 + stopping = true
175 + worker?.cancel()
176 + runCatching { stopSource() }
177 + runCatching { output.finish() }
178 + if (worker == null) {
179 + val result = finishSource(Result.failure(CancellationException("Capture was cancelled.")))
180 + completion.complete(result)
181 + microphoneStopped.complete(result)
182 + }
183 + }
184 +
185 + suspend fun cancel() = withContext(NonCancellable) {
186 + abort()
187 + worker?.join()
188 + finishSource(Result.success(Unit))
189 + }
190 +
191 + @Synchronized private fun stopSource() {
192 + if (sourceStopped) return
193 + sourceStopped = true
194 + source.stop()
195 + }
196 +
197 + @Synchronized private fun finishSource(result: Result<Unit>): Result<Unit> {
198 + if (sourceReleased) return result
199 + sourceReleased = true
200 + var outcome = runCatching { stopSource() }.fold(
201 + onSuccess = { result },
202 + onFailure = { mergeFailure(result, it) }
203 + )
204 + outcome = runCatching { source.release() }.fold(
205 + onSuccess = { outcome },
206 + onFailure = { mergeFailure(outcome, it) }
207 + )
208 + return outcome
209 + }
210 +}
211 +
212 +private fun mergeFailure(result: Result<Unit>, failure: Throwable): Result<Unit> {
213 + val original = result.exceptionOrNull() ?: return Result.failure(failure)
214 + if (original !== failure) original.addSuppressed(failure)
215 + return result
216 +}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/LiveVoiceRecorder.kt new
+198
@@ -0,0 +1,198 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import android.content.Context
4 +import android.media.AudioFormat
5 +import android.media.AudioManager
6 +import android.media.AudioRecord
7 +import android.media.MediaRecorder
8 +import android.net.Uri
9 +import androidx.annotation.RequiresApi
10 +import expo.modules.audio.RecordingInputRoute
11 +import expo.modules.audio.getMapFromDeviceInfo
12 +import expo.modules.audio.recordingInputs
13 +import expo.modules.audio.recordingInputUid
14 +import expo.modules.kotlin.AppContext
15 +import expo.modules.kotlin.sharedobjects.SharedObject
16 +import java.io.File
17 +import java.util.UUID
18 +import kotlinx.coroutines.CoroutineScope
19 +import kotlinx.coroutines.Job
20 +import kotlinx.coroutines.currentCoroutineContext
21 +import kotlinx.coroutines.job
22 +
23 +@RequiresApi(31)
24 +internal class LiveVoiceRecorder(
25 + private val context: Context,
26 + appContext: AppContext,
27 + private val scope: CoroutineScope,
28 + private val onRelease: (String) -> Unit
29 +) : SharedObject(appContext) {
30 + val id = UUID.randomUUID().toString()
31 + private val manager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
32 + private val route = RecordingInputRoute(manager)
33 + private var audio: AudioRecord? = null
34 + private var pipe: PcmPipe? = null
35 + private var capture: LivePcmCapture? = null
36 + private var inputSelection: Job? = null
37 + private var inputCanceled = false
38 + private var inferenceStarted = false
39 + val uri: String? get() = capture?.file?.let { Uri.fromFile(it).toString() }
40 +
41 + @Suppress("TooGenericExceptionCaught") // Release native handles for every setup failure.
42 + fun prepare() {
43 + check(capture?.isRecording != true) { "Another voice recording is active." }
44 + capture?.abort()
45 + pipe?.close()
46 + inputCanceled = false
47 + inferenceStarted = false
48 + val capacity = maxOf(
49 + 32000,
50 + AudioRecord.getMinBufferSize(
51 + 16000,
52 + AudioFormat.CHANNEL_IN_MONO,
53 + AudioFormat.ENCODING_PCM_16BIT
54 + )
55 + )
56 + val recorder = AudioRecord.Builder()
57 + .setAudioSource(MediaRecorder.AudioSource.VOICE_RECOGNITION)
58 + .setAudioFormat(
59 + AudioFormat.Builder().setSampleRate(16000)
60 + .setChannelMask(AudioFormat.CHANNEL_IN_MONO)
61 + .setEncoding(AudioFormat.ENCODING_PCM_16BIT).build()
62 + )
63 + .setBufferSizeInBytes(capacity)
64 + .build()
65 + try {
66 + check(recorder.state == AudioRecord.STATE_INITIALIZED) { "The microphone could not prepare." }
67 + val output = PcmPipe.open()
68 + try {
69 + val file = File.createTempFile("dictation-live-", ".pcm", context.cacheDir)
70 + capture = LivePcmCapture(
71 + object : PcmCaptureSource {
72 + override val capacityBytes = capacity
73 + override fun start() {
74 + recorder.startRecording()
75 + check(recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) {
76 + "The microphone could not start."
77 + }
78 + }
79 + override suspend fun read(buffer: ByteArray, blocking: Boolean) = recorder.read(
80 + buffer,
81 + 0,
82 + buffer.size,
83 + if (blocking) AudioRecord.READ_BLOCKING else AudioRecord.READ_NON_BLOCKING
84 + )
85 + override fun stop() {
86 + if (recorder.recordingState == AudioRecord.RECORDSTATE_RECORDING) recorder.stop()
87 + }
88 + override fun release() {
89 + try {
90 + recorder.release()
91 + } finally {
92 + route.close()
93 + }
94 + }
95 + },
96 + file,
97 + output
98 + )
99 + audio = recorder
100 + pipe = output
101 + } catch (error: Throwable) {
102 + output.close()
103 + throw error
104 + }
105 + } catch (error: Throwable) {
106 + recorder.release()
107 + throw error
108 + }
109 + }
110 +
111 + fun record(seconds: Double) {
112 + val recording = requireNotNull(capture) { "The microphone is not prepared." }
113 + recording.start(scope, seconds) { error ->
114 + emit(
115 + "recordingStatusUpdate",
116 + mapOf(
117 + "id" to id,
118 + "isFinished" to true,
119 + "hasError" to (error != null),
120 + "error" to error?.message,
121 + "url" to Uri.fromFile(recording.file).toString()
122 + )
123 + )
124 + }
125 + }
126 +
127 + suspend fun stop() {
128 + val recording = capture
129 + val output = pipe
130 + try {
131 + val selection = synchronized(this) { inputSelection }
132 + selection?.cancel()
133 + selection?.join()
134 + recording?.stop()
135 + } finally {
136 + // Recognition can start after Stop and still needs the captured PCM pipe.
137 + if (recording?.hasStarted != true) output?.close()
138 + }
139 + }
140 +
141 + fun status() = mapOf(
142 + "canRecord" to (capture != null),
143 + "isRecording" to (capture?.isRecording == true),
144 + "durationMillis" to (capture?.bytes?.div(32) ?: 0),
145 + "fileSize" to (capture?.file?.length() ?: 0),
146 + "metering" to (capture?.metering ?: -160.0),
147 + "mediaServicesDidReset" to false,
148 + "url" to uri
149 + )
150 +
151 + fun inputs() = recordingInputs(manager).let { inputs ->
152 + inputs.map { getMapFromDeviceInfo(it, inputs) }
153 + }
154 +
155 + // Release the preferred route on cancellation as well as selection errors.
156 + @Suppress("TooGenericExceptionCaught")
157 + suspend fun setInput(uid: String) {
158 + val job = currentCoroutineContext().job
159 + synchronized(this) {
160 + check(!inputCanceled) { "Microphone selection was cancelled." }
161 + inputSelection = job
162 + }
163 + try {
164 + val inputs = recordingInputs(manager)
165 + val selected = inputs.firstOrNull { recordingInputUid(it, inputs) == uid }
166 + ?: error("The preferred microphone is disconnected.")
167 + route.select(selected)
168 + val routed = recordingInputs(manager)
169 + val source = routed.firstOrNull { it.isSource && recordingInputUid(it, routed) == uid }
170 + ?: error("The preferred microphone is disconnected.")
171 + check(audio?.setPreferredDevice(source) == true) { "Could not select the microphone." }
172 + } catch (error: Throwable) {
173 + route.close()
174 + throw error
175 + } finally {
176 + synchronized(this) { if (inputSelection === job) inputSelection = null }
177 + }
178 + }
179 +
180 + @Synchronized fun cancelInputSelection() {
181 + inputCanceled = true
182 + inputSelection?.cancel()
183 + }
184 +
185 + suspend fun transcribe(uri: String, client: MlKitSpeechClient): String {
186 + check(this.uri == uri && !inferenceStarted) { "The voice recording is no longer available." }
187 + inferenceStarted = true
188 + return client.transcribeLive(requireNotNull(capture), requireNotNull(pipe))
189 + }
190 +
191 + override fun sharedObjectDidRelease() {
192 + super.sharedObjectDidRelease()
193 + cancelInputSelection()
194 + capture?.abort()
195 + pipe?.close()
196 + onRelease(id)
197 + }
198 +}
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/PcmOutput.kt
+25
@@ -80,6 +80,31 @@ internal fun pacedRecordedResponses(
80 }
81 }
82
83 +internal fun livePcmResponses(
84 + capture: LivePcmCapture,
85 + output: PcmOutput,
86 + responses: () -> Flow<SpeechRecognizerResponse>
87 +): Flow<SpeechRecognizerResponse> = flow {
88 + try {
89 + withTimeout(330000) {
90 + responses().collect { emit(it) }
91 + requireCompleteAudio(capture.inputExhausted) {
92 + "Speech recognition ended before receiving the complete recording."
93 + }
94 + capture.awaitFinished()
95 + output.verifyConsumed()
96 + }
97 + } catch (e: TimeoutCancellationException) {
98 + throw VoiceTranscriptionException(
99 + "incomplete-audio",
100 + "Speech recognition did not finish the complete recording. No transcript was added. Retry dictation.",
101 + e
102 + )
103 + } finally {
104 + capture.cancel()
105 + }
106 +}
107 +
108 internal inline fun requireCompleteAudio(condition: Boolean, message: () -> String) {
109 if (!condition) throw VoiceTranscriptionException("incomplete-audio", message())
110 }
apps/mobile/modules/t3-voice-transcription/android/src/main/java/expo/modules/t3voicetranscription/T3VoiceTranscriptionModule.kt
+55 -1
@@ -6,7 +6,9 @@ import expo.modules.kotlin.Promise
6 import expo.modules.kotlin.functions.Coroutine
7 import expo.modules.kotlin.modules.Module
8 import expo.modules.kotlin.modules.ModuleDefinition
9 +import expo.modules.kotlin.modules.ModuleDefinitionBuilder
10 import java.util.Locale
11 +import java.util.concurrent.ConcurrentHashMap
12 import kotlinx.coroutines.CancellationException
13 import kotlinx.coroutines.CoroutineScope
14 import kotlinx.coroutines.CoroutineStart
@@ -22,6 +24,7 @@ class T3VoiceTranscriptionModule : Module() {
24 private val scope = CoroutineScope(SupervisorJob() + Dispatchers.IO)
25 private val lock = Any()
26 private var operation: Pair<Int, Job>? = null
27 + private val recorders = ConcurrentHashMap<String, LiveVoiceRecorder>()
28
29 override fun definition() = ModuleDefinition {
30 Name("T3VoiceTranscription")
@@ -66,6 +69,27 @@ class T3VoiceTranscriptionModule : Module() {
69 }
70 }
71
72 + AsyncFunction("startLive") {
73 + id: Int,
74 + uri: String,
75 + language: String,
76 + mode: Int,
77 + promise: Promise
78 + ->
79 + runOperation(id, promise, "transcription-failed") {
80 + val recorder = recorders.values.firstOrNull { it.uri == uri }
81 + ?: error("The voice recording is no longer available.")
82 + val context = requireNotNull(appContext.reactContext)
83 + MlKitSpeechClient(context, Locale.forLanguageTag(language), mode).use {
84 + recorder.transcribe(uri, it)
85 + }
86 + }
87 + }
88 +
89 + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) {
90 + defineLiveRecorder()
91 + }
92 +
93 AsyncFunction("cancel") { id: Int, promise: Promise ->
94 scope.launch {
95 val job = synchronized(lock) { operation?.takeIf { it.first == id }?.second }
@@ -74,7 +98,37 @@ class T3VoiceTranscriptionModule : Module() {
98 }
99 }
100
77 - OnDestroy { scope.cancel() }
101 + OnDestroy {
102 + recorders.values.toList().forEach { it.sharedObjectDidRelease() }
103 + scope.cancel()
104 + }
105 + }
106 +
107 + private fun ModuleDefinitionBuilder.defineLiveRecorder() {
108 + Class(LiveVoiceRecorder::class) {
109 + Constructor {
110 + check(Build.VERSION.SDK_INT >= Build.VERSION_CODES.S)
111 + LiveVoiceRecorder(requireNotNull(appContext.reactContext), appContext, scope) {
112 + recorders.remove(it)
113 + }.also { recorders[it.id] = it }
114 + }
115 + Property("uri") { recorder -> recorder.uri }
116 + AsyncFunction("prepareToRecordAsync") Coroutine { recorder: LiveVoiceRecorder ->
117 + withContext(Dispatchers.IO) { recorder.prepare() }
118 + }
119 + Function("record") { recorder: LiveVoiceRecorder, options: Map<String, Double> ->
120 + recorder.record(options["forDuration"] ?: 300.0)
121 + }
122 + AsyncFunction("stop") Coroutine { recorder: LiveVoiceRecorder -> recorder.stop() }
123 + Function("getStatus") { recorder: LiveVoiceRecorder -> recorder.status() }
124 + Function("getAvailableInputs") { recorder: LiveVoiceRecorder -> recorder.inputs() }
125 + AsyncFunction("setInput") Coroutine { recorder: LiveVoiceRecorder, uid: String ->
126 + recorder.setInput(uid)
127 + }
128 + Function("cancelInputSelection") { recorder: LiveVoiceRecorder ->
129 + recorder.cancelInputSelection()
130 + }
131 + }
132 }
133
134 private fun runOperation(
apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LivePcmCaptureTest.kt new
+184
@@ -0,0 +1,184 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import com.google.mlkit.genai.speechrecognition.SpeechRecognizerResponse
4 +import java.io.ByteArrayOutputStream
5 +import java.io.File
6 +import kotlinx.coroutines.CompletableDeferred
7 +import kotlinx.coroutines.CoroutineStart
8 +import kotlinx.coroutines.async
9 +import kotlinx.coroutines.awaitCancellation
10 +import kotlinx.coroutines.channels.Channel
11 +import kotlinx.coroutines.flow.flow
12 +import kotlinx.coroutines.test.runTest
13 +import org.junit.Assert.assertArrayEquals
14 +import org.junit.Assert.assertEquals
15 +import org.junit.Assert.assertFalse
16 +import org.junit.Assert.assertTrue
17 +import org.junit.Test
18 +
19 +class LivePcmCaptureTest {
20 + private class Source : PcmCaptureSource {
21 + override val capacityBytes = 32000
22 + val chunks = Channel<ByteArray>(Channel.UNLIMITED)
23 + val reading = Channel<Unit>(Channel.UNLIMITED)
24 + var released = false
25 + override fun start() = Unit
26 + override suspend fun read(buffer: ByteArray, blocking: Boolean): Int {
27 + reading.send(Unit)
28 + val chunk = if (blocking) chunks.receive() else chunks.tryReceive().getOrNull() ?: return 0
29 + chunk.copyInto(buffer)
30 + return chunk.size
31 + }
32 + override fun stop() = Unit
33 + override fun release() {
34 + released = true
35 + }
36 + }
37 +
38 + private class Output(private val gate: CompletableDeferred<Unit>? = null) : PcmOutput {
39 + val bytes = ByteArrayOutputStream()
40 + val first = CompletableDeferred<Unit>()
41 + val writing = CompletableDeferred<Unit>()
42 + val ended = CompletableDeferred<Unit>()
43 + var unread = false
44 + override suspend fun write(buffer: ByteArray, count: Int) {
45 + writing.complete(Unit)
46 + gate?.await()
47 + bytes.write(buffer, 0, count)
48 + first.complete(Unit)
49 + }
50 + override fun finish() {
51 + ended.complete(Unit)
52 + }
53 + override fun verifyConsumed() = requireCompleteAudio(!unread) { "Unread captured audio." }
54 + override fun close() = Unit
55 + }
56 +
57 + @Test fun transcribesDuringCaptureAndDrainsTheShortTailAfterStop() = runTest {
58 + val source = Source()
59 + val output = Output()
60 + val file = File.createTempFile("live-capture-test-", ".pcm")
61 + val first = ByteArray(3200) { (it % 251).toByte() }
62 + val tail = byteArrayOf(1, 2, 3, 4, 5, 6)
63 + val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
64 + try {
65 + capture.start(this, 300.0) {}
66 + source.chunks.send(first)
67 + output.first.await()
68 + assertTrue(capture.isRecording)
69 + assertFalse(output.ended.isCompleted)
70 + assertArrayEquals(first, output.bytes.toByteArray())
71 + source.reading.receive()
72 + source.reading.receive()
73 +
74 + val stopped = async(start = CoroutineStart.UNDISPATCHED) { capture.stop() }
75 + source.chunks.send(tail)
76 + stopped.await()
77 + assertTrue(source.released)
78 + assertFalse(capture.isRecording)
79 + val text = collectGoogleVoiceTranscript(
80 + livePcmResponses(capture, output) {
81 + flow {
82 + output.ended.await()
83 + emit(
84 + SpeechRecognizerResponse.FinalTextResponse("Opening. Repeat. Repeat.", emptyList())
85 + )
86 + emit(SpeechRecognizerResponse.FinalTextResponse(" 中間。", emptyList()))
87 + emit(SpeechRecognizerResponse.PartialTextResponse(" An unfinished ending"))
88 + emit(SpeechRecognizerResponse.PartialTextResponse(" End."))
89 + emit(SpeechRecognizerResponse.CompletedResponse)
90 + }
91 + }
92 + )
93 + assertEquals("Opening. Repeat. Repeat. 中間。 End.", text)
94 + assertArrayEquals(first + tail, output.bytes.toByteArray())
95 + assertArrayEquals(first + tail, file.readBytes())
96 + } finally {
97 + capture.cancel()
98 + file.delete()
99 + }
100 + }
101 +
102 + @Test fun rejectsIncompleteRecognitionAndUnreadCapturedAudio() = runTest {
103 + for (failure in listOf("early", "unread", "timeout")) {
104 + val early = failure == "early"
105 + val source = Source()
106 + val output = Output().apply { unread = failure == "unread" }
107 + val file = File.createTempFile("live-capture-test-", ".pcm")
108 + val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
109 + try {
110 + capture.start(this, if (early) 300.0 else 0.1) {}
111 + source.chunks.send(ByteArray(3200))
112 + val result = runCatching {
113 + collectGoogleVoiceTranscript(
114 + livePcmResponses(capture, output) {
115 + flow {
116 + if (!early) output.ended.await()
117 + if (failure == "timeout") awaitCancellation()
118 + emit(SpeechRecognizerResponse.FinalTextResponse("Incomplete.", emptyList()))
119 + emit(SpeechRecognizerResponse.CompletedResponse)
120 + }
121 + }
122 + )
123 + }
124 + assertEquals(
125 + "incomplete-audio",
126 + (result.exceptionOrNull() as VoiceTranscriptionException).code
127 + )
128 + assertTrue(source.released)
129 + } finally {
130 + capture.cancel()
131 + file.delete()
132 + }
133 + }
134 + }
135 +
136 + @Test fun durationLimitDrainsQueuedPcmBeforeReleasingTheMicrophone() = runTest {
137 + val source = Source()
138 + val output = Output()
139 + val file = File.createTempFile("live-capture-test-", ".pcm")
140 + val expected = ByteArray(6400) { (it % 251).toByte() } + byteArrayOf(42, 0)
141 + val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
142 + val stopped = CompletableDeferred<Unit>()
143 + try {
144 + source.chunks.send(expected.copyOfRange(0, 3200))
145 + source.chunks.send(expected.copyOfRange(3200, 6400))
146 + source.chunks.send(expected.copyOfRange(6400, expected.size))
147 + capture.start(this, 0.1) { stopped.complete(Unit) }
148 + stopped.await()
149 + assertTrue(source.released)
150 + assertEquals(expected.size.toLong(), file.length())
151 + capture.awaitFinished()
152 + assertArrayEquals(expected, output.bytes.toByteArray())
153 + } finally {
154 + capture.cancel()
155 + file.delete()
156 + }
157 + }
158 +
159 + @Test fun cancellationWaitsForCaptureAndFeederCleanup() = runTest {
160 + for (phase in listOf("prepared", "reading", "writing")) {
161 + val source = Source()
162 + val output = Output(if (phase == "writing") CompletableDeferred() else null)
163 + val file = File.createTempFile("live-capture-test-", ".pcm")
164 + val capture = LivePcmCapture(source, file, output, testScheduler.timeSource)
165 + try {
166 + if (phase != "prepared") {
167 + capture.start(this, 300.0) {}
168 + source.reading.receive()
169 + if (phase == "writing") {
170 + source.chunks.send(ByteArray(3200))
171 + output.writing.await()
172 + }
173 + }
174 + capture.cancel()
175 + assertTrue(source.released)
176 + assertTrue(output.ended.isCompleted)
177 + assertTrue(capture.isFinished)
178 + assertTrue(runCatching { capture.awaitFinished() }.isFailure)
179 + } finally {
180 + file.delete()
181 + }
182 + }
183 + }
184 +}
apps/mobile/modules/t3-voice-transcription/android/src/test/java/expo/modules/t3voicetranscription/LiveVoiceRecorderTest.kt new
+123
@@ -0,0 +1,123 @@
1 +package expo.modules.t3voicetranscription
2 +
3 +import android.content.Context
4 +import android.media.AudioRecord
5 +import android.os.ParcelFileDescriptor
6 +import com.facebook.react.bridge.BridgeReactContext
7 +import expo.modules.kotlin.AppContext
8 +import expo.modules.kotlin.ModulesProvider
9 +import expo.modules.kotlin.modules.Module
10 +import java.io.File
11 +import java.lang.ref.WeakReference
12 +import kotlinx.coroutines.CoroutineStart
13 +import kotlinx.coroutines.async
14 +import kotlinx.coroutines.channels.Channel
15 +import kotlinx.coroutines.test.runTest
16 +import org.junit.Assert.assertEquals
17 +import org.junit.Assert.assertFalse
18 +import org.junit.Assert.assertTrue
19 +import org.junit.Test
20 +import org.junit.runner.RunWith
21 +import org.robolectric.RobolectricTestRunner
22 +import org.robolectric.RuntimeEnvironment
23 +import org.robolectric.annotation.Config
24 +import org.robolectric.util.ReflectionHelpers
25 +
26 +@RunWith(RobolectricTestRunner::class)
27 +@Config(manifest = Config.NONE, sdk = [31])
28 +class LiveVoiceRecorderTest {
29 + private class Source(private val failStop: Boolean = false) : PcmCaptureSource {
30 + override val capacityBytes = 32000
31 + val chunks = Channel<ByteArray>(Channel.UNLIMITED)
32 + val reading = Channel<Unit>(Channel.UNLIMITED)
33 + override fun start() = Unit
34 + override suspend fun read(buffer: ByteArray, blocking: Boolean): Int {
35 + reading.send(Unit)
36 + val chunk = if (blocking) chunks.receive() else chunks.tryReceive().getOrNull() ?: return 0
37 + chunk.copyInto(buffer)
38 + return chunk.size
39 + }
40 + override fun stop() {
41 + check(!failStop) { "The microphone could not stop." }
42 + }
43 + override fun release() = Unit
44 + }
45 +
46 + @Test fun preparingAgainReleasesTheUnusedMicrophone() = runTest {
47 + withAppContext { context, appContext ->
48 + val recorder = LiveVoiceRecorder(context, appContext, this) {}
49 + try {
50 + recorder.prepare()
51 + val previous = ReflectionHelpers.getField<AudioRecord>(recorder, "audio")
52 + assertEquals(AudioRecord.STATE_INITIALIZED, previous.state)
53 + recorder.prepare()
54 + assertEquals(AudioRecord.STATE_UNINITIALIZED, previous.state)
55 + } finally {
56 + recorder.sharedObjectDidRelease()
57 + }
58 + }
59 + }
60 +
61 + @Test fun stoppingBeforeInferencePreservesQueuedAudioAndClosesUnusedPipes() = runTest {
62 + withAppContext { context, appContext ->
63 + for (phase in listOf("prepared", "recording", "draining", "prepared-failure")) {
64 + val started = phase == "recording" || phase == "draining"
65 + val source = Source(phase == "prepared-failure")
66 + val pipe = PcmPipe.open()
67 + val reader = ReflectionHelpers.getField<ParcelFileDescriptor.AutoCloseInputStream>(
68 + pipe,
69 + "reader"
70 + )
71 + val file = File.createTempFile("recorder-stop-test-", ".pcm")
72 + val capture = LivePcmCapture(source, file, pipe, testScheduler.timeSource)
73 + val recorder = LiveVoiceRecorder(context, appContext, this) {}
74 + ReflectionHelpers.setField(recorder, "capture", capture)
75 + ReflectionHelpers.setField(recorder, "pipe", pipe)
76 + try {
77 + assertTrue(reader.fd.valid())
78 + if (started) {
79 + capture.start(this, 300.0) {}
80 + source.reading.receive()
81 + source.chunks.send(ByteArray(3200))
82 + source.reading.receive()
83 + val stopped = async(start = CoroutineStart.UNDISPATCHED) {
84 + if (phase == "draining") capture.stop() else recorder.stop()
85 + }
86 + source.chunks.send(byteArrayOf(1, 2))
87 + stopped.await()
88 + if (phase == "draining") recorder.stop()
89 + assertFalse(capture.isFinished)
90 + } else {
91 + assertEquals(phase == "prepared-failure", runCatching { recorder.stop() }.isFailure)
92 + }
93 + assertEquals(started, reader.fd.valid())
94 + } finally {
95 + capture.cancel()
96 + recorder.sharedObjectDidRelease()
97 + file.delete()
98 + }
99 + }
100 + }
101 + }
102 +
103 + private suspend fun withAppContext(block: suspend (Context, AppContext) -> Unit) {
104 + val context = RuntimeEnvironment.getApplication()
105 + val reactContext = BridgeReactContext(context)
106 + val appContext = AppContext(
107 + object : ModulesProvider {
108 + override fun getModulesMap() = emptyMap<Class<out Module>, String?>()
109 + },
110 + expo.modules.core.ModuleRegistry(emptyList(), emptyList()),
111 + WeakReference(reactContext)
112 + )
113 + try {
114 + block(context, appContext)
115 + } finally {
116 + appContext.javaClass.declaredMethods.first {
117 + it.name.startsWith("onDestroy") && it.parameterCount == 0
118 + }
119 + .invoke(appContext)
120 + reactContext.destroy()
121 + }
122 + }
123 +}
apps/mobile/src/features/voice-input/VoiceInputProvider.tsx
+3 -2
@@ -1,5 +1,5 @@
1 import { useAtomValue } from "@effect/atom-react";
2 -import { AudioModule, RecordingPresets, type RecordingStatus } from "expo-audio";
2 +import { RecordingPresets, type RecordingStatus } from "expo-audio";
3 import { File } from "expo-file-system";
4 import { activateKeepAwakeAsync, deactivateKeepAwake } from "expo-keep-awake";
5 import { useSyncExternalStore } from "react";
@@ -14,6 +14,7 @@ import {
14 rememberMicrophones,
15 } from "../../lib/microphonePriority";
16 import { getLocalVoiceTranscriber } from "../../native/voiceTranscription";
17 +import { createVoiceRecorder } from "../../native/voiceRecorder";
18 import { mobilePreferencesAtom, updateMobilePreferencesAtom } from "../../state/preferences";
19 import { getNativeShowcaseScene } from "../showcase/nativeShowcaseScene";
20 import {
@@ -87,7 +88,7 @@ const voiceInputAtom = Atom.make((get) => {
88 void session.controller.appMovedToBackground();
89 });
90 const recorder = createLazyVoiceRecorder({
90 - create: () => new AudioModule.AudioRecorder(VOICE_RECORDING_OPTIONS),
91 + create: () => createVoiceRecorder(VOICE_RECORDING_OPTIONS),
92 onStatus: (status: RecordingStatus) => {
93 session.controller.handleRecorderStatus({
94 isFinished: status.isFinished,
apps/mobile/src/native/voiceRecorder.android.ts new
+8
@@ -0,0 +1,8 @@
1 +import { AudioModule } from "expo-audio";
2 +import { createLiveVoiceRecorder } from "./voiceTranscription.android";
3 +
4 +export function createVoiceRecorder(
5 + options: ConstructorParameters<typeof AudioModule.AudioRecorder>[0],
6 +) {
7 + return createLiveVoiceRecorder() ?? new AudioModule.AudioRecorder(options);
8 +}
apps/mobile/src/native/voiceRecorder.ts new
+7
@@ -0,0 +1,7 @@
1 +import { AudioModule } from "expo-audio";
2 +
3 +export function createVoiceRecorder(
4 + options: ConstructorParameters<typeof AudioModule.AudioRecorder>[0],
5 +) {
6 + return new AudioModule.AudioRecorder(options);
7 +}
apps/mobile/src/native/voiceTranscription.android.test.ts
+21 -1
@@ -5,6 +5,17 @@ const mocks = vi.hoisted(() => ({
5 prepare: vi.fn<(id: number) => Promise<{ locale: string; mode: number }>>(),
6 transcribe: vi.fn<(id: number, uri: string, locale: string, mode: number) => Promise<string>>(),
7 cancel: vi.fn<(id: number) => Promise<void>>(),
8 + LiveVoiceRecorder: class {
9 + readonly uri = "file:///live.pcm";
10 + },
11 +}));
12 +
13 +vi.mock("react-native", () => ({
14 + Platform: {
15 + get Version() {
16 + return 31;
17 + },
18 + },
19 }));
20
21 vi.mock("expo", () => ({
@@ -15,8 +26,17 @@ import {
26 VoiceInputController,
27 resetVoiceInputGlobalsForTests,
28 } from "../../../../packages/client-runtime/src/voice-input/controller";
29 +import { Platform } from "react-native";
30 +
31 +import { createLiveVoiceRecorder, getLocalVoiceTranscriber } from "./voiceTranscription.android";
32
19 -import { getLocalVoiceTranscriber } from "./voiceTranscription.android";
33 +it.each([
34 + [30, null],
35 + [31, "file:///live.pcm"],
36 +] as const)("offers live recording only on supported Android versions (%s)", (version, uri) => {
37 + using _version = vi.spyOn(Platform, "Version", "get").mockReturnValue(version);
38 + expect(createLiveVoiceRecorder()?.uri ?? null).toBe(uri);
39 +});
40
41 it.each([false, true])(
42 "waits for device support before offering a transcriber (%s)",
apps/mobile/src/native/voiceTranscription.android.ts
+20
@@ -1,4 +1,7 @@
1 import { requireOptionalNativeModule } from "expo";
2 +import type { RecorderState, RecordingStatus } from "expo-audio";
3 +import { Platform } from "react-native";
4 +import type { NativeVoiceRecorder } from "../features/voice-input/lazyVoiceRecorder";
5
6 import {
7 VoiceTranscriptionError,
@@ -13,6 +16,8 @@ type GoogleTranscriptionModule = {
16 prepare(id: number): Promise<PreparedGoogleTranscription>;
17 transcribe(id: number, uri: string, locale: string, mode: number): Promise<string>;
18 cancel(id: number): Promise<void>;
19 + readonly LiveVoiceRecorder?: new () => NativeVoiceRecorder<RecordingStatus, RecorderState>;
20 + startLive?(id: number, uri: string, locale: string, mode: number): Promise<string>;
21 };
22
23 const native = requireOptionalNativeModule<GoogleTranscriptionModule>("T3VoiceTranscription");
@@ -33,6 +38,12 @@ const nativeErrorMessages: Readonly<Record<string, string>> = {
38
39 let nextOperationId = 0;
40
41 +export function createLiveVoiceRecorder() {
42 + return Number(Platform.Version) >= 31 && native?.LiveVoiceRecorder
43 + ? new native.LiveVoiceRecorder()
44 + : null;
45 +}
46 +
47 export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | null> {
48 if (!native || !(await native.checkAvailability())) return null;
49 const module = native;
@@ -41,8 +52,17 @@ export async function getLocalVoiceTranscriber(): Promise<VoiceTranscriber | nul
52 const prepared = await runOperation(module, signal, "preparation-failed", (id) =>
53 module.prepare(id),
54 );
55 + const startLive = module.startLive?.bind(module);
56 return {
57 locale: prepared.locale,
58 + ...(module.LiveVoiceRecorder && startLive
59 + ? {
60 + start: (uri: string, { signal }: { signal: AbortSignal }) =>
61 + runOperation(module, signal, "transcription-failed", (id) =>
62 + startLive(id, uri, prepared.locale, prepared.mode),
63 + ),
64 + }
65 + : {}),
66 transcribe: (uri, { signal }) =>
67 runOperation(module, signal, "transcription-failed", (id) =>
68 module.transcribe(id, uri, prepared.locale, prepared.mode),
docs/user/composer.md
+2 -2
@@ -198,8 +198,8 @@ voice input isn't available. On Android, keep T3 Code open while recording and
198 transcribing.
199
200 The first use may download a speech model and needs a network connection.
201 -Later transcription works offline for that language. On Android, transcription
202 -processes the full recording at roughly its original duration. Recordings can be up to five
201 +Later transcription works offline for that language. Android processes audio while
202 +you record, then finishes the transcript after you stop. Recordings can be up to five
203 minutes long. Canceling or an audio interruption discards the recording and
204 preserves your existing draft. Navigating to another screen keeps dictation active;
205 use the floating controls to finish or cancel. While recording, the screen stays
packages/client-runtime/src/voice-input/controller.test.ts
+93
@@ -243,6 +243,13 @@ describe("VoiceInputController", () => {
243 expect(harness.recorder.record).toHaveBeenCalledWith({
244 forDuration: VOICE_RECORDING_LIMIT_SECONDS,
245 });
246 + await harness.controller.handleRecorderStatus({
247 + isFinished: true,
248 + hasError: false,
249 + error: null,
250 + url: "file:///previous-recording.m4a",
251 + });
252 + expect(harness.controller.currentState.phase).toBe("recording");
253
254 const stopping = harness.controller.stop();
255 harness.controller.handleRecorderStatus({
@@ -342,6 +349,92 @@ describe("VoiceInputController", () => {
349 expect(events).toEqual(["released", "transcribed"]);
350 });
351
352 + it("starts streaming transcription during recording and commits only after stopping", async () => {
353 + const transcript = deferred<string>();
354 + const start = vi.fn(() => transcript.promise);
355 + const transcribe = vi.fn(async () => "replayed audio");
356 + const harness = createHarness({
357 + getTranscriber: () => ({
358 + prepare: async () => ({ locale: "en-US", start, transcribe }),
359 + }),
360 + });
361 + await harness.controller.start();
362 + expect(start).toHaveBeenCalledWith("file:///voice.m4a", {
363 + signal: expect.any(AbortSignal),
364 + });
365 + expect(harness.controller.currentState.phase).toBe("recording");
366 + expect(harness.commits).toEqual([]);
367 +
368 + const stopping = harness.controller.stop();
369 + transcript.resolve("opening middle and final words");
370 + await stopping;
371 + expect(harness.commits[0]?.text).toBe("hello opening middle and final words");
372 + expect(transcribe).not.toHaveBeenCalled();
373 + expect(harness.deleted).toEqual(["file:///voice.m4a"]);
374 + });
375 +
376 + it("waits for canceled live inference before deleting its recording or starting again", async () => {
377 + const transcript = deferred<string>();
378 + const released = deferred<void>();
379 + let signal: AbortSignal | undefined;
380 + const harness = createHarness({
381 + getTranscriber: () => ({
382 + prepare: async () => ({
383 + ...preparedTranscription(),
384 + start: (_uri: string, options: { signal: AbortSignal }) => {
385 + signal = options.signal;
386 + return transcript.promise;
387 + },
388 + }),
389 + }),
390 + deleteRecording: () => released.resolve(undefined),
391 + });
392 + await harness.controller.start();
393 + expect(signal).toBeDefined();
394 + harness.controller.cancel();
395 + expect(signal?.aborted).toBe(true);
396 + const next = createHarness();
397 + const nextStart = next.controller.start();
398 + expect(next.recorder.record).not.toHaveBeenCalled();
399 +
400 + transcript.resolve("discard this");
401 + await released.promise;
402 + await nextStart;
403 + expect(harness.commits).toEqual([]);
404 + expect(next.controller.currentState.phase).toBe("recording");
405 + await next.controller.interruptRecording();
406 + });
407 +
408 + it("stops recording after a live inference failure and preserves the draft for retry", async () => {
409 + const transcript = deferred<string>();
410 + const released = deferred<void>();
411 + const harness = createHarness({
412 + getTranscriber: () => ({
413 + prepare: async () => ({ ...preparedTranscription(), start: () => transcript.promise }),
414 + }),
415 + deleteRecording: () => released.resolve(undefined),
416 + });
417 + await harness.controller.start();
418 + transcript.reject(
419 + new VoiceTranscriptionError({
420 + code: "transcription-failed",
421 + message: "Speech quota reached.",
422 + }),
423 + );
424 + await released.promise;
425 + expect(harness.controller.currentState).toMatchObject({
426 + phase: "error",
427 + error: "Speech quota reached.",
428 + errorAction: "retry",
429 + });
430 + expect(harness.recorder.stop).toHaveBeenCalled();
431 + expect(harness.commits).toEqual([]);
432 + const next = createHarness();
433 + await next.controller.start();
434 + expect(next.controller.currentState.phase).toBe("recording");
435 + await next.controller.interruptRecording();
436 + });
437 +
438 it("ignores a late transcript after the draft owner changes", async () => {
439 const transcription = deferred<string>();
440 const transcriptionEntered = deferred<void>();
packages/client-runtime/src/voice-input/controller.ts
+29 -3
@@ -194,6 +194,7 @@ export class VoiceInputController {
194 private operationToken = 0;
195 private sessionToken: VoiceInputSession | null = null;
196 private transcription: PreparedVoiceTranscription | null = null;
197 + private liveTranscription: Promise<{ text: string } | { error: unknown }> | null = null;
198 private transcriptionAbortController: AbortController | null = null;
199 private capturedDraft: VoiceDraftSnapshot | null = null;
200 private recordingUri: string | null = null;
@@ -278,6 +279,21 @@ export class VoiceInputController {
279 return;
280 }
281 this.capturedDraft = capturedDraft;
282 + if (this.transcription.start && this.recordingUri) {
283 + const start = this.transcription.start;
284 + const uri = this.recordingUri;
285 + this.liveTranscription = runTranscriptionOperation(() =>
286 + start(uri, { signal: abortController.signal }),
287 + ).then(
288 + (text) => ({ text }),
289 + (error: unknown) => {
290 + if (this.isCurrent(operationToken) && this.state.phase === "recording") {
291 + void this.discardRecording(transcriptionErrorMessage(error));
292 + }
293 + return { error };
294 + },
295 + );
296 + }
297 this.dependencies.recorder.record({ forDuration: VOICE_RECORDING_LIMIT_SECONDS });
298 this.setState({ phase: "recording", error: null, errorAction: null });
299 } catch {
@@ -341,6 +357,7 @@ export class VoiceInputController {
357
358 handleRecorderStatus(status: VoiceRecorderStatus): Promise<void> | void {
359 if (this.state.phase !== "recording") return;
360 + if (status.url && this.recordingUri && status.url !== this.recordingUri) return;
361 if (status.hasError) {
362 return this.interruptRecording(
363 status.error ?? "Voice recording was interrupted.",
@@ -402,9 +419,15 @@ export class VoiceInputController {
419 const capturedDraft = this.capturedDraft;
420 let transcript: string;
421 try {
405 - transcript = await runTranscriptionOperation(() =>
406 - transcription.transcribe(recordingUri, { signal }),
407 - );
422 + if (this.liveTranscription) {
423 + const result = await this.liveTranscription;
424 + if ("error" in result) throw result.error;
425 + transcript = result.text;
426 + } else {
427 + transcript = await runTranscriptionOperation(() =>
428 + transcription.transcribe(recordingUri, { signal }),
429 + );
430 + }
431 } catch (error) {
432 if (this.isCurrent(operationToken)) {
433 this.setError(transcriptionErrorMessage(error), "retry");
@@ -454,6 +477,7 @@ export class VoiceInputController {
477 }
478
479 private async releaseResources(): Promise<void> {
480 + if (this.liveTranscription) this.transcriptionAbortController?.abort();
481 this.rememberRecordingUri(this.recordingUri);
482 try {
483 this.rememberRecordingUri(this.dependencies.recorder.uri);
@@ -463,6 +487,8 @@ export class VoiceInputController {
487 await this.dependencies.recorder.stop();
488 } catch {}
489 }
490 + await this.liveTranscription;
491 + this.liveTranscription = null;
492 this.recordingUri = null;
493 for (const uri of this.ownedRecordingUris) {
494 try {
packages/client-runtime/src/voice-input/transcription.ts
+1
@@ -7,6 +7,7 @@ export type VoiceTranscriptionOptions = {
7
8 export type PreparedVoiceTranscription = {
9 readonly locale: string;
10 + readonly start?: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
11 readonly transcribe: (uri: string, options: VoiceTranscriptionOptions) => Promise<string>;
12 };
13
patches/expo-audio@58.0.9.patch
+3 -3
@@ -339,7 +339,7 @@ index 81819c1d771bdeec093bb35a65a5e1e1af0cf9d7..cd0c7624b4cd8a4ebf7ed3b3390cfc76
339 return map
340 }
341
342 -+internal fun recordingInputs(manager: AudioManager): List<AudioDeviceInfo> {
342 ++fun recordingInputs(manager: AudioManager): List<AudioDeviceInfo> {
343 + val inputs = manager.getDevices(AudioManager.GET_DEVICES_INPUTS).filter {
344 + recordingInputType(it.type) != null
345 + }
@@ -367,7 +367,7 @@ index 81819c1d771bdeec093bb35a65a5e1e1af0cf9d7..cd0c7624b4cd8a4ebf7ed3b3390cfc76
367 + else -> null
368 +}
369 +
370 -+internal fun recordingInputUid(device: AudioDeviceInfo, inputs: List<AudioDeviceInfo> = emptyList()): String {
370 ++fun recordingInputUid(device: AudioDeviceInfo, inputs: List<AudioDeviceInfo> = emptyList()): String {
371 + val base = recordingInputBaseUid(device)
372 + val ambiguous = !isBluetoothInput(device.type) && inputs.any {
373 + it.id != device.id && recordingInputBaseUid(it) == base
@@ -401,7 +401,7 @@ index 0000000000000000000000000000000000000000..cdba9724318cf06afd21b4df6b895b38
401 +import kotlinx.coroutines.withTimeout
402 +
403 +@RequiresApi(31)
404 -+internal class RecordingInputRoute(private val manager: AudioManager) : Closeable {
404 ++class RecordingInputRoute(private val manager: AudioManager) : Closeable {
405 + private var previousMode: Int? = null
406 +
407 + suspend fun select(input: AudioDeviceInfo) {
pnpm-lock.yaml
+3 -3
@@ -370,7 +370,7 @@ patchedDependencies:
370 alchemy@2.0.0-beta.85: 07cc8c7e14b3b9af143ddd40a059f117c27e55139fbf20bd5dde2eb30509cedd
371 app-builder-lib@26.16.1: 0fbe9ae51783f5768ebe197d12d1fdb7d477a08c92e5db2f6356bcce62613c53
372 dbus-next@0.10.2: cfff57561b0ee59b5addb3b2e6c6f20906e967507a530ab67e8db8108e520ba4
373 - expo-audio@58.0.9: b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba
373 + expo-audio@58.0.9: cb873a17fbd9aca52ddcecbde4c3d899566f7e265ab49a992963a6276b29bf52
374 expo-modules-core@58.0.22: f0b0fd100550ed76e5c25b4d13eb7c9ff62825f8f197c53edf7bf447c50c9fb7
375 expo-network@58.0.3: cd09b6d1744dde4efa081b5619ebcff38877d98f7502a51cab121629dece8237
376 expo-sharing@58.0.19: 321a86e1471e34c53f1f2937fbb4304ea3f7b7a78ad9e4d35febc2b9e73e9688
@@ -619,7 +619,7 @@ importers:
619 version: 58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)
620 expo-audio:
621 specifier: 58.0.9
622 - version: 58.0.9(patch_hash=b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba)(expo-asset@58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
622 + version: 58.0.9(patch_hash=cb873a17fbd9aca52ddcecbde4c3d899566f7e265ab49a992963a6276b29bf52)(expo-asset@58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)
623 expo-auth-session:
624 specifier: ~58.0.13
625 version: 58.0.13(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)
@@ -18132,7 +18132,7 @@ snapshots:
18132 - supports-color
18133 - typescript
18134
18135 - expo-audio@58.0.9(patch_hash=b391b150cf67acdf23a9d924b791a63f8c4c84ec7a7b18cc82f815f7933fe4ba)(expo-asset@58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18135 + expo-audio@58.0.9(patch_hash=cb873a17fbd9aca52ddcecbde4c3d899566f7e265ab49a992963a6276b29bf52)(expo-asset@58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3))(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0):
18136 dependencies:
18137 expo: 58.0.14(ecd3689b2bda1172f8915f190f5c7383)
18138 expo-asset: 58.0.16(expo@58.0.14)(react-native@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(@react-native/metro-config@0.88.0(@babel/core@7.29.7(supports-color@8.1.1))(bufferutil@4.1.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(@types/react@19.3.0)(bufferutil@4.1.0)(react@19.3.0)(supports-color@8.1.1)(utf-8-validate@6.0.6))(react@19.3.0)(supports-color@8.1.1)(typescript@6.0.3)