The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { BearerConnectionTarget } from "@t3tools/client-runtime/connection"; |
| 2 | import { VoiceInputController } from "@t3tools/client-runtime/voice-input"; |
| 3 | import { |
| 4 | DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, |
| 5 | EnvironmentId, |
| 6 | type EnvironmentSpeechStatus, |
| 7 | } from "@t3tools/contracts"; |
| 8 | import { decodeSpeechPcmRequest } from "@t3tools/shared/speech"; |
| 9 | import { expect, it, vi } from "vite-plus/test"; |
| 10 | |
| 11 | import { stubGlobal } from "../../test/render"; |
| 12 | import { resolveAppleSpeechLocale } from "./appleSpeech"; |
| 13 | import { createBrowserPcmCapture } from "./browserPcmCapture"; |
| 14 | import { |
| 15 | createAppleSpeechVoiceInputPlatform, |
| 16 | createBrowserVoiceInputPlatform, |
| 17 | } from "./browserVoiceInput"; |
| 18 | |
| 19 | vi.mock("../connection/runtime", async () => { |
| 20 | const [{ Atom }, { layerRemoteHttpClient }] = await Promise.all([ |
| 21 | import("effect/reactivity"), |
| 22 | import("@t3tools/client-runtime/rpc"), |
| 23 | ]); |
| 24 | return { |
| 25 | connectionAtomRuntime: Atom.runtime( |
| 26 | layerRemoteHttpClient((input, init) => globalThis.fetch(input, init)), |
| 27 | ), |
| 28 | }; |
| 29 | }); |
| 30 | |
| 31 | function capture() { |
| 32 | let receive: ((event: { data: Float32Array | "stopped" }) => void) | undefined; |
| 33 | const stopTrack = vi.fn(); |
| 34 | const resources = new DisposableStack(); |
| 35 | resources.use( |
| 36 | stubGlobal( |
| 37 | "AudioContext", |
| 38 | class { |
| 39 | audioWorklet = { addModule: async () => {} }; |
| 40 | createMediaStreamSource() { |
| 41 | return { connect() {} }; |
| 42 | } |
| 43 | resume = async () => {}; |
| 44 | close = async () => {}; |
| 45 | }, |
| 46 | ), |
| 47 | ); |
| 48 | resources.use( |
| 49 | stubGlobal( |
| 50 | "AudioWorkletNode", |
| 51 | class { |
| 52 | port = { |
| 53 | set onmessage(handler: typeof receive) { |
| 54 | receive = handler; |
| 55 | }, |
| 56 | postMessage(message: string) { |
| 57 | if (message === "stop") receive?.({ data: "stopped" }); |
| 58 | }, |
| 59 | close() {}, |
| 60 | }; |
| 61 | connect() {} |
| 62 | disconnect() {} |
| 63 | }, |
| 64 | ), |
| 65 | ); |
| 66 | resources.use( |
| 67 | vi.spyOn(navigator.mediaDevices, "getUserMedia").mockResolvedValue({ |
| 68 | getTracks: () => [{ stop: stopTrack, addEventListener() {} }], |
| 69 | } as unknown as MediaStream), |
| 70 | ); |
| 71 | return { |
| 72 | feed: (data: Float32Array) => receive?.({ data }), |
| 73 | stopTrack, |
| 74 | [Symbol.dispose]: () => resources.dispose(), |
| 75 | }; |
| 76 | } |
| 77 | |
| 78 | const target = new BearerConnectionTarget({ |
| 79 | environmentId: EnvironmentId.make("remote-transcription"), |
| 80 | label: "Remote transcription", |
| 81 | connectionId: "remote", |
| 82 | }); |
| 83 | const prepared = { |
| 84 | environmentId: target.environmentId, |
| 85 | label: target.label, |
| 86 | target, |
| 87 | httpBaseUrl: "https://transcription.example.test", |
| 88 | socketUrl: "wss://transcription.example.test/ws", |
| 89 | httpAuthorization: { _tag: "Bearer" as const, token: "audio-grant" }, |
| 90 | }; |
| 91 | const status = { |
| 92 | supported: true, |
| 93 | state: "ready", |
| 94 | modelId: "native-streaming", |
| 95 | model: "Native streaming", |
| 96 | size: 1, |
| 97 | supportsStreaming: true, |
| 98 | supportsBufferedStreaming: true, |
| 99 | supportsTranslation: true, |
| 100 | languages: ["en", "fr"], |
| 101 | supportsLanguageDetection: false, |
| 102 | acceleration: "cpu", |
| 103 | modelUnloadTimeout: "never", |
| 104 | gpuDevices: [], |
| 105 | benchmarkTier: "cpu", |
| 106 | } satisfies EnvironmentSpeechStatus; |
| 107 | |
| 108 | it("retains all PCM until stop, retries batch audio, and captures preferences before status resolves", async () => { |
| 109 | using microphone = capture(); |
| 110 | let streaming = false; |
| 111 | let options = { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "fr" }; |
| 112 | const requests: Array<{ pcm: Uint8Array; options: typeof options }> = []; |
| 113 | let resolveStatus!: (response: Response) => void; |
| 114 | const pendingStatus = new Promise<Response>((resolve) => { |
| 115 | resolveStatus = resolve; |
| 116 | }); |
| 117 | const fetch = globalThis.fetch; |
| 118 | using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { |
| 119 | const request = new Request(input, init); |
| 120 | if (request.url === `${prepared.httpBaseUrl}/api/voice/status`) |
| 121 | return (await pendingStatus).clone(); |
| 122 | if (request.url === `${prepared.httpBaseUrl}/api/voice/transcribe`) { |
| 123 | expect(request.headers.get("authorization")).toBe("Bearer audio-grant"); |
| 124 | requests.push(decodeSpeechPcmRequest(new Uint8Array(await request.arrayBuffer()))); |
| 125 | return requests.length === 1 |
| 126 | ? new Response(null, { status: 503 }) |
| 127 | : Response.json({ text: "bonjour" }); |
| 128 | } |
| 129 | return fetch(input, init); |
| 130 | }); |
| 131 | const onError = vi.fn(); |
| 132 | const platform = createBrowserVoiceInputPlatform({ |
| 133 | prepared, |
| 134 | hasAudioConsent: () => true, |
| 135 | getStreamingEnabled: () => streaming, |
| 136 | getTranscriptionOptions: () => options, |
| 137 | getMicrophoneId: () => "", |
| 138 | onLevel() {}, |
| 139 | onDurationLimit() {}, |
| 140 | onText() {}, |
| 141 | onError, |
| 142 | }); |
| 143 | const commitDraft = vi.fn(); |
| 144 | const controller = new VoiceInputController({ |
| 145 | recorder: platform.recorder, |
| 146 | retryFailedRecording: true, |
| 147 | getTranscriber: () => platform.transcriber, |
| 148 | requestPermission: async () => ({ granted: true, canAskAgain: true }), |
| 149 | configureRecording: async () => {}, |
| 150 | releaseRecording: async () => platform.cancelRecording(), |
| 151 | deleteRecording: platform.deleteRecording, |
| 152 | readDraft: () => ({ |
| 153 | ownerKey: "draft", |
| 154 | text: "", |
| 155 | selection: { start: 0, end: 0 }, |
| 156 | revision: 0, |
| 157 | }), |
| 158 | commitDraft, |
| 159 | onStateChange() {}, |
| 160 | }); |
| 161 | using _controller = { [Symbol.dispose]: () => controller.dispose() }; |
| 162 | const start = controller.start(); |
| 163 | await vi.waitFor(() => expect(_network).toHaveBeenCalled()); |
| 164 | streaming = true; |
| 165 | options = { ...options, speechLanguage: "en" }; |
| 166 | await start; |
| 167 | expect(controller.currentState.phase).toBe("recording"); |
| 168 | microphone.feed(new Float32Array([0.1, 0.2])); |
| 169 | microphone.feed(new Float32Array([0.3])); |
| 170 | expect(requests).toHaveLength(0); |
| 171 | const stop = controller.stop(); |
| 172 | await vi.waitFor(() => expect(microphone.stopTrack).toHaveBeenCalledOnce()); |
| 173 | resolveStatus(Response.json(status)); |
| 174 | await stop; |
| 175 | expect(controller.currentState.phase).toBe("error"); |
| 176 | expect(platform.recorder.uri).not.toBeNull(); |
| 177 | await controller.start(); |
| 178 | expect(controller.currentState.phase).toBe("idle"); |
| 179 | expect(commitDraft).toHaveBeenCalledWith("bonjour", { start: 7, end: 7 }); |
| 180 | expect(requests).toHaveLength(2); |
| 181 | for (const request of requests) { |
| 182 | expect([...new Float32Array(request.pcm.slice().buffer)]).toEqual([ |
| 183 | Math.fround(0.1), |
| 184 | Math.fround(0.2), |
| 185 | Math.fround(0.3), |
| 186 | ]); |
| 187 | expect(request.options.speechLanguage).toBe("fr"); |
| 188 | } |
| 189 | expect(platform.recorder.uri).toBeNull(); |
| 190 | expect(microphone.stopTrack).toHaveBeenCalledOnce(); |
| 191 | expect(onError).not.toHaveBeenCalled(); |
| 192 | streaming = false; |
| 193 | await controller.start(); |
| 194 | microphone.feed(new Float32Array([0.4])); |
| 195 | controller.cancel(); |
| 196 | await vi.waitFor(() => expect(microphone.stopTrack).toHaveBeenCalledTimes(2)); |
| 197 | expect(requests).toHaveLength(2); |
| 198 | expect(platform.recorder.uri).toBeNull(); |
| 199 | }); |
| 200 | |
| 201 | it.each([ |
| 202 | ["fr", ["en-US"], "fr"], |
| 203 | ["auto", ["nl-NL", "en-US"], "en-US"], |
| 204 | ] as const)( |
| 205 | "resolves Apple Speech language %s with Mac preferences %j", |
| 206 | (preference, system, expected) => { |
| 207 | expect(resolveAppleSpeechLocale(["en-US", "fr-FR"], preference, system)).toBe(expected); |
| 208 | }, |
| 209 | ); |
| 210 | |
| 211 | it.each(["prepare", "transcribe"] as const)( |
| 212 | "cancels Apple Speech while native %s is pending", |
| 213 | async (stage) => { |
| 214 | const entered = Promise.withResolvers<void>(); |
| 215 | const pending = Promise.withResolvers<{ ok: true; value: string }>(); |
| 216 | using _bridge = stubGlobal("desktopBridge", { |
| 217 | appleSpeech: { |
| 218 | prepare: async () => { |
| 219 | if (stage !== "prepare") return { ok: true, value: "en-US" }; |
| 220 | entered.resolve(); |
| 221 | return pending.promise; |
| 222 | }, |
| 223 | transcribe: async () => { |
| 224 | entered.resolve(); |
| 225 | return pending.promise; |
| 226 | }, |
| 227 | }, |
| 228 | }); |
| 229 | const platform = createAppleSpeechVoiceInputPlatform({ |
| 230 | hasAudioConsent: () => true, |
| 231 | getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, |
| 232 | getMicrophoneId: () => "", |
| 233 | onLevel() {}, |
| 234 | onDurationLimit() {}, |
| 235 | onError() {}, |
| 236 | }); |
| 237 | const abort = new AbortController(); |
| 238 | using _cleanup = { |
| 239 | [Symbol.dispose]: () => { |
| 240 | abort.abort(); |
| 241 | pending.resolve({ ok: true, value: "late" }); |
| 242 | platform.cancelRecording(); |
| 243 | }, |
| 244 | }; |
| 245 | const options = { signal: abort.signal }; |
| 246 | const transcription = await platform.transcriber.prepare(options); |
| 247 | const uri = URL.createObjectURL(new Blob([new Float32Array([0.5])])); |
| 248 | using _blob = { [Symbol.dispose]: () => URL.revokeObjectURL(uri) }; |
| 249 | const result = transcription.transcribe(uri, options); |
| 250 | await entered.promise; |
| 251 | abort.abort(); |
| 252 | await expect(result).rejects.toThrow(); |
| 253 | }, |
| 254 | ); |
| 255 | |
| 256 | it("streams the opening audio in order when the destination becomes ready during capture", async () => { |
| 257 | using microphone = capture(); |
| 258 | const feed = vi.fn(); |
| 259 | const finish = vi.fn(async () => "hello"); |
| 260 | const ready = Promise.withResolvers<void>(); |
| 261 | const platform = createBrowserPcmCapture( |
| 262 | { |
| 263 | getMicrophoneId: () => "", |
| 264 | onLevel() {}, |
| 265 | onDurationLimit() {}, |
| 266 | onError() {}, |
| 267 | }, |
| 268 | async () => { |
| 269 | await ready.promise; |
| 270 | return { locale: "en", streaming: { feed, finish }, transcribe: async () => "" }; |
| 271 | }, |
| 272 | ); |
| 273 | const abort = new AbortController(); |
| 274 | using _abort = { |
| 275 | [Symbol.dispose]: () => { |
| 276 | abort.abort(); |
| 277 | platform.cancelRecording(); |
| 278 | }, |
| 279 | }; |
| 280 | const transcription = await platform.transcriber.prepare({ signal: abort.signal }); |
| 281 | await platform.recorder.prepareToRecordAsync(abort.signal); |
| 282 | platform.recorder.record({ forDuration: 1 }); |
| 283 | const opening = new Float32Array([0.1]); |
| 284 | const continuing = new Float32Array([0.2]); |
| 285 | microphone.feed(opening); |
| 286 | expect(feed).not.toHaveBeenCalled(); |
| 287 | ready.resolve(); |
| 288 | await vi.waitFor(() => expect(feed).toHaveBeenCalledExactlyOnceWith(opening)); |
| 289 | microphone.feed(continuing); |
| 290 | await platform.recorder.stop(); |
| 291 | expect(feed.mock.calls).toEqual([[opening], [continuing]]); |
| 292 | expect(platform.recorder.uri).toBeNull(); |
| 293 | await expect(transcription.streaming?.finish({ signal: abort.signal })).resolves.toBe("hello"); |
| 294 | }); |
| 295 | |
| 296 | it.each([ |
| 297 | { native: true, enabled: true, buffered: true, live: true }, |
| 298 | { native: true, enabled: false, buffered: true, live: false }, |
| 299 | { native: false, enabled: false, buffered: true, live: true }, |
| 300 | { native: false, enabled: true, buffered: true, live: true }, |
| 301 | { native: false, enabled: false, buffered: false, live: false }, |
| 302 | ])( |
| 303 | "selects the destination model's processing path: %j", |
| 304 | async ({ native, enabled, buffered, live }) => { |
| 305 | const sockets: string[] = []; |
| 306 | using _socket = stubGlobal( |
| 307 | "WebSocket", |
| 308 | class extends EventTarget { |
| 309 | constructor(url: string) { |
| 310 | super(); |
| 311 | sockets.push(url); |
| 312 | } |
| 313 | close() {} |
| 314 | }, |
| 315 | ); |
| 316 | using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => { |
| 317 | const request = new Request(input, init); |
| 318 | if (request.url.endsWith("/api/voice/status")) |
| 319 | return Response.json({ |
| 320 | ...status, |
| 321 | supportsStreaming: native, |
| 322 | supportsBufferedStreaming: buffered, |
| 323 | }); |
| 324 | if (request.url.includes("/api/auth/websocket-ticket/")) |
| 325 | return Response.json({ ticket: "stream-ticket", expiresAt: "2030-01-01T00:00:00.000Z" }); |
| 326 | throw new Error(`Unexpected request: ${request.url}`); |
| 327 | }); |
| 328 | const abort = new AbortController(); |
| 329 | using _abort = { [Symbol.dispose]: () => abort.abort() }; |
| 330 | const platform = createBrowserVoiceInputPlatform({ |
| 331 | prepared, |
| 332 | hasAudioConsent: () => true, |
| 333 | getStreamingEnabled: () => enabled, |
| 334 | getTranscriptionOptions: () => ({ |
| 335 | ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, |
| 336 | speechTranslateToEnglish: true, |
| 337 | speechLanguage: "fr", |
| 338 | }), |
| 339 | getMicrophoneId: () => "", |
| 340 | onLevel() {}, |
| 341 | onDurationLimit() {}, |
| 342 | onText() {}, |
| 343 | onError() {}, |
| 344 | }); |
| 345 | const transcription = await platform.transcriber.prepare({ signal: abort.signal }); |
| 346 | await vi.waitFor(() => expect(transcription.locale).toBe(native && enabled ? "fr" : "en")); |
| 347 | expect(Boolean(transcription.streaming)).toBe(live); |
| 348 | expect(sockets).toEqual( |
| 349 | live ? ["wss://transcription.example.test/ws/voice?wsTicket=stream-ticket"] : [], |
| 350 | ); |
| 351 | }, |
| 352 | ); |