The best way to control Claude Code, Codex and more from any computer or phone

main
ts 352 lines 11.9 KB
Raw
1 import { BearerConnectionTarget } from "@t3tools/client-runtime/connection";
2 import { VoiceInputController } from "@t3tools/client-runtime/voice-input";
3 import {
4 DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
5 EnvironmentId,
6 type EnvironmentSpeechStatus,
7 } from "@t3tools/contracts";
8 import { decodeSpeechPcmRequest } from "@t3tools/shared/speech";
9 import { expect, it, vi } from "vite-plus/test";
10
11 import { stubGlobal } from "../../test/render";
12 import { resolveAppleSpeechLocale } from "./appleSpeech";
13 import { createBrowserPcmCapture } from "./browserPcmCapture";
14 import {
15 createAppleSpeechVoiceInputPlatform,
16 createBrowserVoiceInputPlatform,
17 } from "./browserVoiceInput";
18
19 vi.mock("../connection/runtime", async () => {
20 const [{ Atom }, { layerRemoteHttpClient }] = await Promise.all([
21 import("effect/reactivity"),
22 import("@t3tools/client-runtime/rpc"),
23 ]);
24 return {
25 connectionAtomRuntime: Atom.runtime(
26 layerRemoteHttpClient((input, init) => globalThis.fetch(input, init)),
27 ),
28 };
29 });
30
31 function capture() {
32 let receive: ((event: { data: Float32Array | "stopped" }) => void) | undefined;
33 const stopTrack = vi.fn();
34 const resources = new DisposableStack();
35 resources.use(
36 stubGlobal(
37 "AudioContext",
38 class {
39 audioWorklet = { addModule: async () => {} };
40 createMediaStreamSource() {
41 return { connect() {} };
42 }
43 resume = async () => {};
44 close = async () => {};
45 },
46 ),
47 );
48 resources.use(
49 stubGlobal(
50 "AudioWorkletNode",
51 class {
52 port = {
53 set onmessage(handler: typeof receive) {
54 receive = handler;
55 },
56 postMessage(message: string) {
57 if (message === "stop") receive?.({ data: "stopped" });
58 },
59 close() {},
60 };
61 connect() {}
62 disconnect() {}
63 },
64 ),
65 );
66 resources.use(
67 vi.spyOn(navigator.mediaDevices, "getUserMedia").mockResolvedValue({
68 getTracks: () => [{ stop: stopTrack, addEventListener() {} }],
69 } as unknown as MediaStream),
70 );
71 return {
72 feed: (data: Float32Array) => receive?.({ data }),
73 stopTrack,
74 [Symbol.dispose]: () => resources.dispose(),
75 };
76 }
77
78 const target = new BearerConnectionTarget({
79 environmentId: EnvironmentId.make("remote-transcription"),
80 label: "Remote transcription",
81 connectionId: "remote",
82 });
83 const prepared = {
84 environmentId: target.environmentId,
85 label: target.label,
86 target,
87 httpBaseUrl: "https://transcription.example.test",
88 socketUrl: "wss://transcription.example.test/ws",
89 httpAuthorization: { _tag: "Bearer" as const, token: "audio-grant" },
90 };
91 const status = {
92 supported: true,
93 state: "ready",
94 modelId: "native-streaming",
95 model: "Native streaming",
96 size: 1,
97 supportsStreaming: true,
98 supportsBufferedStreaming: true,
99 supportsTranslation: true,
100 languages: ["en", "fr"],
101 supportsLanguageDetection: false,
102 acceleration: "cpu",
103 modelUnloadTimeout: "never",
104 gpuDevices: [],
105 benchmarkTier: "cpu",
106 } satisfies EnvironmentSpeechStatus;
107
108 it("retains all PCM until stop, retries batch audio, and captures preferences before status resolves", async () => {
109 using microphone = capture();
110 let streaming = false;
111 let options = { ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS, speechLanguage: "fr" };
112 const requests: Array<{ pcm: Uint8Array; options: typeof options }> = [];
113 let resolveStatus!: (response: Response) => void;
114 const pendingStatus = new Promise<Response>((resolve) => {
115 resolveStatus = resolve;
116 });
117 const fetch = globalThis.fetch;
118 using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
119 const request = new Request(input, init);
120 if (request.url === `${prepared.httpBaseUrl}/api/voice/status`)
121 return (await pendingStatus).clone();
122 if (request.url === `${prepared.httpBaseUrl}/api/voice/transcribe`) {
123 expect(request.headers.get("authorization")).toBe("Bearer audio-grant");
124 requests.push(decodeSpeechPcmRequest(new Uint8Array(await request.arrayBuffer())));
125 return requests.length === 1
126 ? new Response(null, { status: 503 })
127 : Response.json({ text: "bonjour" });
128 }
129 return fetch(input, init);
130 });
131 const onError = vi.fn();
132 const platform = createBrowserVoiceInputPlatform({
133 prepared,
134 hasAudioConsent: () => true,
135 getStreamingEnabled: () => streaming,
136 getTranscriptionOptions: () => options,
137 getMicrophoneId: () => "",
138 onLevel() {},
139 onDurationLimit() {},
140 onText() {},
141 onError,
142 });
143 const commitDraft = vi.fn();
144 const controller = new VoiceInputController({
145 recorder: platform.recorder,
146 retryFailedRecording: true,
147 getTranscriber: () => platform.transcriber,
148 requestPermission: async () => ({ granted: true, canAskAgain: true }),
149 configureRecording: async () => {},
150 releaseRecording: async () => platform.cancelRecording(),
151 deleteRecording: platform.deleteRecording,
152 readDraft: () => ({
153 ownerKey: "draft",
154 text: "",
155 selection: { start: 0, end: 0 },
156 revision: 0,
157 }),
158 commitDraft,
159 onStateChange() {},
160 });
161 using _controller = { [Symbol.dispose]: () => controller.dispose() };
162 const start = controller.start();
163 await vi.waitFor(() => expect(_network).toHaveBeenCalled());
164 streaming = true;
165 options = { ...options, speechLanguage: "en" };
166 await start;
167 expect(controller.currentState.phase).toBe("recording");
168 microphone.feed(new Float32Array([0.1, 0.2]));
169 microphone.feed(new Float32Array([0.3]));
170 expect(requests).toHaveLength(0);
171 const stop = controller.stop();
172 await vi.waitFor(() => expect(microphone.stopTrack).toHaveBeenCalledOnce());
173 resolveStatus(Response.json(status));
174 await stop;
175 expect(controller.currentState.phase).toBe("error");
176 expect(platform.recorder.uri).not.toBeNull();
177 await controller.start();
178 expect(controller.currentState.phase).toBe("idle");
179 expect(commitDraft).toHaveBeenCalledWith("bonjour", { start: 7, end: 7 });
180 expect(requests).toHaveLength(2);
181 for (const request of requests) {
182 expect([...new Float32Array(request.pcm.slice().buffer)]).toEqual([
183 Math.fround(0.1),
184 Math.fround(0.2),
185 Math.fround(0.3),
186 ]);
187 expect(request.options.speechLanguage).toBe("fr");
188 }
189 expect(platform.recorder.uri).toBeNull();
190 expect(microphone.stopTrack).toHaveBeenCalledOnce();
191 expect(onError).not.toHaveBeenCalled();
192 streaming = false;
193 await controller.start();
194 microphone.feed(new Float32Array([0.4]));
195 controller.cancel();
196 await vi.waitFor(() => expect(microphone.stopTrack).toHaveBeenCalledTimes(2));
197 expect(requests).toHaveLength(2);
198 expect(platform.recorder.uri).toBeNull();
199 });
200
201 it.each([
202 ["fr", ["en-US"], "fr"],
203 ["auto", ["nl-NL", "en-US"], "en-US"],
204 ] as const)(
205 "resolves Apple Speech language %s with Mac preferences %j",
206 (preference, system, expected) => {
207 expect(resolveAppleSpeechLocale(["en-US", "fr-FR"], preference, system)).toBe(expected);
208 },
209 );
210
211 it.each(["prepare", "transcribe"] as const)(
212 "cancels Apple Speech while native %s is pending",
213 async (stage) => {
214 const entered = Promise.withResolvers<void>();
215 const pending = Promise.withResolvers<{ ok: true; value: string }>();
216 using _bridge = stubGlobal("desktopBridge", {
217 appleSpeech: {
218 prepare: async () => {
219 if (stage !== "prepare") return { ok: true, value: "en-US" };
220 entered.resolve();
221 return pending.promise;
222 },
223 transcribe: async () => {
224 entered.resolve();
225 return pending.promise;
226 },
227 },
228 });
229 const platform = createAppleSpeechVoiceInputPlatform({
230 hasAudioConsent: () => true,
231 getTranscriptionOptions: () => DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
232 getMicrophoneId: () => "",
233 onLevel() {},
234 onDurationLimit() {},
235 onError() {},
236 });
237 const abort = new AbortController();
238 using _cleanup = {
239 [Symbol.dispose]: () => {
240 abort.abort();
241 pending.resolve({ ok: true, value: "late" });
242 platform.cancelRecording();
243 },
244 };
245 const options = { signal: abort.signal };
246 const transcription = await platform.transcriber.prepare(options);
247 const uri = URL.createObjectURL(new Blob([new Float32Array([0.5])]));
248 using _blob = { [Symbol.dispose]: () => URL.revokeObjectURL(uri) };
249 const result = transcription.transcribe(uri, options);
250 await entered.promise;
251 abort.abort();
252 await expect(result).rejects.toThrow();
253 },
254 );
255
256 it("streams the opening audio in order when the destination becomes ready during capture", async () => {
257 using microphone = capture();
258 const feed = vi.fn();
259 const finish = vi.fn(async () => "hello");
260 const ready = Promise.withResolvers<void>();
261 const platform = createBrowserPcmCapture(
262 {
263 getMicrophoneId: () => "",
264 onLevel() {},
265 onDurationLimit() {},
266 onError() {},
267 },
268 async () => {
269 await ready.promise;
270 return { locale: "en", streaming: { feed, finish }, transcribe: async () => "" };
271 },
272 );
273 const abort = new AbortController();
274 using _abort = {
275 [Symbol.dispose]: () => {
276 abort.abort();
277 platform.cancelRecording();
278 },
279 };
280 const transcription = await platform.transcriber.prepare({ signal: abort.signal });
281 await platform.recorder.prepareToRecordAsync(abort.signal);
282 platform.recorder.record({ forDuration: 1 });
283 const opening = new Float32Array([0.1]);
284 const continuing = new Float32Array([0.2]);
285 microphone.feed(opening);
286 expect(feed).not.toHaveBeenCalled();
287 ready.resolve();
288 await vi.waitFor(() => expect(feed).toHaveBeenCalledExactlyOnceWith(opening));
289 microphone.feed(continuing);
290 await platform.recorder.stop();
291 expect(feed.mock.calls).toEqual([[opening], [continuing]]);
292 expect(platform.recorder.uri).toBeNull();
293 await expect(transcription.streaming?.finish({ signal: abort.signal })).resolves.toBe("hello");
294 });
295
296 it.each([
297 { native: true, enabled: true, buffered: true, live: true },
298 { native: true, enabled: false, buffered: true, live: false },
299 { native: false, enabled: false, buffered: true, live: true },
300 { native: false, enabled: true, buffered: true, live: true },
301 { native: false, enabled: false, buffered: false, live: false },
302 ])(
303 "selects the destination model's processing path: %j",
304 async ({ native, enabled, buffered, live }) => {
305 const sockets: string[] = [];
306 using _socket = stubGlobal(
307 "WebSocket",
308 class extends EventTarget {
309 constructor(url: string) {
310 super();
311 sockets.push(url);
312 }
313 close() {}
314 },
315 );
316 using _network = vi.spyOn(globalThis, "fetch").mockImplementation(async (input, init) => {
317 const request = new Request(input, init);
318 if (request.url.endsWith("/api/voice/status"))
319 return Response.json({
320 ...status,
321 supportsStreaming: native,
322 supportsBufferedStreaming: buffered,
323 });
324 if (request.url.includes("/api/auth/websocket-ticket/"))
325 return Response.json({ ticket: "stream-ticket", expiresAt: "2030-01-01T00:00:00.000Z" });
326 throw new Error(`Unexpected request: ${request.url}`);
327 });
328 const abort = new AbortController();
329 using _abort = { [Symbol.dispose]: () => abort.abort() };
330 const platform = createBrowserVoiceInputPlatform({
331 prepared,
332 hasAudioConsent: () => true,
333 getStreamingEnabled: () => enabled,
334 getTranscriptionOptions: () => ({
335 ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
336 speechTranslateToEnglish: true,
337 speechLanguage: "fr",
338 }),
339 getMicrophoneId: () => "",
340 onLevel() {},
341 onDurationLimit() {},
342 onText() {},
343 onError() {},
344 });
345 const transcription = await platform.transcriber.prepare({ signal: abort.signal });
346 await vi.waitFor(() => expect(transcription.locale).toBe(native && enabled ? "fr" : "en"));
347 expect(Boolean(transcription.streaming)).toBe(live);
348 expect(sockets).toEqual(
349 live ? ["wss://transcription.example.test/ws/voice?wsTicket=stream-ticket"] : [],
350 );
351 },
352 );