The best way to control Claude Code, Codex and more from any computer or phone

main
tsx 989 lines 36.4 KB
Raw
1 import { useLayoutEffect } from "react";
2 import { expect, it, vi } from "vite-plus/test";
3 import { page, userEvent } from "vite-plus/test/browser";
4 import * as Deferred from "effect/Deferred";
5 import * as Effect from "effect/Effect";
6 import * as Option from "effect/Option";
7 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
8 import { getEnvironmentSpeechModels } from "@t3tools/client-runtime/voice-input";
9 import {
10 DEFAULT_CLIENT_SETTINGS,
11 DEFAULT_SERVER_SETTINGS,
12 EnvironmentId,
13 type EnvironmentSpeechModel,
14 type ProjectId,
15 type SpeechCustomWords,
16 type SpeechPostProcessingOptions,
17 type SpeechTranscriptionOptions,
18 } from "@t3tools/contracts";
19
20 import { act, render, stubGlobal } from "../../test/render";
21 import { AppRouteModal } from "../components/AppRouteModal";
22 import { GlobalVoiceInputIndicator } from "../components/chat/GlobalVoiceInputIndicator";
23 import { VoiceInputSetup } from "../components/chat/VoiceInputSetup";
24 import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
25 import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
26 import {
27 chooseAudioDestination,
28 chooseTranscriptCleanup,
29 bindAudioGrant,
30 pendingCloudAudioChoice,
31 } from "./voiceConsent";
32 import { writeBrowserClientSettings } from "../clientPersistenceStorage";
33
34 const mocks = vi.hoisted(() => ({
35 voice: undefined as unknown as ReturnType<typeof useEnvironmentSpeechInput>,
36 settings: {} as Record<string, unknown>,
37 connected: new Set<string>(),
38 voiceCapable: new Set<string>(),
39 cloudVoiceCapable: new Set<string>(),
40 canDictate: new Set<string>(),
41 prepared: new Map<string, unknown>(),
42 primaryEnvironmentId: "primary-env",
43 statusRequests: [] as string[],
44 statusUnavailable: false,
45 statusTier: "accelerated" as "cpu" | "accelerated",
46 modelState: "ready" as "ready" | "missing-model",
47 statusModelId: "test-model",
48 models: [] as EnvironmentSpeechModel[],
49 catalogOffline: false,
50 download: vi.fn<(modelId: string) => Promise<boolean | void>>(),
51 selected: [] as string[],
52 selectionFails: false,
53 platforms: [] as string[],
54 microphoneRequests: [] as string[],
55 transcribeCalls: [] as string[],
56 cleanupCalls: [] as string[],
57 cleanupDraft: vi.fn(),
58 cleanupOptions: [] as SpeechPostProcessingOptions[],
59 recordingOptions: [] as SpeechTranscriptionOptions[],
60 projectWords: {} as Record<string, SpeechCustomWords>,
61 transcripts: [] as Array<string | Error>,
62 committed: vi.fn(),
63 draft: "",
64 appleSpeech: false,
65 storedDraft: "",
66 }));
67
68 vi.mock("../components/sidebar/mainAppLocation", () => ({
69 useCloseAppModal: () => () => Promise.resolve(),
70 }));
71 vi.mock("../state/session", () => ({
72 useEnvironmentScope: (id: string) => mocks.canDictate.has(id),
73 usePreparedConnection: (environmentId: string | null) => {
74 if (!environmentId || !mocks.connected.has(environmentId)) return Option.none();
75 let prepared = mocks.prepared.get(environmentId);
76 if (!prepared) {
77 prepared = { environmentId, label: environmentId };
78 mocks.prepared.set(environmentId, prepared);
79 }
80 return Option.some(prepared);
81 },
82 }));
83 vi.mock("../state/environments", () => ({
84 usePrimaryEnvironmentId: () => mocks.primaryEnvironmentId,
85 useEnvironment: (environmentId: string | null) =>
86 environmentId && mocks.connected.has(environmentId)
87 ? {
88 environmentId,
89 serverConfig: {
90 environment: {
91 capabilities: {
92 voiceTranscription: mocks.voiceCapable.has(environmentId),
93 cloudVoiceTranscription: mocks.cloudVoiceCapable.has(environmentId),
94 },
95 },
96 },
97 }
98 : null,
99 }));
100 vi.mock("../hooks/useSettings", () => ({
101 updateClientSettings: vi.fn(),
102 getClientSettings: () => ({ ...DEFAULT_CLIENT_SETTINGS, ...mocks.settings }),
103 isClientSettingsHydrated: () => true,
104 useClientSettingsHydrated: () => true,
105 useClientSettings: (selector?: (settings: Record<string, unknown>) => unknown) => {
106 const settings = { ...DEFAULT_CLIENT_SETTINGS, ...mocks.settings };
107 return selector ? selector(settings) : settings;
108 },
109 useEnvironmentSettings: (
110 environmentId: string,
111 selector: (settings: Record<string, unknown>) => unknown,
112 ) =>
113 selector({
114 ...DEFAULT_SERVER_SETTINGS,
115 projectSettingsOverrides: Object.fromEntries(
116 Object.entries(environmentId === THREAD_ENV ? mocks.projectWords : {}).map(
117 ([projectId, words]) => [projectId, { speechProjectCustomWords: words }],
118 ),
119 ),
120 }),
121 }));
122 vi.mock("./runEnvironmentSpeech", () => ({ runEnvironmentSpeech: Effect.runPromise }));
123 vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } }));
124 vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
125 ...(await importOriginal<typeof import("@t3tools/client-runtime/voice-input")>()),
126 getEnvironmentSpeechStatus: (prepared: PreparedConnection) =>
127 Effect.sync(() => {
128 mocks.statusRequests.push(prepared.environmentId);
129 if (mocks.statusUnavailable) throw new Error("Temporarily offline");
130 return {
131 supported: true,
132 state: mocks.modelState,
133 modelId: mocks.statusModelId,
134 model: "Test",
135 size: 1,
136 supportsStreaming: false,
137 supportsTranslation: false,
138 acceleration: "auto",
139 benchmarkTier: mocks.statusTier,
140 modelUnloadTimeout: "min_15",
141 gpuDevices: [],
142 };
143 }),
144 downloadEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
145 Effect.promise(async () => {
146 const cancelled = await mocks.download(modelId);
147 return { models: mocks.models, cancelled: cancelled ?? false };
148 }),
149 selectEnvironmentSpeechModel: (_prepared: PreparedConnection, modelId: string) =>
150 Effect.sync(() => {
151 mocks.selected.push(modelId);
152 if (mocks.selectionFails) throw new Error("selection failed");
153 mocks.statusModelId = modelId;
154 return { models: mocks.models };
155 }),
156 getEnvironmentSpeechModels: vi.fn((_prepared: PreparedConnection, includeBenchmarks = false) =>
157 Effect.sync(() => {
158 if (mocks.catalogOffline) throw new Error("Catalog offline");
159 return {
160 models: mocks.models.map(({ benchmark, ...model }) =>
161 includeBenchmarks ? { ...model, ...(benchmark ? { benchmark } : {}) } : model,
162 ),
163 };
164 }),
165 ),
166 postProcessEnvironmentTranscript: (
167 prepared: PreparedConnection,
168 transcript: string,
169 draft: unknown,
170 options: SpeechPostProcessingOptions,
171 ) =>
172 Effect.sync(() => {
173 mocks.cleanupCalls.push(prepared.environmentId);
174 mocks.cleanupDraft(draft);
175 mocks.cleanupOptions.push(options);
176 return { text: `Cleaned ${transcript}` };
177 }),
178 }));
179 vi.mock("./appleSpeech", () => ({
180 useAppleSpeechLocales: () => (mocks.appleSpeech ? ["en-US"] : []),
181 }));
182 vi.mock("./browserVoiceInput", () => ({
183 createAppleSpeechVoiceInputPlatform: (
184 input: Omit<Parameters<typeof createBrowserVoiceInputPlatform>[0], "prepared">,
185 ) => mockPlatform("apple-speech", input),
186 createBrowserVoiceInputPlatform: (input: Parameters<typeof createBrowserVoiceInputPlatform>[0]) =>
187 mockPlatform(input.prepared.environmentId, input),
188 }));
189
190 function mockPlatform(
191 environmentId: string,
192 input: Omit<Parameters<typeof createBrowserVoiceInputPlatform>[0], "prepared">,
193 ) {
194 mocks.platforms.push(environmentId);
195 return {
196 recorder: {
197 uri: `blob:${environmentId}`,
198 prepareToRecordAsync: async () => {
199 mocks.microphoneRequests.push(environmentId);
200 },
201 record() {},
202 stop: async () => {},
203 },
204 transcriber: {
205 prepare: async () => {
206 mocks.recordingOptions.push(input.getTranscriptionOptions());
207 return {
208 locale: "en",
209 transcribe: async () => {
210 mocks.transcribeCalls.push(
211 input.cloudProvider
212 ? `${[input.cloudProvider, (input as { cloudModel?: string }).cloudModel].filter(Boolean).join(":")} via ${environmentId}`
213 : environmentId,
214 );
215 const next = mocks.transcripts.shift() ?? "dictated";
216 if (next instanceof Error) throw next;
217 return next;
218 },
219 };
220 },
221 },
222 cancelRecording() {},
223 deleteRecording() {},
224 };
225 }
226
227 const THREAD_ENV = "thread-env";
228
229 function setup(options: { readonly destination?: string | null } = {}) {
230 mocks.settings = {};
231 setDestination(options.destination ?? null);
232 if (!options.destination) localStorage.removeItem("t3code:voice-audio-grant:v1");
233 chooseTranscriptCleanup(false);
234 mocks.primaryEnvironmentId = "primary-env";
235 mocks.connected = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
236 mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
237 mocks.cloudVoiceCapable.clear();
238 mocks.canDictate = new Set(mocks.voiceCapable);
239 mocks.prepared.clear();
240 mocks.statusRequests = [];
241 mocks.statusUnavailable = false;
242 mocks.modelState = "ready";
243 mocks.statusTier = "accelerated";
244 mocks.statusModelId = "test-model";
245 mocks.models = [];
246 mocks.catalogOffline = false;
247 mocks.download.mockReset();
248 mocks.selected = [];
249 mocks.selectionFails = false;
250 mocks.platforms = [];
251 mocks.microphoneRequests = [];
252 mocks.transcribeCalls = [];
253 mocks.cleanupCalls = [];
254 mocks.cleanupDraft.mockClear();
255 mocks.cleanupOptions = [];
256 mocks.recordingOptions = [];
257 mocks.projectWords = {};
258 mocks.transcripts = [];
259 mocks.committed.mockClear();
260 mocks.draft = "";
261 mocks.appleSpeech = false;
262 mocks.storedDraft = "";
263 }
264
265 function setDestination(environmentId: string | null) {
266 chooseAudioDestination(
267 environmentId ? { kind: "environment", environmentId: environmentId as EnvironmentId } : null,
268 );
269 }
270
271 function Probe(props: {
272 readonly projectId?: string;
273 readonly projectName?: string;
274 readonly cleanup?: boolean;
275 readonly showWizard?: boolean;
276 readonly ownerKey?: string;
277 readonly persistent?: boolean;
278 }) {
279 const value = useEnvironmentSpeechInput({
280 environmentId: THREAD_ENV as EnvironmentId,
281 projectId: props.projectId as ProjectId | undefined,
282 projectName: props.projectName,
283 cleanup: props.cleanup,
284 ownerKey: props.ownerKey ?? "draft",
285 persistentDraft: props.persistent
286 ? {
287 readStoredText: () => mocks.storedDraft,
288 commitStoredText: (text) => {
289 mocks.storedDraft = text;
290 },
291 open: () => {},
292 }
293 : undefined,
294 readDraft: () => ({
295 text: mocks.draft,
296 selection: { start: mocks.draft.length, end: mocks.draft.length },
297 }),
298 commitDraft: mocks.committed,
299 });
300 useLayoutEffect(() => {
301 mocks.voice = value;
302 });
303 return props.showWizard ? (
304 <VoiceInputSetup
305 {...value.setup}
306 environmentId={value.transcriptionEnvironmentId}
307 status={value.status}
308 onOpenChange={value.setup.setOpen}
309 onRetryCatalog={value.setup.retryCatalog}
310 onDownload={value.setup.download}
311 onCancelDownload={value.setup.cancelDownload}
312 onStartRecording={value.setup.startRecording}
313 />
314 ) : null;
315 }
316
317 function expectNoAudioSent() {
318 expect(mocks.platforms).toEqual([]);
319 expect(mocks.microphoneRequests).toEqual([]);
320 expect(mocks.transcribeCalls).toEqual([]);
321 expect(mocks.statusRequests).toEqual([]);
322 }
323
324 it.each([
325 ["nothing is configured", {}],
326 // A transcription host picked by an earlier PoC build is not consent.
327 ["a legacy transcription host is configured", { voiceTranscriptionEnvironmentId: "env-a" }],
328 ])("opens destination setup without sending audio when %s", async (_, legacy) => {
329 setup();
330 mocks.settings = { ...legacy, speechSilenceFiltering: true };
331 await using screen = await render(<Probe />);
332 // The composer mic and the dictation shortcut both gate on this.
333 expect(mocks.voice.available).toBe(true);
334 await act(() => mocks.voice.start());
335 expect(screen.router?.state.location).toMatchObject({
336 pathname: "/settings/voice",
337 hash: "voice-transcription-destination",
338 });
339 expectNoAudioSent();
340 expect(mocks.committed).not.toHaveBeenCalled();
341 expect(mocks.voice.state.phase).toBe("idle");
342 expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
343 expect(localStorage.getItem("t3code:voice-audio-grant:v1")).toBeNull();
344 });
345
346 it.each([true, false])(
347 "sends audio with silence filtering %s only to the chosen environment",
348 async (speechSilenceFiltering) => {
349 setup({ destination: "env-a" });
350 mocks.settings = { speechSilenceFiltering };
351 await using _screen = await render(<Probe />);
352 expect(mocks.voice.transcriptionEnvironmentId).toBe("env-a");
353 await act(() => mocks.voice.start());
354 expect(mocks.voice.state.phase).toBe("recording");
355 await act(() => mocks.voice.stop());
356 expect(
357 new Set([...mocks.platforms, ...mocks.microphoneRequests, ...mocks.transcribeCalls]),
358 ).toEqual(new Set(["env-a"]));
359 expect(mocks.statusRequests.every((id) => id === "env-a")).toBe(true);
360 expect(mocks.recordingOptions[0]?.speechSilenceFiltering).toBe(speechSilenceFiltering);
361 // Cleanup is a separate opt-in and stays off by default.
362 expect(mocks.cleanupCalls).toEqual([]);
363 expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything());
364 },
365 );
366
367 it.each([
368 ["is disconnected", () => mocks.connected.delete("env-a")],
369 ["cannot operate", () => mocks.canDictate.delete("env-a")],
370 ["does not support transcription", () => mocks.voiceCapable.delete("env-a")],
371 ])(
372 "does not fall back to another environment when the chosen one %s",
373 async (_, breakDestination) => {
374 setup({ destination: "env-a" });
375 breakDestination();
376 await using _screen = await render(<Probe />);
377 await act(() => mocks.voice.start());
378 expectNoAudioSent();
379 expect(mocks.voice.available).toBe(false);
380 },
381 );
382
383 it.each([
384 ["revoked", null],
385 ["moved to another environment", "env-b"],
386 ["renewed for the same environment", "env-a"],
387 ])("abandons an in-progress recording when consent is %s", async (_, next) => {
388 setup({ destination: "env-a" });
389 await using screen = await render(<Probe />);
390 await act(() => mocks.voice.start());
391 expect(mocks.voice.state.phase).toBe("recording");
392 setDestination(next);
393 await screen.rerender(<Probe />);
394 if (next === null) expect(mocks.voice.available).toBe(false);
395 expect(mocks.voice.state.phase).toBe("idle");
396 await act(() => mocks.voice.stop());
397 expect(mocks.transcribeCalls).toEqual([]);
398 expect(mocks.committed).not.toHaveBeenCalled();
399 });
400
401 it.each([true, false])(
402 "transcribes on this Mac with Apple Speech, never an environment, when available is %s",
403 async (available) => {
404 setup();
405 mocks.appleSpeech = available;
406 chooseAudioDestination({ kind: "device" });
407 await using _screen = await render(<Probe />);
408 expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
409 expect(mocks.voice.available).toBe(available);
410 await act(() => mocks.voice.start());
411 if (!available) {
412 expectNoAudioSent();
413 return;
414 }
415 await act(() => mocks.voice.stop());
416 expect(new Set([...mocks.platforms, ...mocks.transcribeCalls])).toEqual(
417 new Set(["apple-speech"]),
418 );
419 expect(mocks.statusRequests).toEqual([]);
420 expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything());
421 },
422 );
423
424 it("retries a failed transcription on the same chosen environment", async () => {
425 setup({ destination: "env-a" });
426 mocks.transcripts = [new Error("inference failed"), "second try"];
427 await using _screen = await render(<Probe />);
428 await act(() => mocks.voice.start());
429 await act(() => mocks.voice.stop());
430 expect(mocks.voice.state).toMatchObject({ phase: "error", errorAction: "retry" });
431 await act(() => mocks.voice.start());
432 if (mocks.voice.state.phase === "recording") await act(() => mocks.voice.stop());
433 expect(mocks.transcribeCalls).toEqual(["env-a", "env-a"]);
434 expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("second try", expect.anything());
435 });
436
437 it.each([
438 ["a stored thread draft keeps recording into it", true, false],
439 ["an editor-only draft is abandoned", false, false],
440 ["Apple Speech keeps recording into a stored thread draft", true, true],
441 ])("when the editor switches owner mid-recording, %s", async (_name, persistent, onDevice) => {
442 setup({ destination: "env-a" });
443 if (onDevice) {
444 mocks.appleSpeech = true;
445 chooseAudioDestination({ kind: "device" });
446 }
447 mocks.storedDraft = "Earlier text";
448 const view = (ownerKey: string) => (
449 <>
450 <Probe ownerKey={ownerKey} persistent={persistent} />
451 <GlobalVoiceInputIndicator />
452 </>
453 );
454 await using screen = await render(view("thread-a"));
455 await act(() => mocks.voice.start());
456 expect(mocks.voice.state.phase).toBe("recording");
457 // The indicator stays away while the recording's own editor is on screen.
458 expect(page.getByRole("button", { name: "Finish voice input" }).query()).toBeNull();
459
460 await screen.rerender(view("thread-b"));
461 // One microphone: thread-b cannot start its own recording while thread-a's runs.
462 expect(mocks.voice.available).toBe(!persistent);
463 if (!persistent) {
464 await expect
465 .element(page.getByRole("button", { name: "Finish voice input" }))
466 .not.toBeInTheDocument();
467 expect(mocks.transcribeCalls).toEqual([]);
468 expect(mocks.storedDraft).toBe("Earlier text");
469 return;
470 }
471 await page.getByRole("button", { name: "Finish voice input" }).click();
472
473 await vi.waitFor(() => expect(mocks.storedDraft).toBe("Earlier text dictated"));
474 expect(mocks.transcribeCalls).toEqual([onDevice ? "apple-speech" : "env-a"]);
475 expect(mocks.committed).not.toHaveBeenCalled();
476 });
477
478 it("finishes an off-screen recording with the keyboard while Settings is open", async () => {
479 setup({ destination: "env-a" });
480 mocks.storedDraft = "Earlier text";
481 await using screen = await render(
482 <>
483 <Probe ownerKey="thread-a" persistent />
484 <GlobalVoiceInputIndicator />
485 </>,
486 );
487 await act(() => mocks.voice.start());
488 await screen.rerender(
489 <>
490 <AppRouteModal label="Settings">
491 <button type="button">Settings control</button>
492 </AppRouteModal>
493 <GlobalVoiceInputIndicator hidden />
494 </>,
495 );
496 await expect.element(page.getByRole("dialog", { name: "Settings" })).toHaveFocus();
497 await userEvent.keyboard("{Shift>}{Tab}{/Shift}");
498 await expect.element(page.getByRole("button", { name: "Finish voice input" })).toHaveFocus();
499 await userEvent.keyboard("{Enter}");
500 await vi.waitFor(() => expect(mocks.storedDraft).toBe("Earlier text dictated"));
501 });
502
503 it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {
504 setup({ destination: "env-a" });
505 mocks.modelState = "missing-model";
506 mocks.settings = { speechLanguage: "en" };
507 mocks.models = [catalogModel(mocks.statusModelId, ["en"], true)];
508 let finishDownload!: () => void;
509 mocks.download.mockReturnValue(
510 new Promise<void>((resolve) => {
511 finishDownload = resolve;
512 }),
513 );
514 await using screen = await render(<Probe />);
515 await act(() => mocks.voice.start());
516 let done: Promise<void> | undefined;
517 await act(() => {
518 done = mocks.voice.setup.download();
519 });
520 expect(mocks.voice.setup.downloading).toBe(true);
521 mocks.connected.delete("env-a");
522 await screen.rerender(<Probe />);
523 expect(mocks.voice.setup.open).toBe(false);
524 mocks.connected.add("env-a");
525 await screen.rerender(<Probe />);
526 await act(() => mocks.voice.start());
527 expect(mocks.voice.setup).toMatchObject({ open: true, downloading: false, step: 0 });
528 finishDownload();
529 await act(async () => {
530 await done;
531 });
532 expect(mocks.voice.setup.error).toBeNull();
533 });
534
535 function catalogModel(
536 id: string,
537 languages: readonly string[],
538 recommended: boolean,
539 errorPercent = 4,
540 ): EnvironmentSpeechModel {
541 return {
542 id,
543 name: id,
544 description: "",
545 size: 1,
546 languages,
547 accuracy: 0,
548 speed: 0,
549 recommended,
550 benchmark: {
551 accuracy: Object.fromEntries(
552 languages.map((language) => [language, { grade: "A", errorPercent }]),
553 ),
554 secondsPer30s: { accelerated: 2, cpu: 3 },
555 },
556 supportsStreaming: false,
557 supportsLanguageDetection: languages.length > 1,
558 supportsTranslation: false,
559 active: false,
560 state: "downloadable",
561 };
562 }
563
564 it.each([
565 ["the browser prefers German", ["de-DE"], undefined, "auto", "accelerated"],
566 ["the remote environment uses CPU", ["de-DE"], undefined, "auto", "cpu"],
567 ["the desktop exposes no locale methods", ["de-DE"], {}, "auto", "accelerated"],
568 [
569 "an older preload exposes only the OS locale",
570 ["en-US"],
571 { getSystemLocale: () => "de-DE" },
572 "auto",
573 "accelerated",
574 ],
575 [
576 "the desktop OS prefers German",
577 ["en-US"],
578 { getPreferredSystemLanguages: () => ["de-DE"] },
579 "auto",
580 "accelerated",
581 ],
582 ["German is the chosen Voice language", ["en-US"], undefined, "de", "accelerated"],
583 ])(
584 "offers and downloads a German-capable model when %s",
585 async (_, languages, bridge, speechLanguage, tier) => {
586 setup({ destination: "env-a" });
587 mocks.settings = { speechLanguage };
588 mocks.statusTier = tier as "cpu" | "accelerated";
589 using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(languages);
590 using _bridge = stubGlobal("desktopBridge", bridge);
591 const englishDefault = "handy-computer/parakeet-unified-en-0.6b-gguf";
592 const german = "handy-computer/nemotron-3.5-asr-streaming-0.6b-gguf";
593 mocks.modelState = "missing-model";
594 mocks.statusModelId = englishDefault;
595 mocks.models = [
596 { ...catalogModel(englishDefault, ["en"], true), active: true },
597 catalogModel("handy-computer/canary-180m-flash-gguf", ["en", "de", "es", "fr"], true),
598 {
599 ...catalogModel(german, ["en", "de", "fr"], false, 2),
600 benchmark: {
601 accuracy: { de: { grade: "A", errorPercent: 2 } },
602 secondsPer30s: { cpu: 20, accelerated: 2 },
603 },
604 },
605 ];
606 mocks.download.mockResolvedValue(undefined);
607 await using _screen = await render(<Probe />);
608 await act(() => mocks.voice.start());
609 expect(mocks.voice.setup.open).toBe(true);
610 await act(() => mocks.voice.setup.download());
611 const expected = tier === "cpu" ? "handy-computer/canary-180m-flash-gguf" : german;
612 expect(mocks.download).toHaveBeenCalledExactlyOnceWith(expected);
613 expect(mocks.voice.setup.model?.id).toBe(expected);
614 expect(mocks.voice.setup.model?.recommended).toBe(true);
615 },
616 );
617
618 it.each(["cancel", "failed restart"] as const)(
619 "closes pending catalog setup on %s",
620 async (action) => {
621 setup({ destination: "env-a" });
622 mocks.modelState = "missing-model";
623 mocks.settings = { speechLanguage: "en" };
624 mocks.models = [catalogModel("current", ["en"], true)];
625 const response = Deferred.makeUnsafe<{ models: EnvironmentSpeechModel[] }>();
626 vi.mocked(getEnvironmentSpeechModels).mockReturnValueOnce(Deferred.await(response));
627 await using _screen = await render(<Probe showWizard />);
628 let firstStart!: Promise<void>;
629 await act(() => {
630 firstStart = mocks.voice.start();
631 });
632 await expect.element(page.getByText("Loading transcription models…")).toBeVisible();
633 if (action === "cancel") {
634 await act(() => mocks.voice.cancel());
635 } else {
636 mocks.statusUnavailable = true;
637 await act(() => mocks.voice.start());
638 mocks.statusUnavailable = false;
639 }
640 expect(mocks.voice.setup.open).toBe(false);
641
642 await act(() => mocks.voice.start());
643 Deferred.doneUnsafe(
644 response,
645 Effect.succeed({ models: [catalogModel("stale", ["en"], true)] }),
646 );
647 await act(() => firstStart);
648 await expect.element(page.getByRole("heading", { name: "current" })).toBeVisible();
649 expect(mocks.microphoneRequests).toEqual([]);
650 },
651 );
652
653 it.each([true, false])(
654 "never downloads the English default when no German recommendation is available (catalog offline: %s)",
655 async (offline) => {
656 setup({ destination: "env-a" });
657 mocks.settings = { speechLanguage: "de" };
658 mocks.modelState = "missing-model";
659 mocks.statusModelId = "english";
660 mocks.models = [catalogModel("english", ["en"], true)];
661 mocks.catalogOffline = offline;
662 await using _screen = await render(<Probe showWizard />);
663 await act(() => mocks.voice.start());
664 const wizard = page.getByRole("dialog", { name: "Set up voice input" });
665 const retry = wizard.getByRole("button", { name: "Retry loading models" });
666 const download = wizard.getByRole("button", { name: "Download model" });
667 await expect.element(wizard.getByRole("alert")).toBeVisible();
668 await expect.element(download).not.toBeInTheDocument();
669 if (offline) {
670 mocks.catalogOffline = false;
671 mocks.models.push(catalogModel("german", ["de"], true));
672 // Another client can finish setup while the catalog is offline.
673 mocks.modelState = "ready";
674 mocks.statusTier = "accelerated";
675 await userEvent.click(retry);
676 await expect.element(wizard.getByRole("heading", { name: "german" })).toBeVisible();
677 await expect.element(download).toBeEnabled();
678 }
679 await expect.element(retry).not.toBeInTheDocument();
680 expect(mocks.download).not.toHaveBeenCalled();
681 expect(mocks.microphoneRequests).toEqual([]);
682 },
683 );
684
685 it.each([
686 [
687 "selects the offered model once its download installed it",
688 "installed",
689 ["german"],
690 null,
691 false,
692 "english",
693 ],
694 [
695 "selects the downloaded model after another client changed the selection",
696 "installed",
697 ["german"],
698 null,
699 false,
700 "german",
701 ],
702 [
703 "does not advance a cancelled download when another model is ready",
704 "downloadable",
705 [],
706 "The model is not ready. Try downloading it again.",
707 true,
708 "english",
709 ],
710 [
711 "keeps setup open when an external cancellation leaves the weights installed",
712 "installed",
713 [],
714 "The model is not ready. Try downloading it again.",
715 true,
716 "english",
717 ],
718 [
719 "reports a failed selection apart from the download",
720 "installed",
721 ["german"],
722 "The model downloaded, but this environment could not select it. Try again.",
723 false,
724 "english",
725 ],
726 ] as const)("%s", async (_, landedState, selected, error, cancelled, initialModelId) => {
727 setup({ destination: "env-a" });
728 using _languages = vi.spyOn(navigator, "languages", "get").mockReturnValue(["de-DE"]);
729 mocks.selectionFails = error !== null && !cancelled;
730 mocks.modelState = "missing-model";
731 mocks.statusModelId = initialModelId;
732 mocks.models = [catalogModel("english", ["en"], true), catalogModel("german", ["de"], true)];
733 mocks.download.mockImplementation(async () => {
734 mocks.models = [
735 { ...mocks.models[0]!, state: "installed", active: true },
736 { ...mocks.models[1]!, state: landedState, active: false },
737 ];
738 mocks.statusModelId = "english";
739 mocks.modelState = "ready";
740 mocks.statusTier = "accelerated";
741 return cancelled;
742 });
743 await using _screen = await render(<Probe />);
744 await act(() => mocks.voice.start());
745 expect(mocks.voice.setup.language).toBe("de");
746 await act(() => mocks.voice.setup.download());
747 expect(mocks.selected).toEqual(selected);
748 expect(mocks.voice.setup.error).toBe(error);
749 expect(mocks.voice.setup.downloading).toBe(false);
750 expect(mocks.voice.setup.step).toBe(landedState === "installed" && !error ? 1 : 0);
751 });
752
753 it("does not resend a failed recording after consent is revoked", async () => {
754 setup({ destination: "env-a" });
755 mocks.transcripts = [new Error("inference failed")];
756 await using screen = await render(<Probe />);
757 await act(() => mocks.voice.start());
758 await act(() => mocks.voice.stop());
759 expect(mocks.voice.state.phase).toBe("error");
760 setDestination(null);
761 await screen.rerender(<Probe />);
762 await act(() => mocks.voice.start());
763 expect(mocks.transcribeCalls).toEqual(["env-a"]);
764 expect(mocks.microphoneRequests).toEqual(["env-a"]);
765 expect(mocks.committed).not.toHaveBeenCalled();
766 });
767
768 it("revokes across windows even before the storage event and survives stale preference writes", async () => {
769 setup({ destination: "env-a" });
770 await using _screen = await render(<Probe />);
771 await act(() => mocks.voice.start());
772 expect(mocks.voice.state.phase).toBe("recording");
773 const other = document.createElement("iframe");
774 document.body.append(other);
775 try {
776 const storage = other.contentWindow!.localStorage;
777 const key = "t3code:voice-audio-grant:v1";
778 const grant = JSON.parse(storage.getItem(key)!) as { id: string };
779 const canSend = bindAudioGrant(grant.id);
780 expect(canSend()).toBe(true);
781 const saved = pendingCloudAudioChoice({
782 kind: "cloud",
783 provider: "openai",
784 environmentId: EnvironmentId.make("env-a"),
785 });
786 expect(canSend()).toBe(true);
787 storage.removeItem(key);
788 saved();
789 expect(storage.getItem(key)).toBeNull();
790 expect(canSend()).toBe(false);
791 writeBrowserClientSettings({ ...DEFAULT_CLIENT_SETTINGS, wordWrap: false });
792 await vi.waitFor(() => expect(mocks.voice.transcriptionEnvironmentId).toBeNull());
793 expect(mocks.voice.state.phase).toBe("idle");
794 await act(() => mocks.voice.stop());
795 expect(mocks.transcribeCalls).toEqual([]);
796 setDestination("env-a");
797 expect(canSend()).toBe(false);
798 } finally {
799 other.remove();
800 }
801 });
802
803 it.each(["not-json", JSON.stringify("off")])(
804 "does not enable a pending provider over %s before the save completes",
805 async (previous) => {
806 setup();
807 mocks.cloudVoiceCapable.add("env-b");
808 mocks.modelState = "missing-model";
809 window.localStorage.setItem("t3code:voice-audio-grant:v1", previous);
810 const save = pendingCloudAudioChoice({
811 kind: "cloud",
812 environmentId: EnvironmentId.make("env-b"),
813 provider: "xai",
814 });
815 await using _screen = await render(<Probe />);
816 expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
817 if (previous === JSON.stringify("off")) expect(mocks.voice.available).toBe(false);
818 await act(() => mocks.voice.start());
819 expectNoAudioSent();
820 await act(save);
821 await act(() => mocks.voice.start());
822 await act(() => mocks.voice.stop());
823 expect(mocks.transcribeCalls).toEqual(["xai via env-b"]);
824 expect(mocks.statusRequests).toEqual([]);
825 },
826 );
827
828 it("sends audio to OpenRouter only under a device grant naming the exact model", async () => {
829 setup();
830 mocks.cloudVoiceCapable.add("env-b");
831 const key = "t3code:voice-audio-grant:v1";
832 const destination = { kind: "cloud", environmentId: "env-b", provider: "openrouter" };
833 window.localStorage.setItem(key, JSON.stringify({ id: "no-model", destination }));
834 {
835 await using _screen = await render(<Probe />);
836 await act(() => mocks.voice.start());
837 expectNoAudioSent();
838 }
839 const model = "acme/transcribe-1";
840 window.localStorage.setItem(
841 key,
842 JSON.stringify({ id: "model", destination: { ...destination, model } }),
843 );
844 await using _screen = await render(<Probe />);
845 await act(() => mocks.voice.start());
846 await act(() => mocks.voice.stop());
847 expect(mocks.transcribeCalls).toEqual([`openrouter:${model} via env-b`]);
848 expect(mocks.statusRequests).toEqual([]);
849 });
850
851 it("uses neither destination when the cloud save marker cannot be read", async () => {
852 setup({ destination: "env-a" });
853 const key = "t3code:voice-audio-grant:v1";
854 const previous = JSON.parse(window.localStorage.getItem(key)!) as { id: string };
855 const canSendToPrevious = bindAudioGrant(previous.id);
856 const save = pendingCloudAudioChoice({
857 kind: "cloud",
858 environmentId: EnvironmentId.make("env-b"),
859 provider: "openai",
860 });
861 save();
862 const getItem = Storage.prototype.getItem;
863 using _blocked = vi.spyOn(Storage.prototype, "getItem").mockImplementation(function (
864 this: Storage,
865 k,
866 ) {
867 if (k.startsWith(`${key}:saved:`))
868 throw new DOMException("Storage is blocked", "SecurityError");
869 return getItem.call(this, k);
870 });
871 await using _screen = await render(<Probe />);
872 expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
873 expect(canSendToPrevious()).toBe(false);
874 expect(mocks.voice.available).toBe(false);
875 await act(() => mocks.voice.start());
876 expectNoAudioSent();
877 });
878
879 it("cleans text on a cloud-only draft environment only while cleanup is separately enabled", async () => {
880 setup({ destination: "env-a" });
881 mocks.voiceCapable.delete(THREAD_ENV);
882 mocks.cloudVoiceCapable.add(THREAD_ENV);
883 chooseTranscriptCleanup(true);
884 const nearby = "x".repeat(1_024);
885 mocks.draft = "unrelated private paragraph " + nearby;
886 await using screen = await render(<Probe />);
887 await act(() => mocks.voice.start());
888 await act(() => mocks.voice.stop());
889 expect(mocks.transcribeCalls).toEqual(["env-a"]);
890 expect(mocks.cleanupCalls).toEqual([THREAD_ENV]);
891 expect(mocks.cleanupDraft).toHaveBeenCalledExactlyOnceWith({
892 text: nearby,
893 selection: { start: nearby.length, end: nearby.length },
894 });
895 expect(mocks.committed).toHaveBeenCalledExactlyOnceWith(
896 mocks.draft + " Cleaned dictated",
897 expect.anything(),
898 );
899
900 await act(() => mocks.voice.start());
901 chooseTranscriptCleanup(false);
902 await screen.rerender(<Probe />);
903 await act(() => mocks.voice.stop());
904 expect(mocks.cleanupCalls).toHaveLength(1);
905 });
906
907 // The settings model test must show what the model heard, not what a provider rewrote.
908 it.each([
909 { cleanup: false, authorized: true },
910 { cleanup: undefined, authorized: false },
911 ])(
912 "keeps a recording without authorized cleanup off the provider: $cleanup / $authorized",
913 async ({ cleanup, authorized }) => {
914 setup({ destination: "env-a" });
915 mocks.cloudVoiceCapable.add(THREAD_ENV);
916 chooseTranscriptCleanup(true);
917 mocks.settings.speechCorrectionWord = "sorry";
918 if (!authorized) mocks.canDictate.delete(THREAD_ENV);
919 mocks.projectWords = { "proj-1": [{ term: "Acme", aliases: ["ack me"] }] };
920 await using _screen = await render(
921 <Probe {...(cleanup === undefined ? {} : { cleanup })} projectId="proj-1" />,
922 );
923 await act(() => mocks.voice.start());
924 await act(() => mocks.voice.stop());
925 expect(mocks.cleanupCalls).toEqual([]);
926 expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual(mocks.projectWords["proj-1"]);
927 expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined();
928 expect(mocks.transcribeCalls).toEqual(["env-a"]);
929 expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything());
930 },
931 );
932
933 it("sends the project's vocabulary and name as hints, and the cue only with cleanup on", async () => {
934 setup({ destination: "env-a" });
935 mocks.settings.speechCustomWords = [{ term: "Effect", aliases: ["a fact"] }];
936 mocks.settings.speechCorrectionWord = "scratch that";
937 // Stored on the draft's environment under its project, never read from the audio destination.
938 mocks.projectWords = { "proj-1": [{ term: "effect", aliases: ["affect"] }] };
939 await using screen = await render(<Probe projectId="proj-1" projectName="Acme App" />);
940 await act(() => mocks.voice.start());
941 await act(() => mocks.voice.stop());
942 expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual([
943 { term: "effect", aliases: ["affect"] },
944 ]);
945 expect(mocks.recordingOptions.at(-1)?.speechHints).toEqual(["Acme App"]);
946 expect(mocks.cleanupCalls).toEqual([]);
947
948 chooseTranscriptCleanup(true);
949 await screen.rerender(<Probe projectId="proj-1" projectName="Acme App" />);
950 await act(() => mocks.voice.start());
951 await act(() => mocks.voice.stop());
952 expect(mocks.recordingOptions.at(-1)?.speechHints).toEqual(["scratch that", "Acme App"]);
953 expect(mocks.cleanupOptions.at(-1)).toMatchObject({
954 speechCorrectionWord: "scratch that",
955 speechCustomWords: [{ term: "effect", aliases: ["affect"] }],
956 });
957 expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBe("scratch that");
958 chooseTranscriptCleanup(false);
959 await act(() => mocks.voice.start());
960 await act(() => mocks.voice.stop());
961 expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined();
962 expect(mocks.recordingOptions.at(-1)?.speechHints).not.toContain("scratch that");
963 });
964
965 it("shares one status probe per destination across every mounted editor", async () => {
966 setup({ destination: "env-a" });
967 await using _screen = await render(
968 <>
969 <Probe />
970 <Probe />
971 <Probe />
972 </>,
973 );
974 expect(mocks.voice.available).toBe(true);
975 expect(mocks.statusRequests).toEqual(["env-a"]);
976 });
977
978 it("lets the user retry after a status failure without reselecting the destination", async () => {
979 setup({ destination: "env-a" });
980 mocks.statusUnavailable = true;
981 await using _screen = await render(<Probe />);
982 expect(mocks.statusRequests).toEqual(["env-a"]);
983 expect(mocks.voice.available).toBe(true);
984 expect(mocks.microphoneRequests).toEqual([]);
985 mocks.statusUnavailable = false;
986 await act(() => mocks.voice.start());
987 await act(() => mocks.voice.stop());
988 expect(mocks.transcribeCalls).toEqual(["env-a"]);
989 });