The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { |
| 2 | BearerConnectionTarget, |
| 3 | PrimaryConnectionTarget, |
| 4 | RelayConnectionTarget, |
| 5 | type PreparedConnection, |
| 6 | } from "@t3tools/client-runtime/connection"; |
| 7 | import type * as Effect from "effect/Effect"; |
| 8 | import { |
| 9 | EnvironmentId, |
| 10 | type EnvironmentSpeechModel, |
| 11 | type EnvironmentSpeechStatus, |
| 12 | } from "@t3tools/contracts"; |
| 13 | import { expect, it, vi } from "vite-plus/test"; |
| 14 | import { userEvent } from "vite-plus/test/browser"; |
| 15 | import { render } from "../../../test/render"; |
| 16 | import { __setClientSettingsForTests, getClientSettings } from "../../hooks/useSettings"; |
| 17 | import { readBrowserClientSettings } from "../../clientPersistenceStorage"; |
| 18 | import { chooseRecordingMicrophone } from "../../speech/browserMicrophones"; |
| 19 | import { |
| 20 | useEnvironmentSpeechInput, |
| 21 | useVoiceTranscriptionDestination, |
| 22 | } from "../../speech/useEnvironmentSpeechInput"; |
| 23 | import { chooseAudioDestination, chooseTranscriptCleanup } from "../../speech/voiceConsent"; |
| 24 | import { useEnvironmentScope } from "../../state/session"; |
| 25 | import { toastManager } from "../ui/toast"; |
| 26 | import { SettingsScopeProvider } from "./SettingsScopeContext"; |
| 27 | import { SpeechLanguageSelect } from "./SpeechLanguageSelect"; |
| 28 | import { VoiceSettingsPanel } from "./VoiceSettingsPanel"; |
| 29 | |
| 30 | vi.mock(import("../../speech/useEnvironmentSpeechInput"), { spy: true }); |
| 31 | vi.mock(import("../../state/session"), { spy: true }); |
| 32 | |
| 33 | // The real connection runtime authorizes relay requests with a signed-in cloud session. |
| 34 | vi.mock("../../connection/runtime", async () => { |
| 35 | const [{ ManagedRelay }, rpc, { Atom }, Context, Effect, Layer] = await Promise.all([ |
| 36 | import("@t3tools/client-runtime/relay"), |
| 37 | import("@t3tools/client-runtime/rpc"), |
| 38 | import("effect/reactivity"), |
| 39 | import("effect/Context"), |
| 40 | import("effect/Effect"), |
| 41 | import("effect/Layer"), |
| 42 | ]); |
| 43 | type AuthorizedHttp = |
| 44 | import("@t3tools/client-runtime/authorization").AuthorizedRemoteHttpEnvironment; |
| 45 | // Services resolve by key, and client-runtime does not export this one. |
| 46 | class RemoteEnvironmentAuthorization extends Context.Service< |
| 47 | RemoteEnvironmentAuthorization, |
| 48 | { |
| 49 | readonly authorizeDpopHttp: (input: { |
| 50 | readonly expectedEnvironmentId: AuthorizedHttp["environmentId"]; |
| 51 | }) => Effect.Effect<AuthorizedHttp>; |
| 52 | } |
| 53 | // @effect-diagnostics-next-line deterministicKeys:off |
| 54 | >()("@t3tools/client-runtime/authorization/service/RemoteEnvironmentAuthorization") {} |
| 55 | return { |
| 56 | connectionAtomRuntime: Atom.runtime( |
| 57 | Layer.mergeAll( |
| 58 | rpc.layerRemoteHttpClient((input, init) => globalThis.fetch(input, init)), |
| 59 | Layer.succeed(ManagedRelay.ManagedRelayDpopSigner, { |
| 60 | thumbprint: Effect.succeed("thumbprint"), |
| 61 | createProof: () => Effect.succeed("proof"), |
| 62 | }), |
| 63 | Layer.succeed(RemoteEnvironmentAuthorization, { |
| 64 | authorizeDpopHttp: ({ expectedEnvironmentId }) => |
| 65 | Effect.succeed({ |
| 66 | environmentId: expectedEnvironmentId, |
| 67 | label: "MacBook Pro", |
| 68 | httpBaseUrl: window.location.origin, |
| 69 | httpAuthorization: { |
| 70 | _tag: "Dpop", |
| 71 | accessToken: "relay-access-token", |
| 72 | expiresAtEpochMs: 4_102_444_800_000, |
| 73 | }, |
| 74 | }), |
| 75 | }), |
| 76 | ), |
| 77 | ), |
| 78 | }; |
| 79 | }); |
| 80 | |
| 81 | function deferred<T>() { |
| 82 | let resolve!: (value: T) => void; |
| 83 | const promise = new Promise<T>((done) => { |
| 84 | resolve = done; |
| 85 | }); |
| 86 | return { promise, resolve }; |
| 87 | } |
| 88 | |
| 89 | it.each([{ cpu: null, accelerated: 2.1 }, { cpu: 5, accelerated: null }, { accelerated: 2.1 }])( |
| 90 | "uses the destination environment's benchmark tier across switches with measurements %j", |
| 91 | async (secondsPer30s) => { |
| 92 | const target = new PrimaryConnectionTarget({ |
| 93 | environmentId: EnvironmentId.make("voice-local"), |
| 94 | label: "Local CPU", |
| 95 | httpBaseUrl: window.location.origin, |
| 96 | wsBaseUrl: "ws://localhost", |
| 97 | }); |
| 98 | const local: PreparedConnection = { |
| 99 | ...target, |
| 100 | target, |
| 101 | socketUrl: target.wsBaseUrl, |
| 102 | httpAuthorization: null, |
| 103 | }; |
| 104 | const remote: PreparedConnection = { |
| 105 | environmentId: EnvironmentId.make("voice-remote"), |
| 106 | label: "Remote GPU", |
| 107 | httpBaseUrl: "https://voice.example.test", |
| 108 | socketUrl: "wss://voice.example.test/ws", |
| 109 | httpAuthorization: { _tag: "Bearer", token: "test-token" }, |
| 110 | target: new BearerConnectionTarget({ |
| 111 | environmentId: EnvironmentId.make("voice-remote"), |
| 112 | label: "Remote GPU", |
| 113 | connectionId: "remote", |
| 114 | }), |
| 115 | }; |
| 116 | let prepared = local; |
| 117 | const remoteStatus = deferred<Response>(); |
| 118 | const remoteRequested = deferred<void>(); |
| 119 | const status = { |
| 120 | supported: true, |
| 121 | state: "ready", |
| 122 | modelId: "both", |
| 123 | model: "Bilingual", |
| 124 | size: 100, |
| 125 | supportsStreaming: true, |
| 126 | supportsTranslation: false, |
| 127 | acceleration: "auto", |
| 128 | modelUnloadTimeout: "never", |
| 129 | gpuDevices: [], |
| 130 | benchmarkTier: "cpu", |
| 131 | } satisfies EnvironmentSpeechStatus; |
| 132 | const model: EnvironmentSpeechModel = { |
| 133 | id: "both", |
| 134 | name: "Bilingual", |
| 135 | description: "", |
| 136 | size: 100, |
| 137 | languages: ["en"], |
| 138 | accuracy: 1, |
| 139 | speed: 1, |
| 140 | recommended: true, |
| 141 | supportsStreaming: false, |
| 142 | supportsTranslation: false, |
| 143 | supportsLanguageDetection: false, |
| 144 | active: false, |
| 145 | state: "downloadable", |
| 146 | benchmark: { |
| 147 | accuracy: { en: { grade: "A-", errorPercent: 4 } }, |
| 148 | secondsPer30s, |
| 149 | }, |
| 150 | }; |
| 151 | const cpuBenchmark = `Speed: ${secondsPer30s.cpu === undefined ? "update server for this benchmark" : secondsPer30s.cpu === null ? "not benchmarked" : `${secondsPer30s.cpu} s / 30 s`}`; |
| 152 | const gpuBenchmark = `Speed: ${secondsPer30s.accelerated === null ? "not benchmarked" : `${secondsPer30s.accelerated} s / 30 s`}`; |
| 153 | const fetch = globalThis.fetch; |
| 154 | using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { |
| 155 | const url = new URL(input instanceof Request ? input.url : input, window.location.href); |
| 156 | if (url.pathname === "/api/voice/status") { |
| 157 | if (url.origin === remote.httpBaseUrl) { |
| 158 | remoteRequested.resolve(); |
| 159 | return remoteStatus.promise; |
| 160 | } |
| 161 | return Promise.resolve(Response.json(status)); |
| 162 | } |
| 163 | if (url.pathname === "/api/voice/models") |
| 164 | return Promise.resolve(Response.json({ models: [model] })); |
| 165 | return fetch(input, init); |
| 166 | }); |
| 167 | await vi.mocked(useVoiceTranscriptionDestination).withImplementation( |
| 168 | () => ({ |
| 169 | environmentId: prepared.environmentId, |
| 170 | label: prepared.label, |
| 171 | grantId: "test", |
| 172 | prepared, |
| 173 | }), |
| 174 | async () => { |
| 175 | const previous = getClientSettings(); |
| 176 | using cleanup = new DisposableStack(); |
| 177 | cleanup.defer(() => __setClientSettingsForTests(previous)); |
| 178 | __setClientSettingsForTests({ ...previous, speechStreaming: true }); |
| 179 | const panel = () => ( |
| 180 | <SettingsScopeProvider search={{}} onChange={() => {}}> |
| 181 | <VoiceSettingsPanel /> |
| 182 | </SettingsScopeProvider> |
| 183 | ); |
| 184 | await using screen = await render(panel()); |
| 185 | await expect.element(screen.getByText(cpuBenchmark)).toBeVisible(); |
| 186 | await userEvent.click(screen.getByRole("button", { name: "Advanced", exact: true })); |
| 187 | const streaming = screen.getByRole("switch", { name: "Streaming transcription" }); |
| 188 | await expect.element(streaming).toBeEnabled(); |
| 189 | await expect.element(streaming).toBeChecked(); |
| 190 | streaming.element().focus(); |
| 191 | await userEvent.keyboard(" "); |
| 192 | await vi.waitFor(() => expect(readBrowserClientSettings()?.speechStreaming).toBe(false)); |
| 193 | prepared = remote; |
| 194 | await screen.rerender(panel()); |
| 195 | await remoteRequested.promise; |
| 196 | await expect.element(screen.getByText(cpuBenchmark)).not.toBeInTheDocument(); |
| 197 | remoteStatus.resolve( |
| 198 | Response.json({ |
| 199 | ...status, |
| 200 | supportsStreaming: false, |
| 201 | benchmarkTier: "accelerated", |
| 202 | gpuDevices: [{ id: "gpu", name: "Remote GPU" }], |
| 203 | }), |
| 204 | ); |
| 205 | await expect.element(screen.getByText(gpuBenchmark)).toBeVisible(); |
| 206 | await userEvent.click(screen.getByRole("button", { name: "Advanced", exact: true })); |
| 207 | await expect.element(streaming).toBeDisabled(); |
| 208 | await expect.element(streaming).not.toBeChecked(); |
| 209 | await expect |
| 210 | .element(screen.getByText("This model doesn’t support native streaming.")) |
| 211 | .toBeVisible(); |
| 212 | prepared = local; |
| 213 | await screen.rerender(panel()); |
| 214 | await expect.element(screen.getByText(cpuBenchmark)).toBeVisible(); |
| 215 | await userEvent.click(screen.getByRole("button", { name: "Advanced", exact: true })); |
| 216 | await expect.element(screen.getByText(gpuBenchmark)).not.toBeInTheDocument(); |
| 217 | await expect.element(streaming).toBeEnabled(); |
| 218 | await expect.element(streaming).not.toBeChecked(); |
| 219 | streaming.element().focus(); |
| 220 | await userEvent.keyboard(" "); |
| 221 | await vi.waitFor(() => expect(readBrowserClientSettings()?.speechStreaming).toBe(true)); |
| 222 | }, |
| 223 | ); |
| 224 | }, |
| 225 | ); |
| 226 | |
| 227 | it("tests the selected model after download and preserves its transcript across filters", async () => { |
| 228 | const target = new PrimaryConnectionTarget({ |
| 229 | environmentId: EnvironmentId.make("voice-filter"), |
| 230 | label: "Test environment", |
| 231 | httpBaseUrl: window.location.origin, |
| 232 | wsBaseUrl: "ws://localhost", |
| 233 | }); |
| 234 | const prepared = { ...target, target, socketUrl: target.wsBaseUrl, httpAuthorization: null }; |
| 235 | const model: EnvironmentSpeechModel = { |
| 236 | id: "tiny", |
| 237 | name: "Tiny", |
| 238 | description: "", |
| 239 | size: 100, |
| 240 | languages: ["en"], |
| 241 | accuracy: 1, |
| 242 | speed: 1, |
| 243 | recommended: false, |
| 244 | supportsStreaming: false, |
| 245 | supportsTranslation: false, |
| 246 | supportsLanguageDetection: false, |
| 247 | active: true, |
| 248 | state: "installed", |
| 249 | }; |
| 250 | const status = { |
| 251 | supported: true, |
| 252 | state: "ready", |
| 253 | modelId: model.id, |
| 254 | model: model.name, |
| 255 | size: model.size, |
| 256 | supportsStreaming: false, |
| 257 | supportsTranslation: false, |
| 258 | acceleration: "cpu", |
| 259 | benchmarkTier: "cpu", |
| 260 | modelUnloadTimeout: "never", |
| 261 | gpuDevices: [], |
| 262 | } satisfies EnvironmentSpeechStatus; |
| 263 | let installed = false; |
| 264 | const fetch = globalThis.fetch; |
| 265 | using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { |
| 266 | const url = new URL(input instanceof Request ? input.url : input, window.location.href); |
| 267 | if (url.pathname === "/api/voice/status") |
| 268 | return Promise.resolve( |
| 269 | Response.json(installed ? status : { ...status, state: "missing-model" }), |
| 270 | ); |
| 271 | if (url.pathname === "/api/voice/models/download") { |
| 272 | installed = true; |
| 273 | return Promise.resolve(Response.json({ models: [model] })); |
| 274 | } |
| 275 | if (url.pathname === "/api/voice/models") |
| 276 | return Promise.resolve( |
| 277 | Response.json({ |
| 278 | models: [ |
| 279 | installed ? model : { ...model, state: "downloadable", active: false }, |
| 280 | { |
| 281 | ...model, |
| 282 | id: "both", |
| 283 | name: "Bilingual", |
| 284 | active: false, |
| 285 | languages: ["en", "fr"], |
| 286 | ...(url.searchParams.has("benchmarks") |
| 287 | ? { |
| 288 | benchmark: { |
| 289 | accuracy: { |
| 290 | en: { grade: "A-", errorPercent: 4 }, |
| 291 | fr: { grade: "B+", errorPercent: 5.5 }, |
| 292 | }, |
| 293 | secondsPer30s: { accelerated: 2.1, cpu: 5 }, |
| 294 | }, |
| 295 | } |
| 296 | : {}), |
| 297 | }, |
| 298 | ], |
| 299 | }), |
| 300 | ); |
| 301 | return fetch(input, init); |
| 302 | }); |
| 303 | const transcript = "Keep this transcript."; |
| 304 | using _scope = vi.mocked(useEnvironmentScope).mockReturnValue(true); |
| 305 | await vi.mocked(useVoiceTranscriptionDestination).withImplementation( |
| 306 | () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }), |
| 307 | async () => { |
| 308 | await vi.mocked(useEnvironmentSpeechInput).withImplementation( |
| 309 | (input) => ({ |
| 310 | available: true, |
| 311 | status, |
| 312 | transcriptionEnvironmentId: target.environmentId, |
| 313 | transcriptionEnvironmentLabel: target.label, |
| 314 | setup: { |
| 315 | open: false, |
| 316 | step: 0, |
| 317 | model: null, |
| 318 | language: null, |
| 319 | downloading: false, |
| 320 | catalogFailed: false, |
| 321 | retryCatalog: null, |
| 322 | error: null, |
| 323 | setOpen: () => {}, |
| 324 | download: async () => {}, |
| 325 | cancelDownload: async () => {}, |
| 326 | startRecording: async () => {}, |
| 327 | }, |
| 328 | state: { phase: "idle", error: null, errorAction: null }, |
| 329 | preview: null, |
| 330 | level: 0, |
| 331 | recordingStartedAt: null, |
| 332 | preparing: false, |
| 333 | blocksSubmission: false, |
| 334 | freezesEditor: false, |
| 335 | start: async () => { |
| 336 | input.commitDraft(transcript, { start: transcript.length, end: transcript.length }); |
| 337 | }, |
| 338 | submitAfterDictation: () => {}, |
| 339 | stop: async () => {}, |
| 340 | cancel: () => {}, |
| 341 | skipPostProcessing: () => {}, |
| 342 | }), |
| 343 | async () => { |
| 344 | await using screen = await render( |
| 345 | <SettingsScopeProvider search={{}} onChange={() => {}}> |
| 346 | <VoiceSettingsPanel /> |
| 347 | </SettingsScopeProvider>, |
| 348 | ); |
| 349 | const testModel = screen.getByRole("button", { name: "Test model", exact: true }); |
| 350 | const download = screen.getByRole("button", { name: "Download", exact: true }); |
| 351 | await expect.element(download).toBeVisible(); |
| 352 | expect(testModel.query()).toBeNull(); |
| 353 | expect(screen.getByRole("button", { name: "Set up voice input" }).query()).toBeNull(); |
| 354 | await userEvent.click(download); |
| 355 | await expect.element(testModel).toBeVisible(); |
| 356 | expect(screen.getByRole("button", { name: "Set up voice input" }).query()).toBeNull(); |
| 357 | await userEvent.click(testModel); |
| 358 | const result = screen.getByRole("status", { name: "Test transcript" }); |
| 359 | await expect.element(result).toHaveTextContent(transcript); |
| 360 | |
| 361 | const search = screen.getByRole("searchbox", { name: "Search transcription models" }); |
| 362 | await userEvent.fill(search, "no matching model"); |
| 363 | await expect.element(screen.getByText("No models match.")).toBeVisible(); |
| 364 | await userEvent.clear(search); |
| 365 | const languages = screen.getByRole("button", { name: /^Filter by language/ }); |
| 366 | await userEvent.click(languages); |
| 367 | await userEvent.fill(screen.getByRole("textbox", { name: "Search languages" }), "fr"); |
| 368 | const french = screen.getByRole("menuitemcheckbox", { name: "French", exact: true }); |
| 369 | await userEvent.click(french); |
| 370 | await userEvent.keyboard("{Escape}"); |
| 371 | await expect.element(screen.getByText("Bilingual", { exact: true })).toBeVisible(); |
| 372 | await expect.element(screen.getByText("B+", { exact: true })).toBeVisible(); |
| 373 | expect(screen.container.querySelector("#transcription-test")).toHaveAttribute("hidden"); |
| 374 | await userEvent.click(languages); |
| 375 | await userEvent.fill(screen.getByRole("textbox", { name: "Search languages" }), "fr"); |
| 376 | await userEvent.click(french); |
| 377 | await userEvent.keyboard("{Escape}"); |
| 378 | await expect.element(result).toBeVisible(); |
| 379 | await vi.waitFor(() => |
| 380 | expect( |
| 381 | screen.container.querySelector('[aria-label="Test transcript"]')?.textContent, |
| 382 | ).toBe(transcript), |
| 383 | ); |
| 384 | }, |
| 385 | ); |
| 386 | }, |
| 387 | ); |
| 388 | }); |
| 389 | |
| 390 | it.each([ |
| 391 | { tier: "cpu", legacy: false }, |
| 392 | { tier: "accelerated", legacy: false }, |
| 393 | { tier: "cpu", legacy: true }, |
| 394 | ] as const)( |
| 395 | "loads relay models using the server tier $tier (legacy: $legacy)", |
| 396 | async ({ tier, legacy }) => { |
| 397 | const target = new RelayConnectionTarget({ |
| 398 | environmentId: EnvironmentId.make("voice-relay"), |
| 399 | label: "MacBook Pro", |
| 400 | }); |
| 401 | const prepared = { |
| 402 | environmentId: target.environmentId, |
| 403 | label: target.label, |
| 404 | httpBaseUrl: window.location.origin, |
| 405 | socketUrl: "ws://localhost", |
| 406 | httpAuthorization: { |
| 407 | _tag: "Dpop" as const, |
| 408 | accessToken: "stale-access-token", |
| 409 | expiresAtEpochMs: 4_102_444_800_000, |
| 410 | }, |
| 411 | target, |
| 412 | }; |
| 413 | const model = { |
| 414 | id: "gpu", |
| 415 | name: "GPU pick", |
| 416 | description: "", |
| 417 | size: 100, |
| 418 | languages: ["en"], |
| 419 | benchmark: { |
| 420 | accuracy: { en: { grade: "A", errorPercent: 4 } }, |
| 421 | secondsPer30s: { cpu: 20, accelerated: 1 }, |
| 422 | }, |
| 423 | accuracy: 1, |
| 424 | speed: 1, |
| 425 | recommended: false, |
| 426 | supportsStreaming: false, |
| 427 | supportsTranslation: false, |
| 428 | supportsLanguageDetection: false, |
| 429 | active: false, |
| 430 | state: "downloadable", |
| 431 | } satisfies EnvironmentSpeechModel; |
| 432 | const status = { |
| 433 | supported: true, |
| 434 | state: "missing-model", |
| 435 | modelId: model.id, |
| 436 | model: model.name, |
| 437 | size: model.size, |
| 438 | supportsStreaming: false, |
| 439 | supportsTranslation: false, |
| 440 | acceleration: "auto", |
| 441 | benchmarkTier: tier, |
| 442 | modelUnloadTimeout: "never", |
| 443 | gpuDevices: [{ id: "remote-gpu", name: "Remote GPU" }], |
| 444 | } satisfies EnvironmentSpeechStatus; |
| 445 | const cpuModel = { |
| 446 | ...model, |
| 447 | id: "cpu", |
| 448 | name: "CPU pick", |
| 449 | benchmark: { |
| 450 | accuracy: { en: { grade: "B+", errorPercent: 6 } }, |
| 451 | secondsPer30s: { cpu: 1, accelerated: 8 }, |
| 452 | }, |
| 453 | }; |
| 454 | const { benchmarkTier: _tier, ...legacyStatus } = status; |
| 455 | const models = [model, cpuModel]; |
| 456 | const authorizations: Array<string | null> = []; |
| 457 | const fetch = globalThis.fetch; |
| 458 | using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { |
| 459 | const url = new URL(input instanceof Request ? input.url : input, window.location.href); |
| 460 | if (url.pathname === "/api/voice/status") { |
| 461 | authorizations.push(new Headers(init?.headers).get("authorization")); |
| 462 | return Promise.resolve( |
| 463 | Response.json(legacy ? { ...legacyStatus, acceleration: "cpu", gpuDevices: [] } : status), |
| 464 | ); |
| 465 | } |
| 466 | if (url.pathname === "/api/voice/models") |
| 467 | return Promise.resolve( |
| 468 | Response.json({ |
| 469 | models: legacy |
| 470 | ? models.map(({ benchmark, ...entry }) => ({ |
| 471 | ...entry, |
| 472 | benchmark: { |
| 473 | accuracy: benchmark.accuracy, |
| 474 | secondsPer30s: benchmark.secondsPer30s.accelerated, |
| 475 | }, |
| 476 | })) |
| 477 | : models, |
| 478 | }), |
| 479 | ); |
| 480 | return fetch(input, init); |
| 481 | }); |
| 482 | using notify = vi.spyOn(toastManager, "add").mockReturnValue("test"); |
| 483 | // Restored even when an assertion fails, unlike `withImplementation`. |
| 484 | using _destination = vi.mocked(useVoiceTranscriptionDestination).mockImplementation(() => ({ |
| 485 | environmentId: target.environmentId, |
| 486 | label: target.label, |
| 487 | grantId: "test", |
| 488 | prepared, |
| 489 | })); |
| 490 | await using screen = await render( |
| 491 | <SettingsScopeProvider search={{}} onChange={() => {}}> |
| 492 | <VoiceSettingsPanel /> |
| 493 | </SettingsScopeProvider>, |
| 494 | ); |
| 495 | await vi.waitFor(() => expect(authorizations.length + notify.mock.calls.length).not.toBe(0)); |
| 496 | expect(notify).not.toHaveBeenCalled(); |
| 497 | expect(authorizations).toEqual(["DPoP relay-access-token"]); |
| 498 | const region = screen.getByRole("region", { name: "Transcription models" }); |
| 499 | await vi.waitFor(() => expect(region.element().textContent).toContain("CPU pick")); |
| 500 | if (legacy) { |
| 501 | expect(region.element().textContent).toContain("update server for this benchmark"); |
| 502 | expect(region.element().textContent).not.toContain("Recommended"); |
| 503 | } else { |
| 504 | const winner = tier === "cpu" ? "CPU pick" : "GPU pick"; |
| 505 | await vi.waitFor(() => |
| 506 | expect(region.element().firstElementChild?.textContent).toContain(winner + "Recommended"), |
| 507 | ); |
| 508 | expect(region.element().textContent).toContain("Speed: 1 s / 30 s"); |
| 509 | } |
| 510 | }, |
| 511 | ); |
| 512 | |
| 513 | it("recovers from a failed download progress poll without notifying, but reports a failed initial load", async () => { |
| 514 | const target = new PrimaryConnectionTarget({ |
| 515 | environmentId: EnvironmentId.make("voice-progress"), |
| 516 | label: "Test environment", |
| 517 | httpBaseUrl: window.location.origin, |
| 518 | wsBaseUrl: "ws://localhost", |
| 519 | }); |
| 520 | const prepared = { ...target, target, socketUrl: target.wsBaseUrl, httpAuthorization: null }; |
| 521 | const model: EnvironmentSpeechModel = { |
| 522 | id: "tiny", |
| 523 | name: "Tiny", |
| 524 | description: "", |
| 525 | size: 100, |
| 526 | languages: ["en"], |
| 527 | accuracy: 1, |
| 528 | speed: 1, |
| 529 | recommended: false, |
| 530 | supportsStreaming: false, |
| 531 | supportsTranslation: false, |
| 532 | supportsLanguageDetection: false, |
| 533 | active: false, |
| 534 | state: "downloading", |
| 535 | }; |
| 536 | const status = { |
| 537 | supported: true, |
| 538 | state: "missing-model", |
| 539 | modelId: model.id, |
| 540 | model: model.name, |
| 541 | size: model.size, |
| 542 | supportsStreaming: false, |
| 543 | supportsTranslation: false, |
| 544 | acceleration: "cpu", |
| 545 | benchmarkTier: "cpu", |
| 546 | modelUnloadTimeout: "never", |
| 547 | gpuDevices: [], |
| 548 | }; |
| 549 | const unavailable = () => new Response(null, { status: 503 }); |
| 550 | const readModels = vi |
| 551 | .fn((_withBenchmarks: boolean) => unavailable()) |
| 552 | .mockImplementationOnce(() => Response.json({ models: [model] })) |
| 553 | .mockImplementationOnce(unavailable) |
| 554 | .mockImplementationOnce(() => Response.json({ models: [{ ...model, state: "installed" }] })); |
| 555 | const fetch = globalThis.fetch; |
| 556 | using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => { |
| 557 | const url = new URL(input instanceof Request ? input.url : input, window.location.href); |
| 558 | if (url.pathname === "/api/voice/status") return Promise.resolve(Response.json(status)); |
| 559 | if (url.pathname === "/api/voice/models") |
| 560 | return Promise.resolve(readModels(url.searchParams.has("benchmarks"))); |
| 561 | return fetch(input, init); |
| 562 | }); |
| 563 | using notify = vi.spyOn(toastManager, "add").mockReturnValue("test"); |
| 564 | await vi.mocked(useVoiceTranscriptionDestination).withImplementation( |
| 565 | () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }), |
| 566 | async () => { |
| 567 | const panel = (key: string) => ( |
| 568 | <SettingsScopeProvider search={{}} onChange={() => {}}> |
| 569 | <VoiceSettingsPanel key={key} /> |
| 570 | </SettingsScopeProvider> |
| 571 | ); |
| 572 | await using screen = await render(panel("first")); |
| 573 | await expect.element(screen.getByRole("button", { name: "Use model" })).toBeVisible(); |
| 574 | expect(readModels.mock.calls.map(([withBenchmarks]) => withBenchmarks)).toEqual([ |
| 575 | true, |
| 576 | false, |
| 577 | false, |
| 578 | ]); |
| 579 | expect(notify).not.toHaveBeenCalled(); |
| 580 | await screen.rerender(panel("reopened")); |
| 581 | await vi.waitFor(() => |
| 582 | expect(notify).toHaveBeenCalledWith( |
| 583 | expect.objectContaining({ title: "Could not load transcription models" }), |
| 584 | ), |
| 585 | ); |
| 586 | }, |
| 587 | ); |
| 588 | }); |
| 589 | |
| 590 | it.each([ |
| 591 | ["speech-models", "voice-transcription-destination"], |
| 592 | ["speech-hardware", "voice-transcription-destination"], |
| 593 | ["speech-streaming", "voice-transcription-destination"], |
| 594 | ["transcription-test", "voice-transcription-destination"], |
| 595 | ["speech-correction-word", "speech-post-processing"], |
| 596 | ["speech-post-processing-model", "speech-post-processing"], |
| 597 | ])( |
| 598 | "searching for disabled %s focuses its opt-in without enabling it", |
| 599 | async (target, prerequisite) => { |
| 600 | chooseAudioDestination(null); |
| 601 | chooseTranscriptCleanup(false); |
| 602 | await using screen = await render( |
| 603 | <SettingsScopeProvider search={{}} onChange={() => {}}> |
| 604 | <VoiceSettingsPanel /> |
| 605 | </SettingsScopeProvider>, |
| 606 | { router: `/#${target}` }, |
| 607 | ); |
| 608 | await vi.waitFor(() => expect(document.activeElement?.id).toBe(prerequisite)); |
| 609 | await expect |
| 610 | .element(screen.getByRole("combobox", { name: "Transcribe with" })) |
| 611 | .toHaveTextContent("Off"); |
| 612 | await expect |
| 613 | .element(screen.getByRole("switch", { name: "Clean up transcripts" })) |
| 614 | .not.toBeChecked(); |
| 615 | }, |
| 616 | ); |
| 617 | |
| 618 | it("keeps a missing destination selected without claiming it cannot transcribe", async () => { |
| 619 | __setClientSettingsForTests(getClientSettings()); |
| 620 | using cleanup = new DisposableStack(); |
| 621 | cleanup.defer(() => chooseAudioDestination(null)); |
| 622 | chooseAudioDestination({ kind: "environment", environmentId: EnvironmentId.make("missing") }); |
| 623 | await using screen = await render( |
| 624 | <SettingsScopeProvider search={{}} onChange={() => {}}> |
| 625 | <VoiceSettingsPanel /> |
| 626 | </SettingsScopeProvider>, |
| 627 | ); |
| 628 | await expect |
| 629 | .element(screen.getByRole("combobox", { name: "Transcribe with" })) |
| 630 | .toHaveTextContent("Chosen environment (unavailable)"); |
| 631 | await expect.element(screen.getByText(/can’t transcribe audio/)).not.toBeInTheDocument(); |
| 632 | }); |
| 633 | |
| 634 | it("shows the model's effective language and restores the saved preference when supported again", async () => { |
| 635 | const previous = getClientSettings(); |
| 636 | using cleanup = new DisposableStack(); |
| 637 | cleanup.defer(() => __setClientSettingsForTests(previous)); |
| 638 | __setClientSettingsForTests({ ...previous, speechLanguage: "fr" }); |
| 639 | await using screen = await render(<SpeechLanguageSelect model={{ languages: ["en"] }} />); |
| 640 | const picker = screen.getByRole("combobox", { name: "Transcription language" }); |
| 641 | await expect.element(picker).toHaveTextContent("English (French unavailable)"); |
| 642 | await screen.rerender( |
| 643 | <SpeechLanguageSelect model={{ languages: ["en", "de"], supportsLanguageDetection: true }} />, |
| 644 | ); |
| 645 | await expect.element(picker).toHaveTextContent("Auto-detect (French unavailable)"); |
| 646 | await screen.rerender(<SpeechLanguageSelect model={{ languages: ["en", "fr"] }} />); |
| 647 | await expect.element(picker).toHaveTextContent("French"); |
| 648 | expect(getClientSettings().speechLanguage).toBe("fr"); |
| 649 | }); |
| 650 | |
| 651 | it("keeps punctuation in a saved dictionary term and its alias", async () => { |
| 652 | const previous = getClientSettings(); |
| 653 | using cleanup = new DisposableStack(); |
| 654 | cleanup.defer(() => __setClientSettingsForTests(previous)); |
| 655 | __setClientSettingsForTests({ ...previous, speechCustomWords: [] }); |
| 656 | await using screen = await render( |
| 657 | <SettingsScopeProvider search={{}} onChange={() => {}}> |
| 658 | <VoiceSettingsPanel /> |
| 659 | </SettingsScopeProvider>, |
| 660 | ); |
| 661 | await userEvent.type( |
| 662 | screen.getByRole("textbox", { name: "Add a word or phrase to the dictionary" }), |
| 663 | "O'Reilly{Enter}", |
| 664 | ); |
| 665 | expect(getClientSettings().speechCustomWords[0]?.term).toBe("O'Reilly"); |
| 666 | await userEvent.click(screen.getByRole("button", { name: "Add an alias for O'Reilly" })); |
| 667 | await userEvent.type( |
| 668 | screen.getByRole("textbox", { name: "Alias for O'Reilly" }), |
| 669 | '"Oh Really"{Enter}', |
| 670 | ); |
| 671 | expect(getClientSettings().speechCustomWords).toEqual([ |
| 672 | { term: "O'Reilly", aliases: ['"Oh Really"'] }, |
| 673 | ]); |
| 674 | }); |
| 675 | |
| 676 | it("keeps a dictionary term and alias draft on the Enter that confirms an IME candidate", async () => { |
| 677 | const previous = getClientSettings(); |
| 678 | using cleanup = new DisposableStack(); |
| 679 | cleanup.defer(() => __setClientSettingsForTests(previous)); |
| 680 | __setClientSettingsForTests({ |
| 681 | ...previous, |
| 682 | speechCustomWords: [{ term: "日本語", aliases: [] }], |
| 683 | }); |
| 684 | await using screen = await render( |
| 685 | <SettingsScopeProvider search={{}} onChange={() => {}}> |
| 686 | <VoiceSettingsPanel /> |
| 687 | </SettingsScopeProvider>, |
| 688 | ); |
| 689 | const composingEnter = () => |
| 690 | new KeyboardEvent("keydown", { key: "Enter", isComposing: true, bubbles: true }); |
| 691 | const term = screen.getByRole("textbox", { name: "Add a word or phrase to the dictionary" }); |
| 692 | await userEvent.fill(term, "にほん"); |
| 693 | term.element().dispatchEvent(composingEnter()); |
| 694 | await expect.element(term).toHaveValue("にほん"); |
| 695 | await userEvent.click(screen.getByRole("button", { name: "Add an alias for 日本語" })); |
| 696 | const alias = screen.getByRole("textbox", { name: "Alias for 日本語" }); |
| 697 | await userEvent.fill(alias, "にほん"); |
| 698 | alias.element().dispatchEvent(composingEnter()); |
| 699 | expect(getClientSettings().speechCustomWords).toEqual([{ term: "日本語", aliases: [] }]); |
| 700 | await expect.element(alias).toHaveValue("にほん"); |
| 701 | }); |
| 702 | |
| 703 | it("records from the first connected microphone in the saved order", async () => { |
| 704 | const previous = getClientSettings(); |
| 705 | using cleanup = new DisposableStack(); |
| 706 | cleanup.defer(() => __setClientSettingsForTests(previous)); |
| 707 | // A single saved microphone from before the order existed comes first. |
| 708 | __setClientSettingsForTests({ ...previous, voiceMicrophone: "yeti", voiceMicrophones: [] }); |
| 709 | const input = (deviceId: string, label: string) => |
| 710 | ({ deviceId, label, groupId: deviceId, kind: "audioinput" }) as MediaDeviceInfo; |
| 711 | const yeti = input("yeti", "Blue Yeti"); |
| 712 | const airpods = input("airpods", "AirPods Pro"); |
| 713 | using devices = vi |
| 714 | .spyOn(navigator.mediaDevices, "enumerateDevices") |
| 715 | .mockResolvedValue([yeti, airpods]); |
| 716 | const saved = () => |
| 717 | getClientSettings().voiceMicrophones.map((entry) => |
| 718 | entry.kind === "default" ? "System default" : entry.label, |
| 719 | ); |
| 720 | await using screen = await render( |
| 721 | <SettingsScopeProvider search={{}} onChange={() => {}}> |
| 722 | <VoiceSettingsPanel /> |
| 723 | </SettingsScopeProvider>, |
| 724 | ); |
| 725 | const list = screen.getByRole("list", { name: "Microphones, preferred first" }); |
| 726 | await expect.poll(saved).toEqual(["Blue Yeti", "System default", "AirPods Pro"]); |
| 727 | const row = (index: number) => list.getByRole("listitem").nth(index); |
| 728 | await expect.element(row(0)).toHaveTextContent("1Blue YetiIn use"); |
| 729 | expect(await chooseRecordingMicrophone()).toBe("yeti"); |
| 730 | |
| 731 | devices.mockResolvedValue([airpods]); |
| 732 | await userEvent.click(screen.getByRole("button", { name: "Refresh microphones" })); |
| 733 | await expect.element(row(0)).toHaveTextContent("1Blue YetiNot connected"); |
| 734 | await expect.element(row(1)).toHaveTextContent("2System defaultIn use"); |
| 735 | expect(await chooseRecordingMicrophone()).toBe(""); |
| 736 | |
| 737 | await userEvent.click(screen.getByRole("button", { name: "Move AirPods Pro up" })); |
| 738 | await userEvent.click(screen.getByRole("button", { name: "Move AirPods Pro up" })); |
| 739 | expect(saved()).toEqual(["AirPods Pro", "Blue Yeti", "System default"]); |
| 740 | await expect.element(screen.getByRole("button", { name: "Move AirPods Pro up" })).toBeDisabled(); |
| 741 | expect(await chooseRecordingMicrophone()).toBe("airpods"); |
| 742 | |
| 743 | await userEvent.click(screen.getByRole("button", { name: "Forget Blue Yeti" })); |
| 744 | expect(saved()).toEqual(["AirPods Pro", "System default"]); |
| 745 | await expect |
| 746 | .element(screen.getByRole("button", { name: "Forget System default" })) |
| 747 | .not.toBeInTheDocument(); |
| 748 | }); |