The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { |
| 2 | cancelEnvironmentSpeechModelDownload, |
| 3 | downloadEnvironmentSpeechModel, |
| 4 | getEnvironmentSpeechModels, |
| 5 | getEnvironmentSpeechStatus, |
| 6 | getSpeechModelDownloadOutcome, |
| 7 | selectEnvironmentSpeechModel, |
| 8 | voiceInputBlocksSubmission, |
| 9 | } from "@t3tools/client-runtime/voice-input"; |
| 10 | import type { PreparedConnection } from "@t3tools/client-runtime/connection"; |
| 11 | import { AuthOrchestrationOperateScope, CLOUD_SPEECH_LABELS } from "@t3tools/contracts"; |
| 12 | import type { |
| 13 | EnvironmentId, |
| 14 | EnvironmentSpeechModel, |
| 15 | EnvironmentSpeechStatus, |
| 16 | ProjectId, |
| 17 | SpeechBenchmarkTier, |
| 18 | } from "@t3tools/contracts"; |
| 19 | import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; |
| 20 | import { chooseSetupSpeechModel } from "@t3tools/shared/speech"; |
| 21 | import { useNavigate } from "@tanstack/react-router"; |
| 22 | import * as Option from "effect/Option"; |
| 23 | import { msg, useGT } from "gt-react"; |
| 24 | import { useEffect, useLayoutEffect, useRef, useState } from "react"; |
| 25 | |
| 26 | import { useStableCallback } from "~/hooks/useStableCallback"; |
| 27 | import { toastManager } from "../components/ui/toast"; |
| 28 | import { m } from "../i18n/messages"; |
| 29 | import { |
| 30 | useClientSettings, |
| 31 | useClientSettingsHydrated, |
| 32 | useEnvironmentSettings, |
| 33 | } from "../hooks/useSettings"; |
| 34 | import { useEnvironment } from "../state/environments"; |
| 35 | import { useEnvironmentScope, usePreparedConnection } from "../state/session"; |
| 36 | import { useAppleSpeechLocales } from "./appleSpeech"; |
| 37 | import { preferredSpeechLanguages } from "./preferredSpeechLanguages"; |
| 38 | import { runEnvironmentSpeech } from "./runEnvironmentSpeech"; |
| 39 | import { useAudioChoice } from "./voiceConsent"; |
| 40 | import { |
| 41 | IDLE_VOICE_STATE, |
| 42 | presentVoiceOwner, |
| 43 | useRetainedVoiceInputSession, |
| 44 | useVoiceInputSessionActivity, |
| 45 | useVoiceInputSessionSnapshot, |
| 46 | type VoiceInputTarget, |
| 47 | } from "./voiceInputSession"; |
| 48 | |
| 49 | /** A product name, shown untranslated. */ |
| 50 | export const APPLE_SPEECH_LABEL = "Apple Speech"; |
| 51 | |
| 52 | const INITIAL_SETUP = { |
| 53 | open: false, |
| 54 | step: 0, |
| 55 | model: null as EnvironmentSpeechModel | null, |
| 56 | language: null as string | null, |
| 57 | downloading: false, |
| 58 | catalogFailed: false, |
| 59 | error: null as string | null, |
| 60 | }; |
| 61 | |
| 62 | // Every mounted editor shares one status probe per destination; starting a recording refreshes it. |
| 63 | const statusProbes = new WeakMap<PreparedConnection, Promise<EnvironmentSpeechStatus>>(); |
| 64 | function probeSpeechStatus(prepared: PreparedConnection, refresh = false) { |
| 65 | const cached = refresh ? undefined : statusProbes.get(prepared); |
| 66 | if (cached) return cached; |
| 67 | const probe = runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared)); |
| 68 | statusProbes.set(prepared, probe); |
| 69 | probe.catch(() => { |
| 70 | if (statusProbes.get(prepared) === probe) statusProbes.delete(prepared); |
| 71 | }); |
| 72 | return probe; |
| 73 | } |
| 74 | |
| 75 | type DraftInput = { |
| 76 | readonly text: string; |
| 77 | readonly selection: { readonly start: number; readonly end: number }; |
| 78 | }; |
| 79 | |
| 80 | type HookInput = { |
| 81 | /** The draft's own environment. Optional transcript cleanup runs there, never on the audio destination. */ |
| 82 | readonly environmentId: EnvironmentId; |
| 83 | /** The draft's project: its vocabulary on that environment and its name become recognition hints. */ |
| 84 | readonly projectId?: ProjectId | null | undefined; |
| 85 | readonly projectName?: string | null | undefined; |
| 86 | readonly ownerKey: string; |
| 87 | readonly disabled?: boolean | undefined; |
| 88 | /** |
| 89 | * Until a destination is chosen, voice input is offered and its first gesture opens Voice |
| 90 | * settings. False hides it instead, for editors whose draft would not survive the navigation. |
| 91 | */ |
| 92 | readonly offerSetup?: boolean | undefined; |
| 93 | /** False keeps the transcript off the cleanup provider even when cleanup is on; the model test uses it. */ |
| 94 | readonly cleanup?: boolean | undefined; |
| 95 | readonly readDraft: () => DraftInput; |
| 96 | readonly commitDraft: ( |
| 97 | text: string, |
| 98 | selection: { readonly start: number; readonly end: number }, |
| 99 | ) => void; |
| 100 | /** Names the draft in the indicator shown while a recording's editor is off screen. */ |
| 101 | readonly label?: string | null | undefined; |
| 102 | /** False while the editor stays mounted out of sight, so the indicator still shows. */ |
| 103 | readonly presented?: boolean | undefined; |
| 104 | /** |
| 105 | * A draft that is stored outside its editor keeps recording when the editor shows another |
| 106 | * owner or unmounts; the transcript lands in the stored draft instead. |
| 107 | */ |
| 108 | readonly persistentDraft?: |
| 109 | | { |
| 110 | readonly readStoredText: () => string | null; |
| 111 | readonly commitStoredText: (text: string) => void; |
| 112 | readonly open: () => void; |
| 113 | } |
| 114 | | undefined; |
| 115 | }; |
| 116 | |
| 117 | /** |
| 118 | * Microphone audio leaves this device only for the environment the person chose in |
| 119 | * Voice settings. Nothing is inferred from the primary or the draft's environment, and |
| 120 | * the recording is abandoned the moment that choice changes or its connection drops. |
| 121 | * Apple Speech ("device") keeps audio on this Mac and needs no environment. |
| 122 | */ |
| 123 | export function useVoiceTranscriptionDestination() { |
| 124 | const gt = useGT(); |
| 125 | const hydrated = useClientSettingsHydrated(); |
| 126 | const choice = useAudioChoice(); |
| 127 | const grant = choice === "off" ? null : choice; |
| 128 | const appleSpeechAvailable = useAppleSpeechLocales().length > 0; |
| 129 | const device = hydrated && grant?.destination.kind === "device"; |
| 130 | const environmentId = |
| 131 | hydrated && grant && grant.destination.kind !== "device" |
| 132 | ? grant.destination.environmentId |
| 133 | : null; |
| 134 | const environment = useEnvironment(environmentId); |
| 135 | const connection = Option.getOrNull(usePreparedConnection(environmentId)); |
| 136 | const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope); |
| 137 | const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined; |
| 138 | const provider = cloud?.provider; |
| 139 | const model = cloud?.provider === "openrouter" ? cloud.model : undefined; |
| 140 | const supported = |
| 141 | environment?.serverConfig?.environment.capabilities[ |
| 142 | provider ? "cloudVoiceTranscription" : "voiceTranscription" |
| 143 | ] === true; |
| 144 | return { |
| 145 | ...(hydrated && choice === null ? { offerFirstRunSetup: true } : {}), |
| 146 | environmentId, |
| 147 | ...(device ? { device: true, onDevice: appleSpeechAvailable } : {}), |
| 148 | ...(provider ? { cloudProvider: provider } : {}), |
| 149 | ...(model ? { cloudModel: model } : {}), |
| 150 | grantId: grant?.id ?? null, |
| 151 | label: device |
| 152 | ? APPLE_SPEECH_LABEL |
| 153 | : provider && environment |
| 154 | ? gt("{provider} via {environment}", { |
| 155 | provider: CLOUD_SPEECH_LABELS[provider], |
| 156 | environment: environment.label, |
| 157 | }) |
| 158 | : (environment?.label ?? null), |
| 159 | prepared: supported && canDictate ? connection : null, |
| 160 | }; |
| 161 | } |
| 162 | |
| 163 | export function useEnvironmentSpeechInput(input: HookInput) { |
| 164 | const navigate = useNavigate(); |
| 165 | const destination = useVoiceTranscriptionDestination(); |
| 166 | const prepared = destination.prepared; |
| 167 | const onDevice = destination.onDevice === true; |
| 168 | const grantId = destination.grantId; |
| 169 | const cloudProvider = destination.cloudProvider; |
| 170 | const cloudModel = destination.cloudModel; |
| 171 | const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment |
| 172 | .capabilities; |
| 173 | const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId)); |
| 174 | const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope); |
| 175 | const cleanupPrepared = |
| 176 | canCleanUp && |
| 177 | (draftCapabilities?.voiceTranscription || draftCapabilities?.cloudVoiceTranscription) |
| 178 | ? originatingConnection |
| 179 | : null; |
| 180 | const clientSettings = useClientSettings(); |
| 181 | const projectId = input.projectId ?? null; |
| 182 | const projectWords = useEnvironmentSettings( |
| 183 | input.environmentId, |
| 184 | (settings) => resolveProjectSettings(settings, projectId).settings.speechProjectCustomWords, |
| 185 | ); |
| 186 | const [status, setStatus] = useState<{ |
| 187 | readonly prepared: PreparedConnection; |
| 188 | readonly value: EnvironmentSpeechStatus | null; |
| 189 | } | null>(null); |
| 190 | const [setup, setSetup] = useState(INITIAL_SETUP); |
| 191 | const [previousDestination, setPreviousDestination] = useState({ prepared, grantId }); |
| 192 | const [previousOwner, setPreviousOwner] = useState({ |
| 193 | key: input.ownerKey, |
| 194 | disabled: input.disabled, |
| 195 | }); |
| 196 | if (previousDestination.prepared !== prepared || previousDestination.grantId !== grantId) { |
| 197 | setPreviousDestination({ prepared, grantId }); |
| 198 | setSetup(INITIAL_SETUP); |
| 199 | } |
| 200 | if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) { |
| 201 | setPreviousOwner({ key: input.ownerKey, disabled: input.disabled }); |
| 202 | setSetup(INITIAL_SETUP); |
| 203 | } |
| 204 | |
| 205 | const session = useRetainedVoiceInputSession({ |
| 206 | prepared, |
| 207 | grantId, |
| 208 | onDevice, |
| 209 | cloudProvider, |
| 210 | cloudModel, |
| 211 | }); |
| 212 | const snapshot = useVoiceInputSessionSnapshot(session); |
| 213 | const owns = snapshot.target?.ownerKey === input.ownerKey; |
| 214 | const activity = useVoiceInputSessionActivity(session, input.ownerKey); |
| 215 | const state = owns ? snapshot.state : IDLE_VOICE_STATE; |
| 216 | // One microphone: another draft's recording leaves this editor without voice input until it ends. |
| 217 | const busyElsewhere = !owns && voiceInputBlocksSubmission(snapshot.state); |
| 218 | |
| 219 | const startRequestRef = useRef(0); |
| 220 | const pendingSubmissionRef = useRef<{ |
| 221 | ownerKey: string; |
| 222 | submit: (text: string) => void; |
| 223 | text?: string; |
| 224 | } | null>(null); |
| 225 | const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords, session }); |
| 226 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props before an async completion can read them through the session |
| 227 | useLayoutEffect(() => { |
| 228 | latestRef.current = { input, clientSettings, cleanupPrepared, projectWords, session }; |
| 229 | }); |
| 230 | const mountedRef = useRef(false); |
| 231 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons this editor's pending work, and a recording that cannot outlive it, on unmount or a destination change |
| 232 | useEffect(() => { |
| 233 | mountedRef.current = true; |
| 234 | return () => { |
| 235 | mountedRef.current = false; |
| 236 | pendingSubmissionRef.current = null; |
| 237 | startRequestRef.current += 1; |
| 238 | session?.ownerChanged(latestRef.current.input.ownerKey); |
| 239 | }; |
| 240 | }, [session]); |
| 241 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the global indicator this draft is on screen |
| 242 | useEffect( |
| 243 | () => (input.presented === false ? undefined : presentVoiceOwner(input.ownerKey)), |
| 244 | [input.ownerKey, input.presented], |
| 245 | ); |
| 246 | |
| 247 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available |
| 248 | useEffect(() => { |
| 249 | if (!prepared || cloudProvider) return; |
| 250 | let disposed = false; |
| 251 | void probeSpeechStatus(prepared) |
| 252 | .then((value) => { |
| 253 | if (!disposed) setStatus({ prepared, value }); |
| 254 | }) |
| 255 | .catch(() => { |
| 256 | if (!disposed) setStatus({ prepared, value: null }); |
| 257 | }); |
| 258 | return () => { |
| 259 | disposed = true; |
| 260 | }; |
| 261 | }, [prepared, cloudProvider]); |
| 262 | const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null; |
| 263 | const setupModelId = setup.model?.id; |
| 264 | |
| 265 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download |
| 266 | useEffect(() => { |
| 267 | if (!setup.open || !setup.downloading || !prepared || !setupModelId) return; |
| 268 | let disposed = false; |
| 269 | let refreshing = false; |
| 270 | const timer = window.setInterval(() => { |
| 271 | if (refreshing) return; |
| 272 | refreshing = true; |
| 273 | void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared)) |
| 274 | .then((result) => { |
| 275 | if (disposed) return; |
| 276 | const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null; |
| 277 | if (model) |
| 278 | setSetup((current) => ({ |
| 279 | ...current, |
| 280 | model: { ...model, recommended: current.model?.recommended ?? false }, |
| 281 | })); |
| 282 | }) |
| 283 | .catch(() => undefined) |
| 284 | .finally(() => { |
| 285 | refreshing = false; |
| 286 | }); |
| 287 | }, 1_000); |
| 288 | return () => { |
| 289 | disposed = true; |
| 290 | window.clearInterval(timer); |
| 291 | }; |
| 292 | }, [prepared, setup.downloading, setup.open, setupModelId]); |
| 293 | |
| 294 | const previousOwnerRef = useRef(input.ownerKey); |
| 295 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons any queued submission, and a recording that cannot outlive its editor, when the draft's owner changes |
| 296 | useEffect(() => { |
| 297 | const previous = previousOwnerRef.current; |
| 298 | if (previous === input.ownerKey && !input.disabled) return; |
| 299 | previousOwnerRef.current = input.ownerKey; |
| 300 | pendingSubmissionRef.current = null; |
| 301 | startRequestRef.current += 1; |
| 302 | if (previous !== input.ownerKey) session?.ownerChanged(previous); |
| 303 | else if (session?.ownerKey === input.ownerKey) session.cancel(); |
| 304 | }, [input.ownerKey, input.disabled, session]); |
| 305 | |
| 306 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft |
| 307 | useEffect(() => { |
| 308 | const pending = pendingSubmissionRef.current; |
| 309 | if (state.phase === "error") pendingSubmissionRef.current = null; |
| 310 | if (state.phase !== "idle" || !pending) return; |
| 311 | pendingSubmissionRef.current = null; |
| 312 | if (!input.disabled && pending.ownerKey === input.ownerKey && pending.text !== undefined) |
| 313 | pending.submit(pending.text); |
| 314 | }, [state, input.ownerKey, input.disabled]); |
| 315 | |
| 316 | const submitAfterDictation = useStableCallback((submit: (text: string) => void) => { |
| 317 | if (latestRef.current.input.disabled || state.phase === "preparing") return; |
| 318 | if (!session || !voiceInputBlocksSubmission(state)) { |
| 319 | submit(latestRef.current.input.readDraft().text); |
| 320 | return; |
| 321 | } |
| 322 | if (pendingSubmissionRef.current) return; |
| 323 | pendingSubmissionRef.current = { ownerKey: latestRef.current.input.ownerKey, submit }; |
| 324 | if (session.state.phase === "recording") void session.stop(); |
| 325 | }); |
| 326 | |
| 327 | const createTarget = useStableCallback((): VoiceInputTarget => { |
| 328 | const owner = latestRef.current.input.ownerKey; |
| 329 | const readContext = () => { |
| 330 | const current = latestRef.current; |
| 331 | return { |
| 332 | environmentId: current.input.environmentId, |
| 333 | projectName: current.input.projectName ?? null, |
| 334 | projectWords: current.projectWords, |
| 335 | clientSettings: current.clientSettings, |
| 336 | cleanup: current.input.cleanup !== false, |
| 337 | cleanupPrepared: current.cleanupPrepared, |
| 338 | }; |
| 339 | }; |
| 340 | // Once this editor shows another draft, its environment and cleanup must not see this one. |
| 341 | const startContext = readContext(); |
| 342 | const persistent = latestRef.current.input.persistentDraft ?? null; |
| 343 | // While this editor shows the owner, the live editor is the draft; otherwise the stored one. |
| 344 | const live = () => mountedRef.current && latestRef.current.input.ownerKey === owner; |
| 345 | return { |
| 346 | ownerKey: owner, |
| 347 | label: latestRef.current.input.label ?? null, |
| 348 | persistent: persistent !== null, |
| 349 | open: persistent?.open ?? null, |
| 350 | readDraft: () => { |
| 351 | if (live()) return latestRef.current.input.readDraft(); |
| 352 | const text = persistent?.readStoredText(); |
| 353 | return text == null ? null : { text, selection: { start: text.length, end: text.length } }; |
| 354 | }, |
| 355 | commitDraft: (text, selection) => { |
| 356 | if (!live()) { |
| 357 | persistent?.commitStoredText(text); |
| 358 | return; |
| 359 | } |
| 360 | if (latestRef.current.input.disabled) return; |
| 361 | latestRef.current.input.commitDraft(text, selection); |
| 362 | if (pendingSubmissionRef.current?.ownerKey === owner) |
| 363 | pendingSubmissionRef.current.text = text; |
| 364 | }, |
| 365 | readContext: () => (live() ? readContext() : startContext), |
| 366 | }; |
| 367 | }); |
| 368 | |
| 369 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- a remounted editor takes over its draft's running recording so a queued Send still sees the transcript |
| 370 | useEffect(() => { |
| 371 | if (session?.ownerKey === input.ownerKey && voiceInputBlocksSubmission(session.state)) |
| 372 | session.adopt(createTarget()); |
| 373 | }, [session, input.ownerKey, createTarget]); |
| 374 | const loadSetupCatalog = useStableCallback( |
| 375 | async ( |
| 376 | connection: PreparedConnection, |
| 377 | selectedModelId: string, |
| 378 | benchmarkTier: SpeechBenchmarkTier, |
| 379 | request: number, |
| 380 | ) => { |
| 381 | setSetup({ ...INITIAL_SETUP, open: true }); |
| 382 | const catalog = await runEnvironmentSpeech( |
| 383 | getEnvironmentSpeechModels(connection, true), |
| 384 | ).catch(() => null); |
| 385 | if (request !== startRequestRef.current) return; |
| 386 | const choice = chooseSetupSpeechModel(catalog?.models ?? [], { |
| 387 | selectedModelId, |
| 388 | intent: latestRef.current.clientSettings.speechLanguage, |
| 389 | preferred: preferredSpeechLanguages(), |
| 390 | benchmarkTier, |
| 391 | }); |
| 392 | setSetup((current) => ({ |
| 393 | ...current, |
| 394 | model: choice ? { ...choice.model, recommended: choice.recommended } : null, |
| 395 | language: choice?.language ?? null, |
| 396 | catalogFailed: catalog === null, |
| 397 | error: choice |
| 398 | ? null |
| 399 | : catalog |
| 400 | ? m( |
| 401 | msg( |
| 402 | "No model matches your language. Choose a model or change the language in Voice settings.", |
| 403 | ), |
| 404 | ) |
| 405 | : m(msg("Could not load transcription models. Check the connection and try again.")), |
| 406 | })); |
| 407 | }, |
| 408 | ); |
| 409 | |
| 410 | const start = useStableCallback(async () => { |
| 411 | if (latestRef.current.input.disabled || (setup.open && setup.downloading)) return; |
| 412 | if (!grantId) { |
| 413 | if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false) |
| 414 | await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" }); |
| 415 | return; |
| 416 | } |
| 417 | if (!session || voiceInputBlocksSubmission(session.state)) return; |
| 418 | const request = ++startRequestRef.current; |
| 419 | const owner = latestRef.current.input.ownerKey; |
| 420 | if (onDevice) { |
| 421 | await session.start(createTarget()); |
| 422 | return; |
| 423 | } |
| 424 | if (!prepared) return; |
| 425 | if (cloudProvider) { |
| 426 | await session.start(createTarget()); |
| 427 | return; |
| 428 | } |
| 429 | setSetup(INITIAL_SETUP); |
| 430 | const stillCurrent = () => |
| 431 | request === startRequestRef.current && |
| 432 | latestRef.current.session === session && |
| 433 | latestRef.current.input.ownerKey === owner && |
| 434 | !latestRef.current.input.disabled; |
| 435 | const handleStatus = async (latestStatus: EnvironmentSpeechStatus) => { |
| 436 | if (!stillCurrent()) return; |
| 437 | setStatus({ prepared, value: latestStatus }); |
| 438 | if (!latestStatus.supported) { |
| 439 | session.cancel(); |
| 440 | toastManager.add({ |
| 441 | type: "error", |
| 442 | title: m(msg("Voice input is not available")), |
| 443 | description: latestStatus.reason, |
| 444 | }); |
| 445 | } else if (latestStatus.state === "missing-model") { |
| 446 | session.cancel(); |
| 447 | await loadSetupCatalog(prepared, latestStatus.modelId, latestStatus.benchmarkTier, request); |
| 448 | } |
| 449 | }; |
| 450 | |
| 451 | if (currentStatus?.supported && currentStatus.state === "missing-model") { |
| 452 | // Setup must remain usable without opening the microphone. |
| 453 | const latestStatus = await probeSpeechStatus(prepared, true).catch((error: unknown) => { |
| 454 | if (stillCurrent()) { |
| 455 | toastManager.add({ |
| 456 | type: "error", |
| 457 | title: m(msg("Could not start voice input")), |
| 458 | description: error instanceof Error ? error.message : String(error), |
| 459 | }); |
| 460 | } |
| 461 | return null; |
| 462 | }); |
| 463 | if (!stillCurrent() || !latestStatus) return; |
| 464 | await handleStatus(latestStatus); |
| 465 | if (!latestStatus.supported || latestStatus.state === "missing-model" || !stillCurrent()) |
| 466 | return; |
| 467 | } |
| 468 | |
| 469 | await session.start(createTarget(), (latestStatus) => void handleStatus(latestStatus)); |
| 470 | }); |
| 471 | |
| 472 | const downloadSetupModel = useStableCallback(async () => { |
| 473 | if (!prepared || !currentStatus?.supported || !setupModelId || setup.downloading) return; |
| 474 | const modelId = setupModelId; |
| 475 | const request = ++startRequestRef.current; |
| 476 | setSetup((current) => ({ ...current, downloading: true, error: null })); |
| 477 | try { |
| 478 | const downloaded = await runEnvironmentSpeech( |
| 479 | downloadEnvironmentSpeechModel(prepared, modelId), |
| 480 | ); |
| 481 | if (request !== startRequestRef.current) return; |
| 482 | const outcome = getSpeechModelDownloadOutcome(downloaded, modelId); |
| 483 | if (outcome === "needs-selection") { |
| 484 | try { |
| 485 | await runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, modelId)); |
| 486 | } catch { |
| 487 | if (request !== startRequestRef.current) return; |
| 488 | setSetup((current) => ({ |
| 489 | ...current, |
| 490 | downloading: false, |
| 491 | error: m( |
| 492 | msg("The model downloaded, but this environment could not select it. Try again."), |
| 493 | ), |
| 494 | })); |
| 495 | return; |
| 496 | } |
| 497 | } |
| 498 | if (request !== startRequestRef.current) return; |
| 499 | const next = await probeSpeechStatus(prepared, true); |
| 500 | if (request !== startRequestRef.current) return; |
| 501 | setStatus({ prepared, value: next }); |
| 502 | const ready = |
| 503 | outcome !== "incomplete" && |
| 504 | next.supported && |
| 505 | next.modelId === modelId && |
| 506 | next.state !== "missing-model"; |
| 507 | setSetup((current) => ({ |
| 508 | ...current, |
| 509 | downloading: false, |
| 510 | step: ready ? 1 : current.step, |
| 511 | error: ready ? null : m(msg("The model is not ready. Try downloading it again.")), |
| 512 | })); |
| 513 | } catch { |
| 514 | if (request !== startRequestRef.current) return; |
| 515 | setSetup((current) => ({ |
| 516 | ...current, |
| 517 | downloading: false, |
| 518 | error: m( |
| 519 | msg("Could not download the model. Check this environment's connection and try again."), |
| 520 | ), |
| 521 | })); |
| 522 | } |
| 523 | }); |
| 524 | |
| 525 | const cancelSetupDownload = useStableCallback(async () => { |
| 526 | if (!prepared || !setupModelId) return; |
| 527 | startRequestRef.current += 1; |
| 528 | setSetup((current) => ({ ...current, open: false, downloading: false })); |
| 529 | try { |
| 530 | await runEnvironmentSpeech(cancelEnvironmentSpeechModelDownload(prepared, setupModelId)); |
| 531 | } catch (error) { |
| 532 | setSetup((current) => ({ |
| 533 | ...current, |
| 534 | error: error instanceof Error ? error.message : String(error), |
| 535 | })); |
| 536 | } |
| 537 | }); |
| 538 | |
| 539 | const destinationReady = |
| 540 | grantId === null |
| 541 | ? Boolean(destination.offerFirstRunSetup) && input.offerSetup !== false |
| 542 | : onDevice || |
| 543 | (prepared !== null && |
| 544 | (cloudProvider !== undefined || |
| 545 | (status?.prepared === prepared && status.value?.supported !== false))); |
| 546 | return { |
| 547 | available: |
| 548 | destinationReady && |
| 549 | !busyElsewhere && |
| 550 | typeof navigator !== "undefined" && |
| 551 | Boolean(navigator.mediaDevices?.getUserMedia) && |
| 552 | typeof AudioWorkletNode !== "undefined", |
| 553 | status: currentStatus, |
| 554 | transcriptionEnvironmentId: destination.environmentId, |
| 555 | transcriptionEnvironmentLabel: destination.label, |
| 556 | setup: { |
| 557 | ...setup, |
| 558 | setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })), |
| 559 | retryCatalog: |
| 560 | setup.open && setup.catalogFailed && prepared && currentStatus?.supported |
| 561 | ? () => |
| 562 | loadSetupCatalog( |
| 563 | prepared, |
| 564 | currentStatus.modelId, |
| 565 | currentStatus.benchmarkTier, |
| 566 | ++startRequestRef.current, |
| 567 | ) |
| 568 | : null, |
| 569 | download: downloadSetupModel, |
| 570 | cancelDownload: cancelSetupDownload, |
| 571 | startRecording: async () => { |
| 572 | setSetup((current) => ({ ...current, open: false })); |
| 573 | await start(); |
| 574 | }, |
| 575 | }, |
| 576 | state, |
| 577 | preview: |
| 578 | state.phase === "recording" || state.phase === "transcribing" ? activity.preview : null, |
| 579 | level: activity.level, |
| 580 | recordingStartedAt: owns ? snapshot.recordingStartedAt : null, |
| 581 | preparing: state.phase === "preparing", |
| 582 | blocksSubmission: voiceInputBlocksSubmission(state), |
| 583 | freezesEditor: voiceInputBlocksSubmission(state), |
| 584 | start, |
| 585 | submitAfterDictation, |
| 586 | stop: useStableCallback(() => (owns && session ? session.stop() : Promise.resolve())), |
| 587 | cancel: useStableCallback(() => { |
| 588 | pendingSubmissionRef.current = null; |
| 589 | startRequestRef.current += 1; |
| 590 | setSetup(INITIAL_SETUP); |
| 591 | if (owns) session?.cancel(); |
| 592 | }), |
| 593 | skipPostProcessing: useStableCallback(() => { |
| 594 | if (owns) session?.skipPostProcessing(); |
| 595 | }), |
| 596 | }; |
| 597 | } |