@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 597 lines 23.6 KB
Raw
1 import {
2 cancelEnvironmentSpeechModelDownload,
3 downloadEnvironmentSpeechModel,
4 getEnvironmentSpeechModels,
5 getEnvironmentSpeechStatus,
6 getSpeechModelDownloadOutcome,
7 selectEnvironmentSpeechModel,
8 voiceInputBlocksSubmission,
9 } from "@t3tools/client-runtime/voice-input";
10 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
11 import { AuthOrchestrationOperateScope, CLOUD_SPEECH_LABELS } from "@t3tools/contracts";
12 import type {
13 EnvironmentId,
14 EnvironmentSpeechModel,
15 EnvironmentSpeechStatus,
16 ProjectId,
17 SpeechBenchmarkTier,
18 } from "@t3tools/contracts";
19 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
20 import { chooseSetupSpeechModel } from "@t3tools/shared/speech";
21 import { useNavigate } from "@tanstack/react-router";
22 import * as Option from "effect/Option";
23 import { msg, useGT } from "gt-react";
24 import { useEffect, useLayoutEffect, useRef, useState } from "react";
25
26 import { useStableCallback } from "~/hooks/useStableCallback";
27 import { toastManager } from "../components/ui/toast";
28 import { m } from "../i18n/messages";
29 import {
30 useClientSettings,
31 useClientSettingsHydrated,
32 useEnvironmentSettings,
33 } from "../hooks/useSettings";
34 import { useEnvironment } from "../state/environments";
35 import { useEnvironmentScope, usePreparedConnection } from "../state/session";
36 import { useAppleSpeechLocales } from "./appleSpeech";
37 import { preferredSpeechLanguages } from "./preferredSpeechLanguages";
38 import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
39 import { useAudioChoice } from "./voiceConsent";
40 import {
41 IDLE_VOICE_STATE,
42 presentVoiceOwner,
43 useRetainedVoiceInputSession,
44 useVoiceInputSessionActivity,
45 useVoiceInputSessionSnapshot,
46 type VoiceInputTarget,
47 } from "./voiceInputSession";
48
49 /** A product name, shown untranslated. */
50 export const APPLE_SPEECH_LABEL = "Apple Speech";
51
52 const INITIAL_SETUP = {
53 open: false,
54 step: 0,
55 model: null as EnvironmentSpeechModel | null,
56 language: null as string | null,
57 downloading: false,
58 catalogFailed: false,
59 error: null as string | null,
60 };
61
62 // Every mounted editor shares one status probe per destination; starting a recording refreshes it.
63 const statusProbes = new WeakMap<PreparedConnection, Promise<EnvironmentSpeechStatus>>();
64 function probeSpeechStatus(prepared: PreparedConnection, refresh = false) {
65 const cached = refresh ? undefined : statusProbes.get(prepared);
66 if (cached) return cached;
67 const probe = runEnvironmentSpeech(getEnvironmentSpeechStatus(prepared));
68 statusProbes.set(prepared, probe);
69 probe.catch(() => {
70 if (statusProbes.get(prepared) === probe) statusProbes.delete(prepared);
71 });
72 return probe;
73 }
74
75 type DraftInput = {
76 readonly text: string;
77 readonly selection: { readonly start: number; readonly end: number };
78 };
79
80 type HookInput = {
81 /** The draft's own environment. Optional transcript cleanup runs there, never on the audio destination. */
82 readonly environmentId: EnvironmentId;
83 /** The draft's project: its vocabulary on that environment and its name become recognition hints. */
84 readonly projectId?: ProjectId | null | undefined;
85 readonly projectName?: string | null | undefined;
86 readonly ownerKey: string;
87 readonly disabled?: boolean | undefined;
88 /**
89 * Until a destination is chosen, voice input is offered and its first gesture opens Voice
90 * settings. False hides it instead, for editors whose draft would not survive the navigation.
91 */
92 readonly offerSetup?: boolean | undefined;
93 /** False keeps the transcript off the cleanup provider even when cleanup is on; the model test uses it. */
94 readonly cleanup?: boolean | undefined;
95 readonly readDraft: () => DraftInput;
96 readonly commitDraft: (
97 text: string,
98 selection: { readonly start: number; readonly end: number },
99 ) => void;
100 /** Names the draft in the indicator shown while a recording's editor is off screen. */
101 readonly label?: string | null | undefined;
102 /** False while the editor stays mounted out of sight, so the indicator still shows. */
103 readonly presented?: boolean | undefined;
104 /**
105 * A draft that is stored outside its editor keeps recording when the editor shows another
106 * owner or unmounts; the transcript lands in the stored draft instead.
107 */
108 readonly persistentDraft?:
109 | {
110 readonly readStoredText: () => string | null;
111 readonly commitStoredText: (text: string) => void;
112 readonly open: () => void;
113 }
114 | undefined;
115 };
116
117 /**
118 * Microphone audio leaves this device only for the environment the person chose in
119 * Voice settings. Nothing is inferred from the primary or the draft's environment, and
120 * the recording is abandoned the moment that choice changes or its connection drops.
121 * Apple Speech ("device") keeps audio on this Mac and needs no environment.
122 */
123 export function useVoiceTranscriptionDestination() {
124 const gt = useGT();
125 const hydrated = useClientSettingsHydrated();
126 const choice = useAudioChoice();
127 const grant = choice === "off" ? null : choice;
128 const appleSpeechAvailable = useAppleSpeechLocales().length > 0;
129 const device = hydrated && grant?.destination.kind === "device";
130 const environmentId =
131 hydrated && grant && grant.destination.kind !== "device"
132 ? grant.destination.environmentId
133 : null;
134 const environment = useEnvironment(environmentId);
135 const connection = Option.getOrNull(usePreparedConnection(environmentId));
136 const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope);
137 const cloud = grant?.destination.kind === "cloud" ? grant.destination : undefined;
138 const provider = cloud?.provider;
139 const model = cloud?.provider === "openrouter" ? cloud.model : undefined;
140 const supported =
141 environment?.serverConfig?.environment.capabilities[
142 provider ? "cloudVoiceTranscription" : "voiceTranscription"
143 ] === true;
144 return {
145 ...(hydrated && choice === null ? { offerFirstRunSetup: true } : {}),
146 environmentId,
147 ...(device ? { device: true, onDevice: appleSpeechAvailable } : {}),
148 ...(provider ? { cloudProvider: provider } : {}),
149 ...(model ? { cloudModel: model } : {}),
150 grantId: grant?.id ?? null,
151 label: device
152 ? APPLE_SPEECH_LABEL
153 : provider && environment
154 ? gt("{provider} via {environment}", {
155 provider: CLOUD_SPEECH_LABELS[provider],
156 environment: environment.label,
157 })
158 : (environment?.label ?? null),
159 prepared: supported && canDictate ? connection : null,
160 };
161 }
162
163 export function useEnvironmentSpeechInput(input: HookInput) {
164 const navigate = useNavigate();
165 const destination = useVoiceTranscriptionDestination();
166 const prepared = destination.prepared;
167 const onDevice = destination.onDevice === true;
168 const grantId = destination.grantId;
169 const cloudProvider = destination.cloudProvider;
170 const cloudModel = destination.cloudModel;
171 const draftCapabilities = useEnvironment(input.environmentId)?.serverConfig?.environment
172 .capabilities;
173 const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId));
174 const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope);
175 const cleanupPrepared =
176 canCleanUp &&
177 (draftCapabilities?.voiceTranscription || draftCapabilities?.cloudVoiceTranscription)
178 ? originatingConnection
179 : null;
180 const clientSettings = useClientSettings();
181 const projectId = input.projectId ?? null;
182 const projectWords = useEnvironmentSettings(
183 input.environmentId,
184 (settings) => resolveProjectSettings(settings, projectId).settings.speechProjectCustomWords,
185 );
186 const [status, setStatus] = useState<{
187 readonly prepared: PreparedConnection;
188 readonly value: EnvironmentSpeechStatus | null;
189 } | null>(null);
190 const [setup, setSetup] = useState(INITIAL_SETUP);
191 const [previousDestination, setPreviousDestination] = useState({ prepared, grantId });
192 const [previousOwner, setPreviousOwner] = useState({
193 key: input.ownerKey,
194 disabled: input.disabled,
195 });
196 if (previousDestination.prepared !== prepared || previousDestination.grantId !== grantId) {
197 setPreviousDestination({ prepared, grantId });
198 setSetup(INITIAL_SETUP);
199 }
200 if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) {
201 setPreviousOwner({ key: input.ownerKey, disabled: input.disabled });
202 setSetup(INITIAL_SETUP);
203 }
204
205 const session = useRetainedVoiceInputSession({
206 prepared,
207 grantId,
208 onDevice,
209 cloudProvider,
210 cloudModel,
211 });
212 const snapshot = useVoiceInputSessionSnapshot(session);
213 const owns = snapshot.target?.ownerKey === input.ownerKey;
214 const activity = useVoiceInputSessionActivity(session, input.ownerKey);
215 const state = owns ? snapshot.state : IDLE_VOICE_STATE;
216 // One microphone: another draft's recording leaves this editor without voice input until it ends.
217 const busyElsewhere = !owns && voiceInputBlocksSubmission(snapshot.state);
218
219 const startRequestRef = useRef(0);
220 const pendingSubmissionRef = useRef<{
221 ownerKey: string;
222 submit: (text: string) => void;
223 text?: string;
224 } | null>(null);
225 const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords, session });
226 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props before an async completion can read them through the session
227 useLayoutEffect(() => {
228 latestRef.current = { input, clientSettings, cleanupPrepared, projectWords, session };
229 });
230 const mountedRef = useRef(false);
231 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons this editor's pending work, and a recording that cannot outlive it, on unmount or a destination change
232 useEffect(() => {
233 mountedRef.current = true;
234 return () => {
235 mountedRef.current = false;
236 pendingSubmissionRef.current = null;
237 startRequestRef.current += 1;
238 session?.ownerChanged(latestRef.current.input.ownerKey);
239 };
240 }, [session]);
241 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the global indicator this draft is on screen
242 useEffect(
243 () => (input.presented === false ? undefined : presentVoiceOwner(input.ownerKey)),
244 [input.ownerKey, input.presented],
245 );
246
247 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available
248 useEffect(() => {
249 if (!prepared || cloudProvider) return;
250 let disposed = false;
251 void probeSpeechStatus(prepared)
252 .then((value) => {
253 if (!disposed) setStatus({ prepared, value });
254 })
255 .catch(() => {
256 if (!disposed) setStatus({ prepared, value: null });
257 });
258 return () => {
259 disposed = true;
260 };
261 }, [prepared, cloudProvider]);
262 const currentStatus = !cloudProvider && status?.prepared === prepared ? status.value : null;
263 const setupModelId = setup.model?.id;
264
265 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download
266 useEffect(() => {
267 if (!setup.open || !setup.downloading || !prepared || !setupModelId) return;
268 let disposed = false;
269 let refreshing = false;
270 const timer = window.setInterval(() => {
271 if (refreshing) return;
272 refreshing = true;
273 void runEnvironmentSpeech(getEnvironmentSpeechModels(prepared))
274 .then((result) => {
275 if (disposed) return;
276 const model = result.models.find((candidate) => candidate.id === setupModelId) ?? null;
277 if (model)
278 setSetup((current) => ({
279 ...current,
280 model: { ...model, recommended: current.model?.recommended ?? false },
281 }));
282 })
283 .catch(() => undefined)
284 .finally(() => {
285 refreshing = false;
286 });
287 }, 1_000);
288 return () => {
289 disposed = true;
290 window.clearInterval(timer);
291 };
292 }, [prepared, setup.downloading, setup.open, setupModelId]);
293
294 const previousOwnerRef = useRef(input.ownerKey);
295 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons any queued submission, and a recording that cannot outlive its editor, when the draft's owner changes
296 useEffect(() => {
297 const previous = previousOwnerRef.current;
298 if (previous === input.ownerKey && !input.disabled) return;
299 previousOwnerRef.current = input.ownerKey;
300 pendingSubmissionRef.current = null;
301 startRequestRef.current += 1;
302 if (previous !== input.ownerKey) session?.ownerChanged(previous);
303 else if (session?.ownerKey === input.ownerKey) session.cancel();
304 }, [input.ownerKey, input.disabled, session]);
305
306 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft
307 useEffect(() => {
308 const pending = pendingSubmissionRef.current;
309 if (state.phase === "error") pendingSubmissionRef.current = null;
310 if (state.phase !== "idle" || !pending) return;
311 pendingSubmissionRef.current = null;
312 if (!input.disabled && pending.ownerKey === input.ownerKey && pending.text !== undefined)
313 pending.submit(pending.text);
314 }, [state, input.ownerKey, input.disabled]);
315
316 const submitAfterDictation = useStableCallback((submit: (text: string) => void) => {
317 if (latestRef.current.input.disabled || state.phase === "preparing") return;
318 if (!session || !voiceInputBlocksSubmission(state)) {
319 submit(latestRef.current.input.readDraft().text);
320 return;
321 }
322 if (pendingSubmissionRef.current) return;
323 pendingSubmissionRef.current = { ownerKey: latestRef.current.input.ownerKey, submit };
324 if (session.state.phase === "recording") void session.stop();
325 });
326
327 const createTarget = useStableCallback((): VoiceInputTarget => {
328 const owner = latestRef.current.input.ownerKey;
329 const readContext = () => {
330 const current = latestRef.current;
331 return {
332 environmentId: current.input.environmentId,
333 projectName: current.input.projectName ?? null,
334 projectWords: current.projectWords,
335 clientSettings: current.clientSettings,
336 cleanup: current.input.cleanup !== false,
337 cleanupPrepared: current.cleanupPrepared,
338 };
339 };
340 // Once this editor shows another draft, its environment and cleanup must not see this one.
341 const startContext = readContext();
342 const persistent = latestRef.current.input.persistentDraft ?? null;
343 // While this editor shows the owner, the live editor is the draft; otherwise the stored one.
344 const live = () => mountedRef.current && latestRef.current.input.ownerKey === owner;
345 return {
346 ownerKey: owner,
347 label: latestRef.current.input.label ?? null,
348 persistent: persistent !== null,
349 open: persistent?.open ?? null,
350 readDraft: () => {
351 if (live()) return latestRef.current.input.readDraft();
352 const text = persistent?.readStoredText();
353 return text == null ? null : { text, selection: { start: text.length, end: text.length } };
354 },
355 commitDraft: (text, selection) => {
356 if (!live()) {
357 persistent?.commitStoredText(text);
358 return;
359 }
360 if (latestRef.current.input.disabled) return;
361 latestRef.current.input.commitDraft(text, selection);
362 if (pendingSubmissionRef.current?.ownerKey === owner)
363 pendingSubmissionRef.current.text = text;
364 },
365 readContext: () => (live() ? readContext() : startContext),
366 };
367 });
368
369 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- a remounted editor takes over its draft's running recording so a queued Send still sees the transcript
370 useEffect(() => {
371 if (session?.ownerKey === input.ownerKey && voiceInputBlocksSubmission(session.state))
372 session.adopt(createTarget());
373 }, [session, input.ownerKey, createTarget]);
374 const loadSetupCatalog = useStableCallback(
375 async (
376 connection: PreparedConnection,
377 selectedModelId: string,
378 benchmarkTier: SpeechBenchmarkTier,
379 request: number,
380 ) => {
381 setSetup({ ...INITIAL_SETUP, open: true });
382 const catalog = await runEnvironmentSpeech(
383 getEnvironmentSpeechModels(connection, true),
384 ).catch(() => null);
385 if (request !== startRequestRef.current) return;
386 const choice = chooseSetupSpeechModel(catalog?.models ?? [], {
387 selectedModelId,
388 intent: latestRef.current.clientSettings.speechLanguage,
389 preferred: preferredSpeechLanguages(),
390 benchmarkTier,
391 });
392 setSetup((current) => ({
393 ...current,
394 model: choice ? { ...choice.model, recommended: choice.recommended } : null,
395 language: choice?.language ?? null,
396 catalogFailed: catalog === null,
397 error: choice
398 ? null
399 : catalog
400 ? m(
401 msg(
402 "No model matches your language. Choose a model or change the language in Voice settings.",
403 ),
404 )
405 : m(msg("Could not load transcription models. Check the connection and try again.")),
406 }));
407 },
408 );
409
410 const start = useStableCallback(async () => {
411 if (latestRef.current.input.disabled || (setup.open && setup.downloading)) return;
412 if (!grantId) {
413 if (destination.offerFirstRunSetup && latestRef.current.input.offerSetup !== false)
414 await navigate({ to: "/settings/voice", hash: "voice-transcription-destination" });
415 return;
416 }
417 if (!session || voiceInputBlocksSubmission(session.state)) return;
418 const request = ++startRequestRef.current;
419 const owner = latestRef.current.input.ownerKey;
420 if (onDevice) {
421 await session.start(createTarget());
422 return;
423 }
424 if (!prepared) return;
425 if (cloudProvider) {
426 await session.start(createTarget());
427 return;
428 }
429 setSetup(INITIAL_SETUP);
430 const stillCurrent = () =>
431 request === startRequestRef.current &&
432 latestRef.current.session === session &&
433 latestRef.current.input.ownerKey === owner &&
434 !latestRef.current.input.disabled;
435 const handleStatus = async (latestStatus: EnvironmentSpeechStatus) => {
436 if (!stillCurrent()) return;
437 setStatus({ prepared, value: latestStatus });
438 if (!latestStatus.supported) {
439 session.cancel();
440 toastManager.add({
441 type: "error",
442 title: m(msg("Voice input is not available")),
443 description: latestStatus.reason,
444 });
445 } else if (latestStatus.state === "missing-model") {
446 session.cancel();
447 await loadSetupCatalog(prepared, latestStatus.modelId, latestStatus.benchmarkTier, request);
448 }
449 };
450
451 if (currentStatus?.supported && currentStatus.state === "missing-model") {
452 // Setup must remain usable without opening the microphone.
453 const latestStatus = await probeSpeechStatus(prepared, true).catch((error: unknown) => {
454 if (stillCurrent()) {
455 toastManager.add({
456 type: "error",
457 title: m(msg("Could not start voice input")),
458 description: error instanceof Error ? error.message : String(error),
459 });
460 }
461 return null;
462 });
463 if (!stillCurrent() || !latestStatus) return;
464 await handleStatus(latestStatus);
465 if (!latestStatus.supported || latestStatus.state === "missing-model" || !stillCurrent())
466 return;
467 }
468
469 await session.start(createTarget(), (latestStatus) => void handleStatus(latestStatus));
470 });
471
472 const downloadSetupModel = useStableCallback(async () => {
473 if (!prepared || !currentStatus?.supported || !setupModelId || setup.downloading) return;
474 const modelId = setupModelId;
475 const request = ++startRequestRef.current;
476 setSetup((current) => ({ ...current, downloading: true, error: null }));
477 try {
478 const downloaded = await runEnvironmentSpeech(
479 downloadEnvironmentSpeechModel(prepared, modelId),
480 );
481 if (request !== startRequestRef.current) return;
482 const outcome = getSpeechModelDownloadOutcome(downloaded, modelId);
483 if (outcome === "needs-selection") {
484 try {
485 await runEnvironmentSpeech(selectEnvironmentSpeechModel(prepared, modelId));
486 } catch {
487 if (request !== startRequestRef.current) return;
488 setSetup((current) => ({
489 ...current,
490 downloading: false,
491 error: m(
492 msg("The model downloaded, but this environment could not select it. Try again."),
493 ),
494 }));
495 return;
496 }
497 }
498 if (request !== startRequestRef.current) return;
499 const next = await probeSpeechStatus(prepared, true);
500 if (request !== startRequestRef.current) return;
501 setStatus({ prepared, value: next });
502 const ready =
503 outcome !== "incomplete" &&
504 next.supported &&
505 next.modelId === modelId &&
506 next.state !== "missing-model";
507 setSetup((current) => ({
508 ...current,
509 downloading: false,
510 step: ready ? 1 : current.step,
511 error: ready ? null : m(msg("The model is not ready. Try downloading it again.")),
512 }));
513 } catch {
514 if (request !== startRequestRef.current) return;
515 setSetup((current) => ({
516 ...current,
517 downloading: false,
518 error: m(
519 msg("Could not download the model. Check this environment's connection and try again."),
520 ),
521 }));
522 }
523 });
524
525 const cancelSetupDownload = useStableCallback(async () => {
526 if (!prepared || !setupModelId) return;
527 startRequestRef.current += 1;
528 setSetup((current) => ({ ...current, open: false, downloading: false }));
529 try {
530 await runEnvironmentSpeech(cancelEnvironmentSpeechModelDownload(prepared, setupModelId));
531 } catch (error) {
532 setSetup((current) => ({
533 ...current,
534 error: error instanceof Error ? error.message : String(error),
535 }));
536 }
537 });
538
539 const destinationReady =
540 grantId === null
541 ? Boolean(destination.offerFirstRunSetup) && input.offerSetup !== false
542 : onDevice ||
543 (prepared !== null &&
544 (cloudProvider !== undefined ||
545 (status?.prepared === prepared && status.value?.supported !== false)));
546 return {
547 available:
548 destinationReady &&
549 !busyElsewhere &&
550 typeof navigator !== "undefined" &&
551 Boolean(navigator.mediaDevices?.getUserMedia) &&
552 typeof AudioWorkletNode !== "undefined",
553 status: currentStatus,
554 transcriptionEnvironmentId: destination.environmentId,
555 transcriptionEnvironmentLabel: destination.label,
556 setup: {
557 ...setup,
558 setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
559 retryCatalog:
560 setup.open && setup.catalogFailed && prepared && currentStatus?.supported
561 ? () =>
562 loadSetupCatalog(
563 prepared,
564 currentStatus.modelId,
565 currentStatus.benchmarkTier,
566 ++startRequestRef.current,
567 )
568 : null,
569 download: downloadSetupModel,
570 cancelDownload: cancelSetupDownload,
571 startRecording: async () => {
572 setSetup((current) => ({ ...current, open: false }));
573 await start();
574 },
575 },
576 state,
577 preview:
578 state.phase === "recording" || state.phase === "transcribing" ? activity.preview : null,
579 level: activity.level,
580 recordingStartedAt: owns ? snapshot.recordingStartedAt : null,
581 preparing: state.phase === "preparing",
582 blocksSubmission: voiceInputBlocksSubmission(state),
583 freezesEditor: voiceInputBlocksSubmission(state),
584 start,
585 submitAfterDictation,
586 stop: useStableCallback(() => (owns && session ? session.stop() : Promise.resolve())),
587 cancel: useStableCallback(() => {
588 pendingSubmissionRef.current = null;
589 startRequestRef.current += 1;
590 setSetup(INITIAL_SETUP);
591 if (owns) session?.cancel();
592 }),
593 skipPostProcessing: useStableCallback(() => {
594 if (owns) session?.skipPostProcessing();
595 }),
596 };
597 }