@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 448 lines 16.4 KB
Raw
1 import {
2 postProcessEnvironmentTranscript,
3 VoiceInputController,
4 voiceInputBlocksSubmission,
5 type VoiceDraftSnapshot,
6 type VoiceInputControllerDependencies,
7 type VoiceInputErrorCode,
8 type VoiceInputState,
9 } from "@t3tools/client-runtime/voice-input";
10 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
11 import type {
12 CloudSpeechModelId,
13 CloudSpeechProvider,
14 ClientSettings,
15 EnvironmentId,
16 EnvironmentSpeechStatus,
17 SpeechCustomWords,
18 SpeechPostProcessingOptions,
19 SpeechStreamText,
20 SpeechTranscriptionOptions,
21 } from "@t3tools/contracts";
22 import { DEFAULT_CLIENT_SETTINGS } from "@t3tools/contracts";
23 import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
24 import { useEffect, useSyncExternalStore } from "react";
25
26 import { toastManager } from "../components/ui/toast";
27 import { chooseRecordingMicrophone } from "./browserMicrophones";
28 import {
29 createAppleSpeechVoiceInputPlatform,
30 createBrowserVoiceInputPlatform,
31 } from "./browserVoiceInput";
32 import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
33 import { bindAudioGrant, readCleanupGrant } from "./voiceConsent";
34 import { msg } from "gt-react";
35 import { m } from "../i18n/messages";
36
37 export const IDLE_VOICE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
38
39 type DraftInput = {
40 readonly text: string;
41 readonly selection: { readonly start: number; readonly end: number };
42 };
43
44 /** What a recording needs from its editor, read once when it starts. */
45 export type VoiceRecordingContext = {
46 readonly environmentId: EnvironmentId;
47 readonly projectName: string | null;
48 readonly projectWords: SpeechCustomWords;
49 readonly clientSettings: ClientSettings;
50 readonly cleanup: boolean;
51 readonly cleanupPrepared: PreparedConnection | null;
52 };
53
54 export type VoiceInputTarget = {
55 readonly ownerKey: string;
56 /** Names the draft in the indicator shown while its editor is off screen. */
57 readonly label: string | null;
58 /** A target that can outlive its editor (a thread draft) keeps recording when the owner changes. */
59 readonly persistent: boolean;
60 readonly open: (() => void) | null;
61 readonly readDraft: () => DraftInput | null;
62 readonly commitDraft: (text: string, selection: DraftInput["selection"]) => void;
63 readonly readContext: () => VoiceRecordingContext;
64 };
65
66 export type VoiceInputSessionSnapshot = {
67 readonly state: VoiceInputState;
68 readonly target: VoiceInputTarget | null;
69 /** Wall-clock start of the current recording, so every view shows the same elapsed time. */
70 readonly recordingStartedAt: number | null;
71 };
72
73 export type VoiceInputSessionActivity = {
74 readonly level: number;
75 readonly preview: SpeechStreamText | null;
76 };
77
78 type SessionKey = {
79 /** Null is Apple Speech, which transcribes on this Mac without an environment. */
80 readonly prepared: PreparedConnection | null;
81 readonly grantId: string;
82 readonly cloudProvider: CloudSpeechProvider | undefined;
83 readonly cloudModel: CloudSpeechModelId | undefined;
84 };
85
86 const SILENT_ACTIVITY: VoiceInputSessionActivity = { level: 0, preview: null };
87
88 function cleanupContext({ text, selection }: DraftInput): DraftInput {
89 const limit = 1_024;
90 const start = Math.max(0, Math.min(selection.start, text.length));
91 const end = Math.max(start, Math.min(selection.end, text.length));
92 const before = text.slice(Math.max(0, start - limit), start);
93 const selected = text.slice(start, Math.min(end, start + limit));
94 return {
95 text: before + selected + text.slice(end, end + limit),
96 selection: { start: before.length, end: before.length + selected.length },
97 };
98 }
99
100 /**
101 * The browser has one microphone, so every editor shares one dictation per destination. The
102 * recording belongs to the target that started it, not to whichever editor is mounted now.
103 */
104 export class VoiceInputSession {
105 readonly key: SessionKey;
106 private readonly controller: VoiceInputController;
107 private target: VoiceInputTarget | null = null;
108 private snapshot: VoiceInputSessionSnapshot = {
109 state: IDLE_VOICE_STATE,
110 target: null,
111 recordingStartedAt: null,
112 };
113 private activity = SILENT_ACTIVITY;
114 private readonly listeners = new Set<() => void>();
115 private readonly activityListeners = new Set<() => void>();
116 private disposed = false;
117 private onSpeechStatus: ((status: EnvironmentSpeechStatus) => void) | undefined;
118
119 constructor(key: SessionKey) {
120 this.key = key;
121 const hasAudioConsent = bindAudioGrant(key.grantId);
122 let recordingPreferences:
123 | {
124 readonly cleanup: SpeechPostProcessingOptions | null;
125 readonly cleanupGrant: string | null;
126 readonly cleanupPrepared: PreparedConnection | null;
127 }
128 | undefined;
129 const platformInput = {
130 hasAudioConsent,
131 getStatusListener: () => this.onSpeechStatus,
132 getStreamingEnabled: () =>
133 this.target?.readContext().clientSettings.speechStreaming ??
134 DEFAULT_CLIENT_SETTINGS.speechStreaming,
135 getTranscriptionOptions: (): SpeechTranscriptionOptions => {
136 const context = this.target?.readContext();
137 if (!context) throw new Error(m(msg("Voice input has no draft to record into.")));
138 const settings = context.clientSettings;
139 const cleanupGrant = context.cleanup ? readCleanupGrant() : null;
140 const cleanupEnabled = cleanupGrant !== null;
141 const cue = cleanupEnabled && context.cleanupPrepared ? settings.speechCorrectionWord : "";
142 const { speechCustomWords, speechHints } = resolveSpeechVocabulary({
143 personal: settings.speechCustomWords,
144 project: context.projectWords,
145 hints: [cue, context.projectName ?? ""],
146 });
147 recordingPreferences = {
148 cleanupGrant,
149 cleanupPrepared: context.cleanupPrepared,
150 cleanup: cleanupEnabled
151 ? {
152 speechCustomWords,
153 speechPostProcessingPrompt: settings.speechPostProcessingPrompt,
154 ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
155 }
156 : null,
157 };
158 return {
159 speechLanguage: settings.speechLanguage,
160 speechTranslateToEnglish: settings.speechTranslateToEnglish,
161 speechCustomWords,
162 ...(speechHints.length > 0 ? { speechHints } : {}),
163 speechRemoveFillerWords: settings.speechRemoveFillerWords,
164 ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
165 speechSilenceFiltering: settings.speechSilenceFiltering,
166 };
167 },
168 getMicrophoneId: chooseRecordingMicrophone,
169 onLevel: (level: number) => this.publishActivity({ ...this.activity, level }),
170 onDurationLimit: () => void this.controller.stop(),
171 onError: (message: string, code?: VoiceInputErrorCode) => {
172 if (!this.disposed) void this.controller.interruptRecording(message, null, code);
173 },
174 };
175 const platform = key.prepared
176 ? createBrowserVoiceInputPlatform({
177 ...platformInput,
178 prepared: key.prepared,
179 cloudProvider: key.cloudProvider,
180 cloudModel: key.cloudModel,
181 onText: (preview) => {
182 if (!this.disposed) this.publishActivity({ ...this.activity, preview });
183 },
184 })
185 : createAppleSpeechVoiceInputPlatform(platformInput);
186 const postProcess: NonNullable<VoiceInputControllerDependencies["postProcess"]> = (
187 transcript,
188 options,
189 ) => {
190 const preferences = recordingPreferences;
191 if (
192 !preferences?.cleanup ||
193 !hasAudioConsent() ||
194 preferences.cleanupGrant !== readCleanupGrant() ||
195 this.target?.ownerKey !== options.draft.ownerKey
196 )
197 return null;
198 if (!preferences.cleanupPrepared)
199 return Promise.reject(
200 new Error(m(msg("The draft's environment is unavailable for transcript cleanup."))),
201 );
202 return runEnvironmentSpeech(
203 postProcessEnvironmentTranscript(
204 preferences.cleanupPrepared,
205 transcript,
206 cleanupContext(options.draft),
207 preferences.cleanup,
208 () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
209 ),
210 options,
211 ).then((result) => result.text);
212 };
213 this.controller = new VoiceInputController({
214 resolve: m,
215 retryFailedRecording: true,
216 recorder: platform.recorder,
217 getTranscriber: () => platform.transcriber,
218 requestPermission: async () => ({ granted: true, canAskAgain: true }),
219 configureRecording: async () => undefined,
220 releaseRecording: async () => platform.cancelRecording(),
221 deleteRecording: platform.deleteRecording,
222 postProcess,
223 onPostProcessingError: () =>
224 toastManager.add({
225 type: "warning",
226 title: m(msg("Transcript cleanup failed")),
227 description: m(msg("The original transcript was added.")),
228 }),
229 readDraft: (): VoiceDraftSnapshot | null => {
230 const target = this.target;
231 const draft = target?.readDraft();
232 return target && draft ? { ownerKey: target.ownerKey, ...draft } : null;
233 },
234 commitDraft: (text, selection) => this.target?.commitDraft(text, selection),
235 onStateChange: (state) => {
236 if (this.disposed) return;
237 const recordingStartedAt =
238 state.phase !== "recording" ? null : (this.snapshot.recordingStartedAt ?? Date.now());
239 this.publish({ ...this.snapshot, state, recordingStartedAt });
240 if (state.phase !== "recording" && state.phase !== "transcribing")
241 this.publishActivity(SILENT_ACTIVITY);
242 },
243 });
244 }
245
246 get state(): VoiceInputState {
247 return this.controller.currentState;
248 }
249
250 get ownerKey(): string | null {
251 return this.target?.ownerKey ?? null;
252 }
253
254 readonly subscribe = (listener: () => void): (() => void) => {
255 this.listeners.add(listener);
256 return () => this.listeners.delete(listener);
257 };
258
259 readonly getSnapshot = (): VoiceInputSessionSnapshot => this.snapshot;
260
261 readonly subscribeActivity = (listener: () => void): (() => void) => {
262 this.activityListeners.add(listener);
263 return () => this.activityListeners.delete(listener);
264 };
265
266 readonly getActivity = (): VoiceInputSessionActivity => this.activity;
267
268 /** Starts recording into `target`, or does nothing while another draft holds the microphone. */
269 start(
270 target: VoiceInputTarget,
271 onSpeechStatus?: (status: EnvironmentSpeechStatus) => void,
272 ): Promise<void> {
273 if (this.disposed) return Promise.resolve();
274 const busyElsewhere =
275 voiceInputBlocksSubmission(this.controller.currentState) &&
276 this.target?.ownerKey !== target.ownerKey;
277 if (busyElsewhere) return Promise.resolve();
278 this.target = target;
279 this.onSpeechStatus = onSpeechStatus;
280 this.publishActivity(SILENT_ACTIVITY);
281 this.publish({ ...this.snapshot, target });
282 return this.controller.start();
283 }
284
285 /** A newly mounted editor for the recording's draft replaces the target that started it. */
286 adopt(target: VoiceInputTarget): void {
287 if (this.disposed || this.target?.ownerKey !== target.ownerKey) return;
288 this.target = target;
289 this.publish({ ...this.snapshot, target });
290 }
291
292 stop(): Promise<void> {
293 return this.controller.stop();
294 }
295
296 cancel(): void {
297 this.controller.cancel();
298 }
299
300 skipPostProcessing(): void {
301 this.controller.skipPostProcessing();
302 }
303
304 /** The editor for `ownerKey` now shows another draft; only a persistent target keeps going. */
305 ownerChanged(ownerKey: string): void {
306 if (this.target?.ownerKey !== ownerKey || this.target.persistent) return;
307 this.controller.ownerChanged();
308 }
309
310 dispose(): void {
311 this.disposed = true;
312 this.controller.dispose();
313 this.listeners.clear();
314 this.activityListeners.clear();
315 }
316
317 private publish(snapshot: VoiceInputSessionSnapshot): void {
318 this.snapshot = snapshot;
319 for (const listener of this.listeners) listener();
320 }
321
322 private publishActivity(activity: VoiceInputSessionActivity): void {
323 if (activity.level === this.activity.level && activity.preview === this.activity.preview)
324 return;
325 this.activity = activity;
326 for (const listener of this.activityListeners) listener();
327 }
328 }
329
330 const sessions = new Map<VoiceInputSession, number>();
331 const registryListeners = new Set<() => void>();
332
333 function sameKey(session: VoiceInputSession, key: SessionKey): boolean {
334 return (
335 session.key.prepared === key.prepared &&
336 session.key.grantId === key.grantId &&
337 session.key.cloudProvider === key.cloudProvider &&
338 session.key.cloudModel === key.cloudModel
339 );
340 }
341
342 function findSession(key: SessionKey): VoiceInputSession | null {
343 for (const session of sessions.keys()) if (sameKey(session, key)) return session;
344 return null;
345 }
346
347 /** Every holder of a destination shares one session; the last release disposes it. */
348 function retainVoiceInputSession(key: SessionKey): () => void {
349 const session = findSession(key) ?? new VoiceInputSession(key);
350 sessions.set(session, (sessions.get(session) ?? 0) + 1);
351 for (const listener of registryListeners) listener();
352 let released = false;
353 return () => {
354 if (released) return;
355 released = true;
356 const count = (sessions.get(session) ?? 1) - 1;
357 if (count > 0) {
358 sessions.set(session, count);
359 return;
360 }
361 sessions.delete(session);
362 session.dispose();
363 for (const listener of registryListeners) listener();
364 };
365 }
366
367 function subscribeRegistry(listener: () => void): () => void {
368 registryListeners.add(listener);
369 return () => registryListeners.delete(listener);
370 }
371
372 /** Holds the destination's shared session while mounted; null until it is retained. */
373 export function useRetainedVoiceInputSession(input: {
374 readonly prepared: PreparedConnection | null;
375 readonly grantId: string | null;
376 readonly onDevice?: boolean | undefined;
377 readonly cloudProvider?: CloudSpeechProvider | undefined;
378 readonly cloudModel?: CloudSpeechModelId | undefined;
379 }): VoiceInputSession | null {
380 const { grantId, cloudProvider, cloudModel } = input;
381 const prepared = input.onDevice ? null : input.prepared;
382 const ready = (input.onDevice === true || prepared !== null) && grantId !== null;
383 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- retains the shared recording session for this destination while mounted
384 useEffect(() => {
385 if (!ready || !grantId || typeof navigator === "undefined") return;
386 return retainVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel });
387 }, [ready, prepared, grantId, cloudProvider, cloudModel]);
388 return useSyncExternalStore(subscribeRegistry, () =>
389 ready && grantId ? findSession({ prepared, grantId, cloudProvider, cloudModel }) : null,
390 );
391 }
392
393 const IDLE_SNAPSHOT: VoiceInputSessionSnapshot = {
394 state: IDLE_VOICE_STATE,
395 target: null,
396 recordingStartedAt: null,
397 };
398 const noSubscription = () => () => {};
399
400 export function useVoiceInputSessionSnapshot(
401 session: VoiceInputSession | null,
402 ): VoiceInputSessionSnapshot {
403 return useSyncExternalStore(
404 session?.subscribe ?? noSubscription,
405 session ? session.getSnapshot : () => IDLE_SNAPSHOT,
406 );
407 }
408
409 export function useVoiceInputSessionActivity(
410 session: VoiceInputSession | null,
411 ownerKey: string | null,
412 ): VoiceInputSessionActivity {
413 // Activity changes many times a second, so only the owning editor reads it.
414 const owns = useSyncExternalStore(
415 session?.subscribe ?? noSubscription,
416 () => session !== null && ownerKey !== null && session.ownerKey === ownerKey,
417 );
418 return useSyncExternalStore(owns && session ? session.subscribeActivity : noSubscription, () =>
419 owns && session ? session.getActivity() : SILENT_ACTIVITY,
420 );
421 }
422
423 // Editors register the owner they show, so the indicator appears only while its draft is off screen.
424 const presentedOwners = new Map<string, number>();
425 const presentationListeners = new Set<() => void>();
426
427 export function presentVoiceOwner(ownerKey: string): () => void {
428 presentedOwners.set(ownerKey, (presentedOwners.get(ownerKey) ?? 0) + 1);
429 for (const listener of presentationListeners) listener();
430 return () => {
431 const count = (presentedOwners.get(ownerKey) ?? 1) - 1;
432 if (count > 0) presentedOwners.set(ownerKey, count);
433 else presentedOwners.delete(ownerKey);
434 for (const listener of presentationListeners) listener();
435 };
436 }
437
438 function subscribePresentation(listener: () => void): () => void {
439 presentationListeners.add(listener);
440 return () => presentationListeners.delete(listener);
441 }
442
443 export function useVoiceOwnerPresented(ownerKey: string | null): boolean {
444 return useSyncExternalStore(
445 subscribePresentation,
446 () => ownerKey !== null && presentedOwners.has(ownerKey),
447 );
448 }