The best way to control Claude Code, Codex and more from any computer or phone
| 1 | import { thrownErrorMessage } from "@t3tools/client-runtime/errors"; |
| 2 | import { m } from "~/i18n/messages"; |
| 3 | import { scopeProjectRef } from "@t3tools/client-runtime/environment"; |
| 4 | import type { |
| 5 | DesktopPreviewAnnotationVoiceConfig, |
| 6 | DesktopPreviewAnnotationVoiceEvent, |
| 7 | ScopedThreadRef, |
| 8 | } from "@t3tools/contracts"; |
| 9 | import { useNavigate } from "@tanstack/react-router"; |
| 10 | import { useGT, useMessages } from "gt-react"; |
| 11 | import { useEffect, useRef } from "react"; |
| 12 | |
| 13 | import { resolveSpeechPresentation } from "~/components/chat/ComposerSpeechButton"; |
| 14 | import { VoiceInputSetup } from "~/components/chat/VoiceInputSetup"; |
| 15 | import { toastManager } from "~/components/ui/toast"; |
| 16 | import { useComposerDraftStore } from "~/composerDraftStore"; |
| 17 | import { useStableCallback } from "~/hooks/useStableCallback"; |
| 18 | import { useDictationShortcut, type DictationKeyHandlers } from "~/speech/useDictationShortcut"; |
| 19 | import { useEnvironmentSpeechInput } from "~/speech/useEnvironmentSpeechInput"; |
| 20 | import { useProject, useThreadShell } from "~/state/entities"; |
| 21 | import { previewBridge } from "./previewBridge"; |
| 22 | |
| 23 | /** |
| 24 | * The annotation editor lives inside the preview guest, which only forwards trusted gestures |
| 25 | * and mirrors state. Recording, consent, and transcription stay in this renderer. |
| 26 | */ |
| 27 | export function PreviewAnnotationSpeech({ |
| 28 | threadRef, |
| 29 | tabId, |
| 30 | config, |
| 31 | keybindings, |
| 32 | }: { |
| 33 | threadRef: ScopedThreadRef; |
| 34 | tabId: string; |
| 35 | config: DesktopPreviewAnnotationVoiceConfig; |
| 36 | keybindings: Parameters<typeof useDictationShortcut>[0]["keybindings"]; |
| 37 | }) { |
| 38 | const gt = useGT(); |
| 39 | const messages = useMessages(); |
| 40 | const navigate = useNavigate(); |
| 41 | const thread = useThreadShell(threadRef); |
| 42 | const draftThread = useComposerDraftStore((store) => store.getDraftThreadByRef(threadRef)); |
| 43 | const projectId = thread?.projectId ?? draftThread?.projectId ?? null; |
| 44 | const project = useProject( |
| 45 | projectId ? scopeProjectRef(threadRef.environmentId, projectId) : null, |
| 46 | ); |
| 47 | const draft = useRef({ text: "", cursor: 0, selectionEnd: 0 }); |
| 48 | const keys = useRef<DictationKeyHandlers | null>(null); |
| 49 | const guestFocused = useRef(false); |
| 50 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- notices focus returning from the preview guest to this window |
| 51 | useEffect(() => { |
| 52 | const onFocus = () => { |
| 53 | guestFocused.current = false; |
| 54 | }; |
| 55 | window.addEventListener("focus", onFocus); |
| 56 | window.addEventListener("focusin", onFocus); |
| 57 | return () => { |
| 58 | window.removeEventListener("focus", onFocus); |
| 59 | window.removeEventListener("focusin", onFocus); |
| 60 | }; |
| 61 | }, []); |
| 62 | const speech = useEnvironmentSpeechInput({ |
| 63 | environmentId: threadRef.environmentId, |
| 64 | projectId, |
| 65 | projectName: project?.title, |
| 66 | ownerKey: JSON.stringify(["preview-annotation", threadRef, tabId, config.sessionId]), |
| 67 | offerSetup: false, |
| 68 | readDraft: () => ({ |
| 69 | text: draft.current.text, |
| 70 | selection: { start: draft.current.cursor, end: draft.current.selectionEnd }, |
| 71 | }), |
| 72 | commitDraft: (next, selection) => { |
| 73 | draft.current = { text: next, cursor: selection.start, selectionEnd: selection.end }; |
| 74 | void publish(current.current.voiceState, { text: next, cursor: selection.start }); |
| 75 | }, |
| 76 | }); |
| 77 | const subscribeKeys = useStableCallback((handlers: DictationKeyHandlers) => { |
| 78 | keys.current = handlers; |
| 79 | return () => { |
| 80 | keys.current = null; |
| 81 | }; |
| 82 | }); |
| 83 | const shortcutLabel = useDictationShortcut({ |
| 84 | keybindings, |
| 85 | speech, |
| 86 | disabled: false, |
| 87 | terminalOpen: false, |
| 88 | modelPickerOpen: false, |
| 89 | previewFocus: true, |
| 90 | ownsFocus: () => guestFocused.current, |
| 91 | subscribeKeys, |
| 92 | }); |
| 93 | const { available, state, preview, freezesEditor, blocksSubmission, level } = speech; |
| 94 | const voiceState = { |
| 95 | tabId, |
| 96 | sessionId: config.sessionId, |
| 97 | available, |
| 98 | phase: state.phase, |
| 99 | errorAction: state.errorAction, |
| 100 | status: resolveSpeechPresentation(state, null, messages).status, |
| 101 | preview: preview ? preview.committed + preview.tentative : null, |
| 102 | level, |
| 103 | shortcutLabel, |
| 104 | freezesEditor, |
| 105 | blocksSubmission, |
| 106 | }; |
| 107 | const current = useRef({ speech, voiceState }); |
| 108 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed speech actions and state to the desktop IPC listener |
| 109 | useEffect(() => { |
| 110 | current.current = { speech, voiceState }; |
| 111 | }); |
| 112 | const publish = useStableCallback( |
| 113 | async (voice: typeof voiceState, nextDraft: { text: string; cursor: number } | null) => { |
| 114 | try { |
| 115 | await previewBridge?.annotationVoice.update({ ...voice, draft: nextDraft }); |
| 116 | } catch (error) { |
| 117 | current.current.speech.cancel(); |
| 118 | toastManager.add({ |
| 119 | type: "error", |
| 120 | title: gt("Could not update annotation voice input"), |
| 121 | description: thrownErrorMessage(error, m) ?? String(error), |
| 122 | }); |
| 123 | } |
| 124 | }, |
| 125 | ); |
| 126 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- mirrors every committed voice state change into the preview guest |
| 127 | useEffect(() => { |
| 128 | void publish(voiceState, null); |
| 129 | }, [publish, voiceState]); |
| 130 | // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to gestures the preview guest relays for this session |
| 131 | useEffect( |
| 132 | () => |
| 133 | previewBridge?.annotationVoice.onEvent( |
| 134 | (eventTabId, event: DesktopPreviewAnnotationVoiceEvent) => { |
| 135 | if (eventTabId !== tabId || event.sessionId !== config.sessionId) return; |
| 136 | draft.current = { |
| 137 | text: event.text, |
| 138 | cursor: Math.min(event.cursor, event.text.length), |
| 139 | selectionEnd: Math.min(event.selectionEnd ?? event.cursor, event.text.length), |
| 140 | }; |
| 141 | if (event.action === "focus" || event.action === "key" || event.action === "start") |
| 142 | guestFocused.current = true; |
| 143 | if (event.action === "blur") guestFocused.current = false; |
| 144 | const value = current.current.speech; |
| 145 | switch (event.action) { |
| 146 | case "focus": |
| 147 | case "sync": |
| 148 | void publish(current.current.voiceState, null); |
| 149 | break; |
| 150 | case "start": |
| 151 | void value.start(); |
| 152 | break; |
| 153 | case "stop": |
| 154 | void value.stop(); |
| 155 | break; |
| 156 | case "cancel": |
| 157 | value.cancel(); |
| 158 | break; |
| 159 | case "settings": |
| 160 | value.cancel(); |
| 161 | void navigate({ to: "/settings/voice" }); |
| 162 | break; |
| 163 | case "skip": |
| 164 | value.skipPostProcessing(); |
| 165 | break; |
| 166 | case "blur": |
| 167 | keys.current?.blur(); |
| 168 | break; |
| 169 | case "key": { |
| 170 | if (!event.keyboard) break; |
| 171 | keys.current?.[event.keyboard.type]( |
| 172 | new KeyboardEvent(event.keyboard.type, event.keyboard), |
| 173 | ); |
| 174 | break; |
| 175 | } |
| 176 | } |
| 177 | }, |
| 178 | ), |
| 179 | [tabId, config.sessionId, publish, navigate], |
| 180 | ); |
| 181 | return ( |
| 182 | <VoiceInputSetup |
| 183 | environmentId={speech.transcriptionEnvironmentId} |
| 184 | open={speech.setup.open} |
| 185 | step={speech.setup.step} |
| 186 | status={speech.status} |
| 187 | model={speech.setup.model} |
| 188 | language={speech.setup.language} |
| 189 | downloading={speech.setup.downloading} |
| 190 | error={speech.setup.error} |
| 191 | onOpenChange={speech.setup.setOpen} |
| 192 | onRetryCatalog={speech.setup.retryCatalog} |
| 193 | onDownload={() => void speech.setup.download()} |
| 194 | onCancelDownload={() => void speech.setup.cancelDownload()} |
| 195 | onStartRecording={() => void speech.setup.startRecording()} |
| 196 | /> |
| 197 | ); |
| 198 | } |