@devries-cameron20 / t3code Mirror · read-only / commits / c24255826b
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 09:45 UTC

feat(web): dictation keeps recording into its thread when you switch away (#35761)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Nov 16, 2026 at 13:45 UTC c24255826be5aabb0f32ee18f6ac7dec0b3f1347
13 files changed +755 -206
apps/web/src/components/AppRouteModal.browser.test.tsx
+2
@@ -5,6 +5,8 @@ import { render } from "../../test/render";
5 import { useCommitOnBlur } from "../hooks/useCommitOnBlur";
6 import { AppRouteModal } from "./AppRouteModal";
7
8 +vi.mock("./chat/GlobalVoiceInputIndicator", () => ({ GlobalVoiceInputIndicator: () => null }));
9 +
10 function CommitOnBlurField({ onCommit }: { onCommit: (next: string) => void }) {
11 return <input aria-label="Instructions" {...useCommitOnBlur("", onCommit)} />;
12 }
apps/web/src/components/AppRouteModal.tsx
+2
@@ -4,6 +4,7 @@ import { useRef, useState, type CSSProperties, type ReactNode } from "react";
4
5 import { useNativeTitlebarHeight } from "../lib/windowControlsOverlay";
6 import { useCloseAppModal } from "./sidebar/mainAppLocation";
7 +import { GlobalVoiceInputIndicator } from "./chat/GlobalVoiceInputIndicator";
8 import { Button } from "./ui/button";
9 import { DialogBackdrop } from "./ui/dialog";
10 import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip";
@@ -89,6 +90,7 @@ export function AppRouteModal({ label, children }: { label: string; children: Re
90 className="absolute inset-x-0 top-0 hidden h-(--app-modal-titlebar-height) [-webkit-app-region:drag] max-sm:block"
91 />
92 {children}
93 + <GlobalVoiceInputIndicator />
94 </DialogPrimitive.Popup>
95 </DialogPrimitive.Viewport>
96 </DialogPrimitive.Portal>
apps/web/src/components/ThreadRouteView.tsx
+4 -2
@@ -8,7 +8,7 @@ import { Navigate, useNavigate } from "@tanstack/react-router";
8 import { useEffect, useState } from "react";
9
10 import ChatView from "./ChatView";
11 -import { BackgroundThreadContext } from "./chat/backgroundThread";
11 +import { BackgroundThreadContext, HiddenThreadContext } from "./chat/backgroundThread";
12 import { SidebarInset } from "./ui/sidebar";
13 import { useComposerDraftStore } from "../composerDraftStore";
14 import { useSidebarPendingFileDropStore } from "../sidebarPendingFileDropStore";
@@ -229,7 +229,9 @@ export function ThreadRouteView({
229 className="h-svh min-h-0 overflow-hidden overscroll-y-none md:h-dvh"
230 inert={background}
231 >
232 - <BackgroundThreadContext value={background}>{view}</BackgroundThreadContext>
232 + <BackgroundThreadContext value={background}>
233 + <HiddenThreadContext value={background}>{view}</HiddenThreadContext>
234 + </BackgroundThreadContext>
235 {background ? null : redirect}
236 </SidebarInset>
237 );
apps/web/src/components/chat/ChatComposer.tsx
+29 -1
@@ -4,7 +4,7 @@ import { useComposerTypingGuard } from "./useComposerTypingGuard";
4 import { insertAttachmentReferencesForDraft } from "./composerAttachmentReferences";
5 import type { ComposerAttachmentInsertion } from "../composerAttachmentInsertion";
6 import { DESKTOP_PASTE_AS_TEXT_EVENT } from "../../lib/desktopPasteAsText";
7 -import { BackgroundThreadContext } from "./backgroundThread";
7 +import { BackgroundThreadContext, HiddenThreadContext } from "./backgroundThread";
8 import {
9 resolveCompatibleRuntimeMode,
10 runtimeModeConfig,
@@ -107,9 +107,12 @@ import {
107 composerFileNeedsReattach,
108 composerDraftHasUserContent,
109 composerTargetKey,
110 + readComposerDraftPrompt,
111 + resolveLiveComposerDraftTarget,
112 useComposerDraftStore,
113 useComposerThreadDraft,
114 } from "../../composerDraftStore";
115 +import { buildDraftThreadRouteParams, buildThreadRouteParams } from "../../threadRoutes";
116 import { ComposerStashBadge } from "./ComposerStashBadge";
117 import { ComposerStashMenu } from "./ComposerStashMenu";
118 import { useComposerTriggerState } from "./useComposerTriggerState";
@@ -1052,6 +1055,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
1055 });
1056
1057 const backgroundThread = use(BackgroundThreadContext);
1058 + const hiddenThread = use(HiddenThreadContext);
1059 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to keydown, window blur and the desktop paste-as-text event to arm plain-text paste
1060 useEffect(() => {
1061 // A background composer listens on its own form, so it never hears the main composer's keys.
@@ -1846,6 +1850,29 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
1850 projectName,
1851 ownerKey: questionAttachmentTarget ?? composerDraftTargetKey,
1852 disabled: dictationDisabled,
1853 + label: activeThread?.title ?? null,
1854 + presented: !hiddenThread,
1855 + // A question's answer lives only in this composer, so only the thread draft outlives it.
1856 + persistentDraft: questionAttachmentTarget
1857 + ? undefined
1858 + : {
1859 + readStoredText: () => readComposerDraftPrompt(composerDraftTarget),
1860 + commitStoredText: (text) => {
1861 + const live = resolveLiveComposerDraftTarget(composerDraftTarget);
1862 + if (live !== null) setComposerDraftPrompt(live, text);
1863 + },
1864 + // The route, not the draft target: a queued-message edit has a draft id but no route.
1865 + open: () =>
1866 + void (routeKind === "draft" && typeof composerDraftTarget === "string"
1867 + ? navigate({
1868 + to: "/draft/$draftId",
1869 + params: buildDraftThreadRouteParams(composerDraftTarget),
1870 + })
1871 + : navigate({
1872 + to: "/$environmentId/$threadId",
1873 + params: buildThreadRouteParams(routeThreadRef),
1874 + })),
1875 + },
1876 readDraft: () => {
1877 const snapshot = readComposerSnapshot();
1878 return {
@@ -3828,6 +3855,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
3855 finishShortcutLabel={dictationShortcutLabel}
3856 cancelShortcutLabel="Esc"
3857 level={speechInput.level}
3858 + recordingStartedAt={speechInput.recordingStartedAt}
3859 disabled={dictationDisabled}
3860 onStart={() => void speechInput.start()}
3861 onStop={() => void speechInput.stop()}
apps/web/src/components/chat/ComposerSpeechButton.tsx
+13 -4
@@ -43,18 +43,25 @@ function formatElapsed(seconds: number) {
43 return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
44 }
45
46 -function RecordingStatus(props: { level: number; recordingLimitSeconds?: number | undefined }) {
47 - const [elapsedSeconds, setElapsedSeconds] = useState(0);
46 +function RecordingStatus(props: {
47 + level: number;
48 + recordingStartedAt?: number | null | undefined;
49 + recordingLimitSeconds?: number | undefined;
50 +}) {
51 + const { recordingStartedAt } = props;
52 + const [elapsedSeconds, setElapsedSeconds] = useState(() =>
53 + recordingStartedAt ? Math.max(0, Math.floor((Date.now() - recordingStartedAt) / 1_000)) : 0,
54 + );
55
56 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the timer displaying the native recording duration
57 useEffect(() => {
51 - const startedAt = Date.now();
58 + const startedAt = recordingStartedAt ?? Date.now();
59 const timer = window.setInterval(
60 () => setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1_000)),
61 250,
62 );
63 return () => window.clearInterval(timer);
57 - }, []);
64 + }, [recordingStartedAt]);
65
66 return (
67 <>
@@ -125,6 +132,7 @@ export function ComposerSpeechRecordingPill(props: {
132 finishShortcutLabel?: string | null;
133 cancelShortcutLabel?: string | null;
134 level: number;
135 + recordingStartedAt?: number | null | undefined;
136 recordingLimitSeconds?: number | undefined;
137 /** Compact matches the `xs` buttons beside text-field editors. */
138 size?: "default" | "compact";
@@ -181,6 +189,7 @@ export function ComposerSpeechRecordingPill(props: {
189 {recording ? (
190 <RecordingStatus
191 level={props.level}
192 + recordingStartedAt={props.recordingStartedAt}
193 recordingLimitSeconds={props.recordingLimitSeconds}
194 />
195 ) : (
apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx new
+52
@@ -0,0 +1,52 @@
1 +import { useVoiceTranscriptionDestination } from "../../speech/useEnvironmentSpeechInput";
2 +import {
3 + useRetainedVoiceInputSession,
4 + useVoiceInputSessionActivity,
5 + useVoiceInputSessionSnapshot,
6 + useVoiceOwnerPresented,
7 +} from "../../speech/voiceInputSession";
8 +import { Button } from "../ui/button";
9 +import { ComposerSpeechRecordingPill } from "./ComposerSpeechButton";
10 +
11 +/**
12 + * Keeps the chosen destination's dictation alive across navigation and, while the draft it
13 + * records into is off screen, shows it so it can be finished, cancelled, or returned to.
14 + */
15 +export function GlobalVoiceInputIndicator({ hidden = false }: { hidden?: boolean }) {
16 + const { prepared, grantId, cloudProvider, cloudModel } = useVoiceTranscriptionDestination();
17 + const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel });
18 + const { state, target, recordingStartedAt } = useVoiceInputSessionSnapshot(session);
19 + const presented = useVoiceOwnerPresented(target?.ownerKey ?? null);
20 + const { level } = useVoiceInputSessionActivity(
21 + hidden || presented ? null : session,
22 + target?.ownerKey ?? null,
23 + );
24 + if (hidden || !session || !target || state.phase === "idle" || presented) return null;
25 +
26 + return (
27 + <div className="pointer-events-none fixed inset-x-0 top-3 z-100 flex justify-center px-4">
28 + <div className="pointer-events-auto flex max-w-full items-center gap-1 rounded-full bg-popover p-0.5 shadow-lg/5">
29 + <Button
30 + type="button"
31 + size="sm-pill"
32 + variant="ghost"
33 + disabled={!target.open}
34 + onClick={() => target.open?.()}
35 + className="min-w-0 shrink"
36 + >
37 + <span className="truncate">{target.label ?? "Untitled thread"}</span>
38 + </Button>
39 + <ComposerSpeechRecordingPill
40 + state={state}
41 + progress={null}
42 + level={level}
43 + recordingStartedAt={recordingStartedAt}
44 + onStart={() => void session.start(target)}
45 + onStop={() => void session.stop()}
46 + onCancel={() => session.cancel()}
47 + onSkipPostProcessing={() => session.skipPostProcessing()}
48 + />
49 + </div>
50 + </div>
51 + );
52 +}
apps/web/src/components/chat/backgroundThread.ts
+3
@@ -5,3 +5,6 @@ import { createContext } from "react";
5 * parent's panel; its global shortcuts and listeners stand down.
6 */
7 export const BackgroundThreadContext = createContext(false);
8 +
9 +/** True only for a thread kept mounted out of sight under Settings or Usage. */
10 +export const HiddenThreadContext = createContext(false);
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+1
@@ -328,6 +328,7 @@ it("tests the selected model after download and preserves its transcript across
328 state: { phase: "idle", error: null, errorAction: null },
329 preview: null,
330 level: 0,
331 + recordingStartedAt: null,
332 preparing: false,
333 blocksSubmission: false,
334 freezesEditor: false,
apps/web/src/composerDraftStore.ts
+21
@@ -1660,6 +1660,27 @@ function getComposerDraftState(
1660 return state.draftsByThreadKey[threadKey] ?? null;
1661 }
1662
1663 +/** Where a draft lives now: its thread once promoted, or null once it was discarded. */
1664 +export function resolveLiveComposerDraftTarget(
1665 + target: ComposerThreadTarget,
1666 +): ComposerThreadTarget | null {
1667 + if (typeof target !== "string") return target;
1668 + const state = composerDraftStore.getState();
1669 + const promoted = state.promotedDraftRedirects[target];
1670 + if (promoted) return promoted;
1671 + return state.draftThreadsByThreadKey[target] || state.draftsByThreadKey[target] ? target : null;
1672 +}
1673 +
1674 +/** The stored prompt for a target, empty when it has no draft, or null when the target is gone. */
1675 +export function readComposerDraftPrompt(target: ComposerThreadTarget): string | null {
1676 + const live = resolveLiveComposerDraftTarget(target);
1677 + if (live === null) return null;
1678 + const state = composerDraftStore.getState();
1679 + const threadKey = resolveComposerDraftKey(state, live);
1680 + if (!threadKey) return null;
1681 + return state.draftsByThreadKey[threadKey]?.prompt ?? "";
1682 +}
1683 +
1684 function draftThreadHasOpenPreview(draftThread: DraftThreadState | undefined): boolean {
1685 if (draftThread === undefined) return false;
1686 const ref = scopeThreadRef(draftThread.environmentId, draftThread.threadId);
apps/web/src/routes/_chat.tsx
+2
@@ -1,6 +1,7 @@
1 import { Outlet, createFileRoute, redirect } from "@tanstack/react-router";
2
3 import { ThreadRouteView } from "../components/ThreadRouteView";
4 +import { GlobalVoiceInputIndicator } from "../components/chat/GlobalVoiceInputIndicator";
5 import { isAppModalPage } from "../components/sidebar/mainAppLocation";
6 import { SidebarInset } from "../components/ui/sidebar";
7 import { resolveThreadRouteTarget } from "../threadRoutes";
@@ -24,6 +25,7 @@ function ChatRouteLayout() {
25 <Outlet />
26 )}
27 {modalOpen ? <Outlet /> : null}
28 + <GlobalVoiceInputIndicator hidden={modalOpen} />
29 </>
30 );
31 }
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+80 -1
@@ -18,6 +18,8 @@ import {
18 } from "@t3tools/contracts";
19
20 import { act, render, stubGlobal } from "../../test/render";
21 +import { AppRouteModal } from "../components/AppRouteModal";
22 +import { GlobalVoiceInputIndicator } from "../components/chat/GlobalVoiceInputIndicator";
23 import { VoiceInputSetup } from "../components/chat/VoiceInputSetup";
24 import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
25 import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
@@ -58,8 +60,12 @@ const mocks = vi.hoisted(() => ({
60 transcripts: [] as Array<string | Error>,
61 committed: vi.fn(),
62 draft: "",
63 + storedDraft: "",
64 }));
65
66 +vi.mock("../components/sidebar/mainAppLocation", () => ({
67 + useCloseAppModal: () => () => Promise.resolve(),
68 +}));
69 vi.mock("../state/session", () => ({
70 useEnvironmentScope: (id: string) => mocks.canDictate.has(id),
71 usePreparedConnection: (environmentId: string | null) => {
@@ -236,6 +242,7 @@ function setup(options: { readonly destination?: string | null } = {}) {
242 mocks.transcripts = [];
243 mocks.committed.mockClear();
244 mocks.draft = "";
245 + mocks.storedDraft = "";
246 }
247
248 function setDestination(environmentId: string | null) {
@@ -249,13 +256,24 @@ function Probe(props: {
256 readonly projectName?: string;
257 readonly cleanup?: boolean;
258 readonly showWizard?: boolean;
259 + readonly ownerKey?: string;
260 + readonly persistent?: boolean;
261 }) {
262 const value = useEnvironmentSpeechInput({
263 environmentId: THREAD_ENV as EnvironmentId,
264 projectId: props.projectId as ProjectId | undefined,
265 projectName: props.projectName,
266 cleanup: props.cleanup,
258 - ownerKey: "draft",
267 + ownerKey: props.ownerKey ?? "draft",
268 + persistentDraft: props.persistent
269 + ? {
270 + readStoredText: () => mocks.storedDraft,
271 + commitStoredText: (text) => {
272 + mocks.storedDraft = text;
273 + },
274 + open: () => {},
275 + }
276 + : undefined,
277 readDraft: () => ({
278 text: mocks.draft,
279 selection: { start: mocks.draft.length, end: mocks.draft.length },
@@ -376,6 +394,67 @@ it("retries a failed transcription on the same chosen environment", async () =>
394 expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("second try", expect.anything());
395 });
396
397 +it.each([
398 + ["a stored thread draft keeps recording into it", true],
399 + ["an editor-only draft is abandoned", false],
400 +])("when the editor switches owner mid-recording, %s", async (_name, persistent) => {
401 + setup({ destination: "env-a" });
402 + mocks.storedDraft = "Earlier text";
403 + const view = (ownerKey: string) => (
404 + <>
405 + <Probe ownerKey={ownerKey} persistent={persistent} />
406 + <GlobalVoiceInputIndicator />
407 + </>
408 + );
409 + await using screen = await render(view("thread-a"));
410 + await act(() => mocks.voice.start());
411 + expect(mocks.voice.state.phase).toBe("recording");
412 + // The indicator stays away while the recording's own editor is on screen.
413 + expect(page.getByRole("button", { name: "Finish voice input" }).query()).toBeNull();
414 +
415 + await screen.rerender(view("thread-b"));
416 + // One microphone: thread-b cannot start its own recording while thread-a's runs.
417 + expect(mocks.voice.available).toBe(!persistent);
418 + if (!persistent) {
419 + await expect
420 + .element(page.getByRole("button", { name: "Finish voice input" }))
421 + .not.toBeInTheDocument();
422 + expect(mocks.transcribeCalls).toEqual([]);
423 + expect(mocks.storedDraft).toBe("Earlier text");
424 + return;
425 + }
426 + await page.getByRole("button", { name: "Finish voice input" }).click();
427 +
428 + await vi.waitFor(() => expect(mocks.storedDraft).toBe("Earlier text dictated"));
429 + expect(mocks.transcribeCalls).toEqual(["env-a"]);
430 + expect(mocks.committed).not.toHaveBeenCalled();
431 +});
432 +
433 +it("finishes an off-screen recording with the keyboard while Settings is open", async () => {
434 + setup({ destination: "env-a" });
435 + mocks.storedDraft = "Earlier text";
436 + await using screen = await render(
437 + <>
438 + <Probe ownerKey="thread-a" persistent />
439 + <GlobalVoiceInputIndicator />
440 + </>,
441 + );
442 + await act(() => mocks.voice.start());
443 + await screen.rerender(
444 + <>
445 + <AppRouteModal label="Settings">
446 + <button type="button">Settings control</button>
447 + </AppRouteModal>
448 + <GlobalVoiceInputIndicator hidden />
449 + </>,
450 + );
451 + await expect.element(page.getByRole("dialog", { name: "Settings" })).toHaveFocus();
452 + await userEvent.keyboard("{Shift>}{Tab}{/Shift}");
453 + await expect.element(page.getByRole("button", { name: "Finish voice input" })).toHaveFocus();
454 + await userEvent.keyboard("{Enter}");
455 + await vi.waitFor(() => expect(mocks.storedDraft).toBe("Earlier text dictated"));
456 +});
457 +
458 it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {
459 setup({ destination: "env-a" });
460 mocks.modelState = "missing-model";
apps/web/src/speech/useEnvironmentSpeechInput.ts
+122 -198
@@ -3,12 +3,8 @@ import {
3 downloadEnvironmentSpeechModel,
4 getEnvironmentSpeechModels,
5 getEnvironmentSpeechStatus,
6 - postProcessEnvironmentTranscript,
6 selectEnvironmentSpeechModel,
8 - VoiceInputController,
7 voiceInputBlocksSubmission,
10 - type VoiceDraftSnapshot,
11 - type VoiceInputControllerDependencies,
8 type VoiceInputState,
9 } from "@t3tools/client-runtime/voice-input";
10 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
@@ -18,15 +14,12 @@ import type {
14 EnvironmentSpeechModel,
15 EnvironmentSpeechStatus,
16 ProjectId,
21 - SpeechPostProcessingOptions,
22 - SpeechStreamText,
23 - SpeechTranscriptionOptions,
17 } from "@t3tools/contracts";
18 import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
26 -import { chooseSetupSpeechModel, resolveSpeechVocabulary } from "@t3tools/shared/speech";
19 +import { chooseSetupSpeechModel } from "@t3tools/shared/speech";
20 import { useNavigate } from "@tanstack/react-router";
21 import * as Option from "effect/Option";
29 -import { useEffect, useRef, useState } from "react";
22 +import { useEffect, useLayoutEffect, useRef, useState } from "react";
23
24 import { useStableCallback } from "~/hooks/useStableCallback";
25 import { toastManager } from "../components/ui/toast";
@@ -37,13 +30,18 @@ import {
30 } from "../hooks/useSettings";
31 import { useEnvironment } from "../state/environments";
32 import { useEnvironmentScope, usePreparedConnection } from "../state/session";
40 -import { chooseRecordingMicrophone } from "./browserMicrophones";
41 -import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
33 import { preferredSpeechLanguages } from "./preferredSpeechLanguages";
34 import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
44 -import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent";
35 +import { useAudioChoice } from "./voiceConsent";
36 +import {
37 + IDLE_VOICE_STATE,
38 + presentVoiceOwner,
39 + useRetainedVoiceInputSession,
40 + useVoiceInputSessionActivity,
41 + useVoiceInputSessionSnapshot,
42 + type VoiceInputTarget,
43 +} from "./voiceInputSession";
44
46 -const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
45 const WAITING_STATE: VoiceInputState = { phase: "preparing", error: null, errorAction: null };
46 const INITIAL_SETUP = {
47 open: false,
@@ -73,18 +71,6 @@ type DraftInput = {
71 readonly selection: { readonly start: number; readonly end: number };
72 };
73
76 -function cleanupContext({ text, selection }: DraftInput): DraftInput {
77 - const limit = 1_024;
78 - const start = Math.max(0, Math.min(selection.start, text.length));
79 - const end = Math.max(start, Math.min(selection.end, text.length));
80 - const before = text.slice(Math.max(0, start - limit), start);
81 - const selected = text.slice(start, Math.min(end, start + limit));
82 - return {
83 - text: before + selected + text.slice(end, end + limit),
84 - selection: { start: before.length, end: before.length + selected.length },
85 - };
86 -}
87 -
74 type HookInput = {
75 /** The draft's own environment. Optional transcript cleanup runs there, never on the audio destination. */
76 readonly environmentId: EnvironmentId;
@@ -105,6 +91,21 @@ type HookInput = {
91 text: string,
92 selection: { readonly start: number; readonly end: number },
93 ) => void;
94 + /** Names the draft in the indicator shown while a recording's editor is off screen. */
95 + readonly label?: string | null | undefined;
96 + /** False while the editor stays mounted out of sight, so the indicator still shows. */
97 + readonly presented?: boolean | undefined;
98 + /**
99 + * A draft that is stored outside its editor keeps recording when the editor shows another
100 + * owner or unmounts; the transcript lands in the stored draft instead.
101 + */
102 + readonly persistentDraft?:
103 + | {
104 + readonly readStoredText: () => string | null;
105 + readonly commitStoredText: (text: string) => void;
106 + readonly open: () => void;
107 + }
108 + | undefined;
109 };
110
111 /**
@@ -167,188 +168,60 @@ export function useEnvironmentSpeechInput(input: HookInput) {
168 readonly prepared: PreparedConnection;
169 readonly value: EnvironmentSpeechStatus | null;
170 } | null>(null);
170 - const [controllerState, setControllerState] = useState({
171 - prepared,
172 - grantId,
173 - value: INITIAL_STATE,
174 - });
171 const [queuedStart, setQueuedStart] = useState<PreparedConnection | null>(null);
176 - const [level, setLevel] = useState(0);
177 - const [preview, setPreview] = useState<SpeechStreamText | null>(null);
172 const [setup, setSetup] = useState(INITIAL_SETUP);
173 + const [previousDestination, setPreviousDestination] = useState({ prepared, grantId });
174 const [previousOwner, setPreviousOwner] = useState({
175 key: input.ownerKey,
176 disabled: input.disabled,
177 });
183 - if (controllerState.prepared !== prepared || controllerState.grantId !== grantId) {
184 - setControllerState({ prepared, grantId, value: INITIAL_STATE });
178 + if (previousDestination.prepared !== prepared || previousDestination.grantId !== grantId) {
179 + setPreviousDestination({ prepared, grantId });
180 setSetup(INITIAL_SETUP);
181 setQueuedStart(null);
187 - setPreview(null);
188 - setLevel(0);
182 }
183 if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) {
184 setPreviousOwner({ key: input.ownerKey, disabled: input.disabled });
185 setQueuedStart(null);
186 setSetup(INITIAL_SETUP);
187 }
188 +
189 + const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel });
190 + const snapshot = useVoiceInputSessionSnapshot(session);
191 + const owns = snapshot.target?.ownerKey === input.ownerKey;
192 + const activity = useVoiceInputSessionActivity(session, input.ownerKey);
193 const state: VoiceInputState =
196 - queuedStart === prepared && prepared
197 - ? WAITING_STATE
198 - : controllerState.prepared === prepared
199 - ? controllerState.value
200 - : INITIAL_STATE;
194 + queuedStart === prepared && prepared ? WAITING_STATE : owns ? snapshot.state : IDLE_VOICE_STATE;
195 + // One microphone: another draft's recording leaves this editor without voice input until it ends.
196 + const busyElsewhere = !owns && voiceInputBlocksSubmission(snapshot.state);
197
202 - const controllerRef = useRef<VoiceInputController | null>(null);
198 const startRequestRef = useRef(0);
199 const pendingSubmissionRef = useRef<{
200 ownerKey: string;
201 submit: (text: string) => void;
202 text?: string;
203 } | null>(null);
209 - const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords });
210 - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props and settings to the asynchronous recording controller
211 - useEffect(() => {
212 - latestRef.current = { input, clientSettings, cleanupPrepared, projectWords };
204 + const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords, session });
205 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props before an async completion can read them through the session
206 + useLayoutEffect(() => {
207 + latestRef.current = { input, clientSettings, cleanupPrepared, projectWords, session };
208 });
214 - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the recording controller bound to the consented destination; it is disposed with the consent
209 + const mountedRef = useRef(false);
210 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons this editor's pending work, and a recording that cannot outlive it, on unmount or a destination change
211 useEffect(() => {
216 - if (!prepared || !grantId || typeof navigator === "undefined") return;
217 - const hasAudioConsent = bindAudioGrant(grantId);
218 - let disposed = false;
219 - let controller: VoiceInputController;
220 - let recordingPreferences:
221 - | {
222 - readonly cleanup: SpeechPostProcessingOptions | null;
223 - readonly cleanupGrant: string | null;
224 - readonly environmentId: EnvironmentId;
225 - }
226 - | undefined;
227 -
228 - const readDraft = (): VoiceDraftSnapshot => {
229 - const current = latestRef.current.input;
230 - const draft = current.readDraft();
231 - return { ownerKey: current.ownerKey, text: draft.text, selection: draft.selection };
232 - };
233 - const platform = createBrowserVoiceInputPlatform({
234 - prepared,
235 - cloudProvider,
236 - cloudModel,
237 - hasAudioConsent,
238 - getStreamingEnabled: () => latestRef.current.clientSettings.speechStreaming,
239 - getTranscriptionOptions: (): SpeechTranscriptionOptions => {
240 - const {
241 - clientSettings: settings,
242 - projectWords: project,
243 - input: current,
244 - } = latestRef.current;
245 - const cleanupGrant = current.cleanup === false ? null : readCleanupGrant();
246 - const cleanupEnabled = cleanupGrant !== null;
247 - const cue =
248 - cleanupEnabled && latestRef.current.cleanupPrepared ? settings.speechCorrectionWord : "";
249 - const speechCustomWords = resolveSpeechVocabulary({
250 - personal: settings.speechCustomWords,
251 - project,
252 - hints: [cue, current.projectName ?? ""],
253 - });
254 - recordingPreferences = {
255 - cleanupGrant,
256 - environmentId: current.environmentId,
257 - cleanup: cleanupEnabled
258 - ? {
259 - speechCustomWords,
260 - speechPostProcessingPrompt: settings.speechPostProcessingPrompt,
261 - ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
262 - }
263 - : null,
264 - };
265 - return {
266 - speechLanguage: settings.speechLanguage,
267 - speechTranslateToEnglish: settings.speechTranslateToEnglish,
268 - speechCustomWords,
269 - speechRemoveFillerWords: settings.speechRemoveFillerWords,
270 - ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
271 - speechSilenceFiltering: settings.speechSilenceFiltering,
272 - };
273 - },
274 - getMicrophoneId: chooseRecordingMicrophone,
275 - onLevel: setLevel,
276 - onDurationLimit: () => void controller.stop(),
277 - onText: (text) => {
278 - if (!disposed) setPreview(text);
279 - },
280 - onError: (message) => {
281 - if (!disposed) void controller.interruptRecording(message);
282 - },
283 - });
284 - const postProcess: NonNullable<VoiceInputControllerDependencies["postProcess"]> = (
285 - transcript,
286 - options,
287 - ) => {
288 - const preferences = recordingPreferences;
289 - const { input: current, cleanupPrepared: target } = latestRef.current;
290 - if (
291 - !preferences?.cleanup ||
292 - !hasAudioConsent() ||
293 - preferences.cleanupGrant !== readCleanupGrant() ||
294 - current.cleanup === false ||
295 - current.environmentId !== preferences.environmentId ||
296 - current.ownerKey !== options.draft.ownerKey
297 - )
298 - return null;
299 - if (!target)
300 - return Promise.reject(
301 - new Error("The draft's environment is unavailable for transcript cleanup."),
302 - );
303 - return runEnvironmentSpeech(
304 - postProcessEnvironmentTranscript(
305 - target,
306 - transcript,
307 - cleanupContext(options.draft),
308 - preferences.cleanup,
309 - () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
310 - ),
311 - options,
312 - ).then((result) => result.text);
313 - };
314 - controller = new VoiceInputController({
315 - retryFailedRecording: true,
316 - recorder: platform.recorder,
317 - getTranscriber: () => platform.transcriber,
318 - requestPermission: async () => ({ granted: true, canAskAgain: true }),
319 - configureRecording: async () => undefined,
320 - releaseRecording: async () => platform.cancelRecording(),
321 - deleteRecording: platform.deleteRecording,
322 - postProcess,
323 - onPostProcessingError: () =>
324 - toastManager.add({
325 - type: "warning",
326 - title: "Transcript cleanup failed",
327 - description: "The original transcript was added.",
328 - }),
329 - readDraft,
330 - commitDraft: (text, selection) => {
331 - if (latestRef.current.input.disabled) return;
332 - latestRef.current.input.commitDraft(text, selection);
333 - if (pendingSubmissionRef.current) pendingSubmissionRef.current.text = text;
334 - },
335 - onStateChange: (value) => {
336 - if (disposed) return;
337 - if (value.phase === "error") pendingSubmissionRef.current = null;
338 - setControllerState({ prepared, grantId, value });
339 - if (value.phase !== "recording" && value.phase !== "transcribing") setPreview(null);
340 - },
341 - });
342 - controllerRef.current = controller;
212 + mountedRef.current = true;
213 return () => {
344 - disposed = true;
214 + mountedRef.current = false;
215 pendingSubmissionRef.current = null;
216 startRequestRef.current += 1;
347 - setQueuedStart(null);
348 - controller.dispose();
349 - if (controllerRef.current === controller) controllerRef.current = null;
217 + session?.ownerChanged(latestRef.current.input.ownerKey);
218 };
351 - }, [prepared, grantId, cloudProvider, cloudModel]);
219 + }, [session]);
220 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the global indicator this draft is on screen
221 + useEffect(
222 + () => (input.presented === false ? undefined : presentVoiceOwner(input.ownerKey)),
223 + [input.ownerKey, input.presented],
224 + );
225
226 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available
227 useEffect(() => {
@@ -394,18 +267,21 @@ export function useEnvironmentSpeechInput(input: HookInput) {
267 }, [prepared, setup.downloading, setup.open, setupModelId]);
268
269 const previousOwnerRef = useRef(input.ownerKey);
397 - // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes
270 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons any queued submission, and a recording that cannot outlive its editor, when the draft's owner changes
271 useEffect(() => {
399 - if (previousOwnerRef.current === input.ownerKey && !input.disabled) return;
272 + const previous = previousOwnerRef.current;
273 + if (previous === input.ownerKey && !input.disabled) return;
274 previousOwnerRef.current = input.ownerKey;
275 pendingSubmissionRef.current = null;
276 startRequestRef.current += 1;
403 - controllerRef.current?.ownerChanged();
404 - }, [input.ownerKey, input.disabled]);
277 + if (previous !== input.ownerKey) session?.ownerChanged(previous);
278 + else if (session?.ownerKey === input.ownerKey) session.cancel();
279 + }, [input.ownerKey, input.disabled, session]);
280
281 // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft
282 useEffect(() => {
283 const pending = pendingSubmissionRef.current;
284 + if (state.phase === "error") pendingSubmissionRef.current = null;
285 if (state.phase !== "idle" || !pending) return;
286 pendingSubmissionRef.current = null;
287 if (!input.disabled && pending.ownerKey === input.ownerKey && pending.text !== undefined)
@@ -414,16 +290,62 @@ export function useEnvironmentSpeechInput(input: HookInput) {
290
291 const submitAfterDictation = useStableCallback((submit: (text: string) => void) => {
292 if (latestRef.current.input.disabled || state.phase === "preparing") return;
417 - const controller = controllerRef.current;
418 - if (!controller || !voiceInputBlocksSubmission(state)) {
293 + if (!session || !voiceInputBlocksSubmission(state)) {
294 submit(latestRef.current.input.readDraft().text);
295 return;
296 }
297 if (pendingSubmissionRef.current) return;
298 pendingSubmissionRef.current = { ownerKey: latestRef.current.input.ownerKey, submit };
424 - if (controller.currentState.phase === "recording") void controller.stop();
299 + if (session.state.phase === "recording") void session.stop();
300 });
301
302 + const createTarget = useStableCallback((): VoiceInputTarget => {
303 + const owner = latestRef.current.input.ownerKey;
304 + const readContext = () => {
305 + const current = latestRef.current;
306 + return {
307 + environmentId: current.input.environmentId,
308 + projectName: current.input.projectName ?? null,
309 + projectWords: current.projectWords,
310 + clientSettings: current.clientSettings,
311 + cleanup: current.input.cleanup !== false,
312 + cleanupPrepared: current.cleanupPrepared,
313 + };
314 + };
315 + // Once this editor shows another draft, its environment and cleanup must not see this one.
316 + const startContext = readContext();
317 + const persistent = latestRef.current.input.persistentDraft ?? null;
318 + // While this editor shows the owner, the live editor is the draft; otherwise the stored one.
319 + const live = () => mountedRef.current && latestRef.current.input.ownerKey === owner;
320 + return {
321 + ownerKey: owner,
322 + label: latestRef.current.input.label ?? null,
323 + persistent: persistent !== null,
324 + open: persistent?.open ?? null,
325 + readDraft: () => {
326 + if (live()) return latestRef.current.input.readDraft();
327 + const text = persistent?.readStoredText();
328 + return text == null ? null : { text, selection: { start: text.length, end: text.length } };
329 + },
330 + commitDraft: (text, selection) => {
331 + if (!live()) {
332 + persistent?.commitStoredText(text);
333 + return;
334 + }
335 + if (latestRef.current.input.disabled) return;
336 + latestRef.current.input.commitDraft(text, selection);
337 + if (pendingSubmissionRef.current?.ownerKey === owner)
338 + pendingSubmissionRef.current.text = text;
339 + },
340 + readContext: () => (live() ? readContext() : startContext),
341 + };
342 + });
343 +
344 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- a remounted editor takes over its draft's running recording so a queued Send still sees the transcript
345 + useEffect(() => {
346 + if (session?.ownerKey === input.ownerKey && voiceInputBlocksSubmission(session.state))
347 + session.adopt(createTarget());
348 + }, [session, input.ownerKey, createTarget]);
349 const loadSetupCatalog = useStableCallback(
350 async (connection: PreparedConnection, selectedModelId: string, request: number) => {
351 setSetup({ ...INITIAL_SETUP, open: true });
@@ -458,17 +380,15 @@ export function useEnvironmentSpeechInput(input: HookInput) {
380 return;
381 }
382 const request = ++startRequestRef.current;
461 - const controller = controllerRef.current;
383 const owner = latestRef.current.input.ownerKey;
463 - if (!controller || !prepared) return;
384 + if (!session || !prepared || busyElsewhere) return;
385 if (cloudProvider) {
465 - setLevel(0);
466 - await controller.start();
386 + await session.start(createTarget());
387 return;
388 }
389 const stillCurrent = () =>
390 request === startRequestRef.current &&
471 - controllerRef.current === controller &&
391 + latestRef.current.session === session &&
392 latestRef.current.input.ownerKey === owner &&
393 !latestRef.current.input.disabled;
394 setSetup(INITIAL_SETUP);
@@ -500,8 +420,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
420 });
421 return;
422 }
503 - setLevel(0);
504 - await controller.start();
423 + await session.start(createTarget());
424 });
425
426 const downloadSetupModel = useStableCallback(async () => {
@@ -574,6 +493,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
493 return {
494 available:
495 destinationReady &&
496 + !busyElsewhere &&
497 typeof navigator !== "undefined" &&
498 Boolean(navigator.mediaDevices?.getUserMedia) &&
499 typeof AudioWorkletNode !== "undefined",
@@ -595,21 +515,25 @@ export function useEnvironmentSpeechInput(input: HookInput) {
515 },
516 },
517 state,
598 - preview: state.phase === "recording" || state.phase === "transcribing" ? preview : null,
599 - level,
518 + preview:
519 + state.phase === "recording" || state.phase === "transcribing" ? activity.preview : null,
520 + level: activity.level,
521 + recordingStartedAt: owns ? snapshot.recordingStartedAt : null,
522 preparing: state.phase === "preparing",
523 blocksSubmission: voiceInputBlocksSubmission(state),
524 freezesEditor: voiceInputBlocksSubmission(state),
525 start,
526 submitAfterDictation,
605 - stop: useStableCallback(() => controllerRef.current?.stop() ?? Promise.resolve()),
527 + stop: useStableCallback(() => (owns && session ? session.stop() : Promise.resolve())),
528 cancel: useStableCallback(() => {
529 pendingSubmissionRef.current = null;
530 startRequestRef.current += 1;
531 setQueuedStart(null);
532 setSetup(INITIAL_SETUP);
611 - controllerRef.current?.cancel();
533 + if (owns) session?.cancel();
534 + }),
535 + skipPostProcessing: useStableCallback(() => {
536 + if (owns) session?.skipPostProcessing();
537 }),
613 - skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()),
538 };
539 }
apps/web/src/speech/voiceInputSession.ts new
+424
@@ -0,0 +1,424 @@
1 +import {
2 + postProcessEnvironmentTranscript,
3 + VoiceInputController,
4 + voiceInputBlocksSubmission,
5 + type VoiceDraftSnapshot,
6 + type VoiceInputControllerDependencies,
7 + type VoiceInputState,
8 +} from "@t3tools/client-runtime/voice-input";
9 +import type { PreparedConnection } from "@t3tools/client-runtime/connection";
10 +import type {
11 + CloudSpeechModelId,
12 + CloudSpeechProvider,
13 + ClientSettings,
14 + EnvironmentId,
15 + SpeechCustomWords,
16 + SpeechPostProcessingOptions,
17 + SpeechStreamText,
18 + SpeechTranscriptionOptions,
19 +} from "@t3tools/contracts";
20 +import { DEFAULT_CLIENT_SETTINGS } from "@t3tools/contracts";
21 +import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
22 +import { useEffect, useSyncExternalStore } from "react";
23 +
24 +import { toastManager } from "../components/ui/toast";
25 +import { chooseRecordingMicrophone } from "./browserMicrophones";
26 +import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
27 +import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
28 +import { bindAudioGrant, readCleanupGrant } from "./voiceConsent";
29 +
30 +export const IDLE_VOICE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
31 +
32 +type DraftInput = {
33 + readonly text: string;
34 + readonly selection: { readonly start: number; readonly end: number };
35 +};
36 +
37 +/** What a recording needs from its editor, read once when it starts. */
38 +export type VoiceRecordingContext = {
39 + readonly environmentId: EnvironmentId;
40 + readonly projectName: string | null;
41 + readonly projectWords: SpeechCustomWords;
42 + readonly clientSettings: ClientSettings;
43 + readonly cleanup: boolean;
44 + readonly cleanupPrepared: PreparedConnection | null;
45 +};
46 +
47 +export type VoiceInputTarget = {
48 + readonly ownerKey: string;
49 + /** Names the draft in the indicator shown while its editor is off screen. */
50 + readonly label: string | null;
51 + /** A target that can outlive its editor (a thread draft) keeps recording when the owner changes. */
52 + readonly persistent: boolean;
53 + readonly open: (() => void) | null;
54 + readonly readDraft: () => DraftInput | null;
55 + readonly commitDraft: (text: string, selection: DraftInput["selection"]) => void;
56 + readonly readContext: () => VoiceRecordingContext;
57 +};
58 +
59 +export type VoiceInputSessionSnapshot = {
60 + readonly state: VoiceInputState;
61 + readonly target: VoiceInputTarget | null;
62 + /** Wall-clock start of the current recording, so every view shows the same elapsed time. */
63 + readonly recordingStartedAt: number | null;
64 +};
65 +
66 +export type VoiceInputSessionActivity = {
67 + readonly level: number;
68 + readonly preview: SpeechStreamText | null;
69 +};
70 +
71 +type SessionKey = {
72 + readonly prepared: PreparedConnection;
73 + readonly grantId: string;
74 + readonly cloudProvider: CloudSpeechProvider | undefined;
75 + readonly cloudModel: CloudSpeechModelId | undefined;
76 +};
77 +
78 +const SILENT_ACTIVITY: VoiceInputSessionActivity = { level: 0, preview: null };
79 +
80 +function cleanupContext({ text, selection }: DraftInput): DraftInput {
81 + const limit = 1_024;
82 + const start = Math.max(0, Math.min(selection.start, text.length));
83 + const end = Math.max(start, Math.min(selection.end, text.length));
84 + const before = text.slice(Math.max(0, start - limit), start);
85 + const selected = text.slice(start, Math.min(end, start + limit));
86 + return {
87 + text: before + selected + text.slice(end, end + limit),
88 + selection: { start: before.length, end: before.length + selected.length },
89 + };
90 +}
91 +
92 +/**
93 + * The browser has one microphone, so every editor shares one dictation per destination. The
94 + * recording belongs to the target that started it, not to whichever editor is mounted now.
95 + */
96 +export class VoiceInputSession {
97 + readonly key: SessionKey;
98 + private readonly controller: VoiceInputController;
99 + private target: VoiceInputTarget | null = null;
100 + private snapshot: VoiceInputSessionSnapshot = {
101 + state: IDLE_VOICE_STATE,
102 + target: null,
103 + recordingStartedAt: null,
104 + };
105 + private activity = SILENT_ACTIVITY;
106 + private readonly listeners = new Set<() => void>();
107 + private readonly activityListeners = new Set<() => void>();
108 + private disposed = false;
109 +
110 + constructor(key: SessionKey) {
111 + this.key = key;
112 + const hasAudioConsent = bindAudioGrant(key.grantId);
113 + let recordingPreferences:
114 + | {
115 + readonly cleanup: SpeechPostProcessingOptions | null;
116 + readonly cleanupGrant: string | null;
117 + readonly cleanupPrepared: PreparedConnection | null;
118 + }
119 + | undefined;
120 + const platform = createBrowserVoiceInputPlatform({
121 + prepared: key.prepared,
122 + cloudProvider: key.cloudProvider,
123 + cloudModel: key.cloudModel,
124 + hasAudioConsent,
125 + getStreamingEnabled: () =>
126 + this.target?.readContext().clientSettings.speechStreaming ??
127 + DEFAULT_CLIENT_SETTINGS.speechStreaming,
128 + getTranscriptionOptions: (): SpeechTranscriptionOptions => {
129 + const context = this.target?.readContext();
130 + if (!context) throw new Error("Voice input has no draft to record into.");
131 + const settings = context.clientSettings;
132 + const cleanupGrant = context.cleanup ? readCleanupGrant() : null;
133 + const cleanupEnabled = cleanupGrant !== null;
134 + const cue = cleanupEnabled && context.cleanupPrepared ? settings.speechCorrectionWord : "";
135 + const speechCustomWords = resolveSpeechVocabulary({
136 + personal: settings.speechCustomWords,
137 + project: context.projectWords,
138 + hints: [cue, context.projectName ?? ""],
139 + });
140 + recordingPreferences = {
141 + cleanupGrant,
142 + cleanupPrepared: context.cleanupPrepared,
143 + cleanup: cleanupEnabled
144 + ? {
145 + speechCustomWords,
146 + speechPostProcessingPrompt: settings.speechPostProcessingPrompt,
147 + ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
148 + }
149 + : null,
150 + };
151 + return {
152 + speechLanguage: settings.speechLanguage,
153 + speechTranslateToEnglish: settings.speechTranslateToEnglish,
154 + speechCustomWords,
155 + speechRemoveFillerWords: settings.speechRemoveFillerWords,
156 + ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
157 + speechSilenceFiltering: settings.speechSilenceFiltering,
158 + };
159 + },
160 + getMicrophoneId: chooseRecordingMicrophone,
161 + onLevel: (level) => this.publishActivity({ ...this.activity, level }),
162 + onDurationLimit: () => void this.controller.stop(),
163 + onText: (preview) => {
164 + if (!this.disposed) this.publishActivity({ ...this.activity, preview });
165 + },
166 + onError: (message) => {
167 + if (!this.disposed) void this.controller.interruptRecording(message);
168 + },
169 + });
170 + const postProcess: NonNullable<VoiceInputControllerDependencies["postProcess"]> = (
171 + transcript,
172 + options,
173 + ) => {
174 + const preferences = recordingPreferences;
175 + if (
176 + !preferences?.cleanup ||
177 + !hasAudioConsent() ||
178 + preferences.cleanupGrant !== readCleanupGrant() ||
179 + this.target?.ownerKey !== options.draft.ownerKey
180 + )
181 + return null;
182 + if (!preferences.cleanupPrepared)
183 + return Promise.reject(
184 + new Error("The draft's environment is unavailable for transcript cleanup."),
185 + );
186 + return runEnvironmentSpeech(
187 + postProcessEnvironmentTranscript(
188 + preferences.cleanupPrepared,
189 + transcript,
190 + cleanupContext(options.draft),
191 + preferences.cleanup,
192 + () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
193 + ),
194 + options,
195 + ).then((result) => result.text);
196 + };
197 + this.controller = new VoiceInputController({
198 + retryFailedRecording: true,
199 + recorder: platform.recorder,
200 + getTranscriber: () => platform.transcriber,
201 + requestPermission: async () => ({ granted: true, canAskAgain: true }),
202 + configureRecording: async () => undefined,
203 + releaseRecording: async () => platform.cancelRecording(),
204 + deleteRecording: platform.deleteRecording,
205 + postProcess,
206 + onPostProcessingError: () =>
207 + toastManager.add({
208 + type: "warning",
209 + title: "Transcript cleanup failed",
210 + description: "The original transcript was added.",
211 + }),
212 + readDraft: (): VoiceDraftSnapshot | null => {
213 + const target = this.target;
214 + const draft = target?.readDraft();
215 + return target && draft ? { ownerKey: target.ownerKey, ...draft } : null;
216 + },
217 + commitDraft: (text, selection) => this.target?.commitDraft(text, selection),
218 + onStateChange: (state) => {
219 + if (this.disposed) return;
220 + const recordingStartedAt =
221 + state.phase !== "recording" ? null : (this.snapshot.recordingStartedAt ?? Date.now());
222 + this.publish({ ...this.snapshot, state, recordingStartedAt });
223 + if (state.phase !== "recording" && state.phase !== "transcribing")
224 + this.publishActivity(SILENT_ACTIVITY);
225 + },
226 + });
227 + }
228 +
229 + get state(): VoiceInputState {
230 + return this.controller.currentState;
231 + }
232 +
233 + get ownerKey(): string | null {
234 + return this.target?.ownerKey ?? null;
235 + }
236 +
237 + readonly subscribe = (listener: () => void): (() => void) => {
238 + this.listeners.add(listener);
239 + return () => this.listeners.delete(listener);
240 + };
241 +
242 + readonly getSnapshot = (): VoiceInputSessionSnapshot => this.snapshot;
243 +
244 + readonly subscribeActivity = (listener: () => void): (() => void) => {
245 + this.activityListeners.add(listener);
246 + return () => this.activityListeners.delete(listener);
247 + };
248 +
249 + readonly getActivity = (): VoiceInputSessionActivity => this.activity;
250 +
251 + /** Starts recording into `target`, or does nothing while another draft holds the microphone. */
252 + start(target: VoiceInputTarget): Promise<void> {
253 + if (this.disposed) return Promise.resolve();
254 + const busyElsewhere =
255 + voiceInputBlocksSubmission(this.controller.currentState) &&
256 + this.target?.ownerKey !== target.ownerKey;
257 + if (busyElsewhere) return Promise.resolve();
258 + this.target = target;
259 + this.publishActivity(SILENT_ACTIVITY);
260 + this.publish({ ...this.snapshot, target });
261 + return this.controller.start();
262 + }
263 +
264 + /** A newly mounted editor for the recording's draft replaces the target that started it. */
265 + adopt(target: VoiceInputTarget): void {
266 + if (this.disposed || this.target?.ownerKey !== target.ownerKey) return;
267 + this.target = target;
268 + this.publish({ ...this.snapshot, target });
269 + }
270 +
271 + stop(): Promise<void> {
272 + return this.controller.stop();
273 + }
274 +
275 + cancel(): void {
276 + this.controller.cancel();
277 + }
278 +
279 + skipPostProcessing(): void {
280 + this.controller.skipPostProcessing();
281 + }
282 +
283 + /** The editor for `ownerKey` now shows another draft; only a persistent target keeps going. */
284 + ownerChanged(ownerKey: string): void {
285 + if (this.target?.ownerKey !== ownerKey || this.target.persistent) return;
286 + this.controller.ownerChanged();
287 + }
288 +
289 + dispose(): void {
290 + this.disposed = true;
291 + this.controller.dispose();
292 + this.listeners.clear();
293 + this.activityListeners.clear();
294 + }
295 +
296 + private publish(snapshot: VoiceInputSessionSnapshot): void {
297 + this.snapshot = snapshot;
298 + for (const listener of this.listeners) listener();
299 + }
300 +
301 + private publishActivity(activity: VoiceInputSessionActivity): void {
302 + if (activity.level === this.activity.level && activity.preview === this.activity.preview)
303 + return;
304 + this.activity = activity;
305 + for (const listener of this.activityListeners) listener();
306 + }
307 +}
308 +
309 +const sessions = new Map<VoiceInputSession, number>();
310 +const registryListeners = new Set<() => void>();
311 +
312 +function sameKey(session: VoiceInputSession, key: SessionKey): boolean {
313 + return (
314 + session.key.prepared === key.prepared &&
315 + session.key.grantId === key.grantId &&
316 + session.key.cloudProvider === key.cloudProvider &&
317 + session.key.cloudModel === key.cloudModel
318 + );
319 +}
320 +
321 +function findSession(key: SessionKey): VoiceInputSession | null {
322 + for (const session of sessions.keys()) if (sameKey(session, key)) return session;
323 + return null;
324 +}
325 +
326 +/** Every holder of a destination shares one session; the last release disposes it. */
327 +function retainVoiceInputSession(key: SessionKey): () => void {
328 + const session = findSession(key) ?? new VoiceInputSession(key);
329 + sessions.set(session, (sessions.get(session) ?? 0) + 1);
330 + for (const listener of registryListeners) listener();
331 + let released = false;
332 + return () => {
333 + if (released) return;
334 + released = true;
335 + const count = (sessions.get(session) ?? 1) - 1;
336 + if (count > 0) {
337 + sessions.set(session, count);
338 + return;
339 + }
340 + sessions.delete(session);
341 + session.dispose();
342 + for (const listener of registryListeners) listener();
343 + };
344 +}
345 +
346 +function subscribeRegistry(listener: () => void): () => void {
347 + registryListeners.add(listener);
348 + return () => registryListeners.delete(listener);
349 +}
350 +
351 +/** Holds the destination's shared session while mounted; null until it is retained. */
352 +export function useRetainedVoiceInputSession(input: {
353 + readonly prepared: PreparedConnection | null;
354 + readonly grantId: string | null;
355 + readonly cloudProvider?: CloudSpeechProvider | undefined;
356 + readonly cloudModel?: CloudSpeechModelId | undefined;
357 +}): VoiceInputSession | null {
358 + const { prepared, grantId, cloudProvider, cloudModel } = input;
359 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- retains the shared recording session for this destination while mounted
360 + useEffect(() => {
361 + if (!prepared || !grantId || typeof navigator === "undefined") return;
362 + return retainVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel });
363 + }, [prepared, grantId, cloudProvider, cloudModel]);
364 + return useSyncExternalStore(subscribeRegistry, () =>
365 + prepared && grantId ? findSession({ prepared, grantId, cloudProvider, cloudModel }) : null,
366 + );
367 +}
368 +
369 +const IDLE_SNAPSHOT: VoiceInputSessionSnapshot = {
370 + state: IDLE_VOICE_STATE,
371 + target: null,
372 + recordingStartedAt: null,
373 +};
374 +const noSubscription = () => () => {};
375 +
376 +export function useVoiceInputSessionSnapshot(
377 + session: VoiceInputSession | null,
378 +): VoiceInputSessionSnapshot {
379 + return useSyncExternalStore(
380 + session?.subscribe ?? noSubscription,
381 + session ? session.getSnapshot : () => IDLE_SNAPSHOT,
382 + );
383 +}
384 +
385 +export function useVoiceInputSessionActivity(
386 + session: VoiceInputSession | null,
387 + ownerKey: string | null,
388 +): VoiceInputSessionActivity {
389 + // Activity changes many times a second, so only the owning editor reads it.
390 + const owns = useSyncExternalStore(
391 + session?.subscribe ?? noSubscription,
392 + () => session !== null && ownerKey !== null && session.ownerKey === ownerKey,
393 + );
394 + return useSyncExternalStore(owns && session ? session.subscribeActivity : noSubscription, () =>
395 + owns && session ? session.getActivity() : SILENT_ACTIVITY,
396 + );
397 +}
398 +
399 +// Editors register the owner they show, so the indicator appears only while its draft is off screen.
400 +const presentedOwners = new Map<string, number>();
401 +const presentationListeners = new Set<() => void>();
402 +
403 +export function presentVoiceOwner(ownerKey: string): () => void {
404 + presentedOwners.set(ownerKey, (presentedOwners.get(ownerKey) ?? 0) + 1);
405 + for (const listener of presentationListeners) listener();
406 + return () => {
407 + const count = (presentedOwners.get(ownerKey) ?? 1) - 1;
408 + if (count > 0) presentedOwners.set(ownerKey, count);
409 + else presentedOwners.delete(ownerKey);
410 + for (const listener of presentationListeners) listener();
411 + };
412 +}
413 +
414 +function subscribePresentation(listener: () => void): () => void {
415 + presentationListeners.add(listener);
416 + return () => presentationListeners.delete(listener);
417 +}
418 +
419 +export function useVoiceOwnerPresented(ownerKey: string | null): boolean {
420 + return useSyncExternalStore(
421 + subscribePresentation,
422 + () => ownerKey !== null && presentedOwners.has(ownerKey),
423 + );
424 +}