The best way to control Claude Code, Codex and more from any computer or phone
feat(web): dictation keeps recording into its thread when you switch away (#35761)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Julius Marminge committed
Nov 16, 2026 at 13:45 UTC
c24255826be5aabb0f32ee18f6ac7dec0b3f1347
13 files changed
+755
-206
apps/web/src/components/AppRouteModal.browser.test.tsx
+2
@@ -5,6 +5,8 @@ import { render } from "../../test/render";
5
import { useCommitOnBlur } from "../hooks/useCommitOnBlur";
6
import { AppRouteModal } from "./AppRouteModal";
7
8
+vi.mock("./chat/GlobalVoiceInputIndicator", () => ({ GlobalVoiceInputIndicator: () => null }));
9
+
10
function CommitOnBlurField({ onCommit }: { onCommit: (next: string) => void }) {
11
return <input aria-label="Instructions" {...useCommitOnBlur("", onCommit)} />;
12
}
apps/web/src/components/AppRouteModal.tsx
+2
@@ -4,6 +4,7 @@ import { useRef, useState, type CSSProperties, type ReactNode } from "react";
4
5
import { useNativeTitlebarHeight } from "../lib/windowControlsOverlay";
6
import { useCloseAppModal } from "./sidebar/mainAppLocation";
7
+import { GlobalVoiceInputIndicator } from "./chat/GlobalVoiceInputIndicator";
8
import { Button } from "./ui/button";
9
import { DialogBackdrop } from "./ui/dialog";
10
import { Tooltip, TooltipPopup, TooltipTrigger } from "./ui/tooltip";
@@ -89,6 +90,7 @@ export function AppRouteModal({ label, children }: { label: string; children: Re
90
className="absolute inset-x-0 top-0 hidden h-(--app-modal-titlebar-height) [-webkit-app-region:drag] max-sm:block"
91
/>
92
{children}
93
+ <GlobalVoiceInputIndicator />
94
</DialogPrimitive.Popup>
95
</DialogPrimitive.Viewport>
96
</DialogPrimitive.Portal>
apps/web/src/components/ThreadRouteView.tsx
+4
-2
@@ -8,7 +8,7 @@ import { Navigate, useNavigate } from "@tanstack/react-router";
8
import { useEffect, useState } from "react";
9
10
import ChatView from "./ChatView";
11
-import { BackgroundThreadContext } from "./chat/backgroundThread";
11
+import { BackgroundThreadContext, HiddenThreadContext } from "./chat/backgroundThread";
12
import { SidebarInset } from "./ui/sidebar";
13
import { useComposerDraftStore } from "../composerDraftStore";
14
import { useSidebarPendingFileDropStore } from "../sidebarPendingFileDropStore";
@@ -229,7 +229,9 @@ export function ThreadRouteView({
229
className="h-svh min-h-0 overflow-hidden overscroll-y-none md:h-dvh"
230
inert={background}
231
>
232
- <BackgroundThreadContext value={background}>{view}</BackgroundThreadContext>
232
+ <BackgroundThreadContext value={background}>
233
+ <HiddenThreadContext value={background}>{view}</HiddenThreadContext>
234
+ </BackgroundThreadContext>
235
{background ? null : redirect}
236
</SidebarInset>
237
);
apps/web/src/components/chat/ChatComposer.tsx
+29
-1
@@ -4,7 +4,7 @@ import { useComposerTypingGuard } from "./useComposerTypingGuard";
4
import { insertAttachmentReferencesForDraft } from "./composerAttachmentReferences";
5
import type { ComposerAttachmentInsertion } from "../composerAttachmentInsertion";
6
import { DESKTOP_PASTE_AS_TEXT_EVENT } from "../../lib/desktopPasteAsText";
7
-import { BackgroundThreadContext } from "./backgroundThread";
7
+import { BackgroundThreadContext, HiddenThreadContext } from "./backgroundThread";
8
import {
9
resolveCompatibleRuntimeMode,
10
runtimeModeConfig,
@@ -107,9 +107,12 @@ import {
107
composerFileNeedsReattach,
108
composerDraftHasUserContent,
109
composerTargetKey,
110
+ readComposerDraftPrompt,
111
+ resolveLiveComposerDraftTarget,
112
useComposerDraftStore,
113
useComposerThreadDraft,
114
} from "../../composerDraftStore";
115
+import { buildDraftThreadRouteParams, buildThreadRouteParams } from "../../threadRoutes";
116
import { ComposerStashBadge } from "./ComposerStashBadge";
117
import { ComposerStashMenu } from "./ComposerStashMenu";
118
import { useComposerTriggerState } from "./useComposerTriggerState";
@@ -1052,6 +1055,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
1055
});
1056
1057
const backgroundThread = use(BackgroundThreadContext);
1058
+ const hiddenThread = use(HiddenThreadContext);
1059
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to keydown, window blur and the desktop paste-as-text event to arm plain-text paste
1060
useEffect(() => {
1061
// A background composer listens on its own form, so it never hears the main composer's keys.
@@ -1846,6 +1850,29 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
1850
projectName,
1851
ownerKey: questionAttachmentTarget ?? composerDraftTargetKey,
1852
disabled: dictationDisabled,
1853
+ label: activeThread?.title ?? null,
1854
+ presented: !hiddenThread,
1855
+ // A question's answer lives only in this composer, so only the thread draft outlives it.
1856
+ persistentDraft: questionAttachmentTarget
1857
+ ? undefined
1858
+ : {
1859
+ readStoredText: () => readComposerDraftPrompt(composerDraftTarget),
1860
+ commitStoredText: (text) => {
1861
+ const live = resolveLiveComposerDraftTarget(composerDraftTarget);
1862
+ if (live !== null) setComposerDraftPrompt(live, text);
1863
+ },
1864
+ // The route, not the draft target: a queued-message edit has a draft id but no route.
1865
+ open: () =>
1866
+ void (routeKind === "draft" && typeof composerDraftTarget === "string"
1867
+ ? navigate({
1868
+ to: "/draft/$draftId",
1869
+ params: buildDraftThreadRouteParams(composerDraftTarget),
1870
+ })
1871
+ : navigate({
1872
+ to: "/$environmentId/$threadId",
1873
+ params: buildThreadRouteParams(routeThreadRef),
1874
+ })),
1875
+ },
1876
readDraft: () => {
1877
const snapshot = readComposerSnapshot();
1878
return {
@@ -3828,6 +3855,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
3855
finishShortcutLabel={dictationShortcutLabel}
3856
cancelShortcutLabel="Esc"
3857
level={speechInput.level}
3858
+ recordingStartedAt={speechInput.recordingStartedAt}
3859
disabled={dictationDisabled}
3860
onStart={() => void speechInput.start()}
3861
onStop={() => void speechInput.stop()}
apps/web/src/components/chat/ComposerSpeechButton.tsx
+13
-4
@@ -43,18 +43,25 @@ function formatElapsed(seconds: number) {
43
return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
44
}
45
46
-function RecordingStatus(props: { level: number; recordingLimitSeconds?: number | undefined }) {
47
- const [elapsedSeconds, setElapsedSeconds] = useState(0);
46
+function RecordingStatus(props: {
47
+ level: number;
48
+ recordingStartedAt?: number | null | undefined;
49
+ recordingLimitSeconds?: number | undefined;
50
+}) {
51
+ const { recordingStartedAt } = props;
52
+ const [elapsedSeconds, setElapsedSeconds] = useState(() =>
53
+ recordingStartedAt ? Math.max(0, Math.floor((Date.now() - recordingStartedAt) / 1_000)) : 0,
54
+ );
55
56
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the timer displaying the native recording duration
57
useEffect(() => {
51
- const startedAt = Date.now();
58
+ const startedAt = recordingStartedAt ?? Date.now();
59
const timer = window.setInterval(
60
() => setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1_000)),
61
250,
62
);
63
return () => window.clearInterval(timer);
57
- }, []);
64
+ }, [recordingStartedAt]);
65
66
return (
67
<>
@@ -125,6 +132,7 @@ export function ComposerSpeechRecordingPill(props: {
132
finishShortcutLabel?: string | null;
133
cancelShortcutLabel?: string | null;
134
level: number;
135
+ recordingStartedAt?: number | null | undefined;
136
recordingLimitSeconds?: number | undefined;
137
/** Compact matches the `xs` buttons beside text-field editors. */
138
size?: "default" | "compact";
@@ -181,6 +189,7 @@ export function ComposerSpeechRecordingPill(props: {
189
{recording ? (
190
<RecordingStatus
191
level={props.level}
192
+ recordingStartedAt={props.recordingStartedAt}
193
recordingLimitSeconds={props.recordingLimitSeconds}
194
/>
195
) : (
apps/web/src/components/chat/GlobalVoiceInputIndicator.tsx
new
+52
@@ -0,0 +1,52 @@
1
+import { useVoiceTranscriptionDestination } from "../../speech/useEnvironmentSpeechInput";
2
+import {
3
+ useRetainedVoiceInputSession,
4
+ useVoiceInputSessionActivity,
5
+ useVoiceInputSessionSnapshot,
6
+ useVoiceOwnerPresented,
7
+} from "../../speech/voiceInputSession";
8
+import { Button } from "../ui/button";
9
+import { ComposerSpeechRecordingPill } from "./ComposerSpeechButton";
10
+
11
+/**
12
+ * Keeps the chosen destination's dictation alive across navigation and, while the draft it
13
+ * records into is off screen, shows it so it can be finished, cancelled, or returned to.
14
+ */
15
+export function GlobalVoiceInputIndicator({ hidden = false }: { hidden?: boolean }) {
16
+ const { prepared, grantId, cloudProvider, cloudModel } = useVoiceTranscriptionDestination();
17
+ const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel });
18
+ const { state, target, recordingStartedAt } = useVoiceInputSessionSnapshot(session);
19
+ const presented = useVoiceOwnerPresented(target?.ownerKey ?? null);
20
+ const { level } = useVoiceInputSessionActivity(
21
+ hidden || presented ? null : session,
22
+ target?.ownerKey ?? null,
23
+ );
24
+ if (hidden || !session || !target || state.phase === "idle" || presented) return null;
25
+
26
+ return (
27
+ <div className="pointer-events-none fixed inset-x-0 top-3 z-100 flex justify-center px-4">
28
+ <div className="pointer-events-auto flex max-w-full items-center gap-1 rounded-full bg-popover p-0.5 shadow-lg/5">
29
+ <Button
30
+ type="button"
31
+ size="sm-pill"
32
+ variant="ghost"
33
+ disabled={!target.open}
34
+ onClick={() => target.open?.()}
35
+ className="min-w-0 shrink"
36
+ >
37
+ <span className="truncate">{target.label ?? "Untitled thread"}</span>
38
+ </Button>
39
+ <ComposerSpeechRecordingPill
40
+ state={state}
41
+ progress={null}
42
+ level={level}
43
+ recordingStartedAt={recordingStartedAt}
44
+ onStart={() => void session.start(target)}
45
+ onStop={() => void session.stop()}
46
+ onCancel={() => session.cancel()}
47
+ onSkipPostProcessing={() => session.skipPostProcessing()}
48
+ />
49
+ </div>
50
+ </div>
51
+ );
52
+}
apps/web/src/components/chat/backgroundThread.ts
+3
@@ -5,3 +5,6 @@ import { createContext } from "react";
5
* parent's panel; its global shortcuts and listeners stand down.
6
*/
7
export const BackgroundThreadContext = createContext(false);
8
+
9
+/** True only for a thread kept mounted out of sight under Settings or Usage. */
10
+export const HiddenThreadContext = createContext(false);
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx
+1
@@ -328,6 +328,7 @@ it("tests the selected model after download and preserves its transcript across
328
state: { phase: "idle", error: null, errorAction: null },
329
preview: null,
330
level: 0,
331
+ recordingStartedAt: null,
332
preparing: false,
333
blocksSubmission: false,
334
freezesEditor: false,
apps/web/src/composerDraftStore.ts
+21
@@ -1660,6 +1660,27 @@ function getComposerDraftState(
1660
return state.draftsByThreadKey[threadKey] ?? null;
1661
}
1662
1663
+/** Where a draft lives now: its thread once promoted, or null once it was discarded. */
1664
+export function resolveLiveComposerDraftTarget(
1665
+ target: ComposerThreadTarget,
1666
+): ComposerThreadTarget | null {
1667
+ if (typeof target !== "string") return target;
1668
+ const state = composerDraftStore.getState();
1669
+ const promoted = state.promotedDraftRedirects[target];
1670
+ if (promoted) return promoted;
1671
+ return state.draftThreadsByThreadKey[target] || state.draftsByThreadKey[target] ? target : null;
1672
+}
1673
+
1674
+/** The stored prompt for a target, empty when it has no draft, or null when the target is gone. */
1675
+export function readComposerDraftPrompt(target: ComposerThreadTarget): string | null {
1676
+ const live = resolveLiveComposerDraftTarget(target);
1677
+ if (live === null) return null;
1678
+ const state = composerDraftStore.getState();
1679
+ const threadKey = resolveComposerDraftKey(state, live);
1680
+ if (!threadKey) return null;
1681
+ return state.draftsByThreadKey[threadKey]?.prompt ?? "";
1682
+}
1683
+
1684
function draftThreadHasOpenPreview(draftThread: DraftThreadState | undefined): boolean {
1685
if (draftThread === undefined) return false;
1686
const ref = scopeThreadRef(draftThread.environmentId, draftThread.threadId);
apps/web/src/routes/_chat.tsx
+2
@@ -1,6 +1,7 @@
1
import { Outlet, createFileRoute, redirect } from "@tanstack/react-router";
2
3
import { ThreadRouteView } from "../components/ThreadRouteView";
4
+import { GlobalVoiceInputIndicator } from "../components/chat/GlobalVoiceInputIndicator";
5
import { isAppModalPage } from "../components/sidebar/mainAppLocation";
6
import { SidebarInset } from "../components/ui/sidebar";
7
import { resolveThreadRouteTarget } from "../threadRoutes";
@@ -24,6 +25,7 @@ function ChatRouteLayout() {
25
<Outlet />
26
)}
27
{modalOpen ? <Outlet /> : null}
28
+ <GlobalVoiceInputIndicator hidden={modalOpen} />
29
</>
30
);
31
}
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx
+80
-1
@@ -18,6 +18,8 @@ import {
18
} from "@t3tools/contracts";
19
20
import { act, render, stubGlobal } from "../../test/render";
21
+import { AppRouteModal } from "../components/AppRouteModal";
22
+import { GlobalVoiceInputIndicator } from "../components/chat/GlobalVoiceInputIndicator";
23
import { VoiceInputSetup } from "../components/chat/VoiceInputSetup";
24
import type { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
25
import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
@@ -58,8 +60,12 @@ const mocks = vi.hoisted(() => ({
60
transcripts: [] as Array<string | Error>,
61
committed: vi.fn(),
62
draft: "",
63
+ storedDraft: "",
64
}));
65
66
+vi.mock("../components/sidebar/mainAppLocation", () => ({
67
+ useCloseAppModal: () => () => Promise.resolve(),
68
+}));
69
vi.mock("../state/session", () => ({
70
useEnvironmentScope: (id: string) => mocks.canDictate.has(id),
71
usePreparedConnection: (environmentId: string | null) => {
@@ -236,6 +242,7 @@ function setup(options: { readonly destination?: string | null } = {}) {
242
mocks.transcripts = [];
243
mocks.committed.mockClear();
244
mocks.draft = "";
245
+ mocks.storedDraft = "";
246
}
247
248
function setDestination(environmentId: string | null) {
@@ -249,13 +256,24 @@ function Probe(props: {
256
readonly projectName?: string;
257
readonly cleanup?: boolean;
258
readonly showWizard?: boolean;
259
+ readonly ownerKey?: string;
260
+ readonly persistent?: boolean;
261
}) {
262
const value = useEnvironmentSpeechInput({
263
environmentId: THREAD_ENV as EnvironmentId,
264
projectId: props.projectId as ProjectId | undefined,
265
projectName: props.projectName,
266
cleanup: props.cleanup,
258
- ownerKey: "draft",
267
+ ownerKey: props.ownerKey ?? "draft",
268
+ persistentDraft: props.persistent
269
+ ? {
270
+ readStoredText: () => mocks.storedDraft,
271
+ commitStoredText: (text) => {
272
+ mocks.storedDraft = text;
273
+ },
274
+ open: () => {},
275
+ }
276
+ : undefined,
277
readDraft: () => ({
278
text: mocks.draft,
279
selection: { start: mocks.draft.length, end: mocks.draft.length },
@@ -376,6 +394,67 @@ it("retries a failed transcription on the same chosen environment", async () =>
394
expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("second try", expect.anything());
395
});
396
397
+it.each([
398
+ ["a stored thread draft keeps recording into it", true],
399
+ ["an editor-only draft is abandoned", false],
400
+])("when the editor switches owner mid-recording, %s", async (_name, persistent) => {
401
+ setup({ destination: "env-a" });
402
+ mocks.storedDraft = "Earlier text";
403
+ const view = (ownerKey: string) => (
404
+ <>
405
+ <Probe ownerKey={ownerKey} persistent={persistent} />
406
+ <GlobalVoiceInputIndicator />
407
+ </>
408
+ );
409
+ await using screen = await render(view("thread-a"));
410
+ await act(() => mocks.voice.start());
411
+ expect(mocks.voice.state.phase).toBe("recording");
412
+ // The indicator stays away while the recording's own editor is on screen.
413
+ expect(page.getByRole("button", { name: "Finish voice input" }).query()).toBeNull();
414
+
415
+ await screen.rerender(view("thread-b"));
416
+ // One microphone: thread-b cannot start its own recording while thread-a's runs.
417
+ expect(mocks.voice.available).toBe(!persistent);
418
+ if (!persistent) {
419
+ await expect
420
+ .element(page.getByRole("button", { name: "Finish voice input" }))
421
+ .not.toBeInTheDocument();
422
+ expect(mocks.transcribeCalls).toEqual([]);
423
+ expect(mocks.storedDraft).toBe("Earlier text");
424
+ return;
425
+ }
426
+ await page.getByRole("button", { name: "Finish voice input" }).click();
427
+
428
+ await vi.waitFor(() => expect(mocks.storedDraft).toBe("Earlier text dictated"));
429
+ expect(mocks.transcribeCalls).toEqual(["env-a"]);
430
+ expect(mocks.committed).not.toHaveBeenCalled();
431
+});
432
+
433
+it("finishes an off-screen recording with the keyboard while Settings is open", async () => {
434
+ setup({ destination: "env-a" });
435
+ mocks.storedDraft = "Earlier text";
436
+ await using screen = await render(
437
+ <>
438
+ <Probe ownerKey="thread-a" persistent />
439
+ <GlobalVoiceInputIndicator />
440
+ </>,
441
+ );
442
+ await act(() => mocks.voice.start());
443
+ await screen.rerender(
444
+ <>
445
+ <AppRouteModal label="Settings">
446
+ <button type="button">Settings control</button>
447
+ </AppRouteModal>
448
+ <GlobalVoiceInputIndicator hidden />
449
+ </>,
450
+ );
451
+ await expect.element(page.getByRole("dialog", { name: "Settings" })).toHaveFocus();
452
+ await userEvent.keyboard("{Shift>}{Tab}{/Shift}");
453
+ await expect.element(page.getByRole("button", { name: "Finish voice input" })).toHaveFocus();
454
+ await userEvent.keyboard("{Enter}");
455
+ await vi.waitFor(() => expect(mocks.storedDraft).toBe("Earlier text dictated"));
456
+});
457
+
458
it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {
459
setup({ destination: "env-a" });
460
mocks.modelState = "missing-model";
apps/web/src/speech/useEnvironmentSpeechInput.ts
+122
-198
@@ -3,12 +3,8 @@ import {
3
downloadEnvironmentSpeechModel,
4
getEnvironmentSpeechModels,
5
getEnvironmentSpeechStatus,
6
- postProcessEnvironmentTranscript,
6
selectEnvironmentSpeechModel,
8
- VoiceInputController,
7
voiceInputBlocksSubmission,
10
- type VoiceDraftSnapshot,
11
- type VoiceInputControllerDependencies,
8
type VoiceInputState,
9
} from "@t3tools/client-runtime/voice-input";
10
import type { PreparedConnection } from "@t3tools/client-runtime/connection";
@@ -18,15 +14,12 @@ import type {
14
EnvironmentSpeechModel,
15
EnvironmentSpeechStatus,
16
ProjectId,
21
- SpeechPostProcessingOptions,
22
- SpeechStreamText,
23
- SpeechTranscriptionOptions,
17
} from "@t3tools/contracts";
18
import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
26
-import { chooseSetupSpeechModel, resolveSpeechVocabulary } from "@t3tools/shared/speech";
19
+import { chooseSetupSpeechModel } from "@t3tools/shared/speech";
20
import { useNavigate } from "@tanstack/react-router";
21
import * as Option from "effect/Option";
29
-import { useEffect, useRef, useState } from "react";
22
+import { useEffect, useLayoutEffect, useRef, useState } from "react";
23
24
import { useStableCallback } from "~/hooks/useStableCallback";
25
import { toastManager } from "../components/ui/toast";
@@ -37,13 +30,18 @@ import {
30
} from "../hooks/useSettings";
31
import { useEnvironment } from "../state/environments";
32
import { useEnvironmentScope, usePreparedConnection } from "../state/session";
40
-import { chooseRecordingMicrophone } from "./browserMicrophones";
41
-import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
33
import { preferredSpeechLanguages } from "./preferredSpeechLanguages";
34
import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
44
-import { bindAudioGrant, readCleanupGrant, useAudioChoice } from "./voiceConsent";
35
+import { useAudioChoice } from "./voiceConsent";
36
+import {
37
+ IDLE_VOICE_STATE,
38
+ presentVoiceOwner,
39
+ useRetainedVoiceInputSession,
40
+ useVoiceInputSessionActivity,
41
+ useVoiceInputSessionSnapshot,
42
+ type VoiceInputTarget,
43
+} from "./voiceInputSession";
44
46
-const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
45
const WAITING_STATE: VoiceInputState = { phase: "preparing", error: null, errorAction: null };
46
const INITIAL_SETUP = {
47
open: false,
@@ -73,18 +71,6 @@ type DraftInput = {
71
readonly selection: { readonly start: number; readonly end: number };
72
};
73
76
-function cleanupContext({ text, selection }: DraftInput): DraftInput {
77
- const limit = 1_024;
78
- const start = Math.max(0, Math.min(selection.start, text.length));
79
- const end = Math.max(start, Math.min(selection.end, text.length));
80
- const before = text.slice(Math.max(0, start - limit), start);
81
- const selected = text.slice(start, Math.min(end, start + limit));
82
- return {
83
- text: before + selected + text.slice(end, end + limit),
84
- selection: { start: before.length, end: before.length + selected.length },
85
- };
86
-}
87
-
74
type HookInput = {
75
/** The draft's own environment. Optional transcript cleanup runs there, never on the audio destination. */
76
readonly environmentId: EnvironmentId;
@@ -105,6 +91,21 @@ type HookInput = {
91
text: string,
92
selection: { readonly start: number; readonly end: number },
93
) => void;
94
+ /** Names the draft in the indicator shown while a recording's editor is off screen. */
95
+ readonly label?: string | null | undefined;
96
+ /** False while the editor stays mounted out of sight, so the indicator still shows. */
97
+ readonly presented?: boolean | undefined;
98
+ /**
99
+ * A draft that is stored outside its editor keeps recording when the editor shows another
100
+ * owner or unmounts; the transcript lands in the stored draft instead.
101
+ */
102
+ readonly persistentDraft?:
103
+ | {
104
+ readonly readStoredText: () => string | null;
105
+ readonly commitStoredText: (text: string) => void;
106
+ readonly open: () => void;
107
+ }
108
+ | undefined;
109
};
110
111
/**
@@ -167,188 +168,60 @@ export function useEnvironmentSpeechInput(input: HookInput) {
168
readonly prepared: PreparedConnection;
169
readonly value: EnvironmentSpeechStatus | null;
170
} | null>(null);
170
- const [controllerState, setControllerState] = useState({
171
- prepared,
172
- grantId,
173
- value: INITIAL_STATE,
174
- });
171
const [queuedStart, setQueuedStart] = useState<PreparedConnection | null>(null);
176
- const [level, setLevel] = useState(0);
177
- const [preview, setPreview] = useState<SpeechStreamText | null>(null);
172
const [setup, setSetup] = useState(INITIAL_SETUP);
173
+ const [previousDestination, setPreviousDestination] = useState({ prepared, grantId });
174
const [previousOwner, setPreviousOwner] = useState({
175
key: input.ownerKey,
176
disabled: input.disabled,
177
});
183
- if (controllerState.prepared !== prepared || controllerState.grantId !== grantId) {
184
- setControllerState({ prepared, grantId, value: INITIAL_STATE });
178
+ if (previousDestination.prepared !== prepared || previousDestination.grantId !== grantId) {
179
+ setPreviousDestination({ prepared, grantId });
180
setSetup(INITIAL_SETUP);
181
setQueuedStart(null);
187
- setPreview(null);
188
- setLevel(0);
182
}
183
if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) {
184
setPreviousOwner({ key: input.ownerKey, disabled: input.disabled });
185
setQueuedStart(null);
186
setSetup(INITIAL_SETUP);
187
}
188
+
189
+ const session = useRetainedVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel });
190
+ const snapshot = useVoiceInputSessionSnapshot(session);
191
+ const owns = snapshot.target?.ownerKey === input.ownerKey;
192
+ const activity = useVoiceInputSessionActivity(session, input.ownerKey);
193
const state: VoiceInputState =
196
- queuedStart === prepared && prepared
197
- ? WAITING_STATE
198
- : controllerState.prepared === prepared
199
- ? controllerState.value
200
- : INITIAL_STATE;
194
+ queuedStart === prepared && prepared ? WAITING_STATE : owns ? snapshot.state : IDLE_VOICE_STATE;
195
+ // One microphone: another draft's recording leaves this editor without voice input until it ends.
196
+ const busyElsewhere = !owns && voiceInputBlocksSubmission(snapshot.state);
197
202
- const controllerRef = useRef<VoiceInputController | null>(null);
198
const startRequestRef = useRef(0);
199
const pendingSubmissionRef = useRef<{
200
ownerKey: string;
201
submit: (text: string) => void;
202
text?: string;
203
} | null>(null);
209
- const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords });
210
- // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props and settings to the asynchronous recording controller
211
- useEffect(() => {
212
- latestRef.current = { input, clientSettings, cleanupPrepared, projectWords };
204
+ const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords, session });
205
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props before an async completion can read them through the session
206
+ useLayoutEffect(() => {
207
+ latestRef.current = { input, clientSettings, cleanupPrepared, projectWords, session };
208
});
214
- // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the recording controller bound to the consented destination; it is disposed with the consent
209
+ const mountedRef = useRef(false);
210
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons this editor's pending work, and a recording that cannot outlive it, on unmount or a destination change
211
useEffect(() => {
216
- if (!prepared || !grantId || typeof navigator === "undefined") return;
217
- const hasAudioConsent = bindAudioGrant(grantId);
218
- let disposed = false;
219
- let controller: VoiceInputController;
220
- let recordingPreferences:
221
- | {
222
- readonly cleanup: SpeechPostProcessingOptions | null;
223
- readonly cleanupGrant: string | null;
224
- readonly environmentId: EnvironmentId;
225
- }
226
- | undefined;
227
-
228
- const readDraft = (): VoiceDraftSnapshot => {
229
- const current = latestRef.current.input;
230
- const draft = current.readDraft();
231
- return { ownerKey: current.ownerKey, text: draft.text, selection: draft.selection };
232
- };
233
- const platform = createBrowserVoiceInputPlatform({
234
- prepared,
235
- cloudProvider,
236
- cloudModel,
237
- hasAudioConsent,
238
- getStreamingEnabled: () => latestRef.current.clientSettings.speechStreaming,
239
- getTranscriptionOptions: (): SpeechTranscriptionOptions => {
240
- const {
241
- clientSettings: settings,
242
- projectWords: project,
243
- input: current,
244
- } = latestRef.current;
245
- const cleanupGrant = current.cleanup === false ? null : readCleanupGrant();
246
- const cleanupEnabled = cleanupGrant !== null;
247
- const cue =
248
- cleanupEnabled && latestRef.current.cleanupPrepared ? settings.speechCorrectionWord : "";
249
- const speechCustomWords = resolveSpeechVocabulary({
250
- personal: settings.speechCustomWords,
251
- project,
252
- hints: [cue, current.projectName ?? ""],
253
- });
254
- recordingPreferences = {
255
- cleanupGrant,
256
- environmentId: current.environmentId,
257
- cleanup: cleanupEnabled
258
- ? {
259
- speechCustomWords,
260
- speechPostProcessingPrompt: settings.speechPostProcessingPrompt,
261
- ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
262
- }
263
- : null,
264
- };
265
- return {
266
- speechLanguage: settings.speechLanguage,
267
- speechTranslateToEnglish: settings.speechTranslateToEnglish,
268
- speechCustomWords,
269
- speechRemoveFillerWords: settings.speechRemoveFillerWords,
270
- ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
271
- speechSilenceFiltering: settings.speechSilenceFiltering,
272
- };
273
- },
274
- getMicrophoneId: chooseRecordingMicrophone,
275
- onLevel: setLevel,
276
- onDurationLimit: () => void controller.stop(),
277
- onText: (text) => {
278
- if (!disposed) setPreview(text);
279
- },
280
- onError: (message) => {
281
- if (!disposed) void controller.interruptRecording(message);
282
- },
283
- });
284
- const postProcess: NonNullable<VoiceInputControllerDependencies["postProcess"]> = (
285
- transcript,
286
- options,
287
- ) => {
288
- const preferences = recordingPreferences;
289
- const { input: current, cleanupPrepared: target } = latestRef.current;
290
- if (
291
- !preferences?.cleanup ||
292
- !hasAudioConsent() ||
293
- preferences.cleanupGrant !== readCleanupGrant() ||
294
- current.cleanup === false ||
295
- current.environmentId !== preferences.environmentId ||
296
- current.ownerKey !== options.draft.ownerKey
297
- )
298
- return null;
299
- if (!target)
300
- return Promise.reject(
301
- new Error("The draft's environment is unavailable for transcript cleanup."),
302
- );
303
- return runEnvironmentSpeech(
304
- postProcessEnvironmentTranscript(
305
- target,
306
- transcript,
307
- cleanupContext(options.draft),
308
- preferences.cleanup,
309
- () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
310
- ),
311
- options,
312
- ).then((result) => result.text);
313
- };
314
- controller = new VoiceInputController({
315
- retryFailedRecording: true,
316
- recorder: platform.recorder,
317
- getTranscriber: () => platform.transcriber,
318
- requestPermission: async () => ({ granted: true, canAskAgain: true }),
319
- configureRecording: async () => undefined,
320
- releaseRecording: async () => platform.cancelRecording(),
321
- deleteRecording: platform.deleteRecording,
322
- postProcess,
323
- onPostProcessingError: () =>
324
- toastManager.add({
325
- type: "warning",
326
- title: "Transcript cleanup failed",
327
- description: "The original transcript was added.",
328
- }),
329
- readDraft,
330
- commitDraft: (text, selection) => {
331
- if (latestRef.current.input.disabled) return;
332
- latestRef.current.input.commitDraft(text, selection);
333
- if (pendingSubmissionRef.current) pendingSubmissionRef.current.text = text;
334
- },
335
- onStateChange: (value) => {
336
- if (disposed) return;
337
- if (value.phase === "error") pendingSubmissionRef.current = null;
338
- setControllerState({ prepared, grantId, value });
339
- if (value.phase !== "recording" && value.phase !== "transcribing") setPreview(null);
340
- },
341
- });
342
- controllerRef.current = controller;
212
+ mountedRef.current = true;
213
return () => {
344
- disposed = true;
214
+ mountedRef.current = false;
215
pendingSubmissionRef.current = null;
216
startRequestRef.current += 1;
347
- setQueuedStart(null);
348
- controller.dispose();
349
- if (controllerRef.current === controller) controllerRef.current = null;
217
+ session?.ownerChanged(latestRef.current.input.ownerKey);
218
};
351
- }, [prepared, grantId, cloudProvider, cloudModel]);
219
+ }, [session]);
220
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tells the global indicator this draft is on screen
221
+ useEffect(
222
+ () => (input.presented === false ? undefined : presentVoiceOwner(input.ownerKey)),
223
+ [input.ownerKey, input.presented],
224
+ );
225
226
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available
227
useEffect(() => {
@@ -394,18 +267,21 @@ export function useEnvironmentSpeechInput(input: HookInput) {
267
}, [prepared, setup.downloading, setup.open, setupModelId]);
268
269
const previousOwnerRef = useRef(input.ownerKey);
397
- // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes
270
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons any queued submission, and a recording that cannot outlive its editor, when the draft's owner changes
271
useEffect(() => {
399
- if (previousOwnerRef.current === input.ownerKey && !input.disabled) return;
272
+ const previous = previousOwnerRef.current;
273
+ if (previous === input.ownerKey && !input.disabled) return;
274
previousOwnerRef.current = input.ownerKey;
275
pendingSubmissionRef.current = null;
276
startRequestRef.current += 1;
403
- controllerRef.current?.ownerChanged();
404
- }, [input.ownerKey, input.disabled]);
277
+ if (previous !== input.ownerKey) session?.ownerChanged(previous);
278
+ else if (session?.ownerKey === input.ownerKey) session.cancel();
279
+ }, [input.ownerKey, input.disabled, session]);
280
281
// oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft
282
useEffect(() => {
283
const pending = pendingSubmissionRef.current;
284
+ if (state.phase === "error") pendingSubmissionRef.current = null;
285
if (state.phase !== "idle" || !pending) return;
286
pendingSubmissionRef.current = null;
287
if (!input.disabled && pending.ownerKey === input.ownerKey && pending.text !== undefined)
@@ -414,16 +290,62 @@ export function useEnvironmentSpeechInput(input: HookInput) {
290
291
const submitAfterDictation = useStableCallback((submit: (text: string) => void) => {
292
if (latestRef.current.input.disabled || state.phase === "preparing") return;
417
- const controller = controllerRef.current;
418
- if (!controller || !voiceInputBlocksSubmission(state)) {
293
+ if (!session || !voiceInputBlocksSubmission(state)) {
294
submit(latestRef.current.input.readDraft().text);
295
return;
296
}
297
if (pendingSubmissionRef.current) return;
298
pendingSubmissionRef.current = { ownerKey: latestRef.current.input.ownerKey, submit };
424
- if (controller.currentState.phase === "recording") void controller.stop();
299
+ if (session.state.phase === "recording") void session.stop();
300
});
301
302
+ const createTarget = useStableCallback((): VoiceInputTarget => {
303
+ const owner = latestRef.current.input.ownerKey;
304
+ const readContext = () => {
305
+ const current = latestRef.current;
306
+ return {
307
+ environmentId: current.input.environmentId,
308
+ projectName: current.input.projectName ?? null,
309
+ projectWords: current.projectWords,
310
+ clientSettings: current.clientSettings,
311
+ cleanup: current.input.cleanup !== false,
312
+ cleanupPrepared: current.cleanupPrepared,
313
+ };
314
+ };
315
+ // Once this editor shows another draft, its environment and cleanup must not see this one.
316
+ const startContext = readContext();
317
+ const persistent = latestRef.current.input.persistentDraft ?? null;
318
+ // While this editor shows the owner, the live editor is the draft; otherwise the stored one.
319
+ const live = () => mountedRef.current && latestRef.current.input.ownerKey === owner;
320
+ return {
321
+ ownerKey: owner,
322
+ label: latestRef.current.input.label ?? null,
323
+ persistent: persistent !== null,
324
+ open: persistent?.open ?? null,
325
+ readDraft: () => {
326
+ if (live()) return latestRef.current.input.readDraft();
327
+ const text = persistent?.readStoredText();
328
+ return text == null ? null : { text, selection: { start: text.length, end: text.length } };
329
+ },
330
+ commitDraft: (text, selection) => {
331
+ if (!live()) {
332
+ persistent?.commitStoredText(text);
333
+ return;
334
+ }
335
+ if (latestRef.current.input.disabled) return;
336
+ latestRef.current.input.commitDraft(text, selection);
337
+ if (pendingSubmissionRef.current?.ownerKey === owner)
338
+ pendingSubmissionRef.current.text = text;
339
+ },
340
+ readContext: () => (live() ? readContext() : startContext),
341
+ };
342
+ });
343
+
344
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- a remounted editor takes over its draft's running recording so a queued Send still sees the transcript
345
+ useEffect(() => {
346
+ if (session?.ownerKey === input.ownerKey && voiceInputBlocksSubmission(session.state))
347
+ session.adopt(createTarget());
348
+ }, [session, input.ownerKey, createTarget]);
349
const loadSetupCatalog = useStableCallback(
350
async (connection: PreparedConnection, selectedModelId: string, request: number) => {
351
setSetup({ ...INITIAL_SETUP, open: true });
@@ -458,17 +380,15 @@ export function useEnvironmentSpeechInput(input: HookInput) {
380
return;
381
}
382
const request = ++startRequestRef.current;
461
- const controller = controllerRef.current;
383
const owner = latestRef.current.input.ownerKey;
463
- if (!controller || !prepared) return;
384
+ if (!session || !prepared || busyElsewhere) return;
385
if (cloudProvider) {
465
- setLevel(0);
466
- await controller.start();
386
+ await session.start(createTarget());
387
return;
388
}
389
const stillCurrent = () =>
390
request === startRequestRef.current &&
471
- controllerRef.current === controller &&
391
+ latestRef.current.session === session &&
392
latestRef.current.input.ownerKey === owner &&
393
!latestRef.current.input.disabled;
394
setSetup(INITIAL_SETUP);
@@ -500,8 +420,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
420
});
421
return;
422
}
503
- setLevel(0);
504
- await controller.start();
423
+ await session.start(createTarget());
424
});
425
426
const downloadSetupModel = useStableCallback(async () => {
@@ -574,6 +493,7 @@ export function useEnvironmentSpeechInput(input: HookInput) {
493
return {
494
available:
495
destinationReady &&
496
+ !busyElsewhere &&
497
typeof navigator !== "undefined" &&
498
Boolean(navigator.mediaDevices?.getUserMedia) &&
499
typeof AudioWorkletNode !== "undefined",
@@ -595,21 +515,25 @@ export function useEnvironmentSpeechInput(input: HookInput) {
515
},
516
},
517
state,
598
- preview: state.phase === "recording" || state.phase === "transcribing" ? preview : null,
599
- level,
518
+ preview:
519
+ state.phase === "recording" || state.phase === "transcribing" ? activity.preview : null,
520
+ level: activity.level,
521
+ recordingStartedAt: owns ? snapshot.recordingStartedAt : null,
522
preparing: state.phase === "preparing",
523
blocksSubmission: voiceInputBlocksSubmission(state),
524
freezesEditor: voiceInputBlocksSubmission(state),
525
start,
526
submitAfterDictation,
605
- stop: useStableCallback(() => controllerRef.current?.stop() ?? Promise.resolve()),
527
+ stop: useStableCallback(() => (owns && session ? session.stop() : Promise.resolve())),
528
cancel: useStableCallback(() => {
529
pendingSubmissionRef.current = null;
530
startRequestRef.current += 1;
531
setQueuedStart(null);
532
setSetup(INITIAL_SETUP);
611
- controllerRef.current?.cancel();
533
+ if (owns) session?.cancel();
534
+ }),
535
+ skipPostProcessing: useStableCallback(() => {
536
+ if (owns) session?.skipPostProcessing();
537
}),
613
- skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()),
538
};
539
}
apps/web/src/speech/voiceInputSession.ts
new
+424
@@ -0,0 +1,424 @@
1
+import {
2
+ postProcessEnvironmentTranscript,
3
+ VoiceInputController,
4
+ voiceInputBlocksSubmission,
5
+ type VoiceDraftSnapshot,
6
+ type VoiceInputControllerDependencies,
7
+ type VoiceInputState,
8
+} from "@t3tools/client-runtime/voice-input";
9
+import type { PreparedConnection } from "@t3tools/client-runtime/connection";
10
+import type {
11
+ CloudSpeechModelId,
12
+ CloudSpeechProvider,
13
+ ClientSettings,
14
+ EnvironmentId,
15
+ SpeechCustomWords,
16
+ SpeechPostProcessingOptions,
17
+ SpeechStreamText,
18
+ SpeechTranscriptionOptions,
19
+} from "@t3tools/contracts";
20
+import { DEFAULT_CLIENT_SETTINGS } from "@t3tools/contracts";
21
+import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
22
+import { useEffect, useSyncExternalStore } from "react";
23
+
24
+import { toastManager } from "../components/ui/toast";
25
+import { chooseRecordingMicrophone } from "./browserMicrophones";
26
+import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
27
+import { runEnvironmentSpeech } from "./runEnvironmentSpeech";
28
+import { bindAudioGrant, readCleanupGrant } from "./voiceConsent";
29
+
30
+export const IDLE_VOICE_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
31
+
32
+type DraftInput = {
33
+ readonly text: string;
34
+ readonly selection: { readonly start: number; readonly end: number };
35
+};
36
+
37
+/** What a recording needs from its editor, read once when it starts. */
38
+export type VoiceRecordingContext = {
39
+ readonly environmentId: EnvironmentId;
40
+ readonly projectName: string | null;
41
+ readonly projectWords: SpeechCustomWords;
42
+ readonly clientSettings: ClientSettings;
43
+ readonly cleanup: boolean;
44
+ readonly cleanupPrepared: PreparedConnection | null;
45
+};
46
+
47
+export type VoiceInputTarget = {
48
+ readonly ownerKey: string;
49
+ /** Names the draft in the indicator shown while its editor is off screen. */
50
+ readonly label: string | null;
51
+ /** A target that can outlive its editor (a thread draft) keeps recording when the owner changes. */
52
+ readonly persistent: boolean;
53
+ readonly open: (() => void) | null;
54
+ readonly readDraft: () => DraftInput | null;
55
+ readonly commitDraft: (text: string, selection: DraftInput["selection"]) => void;
56
+ readonly readContext: () => VoiceRecordingContext;
57
+};
58
+
59
+export type VoiceInputSessionSnapshot = {
60
+ readonly state: VoiceInputState;
61
+ readonly target: VoiceInputTarget | null;
62
+ /** Wall-clock start of the current recording, so every view shows the same elapsed time. */
63
+ readonly recordingStartedAt: number | null;
64
+};
65
+
66
+export type VoiceInputSessionActivity = {
67
+ readonly level: number;
68
+ readonly preview: SpeechStreamText | null;
69
+};
70
+
71
+type SessionKey = {
72
+ readonly prepared: PreparedConnection;
73
+ readonly grantId: string;
74
+ readonly cloudProvider: CloudSpeechProvider | undefined;
75
+ readonly cloudModel: CloudSpeechModelId | undefined;
76
+};
77
+
78
+const SILENT_ACTIVITY: VoiceInputSessionActivity = { level: 0, preview: null };
79
+
80
+function cleanupContext({ text, selection }: DraftInput): DraftInput {
81
+ const limit = 1_024;
82
+ const start = Math.max(0, Math.min(selection.start, text.length));
83
+ const end = Math.max(start, Math.min(selection.end, text.length));
84
+ const before = text.slice(Math.max(0, start - limit), start);
85
+ const selected = text.slice(start, Math.min(end, start + limit));
86
+ return {
87
+ text: before + selected + text.slice(end, end + limit),
88
+ selection: { start: before.length, end: before.length + selected.length },
89
+ };
90
+}
91
+
92
+/**
93
+ * The browser has one microphone, so every editor shares one dictation per destination. The
94
+ * recording belongs to the target that started it, not to whichever editor is mounted now.
95
+ */
96
+export class VoiceInputSession {
97
+ readonly key: SessionKey;
98
+ private readonly controller: VoiceInputController;
99
+ private target: VoiceInputTarget | null = null;
100
+ private snapshot: VoiceInputSessionSnapshot = {
101
+ state: IDLE_VOICE_STATE,
102
+ target: null,
103
+ recordingStartedAt: null,
104
+ };
105
+ private activity = SILENT_ACTIVITY;
106
+ private readonly listeners = new Set<() => void>();
107
+ private readonly activityListeners = new Set<() => void>();
108
+ private disposed = false;
109
+
110
+ constructor(key: SessionKey) {
111
+ this.key = key;
112
+ const hasAudioConsent = bindAudioGrant(key.grantId);
113
+ let recordingPreferences:
114
+ | {
115
+ readonly cleanup: SpeechPostProcessingOptions | null;
116
+ readonly cleanupGrant: string | null;
117
+ readonly cleanupPrepared: PreparedConnection | null;
118
+ }
119
+ | undefined;
120
+ const platform = createBrowserVoiceInputPlatform({
121
+ prepared: key.prepared,
122
+ cloudProvider: key.cloudProvider,
123
+ cloudModel: key.cloudModel,
124
+ hasAudioConsent,
125
+ getStreamingEnabled: () =>
126
+ this.target?.readContext().clientSettings.speechStreaming ??
127
+ DEFAULT_CLIENT_SETTINGS.speechStreaming,
128
+ getTranscriptionOptions: (): SpeechTranscriptionOptions => {
129
+ const context = this.target?.readContext();
130
+ if (!context) throw new Error("Voice input has no draft to record into.");
131
+ const settings = context.clientSettings;
132
+ const cleanupGrant = context.cleanup ? readCleanupGrant() : null;
133
+ const cleanupEnabled = cleanupGrant !== null;
134
+ const cue = cleanupEnabled && context.cleanupPrepared ? settings.speechCorrectionWord : "";
135
+ const speechCustomWords = resolveSpeechVocabulary({
136
+ personal: settings.speechCustomWords,
137
+ project: context.projectWords,
138
+ hints: [cue, context.projectName ?? ""],
139
+ });
140
+ recordingPreferences = {
141
+ cleanupGrant,
142
+ cleanupPrepared: context.cleanupPrepared,
143
+ cleanup: cleanupEnabled
144
+ ? {
145
+ speechCustomWords,
146
+ speechPostProcessingPrompt: settings.speechPostProcessingPrompt,
147
+ ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
148
+ }
149
+ : null,
150
+ };
151
+ return {
152
+ speechLanguage: settings.speechLanguage,
153
+ speechTranslateToEnglish: settings.speechTranslateToEnglish,
154
+ speechCustomWords,
155
+ speechRemoveFillerWords: settings.speechRemoveFillerWords,
156
+ ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
157
+ speechSilenceFiltering: settings.speechSilenceFiltering,
158
+ };
159
+ },
160
+ getMicrophoneId: chooseRecordingMicrophone,
161
+ onLevel: (level) => this.publishActivity({ ...this.activity, level }),
162
+ onDurationLimit: () => void this.controller.stop(),
163
+ onText: (preview) => {
164
+ if (!this.disposed) this.publishActivity({ ...this.activity, preview });
165
+ },
166
+ onError: (message) => {
167
+ if (!this.disposed) void this.controller.interruptRecording(message);
168
+ },
169
+ });
170
+ const postProcess: NonNullable<VoiceInputControllerDependencies["postProcess"]> = (
171
+ transcript,
172
+ options,
173
+ ) => {
174
+ const preferences = recordingPreferences;
175
+ if (
176
+ !preferences?.cleanup ||
177
+ !hasAudioConsent() ||
178
+ preferences.cleanupGrant !== readCleanupGrant() ||
179
+ this.target?.ownerKey !== options.draft.ownerKey
180
+ )
181
+ return null;
182
+ if (!preferences.cleanupPrepared)
183
+ return Promise.reject(
184
+ new Error("The draft's environment is unavailable for transcript cleanup."),
185
+ );
186
+ return runEnvironmentSpeech(
187
+ postProcessEnvironmentTranscript(
188
+ preferences.cleanupPrepared,
189
+ transcript,
190
+ cleanupContext(options.draft),
191
+ preferences.cleanup,
192
+ () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
193
+ ),
194
+ options,
195
+ ).then((result) => result.text);
196
+ };
197
+ this.controller = new VoiceInputController({
198
+ retryFailedRecording: true,
199
+ recorder: platform.recorder,
200
+ getTranscriber: () => platform.transcriber,
201
+ requestPermission: async () => ({ granted: true, canAskAgain: true }),
202
+ configureRecording: async () => undefined,
203
+ releaseRecording: async () => platform.cancelRecording(),
204
+ deleteRecording: platform.deleteRecording,
205
+ postProcess,
206
+ onPostProcessingError: () =>
207
+ toastManager.add({
208
+ type: "warning",
209
+ title: "Transcript cleanup failed",
210
+ description: "The original transcript was added.",
211
+ }),
212
+ readDraft: (): VoiceDraftSnapshot | null => {
213
+ const target = this.target;
214
+ const draft = target?.readDraft();
215
+ return target && draft ? { ownerKey: target.ownerKey, ...draft } : null;
216
+ },
217
+ commitDraft: (text, selection) => this.target?.commitDraft(text, selection),
218
+ onStateChange: (state) => {
219
+ if (this.disposed) return;
220
+ const recordingStartedAt =
221
+ state.phase !== "recording" ? null : (this.snapshot.recordingStartedAt ?? Date.now());
222
+ this.publish({ ...this.snapshot, state, recordingStartedAt });
223
+ if (state.phase !== "recording" && state.phase !== "transcribing")
224
+ this.publishActivity(SILENT_ACTIVITY);
225
+ },
226
+ });
227
+ }
228
+
229
+ get state(): VoiceInputState {
230
+ return this.controller.currentState;
231
+ }
232
+
233
+ get ownerKey(): string | null {
234
+ return this.target?.ownerKey ?? null;
235
+ }
236
+
237
+ readonly subscribe = (listener: () => void): (() => void) => {
238
+ this.listeners.add(listener);
239
+ return () => this.listeners.delete(listener);
240
+ };
241
+
242
+ readonly getSnapshot = (): VoiceInputSessionSnapshot => this.snapshot;
243
+
244
+ readonly subscribeActivity = (listener: () => void): (() => void) => {
245
+ this.activityListeners.add(listener);
246
+ return () => this.activityListeners.delete(listener);
247
+ };
248
+
249
+ readonly getActivity = (): VoiceInputSessionActivity => this.activity;
250
+
251
+ /** Starts recording into `target`, or does nothing while another draft holds the microphone. */
252
+ start(target: VoiceInputTarget): Promise<void> {
253
+ if (this.disposed) return Promise.resolve();
254
+ const busyElsewhere =
255
+ voiceInputBlocksSubmission(this.controller.currentState) &&
256
+ this.target?.ownerKey !== target.ownerKey;
257
+ if (busyElsewhere) return Promise.resolve();
258
+ this.target = target;
259
+ this.publishActivity(SILENT_ACTIVITY);
260
+ this.publish({ ...this.snapshot, target });
261
+ return this.controller.start();
262
+ }
263
+
264
+ /** A newly mounted editor for the recording's draft replaces the target that started it. */
265
+ adopt(target: VoiceInputTarget): void {
266
+ if (this.disposed || this.target?.ownerKey !== target.ownerKey) return;
267
+ this.target = target;
268
+ this.publish({ ...this.snapshot, target });
269
+ }
270
+
271
+ stop(): Promise<void> {
272
+ return this.controller.stop();
273
+ }
274
+
275
+ cancel(): void {
276
+ this.controller.cancel();
277
+ }
278
+
279
+ skipPostProcessing(): void {
280
+ this.controller.skipPostProcessing();
281
+ }
282
+
283
+ /** The editor for `ownerKey` now shows another draft; only a persistent target keeps going. */
284
+ ownerChanged(ownerKey: string): void {
285
+ if (this.target?.ownerKey !== ownerKey || this.target.persistent) return;
286
+ this.controller.ownerChanged();
287
+ }
288
+
289
+ dispose(): void {
290
+ this.disposed = true;
291
+ this.controller.dispose();
292
+ this.listeners.clear();
293
+ this.activityListeners.clear();
294
+ }
295
+
296
+ private publish(snapshot: VoiceInputSessionSnapshot): void {
297
+ this.snapshot = snapshot;
298
+ for (const listener of this.listeners) listener();
299
+ }
300
+
301
+ private publishActivity(activity: VoiceInputSessionActivity): void {
302
+ if (activity.level === this.activity.level && activity.preview === this.activity.preview)
303
+ return;
304
+ this.activity = activity;
305
+ for (const listener of this.activityListeners) listener();
306
+ }
307
+}
308
+
309
+const sessions = new Map<VoiceInputSession, number>();
310
+const registryListeners = new Set<() => void>();
311
+
312
+function sameKey(session: VoiceInputSession, key: SessionKey): boolean {
313
+ return (
314
+ session.key.prepared === key.prepared &&
315
+ session.key.grantId === key.grantId &&
316
+ session.key.cloudProvider === key.cloudProvider &&
317
+ session.key.cloudModel === key.cloudModel
318
+ );
319
+}
320
+
321
+function findSession(key: SessionKey): VoiceInputSession | null {
322
+ for (const session of sessions.keys()) if (sameKey(session, key)) return session;
323
+ return null;
324
+}
325
+
326
+/** Every holder of a destination shares one session; the last release disposes it. */
327
+function retainVoiceInputSession(key: SessionKey): () => void {
328
+ const session = findSession(key) ?? new VoiceInputSession(key);
329
+ sessions.set(session, (sessions.get(session) ?? 0) + 1);
330
+ for (const listener of registryListeners) listener();
331
+ let released = false;
332
+ return () => {
333
+ if (released) return;
334
+ released = true;
335
+ const count = (sessions.get(session) ?? 1) - 1;
336
+ if (count > 0) {
337
+ sessions.set(session, count);
338
+ return;
339
+ }
340
+ sessions.delete(session);
341
+ session.dispose();
342
+ for (const listener of registryListeners) listener();
343
+ };
344
+}
345
+
346
+function subscribeRegistry(listener: () => void): () => void {
347
+ registryListeners.add(listener);
348
+ return () => registryListeners.delete(listener);
349
+}
350
+
351
+/** Holds the destination's shared session while mounted; null until it is retained. */
352
+export function useRetainedVoiceInputSession(input: {
353
+ readonly prepared: PreparedConnection | null;
354
+ readonly grantId: string | null;
355
+ readonly cloudProvider?: CloudSpeechProvider | undefined;
356
+ readonly cloudModel?: CloudSpeechModelId | undefined;
357
+}): VoiceInputSession | null {
358
+ const { prepared, grantId, cloudProvider, cloudModel } = input;
359
+ // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- retains the shared recording session for this destination while mounted
360
+ useEffect(() => {
361
+ if (!prepared || !grantId || typeof navigator === "undefined") return;
362
+ return retainVoiceInputSession({ prepared, grantId, cloudProvider, cloudModel });
363
+ }, [prepared, grantId, cloudProvider, cloudModel]);
364
+ return useSyncExternalStore(subscribeRegistry, () =>
365
+ prepared && grantId ? findSession({ prepared, grantId, cloudProvider, cloudModel }) : null,
366
+ );
367
+}
368
+
369
+const IDLE_SNAPSHOT: VoiceInputSessionSnapshot = {
370
+ state: IDLE_VOICE_STATE,
371
+ target: null,
372
+ recordingStartedAt: null,
373
+};
374
+const noSubscription = () => () => {};
375
+
376
+export function useVoiceInputSessionSnapshot(
377
+ session: VoiceInputSession | null,
378
+): VoiceInputSessionSnapshot {
379
+ return useSyncExternalStore(
380
+ session?.subscribe ?? noSubscription,
381
+ session ? session.getSnapshot : () => IDLE_SNAPSHOT,
382
+ );
383
+}
384
+
385
+export function useVoiceInputSessionActivity(
386
+ session: VoiceInputSession | null,
387
+ ownerKey: string | null,
388
+): VoiceInputSessionActivity {
389
+ // Activity changes many times a second, so only the owning editor reads it.
390
+ const owns = useSyncExternalStore(
391
+ session?.subscribe ?? noSubscription,
392
+ () => session !== null && ownerKey !== null && session.ownerKey === ownerKey,
393
+ );
394
+ return useSyncExternalStore(owns && session ? session.subscribeActivity : noSubscription, () =>
395
+ owns && session ? session.getActivity() : SILENT_ACTIVITY,
396
+ );
397
+}
398
+
399
+// Editors register the owner they show, so the indicator appears only while its draft is off screen.
400
+const presentedOwners = new Map<string, number>();
401
+const presentationListeners = new Set<() => void>();
402
+
403
+export function presentVoiceOwner(ownerKey: string): () => void {
404
+ presentedOwners.set(ownerKey, (presentedOwners.get(ownerKey) ?? 0) + 1);
405
+ for (const listener of presentationListeners) listener();
406
+ return () => {
407
+ const count = (presentedOwners.get(ownerKey) ?? 1) - 1;
408
+ if (count > 0) presentedOwners.set(ownerKey, count);
409
+ else presentedOwners.delete(ownerKey);
410
+ for (const listener of presentationListeners) listener();
411
+ };
412
+}
413
+
414
+function subscribePresentation(listener: () => void): () => void {
415
+ presentationListeners.add(listener);
416
+ return () => presentationListeners.delete(listener);
417
+}
418
+
419
+export function useVoiceOwnerPresented(ownerKey: string | null): boolean {
420
+ return useSyncExternalStore(
421
+ subscribePresentation,
422
+ () => ownerKey !== null && presentedOwners.has(ownerKey),
423
+ );
424
+}