@devries-cameron20 / t3code Mirror · read-only / commits / a9a1b8c4be
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 22, 03:15 UTC

feat(web): opt into environment dictation on this device (#30623)

Co-authored-by: UtkarshUsername <putkarsh184@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

Julius Marminge committed Nov 5, 2026 at 15:59 UTC a9a1b8c4be75b236a9a521d39d9c13e8a099a32a
84 files changed +7333 -292
apps/server/src/speech/SpeechModelStore.test.ts
+8 -7
@@ -13,7 +13,6 @@ import * as Stream from "effect/Stream";
13 import { HttpClient, HttpClientError, HttpClientRequest, HttpClientResponse } from "effect/http";
14
15 import * as ServerConfig from "../config.ts";
16 -import { getSpeechModel, type SpeechModel } from "./model.ts";
16 import * as SpeechModelStore from "./SpeechModelStore.ts";
17
18 const bytes = Uint8Array.from({ length: 64 }, (_, index) => (index * 7) % 256);
@@ -21,8 +20,9 @@ const SPLIT = 20;
20
21 const harness = Effect.gen(function* () {
22 const digest = yield* Digest.Digest;
24 - const model: SpeechModel = {
25 - ...getSpeechModel("handy-computer/whisper-tiny-gguf")!,
23 + const model = {
24 + id: "fixture/speech-model",
25 + revision: "a".repeat(40),
26 filename: "model.gguf",
27 size: bytes.length,
28 sha256: Hex.encode(yield* digest.digest(Digest.sha256, bytes).pipe(Effect.orDie)),
@@ -121,10 +121,8 @@ it.effect("resumes an interrupted transfer, replaces a corrupt same-size file, a
121 undefined,
122 `bytes=${SPLIT}-`,
123 ]);
124 - expect(h.requests.map((request) => request.url)).toEqual([
125 - "https://huggingface.co/handy-computer/whisper-tiny-gguf/resolve/6687f30c99641ee265df421e582354adbc8848fc/model.gguf",
126 - "https://huggingface.co/handy-computer/whisper-tiny-gguf/resolve/6687f30c99641ee265df421e582354adbc8848fc/model.gguf",
127 - ]);
124 + const pinnedUrl = `https://huggingface.co/${h.model.id}/resolve/${h.model.revision}/${h.model.filename}`;
125 + expect(h.requests.map((request) => request.url)).toEqual([pinnedUrl, pinnedUrl]);
126 expect(progress.at(0)).toBe(0);
127 expect(progress.at(-1)).toBe(bytes.length);
128 expect([...(yield* h.fs.readFile(installed))]).toEqual([...bytes]);
@@ -134,6 +132,9 @@ it.effect("resumes an interrupted transfer, replaces a corrupt same-size file, a
132 ]);
133 expect(yield* h.store.isReady(h.model)).toBe(true);
134 expect(yield* h.store.installedPath(h.model)).toBe(installed);
135 + const requestsBeforeReuse = h.requests.length;
136 + expect(yield* h.store.download(h.model)).toBe(installed);
137 + expect(h.requests).toHaveLength(requestsBeforeReuse);
138 }).pipe(Effect.provide(dependencies)),
139 );
140
apps/server/src/speech/SpeechModelStore.ts
+12 -10
@@ -17,6 +17,8 @@ import * as Stream from "effect/Stream";
17 import * as ServerConfig from "../config.ts";
18 import type { SpeechModel } from "./model.ts";
19
20 +type ModelFile = Pick<SpeechModel, "id" | "filename" | "revision" | "size" | "sha256">;
21 +
22 class SpeechModelDownloadError extends Schema.TaggedError<SpeechModelDownloadError>()(
23 "SpeechModelDownloadError",
24 {
@@ -39,17 +41,17 @@ export class SpeechModelStore extends Context.Service<
41 SpeechModelStore,
42 {
43 /** Cheap size check for status polls; never hashes the file. */
42 - readonly isReady: (model: SpeechModel) => Effect.Effect<boolean>;
44 + readonly isReady: (model: ModelFile) => Effect.Effect<boolean>;
45 /** The installed file once its SHA-256 has been verified, hashing it on first use. */
44 - readonly installedPath: (model: SpeechModel) => Effect.Effect<string | undefined>;
46 + readonly installedPath: (model: ModelFile) => Effect.Effect<string | undefined>;
47 readonly download: (
46 - model: SpeechModel,
48 + model: ModelFile,
49 progress?: SpeechModelDownloadProgress,
50 ) => Effect.Effect<
51 string,
52 SpeechModelDownloadError | PlatformError | HttpClientError.HttpClientError
53 >;
52 - readonly remove: (model: SpeechModel) => Effect.Effect<void, PlatformError>;
54 + readonly remove: (model: ModelFile) => Effect.Effect<void, PlatformError>;
55 }
56 >()("t3/speech/SpeechModelStore") {}
57
@@ -70,11 +72,11 @@ const make = Effect.gen(function* () {
72 yield* fs.remove(path.join(directory, name), { force: true }).pipe(Effect.ignore);
73 }
74 }
73 - const modelPath = (model: SpeechModel) => path.join(directory, model.filename);
75 + const modelPath = (model: ModelFile) => path.join(directory, model.filename);
76 // Files whose digest failed, by identity, so status polls stop calling them ready.
77 const corrupt = new Map<string, string>();
78
77 - const identity = (model: SpeechModel, info: FileSystem.File.Info) =>
79 + const identity = (model: ModelFile, info: FileSystem.File.Info) =>
80 [
81 model.sha256,
82 info.size,
@@ -85,7 +87,7 @@ const make = Effect.gen(function* () {
87
88 const statFile = (file: string) => fs.stat(file).pipe(Effect.orElseSucceed(() => undefined));
89
88 - const expectedFile = (model: SpeechModel, file: string) =>
90 + const expectedFile = (model: ModelFile, file: string) =>
91 statFile(file).pipe(
92 Effect.map((info) =>
93 info?.type === "File" && Number(info.size) === model.size ? info : undefined,
@@ -93,7 +95,7 @@ const make = Effect.gen(function* () {
95 );
96
97 const digestMatches = Effect.fn("SpeechModelStore.digest")(
96 - function* (model: SpeechModel, file: string) {
98 + function* (model: ModelFile, file: string) {
99 const hash = NodeCrypto.createHash("sha256");
100 yield* fs
101 .stream(file)
@@ -105,7 +107,7 @@ const make = Effect.gen(function* () {
107
108 // A verified file is trusted while its size, mtime, device and inode are unchanged.
109 const verifiedPath = Effect.fn("SpeechModelStore.verifiedPath")(function* (
108 - model: SpeechModel,
110 + model: ModelFile,
111 file: string,
112 onVerificationStart?: () => void,
113 ) {
@@ -127,7 +129,7 @@ const make = Effect.gen(function* () {
129 });
130
131 const fetchInto = Effect.fn("SpeechModelStore.fetch")(function* (
130 - model: SpeechModel,
132 + model: ModelFile,
133 url: string,
134 partialPath: string,
135 offset: number,
apps/server/src/speech/SpeechService.test.ts
+185 -17
@@ -10,19 +10,22 @@ import * as Result from "effect/Result";
10 import * as Scheduler from "effect/Scheduler";
11 import * as Scope from "effect/Scope";
12 import * as Exit from "effect/Exit";
13 +import * as PlatformError from "effect/PlatformError";
14 import * as HostProcess from "@t3tools/shared/HostProcess";
15 import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS } from "@t3tools/contracts";
16 import * as ServerSettings from "../serverSettings.ts";
17 import { listNativeSpeechGpuDevices } from "./native.ts";
18 import * as SpeechModelStore from "./SpeechModelStore.ts";
19 import * as SpeechService from "./SpeechService.ts";
20 +import { SILERO_VAD } from "./silero.ts";
21
22 const native = vi.hoisted(() => ({
23 backend: "Vulkan0",
24 supportsStreaming: true,
25 + supportsLongForm: false,
26 supportsTranslation: true,
27 supportsVocabulary: false,
25 - begin: vi.fn(async (_language?: string, _vocabulary?: readonly string[]) => {}),
28 + begin: vi.fn(async (_options?: unknown, _filterSilence?: boolean, _vadPath?: string) => {}),
29 feed: vi.fn(async (_pcm: Float32Array) => ({
30 revision: 1,
31 text: { committed: "", tentative: "hello" },
@@ -66,13 +69,13 @@ const modelDefinitions = vi.hoisted(() => [
69 name: "fallback",
70 description: "fallback",
71 size: 8,
69 - languages: ["en"],
72 + languages: ["en", "fr"],
73 accuracy: 2,
74 speed: 2,
75 recommended: false,
76 supportsStreaming: false,
77 supportsLanguageDetection: false,
75 - supportsTranslation: false,
78 + supportsTranslation: true,
79 },
80 ]);
81 vi.mock("./native.ts", () => ({
@@ -112,8 +115,10 @@ const pcm = () => new Uint8Array(new Float32Array([0.25]).buffer);
115 beforeEach(() => {
116 vi.clearAllMocks();
117 loadNative.mockReset();
118 + native.finish.mockReset();
119 native.backend = "Vulkan0";
120 native.supportsVocabulary = false;
121 + native.supportsStreaming = true;
122 readyModels.clear();
123 readyModels.add("test-model");
124 readyModels.add("fallback-model");
@@ -146,6 +151,21 @@ it.effect("prepares a batch model before audio arrives and reuses it for transcr
151 yield* speech.prepareModel;
152 expect(loadNative).toHaveBeenCalledTimes(1);
153 expect(native.transcribe).not.toHaveBeenCalled();
154 + native.supportsStreaming = false;
155 + yield* Effect.gen(function* () {
156 + const stream = yield* speech.startStream({
157 + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
158 + speechLanguage: "fr",
159 + speechTranslateToEnglish: true,
160 + speechSilenceFiltering: false,
161 + });
162 + expect(native.begin).toHaveBeenLastCalledWith(
163 + { timestamps: "none", language: "fr", task: "translate", targetLanguage: "en" },
164 + false,
165 + "test.gguf",
166 + );
167 + expect(yield* stream.finish).toBe("hello");
168 + }).pipe(Effect.scoped);
169 expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
170 expect(loadNative).toHaveBeenCalledTimes(1);
171 }).pipe(Effect.provide(layer)),
@@ -223,28 +243,112 @@ it.effect("keeps a detached download exclusive until the transfer finishes", ()
243 const speech = yield* SpeechService.SpeechService;
244 const started = yield* Deferred.make<void>();
245 const finish = yield* Deferred.make<void>();
226 - download.mockImplementationOnce(() =>
227 - Deferred.succeed(started, undefined).pipe(
228 - Effect.andThen(Deferred.await(finish)),
229 - Effect.as("test.gguf"),
230 - ),
231 - );
246 + const helperStarted = yield* Deferred.make<void>();
247 + let helperStartedDuringInstall = false;
248 + let helperInterrupted = false;
249 + download
250 + .mockImplementationOnce(() =>
251 + Deferred.succeed(started, undefined).pipe(
252 + Effect.andThen(Deferred.await(finish)),
253 + Effect.as("test.gguf"),
254 + ),
255 + )
256 + .mockImplementationOnce(() =>
257 + Deferred.isDone(finish).pipe(
258 + Effect.tap((done) =>
259 + Effect.sync(() => {
260 + helperStartedDuringInstall = !done;
261 + }),
262 + ),
263 + Effect.andThen(Deferred.succeed(helperStarted, undefined)),
264 + Effect.andThen(Effect.never),
265 + Effect.onInterrupt(() =>
266 + Effect.sync(() => {
267 + helperInterrupted = true;
268 + }),
269 + ),
270 + ),
271 + );
272 const request = yield* speech.downloadModel("test-model").pipe(Effect.forkChild);
273 yield* Deferred.await(started);
274 + yield* Deferred.await(helperStarted);
275 yield* Fiber.interrupt(request);
276 + yield* Deferred.succeed(finish, undefined);
277 + yield* TestClock.adjust("16 seconds");
278 + expect(helperInterrupted).toBe(false);
279 const removed = yield* Effect.result(speech.removeModel("test-model"));
280 expect(Result.isFailure(removed) && removed.failure).toMatchObject({ _tag: "SpeechBusyError" });
281 const join = yield* speech
282 .downloadModel("test-model")
283 .pipe(Effect.forkChild({ startImmediately: true }));
240 - yield* Deferred.succeed(finish, undefined);
284 + yield* TestClock.adjust("2 minutes");
285 + expect(helperInterrupted).toBe(true);
286 yield* Fiber.join(join);
242 - expect(download).toHaveBeenCalledTimes(1);
287 + expect(download.mock.calls.map(([model]) => model.id)).toEqual(["test-model", SILERO_VAD.id]);
288 + expect(helperStartedDuringInstall).toBe(true);
289 yield* speech.removeModel("test-model");
290 expect(readyModels.has("test-model")).toBe(false);
291 }).pipe(Effect.provide(layer)),
292 );
293
294 +it.effect("bounds missing Silero recovery during recording and continues with raw audio", () =>
295 + Effect.gen(function* () {
296 + const helperStarted = yield* Deferred.make<void>();
297 + const helperInterrupted = yield* Deferred.make<void>();
298 + download.mockImplementationOnce(() =>
299 + Deferred.succeed(helperStarted, undefined).pipe(
300 + Effect.andThen(Effect.never),
301 + Effect.onInterrupt(() => Deferred.succeed(helperInterrupted, undefined)),
302 + ),
303 + );
304 + const speech = yield* SpeechService.SpeechService;
305 + const recording = yield* speech
306 + .startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)
307 + .pipe(Effect.forkChild);
308 + yield* Deferred.await(helperStarted);
309 + yield* TestClock.adjust("14999 millis");
310 + expect(native.begin).not.toHaveBeenCalled();
311 + yield* TestClock.adjust("1 millis");
312 + yield* Deferred.await(helperInterrupted);
313 + const stream = yield* Fiber.join(recording);
314 + expect(native.begin).toHaveBeenLastCalledWith(expect.anything(), true, undefined);
315 + yield* stream.feed(pcm());
316 + expect(yield* stream.finish).toBe("hello");
317 + }).pipe(Effect.scoped, Effect.provide(layer)),
318 +);
319 +
320 +it.effect("keeps the installed speech model usable when optional Silero acquisition fails", () =>
321 + Effect.gen(function* () {
322 + download
323 + .mockImplementationOnce(() => Effect.succeed("test.gguf"))
324 + .mockImplementationOnce(() =>
325 + Effect.fail(
326 + new PlatformError.PlatformError(
327 + new PlatformError.SystemError({
328 + _tag: "NotFound",
329 + module: "FileSystem",
330 + method: "open",
331 + }),
332 + ),
333 + ),
334 + );
335 + const speech = yield* SpeechService.SpeechService;
336 + const installed = yield* speech.downloadModel("test-model");
337 + expect(installed.models.find((model) => model.id === "test-model")?.state).toBe("installed");
338 + expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
339 + yield* Effect.scoped(
340 + Effect.gen(function* () {
341 + const stream = yield* speech.startStream(DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
342 + expect(yield* stream.finish).toBe("hello");
343 + }),
344 + );
345 + expect(download).toHaveBeenCalledTimes(2);
346 + yield* TestClock.adjust("10 minutes");
347 + expect(yield* speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS)).toBe("hello");
348 + expect(download).toHaveBeenCalledTimes(3);
349 + }).pipe(Effect.provide(layer)),
350 +);
351 +
352 it.effect(
353 "allows another model operation after an already-installed download completes immediately",
354 () =>
@@ -333,7 +437,11 @@ it.effect("uses vocabulary only for supported transcription and always applies a
437 text: { committed: vocabulary ? "git hub" : "GitHub", tentative: "Bob" },
438 });
439 expect(yield* stream.finish).toBe(vocabulary ? "git hub and Bob" : "GitHub and Bob");
336 - expect(native.begin.mock.lastCall).toEqual(["fr", vocabulary]);
440 + expect(native.begin.mock.lastCall).toEqual([
441 + { timestamps: "none", language: "fr", ...(vocabulary ? { vocabulary } : {}) },
442 + true,
443 + "test.gguf",
444 + ]);
445 }).pipe(Effect.scoped, Effect.provide(layer));
446 }
447 }
@@ -354,6 +462,58 @@ it.effect("unloads after the chosen idle period and reloads on the next recordin
462 }).pipe(Effect.provide(layer)),
463 );
464
465 +it.effect.each([
466 + { cue: "um", remove: true, text: "Make it red, um, blue.", expected: "Make it red, um, blue." },
467 + {
468 + cue: "uh no",
469 + remove: true,
470 + text: "Make it red, uh no, blue.",
471 + expected: "Make it red, uh no, blue.",
472 + },
473 + {
474 + cue: "uh no",
475 + remove: true,
476 + text: "Uh, Make it red, UH NO, blue. Uh nope.",
477 + expected: "Make it red, UH NO, blue. nope.",
478 + },
479 + {
480 + cue: " UM ",
481 + remove: true,
482 + text: "Uh, Make it red, um, blue.",
483 + expected: "Make it red, um, blue.",
484 + },
485 + {
486 + cue: undefined,
487 + remove: true,
488 + text: "Uh, Make it red, um, blue.",
489 + expected: "Make it red, blue.",
490 + },
491 + {
492 + cue: "um",
493 + remove: false,
494 + text: "Uh, Make it red, um, blue.",
495 + expected: "Uh, Make it red, um, blue.",
496 + },
497 +])(
498 + "preserves only an explicit correction cue during batch and stream filler removal: $cue / $remove",
499 + ({ cue, remove, text, expected }) =>
500 + Effect.gen(function* () {
501 + const speech = yield* SpeechService.SpeechService;
502 + const options = {
503 + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
504 + speechLanguage: "en",
505 + speechRemoveFillerWords: remove,
506 + speechCustomWords: [{ term: "um", aliases: [] }],
507 + ...(cue === undefined ? {} : { speechCorrectionWord: cue }),
508 + };
509 + native.transcribe.mockResolvedValueOnce({ text });
510 + native.finish.mockResolvedValueOnce(text);
511 + expect(yield* speech.transcribe(pcm(), options)).toBe(expected);
512 + const stream = yield* speech.startStream(options);
513 + expect(yield* stream.finish).toBe(expected);
514 + }).pipe(Effect.scoped, Effect.provide(layer)),
515 +);
516 +
517 it.effect("holds model ownership until the stream scope closes and preserves silence", () =>
518 Effect.gen(function* () {
519 const speech = yield* SpeechService.SpeechService;
@@ -560,9 +720,13 @@ it.effect("replays streamed audio on CPU after an Auto-placed feed crashes, once
720 };
721 expect(yield* stream.feed(pcm())).toEqual({ revision: 5, text: recovered });
722 expect(loadNative.mock.calls.slice(-2).map((call) => call[3])).toEqual(["auto", "cpu"]);
563 - expect(native.begin.mock.calls.slice(-2)).toEqual([
564 - ["en", ["GitHub", "Bob"]],
565 - ["en", cpuSupportsVocabulary ? ["GitHub", "Bob"] : undefined],
723 + expect(native.begin.mock.calls.slice(-2).map(([options]) => options)).toEqual([
724 + { timestamps: "none", language: "en", vocabulary: ["GitHub", "Bob"] },
725 + {
726 + timestamps: "none",
727 + language: "en",
728 + ...(cpuSupportsVocabulary ? { vocabulary: ["GitHub", "Bob"] } : {}),
729 + },
730 ]);
731 expect(native.feed).toHaveBeenCalledTimes(feedCalls + 4);
732 expect(yield* stream.feed(pcm())).toEqual({ revision: 6, text: recovered });
@@ -583,7 +747,7 @@ it.effect.each([false, true])(
747 const cpu = {
748 ...native,
749 backend: "CPU",
586 - begin: vi.fn(async (_language?: string) => {}),
750 + begin: vi.fn(async (_options?: unknown) => {}),
751 feed: vi.fn(async (_pcm: Float32Array) => ({
752 revision: 1,
753 text: { committed: "", tentative: "" },
@@ -611,7 +775,11 @@ it.effect.each([false, true])(
775 expect(cpu.feed.mock.calls.map(([chunk]) => [...chunk])).toEqual(
776 chunks.map((chunk) => [...chunk]),
777 );
614 - expect(cpu.begin).toHaveBeenCalledWith("fr", undefined);
778 + expect(cpu.begin).toHaveBeenCalledWith(
779 + { timestamps: "none", language: "fr" },
780 + true,
781 + "test.gguf",
782 + );
783 expect(gpu.dispose).toHaveBeenCalledOnce();
784 expect(cpu.dispose).not.toHaveBeenCalled();
785 const busy = yield* Effect.result(speech.selectModel("fallback-model"));
apps/server/src/speech/SpeechService.ts
+72 -22
@@ -32,6 +32,7 @@ import {
32 type SpeechModel,
33 } from "./model.ts";
34 import { listNativeSpeechGpuDevices, loadNativeSpeechModel } from "./native.ts";
35 +import { SILERO_VAD } from "./silero.ts";
36 import { isSpeechPlatformSupported } from "./platform.ts";
37 import * as SpeechModelStore from "./SpeechModelStore.ts";
38
@@ -39,14 +40,13 @@ const SAMPLE_RATE = 16_000;
40 const MAX_SPEECH_DURATION_SECONDS = 5 * 60;
41 export const MAX_SPEECH_BYTES =
42 SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * MAX_SPEECH_DURATION_SECONDS;
42 -const MIN_CAPTURE_RMS = 0.0005;
43
44 export class SpeechInvalidAudioError extends Schema.TaggedError<SpeechInvalidAudioError>()(
45 "SpeechInvalidAudioError",
46 { byteLength: Schema.Number, message: Schema.String },
47 ) {}
48
49 -export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false): Float32Array {
49 +export function decodeSpeechPcm(pcmBytes: Uint8Array): Float32Array {
50 if (
51 pcmBytes.byteLength === 0 ||
52 pcmBytes.byteLength > MAX_SPEECH_BYTES ||
@@ -60,7 +60,6 @@ export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false):
60 // WebSocket frames can be pooled Node Buffers whose backing storage is larger
61 // than the frame. Copy the view so only the received bytes are decoded.
62 const pcm = new Float32Array(Uint8Array.from(pcmBytes).buffer);
63 - let energy = 0;
63 for (const sample of pcm) {
64 if (!Number.isFinite(sample) || sample < -1 || sample > 1) {
65 throw new SpeechInvalidAudioError({
@@ -68,11 +67,8 @@ export function decodeSpeechPcm(pcmBytes: Uint8Array, preserveSilence = false):
67 message: "audio contains an invalid PCM sample",
68 });
69 }
71 - energy += sample * sample;
70 }
73 - return !preserveSilence && Math.sqrt(energy / pcm.length) < MIN_CAPTURE_RMS
74 - ? new Float32Array()
75 - : pcm;
71 + return pcm;
72 }
73
74 export class SpeechOperationError extends Schema.TaggedError<SpeechOperationError>()(
@@ -281,6 +277,23 @@ const make = Effect.gen(function* () {
277 const selectedModel = (settings: SettingsSnapshot): SpeechModel =>
278 getSpeechModel(settings.speechModelId) ?? getSpeechModel(DEFAULT_SPEECH_MODEL_ID)!;
279
280 + let silenceRetryAt = 0;
281 + const acquireSilenceModel = Effect.fn("SpeechService.acquireSilenceModel")(function* (
282 + purpose: "installation" | "recording",
283 + ) {
284 + if (purpose === "recording" && now() < silenceRetryAt) return undefined;
285 + const path = yield* store.download(SILERO_VAD).pipe(
286 + Effect.timeout(purpose === "recording" ? "15 seconds" : "2 minutes"),
287 + Effect.orElseSucceed(() => undefined),
288 + );
289 + if (path !== undefined) {
290 + silenceRetryAt = 0;
291 + return path;
292 + }
293 + silenceRetryAt = now() + 10 * 60_000;
294 + yield* Effect.logWarning("Silero unavailable; using raw audio with bounded ASR processing.");
295 + });
296 +
297 // Publish the reservation before forking: a cached transfer can finish before forkIn returns.
298 const downloadFiber = Effect.fn("SpeechService.downloadFiber")(function* (modelId: string) {
299 const definition = getSpeechModel(modelId);
@@ -306,6 +319,9 @@ const make = Effect.gen(function* () {
319 },
320 })
321 .pipe(
322 + (modelDownload) =>
323 + Effect.all([modelDownload, acquireSilenceModel("installation")], { concurrency: 2 }),
324 + Effect.map(([modelPath]) => modelPath),
325 Effect.mapError((cause) => speechError("model download", cause)),
326 Effect.forkIn(scope),
327 );
@@ -442,6 +458,7 @@ const make = Effect.gen(function* () {
458 model: definition.name,
459 size: definition.size,
460 supportsStreaming: definition.supportsStreaming,
461 + supportsBufferedStreaming: true,
462 supportsTranslation: definition.supportsTranslation,
463 languages: definition.languages,
464 supportsLanguageDetection: definition.supportsLanguageDetection,
@@ -512,7 +529,9 @@ const make = Effect.gen(function* () {
529 language: string | undefined,
530 ) => {
531 return (
515 - options.speechRemoveFillerWords ? removeSpeechFillerWords(text, language) : text
532 + options.speechRemoveFillerWords
533 + ? removeSpeechFillerWords(text, language, options.speechCorrectionWord)
534 + : text
535 ).trim();
536 };
537
@@ -543,6 +562,15 @@ const make = Effect.gen(function* () {
562 ),
563 );
564
565 + const silenceModelPath = Effect.fn("SpeechService.silenceModelPath")(function* (
566 + loaded: LoadedModel,
567 + options: SpeechTranscriptionOptions,
568 + ) {
569 + if (!options.speechSilenceFiltering && (loaded.supportsStreaming || loaded.supportsLongForm))
570 + return undefined;
571 + return yield* acquireSilenceModel("recording");
572 + });
573 +
574 const transcribe: SpeechService["Service"]["transcribe"] = Effect.fn("SpeechService.transcribe")(
575 function* (pcmBytes, options) {
576 const settings = yield* readSettings("transcription");
@@ -574,6 +602,7 @@ const make = Effect.gen(function* () {
602 ),
603 catch: (cause) => speechError("model preparation", cause),
604 });
605 + const vadPath = yield* silenceModelPath(loaded, options);
606 const dictionary = normalizeSpeechCustomWords(options.speechCustomWords);
607 const customWords = dictionary.map(({ term }) => term);
608 // The vocabulary capability guarantees plain transcription, not translation.
@@ -584,14 +613,19 @@ const make = Effect.gen(function* () {
613 try: (signal) => {
614 // Interruption kills the isolated process; the next request reloads the model.
615 signal.addEventListener("abort", () => dropModel(inference), { once: true });
587 - return inference.transcribe(pcm, {
588 - timestamps: "none",
589 - ...(language === "auto" ? {} : { language }),
590 - ...(translateToEnglish
591 - ? { task: "translate" as const, targetLanguage: "en" as const }
592 - : {}),
593 - ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}),
594 - });
616 + return inference.transcribe(
617 + pcm,
618 + {
619 + timestamps: "none",
620 + ...(language === "auto" ? {} : { language }),
621 + ...(translateToEnglish
622 + ? { task: "translate" as const, targetLanguage: "en" as const }
623 + : {}),
624 + ...(vocabularyFor(inference).length > 0 ? { vocabulary: customWords } : {}),
625 + },
626 + options.speechSilenceFiltering,
627 + vadPath,
628 + );
629 },
630 catch: (cause) => {
631 dropModel(inference);
@@ -703,12 +737,28 @@ const make = Effect.gen(function* () {
737 loadModel(modelPath, definition, signal, settings.speechAcceleration),
738 );
739 loaded = streamModel;
740 + const vadPath = yield* silenceModelPath(streamModel, options);
741 + let translateToEnglish = false;
742 const beginStream = async () => {
707 - if (!streamModel.supportsStreaming)
708 - throw new Error("The selected model does not support streaming.");
743 + translateToEnglish =
744 + options.speechTranslateToEnglish &&
745 + !streamModel.supportsStreaming &&
746 + streamModel.supportsTranslation &&
747 + language !== "en";
748 const vocabulary =
710 - streamModel.supportsVocabulary && customWords.length > 0 ? customWords : undefined;
711 - await streamModel.begin(language === "auto" ? undefined : language, vocabulary);
749 + streamModel.supportsVocabulary && !translateToEnglish && customWords.length
750 + ? customWords
751 + : undefined;
752 + await streamModel.begin(
753 + {
754 + timestamps: "none",
755 + ...(language === "auto" ? {} : { language }),
756 + ...(translateToEnglish ? { task: "translate", targetLanguage: "en" } : {}),
757 + ...(vocabulary ? { vocabulary } : {}),
758 + },
759 + options.speechSilenceFiltering,
760 + vadPath,
761 + );
762 correct = makeSpeechTextCorrector(vocabulary ? [] : customWords, dictionary);
763 started = true;
764 };
@@ -768,7 +818,7 @@ const make = Effect.gen(function* () {
818 byteLength: bytes.byteLength,
819 message: "Speech stream audio limit exceeded.",
820 });
771 - const pcm = decodeSpeechPcm(bytes, true);
821 + const pcm = decodeSpeechPcm(bytes);
822 byteLength += bytes.byteLength;
823 received.push(pcm);
824 let update: Awaited<ReturnType<LoadedModel["feed"]>> | undefined;
@@ -806,7 +856,7 @@ const make = Effect.gen(function* () {
856 return finalizeTranscript(
857 correct(text),
858 options,
809 - language === "auto" ? undefined : language,
859 + translateToEnglish ? "en" : language === "auto" ? undefined : language,
860 );
861 }),
862 };
apps/server/src/speech/audio.test.ts new
+92
@@ -0,0 +1,92 @@
1 +import { expect, it } from "vite-plus/test";
2 +import { createSpeechAudio } from "./audio.ts";
3 +
4 +const frame = 512;
5 +const make = (overrides = {}) =>
6 + createSpeechAudio({
7 + frameSamples: frame,
8 + filterSilence: true,
9 + nativeStreaming: false,
10 + longForm: false,
11 + maxAudioMs: 0,
12 + ...overrides,
13 + });
14 +const frames = (count: number) =>
15 + Float32Array.from({ length: count * frame }, (_, i) => (Math.floor(i / frame) + 1) / 1000);
16 +const seconds = (n: number) => Math.floor((n * 16000) / frame);
17 +
18 +it("retains pre-roll, onset and offline/streaming tails without duplicating edges", () => {
19 + for (const nativeStreaming of [false, true]) {
20 + const audio = make({ nativeStreaming });
21 + const pcm = frames(100);
22 + const chunks: Float32Array[] = [];
23 + for (let i = 0; i < 100; i++) {
24 + audio.append(pcm.subarray(i * frame, (i + 1) * frame));
25 + audio.score(i, [i >= 20 && i < 24 ? 0.9 : 0]);
26 + chunks.push(...audio.take());
27 + }
28 + chunks.push(...audio.take(true));
29 + const retained = chunks.flatMap((chunk) => [...chunk]);
30 + expect(retained).toEqual([
31 + ...pcm.subarray(6 * frame, (24 + (nativeStreaming ? 52 : 15)) * frame),
32 + ]);
33 + }
34 +});
35 +
36 +it("handles silence, short speech, a partial last frame and disabled filtering", () => {
37 + const silence = make();
38 + silence.append(new Float32Array(frame * 4 + 7));
39 + silence.score(0, [0, 0, 0, 0, 0]);
40 + expect(silence.take(true)).toEqual([]);
41 + const short = make();
42 + short.append(new Float32Array([0.2, 0.1]));
43 + short.score(0, [0.8]);
44 + expect([...short.take(true)[0]!]).toEqual([...new Float32Array([0.2, 0.1])]);
45 + const raw = make({ filterSilence: false });
46 + const pcm = frames(50).subarray(0, 50 * frame - 3);
47 + raw.append(pcm);
48 + raw.score(0, new Array(50).fill(0));
49 + expect(raw.take(true)[0]).toEqual(pcm);
50 +});
51 +
52 +it("plans at pauses while recording and preserves ordered audio and final tails", () => {
53 + const audio = make({ filterSilence: false });
54 + const pcm = frames(seconds(100));
55 + const chunks: Float32Array[] = [];
56 + for (let start = 0; start < pcm.length; start += frame * 7) {
57 + const piece = pcm.subarray(start, start + frame * 7);
58 + audio.append(piece);
59 + audio.score(
60 + start / frame,
61 + Array.from({ length: piece.length / frame }, (_, i) => {
62 + const t = ((start / frame + i) * frame) / 16000;
63 + return (t >= 28 && t < 29) || (t >= 55 && t < 56) ? 0 : 1;
64 + }),
65 + );
66 + chunks.push(...audio.take());
67 + }
68 + expect(chunks.length).toBeGreaterThan(0);
69 + expect(chunks[0]!.length / 16000).toBeCloseTo(28.5, 1);
70 + chunks.push(...audio.take(true));
71 + expect(Float32Array.from(chunks.flatMap((chunk) => [...chunk]))).toEqual(pcm);
72 + expect(chunks.every((chunk) => chunk.length <= 40 * 16000)).toBe(true);
73 + expect(chunks.at(-1)!.length).toBeGreaterThanOrEqual(5 * 16000);
74 +});
75 +
76 +it("respects smaller engine limits, hard cuts, and native/internal-long-form eligibility", () => {
77 + for (const config of [{}, { maxAudioMs: 4000 }, { nativeStreaming: true }, { longForm: true }]) {
78 + const audio = make({ filterSilence: false, ...config });
79 + const pcm = frames(seconds(84));
80 + audio.append(pcm);
81 + audio.score(0, new Array(pcm.length / frame).fill(1));
82 + const live = audio.take();
83 + const chunks = [...live, ...audio.take(true)];
84 + expect(chunks.reduce((n, chunk) => n + chunk.length, 0)).toBe(pcm.length);
85 + if ("nativeStreaming" in config || "longForm" in config) expect(chunks).toHaveLength(1);
86 + else
87 + expect(
88 + chunks.every((chunk) => chunk.length <= ("maxAudioMs" in config ? 32000 : 640000)),
89 + ).toBe(true);
90 + if ("longForm" in config) expect(live).toEqual([]);
91 + }
92 +});
apps/server/src/speech/audio.ts new
+174
@@ -0,0 +1,174 @@
1 +// Self-contained because the native worker embeds this factory in Node, Electron and SEA.
2 +export function createSpeechAudio(config: {
3 + frameSamples: number;
4 + filterSilence: boolean;
5 + nativeStreaming: boolean;
6 + longForm: boolean;
7 + maxAudioMs: number;
8 +}) {
9 + const rate = 16_000;
10 + const frame = config.frameSamples;
11 + const prefill = Math.ceil((450 * rate) / 1000 / frame);
12 + const onset = Math.ceil((60 * rate) / 1000 / frame);
13 + const tail = Math.ceil(((config.nativeStreaming ? 1650 : 450) * rate) / 1000 / frame);
14 + const eligible = !config.nativeStreaming && !config.longForm;
15 + // Preserve the existing output-limit headroom and bounded offline retries.
16 + const maximum = Math.min(
17 + 40 * rate,
18 + config.maxAudioMs > 0 ? Math.max(1, Math.floor((config.maxAudioMs * rate) / 2000)) : 40 * rate,
19 + );
20 + const minimum = Math.min(5 * rate, Math.floor(maximum / 4));
21 + const target = Math.min(30 * rate, maximum * 0.75);
22 + const blocks: { start: number; pcm: Float32Array }[] = [];
23 + const probs: number[] = [];
24 + const keep: boolean[] = [];
25 + let length = 0;
26 + let start = 0;
27 + let streamAt = 0;
28 + let inSpeech = false;
29 + let onsetCount = 0;
30 + let hangover = 0;
31 +
32 + const raw = (from: number, to: number) => {
33 + const output = new Float32Array(to - from);
34 + for (const block of blocks) {
35 + const left = Math.max(from, block.start);
36 + const right = Math.min(to, block.start + block.pcm.length);
37 + if (left < right)
38 + output.set(block.pcm.subarray(left - block.start, right - block.start), left - from);
39 + }
40 + return output;
41 + };
42 + const read = (from: number, to: number) => {
43 + const pcm = raw(from, to);
44 + if (!config.filterSilence) return pcm;
45 + let count = 0;
46 + for (let at = from; at < to;) {
47 + const end = Math.min(to, (Math.floor(at / frame) + 1) * frame);
48 + if (keep[Math.floor(at / frame)]) {
49 + pcm.copyWithin(count, at - from, end - from);
50 + count += end - at;
51 + }
52 + at = end;
53 + }
54 + return pcm.subarray(0, count);
55 + };
56 + // Silero timestamp hysteresis is separate from the capture filter's 0.3 threshold.
57 + const pauses = (samples: number, final: boolean) => {
58 + const segments: { start: number; end: number }[] = [];
59 + let speechStart: number | undefined;
60 + let silenceStart: number | undefined;
61 + for (let i = 0; i < probs.length; i++) {
62 + const at = i * frame;
63 + if (probs[i]! >= 0.5) {
64 + silenceStart = undefined;
65 + speechStart ??= at;
66 + } else if (probs[i]! < 0.35 && speechStart !== undefined) {
67 + silenceStart ??= at;
68 + if (at - silenceStart >= 0.1 * rate) {
69 + if (silenceStart - speechStart > 0.25 * rate)
70 + segments.push({ start: speechStart, end: silenceStart });
71 + speechStart = silenceStart = undefined;
72 + }
73 + }
74 + }
75 + if (speechStart !== undefined && samples - speechStart > 0.25 * rate)
76 + segments.push({ start: speechStart, end: samples });
77 + const candidates: { cut: number; gap: number }[] = [];
78 + let end = 0;
79 + for (const segment of segments) {
80 + if (segment.start > end)
81 + candidates.push({
82 + cut: Math.floor((end + segment.start) / 2),
83 + gap: (segment.start - end) / rate,
84 + });
85 + end = segment.end;
86 + }
87 + // An open pause's midpoint can still move, so only final audio uses it.
88 + if (final && samples > end)
89 + candidates.push({ cut: Math.floor((end + samples) / 2), gap: (samples - end) / rate });
90 + return candidates;
91 + };
92 +
93 + return {
94 + append(pcm: Float32Array) {
95 + blocks.push({ start: length, pcm });
96 + length += pcm.length;
97 + },
98 + score(firstFrame: number, scores: readonly number[]) {
99 + if (firstFrame !== probs.length || scores.some((p) => !Number.isFinite(p) || p < 0 || p > 1))
100 + throw new Error("Invalid Silero frame sequence.");
101 + for (const probability of scores) {
102 + const i = probs.length;
103 + probs.push(probability);
104 + keep.push(false);
105 + const voiced = probability > 0.3;
106 + if (!inSpeech && voiced) {
107 + if (++onsetCount >= onset) {
108 + inSpeech = true;
109 + hangover = tail;
110 + onsetCount = 0;
111 + for (let j = Math.max(0, i - prefill); j <= i; j++) keep[j] = true;
112 + }
113 + } else if (inSpeech && voiced) {
114 + hangover = tail;
115 + keep[i] = true;
116 + } else if (inSpeech && hangover > 0) {
117 + hangover--;
118 + keep[i] = true;
119 + } else {
120 + inSpeech = false;
121 + onsetCount = 0;
122 + }
123 + }
124 + },
125 + audio: () => raw(0, length),
126 + samples: () => length,
127 + take(final = false): Float32Array[] {
128 + const samples = Math.min(length, probs.length * frame);
129 + // Preserve a very short utterance or syllable stopped before onset confirms.
130 + if (final && onsetCount > 0)
131 + for (let i = Math.max(0, probs.length - 1 - prefill); i < probs.length; i++) keep[i] = true;
132 + if (config.nativeStreaming) {
133 + const output = read(streamAt, samples);
134 + let lastKept = streamAt;
135 + for (let i = Math.floor(streamAt / frame); i < probs.length; i++)
136 + if (!config.filterSilence || keep[i]) lastKept = Math.min(samples, (i + 1) * frame);
137 + streamAt = Math.max(lastKept, Math.max(0, samples - prefill * frame));
138 + return output.length ? [output] : [];
139 + }
140 + const chunks: Float32Array[] = [];
141 + const lookahead = Math.max(minimum, (prefill + onset) * frame);
142 + if (eligible && samples - start > maximum + (final ? 0 : lookahead)) {
143 + const candidates = pauses(samples, final);
144 + while (samples - start > maximum + (final ? 0 : lookahead)) {
145 + let best: { cut: number; score: number } | undefined;
146 + let fallback: typeof best;
147 + for (const candidate of candidates) {
148 + const position = candidate.cut - start;
149 + if (position < minimum || position > maximum) continue;
150 + const offTarget = (position - target) / rate;
151 + const score =
152 + Math.min(candidate.gap, 1) - Math.abs(offTarget) * (offTarget > 0 ? 0.1 : 0.05);
153 + if (!fallback || score > fallback.score) fallback = { cut: candidate.cut, score };
154 + if (samples - candidate.cut >= minimum && (!best || score > best.score))
155 + best = { cut: candidate.cut, score };
156 + }
157 + const hard =
158 + start +
159 + (samples - start < maximum + minimum ? Math.floor((samples - start) / 2) : maximum);
160 + const cut = (best ?? fallback)?.cut ?? hard;
161 + const chunk = read(start, cut);
162 + if (chunk.length) chunks.push(chunk);
163 + start = cut;
164 + }
165 + }
166 + if (final && samples > start) {
167 + const chunk = read(start, samples);
168 + if (chunk.length) chunks.push(chunk);
169 + start = samples;
170 + }
171 + return chunks;
172 + },
173 + };
174 +}
apps/server/src/speech/fillerWords.ts
+11 -9
@@ -22,19 +22,21 @@ const LANGUAGE_FILLER_WORDS: Readonly<Record<string, readonly string[]>> = {
22
23 const escapeRegex = (value: string) => value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
24
25 -export function removeSpeechFillerWords(text: string, language?: string): string {
25 +export function removeSpeechFillerWords(
26 + text: string,
27 + language?: string,
28 + correctionCue?: string,
29 +): string {
30 const baseLanguage = language?.split(/[-_]/, 1)[0]?.toLowerCase();
31 const words = [
32 ...UNIVERSAL_FILLER_WORDS,
33 ...(baseLanguage ? (LANGUAGE_FILLER_WORDS[baseLanguage] ?? []) : []),
34 ];
31 - let filtered = text;
32 - for (const word of words) {
33 - const pattern = new RegExp(
34 - `(?<![\\p{L}\\p{N}_])${escapeRegex(word)}(?![\\p{L}\\p{N}_])[,.]?`,
35 - "giu",
36 - );
37 - filtered = filtered.replace(pattern, "");
38 - }
35 + const cuePattern = correctionCue?.trim().split(/\s+/).map(escapeRegex).join("\\s+");
36 + const pattern = new RegExp(
37 + `(?<![\\p{L}\\p{N}_])(?:(${cuePattern || "(?!)"})(?![\\p{L}\\p{N}_])|(?:${words.map(escapeRegex).join("|")})(?![\\p{L}\\p{N}_])[,.]?)`,
38 + "giu",
39 + );
40 + const filtered = text.replace(pattern, (_match, cue: string | undefined) => cue ?? "");
41 return filtered.replace(/\s{2,}/g, " ").trim();
42 }
apps/server/src/speech/native.test.ts
+188 -14
@@ -39,7 +39,11 @@ it("returns transcription from an isolated process and disposes it", async () =>
39 );
40 try {
41 expect(
42 - await model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }),
42 + await model.transcribe(
43 + new Float32Array([0.25]),
44 + { timestamps: "none", language: "en" },
45 + false,
46 + ),
47 ).toEqual({ text: "0.25" });
48 } finally {
49 await model.dispose();
@@ -60,7 +64,7 @@ it("probes generic vocabulary support and delivers ordered terms to offline infe
64 language: "en",
65 ...(supported ? { vocabulary: ["T3 Code", "GitHub"] } : {}),
66 };
63 - const result = await model.transcribe(new Float32Array([0.25]), options);
67 + const result = await model.transcribe(new Float32Array([0.25]), options, false);
68 expect(JSON.parse(result.text)).toEqual(options);
69 } finally {
70 await model.dispose();
@@ -76,7 +80,7 @@ it("ignores IPC messages that do not belong to the speech protocol", async () =>
80 );
81 try {
82 await expect(
79 - model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }),
83 + model.transcribe(new Float32Array([0.25]), { timestamps: "none", language: "en" }, false),
84 ).resolves.toEqual({ text: "hello" });
85 } finally {
86 await model.dispose();
@@ -90,10 +94,14 @@ it("terminates hung native inference when the service shuts down", async () => {
94 controller.signal,
95 fixture("async () => { while (true) {} }"),
96 );
93 - const pending = model.transcribe(new Float32Array([0.25]), {
94 - timestamps: "none",
95 - language: "en",
96 - });
97 + const pending = model.transcribe(
98 + new Float32Array([0.25]),
99 + {
100 + timestamps: "none",
101 + language: "en",
102 + },
103 + false,
104 + );
105 const rejected = expect(pending).rejects.toThrow();
106 controller.abort();
107 await model.dispose();
@@ -111,17 +119,22 @@ it("returns streaming previews and final text from an isolated process", async (
119 );
120 try {
121 expect(model.supportsStreaming).toBe(true);
114 - await model.begin();
115 - expect(await model.feed(new Float32Array([0.25]))).toEqual({
122 + await model.begin({ timestamps: "none" }, false);
123 + expect(await model.feed(new Float32Array(512).fill(0.25))).toEqual({
124 revision: 1,
125 text: { committed: "hello ", tentative: "world" },
126 });
127 expect(await model.finish()).toBe("hello world");
120 - await model.begin("fr");
128 + await model.begin({ timestamps: "none", language: "fr" }, false);
129 await model.reset();
122 - await model.begin("fr");
130 + await model.begin({ timestamps: "none", language: "fr" }, false);
131 + await model.feed(new Float32Array(512));
132 expect(await model.finish()).toBe("fr");
124 - await model.begin("fr", ["T3 Code", "GitHub"]);
133 + await model.begin(
134 + { timestamps: "none", language: "fr", vocabulary: ["T3 Code", "GitHub"] },
135 + false,
136 + );
137 + await model.feed(new Float32Array(512).fill(0.25));
138 expect(await model.finish()).toBe("T3 Code / GitHub");
139 } finally {
140 await model.dispose();
@@ -135,9 +148,170 @@ it("kills a hung streaming feed without waiting for native cleanup", async () =>
148 controller.signal,
149 streamingFixture("async () => { while (true) {} }"),
150 );
138 - await model.begin();
139 - const rejected = expect(model.feed(new Float32Array([0.25]))).rejects.toThrow();
151 + await model.begin({ timestamps: "none" }, false);
152 + const rejected = expect(model.feed(new Float32Array(512).fill(0.25))).rejects.toThrow();
153 controller.abort();
154 await model.dispose();
155 await rejected;
156 });
157 +const vadFixture = ({
158 + streaming = false,
159 + failVad = false,
160 + failVadLoad = false,
161 + failAsr = false,
162 + maxAudioMs = 4000,
163 +} = {}) =>
164 + `data:text/javascript,${encodeURIComponent(`
165 + const summarize = pcm => String(pcm.length) + ":" + pcm[0].toFixed(2) + ":" + pcm[pcm.length - 1].toFixed(2);
166 + export const TranscribeModel = { load: async (path, options) => {
167 + if (path === "vad.gguf") {
168 + if (${failVadLoad}) throw new Error("VAD unavailable");
169 + if (options.backend !== "cpu") throw new Error("VAD must use CPU");
170 + return { vadInfo: { sampleRate: 16000, frameSamples: 512 }, dispose() {},
171 + createVadSession: ({ nThreads }) => {
172 + if (nThreads !== 1) throw new Error("VAD thread count");
173 + let pending = []; let at = 0;
174 + const score = final => {
175 + const firstFrame = at;
176 + const probs = [];
177 + while (pending.length >= 512 || final && pending.length) {
178 + probs.push(pending[0] > 0.5 ? 0.9 : 0);
179 + pending.splice(0, 512); at++;
180 + }
181 + return { firstFrame, probs };
182 + };
183 + return { dispose() {}, streamReset: async () => { pending = []; at = 0; },
184 + streamFeed: async pcm => {
185 + if (${failVad} && pcm[0] < 0) throw new Error("VAD fixture failed");
186 + pending.push(...pcm); return score(false);
187 + },
188 + streamFlush: async () => score(true),
189 + };
190 + },
191 + };
192 + }
193 + return { backend: "CPU", supports: () => false,
194 + capabilities: { supportsStreaming: ${streaming}, maxAudioMs: ${maxAudioMs} },
195 + transcribe: async pcm => { if (${failAsr}) throw new Error("ASR fixture failed"); return { text: summarize(pcm) }; },
196 + createSession: () => {
197 + let all = [];
198 + return { dispose() {}, stream: async () => ({
199 + feed: async pcm => { all.push(...pcm); return { committedChanged: true }; },
200 + finalize: async () => {}, reset() { all = []; },
201 + get text() { const full = summarize(all); return { committed: full, tentative: "", full }; },
202 + }) };
203 + },
204 + };
205 + }};
206 + `)}`;
207 +
208 +it("filters silence and preserves partial speech through the production worker and resets between recordings", async () => {
209 + for (const streaming of [false, true]) {
210 + const model = await loadNativeSpeechModel(
211 + "asr.gguf",
212 + new AbortController().signal,
213 + vadFixture({ streaming }),
214 + "cpu",
215 + );
216 + try {
217 + expect(
218 + await model.transcribe(new Float32Array(2000), { timestamps: "none" }, true, "vad.gguf"),
219 + ).toEqual({ text: "" });
220 + expect(
221 + await model.transcribe(
222 + new Float32Array([0.8, 0.6]),
223 + { timestamps: "none" },
224 + true,
225 + "vad.gguf",
226 + ),
227 + ).toEqual({ text: "2:0.80:0.60" });
228 + expect(
229 + await model.transcribe(new Float32Array(2000), { timestamps: "none" }, false, "vad.gguf"),
230 + ).toEqual({ text: "2000:0.00:0.00" });
231 + for (const cancel of [true, false]) {
232 + await model.begin({ timestamps: "none" }, true, "vad.gguf");
233 + await model.feed(new Float32Array(600));
234 + expect(await model.finish()).toBe("");
235 + await model.begin({ timestamps: "none" }, true, "vad.gguf");
236 + await model.feed(new Float32Array(511).fill(0.8));
237 + if (cancel) await model.reset();
238 + else expect(await model.finish()).toBe("511:0.80:0.80");
239 + }
240 + } finally {
241 + await model.dispose();
242 + }
243 + }
244 +});
245 +
246 +it("decodes complete stable chunks before Stop and returns the ordered final tail", async () => {
247 + const model = await loadNativeSpeechModel(
248 + "asr.gguf",
249 + new AbortController().signal,
250 + vadFixture(),
251 + "cpu",
252 + );
253 + try {
254 + await model.begin({ timestamps: "none" }, false, "vad.gguf");
255 + await model.feed(new Float32Array(16000).fill(0.1));
256 + await model.feed(new Float32Array(16000).fill(0.2));
257 + const update = await model.feed(new Float32Array(16000).fill(0.3));
258 + expect(update.text).toEqual({ committed: "32000:0.10:0.20", tentative: "" });
259 + await model.feed(new Float32Array([0.4, 0.5]));
260 + expect(await model.finish()).toBe("32000:0.10:0.20 16002:0.30:0.50");
261 + } finally {
262 + await model.dispose();
263 + }
264 +});
265 +
266 +it("preserves withheld audio on VAD failure while keeping ASR failures distinct", async () => {
267 + for (const streaming of [false, true]) {
268 + const model = await loadNativeSpeechModel(
269 + "asr.gguf",
270 + new AbortController().signal,
271 + vadFixture({ streaming, failVad: true }),
272 + "cpu",
273 + );
274 + try {
275 + await model.begin({ timestamps: "none" }, true, "vad.gguf");
276 + await model.feed(new Float32Array(512).fill(0.1));
277 + await model.feed(new Float32Array(512).fill(-0.2));
278 + expect(await model.finish()).toBe("1024:0.10:-0.20");
279 + await model.begin({ timestamps: "none" }, true, "vad.gguf");
280 + await model.feed(new Float32Array(512));
281 + expect(await model.finish()).toBe("");
282 + } finally {
283 + await model.dispose();
284 + }
285 + }
286 + const unavailable = await loadNativeSpeechModel(
287 + "asr.gguf",
288 + new AbortController().signal,
289 + vadFixture({ failVadLoad: true }),
290 + "cpu",
291 + );
292 + try {
293 + expect(
294 + await unavailable.transcribe(
295 + new Float32Array(2000).fill(0.1),
296 + { timestamps: "none" },
297 + true,
298 + "vad.gguf",
299 + ),
300 + ).toEqual({ text: "2000:0.10:0.10" });
301 + } finally {
302 + await unavailable.dispose();
303 + }
304 + const model = await loadNativeSpeechModel(
305 + "asr.gguf",
306 + new AbortController().signal,
307 + vadFixture({ failAsr: true }),
308 + "cpu",
309 + );
310 + try {
311 + await expect(
312 + model.transcribe(new Float32Array(512).fill(0.8), { timestamps: "none" }, true, "vad.gguf"),
313 + ).rejects.toThrow("ASR fixture failed");
314 + } finally {
315 + await model.dispose();
316 + }
317 +});
apps/server/src/speech/native.ts
+163 -71
@@ -2,16 +2,142 @@
2 import * as NodeChildProcess from "node:child_process";
3 import * as NodeSea from "node:sea";
4 import type { SpeechStreamText } from "@t3tools/contracts";
5 +import { createSpeechAudio } from "./audio.ts";
6
7 // Inline the small child entry so the packaged CLI does not need a separate worker artifact.
8 const entry = `
9 +const createSpeechAudio = ${createSpeechAudio.toString()};
10 +let speech;
11 let model;
12 +let vadModel;
13 +let vad;
14 let session;
15 let stream;
16 +let audio;
17 +let options;
18 +let nativeStreaming;
19 +let useVad;
20 +let scored;
21 +let revision;
22 +let texts;
23 +let heardAudio;
24 +const warnVad = () => console.warn("Silero silence filtering unavailable; preserving raw recording and using bounded ASR chunks.");
25 +const resetAsr = () => {
26 + if (stream) stream.reset();
27 + if (session) session.dispose();
28 + stream = session = undefined;
29 +};
30 +const newAudio = (filterSilence) => createSpeechAudio({
31 + frameSamples: 512, filterSilence, nativeStreaming: model.capabilities.supportsStreaming,
32 + longForm: model.supports("long_form"), maxAudioMs: model.capabilities.maxAudioMs,
33 +});
34 +const openAsr = async () => {
35 + if (nativeStreaming) {
36 + session = model.createSession();
37 + stream = await session.stream(options);
38 + }
39 +};
40 +const begin = async (message, live) => {
41 + resetAsr();
42 + options = message.options;
43 + nativeStreaming = live && model.capabilities.supportsStreaming;
44 + texts = [];
45 + scored = revision = 0;
46 + heardAudio = false;
47 + useVad = Boolean(message.vadPath) && (message.filterSilence ||
48 + (!model.capabilities.supportsStreaming && !model.supports("long_form")));
49 + if (useVad) {
50 + try {
51 + if (!vadModel) {
52 + vadModel = await speech.TranscribeModel.load(message.vadPath, { backend: "cpu" });
53 + if (vadModel.vadInfo.sampleRate !== 16000 || vadModel.vadInfo.frameSamples !== 512)
54 + throw new Error("Unsupported Silero model frames.");
55 + vad = vadModel.createVadSession({ nThreads: 1 });
56 + }
57 + await vad.streamReset();
58 + } catch {
59 + vad?.dispose();
60 + vadModel?.dispose();
61 + vad = vadModel = undefined;
62 + useVad = false;
63 + warnVad();
64 + }
65 + }
66 + audio = newAudio(useVad && message.filterSilence);
67 + await openAsr();
68 +};
69 +const transcribe = async (pcm) => {
70 + try {
71 + const result = await model.transcribe(pcm, options);
72 + texts.push(result.text);
73 + } catch (error) {
74 + if ((error?.constructor?.name !== "OutputTruncated" &&
75 + error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error;
76 + const middle = Math.floor(pcm.length / 2);
77 + await transcribe(pcm.subarray(0, middle));
78 + await transcribe(pcm.subarray(middle));
79 + }
80 +};
81 +const decode = async (final) => {
82 + let preview = null;
83 + for (const pcm of audio.take(final)) {
84 + if (!pcm.length) continue;
85 + heardAudio = true;
86 + if (nativeStreaming) {
87 + const update = await stream.feed(pcm);
88 + if (update.committedChanged || update.tentativeChanged) {
89 + const text = stream.text;
90 + preview = { committed: text.committed, tentative: text.tentative };
91 + }
92 + } else {
93 + await transcribe(pcm);
94 + preview = { committed: texts.join(" ").trim(), tentative: "" };
95 + }
96 + }
97 + return preview;
98 +};
99 +const feed = async (pcm, final = false) => {
100 + if (pcm) audio.append(pcm);
101 + try {
102 + if (useVad) {
103 + if (pcm) {
104 + const result = await vad.streamFeed(pcm);
105 + audio.score(result.firstFrame, result.probs);
106 + if (result.firstFrame + result.probs.length !== Math.floor(audio.samples() / 512))
107 + throw new Error("Incomplete Silero frames.");
108 + }
109 + if (final) {
110 + const result = await vad.streamFlush();
111 + audio.score(result.firstFrame, result.probs);
112 + if (result.firstFrame + result.probs.length !== Math.ceil(audio.samples() / 512))
113 + throw new Error("Incomplete Silero tail.");
114 + }
115 + } else {
116 + const count = final ? Math.ceil(audio.samples() / 512) : Math.floor(audio.samples() / 512);
117 + audio.score(scored, new Array(count - scored).fill(1));
118 + scored = count;
119 + }
120 + } catch {
121 + // Replay on optional VAD failure, including audio it had withheld; ASR errors stay separate.
122 + const raw = audio.audio();
123 + useVad = false;
124 + warnVad();
125 + resetAsr();
126 + texts = [];
127 + heardAudio = false;
128 + audio = newAudio(false);
129 + audio.append(raw);
130 + scored = final ? Math.ceil(raw.length / 512) : Math.floor(raw.length / 512);
131 + audio.score(0, new Array(scored).fill(1));
132 + await openAsr();
133 + }
134 + return await decode(final);
135 +};
136 process.on("message", async (message) => {
137 try {
138 if (message.kind === "load") {
14 - const { TranscribeModel, getAvailableBackends } = await loadSpeechModule(message.moduleUrl);
139 + speech = await loadSpeechModule(message.moduleUrl);
140 + const { TranscribeModel, getAvailableBackends } = speech;
141 const device = message.acceleration.startsWith("gpu:")
142 ? getAvailableBackends().find((item) =>
143 (item.deviceType === "gpu" || item.deviceType === "igpu") &&
@@ -19,80 +145,40 @@ process.on("message", async (message) => {
145 : undefined;
146 if (message.acceleration.startsWith("gpu:") && !device)
147 throw new Error("The selected speech GPU is unavailable.");
22 - model = await TranscribeModel.load(message.path, device
23 - ? { device }
24 - : { backend: message.acceleration });
148 + model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration });
149 process.send({ type: "t3-speech-reply", ok: true, backend: model.backend,
150 supportsStreaming: model.capabilities.supportsStreaming,
151 + supportsLongForm: model.supports("long_form"),
152 supportsTranslation: model.capabilities.supportsTranslate,
153 supportsVocabulary: model.supports("vocabulary") });
154 } else if (message.kind === "begin") {
30 - session = model.createSession();
31 - stream = await session.stream({ timestamps: "none", language: message.language,
32 - vocabulary: message.vocabulary });
155 + await begin(message, true);
156 process.send({ type: "t3-speech-reply", ok: true });
157 } else if (message.kind === "feed") {
35 - const update = await stream.feed(message.pcm);
36 - const text = update.committedChanged || update.tentativeChanged ? stream.text : null;
37 - process.send({ type: "t3-speech-reply", ok: true, revision: update.revision,
38 - preview: text ? { committed: text.committed, tentative: text.tentative } : null });
158 + const preview = await feed(message.pcm);
159 + process.send({ type: "t3-speech-reply", ok: true, revision: ++revision, preview });
160 } else if (message.kind === "finish") {
40 - await stream.finalize();
41 - const text = stream.text.full;
42 - stream.reset();
43 - session.dispose();
44 - stream = session = undefined;
161 + await feed(undefined, true);
162 + if (nativeStreaming && heardAudio) await stream.finalize();
163 + const text = nativeStreaming ? (heardAudio ? stream.text.full : "") : texts.join(" ").trim();
164 + resetAsr();
165 + audio = undefined;
166 process.send({ type: "t3-speech-reply", ok: true, text });
167 } else if (message.kind === "reset") {
47 - stream.reset();
48 - session.dispose();
49 - stream = session = undefined;
168 + resetAsr();
169 + if (vad) await vad.streamReset();
170 + audio = undefined;
171 process.send({ type: "t3-speech-reply", ok: true });
172 } else if (message.kind === "transcribe") {
52 - // Leave unbounded models alone. Use headroom for output-bound models:
53 - // maxAudioMs is an estimate, so dense speech may still need smaller pieces.
54 - const limit = model.capabilities.maxAudioMs;
55 - const chunkSamples = limit > 0 ? Math.max(4000, Math.floor(limit * 16 / 2)) : message.pcm.length;
56 - const texts = [];
57 - const transcribe = async (pcm) => {
58 - try {
59 - const result = await model.transcribe(pcm, message.options);
60 - texts.push(result.text);
61 - } catch (error) {
62 - if ((error?.constructor?.name !== "OutputTruncated" &&
63 - error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error;
64 - const middle = Math.floor(pcm.length / 2);
65 - await transcribe(pcm.subarray(0, middle));
66 - await transcribe(pcm.subarray(middle));
67 - }
68 - };
69 - let offset = 0;
70 - while (offset < message.pcm.length) {
71 - let end = Math.min(offset + chunkSamples, message.pcm.length);
72 - if (end < message.pcm.length) {
73 - // Prefer the quietest 20 ms frame in the final quarter of the window.
74 - let quietest = Infinity;
75 - const windowEnd = end;
76 - for (let start = offset + Math.floor(chunkSamples * 0.75); start + 320 <= windowEnd; start += 320) {
77 - let energy = 0;
78 - for (let i = start; i < start + 320; i++) energy += message.pcm[i] ** 2;
79 - if (energy < quietest) {
80 - quietest = energy;
81 - end = start + 160;
82 - }
83 - }
84 - }
85 - await transcribe(message.pcm.subarray(offset, end));
86 - offset = end;
87 - }
88 - process.send({ type: "t3-speech-reply", ok: true, text: texts.join(" ").trim() });
173 + await begin(message, false);
174 + await feed(message.pcm, true);
175 + const text = texts.join(" ").trim();
176 + audio = undefined;
177 + process.send({ type: "t3-speech-reply", ok: true, text });
178 }
179 } catch (error) {
91 - process.send({
92 - type: "t3-speech-reply",
93 - ok: false,
94 - error: error instanceof Error ? error.stack ?? error.message : String(error),
95 - });
180 + process.send({ type: "t3-speech-reply", ok: false,
181 + error: error instanceof Error ? error.stack ?? error.message : String(error) });
182 }
183 });
184 process.on("disconnect", () => process.exit(0));
@@ -104,6 +190,7 @@ type Reply = {
190 readonly text?: string;
191 readonly error?: string;
192 readonly supportsStreaming?: boolean;
193 + readonly supportsLongForm?: boolean;
194 readonly supportsTranslation?: boolean;
195 readonly supportsVocabulary?: boolean;
196 readonly backend?: string;
@@ -305,10 +392,11 @@ async function loadNativeSpeechModelOnce(
392 return {
393 backend: loaded.backend ?? "unknown",
394 supportsStreaming: loaded.supportsStreaming === true,
395 + supportsLongForm: loaded.supportsLongForm === true,
396 supportsTranslation: loaded.supportsTranslation === true,
397 supportsVocabulary: loaded.supportsVocabulary === true,
310 - begin: async (language?: string, vocabulary?: readonly string[]) => {
311 - await send({ kind: "begin", language, vocabulary });
398 + begin: async (options: NativeSpeechOptions, filterSilence: boolean, vadPath?: string) => {
399 + await send({ kind: "begin", options, filterSilence, vadPath });
400 },
401 feed: async (pcm: Float32Array) => {
402 const reply = await send({ kind: "feed", pcm });
@@ -343,18 +431,22 @@ async function loadNativeSpeechModelOnce(
431 },
432 transcribe: async (
433 pcm: Float32Array,
346 - options: {
347 - readonly timestamps: "none";
348 - readonly task?: "translate";
349 - readonly targetLanguage?: "en";
350 - readonly language?: string;
351 - readonly vocabulary?: readonly string[];
352 - },
434 + options: NativeSpeechOptions,
435 + filterSilence: boolean,
436 + vadPath?: string,
437 ) => {
354 - const result = await send({ kind: "transcribe", pcm, options });
438 + const result = await send({ kind: "transcribe", pcm, options, filterSilence, vadPath });
439 if (typeof result.text !== "string") throw new Error("Invalid speech process response.");
440 return { text: result.text };
441 },
442 dispose,
443 };
444 }
445 +
446 +type NativeSpeechOptions = {
447 + readonly timestamps: "none";
448 + readonly task?: "translate";
449 + readonly targetLanguage?: "en";
450 + readonly language?: string;
451 + readonly vocabulary?: readonly string[];
452 +};
apps/server/src/speech/silero.ts new
+8
@@ -0,0 +1,8 @@
1 +// MIT-licensed internal helper; acquired and verified by the existing model store.
2 +export const SILERO_VAD = {
3 + id: "handy-computer/silero-vad-v6.2-gguf",
4 + revision: "97bbd75179f535e4708bccdf61e9278b8b60dfc7",
5 + filename: "silero-vad-v6.2-F32.gguf",
6 + size: 1_241_056,
7 + sha256: "0d871a800c420fdd5d1b17a3aea4296d1c39608f0ff4a9295eb76457eb1c88e6",
8 +};
apps/web/src/components/ChatView.tsx
+2
@@ -3245,6 +3245,7 @@ export default function ChatView(props: ChatViewProps) {
3245 : undefined,
3246 }}
3247 files={{
3248 + projectId: activeProject?.id,
3249 projectName: activeProject?.title ?? null,
3250 workspaceRoot: activeWorkspaceRoot,
3251 availableEditors,
@@ -3694,6 +3695,7 @@ export default function ChatView(props: ChatViewProps) {
3695 }
3696 activeThreadModelSelection={activeThread?.modelSelection}
3697 activeProjectId={activeProject?.id ?? null}
3698 + projectName={activeProject?.title}
3699 contextWindowEntries={serverVisibleTurnItems}
3700 contextWindowLiveUsage={activeThreadLiveTokenUsage}
3701 contextWindowProviderThread={activeProviderThread}
apps/web/src/components/DiffPanel.browser.test.tsx
+2 -1
@@ -84,6 +84,7 @@ vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => async () => {
84 vi.mock("~/editorPreferences", () => ({ useOpenInPreferredEditor: () => async () => {} }));
85 vi.mock("~/fileContextMenu", () => ({ useFileContextMenuHandler: () => () => {} }));
86 vi.mock("~/browser/useOpenLink", () => ({ useOpenLink: () => async () => {} }));
87 +vi.mock("./diffs/DiffCommentAnnotation", () => ({ DiffCommentAnnotation: () => null }));
88 vi.mock("./diffs/DiffFileTree", () => ({ DiffFileTree: () => null }));
89 vi.mock("./DiffFilePathCopyButton", () => ({ DiffFilePathCopyButton: () => null }));
90
@@ -250,7 +251,7 @@ it.each([false, true])(
251 const toggle = screen.getByRole("button", { name: `Expand ${path}`, exact: true });
252 await expect.element(toggle).toHaveAttribute("aria-expanded", "false");
253 await userEvent.click(toggle);
253 - act(() =>
254 + await act(() =>
255 context.registry.set(
256 context.patch,
257 AsyncResult.success({
apps/web/src/components/DiffPanel.tsx
+2
@@ -1425,6 +1425,8 @@ export default function DiffPanel({
1425 >
1426 <AnnotatableCodeView
1427 key={collapseScopeKey ?? reviewSectionId}
1428 + environmentId={activeThread?.environmentId}
1429 + projectId={activeThread?.projectId}
1430 viewerRef={codeViewReveal.attach}
1431 codeViewKey={codeViewMountKey}
1432 className="h-full min-h-0 overflow-auto"
apps/web/src/components/chat/AssistantCitationChip.tsx
+5 -1
@@ -55,6 +55,7 @@ export function AssistantCitationChip({
55 const commentInputRef = useRef<HTMLTextAreaElement>(null);
56 const commentPopupRef = useRef<HTMLDivElement>(null);
57 const draftCommentRef = useRef<string | null>(null);
58 + const [voiceSetupOpen, setVoiceSetupOpen] = useState(false);
59 const [unavailableSourceAnchor, setUnavailableSourceAnchor] =
60 useState<AssistantCitationSourceAnchor | null>(null);
61 const commentOpen = commentEditor?.open ?? false;
@@ -167,7 +168,7 @@ export function AssistantCitationChip({
168 {composerSourceLink}
169 {commentEditor ? (
170 <Popover
170 - open={commentEditor.open}
171 + open={commentEditor.open && !voiceSetupOpen}
172 onOpenChange={(open, eventDetails) => {
173 if (!open && !settleDraftOnClose(eventDetails.reason)) {
174 eventDetails.cancel();
@@ -185,6 +186,8 @@ export function AssistantCitationChip({
186 </PopoverTrigger>
187 {commentEditor.open ? (
188 <PopoverPopup
189 + // The editor stays mounted while its voice setup dialog replaces the popover.
190 + keepMounted
191 {...composerFloatingLayerProps}
192 side={activeSourceAnchor ? "bottom" : "top"}
193 align="end"
@@ -217,6 +220,7 @@ export function AssistantCitationChip({
220 key={serializeAssistantCitation(citation)}
221 citation={citation}
222 inputRef={commentInputRef}
223 + onVoiceSetupOpenChange={setVoiceSetupOpen}
224 onDraftChange={(comment) => {
225 draftCommentRef.current = comment;
226 commentEditor.onDraftChange?.(comment);
apps/web/src/components/chat/AssistantCitationCommentEditor.tsx
+49 -17
@@ -1,7 +1,10 @@
1 import { ASSISTANT_CITATION_MAX_COMMENT_LENGTH, type AssistantCitation } from "@t3tools/contracts";
2 -import { useState, type Ref } from "react";
2 +import { use, useEffect, useRef, useState, type RefObject } from "react";
3 +
4 +import { TextFieldSpeechControls, useTextFieldSpeech } from "~/speech/useTextFieldSpeech";
5 import { textDirection } from "~/textDirection";
6
7 +import { ComposerContextActionsContext } from "../composerContextActions";
8 import { Button } from "../ui/button";
9
10 export function AssistantCitationCommentEditor({
@@ -11,26 +14,55 @@ export function AssistantCitationCommentEditor({
14 onSubmitAndSend,
15 onCancel,
16 onDraftChange,
17 + onVoiceSetupOpenChange,
18 }: {
19 citation: AssistantCitation;
16 - inputRef?: Ref<HTMLTextAreaElement>;
20 + inputRef?: RefObject<HTMLTextAreaElement | null>;
21 onSubmit: (comment: string) => boolean;
22 onSubmitAndSend?: (comment: string) => boolean;
23 onCancel: () => void;
24 onDraftChange?: (comment: string) => void;
25 + onVoiceSetupOpenChange?: (open: boolean) => void;
26 }) {
27 const [comment, setComment] = useState(citation.comment ?? "");
28 + const localInputRef = useRef<HTMLTextAreaElement | null>(null);
29 + const textareaRef = inputRef ?? localInputRef;
30 + const { environmentId, projectId } = use(ComposerContextActionsContext);
31 + const updateComment = (text: string) => {
32 + setComment(text);
33 + onDraftChange?.(text);
34 + };
35 + const speech = useTextFieldSpeech({
36 + environmentId: environmentId ?? citation.environmentId,
37 + projectId,
38 + ownerKey: JSON.stringify([
39 + "citation",
40 + environmentId,
41 + citation.threadId,
42 + citation.messageId,
43 + citation.start,
44 + citation.end,
45 + ]),
46 + text: comment,
47 + textareaRef,
48 + onTextChange: updateComment,
49 + });
50 + const setupOpen = speech.setup.open;
51 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- keeps the citation popover open while the voice setup dialog owns focus
52 + useEffect(() => {
53 + onVoiceSetupOpenChange?.(setupOpen);
54 + return () => onVoiceSetupOpenChange?.(false);
55 + }, [onVoiceSetupOpenChange, setupOpen]);
56 const commentTooLong = comment.length > ASSISTANT_CITATION_MAX_COMMENT_LENGTH;
57 const submit = () => {
25 - if (!commentTooLong) onSubmit(comment);
58 + speech.submitAfterDictation((text) => {
59 + if (text.length <= ASSISTANT_CITATION_MAX_COMMENT_LENGTH) onSubmit(text);
60 + });
61 };
62 const submitAndSend = () => {
28 - if (commentTooLong) return;
29 - if (onSubmitAndSend) {
30 - onSubmitAndSend(comment);
31 - } else {
32 - onSubmit(comment);
33 - }
63 + speech.submitAfterDictation((text) => {
64 + if (text.length <= ASSISTANT_CITATION_MAX_COMMENT_LENGTH) (onSubmitAndSend ?? onSubmit)(text);
65 + });
66 };
67
68 return (
@@ -41,12 +73,14 @@ export function AssistantCitationCommentEditor({
73 if (event.nativeEvent.isComposing || event.keyCode === 229) return;
74 if (event.key === "Escape") {
75 event.preventDefault();
44 - onCancel();
76 + if (speech.state.phase !== "idle") speech.cancel();
77 + else onCancel();
78 }
79 }}
80 >
81 <textarea
49 - ref={inputRef}
82 + ref={textareaRef}
83 + readOnly={speech.freezesEditor}
84 aria-label="Comment on selected text"
85 aria-description="Enter to save the citation comment; Command/Ctrl+Enter to save and send; Shift+Enter for a new line."
86 aria-invalid={commentTooLong || undefined}
@@ -55,10 +89,7 @@ export function AssistantCitationCommentEditor({
89 dir={textDirection(comment)}
90 className="field-sizing-content block max-h-40 min-h-16 w-full resize-none bg-transparent px-1 py-1.5 text-base outline-none placeholder:text-muted-foreground sm:text-sm"
91 value={comment}
58 - onChange={(event) => {
59 - setComment(event.currentTarget.value);
60 - onDraftChange?.(event.currentTarget.value);
61 - }}
92 + onChange={(event) => updateComment(event.currentTarget.value)}
93 onKeyDown={(event) => {
94 if (
95 event.key === "Enter" &&
@@ -81,7 +112,8 @@ export function AssistantCitationCommentEditor({
112 characters.
113 </p>
114 ) : null}
84 - <div className="mt-2 flex items-center justify-end gap-2">
115 + <div className="mt-2 flex flex-wrap items-center justify-end gap-2">
116 + <TextFieldSpeechControls speech={speech} />
117 <Button
118 variant="outline"
119 size="xs"
@@ -92,7 +124,7 @@ export function AssistantCitationCommentEditor({
124 </Button>
125 <Button
126 size="xs"
95 - disabled={commentTooLong}
127 + disabled={speech.preparing || (commentTooLong && !speech.blocksSubmission)}
128 onPointerDown={(event) => event.preventDefault()}
129 onClick={submit}
130 >
apps/web/src/components/chat/ChatComposer.tsx
+157 -8
@@ -1,3 +1,4 @@
1 +import { VoiceInputSetup } from "./VoiceInputSetup";
2 import { useComposerTypingGuard } from "./useComposerTypingGuard";
3 import { insertAttachmentReferencesForDraft } from "./composerAttachmentReferences";
4 import { DESKTOP_PASTE_AS_TEXT_EVENT } from "../../lib/desktopPasteAsText";
@@ -71,6 +72,7 @@ import {
72 type ComposerSubmissionIntent,
73 collapseExpandedComposerCursor,
74 composerRequiresModifier,
75 + composerSubmissionIntentForKey,
76 composerStateAtPromptEnd,
77 detectComposerTrigger,
78 expandCollapsedComposerCursor,
@@ -135,6 +137,10 @@ import {
137 useAttachmentUploadStore,
138 } from "../../lib/attachmentUploadQueue";
139 import { attachmentUploadBlockReason } from "../../lib/attachmentUploadState";
140 +import { useDictationShortcut } from "../../speech/useDictationShortcut";
141 +import { useDictationSubmissionShortcut } from "../../speech/useDictationSubmissionShortcut";
142 +import { useEnvironmentSpeechInput } from "../../speech/useEnvironmentSpeechInput";
143 +import { ComposerSpeechButton, ComposerSpeechRecordingPill } from "./ComposerSpeechButton";
144 import type { AssistantCitationSourceAnchor } from "~/lib/assistantTextSelection";
145 import { shortcutLabelForCommand } from "../../keybindings";
146 import {
@@ -427,6 +433,7 @@ export interface ChatComposerProps {
433 activeThreadModelSelection: ModelSelection | null | undefined;
434 reportedModelSelection?: ModelSelection | null;
435 activeProjectId: ProjectId | null;
436 + projectName?: string | undefined;
437
438 contextWindowEntries: Parameters<typeof deriveLatestContextWindowSnapshot>[0];
439 contextWindowLiveUsage: Parameters<typeof deriveLatestContextWindowSnapshot>[1];
@@ -554,6 +561,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
561 activeThreadModelSelection,
562 reportedModelSelection,
563 activeProjectId,
564 + projectName,
565 contextWindowEntries,
566 contextWindowLiveUsage,
567 contextWindowProviderThread,
@@ -728,6 +736,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
736 const composerContextActions = {
737 environmentId,
738 cwd: gitCwd ?? undefined,
739 + projectId: activeProjectId ?? undefined,
740 expandImage: (imageId: string) => {
741 const preview = buildExpandedImagePreview(composerImages, imageId);
742 if (preview) onExpandImage(preview);
@@ -1833,6 +1842,54 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
1842 };
1843 const readComposerSnapshotStable = useStableCallback(readComposerSnapshot);
1844
1845 + const dictationDisabled =
1846 + !canOperateThread ||
1847 + environmentUnavailable !== null ||
1848 + projectSelectionRequired ||
1849 + isComposerApprovalState ||
1850 + activePendingIsResponding ||
1851 + activePendingProgress?.activeQuestion?.allowCustomAnswer === false;
1852 + const speechInput = useEnvironmentSpeechInput({
1853 + environmentId,
1854 + projectId: activeProjectId,
1855 + projectName,
1856 + ownerKey: questionAttachmentTarget ?? composerDraftTargetKey,
1857 + draftText: prompt,
1858 + disabled: dictationDisabled,
1859 + readDraft: () => {
1860 + const snapshot = readComposerSnapshot();
1861 + return {
1862 + text: snapshot.value,
1863 + selection: composerEditorRef.current?.readSelectionRange() ?? {
1864 + start: snapshot.expandedCursor,
1865 + end: snapshot.expandedCursor,
1866 + },
1867 + };
1868 + },
1869 + commitDraft: (text, selection) => {
1870 + const snapshot = readComposerSnapshot();
1871 + const applied = applyPromptReplacement(0, snapshot.value.length, text, {
1872 + expectedText: snapshot.value,
1873 + focusEditorAfterReplace: false,
1874 + });
1875 + if (!applied) return;
1876 + const cursor = collapseExpandedComposerCursor(text, selection.start, isLiteralPendingAnswer);
1877 + setComposerCursor(cursor);
1878 + window.requestAnimationFrame(() => composerEditorRef.current?.focusAt(cursor));
1879 + },
1880 + });
1881 + const { blocksSubmission: voiceInputActive, submitAfterDictation } = speechInput;
1882 + const voiceSendDisabledReason =
1883 + sendDisabledReason ?? (speechInput.preparing ? "Preparing voice input" : null);
1884 + const dictationShortcutLabel = useDictationShortcut({
1885 + keybindings,
1886 + speech: speechInput,
1887 + disabled: dictationDisabled,
1888 + terminalOpen,
1889 + modelPickerOpen: isComposerModelPickerOpen,
1890 + ...(backgroundThread ? { targetRef: composerFormRef } : {}),
1891 + });
1892 +
1893 const resolveActiveComposerTrigger = (): {
1894 snapshot: { value: string; cursor: number; expandedCursor: number };
1895 trigger: ComposerTrigger | null;
@@ -2076,11 +2133,19 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
2133 return showPlanFollowUpPrompt || composerSendState.hasSendableContent;
2134 };
2135
2136 + const submitComposerRef = useRef<
2137 + (dispatchMode?: ComposerDispatchMode, submissionIntent?: ComposerSubmissionIntent) => void
2138 + >(() => {});
2139 const submitComposer = (
2140 event?: { preventDefault: () => void },
2141 dispatchMode?: ComposerDispatchMode,
2142 submissionIntent?: ComposerSubmissionIntent,
2143 ) => {
2144 + if (voiceInputActive) {
2145 + event?.preventDefault();
2146 + submitAfterDictation(() => submitComposerRef.current(dispatchMode, submissionIntent));
2147 + return;
2148 + }
2149 if (
2150 noProviderAvailable ||
2151 isSendDisabled ||
@@ -2138,6 +2203,38 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
2203 }
2204 };
2205 const submitComposerStable = useStableCallback(submitComposer);
2206 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes the committed submission handler to the asynchronous dictation completion
2207 + useLayoutEffect(() => {
2208 + submitComposerRef.current = (mode, intent) => submitComposerStable(undefined, mode, intent);
2209 + }, [submitComposerStable]);
2210 + useDictationSubmissionShortcut({
2211 + enabled: voiceInputActive && !speechInput.preparing,
2212 + formRef: composerFormRef,
2213 + onKeyDown: (event) => {
2214 + const intent = composerSubmissionIntentForKey({
2215 + event,
2216 + keybindings,
2217 + isSoftwareKeyboardOpen,
2218 + isDraftThread: routeKind === "draft",
2219 + isRunning: phase === "running",
2220 + sendShortcut: settings.sendShortcut,
2221 + prompt: promptRef.current,
2222 + });
2223 + if (!intent) return;
2224 + event.preventDefault();
2225 + event.stopImmediatePropagation();
2226 + submitComposer(
2227 + undefined,
2228 + resolveComposerDispatchMode({
2229 + running: phase === "running",
2230 + alternateModifier: intent === "alternate",
2231 + canSteer: !isCompacting,
2232 + activeTurnDefault: followUpBehavior,
2233 + }),
2234 + intent,
2235 + );
2236 + },
2237 + });
2238 const handleSubmitMessage = (event: React.MouseEvent<HTMLButtonElement>) => {
2239 event.preventDefault();
2240 submitComposer(
@@ -3444,7 +3541,8 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
3541 disabled={
3542 projectSelectionRequired ||
3543 isChoiceOnlyPendingQuestion ||
3447 - activePendingIsResponding
3544 + activePendingIsResponding ||
3545 + speechInput.freezesEditor
3546 }
3547 />
3548 </ComposerContextActionsContext>
@@ -3462,7 +3560,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
3560 showPlanFollowUpPrompt={false}
3561 promptHasText={false}
3562 isSendBusy={isSendBusy}
3465 - sendDisabledReason={sendDisabledReason}
3563 + sendDisabledReason={voiceSendDisabledReason}
3564 isEnvironmentUnavailable={
3565 environmentUnavailable !== null ||
3566 noProviderAvailable ||
@@ -3484,13 +3582,25 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
3582 message={providerInputSubmissionError ?? composerSubmissionError}
3583 />
3584
3487 - {isComposerCollapsedMobile ? null : (
3585 + {speechInput.preview ? (
3586 + <div
3587 + aria-label="Live transcription"
3588 + className="mx-3 mb-3 max-h-32 overflow-y-auto rounded-lg border border-border/60 bg-muted/30 px-3 py-2 text-sm leading-relaxed break-words whitespace-pre-wrap sm:mx-4"
3589 + >
3590 + <span>{speechInput.preview.committed}</span>
3591 + <span className="text-muted-foreground">{speechInput.preview.tentative}</span>
3592 + </div>
3593 + ) : null}
3594 +
3595 + {isComposerCollapsedMobile && speechInput.state.phase === "idle" ? null : (
3596 <div
3597 data-chat-composer-footer="true"
3598 className={cn(
3599 "flex min-w-0 flex-nowrap items-center overflow-visible px-2 pb-2",
3600 pendingUserInputs.length > 0 && "pt-2",
3493 - showMobilePendingAnswerActions && "hidden sm:flex",
3601 + showMobilePendingAnswerActions &&
3602 + speechInput.state.phase === "idle" &&
3603 + "hidden sm:flex",
3604 )}
3605 >
3606 {composerAttachButton ? <div className="me-2">{composerAttachButton}</div> : null}
@@ -3541,16 +3651,17 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
3651 showPlanFollowUpPrompt={
3652 pendingUserInputs.length === 0 && showPlanFollowUpPrompt
3653 }
3544 - promptHasText={prompt.trim().length > 0}
3654 + promptHasText={prompt.trim().length > 0 || voiceInputActive}
3655 + voiceInputActive={voiceInputActive}
3656 isSendBusy={isSendBusy}
3546 - sendDisabledReason={sendDisabledReason}
3657 + sendDisabledReason={voiceSendDisabledReason}
3658 isEnvironmentUnavailable={
3659 environmentUnavailable !== null ||
3660 noProviderAvailable ||
3661 projectSelectionRequired
3662 }
3663 isPreparingWorktree={isPreparingWorktree}
3553 - hasSendableContent={composerSendState.hasSendableContent}
3664 + hasSendableContent={composerSendState.hasSendableContent || voiceInputActive}
3665 canResume={showResumeAction}
3666 preserveComposerFocusOnPointerDown={isMobileViewport}
3667 isEditingQueuedMessage={isEditingQueuedMessage}
@@ -3567,7 +3678,45 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
3678 {...(compactCommandAvailable
3679 ? { onCompactContext: compactThreadContextStable }
3680 : {})}
3570 - />
3681 + >
3682 + {speechInput.available ? (
3683 + <>
3684 + <ComposerSpeechRecordingPill
3685 + state={speechInput.state}
3686 + progress={null}
3687 + finishShortcutLabel={dictationShortcutLabel}
3688 + cancelShortcutLabel="Esc"
3689 + level={speechInput.level}
3690 + disabled={dictationDisabled}
3691 + onStart={() => void speechInput.start()}
3692 + onStop={() => void speechInput.stop()}
3693 + onCancel={() => speechInput.cancel()}
3694 + onSkipPostProcessing={() => speechInput.skipPostProcessing()}
3695 + />
3696 + <ComposerSpeechButton
3697 + state={speechInput.state}
3698 + progress={null}
3699 + shortcutLabel={dictationShortcutLabel}
3700 + disabled={dictationDisabled}
3701 + onStart={() => void speechInput.start()}
3702 + onCancel={() => speechInput.cancel()}
3703 + />
3704 + </>
3705 + ) : null}
3706 + <VoiceInputSetup
3707 + environmentId={speechInput.transcriptionEnvironmentId}
3708 + open={speechInput.setup.open}
3709 + step={speechInput.setup.step}
3710 + status={speechInput.status}
3711 + model={speechInput.setup.model}
3712 + downloading={speechInput.setup.downloading}
3713 + error={speechInput.setup.error}
3714 + onOpenChange={speechInput.setup.setOpen}
3715 + onDownload={() => void speechInput.setup.download()}
3716 + onCancelDownload={() => void speechInput.setup.cancelDownload()}
3717 + onStartRecording={() => void speechInput.setup.startRecording()}
3718 + />
3719 + </ComposerFooterPrimaryActions>
3720 </div>
3721 </div>
3722 )}
apps/web/src/components/chat/ComposerFooterControls.tsx
+4
@@ -246,6 +246,7 @@ export function ComposerPermissionsSection(props: {
246 }
247
248 export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryActions(props: {
249 + children?: React.ReactNode;
250 compact: boolean;
251 canOperateThread: boolean;
252 activeContextWindow: ContextWindowSnapshot | null;
@@ -266,6 +267,7 @@ export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryA
267 alternateShortcutLabel: string | null;
268 showPlanFollowUpPrompt: boolean;
269 promptHasText: boolean;
270 + voiceInputActive?: boolean;
271 isSendBusy: boolean;
272 sendDisabledReason: string | null;
273 isEnvironmentUnavailable: boolean;
@@ -298,6 +300,7 @@ export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryA
300 ) : props.reserveContextWindowMeter ? (
301 <ContextWindowMeterPlaceholder />
302 ) : null}
303 + {props.children}
304 <ComposerPrimaryActions
305 compact={props.compact}
306 canOperateThread={props.canOperateThread}
@@ -309,6 +312,7 @@ export const ComposerFooterPrimaryActions = memo(function ComposerFooterPrimaryA
312 alternateShortcutLabel={props.alternateShortcutLabel}
313 showPlanFollowUpPrompt={props.showPlanFollowUpPrompt}
314 promptHasText={props.promptHasText}
315 + voiceInputActive={props.voiceInputActive ?? false}
316 isSendBusy={props.isSendBusy}
317 sendDisabledReason={props.sendDisabledReason}
318 isEnvironmentUnavailable={props.isEnvironmentUnavailable}
apps/web/src/components/chat/ComposerPrimaryActions.tsx
+8 -1
@@ -36,6 +36,8 @@ interface ComposerPrimaryActionsProps {
36 alternateShortcutLabel?: string | null;
37 showPlanFollowUpPrompt: boolean;
38 promptHasText: boolean;
39 + /** Dictation is running; Send finishes it and submits once the transcript lands. */
40 + voiceInputActive?: boolean;
41 isSendBusy: boolean;
42 sendDisabledReason: string | null;
43 isEnvironmentUnavailable: boolean;
@@ -90,6 +92,7 @@ export const ComposerPrimaryActions = memo(function ComposerPrimaryActions({
92 alternateShortcutLabel = null,
93 showPlanFollowUpPrompt,
94 promptHasText,
95 + voiceInputActive = false,
96 isSendBusy,
97 sendDisabledReason,
98 isEnvironmentUnavailable,
@@ -186,8 +189,12 @@ export const ComposerPrimaryActions = memo(function ComposerPrimaryActions({
189 disabled={
190 !canOperateThread ||
191 isEnvironmentUnavailable ||
192 + isSendDisabled ||
193 pendingAction.isResponding ||
190 - (pendingAction.isLastQuestion ? !pendingAction.isComplete : !pendingAction.canAdvance)
194 + (!voiceInputActive &&
195 + (pendingAction.isLastQuestion
196 + ? !pendingAction.isComplete
197 + : !pendingAction.canAdvance))
198 }
199 >
200 {formatPendingPrimaryActionLabel({
apps/web/src/components/chat/ComposerSpeechButton.tsx new
+302
@@ -0,0 +1,302 @@
1 +import type { VoiceInputState } from "@t3tools/client-runtime/voice-input";
2 +import { CheckIcon, MicIcon, XIcon } from "lucide-react";
3 +import { useEffect, useState } from "react";
4 +
5 +import { VoiceInputErrorPill } from "./VoiceInputErrorPill";
6 +import { VoiceInputPill, VoiceWaveform } from "./VoiceInputPill";
7 +import { cn } from "~/lib/utils";
8 +import { Button } from "../ui/button";
9 +import { Spinner } from "../ui/spinner";
10 +import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
11 +
12 +type SpeechPresentation = {
13 + status: string | null;
14 + showsCancel: boolean;
15 + showsConfirm: boolean;
16 + confirmEnabled: boolean;
17 + showsSend: boolean;
18 +};
19 +
20 +function resolveSpeechPresentation(
21 + state: VoiceInputState,
22 + progress: { downloaded: number; total: number } | null,
23 +): SpeechPresentation {
24 + switch (state.phase) {
25 + case "idle":
26 + return {
27 + status: null,
28 + showsCancel: false,
29 + showsConfirm: false,
30 + confirmEnabled: false,
31 + showsSend: true,
32 + };
33 + case "error":
34 + return {
35 + status: state.error ?? "Voice input failed",
36 + showsCancel: false,
37 + showsConfirm: false,
38 + confirmEnabled: false,
39 + showsSend: true,
40 + };
41 + case "preparing":
42 + return {
43 + status: progress
44 + ? `Downloading speech model ${Math.round((progress.downloaded / Math.max(1, progress.total)) * 100)}%`
45 + : "Preparing",
46 + showsCancel: true,
47 + showsConfirm: true,
48 + confirmEnabled: false,
49 + showsSend: true,
50 + };
51 + case "recording":
52 + return {
53 + status: "Recording",
54 + showsCancel: true,
55 + showsConfirm: true,
56 + confirmEnabled: true,
57 + showsSend: true,
58 + };
59 + case "transcribing":
60 + return {
61 + status: "Transcribing",
62 + showsCancel: true,
63 + showsConfirm: true,
64 + confirmEnabled: false,
65 + showsSend: true,
66 + };
67 + case "post-processing":
68 + return {
69 + status: "Post-processing",
70 + showsCancel: true,
71 + showsConfirm: true,
72 + confirmEnabled: false,
73 + showsSend: true,
74 + };
75 + }
76 +}
77 +
78 +function formatElapsed(seconds: number) {
79 + return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
80 +}
81 +
82 +function RecordingStatus(props: { level: number; recordingLimitSeconds?: number | undefined }) {
83 + const [elapsedSeconds, setElapsedSeconds] = useState(0);
84 +
85 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the timer displaying the native recording duration
86 + useEffect(() => {
87 + const startedAt = Date.now();
88 + const timer = window.setInterval(
89 + () => setElapsedSeconds(Math.floor((Date.now() - startedAt) / 1_000)),
90 + 250,
91 + );
92 + return () => window.clearInterval(timer);
93 + }, []);
94 +
95 + return (
96 + <>
97 + <VoiceWaveform level={props.level} />
98 + <span aria-hidden className="size-1.5 shrink-0 rounded-full bg-primary" />
99 + <span className="shrink-0 text-secondary-label text-xs tabular-nums">
100 + {formatElapsed(elapsedSeconds)}
101 + {props.recordingLimitSeconds !== undefined
102 + ? ` / ${formatElapsed(props.recordingLimitSeconds)}`
103 + : ""}
104 + </span>
105 + </>
106 + );
107 +}
108 +
109 +function ComposerSpeechStatus(props: {
110 + state: VoiceInputState;
111 + progress: { downloaded: number; total: number } | null;
112 + level: number;
113 + recordingLimitSeconds?: number | undefined;
114 +}) {
115 + const presentation = resolveSpeechPresentation(props.state, props.progress);
116 +
117 + if (!presentation.status) return null;
118 + const isRecording = props.state.phase === "recording";
119 + const isError = props.state.phase === "error";
120 +
121 + return (
122 + <div
123 + className="flex h-7 min-w-0 flex-1 items-center gap-2"
124 + role="status"
125 + aria-live={isRecording ? "off" : "polite"}
126 + aria-label={presentation.status}
127 + >
128 + {isRecording ? (
129 + <RecordingStatus level={props.level} recordingLimitSeconds={props.recordingLimitSeconds} />
130 + ) : (
131 + <span
132 + className={cn(
133 + "min-w-0 flex-1 truncate text-right text-sm text-secondary-label",
134 + isError && "text-destructive",
135 + )}
136 + >
137 + {presentation.status}
138 + </span>
139 + )}
140 + </div>
141 + );
142 +}
143 +
144 +function ComposerSpeechCancelButton(props: {
145 + state: VoiceInputState;
146 + shortcutLabel?: string | null;
147 + onCancel(): void;
148 +}) {
149 + if (props.state.phase === "idle") return null;
150 + const label = props.state.phase === "error" ? "Dismiss voice input error" : "Cancel voice input";
151 +
152 + return (
153 + <Tooltip>
154 + <TooltipTrigger
155 + render={
156 + <Button
157 + type="button"
158 + size="icon-sm-pill"
159 + variant="ghost"
160 + aria-label={label}
161 + onPointerDown={(event) => event.preventDefault()}
162 + onClick={props.onCancel}
163 + className="shrink-0"
164 + >
165 + <XIcon />
166 + </Button>
167 + }
168 + />
169 + <TooltipPopup side="top">
170 + {label}
171 + {props.shortcutLabel ? ` (${props.shortcutLabel})` : ""}
172 + </TooltipPopup>
173 + </Tooltip>
174 + );
175 +}
176 +
177 +export function ComposerSpeechButton(props: {
178 + state: VoiceInputState;
179 + progress: { downloaded: number; total: number } | null;
180 + shortcutLabel?: string | null;
181 + disabled?: boolean;
182 + onStart(): void;
183 + onCancel(): void;
184 +}) {
185 + const presentation = resolveSpeechPresentation(props.state, props.progress);
186 + const label = "Start voice input";
187 +
188 + if (presentation.showsConfirm || props.state.phase === "error") return null;
189 +
190 + return (
191 + <Tooltip>
192 + <TooltipTrigger
193 + render={
194 + <Button
195 + type="button"
196 + size="icon-sm"
197 + variant="ghost"
198 + aria-label={label}
199 + disabled={props.disabled}
200 + onPointerDown={(event) => event.preventDefault()}
201 + onClick={() => {
202 + if (props.disabled) return;
203 + props.onStart();
204 + }}
205 + className="relative shrink-0"
206 + >
207 + <MicIcon />
208 + </Button>
209 + }
210 + />
211 + <TooltipPopup side="top">
212 + {label}
213 + {props.shortcutLabel ? ` (${props.shortcutLabel})` : ""}
214 + </TooltipPopup>
215 + </Tooltip>
216 + );
217 +}
218 +
219 +export function ComposerSpeechRecordingPill(props: {
220 + state: VoiceInputState;
221 + progress: { downloaded: number; total: number } | null;
222 + finishShortcutLabel?: string | null;
223 + cancelShortcutLabel?: string | null;
224 + level: number;
225 + recordingLimitSeconds?: number | undefined;
226 + onStart(): void;
227 + disabled?: boolean;
228 + onStop(): void;
229 + onCancel(): void;
230 + onSkipPostProcessing(): void;
231 +}) {
232 + const presentation = resolveSpeechPresentation(props.state, props.progress);
233 + if (props.state.phase === "error") {
234 + return (
235 + <VoiceInputErrorPill
236 + error={props.state.error}
237 + errorAction={props.state.errorAction}
238 + disabled={props.disabled ?? false}
239 + cancelShortcutLabel={props.cancelShortcutLabel ?? null}
240 + onRetry={props.onStart}
241 + onDismiss={props.onCancel}
242 + />
243 + );
244 + }
245 + if (!presentation.showsConfirm) return null;
246 +
247 + const label = presentation.confirmEnabled
248 + ? "Finish voice input"
249 + : (presentation.status ?? "Voice input is busy");
250 +
251 + return (
252 + <VoiceInputPill>
253 + <ComposerSpeechCancelButton
254 + state={props.state}
255 + shortcutLabel={props.cancelShortcutLabel ?? null}
256 + onCancel={props.onCancel}
257 + />
258 + <ComposerSpeechStatus
259 + state={props.state}
260 + progress={props.progress}
261 + level={props.level}
262 + recordingLimitSeconds={props.recordingLimitSeconds}
263 + />
264 + {props.state.phase === "post-processing" ? (
265 + <Button
266 + type="button"
267 + size="sm-pill"
268 + variant="ghost"
269 + onPointerDown={(event) => event.preventDefault()}
270 + onClick={props.onSkipPostProcessing}
271 + className="shrink-0"
272 + >
273 + Skip
274 + </Button>
275 + ) : (
276 + <Tooltip>
277 + <TooltipTrigger
278 + render={
279 + <Button
280 + type="button"
281 + size="icon-sm-pill"
282 + aria-label={label}
283 + disabled={!presentation.confirmEnabled}
284 + onPointerDown={(event) => event.preventDefault()}
285 + onClick={props.onStop}
286 + className="shrink-0"
287 + >
288 + {presentation.confirmEnabled ? <CheckIcon /> : <Spinner aria-hidden />}
289 + </Button>
290 + }
291 + />
292 + <TooltipPopup side="top">
293 + {label}
294 + {presentation.confirmEnabled && props.finishShortcutLabel
295 + ? ` (${props.finishShortcutLabel})`
296 + : ""}
297 + </TooltipPopup>
298 + </Tooltip>
299 + )}
300 + </VoiceInputPill>
301 + );
302 +}
apps/web/src/components/chat/RightPanelSurfaceContent.tsx
+2
@@ -50,6 +50,7 @@ interface RightPanelPullRequestProps {
50 }
51
52 interface RightPanelFileProps {
53 + readonly projectId?: ProjectId | undefined;
54 readonly projectName: string | null;
55 readonly workspaceRoot: string | undefined;
56 readonly availableEditors: ReadonlyArray<EditorId>;
@@ -209,6 +210,7 @@ export function RightPanelSurfaceContent(props: {
210 environmentId={props.environmentId}
211 cwd={files.workspaceRoot ?? ""}
212 projectName={files.projectName ?? ""}
213 + projectId={files.projectId}
214 threadRef={threadRef}
215 composerDraftTarget={props.composerDraftTarget}
216 keybindings={props.keybindings}
apps/web/src/components/chat/VoiceInputErrorPill.tsx new
+115
@@ -0,0 +1,115 @@
1 +import { MICROPHONE_PERMISSION_ERROR } from "@t3tools/client-runtime/voice-input";
2 +import { useNavigate } from "@tanstack/react-router";
3 +import { RotateCcwIcon, SettingsIcon, XIcon } from "lucide-react";
4 +
5 +import { Button } from "../ui/button";
6 +import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
7 +import { VoiceInputPill } from "./VoiceInputPill";
8 +
9 +const VOICE_ERROR_LABELS: Record<string, string> = {
10 + [MICROPHONE_PERMISSION_ERROR]: "Microphone access needed",
11 + "Microphone access is required for voice input.": "Microphone access needed",
12 + "This draft is no longer available.": "Draft unavailable",
13 + "Another voice recording is already active.": "Another recording active",
14 + "Voice transcription is not available.": "Voice input unavailable",
15 + "Voice transcription is still finishing. Try again shortly.": "Transcription still busy",
16 + "Voice transcription is not available for this language.": "Language unsupported",
17 + "Could not prepare voice transcription.": "Couldn’t prepare dictation",
18 + "Could not start voice recording.": "Couldn’t start recording",
19 + "Voice recording was interrupted.": "Recording interrupted",
20 + "Could not finish voice recording.": "Couldn’t finish recording",
21 + "Could not transcribe this recording.": "Transcription failed",
22 + "Voice transcription on this environment failed.": "Transcription failed",
23 + "The draft changed while voice input was running. The transcript was not added.": "Draft changed",
24 + "No speech was detected.": "No speech detected",
25 + "Microphone processing failed.": "Microphone processing failed",
26 + "The microphone disconnected.": "Microphone disconnected",
27 +};
28 +
29 +function voiceErrorLabel(error: string | null): string {
30 + return (error && VOICE_ERROR_LABELS[error]) || "Voice input failed";
31 +}
32 +
33 +export function VoiceInputErrorPill(props: {
34 + error: string | null;
35 + errorAction: "retry" | "settings" | null;
36 + disabled?: boolean;
37 + cancelShortcutLabel?: string | null;
38 + onRetry(): void;
39 + onDismiss(): void;
40 +}) {
41 + const navigate = useNavigate();
42 + const openSettings = props.errorAction === "settings";
43 + const actionLabel = openSettings
44 + ? "Open voice settings"
45 + : props.error === "Could not transcribe this recording." ||
46 + props.error === "Voice transcription on this environment failed."
47 + ? "Retry transcription"
48 + : "Record again";
49 + return (
50 + <VoiceInputPill>
51 + <Tooltip>
52 + <TooltipTrigger
53 + render={
54 + <Button
55 + type="button"
56 + size="icon-sm-pill"
57 + variant="ghost"
58 + aria-label="Dismiss voice input error"
59 + onPointerDown={(event) => event.preventDefault()}
60 + onClick={props.onDismiss}
61 + >
62 + <XIcon />
63 + </Button>
64 + }
65 + />
66 + <TooltipPopup side="top">
67 + Dismiss voice input error
68 + {props.cancelShortcutLabel ? ` (${props.cancelShortcutLabel})` : ""}
69 + </TooltipPopup>
70 + </Tooltip>
71 + <Tooltip>
72 + <TooltipTrigger
73 + render={
74 + <span
75 + tabIndex={0}
76 + role="status"
77 + aria-label={props.error ?? "Voice input failed"}
78 + className="min-w-0 flex-1 truncate text-sm text-destructive"
79 + />
80 + }
81 + >
82 + {voiceErrorLabel(props.error)}
83 + </TooltipTrigger>
84 + <TooltipPopup side="top">{props.error ?? "Voice input failed"}</TooltipPopup>
85 + </Tooltip>
86 + {props.errorAction ? (
87 + <Tooltip>
88 + <TooltipTrigger
89 + render={
90 + <Button
91 + type="button"
92 + size="icon-sm-pill"
93 + variant="ghost"
94 + aria-label={actionLabel}
95 + disabled={props.disabled}
96 + onPointerDown={(event) => event.preventDefault()}
97 + onClick={() => {
98 + if (openSettings) {
99 + props.onDismiss();
100 + void navigate({ to: "/settings/voice" });
101 + } else {
102 + props.onRetry();
103 + }
104 + }}
105 + >
106 + {openSettings ? <SettingsIcon /> : <RotateCcwIcon />}
107 + </Button>
108 + }
109 + />
110 + <TooltipPopup side="top">{actionLabel}</TooltipPopup>
111 + </Tooltip>
112 + ) : null}
113 + </VoiceInputPill>
114 + );
115 +}
apps/web/src/components/chat/VoiceInputPill.tsx new
+51
@@ -0,0 +1,51 @@
1 +import { memo, useEffect, useRef, type ReactNode } from "react";
2 +import { useMediaQuery } from "~/hooks/useMediaQuery";
3 +
4 +const WAVEFORM_BAR_COUNT = 28;
5 +const WAVEFORM_BAR_IDS = Array.from({ length: WAVEFORM_BAR_COUNT }, (_, index) => String(index));
6 +
7 +export const VoiceWaveform = memo(function VoiceWaveform(props: { level: number }) {
8 + const prefersReducedMotion = useMediaQuery("(prefers-reduced-motion: reduce)");
9 + const barsRef = useRef<Array<HTMLSpanElement | null>>([]);
10 + const levelsRef = useRef(Array<number>(WAVEFORM_BAR_COUNT).fill(0));
11 +
12 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- updates native waveform bar styles when microphone levels arrive
13 + useEffect(() => {
14 + const levels = levelsRef.current;
15 + levels.copyWithin(0, 1);
16 + levels[levels.length - 1] = props.level;
17 + barsRef.current.forEach((bar, index) => {
18 + if (!bar) return;
19 + const level = levels[index] ?? 0;
20 + bar.style.opacity = String(0.22 + level * 0.78);
21 + bar.style.transform = `scaleY(${prefersReducedMotion ? 0.35 : Math.max(0.08, level)})`;
22 + });
23 + }, [prefersReducedMotion, props.level]);
24 +
25 + return (
26 + <div
27 + aria-hidden
28 + className="flex h-5 min-w-0 flex-1 items-center justify-between gap-px overflow-hidden"
29 + >
30 + {WAVEFORM_BAR_IDS.map((id, index) => (
31 + <span
32 + key={id}
33 + ref={(bar) => {
34 + barsRef.current[index] = bar;
35 + }}
36 + className="h-full w-0.5 shrink-0 origin-center rounded-full bg-primary opacity-25 transition-[transform,opacity] duration-100 ease-out motion-reduce:transition-none"
37 + style={{ transform: `scaleY(${prefersReducedMotion ? 0.35 : 0.08})` }}
38 + />
39 + ))}
40 + </div>
41 + );
42 +});
43 +
44 +export function VoiceInputPill({ children }: { children: ReactNode }) {
45 + return (
46 + // Same height as the toolbar's icon buttons, so starting a recording never resizes the composer.
47 + <div className="flex h-8 w-48 min-w-0 shrink-0 items-center gap-2 sm:h-7 sm:w-64">
48 + {children}
49 + </div>
50 + );
51 +}
apps/web/src/components/chat/VoiceInputSetup.tsx new
+243
@@ -0,0 +1,243 @@
1 +import { useStableCallback } from "~/hooks/useStableCallback";
2 +import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";
3 +import type {
4 + EnvironmentId,
5 + EnvironmentSpeechModel,
6 + EnvironmentSpeechStatus,
7 +} from "@t3tools/contracts";
8 +import { useNavigate } from "@tanstack/react-router";
9 +import { RefreshCwIcon } from "lucide-react";
10 +import { useEffect, useState } from "react";
11 +
12 +import { useClientSettings, updateClientSettings } from "../../hooks/useSettings";
13 +import { useEnvironment } from "../../state/environments";
14 +import { AuthSettingsWriteScope } from "@t3tools/contracts";
15 +import { useEnvironmentScope } from "../../state/session";
16 +import { Button } from "../ui/button";
17 +import { Dialog } from "../ui/dialog";
18 +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
19 +import { WizardFooter, WizardHeader, WizardPanel, WizardPopup, WizardSteps } from "../ui/wizard";
20 +
21 +export function VoiceInputSetup(props: {
22 + environmentId: EnvironmentId | null;
23 + open: boolean;
24 + step: number;
25 + status: EnvironmentSpeechStatus | null;
26 + model: EnvironmentSpeechModel | null;
27 + downloading: boolean;
28 + error: string | null;
29 + onOpenChange(open: boolean): void;
30 + onDownload(): void;
31 + onCancelDownload(): void;
32 + onStartRecording(): void;
33 +}) {
34 + const navigate = useNavigate();
35 + const environment = useEnvironment(props.environmentId);
36 + const canManage = useEnvironmentScope(props.environmentId, AuthSettingsWriteScope);
37 + const selectedMicrophone = useClientSettings((settings) => settings.voiceMicrophone);
38 + const [microphones, setMicrophones] = useState<BrowserMicrophone[]>([]);
39 + const [loadingMicrophones, setLoadingMicrophones] = useState(false);
40 + const [microphoneError, setMicrophoneError] = useState<string | null>(null);
41 + const refreshMicrophones = useStableCallback(async () => {
42 + if (!navigator.mediaDevices?.enumerateDevices) return;
43 + setLoadingMicrophones(true);
44 + try {
45 + setMicrophones(await listBrowserMicrophones());
46 + setMicrophoneError(null);
47 + } catch {
48 + setMicrophoneError("Could not list microphones. You can still use the system default.");
49 + }
50 + setLoadingMicrophones(false);
51 + });
52 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to native microphone changes while the device setup step is visible
53 + useEffect(() => {
54 + if (!props.open || props.step !== 1) return;
55 + queueMicrotask(() => void refreshMicrophones());
56 + const mediaDevices = navigator.mediaDevices;
57 + mediaDevices?.addEventListener?.("devicechange", refreshMicrophones);
58 + return () => mediaDevices?.removeEventListener?.("devicechange", refreshMicrophones);
59 + }, [props.open, props.step, refreshMicrophones]);
60 + const selectedIsUnavailable = Boolean(
61 + selectedMicrophone && !microphones.some((device) => device.deviceId === selectedMicrophone),
62 + );
63 + const selectedMicrophoneLabel = selectedMicrophone
64 + ? (microphones.find((device) => device.deviceId === selectedMicrophone)?.label ??
65 + "Selected microphone (Unavailable)")
66 + : "System default";
67 + const status = props.status?.supported ? props.status : null;
68 + const model = props.model;
69 + const progress =
70 + model?.downloaded === undefined ? null : Math.round((model.downloaded / model.size) * 100);
71 + const openSettings = (hash: string) => {
72 + props.onOpenChange(false);
73 + void navigate({ to: "/settings/voice", hash });
74 + };
75 +
76 + return (
77 + <Dialog open={props.open} onOpenChange={props.onOpenChange}>
78 + <WizardPopup>
79 + <WizardHeader
80 + title="Set up voice input"
81 + description={`Transcribe speech on ${environment?.label ?? "your transcription environment"} and use voice input across environments.`}
82 + >
83 + <WizardSteps steps={["Model", "Ready to dictate"]} currentStep={props.step} />
84 + </WizardHeader>
85 + <WizardPanel>
86 + {props.step === 0 ? (
87 + <section className="space-y-3 text-sm">
88 + <div className="rounded-lg border border-border/70 bg-card/60 p-4">
89 + <div className="flex items-center justify-between gap-2">
90 + <h3 className="font-medium">{status?.model ?? "English transcription model"}</h3>
91 + {status?.modelId === "handy-computer/parakeet-unified-en-0.6b-gguf" ? (
92 + <span className="text-xs font-medium text-primary">
93 + Recommended for English
94 + </span>
95 + ) : null}
96 + </div>
97 + <p className="mt-2 text-muted-foreground">
98 + Download the {status ? Math.round(status.size / 1024 / 1024) : 697} MB model to{" "}
99 + {environment?.label ?? "your transcription environment"}. Microphone audio is sent
100 + there for transcription, as chosen in Voice settings.
101 + </p>
102 + <p className="mt-2 text-muted-foreground">
103 + Silero (1.2 MB), used for silence filtering and pause detection, downloads
104 + alongside the speech model. An installed copy is reused.
105 + </p>
106 + {props.downloading ? (
107 + <div role="status" className="mt-4 space-y-1 text-muted-foreground">
108 + <div
109 + role="progressbar"
110 + aria-label="Speech model download progress"
111 + aria-valuemin={0}
112 + aria-valuemax={100}
113 + aria-valuenow={progress ?? undefined}
114 + className="h-1 overflow-hidden rounded-full bg-muted"
115 + >
116 + <div
117 + className="h-full bg-primary transition-[width] duration-200"
118 + style={{ width: `${progress ?? 0}%` }}
119 + />
120 + </div>
121 + <p className="text-xs">
122 + {model?.state === "verifying"
123 + ? "Verifying download…"
124 + : progress === null
125 + ? "Starting download…"
126 + : `${progress}% downloaded`}
127 + </p>
128 + </div>
129 + ) : null}
130 + </div>
131 + {props.error ? (
132 + <p role="alert" className="text-destructive">
133 + {props.error}
134 + </p>
135 + ) : null}
136 + </section>
137 + ) : (
138 + <section className="space-y-4 text-sm">
139 + <div className="space-y-2">
140 + <label className="font-medium" htmlFor="voice-setup-microphone">
141 + Microphone
142 + </label>
143 + <div className="flex items-center gap-2">
144 + <Select
145 + value={selectedMicrophone ? `device:${selectedMicrophone}` : "system-default"}
146 + disabled={loadingMicrophones}
147 + onValueChange={(value) => {
148 + if (value)
149 + void updateClientSettings({
150 + voiceMicrophone:
151 + value === "system-default" ? "" : value.slice("device:".length),
152 + });
153 + }}
154 + >
155 + <SelectTrigger
156 + id="voice-setup-microphone"
157 + size="sm"
158 + aria-label="Microphone"
159 + className="min-w-0 flex-1"
160 + >
161 + <SelectValue>{selectedMicrophoneLabel}</SelectValue>
162 + </SelectTrigger>
163 + <SelectPopup align="end" alignItemWithTrigger={false}>
164 + <SelectItem value="system-default">System default</SelectItem>
165 + {selectedIsUnavailable ? (
166 + <SelectItem value={`device:${selectedMicrophone}`}>
167 + Selected microphone (Unavailable)
168 + </SelectItem>
169 + ) : null}
170 + {microphones.map((device) => (
171 + <SelectItem key={device.deviceId} value={`device:${device.deviceId}`}>
172 + {device.label}
173 + </SelectItem>
174 + ))}
175 + </SelectPopup>
176 + </Select>
177 + <Button
178 + type="button"
179 + variant="ghost"
180 + size="icon-sm"
181 + disabled={loadingMicrophones}
182 + aria-label="Refresh microphones"
183 + onClick={() => void refreshMicrophones()}
184 + >
185 + <RefreshCwIcon className="size-3.5" />
186 + </Button>
187 + </div>
188 + {selectedIsUnavailable ? (
189 + <p className="text-xs text-destructive">
190 + The selected microphone is unavailable. Choose another or use the system
191 + default.
192 + </p>
193 + ) : null}
194 + {microphoneError ? (
195 + <p role="alert" className="text-xs text-muted-foreground">
196 + {microphoneError}
197 + </p>
198 + ) : null}
199 + </div>
200 + <p className="text-muted-foreground">
201 + Speak and transcribe in multiple languages or translate into English, and get names
202 + and technical terms right with the dictionary in Voice settings.
203 + </p>
204 + </section>
205 + )}
206 + </WizardPanel>
207 + <WizardFooter>
208 + {props.step === 0 ? (
209 + <>
210 + <Button variant="outline" onClick={() => openSettings("speech-models")}>
211 + Choose another model or language
212 + </Button>
213 + {props.downloading ? (
214 + <Button variant="outline" disabled={!canManage} onClick={props.onCancelDownload}>
215 + Cancel download
216 + </Button>
217 + ) : null}
218 + <Button
219 + disabled={!canManage || props.downloading || !status}
220 + onClick={props.onDownload}
221 + >
222 + {props.downloading
223 + ? "Downloading…"
224 + : props.error
225 + ? "Retry download"
226 + : canManage
227 + ? "Download model"
228 + : "Ask an environment admin to download"}
229 + </Button>
230 + </>
231 + ) : (
232 + <>
233 + <Button variant="outline" onClick={() => openSettings("")}>
234 + Customize in Voice settings
235 + </Button>
236 + <Button onClick={props.onStartRecording}>Start dictating</Button>
237 + </>
238 + )}
239 + </WizardFooter>
240 + </WizardPopup>
241 + </Dialog>
242 + );
243 +}
apps/web/src/components/composerContextActions.ts new
+22
@@ -0,0 +1,22 @@
1 +import type { EnvironmentId, ProjectId } from "@t3tools/contracts";
2 +import { createContext, type MouseEvent } from "react";
3 +
4 +export interface ComposerContextActions {
5 + environmentId: EnvironmentId | null;
6 + cwd?: string | undefined;
7 + projectId?: ProjectId | undefined;
8 + expandImage: (imageId: string) => void;
9 + expandVideo: (fileId: string) => void;
10 + openFile: (fileId: string) => void;
11 + openMention: (path: string) => void;
12 + openPullRequest: (event: MouseEvent<HTMLElement>, url: string) => void;
13 +}
14 +
15 +export const ComposerContextActionsContext = createContext<ComposerContextActions>({
16 + environmentId: null,
17 + expandImage: () => {},
18 + expandVideo: () => {},
19 + openFile: () => {},
20 + openMention: () => {},
21 + openPullRequest: () => {},
22 +});
apps/web/src/components/composerContextPresentation.tsx
+6 -20
@@ -9,8 +9,8 @@ import type {
9 import { formatAttachmentSize } from "@t3tools/client-runtime/state/attachments";
10 import { videoMimeType } from "@t3tools/shared/video";
11 import { MessageCircleIcon, MousePointerClickIcon } from "lucide-react";
12 -import { createContext, type MouseEvent, type ReactElement, type ReactNode, use } from "react";
13 -import type { EnvironmentId } from "@t3tools/contracts";
12 +import { createContext, type ReactElement, type ReactNode, use } from "react";
13 +import { ComposerContextActionsContext } from "./composerContextActions";
14
15 import type { ComposerFileAttachment, ComposerImageAttachment } from "~/composerDraftStore";
16 import { composerFileNeedsReattach } from "~/composerDraftStore";
@@ -68,24 +68,10 @@ export type ComposerDraftContextRecord =
68 | { kind: "thread"; record: ThreadContextRecord }
69 | { kind: "environment"; record: EnvironmentContextRecord };
70
71 -export interface ComposerContextActions {
72 - environmentId: EnvironmentId | null;
73 - cwd?: string | undefined;
74 - expandImage: (imageId: string) => void;
75 - expandVideo: (fileId: string) => void;
76 - openFile: (fileId: string) => void;
77 - openMention: (path: string) => void;
78 - openPullRequest: (event: MouseEvent<HTMLElement>, url: string) => void;
79 -}
80 -
81 -export const ComposerContextActionsContext = createContext<ComposerContextActions>({
82 - environmentId: null,
83 - expandImage: () => {},
84 - expandVideo: () => {},
85 - openFile: () => {},
86 - openMention: () => {},
87 - openPullRequest: () => {},
88 -});
71 +export {
72 + ComposerContextActionsContext,
73 + type ComposerContextActions,
74 +} from "./composerContextActions";
75
76 export type ComposerDraftContextRecords = ReadonlyMap<string, ComposerDraftContextRecord>;
77
apps/web/src/components/diffs/AnnotatableCodeView.tsx
+13 -1
@@ -7,7 +7,7 @@ import type {
7 SelectedLineRange,
8 } from "@pierre/diffs";
9 import type { CodeViewHandle } from "@pierre/diffs/react";
10 -import type { ScopedThreadRef } from "@t3tools/contracts";
10 +import type { EnvironmentId, ProjectId, ScopedThreadRef } from "@t3tools/contracts";
11 import { useState, type ReactNode, type Ref } from "react";
12
13 import { type DraftId, useComposerDraftStore } from "~/composerDraftStore";
@@ -72,6 +72,8 @@ function appendAnnotationEntry(
72 }
73
74 interface AnnotatableCodeViewProps {
75 + environmentId?: EnvironmentId | undefined;
76 + projectId?: ProjectId | null | undefined;
77 codeViewKey: string;
78 files: ReadonlyArray<{
79 fileDiff: FileDiffMetadata;
@@ -104,6 +106,8 @@ interface DiffSelectionContext {
106 }
107
108 export function AnnotatableCodeView({
109 + environmentId,
110 + projectId,
111 codeViewKey,
112 files,
113 pendingFiles,
@@ -292,6 +296,14 @@ export function AnnotatableCodeView({
296 {annotation.metadata.entries.map((entry) => (
297 <DiffCommentAnnotation
298 key={entry.id}
299 + environmentId={environmentId}
300 + projectId={projectId}
301 + ownerKey={JSON.stringify([
302 + environmentId,
303 + composerDraftTarget,
304 + codeViewKey,
305 + entry.id,
306 + ])}
307 kind={entry.kind}
308 rangeLabel={entry.rangeLabel}
309 text={entry.kind === "draft" ? draftText : entry.text}
apps/web/src/components/diffs/DiffCommentAnnotation.tsx
+60 -8
@@ -1,8 +1,14 @@
1 +import type { EnvironmentId, ProjectId } from "@t3tools/contracts";
2 import { MessageCircle, Trash2 } from "lucide-react";
3 import { useLayoutEffect, useRef, useState, type ReactNode } from "react";
4
5 import { Button } from "~/components/ui/button";
6 import { Textarea } from "~/components/ui/textarea";
7 +import {
8 + SpeechTextField,
9 + TextFieldSpeechControls,
10 + type TextFieldSpeech,
11 +} from "~/speech/useTextFieldSpeech";
12 import { textDirection } from "~/textDirection";
13
14 import { isCommentSubmitShortcut } from "./commentSubmitShortcut";
@@ -15,6 +21,10 @@ interface DiffCommentSecondaryAction {
21 }
22
23 interface DiffCommentAnnotationProps {
24 + /** With an environment, the draft can be dictated; its project supplies vocabulary. */
25 + environmentId?: EnvironmentId | undefined;
26 + projectId?: ProjectId | null | undefined;
27 + ownerKey?: string;
28 kind: "draft" | "comment";
29 rangeLabel: string;
30 text: string;
@@ -30,6 +40,9 @@ interface DiffCommentAnnotationProps {
40 }
41
42 export function DiffCommentAnnotation({
43 + environmentId,
44 + projectId,
45 + ownerKey,
46 kind,
47 rangeLabel,
48 text,
@@ -88,8 +101,20 @@ export function DiffCommentAnnotation({
101 );
102 }
103
104 + const submit = (
105 + speech: TextFieldSpeech | undefined,
106 + action: (text: string) => void,
107 + allowEmpty = false,
108 + ) => {
109 + const finish = (value: string) => {
110 + if (!pending && (allowEmpty || value.trim())) action(value.trim());
111 + };
112 + if (speech) speech.submitAfterDictation(finish);
113 + else finish(displayedText);
114 + };
115 +
116 // Pierre hides native carets and selection; textarea comments need their own visible feedback.
92 - return (
117 + const renderDraft = (speech?: TextFieldSpeech) => (
118 <div
119 data-diff-comment-annotation
120 className="px-3 py-2 font-sans text-foreground caret-foreground selection:bg-primary selection:text-primary-foreground"
@@ -98,6 +123,7 @@ export function DiffCommentAnnotation({
123 >
124 <Textarea
125 ref={textareaRef}
126 + readOnly={speech?.freezesEditor}
127 autoFocus={focusOnMount}
128 size="sm"
129 value={displayedText}
@@ -112,16 +138,18 @@ export function DiffCommentAnnotation({
138 onKeyDown={(event) => {
139 if (event.key === "Escape") {
140 event.preventDefault();
115 - onCancel();
141 + if (speech && speech.state.phase !== "idle") speech.cancel();
142 + else onCancel();
143 }
117 - if (isCommentSubmitShortcut(event, trimmedText, pending)) {
144 + if (isCommentSubmitShortcut(event, trimmedText, pending, speech?.blocksSubmission)) {
145 event.preventDefault();
119 - onComment(trimmedText);
146 + submit(speech, onComment);
147 }
148 }}
149 />
123 - <div className="mt-1.5 flex items-center gap-1">
150 + <div className="mt-1.5 flex flex-wrap items-center gap-1">
151 <span className="mr-auto text-3xs text-muted-foreground/70">⌘/Ctrl Enter to send</span>
152 + {speech ? <TextFieldSpeechControls speech={speech} disabled={pending} /> : null}
153 <Button variant="ghost-muted" size="xs" onClick={onCancel}>
154 Cancel
155 </Button>
@@ -129,17 +157,41 @@ export function DiffCommentAnnotation({
157 <Button
158 size="xs"
159 variant="outline"
132 - disabled={!secondaryAction.allowEmpty && !trimmedText}
133 - onClick={() => secondaryAction.onAction(trimmedText)}
160 + disabled={
161 + speech?.preparing ||
162 + pending ||
163 + (!speech?.blocksSubmission && !secondaryAction.allowEmpty && !trimmedText)
164 + }
165 + onClick={() => submit(speech, secondaryAction.onAction, secondaryAction.allowEmpty)}
166 >
167 {secondaryAction.icon}
168 {secondaryAction.label}
169 </Button>
170 ) : null}
139 - <Button size="xs" disabled={pending || !trimmedText} onClick={() => onComment(trimmedText)}>
171 + <Button
172 + size="xs"
173 + disabled={speech?.preparing || pending || (!speech?.blocksSubmission && !trimmedText)}
174 + onClick={() => submit(speech, onComment)}
175 + >
176 {submitLabel}
177 </Button>
178 </div>
179 </div>
180 );
181 +
182 + return environmentId ? (
183 + <SpeechTextField
184 + environmentId={environmentId}
185 + projectId={projectId}
186 + ownerKey={ownerKey ?? rangeLabel}
187 + text={displayedText}
188 + textareaRef={textareaRef}
189 + onTextChange={onTextChange ?? setLocalDraftText}
190 + disabled={pending}
191 + >
192 + {renderDraft}
193 + </SpeechTextField>
194 + ) : (
195 + renderDraft()
196 + );
197 }
apps/web/src/components/diffs/commentSubmitShortcut.ts
+5 -1
@@ -8,8 +8,12 @@ export function isCommentSubmitShortcut(
8 event: CommentSubmitShortcutEvent,
9 value: string,
10 pending: boolean,
11 + allowEmpty = false,
12 ): boolean {
13 return (
13 - !pending && (event.metaKey || event.ctrlKey) && event.key === "Enter" && value.trim().length > 0
14 + !pending &&
15 + (event.metaKey || event.ctrlKey) &&
16 + event.key === "Enter" &&
17 + (allowEmpty || value.trim().length > 0)
18 );
19 }
apps/web/src/components/diffs/useReviewFilePatches.browser.test.tsx
+5 -5
@@ -89,10 +89,10 @@ it("retains automatic and explicitly opened files through metadata insertion, re
89 diff: "diff --git a/f.txt b/f.txt\n--- a/f.txt\n+++ b/f.txt\n@@ -1 +1 @@\n-old\n+new\n",
90 files: tracked.files!.filter((f) => f.path === "f.txt"),
91 });
92 - act(() => registry.set(atom, loaded));
92 + await act(() => registry.set(atom, loaded));
93 await expect.element(screen.getByText("f.txt:loaded")).toBeVisible();
94 const automatic = [...state.patches].find(([key]) => JSON.parse(key)[1] === "e.txt")![1];
95 - act(() =>
95 + await act(() =>
96 registry.set(
97 automatic,
98 AsyncResult.success({
@@ -108,14 +108,14 @@ it("retains automatic and explicitly opened files through metadata insertion, re
108 );
109 await expect.element(screen.getByText("e.txt:loaded")).toBeVisible();
110 await expect.element(screen.getByText("f.txt:loaded")).toBeVisible();
111 - act(() =>
111 + await act(() =>
112 registry.set(
113 atom,
114 AsyncResult.fail(new Error("refresh failed"), { previousSuccess: Option.some(loaded) }),
115 ),
116 );
117 await expect.element(screen.getByText("f.txt:loaded")).toBeVisible();
118 - act(() => registry.set(atom, loaded));
118 + await act(() => registry.set(atom, loaded));
119 await expect.element(screen.getByText("f.txt:loaded")).toBeVisible();
120 });
121
@@ -130,7 +130,7 @@ it("preserves deletion and addition patches for a file replaced by a symlink", a
130 </RegistryContext.Provider>,
131 );
132 const atom = [...state.patches.values()][0]!;
133 - act(() =>
133 + await act(() =>
134 registry.set(
135 atom,
136 AsyncResult.success({
apps/web/src/components/files/FilePreviewPanel.tsx
+14
@@ -5,6 +5,7 @@ import {
5 AuthPreviewOperateScope,
6 type EditorId,
7 type EnvironmentId,
8 + type ProjectId,
9 type ResolvedKeybindingsConfig,
10 type ScopedThreadRef,
11 } from "@t3tools/contracts";
@@ -134,6 +135,7 @@ interface FilePreviewPanelProps {
135 environmentId: EnvironmentId;
136 cwd: string;
137 projectName: string;
138 + projectId?: ProjectId | undefined;
139 relativePath: string | null;
140 attachment?: ChatFileAttachment;
141 threadRef: ScopedThreadRef;
@@ -650,6 +652,7 @@ function useEditableAfterHighlight(file: FileContents) {
652
653 interface EditableFileSurfaceProps {
654 environmentId: EnvironmentId;
655 + projectId?: ProjectId | undefined;
656 cwd: string;
657 relativePath: string;
658 composerDraftTarget: ScopedThreadRef | DraftId;
@@ -668,6 +671,7 @@ interface FileSelectionOverride {
671
672 function EditableFileSurface({
673 environmentId,
674 + projectId,
675 cwd,
676 relativePath,
677 composerDraftTarget,
@@ -897,6 +901,14 @@ function EditableFileSurface({
901 {annotation.metadata.entries.map((entry) => (
902 <DiffCommentAnnotation
903 key={entry.id}
904 + environmentId={environmentId}
905 + projectId={projectId}
906 + ownerKey={JSON.stringify([
907 + environmentId,
908 + composerDraftTarget,
909 + relativePath,
910 + entry.id,
911 + ])}
912 kind={entry.kind}
913 rangeLabel={formatFileCommentRange(entry.startLine, entry.endLine)}
914 text={entry.text}
@@ -989,6 +1001,7 @@ export default function FilePreviewPanel({
1001 environmentId,
1002 cwd,
1003 projectName,
1004 + projectId,
1005 relativePath: requestedPath,
1006 attachment,
1007 threadRef,
@@ -1511,6 +1524,7 @@ export default function FilePreviewPanel({
1524 <EditableFileSurface
1525 key={`${relativePath}:${resolvedTheme}`}
1526 environmentId={environmentId}
1527 + projectId={projectId}
1528 cwd={cwd}
1529 relativePath={relativePath}
1530 composerDraftTarget={composerDraftTarget}
apps/web/src/components/pullRequest/PullRequestCodeTab.browser.test.tsx
+3 -4
@@ -168,7 +168,6 @@ function pagedContainer() {
168 return container;
169 }
170
171 -// Rows are virtualized, so a header far down is not rendered until the view scrolls to it.
171 async function expectFileHeaderShown(
172 screen: Awaited<ReturnType<typeof render>>,
173 container: HTMLElement,
@@ -177,10 +176,10 @@ async function expectFileHeaderShown(
176 const header = screen.getByText(path, { exact: true }).first();
177 await expect.element(header).toBeInTheDocument();
178 await vi.waitFor(() => {
180 - const top = header.element().getBoundingClientRect().top;
181 - const bounds = container.getBoundingClientRect();
179 + const bounds = container.querySelector(".diff-render-surface")!.getBoundingClientRect();
180 + const { top, bottom } = header.element().getBoundingClientRect();
181 expect(top).toBeGreaterThanOrEqual(bounds.top);
183 - expect(top).toBeLessThan(bounds.top + bounds.height / 2);
182 + expect(bottom).toBeLessThanOrEqual(bounds.bottom);
183 });
184 }
185
apps/web/src/components/pullRequest/PullRequestCodeTab.tsx
+3
@@ -1025,6 +1025,9 @@ function PullRequestCodeTab({
1025 ))}
1026 {annotation.metadata.draft && draft && canSelectLines ? (
1027 <DiffCommentAnnotation
1028 + environmentId={environmentId}
1029 + projectId={reference.projectId}
1030 + ownerKey={JSON.stringify([environmentId, reviewKey, draft])}
1031 kind="draft"
1032 rangeLabel={`${draft.path}:${getReviewPositionAnchor(draft.position).line}`}
1033 text=""
apps/web/src/components/pullRequest/PullRequestCommentForm.tsx
+38 -11
@@ -2,8 +2,9 @@ import { sourceControlClients } from "@t3tools/client-runtime/source-control-cli
2 import { useAtomCommand } from "~/state/use-atom-command";
3 import type { EnvironmentId, PullRequestDetailView, PullRequestRef } from "@t3tools/contracts";
4 import { SendIcon } from "lucide-react";
5 -import { useState, type RefObject } from "react";
5 +import { useEffect, useState, type RefObject } from "react";
6
7 +import { TextFieldSpeechControls, useTextFieldSpeech } from "~/speech/useTextFieldSpeech";
8 import { pullRequestEnvironment } from "~/state/pullRequests";
9
10 import { Button } from "../ui/button";
@@ -15,6 +16,7 @@ import { useInlineActionConfirm } from "./useInlineActionConfirm";
16
17 export function PullRequestCommentForm({
18 environmentId,
19 + active,
20 reference,
21 detail,
22 actionPending,
@@ -24,6 +26,8 @@ export function PullRequestCommentForm({
26 onClose,
27 }: {
28 environmentId: EnvironmentId;
29 + /** False while the form stays mounted but hidden; dictation is cancelled then. */
30 + active: boolean;
31 reference: PullRequestRef;
32 detail: PullRequestDetailView;
33 actionPending: boolean;
@@ -37,6 +41,21 @@ export function PullRequestCommentForm({
41 }) {
42 const [body, setBody] = useState("");
43 const [submitting, setSubmitting] = useState<"comment" | "close" | "reopen" | null>(null);
44 + const locked = submitting !== null || actionPending;
45 + const speech = useTextFieldSpeech({
46 + environmentId,
47 + projectId: reference.projectId,
48 + ownerKey: JSON.stringify(["pr-comment", environmentId, reference]),
49 + text: body,
50 + textareaRef,
51 + onTextChange: setBody,
52 + disabled: !active || locked,
53 + });
54 + const { cancel } = speech;
55 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- discards a recording when the hidden form can no longer be submitted
56 + useEffect(() => {
57 + if (!active) cancel();
58 + }, [active, cancel]);
59 const postComment = useAtomCommand(pullRequestEnvironment.comment, {
60 reportFailure: false,
61 });
@@ -52,8 +71,8 @@ export function PullRequestCommentForm({
71 ? ("reopen" as const)
72 : null;
73
55 - const submit = async (action: "comment" | "close" | "reopen") => {
56 - const trimmed = body.trim();
74 + const submit = async (action: "comment" | "close" | "reopen", text: string) => {
75 + const trimmed = text.trim();
76 if (trimmed.length === 0 || submitting !== null || actionPending) return;
77 setSubmitting(action);
78 if (action !== "comment") {
@@ -82,13 +101,18 @@ export function PullRequestCommentForm({
101 onClose();
102 onCommented();
103 };
104 + const submitAfterDictation = (action: "comment" | "close" | "reopen") =>
105 + speech.submitAfterDictation((text) => void submit(action, text));
106 + const submitDisabled =
107 + speech.preparing || (body.trim().length === 0 && !speech.blocksSubmission) || locked;
108
109 return (
110 <div className="space-y-2">
111 <Textarea
112 ref={textareaRef}
113 // Lock while posting because success clears the draft.
91 - disabled={submitting !== null || actionPending}
114 + disabled={locked}
115 + readOnly={speech.freezesEditor}
116 value={body}
117 rows={3}
118 placeholder="Leave a comment"
@@ -104,19 +128,22 @@ export function PullRequestCommentForm({
128 ) {
129 event.preventDefault();
130 event.stopPropagation();
107 - if (!event.repeat) void submit("comment");
131 + if (!event.repeat) submitAfterDictation("comment");
132 }
133 }}
134 />
111 - <div className="flex flex-wrap justify-end gap-2">
135 + <div className="flex flex-wrap items-center justify-end gap-2">
136 + <div className="mr-auto flex min-w-0 flex-wrap items-center gap-1">
137 + <TextFieldSpeechControls speech={speech} disabled={locked} />
138 + </div>
139 {followUpAction === null ? null : (
140 <Button
141 size="xs"
142 variant={followUpAction === "close" ? "destructive-outline" : "outline"}
116 - disabled={body.trim().length === 0 || submitting !== null || actionPending}
143 + disabled={submitDisabled}
144 {...(followUpAction === "close"
118 - ? inlineConfirm.props("close", () => void submit("close"))
119 - : { onClick: () => void submit("reopen") })}
145 + ? inlineConfirm.props("close", () => submitAfterDictation("close"))
146 + : { onClick: () => submitAfterDictation("reopen") })}
147 >
148 {followUpAction === "close" ? (
149 <PullRequestGlyph.closed className="size-3.5" />
@@ -147,8 +174,8 @@ export function PullRequestCommentForm({
174 <Button
175 size="xs"
176 variant="outline"
150 - disabled={body.trim().length === 0 || submitting !== null || actionPending}
151 - onClick={() => void submit("comment")}
177 + disabled={submitDisabled}
178 + onClick={() => submitAfterDictation("comment")}
179 >
180 <SendIcon className="size-3.5" />
181 {submitting === "comment" ? "Posting..." : "Comment"}
apps/web/src/components/pullRequest/PullRequestComposer.tsx
+2
@@ -151,6 +151,7 @@ export function PullRequestComposer({
151 {verdicts.length > 0 ? (
152 <div hidden={mode !== "review"}>
153 <PullRequestReviewForm
154 + active={open && mode === "review"}
155 environmentId={environmentId}
156 reference={reference}
157 term={term}
@@ -171,6 +172,7 @@ export function PullRequestComposer({
172 {canComment ? (
173 <div hidden={mode !== "comment"}>
174 <PullRequestCommentForm
175 + active={open && mode === "comment"}
176 environmentId={environmentId}
177 reference={reference}
178 detail={detail}
apps/web/src/components/pullRequest/PullRequestReviewAnnotation.tsx
+55 -33
@@ -14,6 +14,7 @@ import { settlePromise, squashAtomCommandFailure } from "@t3tools/client-runtime
14
15 import { RelativeTime } from "~/components/RelativeTime";
16 import { cn } from "~/lib/utils";
17 +import { SpeechTextField, TextFieldSpeechControls } from "~/speech/useTextFieldSpeech";
18
19 import { Button } from "../ui/button";
20 import { MorphIcon } from "~/components/MorphIcon";
@@ -36,6 +37,7 @@ const CARD_CLASS =
37 function submitKeys(input: {
38 readonly value: string;
39 readonly pending: boolean;
40 + readonly allowEmpty?: boolean;
41 readonly onSubmit: () => void;
42 readonly onCancel?: (() => void) | undefined;
43 }) {
@@ -44,7 +46,7 @@ function submitKeys(input: {
46 event.preventDefault();
47 input.onCancel();
48 }
47 - if (isCommentSubmitShortcut(event, input.value, input.pending)) {
49 + if (isCommentSubmitShortcut(event, input.value, input.pending, input.allowEmpty)) {
50 event.preventDefault();
51 input.onSubmit();
52 }
@@ -146,6 +148,7 @@ export function ReviewThreadCard({
148 const [editingId, setEditingId] = useState<string | null>(null);
149 const [savingEdit, setSavingEdit] = useState(false);
150 const sendingRef = useRef(false);
151 + const replyRef = useRef<HTMLTextAreaElement>(null);
152 const [loadedPage, setLoadedPage] = useState<
153 (PullRequestThreadCommentsResult & { readonly threadId: string }) | null
154 >(null);
@@ -174,8 +177,8 @@ export function ReviewThreadCard({
177 }
178 };
179
177 - const send = async () => {
178 - const trimmed = reply.trim();
180 + const send = async (text: string) => {
181 + const trimmed = text.trim();
182 if (trimmed.length === 0 || pending || sendingRef.current) return;
183 sendingRef.current = true;
184 const result = await settlePromise(() => onReply(trimmed));
@@ -323,36 +326,55 @@ export function ReviewThreadCard({
326
327 {canReply ? (
328 replying ? (
326 - <div className="mt-2">
327 - <Textarea
328 - autoFocus
329 - size="sm"
330 - value={reply}
331 - placeholder="Reply"
332 - aria-label="Reply to this conversation"
333 - onChange={(event) => setReply(event.target.value)}
334 - onKeyDown={(event) =>
335 - submitKeys({
336 - value: reply,
337 - pending,
338 - onSubmit: () => void send(),
339 - onCancel: () => setReplying(false),
340 - })(event)
341 - }
342 - />
343 - <div className="mt-2 flex justify-end gap-2">
344 - <Button size="xs" variant="ghost" onClick={() => setReplying(false)}>
345 - Cancel
346 - </Button>
347 - <Button
348 - size="xs"
349 - disabled={pending || reply.trim().length === 0}
350 - onClick={() => void send()}
351 - >
352 - Reply
353 - </Button>
354 - </div>
355 - </div>
329 + <SpeechTextField
330 + environmentId={environmentId}
331 + projectId={reference.projectId}
332 + ownerKey={JSON.stringify(["pr-reply", environmentId, reference, thread.id])}
333 + text={reply}
334 + textareaRef={replyRef}
335 + onTextChange={setReply}
336 + disabled={pending}
337 + >
338 + {(speech) => (
339 + <div className="mt-2">
340 + <Textarea
341 + ref={replyRef}
342 + readOnly={speech.freezesEditor}
343 + autoFocus
344 + size="sm"
345 + value={reply}
346 + placeholder="Reply"
347 + aria-label="Reply to this conversation"
348 + onChange={(event) => setReply(event.target.value)}
349 + onKeyDown={submitKeys({
350 + value: reply,
351 + pending,
352 + allowEmpty: speech.blocksSubmission,
353 + onSubmit: () => speech.submitAfterDictation((text) => void send(text)),
354 + onCancel: () =>
355 + speech.state.phase === "idle" ? setReplying(false) : speech.cancel(),
356 + })}
357 + />
358 + <div className="mt-2 flex flex-wrap items-center justify-end gap-2">
359 + <TextFieldSpeechControls speech={speech} disabled={pending} />
360 + <Button size="xs" variant="ghost" onClick={() => setReplying(false)}>
361 + Cancel
362 + </Button>
363 + <Button
364 + size="xs"
365 + disabled={
366 + speech.preparing ||
367 + pending ||
368 + (!speech.blocksSubmission && reply.trim().length === 0)
369 + }
370 + onClick={() => speech.submitAfterDictation((text) => void send(text))}
371 + >
372 + Reply
373 + </Button>
374 + </div>
375 + </div>
376 + )}
377 + </SpeechTextField>
378 ) : (
379 <Button size="xs" variant="ghost" className="mt-2" onClick={() => setReplying(true)}>
380 Reply
apps/web/src/components/pullRequest/PullRequestReviewForm.browser.test.tsx new
+54
@@ -0,0 +1,54 @@
1 +import { createRef } from "react";
2 +import { EnvironmentId, ProjectId } from "@t3tools/contracts";
3 +import { expect, it, vi } from "vite-plus/test";
4 +import { act, render } from "../../../test/render";
5 +import { PullRequestReviewForm } from "./PullRequestReviewForm";
6 +import { pullRequestReviewKey, usePullRequestReviewStore } from "./pullRequestReviewStore";
7 +
8 +const mocks = vi.hoisted(() => ({
9 + submit: vi.fn().mockResolvedValue({ _tag: "Success" }),
10 + queue: vi.fn<(submit: (text: string) => void) => void>(),
11 +}));
12 +vi.mock("~/speech/useTextFieldSpeech", () => ({
13 + TextFieldSpeechControls: () => null,
14 + useTextFieldSpeech: () => ({ blocksSubmission: true, submitAfterDictation: mocks.queue }),
15 +}));
16 +vi.mock("~/state/use-atom-command", () => ({ useAtomCommand: () => mocks.submit }));
17 +vi.mock("~/state/pullRequests", () => ({ pullRequestEnvironment: { submitReview: {} } }));
18 +
19 +it("submits only the line comments still in the draft when dictation finishes", async () => {
20 + mocks.submit.mockClear();
21 + mocks.queue.mockClear();
22 + const reference = { projectId: ProjectId.make("test"), repository: "test/repo", number: 1 };
23 + const key = pullRequestReviewKey(reference);
24 + const store = usePullRequestReviewStore.getState();
25 + store.clear(key);
26 + store.addComment(key, {
27 + id: "removed",
28 + body: "Do not post",
29 + path: "a.ts",
30 + position: { kind: "added", newLine: 1 },
31 + });
32 + await using screen = await render(
33 + <PullRequestReviewForm
34 + active
35 + environmentId={EnvironmentId.make("test")}
36 + reference={reference}
37 + term="pull request"
38 + verdicts={["comment"]}
39 + requestChangesSummaryRequired={false}
40 + textareaRef={createRef()}
41 + pending={false}
42 + onPendingChange={vi.fn()}
43 + onSubmitted={vi.fn()}
44 + />,
45 + );
46 + await screen.getByRole("button", { name: "Submit review" }).click();
47 + await act(() => store.clear(key));
48 + await act(() => mocks.queue.mock.calls[0]![0]("Dictated summary"));
49 + expect(mocks.submit).toHaveBeenCalledWith(
50 + expect.objectContaining({
51 + input: expect.objectContaining({ body: "Dictated summary", comments: [] }),
52 + }),
53 + );
54 +});
apps/web/src/components/pullRequest/PullRequestReviewForm.tsx
+50 -12
@@ -2,11 +2,13 @@ import { useAtomCommand } from "~/state/use-atom-command";
2 import { capitalizeTerm } from "@t3tools/client-runtime/source-control-clients";
3 import type { EnvironmentId, PullRequestRef, PullRequestReviewVerdict } from "@t3tools/contracts";
4 import { CheckIcon, MessageSquareIcon, XCircleIcon } from "lucide-react";
5 -import { useState, type ReactNode, type RefObject } from "react";
5 +import { useEffect, useState, type ReactNode, type RefObject } from "react";
6
7 +import { TextFieldSpeechControls, useTextFieldSpeech } from "~/speech/useTextFieldSpeech";
8 import { pullRequestEnvironment } from "~/state/pullRequests";
9 import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime";
10
11 +import { isCommentSubmitShortcut } from "../diffs/commentSubmitShortcut";
12 import { Button } from "../ui/button";
13 import { Select, SelectItem, SelectPopup, SelectTrigger } from "../ui/select";
14 import { Textarea } from "../ui/textarea";
@@ -46,6 +48,7 @@ const VERDICTS: ReadonlyArray<{
48
49 export function PullRequestReviewForm({
50 environmentId,
51 + active,
52 reference,
53 term,
54 verdicts,
@@ -56,6 +59,8 @@ export function PullRequestReviewForm({
59 onSubmitted,
60 }: {
61 environmentId: EnvironmentId;
62 + /** False while the form stays mounted but hidden; dictation is cancelled then. */
63 + active: boolean;
64 reference: PullRequestRef;
65 /** The host's word for a change request, as in "pull request". */
66 term: string;
@@ -72,6 +77,20 @@ export function PullRequestReviewForm({
77 const body = usePullRequestReviewStore((store) => store.summaries[reviewKey] ?? "");
78 const removeComments = usePullRequestReviewStore((store) => store.removeComments);
79 const setSummary = usePullRequestReviewStore((store) => store.setSummary);
80 + const speech = useTextFieldSpeech({
81 + environmentId,
82 + projectId: reference.projectId,
83 + ownerKey: JSON.stringify(["pr-review", environmentId, reviewKey]),
84 + text: body,
85 + textareaRef,
86 + onTextChange: (text) => setSummary(reviewKey, text),
87 + disabled: !active || pending,
88 + });
89 + const { cancel } = speech;
90 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- discards a recording when the hidden form can no longer be submitted
91 + useEffect(() => {
92 + if (!active) cancel();
93 + }, [active, cancel]);
94 const clearSummary = usePullRequestReviewStore((store) => store.clearSummary);
95 const submitReview = useAtomCommand(pullRequestEnvironment.submitReview, {
96 reportFailure: false,
@@ -81,10 +100,10 @@ export function PullRequestReviewForm({
100 const selectedVerdict =
101 offered.find((verdict) => verdict.value === requestedVerdict) ?? offered[0];
102
84 - const submit = async (verdict: (typeof VERDICTS)[number]) => {
85 - if (pending) return;
86 - const submittedBody = body;
87 - const submittedComments = comments;
103 + const submit = async (verdict: (typeof VERDICTS)[number], text: string) => {
104 + const submittedComments = usePullRequestReviewStore.getState().drafts[reviewKey] ?? [];
105 + if (pending || !canSubmit(verdict.value, text, submittedComments.length)) return;
106 + const submittedBody = text;
107 onPendingChange(true);
108 const result = await submitReview({
109 environmentId,
@@ -118,12 +137,12 @@ export function PullRequestReviewForm({
137 };
138
139 // Forgejo requires a summary when requesting changes, even with inline comments.
121 - const canSubmit = (verdict: PullRequestReviewVerdict, summary: string) =>
140 + const canSubmit = (verdict: PullRequestReviewVerdict, summary: string, commentCount: number) =>
141 verdict === "request-changes" && requestChangesSummaryRequired
142 ? summary.trim().length > 0
124 - : verdict === "approve" || summary.trim().length > 0 || comments.length > 0;
143 + : verdict === "approve" || summary.trim().length > 0 || commentCount > 0;
144 const placeholder =
126 - selectedVerdict === undefined || canSubmit(selectedVerdict.value, "")
145 + selectedVerdict === undefined || canSubmit(selectedVerdict.value, "", comments.length)
146 ? "Summarize your review (optional)"
147 : selectedVerdict.value === "request-changes" && requestChangesSummaryRequired
148 ? "Summarize your review (required to request changes)"
@@ -134,12 +153,27 @@ export function PullRequestReviewForm({
153 <Textarea
154 ref={textareaRef}
155 rows={3}
156 + readOnly={speech.freezesEditor}
157 value={body}
158 placeholder={placeholder}
159 aria-label="Review summary"
160 onChange={(event) => setSummary(reviewKey, event.target.value)}
161 + onKeyDown={(event) => {
162 + if (event.nativeEvent.isComposing || event.keyCode === 229 || event.repeat) return;
163 + if (
164 + selectedVerdict !== undefined &&
165 + isCommentSubmitShortcut(event, body, pending || speech.preparing, true)
166 + ) {
167 + event.preventDefault();
168 + event.stopPropagation();
169 + speech.submitAfterDictation((text) => void submit(selectedVerdict, text));
170 + }
171 + }}
172 />
142 - <div className="mt-2 flex justify-end gap-2">
173 + <div className="mt-2 flex flex-wrap items-center justify-end gap-2">
174 + <div className="mr-auto flex min-w-0 flex-wrap items-center gap-1">
175 + <TextFieldSpeechControls speech={speech} disabled={pending} />
176 + </div>
177 {offered.length > 1 ? (
178 <Select
179 value={selectedVerdict?.value ?? null}
@@ -148,7 +182,7 @@ export function PullRequestReviewForm({
182 if (value !== null) setRequestedVerdict(value);
183 }}
184 >
151 - <SelectTrigger size="xs" className="mr-auto w-auto min-w-0" aria-label="Review verdict">
185 + <SelectTrigger size="xs" className="w-auto min-w-0" aria-label="Review verdict">
186 <span className="flex items-center gap-1.5">
187 {selectedVerdict?.icon}
188 {selectedVerdict?.label}
@@ -169,10 +203,14 @@ export function PullRequestReviewForm({
203 <Button
204 size="xs"
205 disabled={
172 - pending || selectedVerdict === undefined || !canSubmit(selectedVerdict.value, body)
206 + speech.preparing ||
207 + pending ||
208 + selectedVerdict === undefined ||
209 + (!canSubmit(selectedVerdict.value, body, comments.length) && !speech.blocksSubmission)
210 }
211 onClick={() => {
175 - if (selectedVerdict !== undefined) void submit(selectedVerdict);
212 + if (selectedVerdict !== undefined)
213 + speech.submitAfterDictation((text) => void submit(selectedVerdict, text));
214 }}
215 >
216 {pending ? "Submitting..." : "Submit review"}
apps/web/src/components/settings/KeybindingsSettings.logic.ts
+1
@@ -373,6 +373,7 @@ export function commandLabel(command: KeybindingCommand): string {
373 if (command === "composer.sendAlternate") return "Composer: Opposite Queue or Steer Action";
374 if (command === "composer.sendBackground") return "Composer: Start in Background";
375 if (command === "composer.sendAndNewThread") return "Composer: Send and Start New Thread";
376 + if (command === "composer.dictation") return "Composer: Start or Finish Dictation";
377 if (command === "thread.steerQueuedMessage") return "Queue: Send First Queued Message";
378 if (command === "thread.editQueuedMessage") return "Queue: Edit Last Queued Message";
379 if (command === "thread.copyReference") return "Pull Request: Copy Link or Thread";
apps/web/src/components/settings/MicrophoneTest.tsx new
+350
@@ -0,0 +1,350 @@
1 +import { MICROPHONE_PERMISSION_ERROR } from "@t3tools/client-runtime/voice-input";
2 +import { CheckIcon, MicIcon, PauseIcon, PlayIcon, RotateCcwIcon, XIcon } from "lucide-react";
3 +import { useEffect, useRef, useState } from "react";
4 +
5 +import { useStableCallback } from "~/hooks/useStableCallback";
6 +import workletUrl from "../../speech/pcmWorklet.ts?worker&url";
7 +import { VoiceInputErrorPill } from "../chat/VoiceInputErrorPill";
8 +import { VoiceInputPill, VoiceWaveform } from "../chat/VoiceInputPill";
9 +import { Button } from "../ui/button";
10 +
11 +const LOOPBACK_SECONDS = 10;
12 +const formatTime = (seconds: number) =>
13 + `${Math.floor(seconds / 60)}:${String(Math.floor(seconds % 60)).padStart(2, "0")}`;
14 +
15 +type Capture = {
16 + stream?: MediaStream;
17 + recorder?: MediaRecorder;
18 + context?: AudioContext;
19 + worklet?: AudioWorkletNode;
20 + timer?: ReturnType<typeof setTimeout>;
21 + elapsedTimer?: ReturnType<typeof setInterval>;
22 + startedAt?: number;
23 +};
24 +
25 +/**
26 + * Records a few seconds and plays them back in the browser. The audio never leaves this
27 + * device: no environment, transcriber, or provider is involved, so a silent result points
28 + * at the microphone rather than at transcription.
29 + */
30 +export function MicrophoneTest({ microphoneId }: { microphoneId: string }) {
31 + const [phase, setPhase] = useState<"idle" | "starting" | "recording" | "ready">("idle");
32 + const [recording, setRecording] = useState<{ url: string; duration: number } | null>(null);
33 + const [error, setError] = useState<string | null>(null);
34 + const [level, setLevel] = useState(0);
35 + const [elapsed, setElapsed] = useState(0);
36 + const capture = useRef<Capture | null>(null);
37 + const recordingUrl = useRef<string | null>(null);
38 +
39 + const releaseCapture = useStableCallback(() => {
40 + const current = capture.current;
41 + if (!current) return;
42 + capture.current = null;
43 + clearTimeout(current.timer);
44 + clearInterval(current.elapsedTimer);
45 + current.worklet?.disconnect();
46 + current.worklet?.port.close();
47 + if (current.recorder && current.recorder.state !== "inactive") current.recorder.stop();
48 + current.stream?.getTracks().forEach((track) => track.stop());
49 + void current.context?.close().catch(() => undefined);
50 + });
51 + const discard = () => {
52 + if (recordingUrl.current) URL.revokeObjectURL(recordingUrl.current);
53 + recordingUrl.current = null;
54 + setRecording(null);
55 + setPhase("idle");
56 + };
57 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- releases the microphone and the recording's object URL on unmount
58 + useEffect(
59 + () => () => {
60 + releaseCapture();
61 + if (recordingUrl.current) URL.revokeObjectURL(recordingUrl.current);
62 + },
63 + [releaseCapture],
64 + );
65 +
66 + const start = async () => {
67 + releaseCapture();
68 + discard();
69 + setError(null);
70 + setLevel(0);
71 + setElapsed(0);
72 + setPhase("starting");
73 + if (
74 + !navigator.mediaDevices?.getUserMedia ||
75 + typeof MediaRecorder === "undefined" ||
76 + typeof AudioWorkletNode === "undefined"
77 + ) {
78 + setError("Microphone testing is unavailable in this browser. Use HTTPS or the desktop app.");
79 + setPhase("idle");
80 + return;
81 + }
82 + const current: Capture = {};
83 + capture.current = current;
84 + try {
85 + const stream = await navigator.mediaDevices.getUserMedia({
86 + audio: microphoneId ? { deviceId: { exact: microphoneId } } : true,
87 + });
88 + if (capture.current !== current) {
89 + stream.getTracks().forEach((track) => track.stop());
90 + return;
91 + }
92 + current.stream = stream;
93 + const recorder = new MediaRecorder(stream);
94 + current.recorder = recorder;
95 + const chunks: Blob[] = [];
96 + recorder.addEventListener("dataavailable", ({ data }) => {
97 + if (capture.current === current && data.size) chunks.push(data);
98 + });
99 + recorder.addEventListener("stop", () => {
100 + if (capture.current !== current) return;
101 + const blob = new Blob(chunks, { type: recorder.mimeType });
102 + const duration =
103 + current.startedAt === undefined ? 0 : (performance.now() - current.startedAt) / 1_000;
104 + releaseCapture();
105 + setLevel(0);
106 + if (!blob.size) {
107 + setError("No audio was recorded. Try again.");
108 + setPhase("idle");
109 + return;
110 + }
111 + recordingUrl.current = URL.createObjectURL(blob);
112 + setRecording({ url: recordingUrl.current, duration });
113 + setPhase("ready");
114 + });
115 + const onRecordingError = () => {
116 + if (capture.current !== current) return;
117 + releaseCapture();
118 + setLevel(0);
119 + setError("Could not record microphone audio. Try again.");
120 + setPhase("idle");
121 + };
122 + recorder.addEventListener("error", onRecordingError);
123 + for (const track of stream.getAudioTracks()) {
124 + track.addEventListener(
125 + "ended",
126 + () => {
127 + if (capture.current !== current) return;
128 + setError("The microphone disconnected.");
129 + if (recorder.state !== "inactive") recorder.stop();
130 + },
131 + { once: true },
132 + );
133 + }
134 + const context = new AudioContext();
135 + current.context = context;
136 + await context.audioWorklet.addModule(workletUrl);
137 + if (capture.current !== current) return;
138 + const worklet = new AudioWorkletNode(context, "t3-pcm-capture", {
139 + channelCount: 1,
140 + processorOptions: { meterOnly: true },
141 + });
142 + current.worklet = worklet;
143 + worklet.port.onmessage = ({ data }: MessageEvent<number | "stopped">) => {
144 + if (capture.current === current && typeof data === "number") setLevel(data);
145 + };
146 + worklet.onprocessorerror = onRecordingError;
147 + context.createMediaStreamSource(stream).connect(worklet);
148 + worklet.connect(context.destination);
149 + await context.resume();
150 + if (capture.current !== current) return;
151 + const startedAt = performance.now();
152 + current.startedAt = startedAt;
153 + current.elapsedTimer = setInterval(() => {
154 + setElapsed(Math.min(LOOPBACK_SECONDS, Math.floor((performance.now() - startedAt) / 1_000)));
155 + }, 100);
156 + recorder.start();
157 + worklet.port.postMessage("start", []);
158 + setPhase("recording");
159 + current.timer = setTimeout(() => {
160 + if (recorder.state !== "inactive") recorder.stop();
161 + }, LOOPBACK_SECONDS * 1_000);
162 + } catch (cause) {
163 + if (capture.current !== current) return;
164 + releaseCapture();
165 + setPhase("idle");
166 + setError(
167 + typeof cause === "object" &&
168 + cause !== null &&
169 + "name" in cause &&
170 + cause.name === "NotAllowedError"
171 + ? MICROPHONE_PERMISSION_ERROR
172 + : cause instanceof Error
173 + ? cause.message
174 + : "Could not access the microphone.",
175 + );
176 + }
177 + };
178 + const cancel = () => {
179 + releaseCapture();
180 + discard();
181 + setError(null);
182 + };
183 +
184 + if (error) {
185 + return (
186 + <VoiceInputErrorPill
187 + error={error}
188 + errorAction="retry"
189 + onRetry={() => void start()}
190 + onDismiss={cancel}
191 + />
192 + );
193 + }
194 + if (phase === "idle") {
195 + return (
196 + <Button size="sm" variant="outline" onClick={() => void start()}>
197 + <MicIcon />
198 + Test microphone
199 + </Button>
200 + );
201 + }
202 + if (phase === "ready" && recording) {
203 + return (
204 + <MicrophonePlayback
205 + key={recording.url}
206 + src={recording.url}
207 + duration={recording.duration}
208 + onRetry={() => void start()}
209 + onDone={cancel}
210 + />
211 + );
212 + }
213 + return (
214 + <VoiceInputPill>
215 + <Button
216 + size="icon-sm-pill"
217 + variant="ghost"
218 + aria-label="Cancel microphone test"
219 + onClick={cancel}
220 + >
221 + <XIcon />
222 + </Button>
223 + {phase === "recording" ? (
224 + <>
225 + <VoiceWaveform level={level} />
226 + <span aria-hidden className="size-1.5 shrink-0 rounded-full bg-primary" />
227 + <span
228 + className="shrink-0 text-xs tabular-nums text-muted-foreground"
229 + aria-label="Recording time"
230 + >
231 + {formatTime(elapsed)} / {formatTime(LOOPBACK_SECONDS)}
232 + </span>
233 + <meter
234 + aria-label="Microphone input level"
235 + min={0}
236 + max={1}
237 + value={level}
238 + className="sr-only"
239 + />
240 + </>
241 + ) : (
242 + <span role="status" className="min-w-0 flex-1 truncate text-xs text-muted-foreground">
243 + Opening microphone…
244 + </span>
245 + )}
246 + <Button
247 + size="icon-sm-pill"
248 + aria-label="Stop recording"
249 + disabled={phase !== "recording"}
250 + onClick={() => {
251 + const recorder = capture.current?.recorder;
252 + if (recorder && recorder.state !== "inactive") recorder.stop();
253 + }}
254 + >
255 + <CheckIcon />
256 + </Button>
257 + </VoiceInputPill>
258 + );
259 +}
260 +
261 +function MicrophonePlayback({
262 + src,
263 + duration,
264 + onRetry,
265 + onDone,
266 +}: {
267 + src: string;
268 + duration: number;
269 + onRetry(): void;
270 + onDone(): void;
271 +}) {
272 + const audio = useRef<HTMLAudioElement>(null);
273 + const [playing, setPlaying] = useState(false);
274 + const [position, setPosition] = useState(0);
275 + const [failed, setFailed] = useState(false);
276 + const progress = duration > 0 ? Math.min(position / duration, 1) * 100 : 0;
277 +
278 + const togglePlayback = async () => {
279 + const player = audio.current;
280 + if (!player) return;
281 + if (!player.paused) {
282 + player.pause();
283 + return;
284 + }
285 + setFailed(false);
286 + if (player.ended) player.currentTime = 0;
287 + try {
288 + await player.play();
289 + } catch {
290 + setFailed(true);
291 + }
292 + };
293 +
294 + return (
295 + <div className="flex flex-col gap-1">
296 + <audio
297 + ref={audio}
298 + src={src}
299 + preload="metadata"
300 + onPlay={() => setPlaying(true)}
301 + onPause={() => setPlaying(false)}
302 + onEnded={() => setPlaying(false)}
303 + onTimeUpdate={() => setPosition(audio.current?.currentTime ?? 0)}
304 + onError={() => setFailed(true)}
305 + />
306 + <VoiceInputPill>
307 + <Button
308 + size="icon-sm-pill"
309 + variant="ghost"
310 + aria-label={playing ? "Pause recording" : "Play recording"}
311 + onClick={() => void togglePlayback()}
312 + >
313 + {playing ? <PauseIcon /> : <PlayIcon />}
314 + </Button>
315 + <input
316 + type="range"
317 + aria-label="Recording playback position"
318 + min={0}
319 + max={duration}
320 + step={0.01}
321 + value={Math.min(position, duration)}
322 + style={{
323 + background: `linear-gradient(to right, var(--primary) ${progress}%, var(--muted) ${progress}%)`,
324 + }}
325 + disabled={duration <= 0}
326 + onChange={(event) => {
327 + const next = Number(event.target.value);
328 + if (audio.current) audio.current.currentTime = next;
329 + setPosition(next);
330 + }}
331 + className="h-1 min-w-0 flex-1 cursor-pointer appearance-none rounded-full bg-muted accent-primary focus-visible:outline-2 focus-visible:-outline-offset-2 focus-visible:outline-ring [&::-webkit-slider-thumb]:size-2.5 [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-primary [&::-moz-range-thumb]:size-2.5 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:border-0 [&::-moz-range-thumb]:bg-primary"
332 + />
333 + <span className="shrink-0 text-xs tabular-nums text-muted-foreground">
334 + {formatTime(position)}
335 + </span>
336 + <Button size="icon-sm-pill" variant="ghost" aria-label="Record again" onClick={onRetry}>
337 + <RotateCcwIcon />
338 + </Button>
339 + <Button size="icon-sm-pill" aria-label="Done" onClick={onDone}>
340 + <CheckIcon />
341 + </Button>
342 + </VoiceInputPill>
343 + {failed ? (
344 + <p role="alert" className="text-xs text-destructive">
345 + Could not play this recording. Try recording again.
346 + </p>
347 + ) : null}
348 + </div>
349 + );
350 +}
apps/web/src/components/settings/SettingsSidebarNav.tsx
+2
@@ -23,6 +23,7 @@ import {
23 PanelsTopLeftIcon,
24 KeyboardIcon,
25 KeyRoundIcon,
26 + MicIcon,
27 PaletteIcon,
28 SearchIcon,
29 ServerIcon,
@@ -88,6 +89,7 @@ const SETTINGS_SECTION_ICONS: Readonly<
89 > = {
90 "/settings/general": Settings2Icon,
91 "/settings/diagnostics": WrenchIcon,
92 + "/settings/voice": MicIcon,
93 "/settings/appearance": PaletteIcon,
94 "/settings/chat": MessageSquareTextIcon,
95 "/settings/diffs": FileDiffIcon,
apps/web/src/components/settings/SpeechLanguageSelect.tsx new
+61
@@ -0,0 +1,61 @@
1 +import type { EnvironmentSpeechStatus } from "@t3tools/contracts";
2 +import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
3 +import { updateClientSettings, useClientSettings } from "../../hooks/useSettings";
4 +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
5 +
6 +const languageNames = new Intl.DisplayNames(["en"], { type: "language" });
7 +export const languageLabel = (code: string) => languageNames.of(code) ?? code;
8 +
9 +type ModelLanguages = Pick<
10 + Extract<EnvironmentSpeechStatus, { supported: true }>,
11 + "languages" | "supportsLanguageDetection"
12 +>;
13 +
14 +export function SpeechLanguageSelect({ model }: { readonly model: ModelLanguages | null }) {
15 + const preference = useClientSettings((settings) => settings.speechLanguage);
16 + const languages = model?.languages ?? [];
17 + const resolved = model
18 + ? effectiveSpeechLanguage(
19 + { languages, supportsLanguageDetection: model.supportsLanguageDetection ?? false },
20 + preference,
21 + )
22 + : preference;
23 + const effectiveLabel = resolved === "auto" ? "Auto-detect" : languageLabel(resolved);
24 + const unavailable =
25 + preference !== "auto" && preference !== resolved
26 + ? `${languageLabel(preference)} unavailable`
27 + : null;
28 + const label = unavailable
29 + ? `${effectiveLabel} (${unavailable})`
30 + : preference === "auto" && resolved !== "auto"
31 + ? `Auto (${effectiveLabel})`
32 + : effectiveLabel;
33 + return (
34 + <Select
35 + value={preference}
36 + disabled={!model}
37 + onValueChange={(value) => {
38 + if (value) void updateClientSettings({ speechLanguage: value });
39 + }}
40 + >
41 + <SelectTrigger size="sm" aria-label="Transcription language" className="max-w-64">
42 + <SelectValue>{label}</SelectValue>
43 + </SelectTrigger>
44 + <SelectPopup align="end" alignItemWithTrigger={false}>
45 + <SelectItem value="auto">Auto</SelectItem>
46 + {unavailable ? (
47 + <SelectItem value={preference} disabled>
48 + {unavailable}
49 + </SelectItem>
50 + ) : null}
51 + {languages
52 + .toSorted((left, right) => languageLabel(left).localeCompare(languageLabel(right)))
53 + .map((code) => (
54 + <SelectItem key={code} value={code}>
55 + {languageLabel(code)}
56 + </SelectItem>
57 + ))}
58 + </SelectPopup>
59 + </Select>
60 + );
61 +}
apps/web/src/components/settings/SpeechModelComparison.tsx new
+126
@@ -0,0 +1,126 @@
1 +import type { EnvironmentSpeechModel } from "@t3tools/contracts";
2 +import { CheckIcon } from "lucide-react";
3 +
4 +import { cn } from "../../lib/utils";
5 +import { languageLabel } from "./SpeechLanguageSelect";
6 +
7 +const commonLanguages = [
8 + "en",
9 + "zh",
10 + "hi",
11 + "es",
12 + "fr",
13 + "bn",
14 + "pt",
15 + "id",
16 + "ur",
17 + "ru",
18 + "de",
19 + "ja",
20 + "mr",
21 + "vi",
22 + "te",
23 + "sw",
24 + "ha",
25 + "tr",
26 + "tl",
27 + "ta",
28 + "jw",
29 + "ko",
30 + "am",
31 + "th",
32 + "it",
33 + "gu",
34 + "kn",
35 + "yo",
36 +];
37 +
38 +export function orderSpeechLanguages(languages: readonly string[], preferred: readonly string[]) {
39 + const order = [...new Set([...preferred, ...commonLanguages])];
40 + const rank = new Map(order.map((code, index) => [code, index]));
41 + return languages.toSorted(
42 + (left, right) =>
43 + (rank.get(left) ?? order.length) - (rank.get(right) ?? order.length) ||
44 + languageLabel(left).localeCompare(languageLabel(right)),
45 + );
46 +}
47 +
48 +export function LanguageRow({
49 + label,
50 + selected,
51 + onClick,
52 +}: {
53 + label: string;
54 + selected: boolean;
55 + onClick: () => void;
56 +}) {
57 + return (
58 + <button
59 + type="button"
60 + aria-pressed={selected}
61 + onClick={onClick}
62 + className={cn(
63 + "flex shrink-0 cursor-pointer items-center justify-between gap-2 rounded-md px-2.5 py-1.5 text-left text-sm outline-none hover:bg-muted/50 focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-ring",
64 + selected && "bg-muted font-medium",
65 + )}
66 + >
67 + {label}
68 + {selected && <CheckIcon className="size-3.5 shrink-0" />}
69 + </button>
70 + );
71 +}
72 +
73 +export function ModelBenchmarks({
74 + model,
75 + languages,
76 + facts,
77 +}: {
78 + model: EnvironmentSpeechModel;
79 + languages: readonly string[];
80 + /** Plain facts, such as language count and size, shown after the scores. */
81 + facts: readonly string[];
82 +}) {
83 + const data = model.benchmark;
84 + const visible = !data
85 + ? []
86 + : languages.length
87 + ? languages
88 + : [model.languages.find((code) => data.accuracy[code]) ?? model.languages[0]].filter(
89 + (code): code is string => code !== undefined,
90 + );
91 + return (
92 + <div className="mt-1 flex flex-wrap gap-x-3 gap-y-1 text-xs text-muted-foreground tabular-nums">
93 + {visible.map((code) => {
94 + const score = data?.accuracy[code];
95 + return (
96 + <span key={code}>
97 + {languageLabel(code)}:{" "}
98 + {score ? (
99 + <>
100 + <strong className="font-medium text-foreground">{score.grade}</strong> (
101 + {score.errorPercent}% error)
102 + </>
103 + ) : (
104 + "not benchmarked"
105 + )}
106 + </span>
107 + );
108 + })}
109 + {data ? (
110 + <span>
111 + Speed:{" "}
112 + {data.secondsPer30s === null ? (
113 + "not benchmarked"
114 + ) : (
115 + <>
116 + <strong className="font-medium text-foreground">{data.secondsPer30s} s</strong> / 30 s
117 + </>
118 + )}
119 + </span>
120 + ) : null}
121 + {facts.map((fact) => (
122 + <span key={fact}>{fact}</span>
123 + ))}
124 + </div>
125 + );
126 +}
apps/web/src/components/settings/VoiceSettingsPanel.browser.test.tsx new
+319
@@ -0,0 +1,319 @@
1 +import { PrimaryConnectionTarget } from "@t3tools/client-runtime/connection";
2 +import {
3 + EnvironmentId,
4 + type EnvironmentSpeechModel,
5 + type EnvironmentSpeechStatus,
6 +} from "@t3tools/contracts";
7 +import { expect, it, vi } from "vite-plus/test";
8 +import { userEvent } from "vite-plus/test/browser";
9 +import { render } from "../../../test/render";
10 +import { __setClientSettingsForTests, getClientSettings } from "../../hooks/useSettings";
11 +import {
12 + useEnvironmentSpeechInput,
13 + useVoiceTranscriptionDestination,
14 +} from "../../speech/useEnvironmentSpeechInput";
15 +import { chooseAudioDestination, chooseTranscriptCleanup } from "../../speech/voiceConsent";
16 +import { toastManager } from "../ui/toast";
17 +import { SettingsScopeProvider } from "./SettingsScopeContext";
18 +import { SpeechLanguageSelect } from "./SpeechLanguageSelect";
19 +import { VoiceSettingsPanel } from "./VoiceSettingsPanel";
20 +
21 +vi.mock(import("../../speech/useEnvironmentSpeechInput"), { spy: true });
22 +
23 +it("preserves the model test transcript when its model is filtered out", async () => {
24 + const target = new PrimaryConnectionTarget({
25 + environmentId: EnvironmentId.make("voice-filter"),
26 + label: "Test environment",
27 + httpBaseUrl: window.location.origin,
28 + wsBaseUrl: "ws://localhost",
29 + });
30 + const prepared = { ...target, target, socketUrl: target.wsBaseUrl, httpAuthorization: null };
31 + const model: EnvironmentSpeechModel = {
32 + id: "tiny",
33 + name: "Tiny",
34 + description: "",
35 + size: 100,
36 + languages: ["en"],
37 + accuracy: 1,
38 + speed: 1,
39 + recommended: false,
40 + supportsStreaming: false,
41 + supportsTranslation: false,
42 + supportsLanguageDetection: false,
43 + active: true,
44 + state: "installed",
45 + };
46 + const status = {
47 + supported: true,
48 + state: "ready",
49 + modelId: model.id,
50 + model: model.name,
51 + size: model.size,
52 + supportsStreaming: false,
53 + supportsTranslation: false,
54 + acceleration: "cpu",
55 + modelUnloadTimeout: "never",
56 + gpuDevices: [],
57 + } satisfies EnvironmentSpeechStatus;
58 + const fetch = globalThis.fetch;
59 + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
60 + const url = new URL(input instanceof Request ? input.url : input, window.location.href);
61 + if (url.pathname === "/api/voice/status") return Promise.resolve(Response.json(status));
62 + if (url.pathname === "/api/voice/models")
63 + return Promise.resolve(
64 + Response.json({
65 + models: [
66 + model,
67 + {
68 + ...model,
69 + id: "both",
70 + name: "Bilingual",
71 + active: false,
72 + languages: ["en", "fr"],
73 + ...(url.searchParams.has("benchmarks")
74 + ? {
75 + benchmark: {
76 + accuracy: {
77 + en: { grade: "A-", errorPercent: 4 },
78 + fr: { grade: "B+", errorPercent: 5.5 },
79 + },
80 + secondsPer30s: 2.1,
81 + },
82 + }
83 + : {}),
84 + },
85 + ],
86 + }),
87 + );
88 + return fetch(input, init);
89 + });
90 + const transcript = "Keep this transcript.";
91 + await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
92 + () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }),
93 + async () => {
94 + await vi.mocked(useEnvironmentSpeechInput).withImplementation(
95 + (input) => ({
96 + available: true,
97 + status,
98 + transcriptionEnvironmentId: target.environmentId,
99 + transcriptionEnvironmentLabel: target.label,
100 + setup: {
101 + open: false,
102 + step: 0,
103 + model: null,
104 + downloading: false,
105 + error: null,
106 + setOpen: () => {},
107 + download: async () => {},
108 + cancelDownload: async () => {},
109 + startRecording: async () => {},
110 + },
111 + state: { phase: "idle", error: null, errorAction: null },
112 + preview: null,
113 + level: 0,
114 + preparing: false,
115 + blocksSubmission: false,
116 + freezesEditor: false,
117 + start: async () => {
118 + input.commitDraft(transcript, { start: transcript.length, end: transcript.length });
119 + },
120 + submitAfterDictation: () => {},
121 + stop: async () => {},
122 + cancel: () => {},
123 + skipPostProcessing: () => {},
124 + }),
125 + async () => {
126 + await using screen = await render(
127 + <SettingsScopeProvider search={{}} onChange={() => {}}>
128 + <VoiceSettingsPanel />
129 + </SettingsScopeProvider>,
130 + );
131 + const testModel = screen.getByRole("button", { name: "Test model", exact: true });
132 + await expect.element(testModel).toBeVisible();
133 + await userEvent.click(testModel);
134 + const result = screen.getByRole("status", { name: "Test transcript" });
135 + await expect.element(result).toHaveTextContent(transcript);
136 +
137 + const search = screen.getByRole("textbox", { name: "Search transcription models" });
138 + await userEvent.fill(search, "no matching model");
139 + await expect.element(screen.getByText("No models match.")).toBeVisible();
140 + await userEvent.clear(search);
141 + await userEvent.fill(screen.getByRole("textbox", { name: "Search languages" }), "fr");
142 + const french = screen.getByRole("button", { name: "French", exact: true });
143 + await userEvent.click(french);
144 + await expect.element(screen.getByText("Bilingual", { exact: true })).toBeVisible();
145 + await expect.element(screen.getByText("B+", { exact: true })).toBeVisible();
146 + expect(screen.container.querySelector("#transcription-test")).toHaveAttribute("hidden");
147 + await userEvent.click(french);
148 + await expect.element(result).toBeVisible();
149 + await vi.waitFor(() =>
150 + expect(
151 + screen.container.querySelector('[aria-label="Test transcript"]')?.textContent,
152 + ).toBe(transcript),
153 + );
154 + },
155 + );
156 + },
157 + );
158 +});
159 +
160 +it("recovers from a failed download progress poll without notifying, but reports a failed initial load", async () => {
161 + const target = new PrimaryConnectionTarget({
162 + environmentId: EnvironmentId.make("voice-progress"),
163 + label: "Test environment",
164 + httpBaseUrl: window.location.origin,
165 + wsBaseUrl: "ws://localhost",
166 + });
167 + const prepared = { ...target, target, socketUrl: target.wsBaseUrl, httpAuthorization: null };
168 + const model: EnvironmentSpeechModel = {
169 + id: "tiny",
170 + name: "Tiny",
171 + description: "",
172 + size: 100,
173 + languages: ["en"],
174 + accuracy: 1,
175 + speed: 1,
176 + recommended: false,
177 + supportsStreaming: false,
178 + supportsTranslation: false,
179 + supportsLanguageDetection: false,
180 + active: false,
181 + state: "downloading",
182 + };
183 + const status = {
184 + supported: true,
185 + state: "missing-model",
186 + modelId: model.id,
187 + model: model.name,
188 + size: model.size,
189 + supportsStreaming: false,
190 + supportsTranslation: false,
191 + acceleration: "cpu",
192 + modelUnloadTimeout: "never",
193 + gpuDevices: [],
194 + };
195 + const unavailable = () => new Response(null, { status: 503 });
196 + const readModels = vi
197 + .fn((_withBenchmarks: boolean) => unavailable())
198 + .mockImplementationOnce(() => Response.json({ models: [model] }))
199 + .mockImplementationOnce(unavailable)
200 + .mockImplementationOnce(() => Response.json({ models: [{ ...model, state: "installed" }] }));
201 + const fetch = globalThis.fetch;
202 + using _network = vi.spyOn(globalThis, "fetch").mockImplementation((input, init) => {
203 + const url = new URL(input instanceof Request ? input.url : input, window.location.href);
204 + if (url.pathname === "/api/voice/status") return Promise.resolve(Response.json(status));
205 + if (url.pathname === "/api/voice/models")
206 + return Promise.resolve(readModels(url.searchParams.has("benchmarks")));
207 + return fetch(input, init);
208 + });
209 + using notify = vi.spyOn(toastManager, "add").mockReturnValue("test");
210 + await vi.mocked(useVoiceTranscriptionDestination).withImplementation(
211 + () => ({ environmentId: target.environmentId, label: target.label, grantId: "test", prepared }),
212 + async () => {
213 + const panel = (key: string) => (
214 + <SettingsScopeProvider search={{}} onChange={() => {}}>
215 + <VoiceSettingsPanel key={key} />
216 + </SettingsScopeProvider>
217 + );
218 + await using screen = await render(panel("first"));
219 + await expect.element(screen.getByRole("button", { name: "Use model" })).toBeVisible();
220 + expect(readModels.mock.calls.map(([withBenchmarks]) => withBenchmarks)).toEqual([
221 + true,
222 + false,
223 + false,
224 + ]);
225 + expect(notify).not.toHaveBeenCalled();
226 + await screen.rerender(panel("reopened"));
227 + await vi.waitFor(() =>
228 + expect(notify).toHaveBeenCalledWith(
229 + expect.objectContaining({ title: "Could not load transcription models" }),
230 + ),
231 + );
232 + },
233 + );
234 +});
235 +
236 +it.each([
237 + ["speech-models", "voice-transcription-destination"],
238 + ["speech-hardware", "voice-transcription-destination"],
239 + ["transcription-test", "voice-transcription-destination"],
240 + ["speech-correction-word", "speech-post-processing"],
241 + ["speech-post-processing-model", "speech-post-processing"],
242 +])(
243 + "searching for disabled %s focuses its opt-in without enabling it",
244 + async (target, prerequisite) => {
245 + chooseAudioDestination(null);
246 + chooseTranscriptCleanup(false);
247 + await using screen = await render(
248 + <SettingsScopeProvider search={{}} onChange={() => {}}>
249 + <VoiceSettingsPanel />
250 + </SettingsScopeProvider>,
251 + { router: `/#${target}` },
252 + );
253 + await vi.waitFor(() => expect(document.activeElement?.id).toBe(prerequisite));
254 + await expect
255 + .element(screen.getByRole("combobox", { name: "Send microphone audio to" }))
256 + .toHaveTextContent("Off");
257 + await expect
258 + .element(screen.getByRole("switch", { name: "Clean up transcripts with a provider" }))
259 + .not.toBeChecked();
260 + },
261 +);
262 +
263 +it("keeps a missing destination selected without claiming it cannot transcribe", async () => {
264 + __setClientSettingsForTests(getClientSettings());
265 + using cleanup = new DisposableStack();
266 + cleanup.defer(() => chooseAudioDestination(null));
267 + chooseAudioDestination({ kind: "environment", environmentId: EnvironmentId.make("missing") });
268 + await using screen = await render(
269 + <SettingsScopeProvider search={{}} onChange={() => {}}>
270 + <VoiceSettingsPanel />
271 + </SettingsScopeProvider>,
272 + );
273 + await expect
274 + .element(screen.getByRole("combobox", { name: "Send microphone audio to" }))
275 + .toHaveTextContent("Chosen environment (unavailable)");
276 + await expect.element(screen.getByText(/can’t transcribe audio/)).not.toBeInTheDocument();
277 +});
278 +
279 +it("shows the model's effective language and restores the saved preference when supported again", async () => {
280 + const previous = getClientSettings();
281 + using cleanup = new DisposableStack();
282 + cleanup.defer(() => __setClientSettingsForTests(previous));
283 + __setClientSettingsForTests({ ...previous, speechLanguage: "fr" });
284 + await using screen = await render(<SpeechLanguageSelect model={{ languages: ["en"] }} />);
285 + const picker = screen.getByRole("combobox", { name: "Transcription language" });
286 + await expect.element(picker).toHaveTextContent("English (French unavailable)");
287 + await screen.rerender(
288 + <SpeechLanguageSelect model={{ languages: ["en", "de"], supportsLanguageDetection: true }} />,
289 + );
290 + await expect.element(picker).toHaveTextContent("Auto-detect (French unavailable)");
291 + await screen.rerender(<SpeechLanguageSelect model={{ languages: ["en", "fr"] }} />);
292 + await expect.element(picker).toHaveTextContent("French");
293 + expect(getClientSettings().speechLanguage).toBe("fr");
294 +});
295 +
296 +it("keeps punctuation in a saved dictionary term and its alias", async () => {
297 + const previous = getClientSettings();
298 + using cleanup = new DisposableStack();
299 + cleanup.defer(() => __setClientSettingsForTests(previous));
300 + __setClientSettingsForTests({ ...previous, speechCustomWords: [] });
301 + await using screen = await render(
302 + <SettingsScopeProvider search={{}} onChange={() => {}}>
303 + <VoiceSettingsPanel />
304 + </SettingsScopeProvider>,
305 + );
306 + await userEvent.type(
307 + screen.getByRole("textbox", { name: "Add a word or phrase to the dictionary" }),
308 + "O'Reilly{Enter}",
309 + );
310 + expect(getClientSettings().speechCustomWords[0]?.term).toBe("O'Reilly");
311 + await userEvent.click(screen.getByRole("button", { name: "Add an alias for O'Reilly" }));
312 + await userEvent.type(
313 + screen.getByRole("textbox", { name: "Alias for O'Reilly" }),
314 + '"Oh Really"{Enter}',
315 + );
316 + expect(getClientSettings().speechCustomWords).toEqual([
317 + { term: "O'Reilly", aliases: ['"Oh Really"'] },
318 + ]);
319 +});
apps/web/src/components/settings/VoiceSettingsPanel.tsx new
+1371
@@ -0,0 +1,1371 @@
1 +import {
2 + cancelEnvironmentSpeechModelDownload,
3 + downloadEnvironmentSpeechModel,
4 + getEnvironmentSpeechModels,
5 + getEnvironmentSpeechStatus,
6 + removeEnvironmentSpeechModel,
7 + selectEnvironmentSpeechModel,
8 + updateEnvironmentSpeechAcceleration,
9 + updateEnvironmentSpeechModelUnloadTimeout,
10 +} from "@t3tools/client-runtime/voice-input";
11 +import type { PreparedConnection } from "@t3tools/client-runtime/connection";
12 +import {
13 + AuthSettingsWriteScope,
14 + DEFAULT_SPEECH_POST_PROCESSING_PROMPT,
15 + type EnvironmentId,
16 + type EnvironmentSpeechModel,
17 + type EnvironmentSpeechStatus,
18 + type ProjectId,
19 + type SpeechAcceleration,
20 + type SpeechCustomWords,
21 + type SpeechModelUnloadTimeout,
22 + type VoiceShortcutMode,
23 +} from "@t3tools/contracts";
24 +import { createModelSelection } from "@t3tools/shared/model";
25 +import { resolveSpeechPostProcessingModelSelection } from "@t3tools/shared/serverSettings";
26 +import {
27 + AudioLinesIcon,
28 + CheckIcon,
29 + DownloadIcon,
30 + LanguagesIcon,
31 + MicIcon,
32 + RefreshCwIcon,
33 + StarIcon,
34 + Trash2Icon,
35 + XIcon,
36 +} from "lucide-react";
37 +import { useEffect, useRef, useState, type ReactNode } from "react";
38 +
39 +import { useStableCallback } from "~/hooks/useStableCallback";
40 +import { listBrowserMicrophones, type BrowserMicrophone } from "../../speech/browserMicrophones";
41 +import {
42 + chooseAudioDestination,
43 + chooseTranscriptCleanup,
44 + useCleanupGrant,
45 +} from "../../speech/voiceConsent";
46 +import { updateClientSettings, useClientSettings } from "../../hooks/useSettings";
47 +import { runtime } from "../../lib/runtime";
48 +import {
49 + getCustomModelOptionsByInstance,
50 + resolveAppModelSelectionState,
51 +} from "../../modelSelection";
52 +import { providerInstanceEntriesFor } from "../../providerInstances";
53 +import { useEnvironmentProviders, useEnvironments } from "../../state/environments";
54 +import { useEnvironmentScope } from "../../state/session";
55 +import { EMPTY_SERVER_PROVIDERS } from "../../state/server";
56 +import {
57 + useEnvironmentSpeechInput,
58 + useVoiceTranscriptionDestination,
59 +} from "../../speech/useEnvironmentSpeechInput";
60 +import { ComposerSpeechRecordingPill } from "../chat/ComposerSpeechButton";
61 +import { ProviderModelPicker } from "../chat/ProviderModelPicker";
62 +import { TraitsPicker } from "../chat/TraitsPicker";
63 +import { VoiceInputSetup } from "../chat/VoiceInputSetup";
64 +import { Badge } from "../ui/badge";
65 +import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
66 +import { Button } from "../ui/button";
67 +import { Input } from "../ui/input";
68 +import { Select, SelectItem, SelectPopup, SelectTrigger, SelectValue } from "../ui/select";
69 +import { Switch } from "../ui/switch";
70 +import { Textarea } from "../ui/textarea";
71 +import { toastManager } from "../ui/toast";
72 +import { MicrophoneTest } from "./MicrophoneTest";
73 +import { LanguageRow, ModelBenchmarks, orderSpeechLanguages } from "./SpeechModelComparison";
74 +import { languageLabel, SpeechLanguageSelect } from "./SpeechLanguageSelect";
75 +import { AdvancedSettings } from "./AdvancedSettings";
76 +import {
77 + SETTINGS_PICKER_TRIGGER_CLASSNAME,
78 + SettingResetButton,
79 + SettingsPageContainer,
80 + SettingsRow,
81 + SettingsSearchTarget,
82 + SettingsSection,
83 +} from "./settingsLayout";
84 +import { searchableSetting } from "./settingsSearch";
85 +import { useOptionalSettingsScope, useSettingsScope } from "./SettingsScopeContext";
86 +import { useScopedModelDisabledReason } from "./useScopedModelAvailability";
87 +import { useScopedSettings, useUpdateScopedSettings } from "./useScopedSettings";
88 +
89 +const OFF = "off";
90 +const SYSTEM_DEFAULT = "system-default";
91 +const formatSize = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`;
92 +const SHORTCUT_MODE_LABELS: Record<VoiceShortcutMode, string> = {
93 + auto: "Auto: press to toggle, hold to record",
94 + hold: "Hold: record while held",
95 + toggle: "Toggle: press to start, press again to stop",
96 +};
97 +const UNLOAD_LABELS: Record<SpeechModelUnloadTimeout, string> = {
98 + never: "Never",
99 + immediately: "Immediately",
100 + min_2: "After 2 minutes",
101 + min_5: "After 5 minutes",
102 + min_10: "After 10 minutes",
103 + min_15: "After 15 minutes",
104 + hour_1: "After 1 hour",
105 +};
106 +const normalizeTerm = (value: string) => value.replace(/\s+/g, " ").trim().normalize("NFC");
107 +const spellingsOf = (words: SpeechCustomWords) =>
108 + new Set(
109 + words.flatMap(({ term, aliases }) =>
110 + [term, ...aliases].map((spelling) => spelling.toLocaleLowerCase()),
111 + ),
112 + );
113 +const modelSortOrder = (model: EnvironmentSpeechModel) => {
114 + if (model.active) return 0;
115 + if (model.state === "installed") return 1;
116 + if (model.state === "downloading" || model.state === "verifying") return 2;
117 + return model.recommended ? 3 : 4;
118 +};
119 +
120 +type ModelCardProps = {
121 + readonly model: EnvironmentSpeechModel;
122 + readonly comparisonLanguages: readonly string[];
123 + readonly busy: boolean;
124 + readonly canCancel: boolean;
125 + readonly onDownload: () => void;
126 + readonly onSelect: () => void;
127 + readonly onCancel: () => void;
128 + readonly onDelete: () => void;
129 + readonly testControls?: ReactNode;
130 + readonly children?: ReactNode;
131 +};
132 +
133 +function ModelBadge(props: { icon: ReactNode; label: string; tooltip: string }) {
134 + return (
135 + <Tooltip>
136 + <TooltipTrigger render={<Badge variant="outline" />}>
137 + {props.icon}
138 + {props.label}
139 + </TooltipTrigger>
140 + <TooltipPopup side="top">{props.tooltip}</TooltipPopup>
141 + </Tooltip>
142 + );
143 +}
144 +
145 +function ModelCard(props: ModelCardProps) {
146 + const { model } = props;
147 + const downloading = model.state === "downloading" || model.state === "verifying";
148 + const progress = model.downloaded === undefined ? 0 : (model.downloaded / model.size) * 100;
149 + return (
150 + <div className="flex flex-wrap items-start gap-3 px-3 py-2.5 sm:px-4">
151 + <div className="min-w-0 flex-1">
152 + <div className="flex flex-wrap items-center gap-1.5">
153 + <span className="text-sm font-medium leading-5">{model.name}</span>
154 + {model.recommended ? (
155 + <ModelBadge
156 + icon={<StarIcon className="mr-1 size-3" />}
157 + label="Recommended"
158 + tooltip="A good default pick for the languages it covers."
159 + />
160 + ) : null}
161 + {model.supportsStreaming ? (
162 + <ModelBadge
163 + icon={<AudioLinesIcon className="mr-1 size-3" />}
164 + label="Live preview"
165 + tooltip="Shows words in your draft as you speak, before the recording finishes."
166 + />
167 + ) : null}
168 + {model.supportsTranslation ? (
169 + <ModelBadge
170 + icon={<LanguagesIcon className="mr-1 size-3" />}
171 + label="Translate"
172 + tooltip="Can write English text from speech in another language. Turn on Translate to English to use it."
173 + />
174 + ) : null}
175 + </div>
176 + <p className="mt-0.5 text-xs leading-relaxed text-muted-foreground">{model.description}</p>
177 + <ModelBenchmarks
178 + model={model}
179 + languages={props.comparisonLanguages}
180 + facts={[
181 + ...(model.languages.length > 1
182 + ? [`${model.languages.length} languages`]
183 + : model.benchmark
184 + ? []
185 + : [languageLabel(model.languages[0] ?? "")]),
186 + formatSize(model.size),
187 + ]}
188 + />
189 + {downloading ? (
190 + <div className="mt-2">
191 + <div
192 + role="progressbar"
193 + aria-label={`${model.name} download progress`}
194 + aria-valuemin={0}
195 + aria-valuemax={100}
196 + aria-valuenow={Math.round(progress)}
197 + className="h-1 overflow-hidden rounded-full bg-muted"
198 + >
199 + <div
200 + className="h-full bg-primary transition-[width] duration-200"
201 + style={{ width: `${progress}%` }}
202 + />
203 + </div>
204 + <p className="mt-1 text-2xs text-muted-foreground">
205 + {model.state === "verifying"
206 + ? "Verifying download…"
207 + : `${Math.round(progress)}% downloaded`}
208 + </p>
209 + </div>
210 + ) : null}
211 + </div>
212 + <div className="flex max-w-full flex-wrap items-center justify-end gap-1">
213 + {props.testControls}
214 + {downloading ? (
215 + <Button
216 + size="icon-sm"
217 + variant="ghost"
218 + aria-label={`Cancel ${model.name} download`}
219 + disabled={!props.canCancel}
220 + onClick={props.onCancel}
221 + >
222 + <XIcon className="size-3.5" />
223 + </Button>
224 + ) : model.state === "downloadable" ? (
225 + <Button size="sm" variant="outline" disabled={props.busy} onClick={props.onDownload}>
226 + <DownloadIcon className="mr-1.5 size-3.5" />
227 + Download
228 + </Button>
229 + ) : model.active ? (
230 + <Button size="sm" variant="secondary" disabled>
231 + <CheckIcon className="mr-1.5 size-3.5" />
232 + Selected
233 + </Button>
234 + ) : (
235 + <Button size="sm" variant="outline" disabled={props.busy} onClick={props.onSelect}>
236 + Use model
237 + </Button>
238 + )}
239 + {model.state === "installed" ? (
240 + <Button
241 + size="icon-sm"
242 + variant="ghost"
243 + disabled={props.busy}
244 + aria-label={`Delete ${model.name}`}
245 + onClick={props.onDelete}
246 + >
247 + <Trash2Icon className="size-3.5" />
248 + </Button>
249 + ) : null}
250 + </div>
251 + {props.children}
252 + </div>
253 + );
254 +}
255 +
256 +function DictionaryEditor(props: {
257 + readonly label: string;
258 + readonly words: SpeechCustomWords;
259 + readonly disabled?: boolean;
260 + readonly onChange: (words: SpeechCustomWords) => void;
261 +}) {
262 + const { words, label, disabled = false, onChange } = props;
263 + const [termDraft, setTermDraft] = useState("");
264 + const [aliasDraft, setAliasDraft] = useState<{ term: string; value: string } | null>(null);
265 + const spellings = spellingsOf(words);
266 + const seen = new Set(spellings);
267 + const newTerms = termDraft
268 + .split(/[,\n]/)
269 + .map(normalizeTerm)
270 + .filter((term) => {
271 + const key = term.toLocaleLowerCase();
272 + if (!key || seen.has(key)) return false;
273 + seen.add(key);
274 + return true;
275 + });
276 + const canAddTerm =
277 + !disabled &&
278 + newTerms.length > 0 &&
279 + newTerms.every((term) => term.length <= 50) &&
280 + words.length + newTerms.length <= 100;
281 + const addTerm = () => {
282 + if (!canAddTerm) return;
283 + onChange([...words, ...newTerms.map((term) => ({ term, aliases: [] }))]);
284 + setTermDraft("");
285 + };
286 + const addAlias = () => {
287 + if (!aliasDraft) return;
288 + const value = normalizeTerm(aliasDraft.value);
289 + if (!value || value.length > 50 || spellings.has(value.toLocaleLowerCase())) return;
290 + onChange(
291 + words.map((entry) =>
292 + entry.term === aliasDraft.term && entry.aliases.length < 8
293 + ? { term: entry.term, aliases: [...entry.aliases, value] }
294 + : entry,
295 + ),
296 + );
297 + setAliasDraft(null);
298 + };
299 + return (
300 + <>
301 + <div className="flex items-start gap-1.5">
302 + <Textarea
303 + value={termDraft}
304 + rows={1}
305 + maxLength={5_100}
306 + disabled={disabled}
307 + placeholder="Add words or phrases, separated by commas or line breaks"
308 + aria-label={`Add a word or phrase to the ${label}`}
309 + onChange={(event) => setTermDraft(event.target.value)}
310 + onKeyDown={(event) => {
311 + if (event.key !== "Enter" || event.shiftKey) return;
312 + event.preventDefault();
313 + addTerm();
314 + }}
315 + />
316 + <Button type="button" disabled={!canAddTerm} onClick={addTerm}>
317 + Add
318 + </Button>
319 + </div>
320 + {words.length > 0 ? (
321 + <ul aria-label={`${label} entries`} className="mt-3 flex flex-wrap gap-1.5">
322 + {words.map(({ term, aliases }) => (
323 + <li
324 + key={term}
325 + className="flex items-center gap-1 rounded-md border border-border/60 bg-muted/30 py-0.5 pl-2 pr-0.5 text-xs"
326 + >
327 + <span className="font-medium">{term}</span>
328 + {aliases.length > 0 ? (
329 + <span className="text-muted-foreground">← {aliases.join(", ")}</span>
330 + ) : null}
331 + {aliasDraft?.term === term ? (
332 + <Input
333 + autoFocus
334 + value={aliasDraft.value}
335 + maxLength={50}
336 + aria-label={`Alias for ${term}`}
337 + placeholder="Misspelling"
338 + className="w-32"
339 + onChange={(event) => setAliasDraft({ term, value: event.target.value })}
340 + onBlur={() => setAliasDraft(null)}
341 + onKeyDown={(event) => {
342 + if (event.key === "Escape") setAliasDraft(null);
343 + if (event.key !== "Enter") return;
344 + event.preventDefault();
345 + addAlias();
346 + }}
347 + />
348 + ) : (
349 + <Button
350 + type="button"
351 + size="xs"
352 + variant="ghost-muted"
353 + disabled={disabled || aliases.length >= 8}
354 + aria-label={`Add an alias for ${term}`}
355 + onClick={() => setAliasDraft({ term, value: "" })}
356 + >
357 + Alias
358 + </Button>
359 + )}
360 + <Button
361 + type="button"
362 + size="icon-xs"
363 + variant="ghost-muted"
364 + disabled={disabled}
365 + aria-label={`Remove ${term}`}
366 + onClick={() => onChange(words.filter((entry) => entry.term !== term))}
367 + >
368 + <XIcon />
369 + </Button>
370 + </li>
371 + ))}
372 + </ul>
373 + ) : null}
374 + </>
375 + );
376 +}
377 +
378 +/** Vocabulary stored on the environment with the project, read by everyone who dictates there. */
379 +function ProjectDictionaryRow() {
380 + const scope = useOptionalSettingsScope();
381 + const words = useScopedSettings((settings) => settings.speechProjectCustomWords);
382 + const updateScopedSettings = useUpdateScopedSettings();
383 + if (scope === null) return null;
384 + const isProjectScope = scope.scope.kind === "project" || scope.scope.kind === "checkout";
385 + return (
386 + <SettingsRow
387 + serverScoped
388 + settingKeys={["speechProjectCustomWords"]}
389 + {...searchableSetting("speech-project-dictionary")}
390 + title={isProjectScope ? "Project dictionary" : "Shared dictionary"}
391 + description={
392 + isProjectScope
393 + ? `Saved with ${scope.scope.label} on its environment. Everyone who dictates in this project gets these spellings, along with their own dictionary. Project spellings win when the two overlap.`
394 + : "Saved on the environment and used in every project there, along with each person’s own dictionary. Pick a project above to limit these words to one project."
395 + }
396 + control={
397 + <span className="text-xs text-muted-foreground">
398 + {words.length} {words.length === 1 ? "entry" : "entries"}
399 + </span>
400 + }
401 + >
402 + <div className="mt-2 pb-3.5">
403 + <DictionaryEditor
404 + label={isProjectScope ? "project dictionary" : "shared dictionary"}
405 + words={words}
406 + disabled={scope.connectedEnvironments.length === 0}
407 + onChange={(next) => updateScopedSettings({ speechProjectCustomWords: next })}
408 + />
409 + </div>
410 + </SettingsRow>
411 + );
412 +}
413 +
414 +/** Cleanup runs on the draft's environment, so its model is an environment setting there. */
415 +function CleanupModelRow() {
416 + const { environment, connectedEnvironments } = useSettingsScope();
417 + const settings = useScopedSettings();
418 + const updateSettings = useUpdateScopedSettings();
419 + const providersByEnvironment = useEnvironmentProviders();
420 + const providers =
421 + (environment?.serverConfig
422 + ? providersByEnvironment.get(environment.environmentId)
423 + : undefined) ?? EMPTY_SERVER_PROVIDERS;
424 + const textGenerationProviders = providers.filter(
425 + (provider) => provider.supportsTextGeneration !== false,
426 + );
427 + const instanceEntries = providerInstanceEntriesFor(textGenerationProviders, settings, null);
428 + const hasProvider = instanceEntries.some((entry) => entry.enabled && entry.isAvailable);
429 + const dedicated = settings.speechPostProcessingModelSelection !== null;
430 + const selection = resolveAppModelSelectionState(
431 + {
432 + ...settings,
433 + textGenerationModelSelection: resolveSpeechPostProcessingModelSelection(
434 + settings,
435 + textGenerationProviders,
436 + ),
437 + },
438 + textGenerationProviders,
439 + );
440 + const modelOptionsByInstance = getCustomModelOptionsByInstance(
441 + settings,
442 + textGenerationProviders,
443 + selection.instanceId,
444 + selection.model,
445 + );
446 + const instanceEntry = instanceEntries.find((entry) => entry.instanceId === selection.instanceId);
447 + const modelDisabledReason = useScopedModelDisabledReason(settings, instanceEntries);
448 + const setModel = (next: ReturnType<typeof createModelSelection>) => {
449 + const reason = modelDisabledReason(next.instanceId, next.model);
450 + if (reason) {
451 + toastManager.add({ type: "error", title: "Cleanup model not saved", description: reason });
452 + return;
453 + }
454 + updateSettings({ speechPostProcessingModelSelection: next });
455 + };
456 + return (
457 + <SettingsRow
458 + serverScoped
459 + settingKeys={["speechPostProcessingModelSelection"]}
460 + {...searchableSetting("speech-post-processing-model")}
461 + title="Cleanup model"
462 + description={
463 + dedicated
464 + ? `Drafts on ${environment?.label ?? "this environment"} use this model for cleanup instead of the environment’s text generation model.`
465 + : `Cleanup currently uses the text generation model on ${environment?.label ?? "this environment"}. Pick another model if you want a cheaper or faster one.`
466 + }
467 + resetAction={
468 + dedicated ? (
469 + <SettingResetButton
470 + label="cleanup model"
471 + onClick={() => updateSettings({ speechPostProcessingModelSelection: null })}
472 + />
473 + ) : null
474 + }
475 + control={
476 + connectedEnvironments.length === 0 ? (
477 + <span className="text-sm text-muted-foreground">Connect an environment first.</span>
478 + ) : !hasProvider || !instanceEntry ? (
479 + <span className="text-sm text-muted-foreground">No providers available.</span>
480 + ) : (
481 + <div className="flex min-w-0 flex-wrap items-center justify-end gap-1.5">
482 + <ProviderModelPicker
483 + environmentId={environment?.environmentId ?? null}
484 + activeInstanceId={selection.instanceId}
485 + model={selection.model}
486 + lockedProvider={null}
487 + instanceEntries={instanceEntries}
488 + modelOptionsByInstance={modelOptionsByInstance}
489 + triggerClassName={SETTINGS_PICKER_TRIGGER_CLASSNAME}
490 + getModelDisabledReason={modelDisabledReason}
491 + onInstanceModelChange={(instanceId, model) =>
492 + setModel(createModelSelection(instanceId, model))
493 + }
494 + />
495 + <TraitsPicker
496 + provider={instanceEntry.driverKind}
497 + models={instanceEntry.models}
498 + model={selection.model}
499 + modelOptions={selection.options ?? []}
500 + allowPromptInjectedEffort={false}
501 + planModeEnabled={settings.planModeEnabled}
502 + triggerClassName={SETTINGS_PICKER_TRIGGER_CLASSNAME}
503 + onModelOptionsChange={(options) =>
504 + setModel(createModelSelection(selection.instanceId, selection.model, options))
505 + }
506 + />
507 + </div>
508 + )
509 + }
510 + />
511 + );
512 +}
513 +
514 +function useVoiceModels(prepared: PreparedConnection | null) {
515 + const canManage = useEnvironmentScope(prepared?.environmentId ?? null, AuthSettingsWriteScope);
516 + const [models, setModels] = useState<readonly EnvironmentSpeechModel[]>([]);
517 + const [status, setStatus] = useState<EnvironmentSpeechStatus | null>(null);
518 + const [busy, setBusy] = useState(false);
519 + const [requestedDownload, setRequestedDownload] = useState<string | null>(null);
520 + const [actionConnection, setActionConnection] = useState(prepared);
521 + if (actionConnection !== prepared) {
522 + setActionConnection(prepared);
523 + setBusy(false);
524 + setRequestedDownload(null);
525 + setModels([]);
526 + setStatus(null);
527 + }
528 + const epoch = useRef(0);
529 + const refreshRevision = useRef(0);
530 + const refresh = useStableCallback(
531 + async (connection: PreparedConnection, mode: "foreground" | "progress") => {
532 + const current = epoch.current;
533 + const revision = ++refreshRevision.current;
534 + try {
535 + const [nextStatus, nextModels] = await Promise.all([
536 + runtime.runPromise(getEnvironmentSpeechStatus(connection)),
537 + runtime.runPromise(getEnvironmentSpeechModels(connection, mode === "foreground")),
538 + ]);
539 + if (current !== epoch.current || revision !== refreshRevision.current) return;
540 + setStatus(nextStatus);
541 + setModels((previous) => {
542 + if (mode === "foreground") return nextModels.models;
543 + const benchmarks = new Map(previous.map((model) => [model.id, model.benchmark]));
544 + return nextModels.models.map((model) => {
545 + const benchmark = benchmarks.get(model.id);
546 + return benchmark ? { ...model, benchmark } : model;
547 + });
548 + });
549 + } catch (error) {
550 + if (mode === "progress") return;
551 + if (current !== epoch.current || revision !== refreshRevision.current) return;
552 + toastManager.add({
553 + type: "error",
554 + title: "Could not load transcription models",
555 + description: error instanceof Error ? error.message : String(error),
556 + });
557 + }
558 + },
559 + );
560 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- loads the environment's model catalog when its connection becomes available
561 + useEffect(() => {
562 + epoch.current += 1;
563 + if (prepared) void refresh(prepared, "foreground");
564 + return () => {
565 + epoch.current += 1;
566 + };
567 + }, [prepared, refresh]);
568 + const downloading =
569 + requestedDownload !== null ||
570 + models.some((model) => model.state === "downloading" || model.state === "verifying");
571 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls progress while a local request or the environment reports an active download
572 + useEffect(() => {
573 + if (!prepared || !downloading) return;
574 + let cancelled = false;
575 + let timer: number | undefined;
576 + const poll = async () => {
577 + await refresh(prepared, "progress");
578 + if (!cancelled) timer = window.setTimeout(() => void poll(), 1_000);
579 + };
580 + void poll();
581 + return () => {
582 + cancelled = true;
583 + window.clearTimeout(timer);
584 + };
585 + }, [downloading, prepared, refresh]);
586 +
587 + const run = async (label: string, work: () => Promise<unknown>, downloadId?: string) => {
588 + if (!prepared || !canManage) return;
589 + const current = epoch.current;
590 + if (downloadId) setRequestedDownload(downloadId);
591 + else setBusy(true);
592 + try {
593 + await work();
594 + } catch (error) {
595 + if (current === epoch.current)
596 + toastManager.add({
597 + type: "error",
598 + title: label,
599 + description: error instanceof Error ? error.message : String(error),
600 + });
601 + }
602 + if (current === epoch.current) {
603 + if (downloadId) setRequestedDownload(null);
604 + else setBusy(false);
605 + await refresh(prepared, "foreground");
606 + }
607 + };
608 + return {
609 + models,
610 + status,
611 + run,
612 + controlsDisabled: busy || downloading || !canManage,
613 + canCancel: canManage && !busy,
614 + requestedDownload,
615 + };
616 +}
617 +
618 +type VoiceModels = ReturnType<typeof useVoiceModels>;
619 +
620 +function AdvancedSpeechSettings({
621 + prepared,
622 + catalog,
623 +}: {
624 + prepared: PreparedConnection;
625 + catalog: VoiceModels;
626 +}) {
627 + const { run, controlsDisabled, status } = catalog;
628 + if (!status?.supported) return null;
629 + const supported = status;
630 + return (
631 + <AdvancedSettings targetIds={[searchableSetting("speech-hardware").id]}>
632 + <SettingsRow
633 + {...searchableSetting("speech-hardware")}
634 + title="Hardware"
635 + description="Auto uses the environment’s GPU when it has one."
636 + control={
637 + <Select
638 + value={supported.acceleration}
639 + disabled={controlsDisabled}
640 + onValueChange={(value) => {
641 + if (value)
642 + void run("Could not change transcription hardware", () =>
643 + runtime.runPromise(
644 + updateEnvironmentSpeechAcceleration(prepared, value as SpeechAcceleration),
645 + ),
646 + );
647 + }}
648 + >
649 + <SelectTrigger size="sm" aria-label="Transcription hardware" className="max-w-72">
650 + <SelectValue>
651 + {supported.acceleration === "auto"
652 + ? "Auto"
653 + : supported.acceleration === "cpu"
654 + ? "CPU"
655 + : (supported.gpuDevices.find(
656 + (device) => `gpu:${device.id}` === supported.acceleration,
657 + )?.name ?? "Selected GPU")}
658 + </SelectValue>
659 + </SelectTrigger>
660 + <SelectPopup align="end" alignItemWithTrigger={false}>
661 + <SelectItem value="auto">Auto</SelectItem>
662 + <SelectItem value="cpu">CPU</SelectItem>
663 + {supported.gpuDevices.map((device) => (
664 + <SelectItem key={device.id} value={`gpu:${device.id}`}>
665 + {device.name}
666 + </SelectItem>
667 + ))}
668 + </SelectPopup>
669 + </Select>
670 + }
671 + />
672 + <SettingsRow
673 + title="Unload model"
674 + description="Release the model’s memory on the environment after it sits idle."
675 + control={
676 + <Select
677 + value={supported.modelUnloadTimeout}
678 + disabled={controlsDisabled}
679 + onValueChange={(value) => {
680 + if (value)
681 + void run("Could not change when the model unloads", () =>
682 + runtime.runPromise(
683 + updateEnvironmentSpeechModelUnloadTimeout(
684 + prepared,
685 + value as SpeechModelUnloadTimeout,
686 + ),
687 + ),
688 + );
689 + }}
690 + >
691 + <SelectTrigger size="sm" aria-label="Unload model" className="max-w-72">
692 + <SelectValue>{UNLOAD_LABELS[supported.modelUnloadTimeout]}</SelectValue>
693 + </SelectTrigger>
694 + <SelectPopup align="end" alignItemWithTrigger={false}>
695 + {(Object.keys(UNLOAD_LABELS) as SpeechModelUnloadTimeout[]).map((timeout) => (
696 + <SelectItem key={timeout} value={timeout}>
697 + {UNLOAD_LABELS[timeout]}
698 + </SelectItem>
699 + ))}
700 + </SelectPopup>
701 + </Select>
702 + }
703 + />
704 + </AdvancedSettings>
705 + );
706 +}
707 +
708 +function SpeechModelBrowser(props: {
709 + prepared: PreparedConnection;
710 + catalog: VoiceModels;
711 + testEnvironmentId: EnvironmentId;
712 + projectId: ProjectId | null;
713 + projectName: string | null;
714 +}) {
715 + const {
716 + prepared,
717 + catalog: { status, models, run, controlsDisabled, canCancel, requestedDownload },
718 + } = props;
719 + const [search, setSearch] = useState("");
720 + const [languageSearch, setLanguageSearch] = useState("");
721 + const [selection, setSelection] = useState<string[] | null>(null);
722 + const preferred = navigator.languages.map((language) => language.toLowerCase().split("-")[0]!);
723 + const supportedLanguages = [...new Set(models.flatMap((model) => model.languages))];
724 + const defaultLanguage = preferred.find((code) => supportedLanguages.includes(code));
725 + const selectedLanguages = selection ?? (defaultLanguage ? [defaultLanguage] : []);
726 + const languages = orderSpeechLanguages(supportedLanguages, preferred);
727 + const needle = languageSearch.trim().toLocaleLowerCase();
728 + const visibleLanguages = languages.filter(
729 + (code) =>
730 + !needle || code.includes(needle) || languageLabel(code).toLocaleLowerCase().includes(needle),
731 + );
732 + const query = search.trim().toLocaleLowerCase();
733 + const sortedModels = models
734 + .filter(
735 + (model) =>
736 + selectedLanguages.every((language) => model.languages.includes(language)) &&
737 + (!query || (model.name + " " + model.description).toLocaleLowerCase().includes(query)),
738 + )
739 + .toSorted((left, right) => modelSortOrder(left) - modelSortOrder(right));
740 + const activeModel = models.find((model) => model.active);
741 + const renderedModels =
742 + activeModel && !sortedModels.includes(activeModel)
743 + ? [activeModel, ...sortedModels]
744 + : sortedModels;
745 +
746 + return (
747 + <SettingsSearchTarget
748 + {...searchableSetting("speech-models")}
749 + className="@container/voice-models"
750 + >
751 + {status && !status.supported ? (
752 + <p className="px-4 py-2 text-sm text-muted-foreground">
753 + This environment can’t transcribe audio: {status.reason}.
754 + </p>
755 + ) : null}
756 + {status?.supported ? (
757 + <>
758 + <p className="border-b border-border/60 px-3 py-2 text-xs text-muted-foreground sm:px-4">
759 + Pick languages to see models that support all of them. Accuracy is word or character
760 + error rate: lower is better. Speed is the time to transcribe 30 seconds of audio on
761 + Handy’s benchmark GPU.
762 + </p>
763 + <div className="@min-[32rem]/voice-models:grid @min-[32rem]/voice-models:grid-cols-[11rem_minmax(0,1fr)]">
764 + <div className="flex min-h-0 flex-col border-b border-border/60 bg-muted/10 @min-[32rem]/voice-models:border-r @min-[32rem]/voice-models:border-b-0">
765 + <div className="space-y-2 p-3">
766 + <p className="text-sm font-medium">Languages</p>
767 + <Input
768 + size="compact"
769 + aria-label="Search languages"
770 + placeholder="Search languages"
771 + value={languageSearch}
772 + onChange={(e) => setLanguageSearch(e.target.value)}
773 + />
774 + </div>
775 + <div
776 + role="group"
777 + aria-label="Browse models by language"
778 + className="flex gap-1 overflow-x-auto px-2 pb-2 @min-[32rem]/voice-models:h-72 @min-[32rem]/voice-models:flex-col @min-[32rem]/voice-models:overflow-y-auto"
779 + >
780 + <LanguageRow
781 + label="All languages"
782 + selected={selectedLanguages.length === 0}
783 + onClick={() => setSelection([])}
784 + />
785 + {visibleLanguages.map((code) => (
786 + <LanguageRow
787 + key={code}
788 + label={languageLabel(code)}
789 + selected={selectedLanguages.includes(code)}
790 + onClick={() =>
791 + setSelection(
792 + selectedLanguages.includes(code)
793 + ? selectedLanguages.filter((language) => language !== code)
794 + : [...selectedLanguages, code],
795 + )
796 + }
797 + />
798 + ))}
799 + {visibleLanguages.length === 0 && (
800 + <p className="px-2 py-1 text-xs text-muted-foreground">No languages match.</p>
801 + )}
802 + </div>
803 + </div>
804 + <div className="flex min-w-0 flex-col">
805 + <div className="space-y-2 p-3">
806 + <p className="text-sm font-medium">
807 + {selectedLanguages.length
808 + ? "Models for " + selectedLanguages.map(languageLabel).join(", ")
809 + : "All models"}
810 + </p>
811 + <Input
812 + size="compact"
813 + aria-label="Search transcription models"
814 + placeholder="Search models"
815 + value={search}
816 + onChange={(e) => setSearch(e.target.value)}
817 + />
818 + </div>
819 + <div
820 + className="h-72 overflow-y-auto divide-y divide-border/60"
821 + role="region"
822 + aria-label="Transcription models"
823 + tabIndex={0}
824 + >
825 + {sortedModels.length === 0 && (
826 + <p className="px-4 py-2 text-sm text-muted-foreground">No models match.</p>
827 + )}
828 + {renderedModels.map((model) => {
829 + const card: ModelCardProps = {
830 + model:
831 + model.id === requestedDownload && model.state === "downloadable"
832 + ? { ...model, state: "downloading" }
833 + : model,
834 + busy: controlsDisabled,
835 + canCancel,
836 + comparisonLanguages: selectedLanguages,
837 + onDownload: () =>
838 + void run(
839 + "Could not download the model",
840 + () =>
841 + runtime.runPromise(downloadEnvironmentSpeechModel(prepared, model.id)),
842 + model.id,
843 + ),
844 + onSelect: () =>
845 + void run("Could not select the model", () =>
846 + runtime.runPromise(selectEnvironmentSpeechModel(prepared, model.id)),
847 + ),
848 + onCancel: () =>
849 + void run("Could not cancel the download", () =>
850 + runtime.runPromise(
851 + cancelEnvironmentSpeechModelDownload(prepared, model.id),
852 + ),
853 + ),
854 + onDelete: () =>
855 + void run("Could not delete the model", () =>
856 + runtime.runPromise(removeEnvironmentSpeechModel(prepared, model.id)),
857 + ),
858 + };
859 + return model.active ? (
860 + <TranscriptionTest
861 + key={model.id}
862 + card={card}
863 + hidden={!sortedModels.includes(model)}
864 + environmentId={props.testEnvironmentId}
865 + projectId={props.projectId}
866 + projectName={props.projectName}
867 + />
868 + ) : (
869 + <ModelCard key={model.id} {...card} />
870 + );
871 + })}
872 + </div>
873 + </div>
874 + </div>
875 + </>
876 + ) : null}
877 + </SettingsSearchTarget>
878 + );
879 +}
880 +
881 +function TranscriptionTest(props: {
882 + readonly card: ModelCardProps;
883 + readonly hidden: boolean;
884 + readonly environmentId: EnvironmentId;
885 + readonly projectId: ProjectId | null;
886 + readonly projectName: string | null;
887 +}) {
888 + const [transcript, setTranscript] = useState<string | null>(null);
889 + const draft = transcript ?? "";
890 + const [showTranscript, setShowTranscript] = useState(false);
891 + const draftRef = useRef(draft);
892 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes the committed draft to the asynchronous transcription controller
893 + useEffect(() => {
894 + draftRef.current = draft;
895 + }, [draft]);
896 + const speech = useEnvironmentSpeechInput({
897 + environmentId: props.environmentId,
898 + projectId: props.projectId,
899 + projectName: props.projectName,
900 + ownerKey: JSON.stringify([props.environmentId, props.projectId]),
901 + draftText: draft,
902 + cleanup: false,
903 + readDraft: () => ({
904 + text: draftRef.current,
905 + selection: { start: draftRef.current.length, end: draftRef.current.length },
906 + }),
907 + commitDraft: (text) => setTranscript(text),
908 + });
909 + return (
910 + <SettingsSearchTarget {...searchableSetting("transcription-test")} hidden={props.hidden}>
911 + <ModelCard
912 + {...props.card}
913 + testControls={
914 + <div className="flex min-w-0 max-w-full items-center gap-1">
915 + <ComposerSpeechRecordingPill
916 + state={speech.state}
917 + progress={null}
918 + level={speech.level}
919 + onStart={() => void speech.start()}
920 + onStop={() => void speech.stop()}
921 + onCancel={() => speech.cancel()}
922 + onSkipPostProcessing={() => speech.skipPostProcessing()}
923 + />
924 + {speech.available ? (
925 + speech.state.phase === "idle" ? (
926 + <Button
927 + size="sm"
928 + variant="outline"
929 + onClick={() => {
930 + setShowTranscript(true);
931 + void speech.start();
932 + }}
933 + >
934 + <MicIcon />
935 + Test model
936 + </Button>
937 + ) : null
938 + ) : (
939 + <span className="text-xs text-muted-foreground">
940 + {speech.status?.supported === false
941 + ? speech.status.reason
942 + : "Waiting for the environment to connect"}
943 + </span>
944 + )}
945 + </div>
946 + }
947 + >
948 + {showTranscript ? (
949 + <div className="w-full">
950 + <div className="flex items-start gap-2">
951 + <p className="min-w-0 flex-1 text-xs leading-relaxed text-muted-foreground">
952 + Say a sentence in the language above. Cleanup is skipped, so you see exactly what
953 + the model heard. The transcript stays on this page.
954 + </p>
955 + <Button
956 + size="icon-sm"
957 + variant="ghost"
958 + aria-label="Close model test"
959 + onClick={() => {
960 + speech.cancel();
961 + setShowTranscript(false);
962 + }}
963 + >
964 + <XIcon />
965 + </Button>
966 + </div>
967 + {speech.preview || transcript !== null ? (
968 + <p
969 + role="status"
970 + aria-label="Test transcript"
971 + className="mt-2 select-text whitespace-pre-wrap break-words text-sm leading-relaxed"
972 + >
973 + {speech.preview ? (
974 + <>
975 + {speech.preview.committed}
976 + <span className="text-muted-foreground">{speech.preview.tentative}</span>
977 + </>
978 + ) : (
979 + transcript || "No speech recognized. Try again."
980 + )}
981 + </p>
982 + ) : null}
983 + </div>
984 + ) : null}
985 + </ModelCard>
986 + <VoiceInputSetup
987 + environmentId={speech.transcriptionEnvironmentId}
988 + open={speech.setup.open}
989 + step={speech.setup.step}
990 + status={speech.status}
991 + model={speech.setup.model}
992 + downloading={speech.setup.downloading}
993 + error={speech.setup.error}
994 + onOpenChange={speech.setup.setOpen}
995 + onDownload={() => void speech.setup.download()}
996 + onCancelDownload={() => void speech.setup.cancelDownload()}
997 + onStartRecording={() => void speech.setup.startRecording()}
998 + />
999 + </SettingsSearchTarget>
1000 + );
1001 +}
1002 +
1003 +export function VoiceSettingsPanel() {
1004 + const cleanupEnabled = useCleanupGrant() !== null;
1005 + const settings = useClientSettings();
1006 + const settingsScope = useOptionalSettingsScope();
1007 + const destination = useVoiceTranscriptionDestination();
1008 + const { environments } = useEnvironments();
1009 + const [microphones, setMicrophones] = useState<BrowserMicrophone[]>([]);
1010 + const prepared = destination.prepared;
1011 + const catalog = useVoiceModels(prepared);
1012 +
1013 + const refreshMicrophones = useStableCallback(async () => {
1014 + if (!navigator.mediaDevices?.enumerateDevices) return;
1015 + try {
1016 + setMicrophones(await listBrowserMicrophones());
1017 + } catch {
1018 + setMicrophones([]);
1019 + }
1020 + });
1021 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to microphone device changes while settings are open
1022 + useEffect(() => {
1023 + queueMicrotask(() => void refreshMicrophones());
1024 + const mediaDevices = navigator.mediaDevices;
1025 + if (!mediaDevices?.addEventListener) return;
1026 + const onChange = () => void refreshMicrophones();
1027 + mediaDevices.addEventListener("devicechange", onChange);
1028 + return () => mediaDevices.removeEventListener("devicechange", onChange);
1029 + }, [refreshMicrophones]);
1030 + const modelStatus = catalog.status;
1031 + const target = settingsScope?.target;
1032 + const members = settingsScope?.scope.members;
1033 + const project =
1034 + members?.find(
1035 + (member) => member.environmentId === target?.environmentId && member.id === target.projectId,
1036 + ) ?? members?.[0];
1037 +
1038 + const selectedMicrophone = settings.voiceMicrophone;
1039 + const selectedMicrophoneLabel = selectedMicrophone
1040 + ? (microphones.find((device) => device.deviceId === selectedMicrophone)?.label ??
1041 + "Selected microphone (unavailable)")
1042 + : "System default";
1043 + const destinationValue = destination.environmentId ?? OFF;
1044 + const chosenEnvironment = environments.find(
1045 + (environment) => environment.environmentId === destination.environmentId,
1046 + );
1047 + const chosenUnsupported =
1048 + chosenEnvironment?.serverConfig != null &&
1049 + chosenEnvironment.serverConfig.environment.capabilities.voiceTranscription !== true;
1050 + const model = prepared && modelStatus?.supported ? modelStatus : null;
1051 +
1052 + return (
1053 + <SettingsPageContainer
1054 + searchTargetFallbacks={{
1055 + "speech-models": prepared ? undefined : "voice-transcription-destination",
1056 + "speech-hardware": prepared && model ? undefined : "voice-transcription-destination",
1057 + "transcription-test": !prepared
1058 + ? "voice-transcription-destination"
1059 + : !model || model.state === "missing-model"
1060 + ? "speech-models"
1061 + : undefined,
1062 + "speech-post-processing-model":
1063 + cleanupEnabled && settingsScope ? undefined : "speech-post-processing",
1064 + "speech-correction-word": cleanupEnabled ? undefined : "speech-post-processing",
1065 + }}
1066 + >
1067 + <SettingsSection id="voice-input" title="Input">
1068 + <SettingsRow
1069 + {...searchableSetting("voice-microphone")}
1070 + title="Microphone"
1071 + description="System default follows your operating system. Record a ten-second test and play it back here. Test audio stays on this device and is never transcribed."
1072 + control={
1073 + <div className="flex min-w-0 flex-wrap items-center justify-end gap-1.5">
1074 + <SettingsSearchTarget {...searchableSetting("microphone-test")}>
1075 + <MicrophoneTest key={selectedMicrophone} microphoneId={selectedMicrophone} />
1076 + </SettingsSearchTarget>
1077 + <div className="flex min-w-0 items-center gap-1.5">
1078 + <Select
1079 + value={selectedMicrophone ? `device:${selectedMicrophone}` : SYSTEM_DEFAULT}
1080 + onValueChange={(value) => {
1081 + if (value)
1082 + void updateClientSettings({
1083 + voiceMicrophone:
1084 + value === SYSTEM_DEFAULT ? "" : value.slice("device:".length),
1085 + });
1086 + }}
1087 + >
1088 + <SelectTrigger size="sm" aria-label="Microphone" className="max-w-64">
1089 + <SelectValue>{selectedMicrophoneLabel}</SelectValue>
1090 + </SelectTrigger>
1091 + <SelectPopup align="end" alignItemWithTrigger={false}>
1092 + <SelectItem value={SYSTEM_DEFAULT}>System default</SelectItem>
1093 + {microphones.map((device) => (
1094 + <SelectItem key={device.deviceId} value={`device:${device.deviceId}`}>
1095 + {device.label}
1096 + </SelectItem>
1097 + ))}
1098 + </SelectPopup>
1099 + </Select>
1100 + <Button
1101 + type="button"
1102 + variant="ghost"
1103 + size="icon-sm"
1104 + aria-label="Refresh microphones"
1105 + onClick={() => void refreshMicrophones()}
1106 + >
1107 + <RefreshCwIcon className="size-3.5" />
1108 + </Button>
1109 + </div>
1110 + </div>
1111 + }
1112 + />
1113 + <SettingsRow
1114 + {...searchableSetting("dictation-shortcut-mode")}
1115 + title="Dictation shortcut"
1116 + description="How the dictation shortcut starts and stops recording."
1117 + control={
1118 + <Select
1119 + value={settings.voiceShortcutMode}
1120 + onValueChange={(value) => {
1121 + if (value)
1122 + void updateClientSettings({ voiceShortcutMode: value as VoiceShortcutMode });
1123 + }}
1124 + >
1125 + <SelectTrigger size="sm" aria-label="Dictation shortcut mode" className="max-w-72">
1126 + <SelectValue>{SHORTCUT_MODE_LABELS[settings.voiceShortcutMode]}</SelectValue>
1127 + </SelectTrigger>
1128 + <SelectPopup align="end" alignItemWithTrigger={false}>
1129 + {(Object.keys(SHORTCUT_MODE_LABELS) as VoiceShortcutMode[]).map((mode) => (
1130 + <SelectItem key={mode} value={mode}>
1131 + {SHORTCUT_MODE_LABELS[mode]}
1132 + </SelectItem>
1133 + ))}
1134 + </SelectPopup>
1135 + </Select>
1136 + }
1137 + />
1138 + </SettingsSection>
1139 +
1140 + <SettingsSection id="voice-transcription" title="Transcription">
1141 + <SettingsRow
1142 + {...searchableSetting("voice-transcription-destination")}
1143 + title="Send microphone audio to"
1144 + description={
1145 + <>
1146 + Microphone audio is sent to this environment to transcribe, then discarded. Choose Off
1147 + to turn voice input off on this device. This choice is saved on this device only.
1148 + </>
1149 + }
1150 + control={
1151 + <Select
1152 + value={destinationValue}
1153 + onValueChange={(value) => {
1154 + if (!value) return;
1155 + chooseAudioDestination(
1156 + value === OFF
1157 + ? null
1158 + : { kind: "environment", environmentId: value as EnvironmentId },
1159 + );
1160 + }}
1161 + >
1162 + <SelectTrigger size="sm" aria-label="Send microphone audio to" className="max-w-72">
1163 + <SelectValue>
1164 + {destinationValue === OFF
1165 + ? "Off"
1166 + : (chosenEnvironment?.label ?? "Chosen environment (unavailable)")}
1167 + </SelectValue>
1168 + </SelectTrigger>
1169 + <SelectPopup align="end" alignItemWithTrigger={false}>
1170 + <SelectItem value={OFF}>Off</SelectItem>
1171 + {environments.map((environment) => (
1172 + <SelectItem key={environment.environmentId} value={environment.environmentId}>
1173 + {environment.label}
1174 + </SelectItem>
1175 + ))}
1176 + {destination.environmentId && !chosenEnvironment ? (
1177 + <SelectItem value={destination.environmentId}>
1178 + Chosen environment (unavailable)
1179 + </SelectItem>
1180 + ) : null}
1181 + </SelectPopup>
1182 + </Select>
1183 + }
1184 + >
1185 + {destination.environmentId && chosenUnsupported ? (
1186 + <p className="mt-2 text-xs text-destructive">
1187 + {chosenEnvironment.label} can’t transcribe audio. Pick an environment that can to use
1188 + voice input.
1189 + </p>
1190 + ) : null}
1191 + </SettingsRow>
1192 +
1193 + <SettingsRow
1194 + {...searchableSetting("speech-language")}
1195 + title="Language"
1196 + description={
1197 + model
1198 + ? `${model.model} ${model.supportsLanguageDetection ? "detects the language automatically" : "uses one fixed language"}.`
1199 + : "Choose an environment above to see the languages its transcription model supports."
1200 + }
1201 + control={<SpeechLanguageSelect model={model} />}
1202 + />
1203 + {model?.supportsTranslation ? (
1204 + <SettingsRow
1205 + title="Translate to English"
1206 + description="Speak any language and get English text."
1207 + control={
1208 + <Switch
1209 + aria-label="Translate to English"
1210 + checked={settings.speechTranslateToEnglish}
1211 + onCheckedChange={(checked) =>
1212 + void updateClientSettings({ speechTranslateToEnglish: checked })
1213 + }
1214 + />
1215 + }
1216 + />
1217 + ) : null}
1218 + {prepared ? (
1219 + <SpeechModelBrowser
1220 + key={prepared.environmentId}
1221 + prepared={prepared}
1222 + catalog={catalog}
1223 + testEnvironmentId={
1224 + project?.environmentId ?? target?.environmentId ?? prepared.environmentId
1225 + }
1226 + projectId={project?.id ?? null}
1227 + projectName={project?.title ?? null}
1228 + />
1229 + ) : null}
1230 + {prepared && <AdvancedSpeechSettings prepared={prepared} catalog={catalog} />}
1231 + </SettingsSection>
1232 + <SettingsSection id="voice-recognition" title="Transcription options">
1233 + <SettingsRow
1234 + {...searchableSetting("speech-silence-filtering")}
1235 + title="Silence filtering"
1236 + description="Skip silence and other non-speech audio before transcription, keeping a short buffer so words aren’t cut off."
1237 + control={
1238 + <Switch
1239 + aria-label="Silence filtering"
1240 + checked={settings.speechSilenceFiltering}
1241 + onCheckedChange={(checked) =>
1242 + void updateClientSettings({ speechSilenceFiltering: checked })
1243 + }
1244 + />
1245 + }
1246 + />
1247 + <SettingsRow
1248 + {...searchableSetting("remove-filler-words")}
1249 + title="Remove filler words"
1250 + description="Remove filler words like “um” and “uh” from transcripts."
1251 + control={
1252 + <Switch
1253 + aria-label="Remove filler words"
1254 + checked={settings.speechRemoveFillerWords}
1255 + onCheckedChange={(checked) =>
1256 + void updateClientSettings({ speechRemoveFillerWords: checked })
1257 + }
1258 + />
1259 + }
1260 + />
1261 + <SettingsRow
1262 + {...searchableSetting("speech-dictionary")}
1263 + title="Dictionary"
1264 + description="Names and terms transcription should spell your way, saved on this device. Add an alias when it keeps hearing a term wrong. The name of the project you dictate in is also sent as a hint."
1265 + control={
1266 + <span className="text-xs text-muted-foreground">
1267 + {settings.speechCustomWords.length}{" "}
1268 + {settings.speechCustomWords.length === 1 ? "entry" : "entries"}
1269 + </span>
1270 + }
1271 + >
1272 + <div className="mt-2 pb-3.5">
1273 + <DictionaryEditor
1274 + label="dictionary"
1275 + words={settings.speechCustomWords}
1276 + onChange={(words) => void updateClientSettings({ speechCustomWords: words })}
1277 + />
1278 + </div>
1279 + </SettingsRow>
1280 + <ProjectDictionaryRow />
1281 + </SettingsSection>
1282 +
1283 + <SettingsSection id={searchableSetting("speech-post-processing").id} title="Cleanup">
1284 + <SettingsRow
1285 + title="Clean up transcripts with a provider"
1286 + description="Sends each transcript, plus the draft text around your cursor, to the text generation provider on the draft’s environment to fix punctuation, numbers, and spoken corrections. Your audio is never sent. Off by default."
1287 + control={
1288 + <Switch
1289 + aria-label="Clean up transcripts with a provider"
1290 + checked={cleanupEnabled}
1291 + onCheckedChange={chooseTranscriptCleanup}
1292 + />
1293 + }
1294 + />
1295 + {cleanupEnabled ? (
1296 + <SettingsRow
1297 + title="Cleanup instructions"
1298 + description="Use the built-in instructions or write your own."
1299 + control={
1300 + <Select
1301 + value={settings.speechPostProcessingPrompt.mode}
1302 + onValueChange={(value) => {
1303 + if (value === "default" || value === "custom")
1304 + void updateClientSettings({
1305 + speechPostProcessingPrompt: {
1306 + ...settings.speechPostProcessingPrompt,
1307 + mode: value,
1308 + },
1309 + });
1310 + }}
1311 + >
1312 + <SelectTrigger size="sm" aria-label="Cleanup instructions" className="max-w-64">
1313 + <SelectValue>
1314 + {settings.speechPostProcessingPrompt.mode === "custom" ? "Custom" : "Built-in"}
1315 + </SelectValue>
1316 + </SelectTrigger>
1317 + <SelectPopup align="end" alignItemWithTrigger={false}>
1318 + <SelectItem value="default">Built-in</SelectItem>
1319 + <SelectItem value="custom">Custom</SelectItem>
1320 + </SelectPopup>
1321 + </Select>
1322 + }
1323 + >
1324 + <Textarea
1325 + aria-label="Custom cleanup instructions"
1326 + className="mt-2"
1327 + maxLength={10_000}
1328 + disabled={settings.speechPostProcessingPrompt.mode !== "custom"}
1329 + value={
1330 + settings.speechPostProcessingPrompt.mode === "custom"
1331 + ? settings.speechPostProcessingPrompt.customInstructions
1332 + : DEFAULT_SPEECH_POST_PROCESSING_PROMPT
1333 + }
1334 + onChange={(event) =>
1335 + void updateClientSettings({
1336 + speechPostProcessingPrompt: {
1337 + mode: "custom",
1338 + customInstructions: event.target.value,
1339 + },
1340 + })
1341 + }
1342 + />
1343 + </SettingsRow>
1344 + ) : null}
1345 + {cleanupEnabled && settingsScope !== null ? <CleanupModelRow /> : null}
1346 + {cleanupEnabled ? (
1347 + <SettingsRow
1348 + {...searchableSetting("speech-correction-word")}
1349 + title="Correction cue"
1350 + description="The word or phrase you say to flag a correction, like “sorry” or “scratch that”. Cleanup applies the correction only where one is meant, and hears the cue as a hint."
1351 + control={
1352 + <Input
1353 + key={settings.speechCorrectionWord}
1354 + defaultValue={settings.speechCorrectionWord}
1355 + maxLength={50}
1356 + placeholder="sorry"
1357 + aria-label="Correction cue"
1358 + className="w-40"
1359 + onBlur={(event) => {
1360 + const next = event.target.value.trim();
1361 + if (next !== settings.speechCorrectionWord)
1362 + void updateClientSettings({ speechCorrectionWord: next });
1363 + }}
1364 + />
1365 + }
1366 + />
1367 + ) : null}
1368 + </SettingsSection>
1369 + </SettingsPageContainer>
1370 + );
1371 +}
apps/web/src/components/settings/settingsLayout.tsx
+5 -1
@@ -172,6 +172,8 @@ export function useSettingsSearchTarget<T extends HTMLElement>(id: string | unde
172
173 export function SettingsSearchTarget({
174 children,
175 + // `searchableSetting` names the target for search; as an HTML title it shows a native tooltip.
176 + title: _searchTitle,
177 ...targetProps
178 }: ComponentPropsWithoutRef<"div">) {
179 const targetRef = useSettingsSearchTarget<HTMLDivElement>(targetProps.id);
@@ -583,10 +585,12 @@ export function SettingsPageContainer({
585 children,
586 className,
587 width = "readable",
588 + searchTargetFallbacks,
589 }: {
590 children: ReactNode;
591 className?: string;
592 width?: WorkspacePageWidth;
593 + searchTargetFallbacks?: Readonly<Record<string, string | undefined>>;
594 }) {
595 const navigate = useNavigate();
596 const hash = useLocation({ select: (location) => location.hash });
@@ -606,7 +610,7 @@ export function SettingsPageContainer({
610
611 return (
612 <SettingsSearchTargetProvider
609 - targetId={targetId}
613 + targetId={targetId === null ? null : (searchTargetFallbacks?.[targetId] ?? targetId)}
614 highlightTarget={highlightTarget}
615 onTargetHandled={clearTargetHash}
616 >
apps/web/src/components/settings/settingsSearch.test.ts
+6 -3
@@ -232,9 +232,12 @@ describe("settings search targets", () => {
232 expect(isSettingsSearchScopeAvailable(fallback.scope, "environment")).toBe(true);
233 expect(isSettingsSearchScopeAvailable(fallback.scope, "all")).toBe(true);
234 expect(isSettingsSearchScopeAvailable(fallback.scope, "project")).toBe(false);
235 - const streaming = getSettingsSearchTargetScope("response-streaming")!;
236 - expect(streaming.scope).toBe("project-defaults");
237 - expect(isSettingsSearchScopeAvailable(streaming.scope, "project")).toBe(true);
235 + for (const id of ["response-streaming", "speech-project-dictionary"]) {
236 + expect(getSettingsSearchTargetScope(id)?.scope).toBe("project-defaults");
237 + }
238 + for (const scope of ["all", "environment", "project"] as const) {
239 + expect(isSettingsSearchScopeAvailable("project-defaults", scope)).toBe(true);
240 + }
241 for (const id of ["legacy-plan-mode", "legacy-context-window-indicator", "boring-sidebar"]) {
242 expect(getSettingsSearchTargetScope(id)!.scope).toBeNull();
243 }
apps/web/src/components/settings/settingsSearch.ts
+100
@@ -19,6 +19,7 @@ export type SettingsPath =
19 | "/settings/projects"
20 | "/settings/general"
21 | "/settings/diagnostics"
22 + | "/settings/voice"
23 | "/settings/appearance"
24 | "/settings/chat"
25 | "/settings/diffs"
@@ -107,6 +108,7 @@ export const SETTINGS_SECTION_LABELS: Readonly<Record<SettingsPath, string>> = {
108 "/settings/maintenance": "Maintenance",
109 "/settings/labs": "Labs",
110 "/settings/about": "About",
111 + "/settings/voice": "Voice",
112 "/settings/archived": "Archive",
113 };
114
@@ -123,6 +125,7 @@ export const SETTINGS_NAV_GROUPS: ReadonlyArray<{
125 "/settings/chat",
126 "/settings/diffs",
127 "/settings/keybindings",
128 + "/settings/voice",
129 "/settings/snap-shot",
130 ],
131 },
@@ -486,6 +489,102 @@ export const SETTINGS_SEARCH_ITEMS = [
489 scope: "environment",
490 searchTerms: ["beta alpha simlock callstack simulator emulator ios android pool lease gateway"],
491 },
492 + {
493 + id: "voice-transcription-destination",
494 + title: "Send microphone audio to",
495 + to: "/settings/voice",
496 + searchTerms: ["speech voice dictation stt transcription environment machine consent privacy"],
497 + },
498 + {
499 + id: "voice-microphone",
500 + title: "Microphone",
501 + to: "/settings/voice",
502 + searchTerms: ["mic input audio device"],
503 + },
504 + {
505 + id: "dictation-shortcut-mode",
506 + title: "Dictation shortcut",
507 + to: "/settings/voice",
508 + searchTerms: ["speech voice keyboard hotkey hold toggle auto"],
509 + },
510 + {
511 + id: "speech-language",
512 + title: "Transcription language",
513 + to: "/settings/voice",
514 + searchTerms: ["speech voice translate english auto detect"],
515 + },
516 + {
517 + id: "speech-dictionary",
518 + title: "Dictionary",
519 + to: "/settings/voice",
520 + searchTerms: ["speech voice transcription vocabulary names glossary alias"],
521 + },
522 + {
523 + id: "speech-project-dictionary",
524 + title: "Project dictionary",
525 + to: "/settings/voice",
526 + scope: "project-defaults",
527 + searchTerms: [
528 + "speech voice transcription vocabulary shared dictionary team environment glossary",
529 + ],
530 + },
531 + {
532 + id: "speech-post-processing-model",
533 + title: "Cleanup model",
534 + to: "/settings/voice",
535 + scope: "environment",
536 + searchTerms: ["speech voice transcript cleanup provider model post-processing"],
537 + },
538 + {
539 + id: "speech-correction-word",
540 + title: "Correction cue",
541 + to: "/settings/voice",
542 + searchTerms: ["speech voice cleanup sorry scratch that fix mistake"],
543 + },
544 + {
545 + id: "speech-silence-filtering",
546 + title: "Silence filtering",
547 + to: "/settings/voice",
548 + searchTerms: ["voice speech vad silero pauses noise"],
549 + },
550 + {
551 + id: "remove-filler-words",
552 + title: "Remove filler words",
553 + to: "/settings/voice",
554 + searchTerms: ["voice speech transcription hesitation um uh cleanup"],
555 + },
556 + {
557 + id: "speech-post-processing",
558 + title: "Clean up transcripts with a provider",
559 + to: "/settings/voice",
560 + searchTerms: ["speech transcription polish clean provider prompt post-processing"],
561 + },
562 + {
563 + id: "speech-models",
564 + title: "Transcription models",
565 + to: "/settings/voice",
566 + searchTerms: [
567 + "speech voice stt download whisper parakeet canary moonshine accuracy speed benchmark languages",
568 + ],
569 + },
570 + {
571 + id: "speech-hardware",
572 + title: "Transcription hardware and model unloading",
573 + to: "/settings/voice",
574 + searchTerms: ["speech voice stt advanced gpu cpu acceleration memory unload idle"],
575 + },
576 + {
577 + id: "microphone-test",
578 + title: "Test microphone",
579 + to: "/settings/voice",
580 + searchTerms: ["test mic audio playback loopback record input level"],
581 + },
582 + {
583 + id: "transcription-test",
584 + title: "Test model",
585 + to: "/settings/voice",
586 + searchTerms: ["test voice speech model transcription recognition"],
587 + },
588 {
589 id: "auto-settle-inactive-threads",
590 title: "Auto-settle inactive threads",
@@ -1202,6 +1301,7 @@ const SETTINGS_CATEGORY_SCOPES: Readonly<Record<SettingsPath, SettingsSearchScop
1301 "/settings/diagnostics": "environment",
1302 "/settings/projects": "project",
1303 "/settings/general": null,
1304 + "/settings/voice": null,
1305 "/settings/appearance": null,
1306 "/settings/chat": null,
1307 "/settings/diffs": null,
apps/web/src/components/ui/button.tsx
+2
@@ -23,6 +23,7 @@ const buttonVariants = cva(
23 "size-5 rounded-sm p-0 before:rounded-[calc(var(--radius-sm)-1px)] [&_svg:not([class*='size-'])]:size-3",
24 "icon-tiny": "size-4 p-0 [&_svg:not([class*='size-'])]:size-3",
25 "icon-sm": "size-8 sm:size-7",
26 + "icon-sm-pill": "size-8 rounded-full before:rounded-full sm:size-7",
27 "icon-xl":
28 "size-11 sm:size-10 [&_svg:not([class*='size-'])]:size-5 sm:[&_svg:not([class*='size-'])]:size-4.5",
29 "icon-xs":
@@ -33,6 +34,7 @@ const buttonVariants = cva(
34 multiline:
35 "min-h-9 px-[calc(--spacing(3)-1px)] py-[calc(--spacing(1.5)-1px)] whitespace-normal sm:min-h-8",
36 sm: "h-8 gap-1.5 px-[calc(--spacing(2.5)-1px)] sm:h-7",
37 + "sm-pill": "h-8 gap-1.5 rounded-full px-2.5 sm:h-7",
38 "sm-multiline":
39 "min-h-8 gap-1.5 px-[calc(--spacing(2.5)-1px)] py-[calc(--spacing(1)-1px)] whitespace-normal sm:min-h-7",
40 xl: "h-11 px-[calc(--spacing(4)-1px)] text-lg sm:h-10 sm:text-base [&_svg:not([class*='size-'])]:size-5 sm:[&_svg:not([class*='size-'])]:size-4.5",
apps/web/src/components/ui/textarea.tsx
+22 -5
@@ -9,7 +9,13 @@ type TextareaProps = React.ComponentProps<"textarea"> & {
9 unstyled?: boolean;
10 };
11
12 -function Textarea({ className, size = "default", unstyled = false, ...props }: TextareaProps) {
12 +function Textarea({
13 + className,
14 + size = "default",
15 + unstyled = false,
16 + rows,
17 + ...props
18 +}: TextareaProps) {
19 return (
20 <span
21 className={
@@ -26,12 +32,23 @@ function Textarea({ className, size = "default", unstyled = false, ...props }: T
32 render={(defaultProps) => (
33 <textarea
34 className={cn(
29 - "field-sizing-content min-h-17.5 max-h-64 w-full rounded-[inherit] px-[calc(--spacing(3)-1px)] py-[calc(--spacing(1.5)-1px)] outline-none wrap-anywhere max-sm:min-h-20.5",
30 - size === "sm" &&
31 - "min-h-16.5 px-[calc(--spacing(2.5)-1px)] py-[calc(--spacing(1)-1px)] max-sm:min-h-19.5",
32 - size === "lg" && "min-h-18.5 py-[calc(--spacing(2)-1px)] max-sm:min-h-21.5",
35 + "field-sizing-content max-h-64 w-full rounded-[inherit] px-[calc(--spacing(3)-1px)] py-[calc(--spacing(1.5)-1px)] outline-none wrap-anywhere",
36 + rows === undefined && "min-h-17.5 max-sm:min-h-20.5",
37 + size === "sm" && "px-[calc(--spacing(2.5)-1px)] py-[calc(--spacing(1)-1px)]",
38 + size === "sm" && rows === undefined && "min-h-16.5 max-sm:min-h-19.5",
39 + size === "lg" && "py-[calc(--spacing(2)-1px)]",
40 + size === "lg" && rows === undefined && "min-h-18.5 max-sm:min-h-21.5",
41 )}
42 + // field-sizing: content ignores `rows`, so an explicit row count sets the minimum height.
43 + style={
44 + rows === undefined
45 + ? undefined
46 + : {
47 + minHeight: `calc(${rows}lh + 2 * (var(--spacing) * ${size === "sm" ? 1 : size === "lg" ? 2 : 1.5} - 1px))`,
48 + }
49 + }
50 data-slot="textarea"
51 + rows={rows}
52 {...mergeProps(defaultProps, props)}
53 />
54 )}
apps/web/src/hooks/useLocalStorage.ts
+1 -1
@@ -134,7 +134,7 @@ export function readLocalStorageValue<T, E>(
134 /** Fires on writes from this tab and from other tabs. */
135 export function subscribeLocalStorageKey(key: string, onChange: () => void): () => void {
136 const handleStorageChange = (event: StorageEvent) => {
137 - if (event.key === key) {
137 + if (event.key === key || event.key === null) {
138 onChange();
139 }
140 };
apps/web/src/keybindings.test.ts
+17
@@ -767,6 +767,23 @@ describe("isRichTextBoldShortcut", () => {
767 });
768
769 describe("composer and pull request shortcuts", () => {
770 + it("uses the dictation chord outside the terminal and preserves terminal split", () => {
771 + const chord = event({ key: "d", ctrlKey: true, shiftKey: true });
772 + assert.strictEqual(
773 + resolveShortcutCommand(chord, DEFAULT_RESOLVED_KEYBINDINGS, {
774 + platform: "Linux",
775 + context: { terminalFocus: false },
776 + }),
777 + "composer.dictation",
778 + );
779 + assert.strictEqual(
780 + resolveShortcutCommand(chord, DEFAULT_RESOLVED_KEYBINDINGS, {
781 + platform: "Linux",
782 + context: { terminalFocus: true },
783 + }),
784 + "terminal.splitVertical",
785 + );
786 + });
787 it("fills missing number shortcuts without replacing the saved URL binding", () => {
788 const olderServerBindings = DEFAULT_RESOLVED_KEYBINDINGS.filter(
789 (binding) =>
apps/web/src/routeTree.gen.ts
+21
@@ -23,6 +23,7 @@ import { Route as ChatPullRequestsRouteImport } from './routes/_chat.pull-reques
23 import { Route as ChatNewRouteImport } from './routes/_chat.new'
24 import { Route as ThreadsEnvironmentIdThreadIdRouteImport } from './routes/threads.$environmentId.$threadId'
25 import { Route as ChatSettingsWorktreesRouteImport } from './routes/_chat.settings.worktrees'
26 +import { Route as ChatSettingsVoiceRouteImport } from './routes/_chat.settings.voice'
27 import { Route as ChatSettingsThreadsRouteImport } from './routes/_chat.settings.threads'
28 import { Route as ChatSettingsStorageRouteImport } from './routes/_chat.settings.storage'
29 import { Route as ChatSettingsSourceControlRouteImport } from './routes/_chat.settings.source-control'
@@ -121,6 +122,11 @@ const ChatSettingsWorktreesRoute = ChatSettingsWorktreesRouteImport.update({
122 path: '/worktrees',
123 getParentRoute: () => ChatSettingsRoute,
124 } as any)
125 +const ChatSettingsVoiceRoute = ChatSettingsVoiceRouteImport.update({
126 + id: '/voice',
127 + path: '/voice',
128 + getParentRoute: () => ChatSettingsRoute,
129 +} as any)
130 const ChatSettingsThreadsRoute = ChatSettingsThreadsRouteImport.update({
131 id: '/threads',
132 path: '/threads',
@@ -301,6 +307,7 @@ export interface FileRoutesByFullPath {
307 '/settings/source-control': typeof ChatSettingsSourceControlRoute
308 '/settings/storage': typeof ChatSettingsStorageRoute
309 '/settings/threads': typeof ChatSettingsThreadsRoute
310 + '/settings/voice': typeof ChatSettingsVoiceRoute
311 '/settings/worktrees': typeof ChatSettingsWorktreesRoute
312 '/threads/$environmentId/$threadId': typeof ThreadsEnvironmentIdThreadIdRoute
313 }
@@ -343,6 +350,7 @@ export interface FileRoutesByTo {
350 '/settings/source-control': typeof ChatSettingsSourceControlRoute
351 '/settings/storage': typeof ChatSettingsStorageRoute
352 '/settings/threads': typeof ChatSettingsThreadsRoute
353 + '/settings/voice': typeof ChatSettingsVoiceRoute
354 '/settings/worktrees': typeof ChatSettingsWorktreesRoute
355 '/threads/$environmentId/$threadId': typeof ThreadsEnvironmentIdThreadIdRoute
356 }
@@ -387,6 +395,7 @@ export interface FileRoutesById {
395 '/_chat/settings/source-control': typeof ChatSettingsSourceControlRoute
396 '/_chat/settings/storage': typeof ChatSettingsStorageRoute
397 '/_chat/settings/threads': typeof ChatSettingsThreadsRoute
398 + '/_chat/settings/voice': typeof ChatSettingsVoiceRoute
399 '/_chat/settings/worktrees': typeof ChatSettingsWorktreesRoute
400 '/threads/$environmentId/$threadId': typeof ThreadsEnvironmentIdThreadIdRoute
401 }
@@ -431,6 +440,7 @@ export interface FileRouteTypes {
440 | '/settings/source-control'
441 | '/settings/storage'
442 | '/settings/threads'
443 + | '/settings/voice'
444 | '/settings/worktrees'
445 | '/threads/$environmentId/$threadId'
446 fileRoutesByTo: FileRoutesByTo
@@ -473,6 +483,7 @@ export interface FileRouteTypes {
483 | '/settings/source-control'
484 | '/settings/storage'
485 | '/settings/threads'
486 + | '/settings/voice'
487 | '/settings/worktrees'
488 | '/threads/$environmentId/$threadId'
489 id:
@@ -516,6 +527,7 @@ export interface FileRouteTypes {
527 | '/_chat/settings/source-control'
528 | '/_chat/settings/storage'
529 | '/_chat/settings/threads'
530 + | '/_chat/settings/voice'
531 | '/_chat/settings/worktrees'
532 | '/threads/$environmentId/$threadId'
533 fileRoutesById: FileRoutesById
@@ -631,6 +643,13 @@ declare module '@tanstack/react-router' {
643 preLoaderRoute: typeof ChatSettingsWorktreesRouteImport
644 parentRoute: typeof ChatSettingsRoute
645 }
646 + '/_chat/settings/voice': {
647 + id: '/_chat/settings/voice'
648 + path: '/voice'
649 + fullPath: '/settings/voice'
650 + preLoaderRoute: typeof ChatSettingsVoiceRouteImport
651 + parentRoute: typeof ChatSettingsRoute
652 + }
653 '/_chat/settings/threads': {
654 id: '/_chat/settings/threads'
655 path: '/threads'
@@ -849,6 +868,7 @@ interface ChatSettingsRouteChildren {
868 ChatSettingsSourceControlRoute: typeof ChatSettingsSourceControlRoute
869 ChatSettingsStorageRoute: typeof ChatSettingsStorageRoute
870 ChatSettingsThreadsRoute: typeof ChatSettingsThreadsRoute
871 + ChatSettingsVoiceRoute: typeof ChatSettingsVoiceRoute
872 ChatSettingsWorktreesRoute: typeof ChatSettingsWorktreesRoute
873 }
874
@@ -878,6 +898,7 @@ const ChatSettingsRouteChildren: ChatSettingsRouteChildren = {
898 ChatSettingsSourceControlRoute: ChatSettingsSourceControlRoute,
899 ChatSettingsStorageRoute: ChatSettingsStorageRoute,
900 ChatSettingsThreadsRoute: ChatSettingsThreadsRoute,
901 + ChatSettingsVoiceRoute: ChatSettingsVoiceRoute,
902 ChatSettingsWorktreesRoute: ChatSettingsWorktreesRoute,
903 }
904
apps/web/src/routes/_chat.settings.voice.tsx new
+7
@@ -0,0 +1,7 @@
1 +import { createFileRoute } from "@tanstack/react-router";
2 +
3 +import { VoiceSettingsPanel } from "../components/settings/VoiceSettingsPanel";
4 +
5 +export const Route = createFileRoute("/_chat/settings/voice")({
6 + component: VoiceSettingsPanel,
7 +});
apps/web/src/speech/browserMicrophones.ts new
+11
@@ -0,0 +1,11 @@
1 +export type BrowserMicrophone = Pick<MediaDeviceInfo, "deviceId" | "label">;
2 +
3 +export async function listBrowserMicrophones(): Promise<BrowserMicrophone[]> {
4 + const devices = await navigator.mediaDevices.enumerateDevices();
5 + return devices
6 + .filter((device) => device.kind === "audioinput")
7 + .map(({ deviceId, label }, index) => ({
8 + deviceId,
9 + label: label || `Microphone ${index + 1}`,
10 + }));
11 +}
apps/web/src/speech/browserPcmCapture.browser.test.ts new
+28
@@ -0,0 +1,28 @@
1 +import { expect, it, vi } from "vite-plus/test";
2 +import { createBrowserPcmCapture } from "./browserPcmCapture";
3 +
4 +it("cancels an unanswered permission prompt and stops a late microphone stream", async () => {
5 + let grantPermission!: (media: MediaStream) => void;
6 + const permission = new Promise<MediaStream>((resolve) => {
7 + grantPermission = resolve;
8 + });
9 + using microphone = vi.spyOn(navigator.mediaDevices, "getUserMedia").mockReturnValue(permission);
10 + const capture = createBrowserPcmCapture(
11 + { getMicrophoneId: () => "", onLevel: vi.fn(), onDurationLimit: vi.fn(), onError: vi.fn() },
12 + async () => ({ locale: "en", transcribe: async () => "" }),
13 + );
14 + const abort = new AbortController();
15 + await capture.transcriber.prepare({ signal: abort.signal });
16 + const rejected = vi.fn();
17 + const preparing = capture.recorder.prepareToRecordAsync(abort.signal).catch(rejected);
18 + expect(microphone).toHaveBeenCalledOnce();
19 + abort.abort();
20 + const stop = vi.fn();
21 + try {
22 + await vi.waitFor(() => expect(rejected).toHaveBeenCalledWith(abort.signal.reason));
23 + } finally {
24 + grantPermission({ getTracks: () => [{ stop }] } as unknown as MediaStream);
25 + await preparing;
26 + }
27 + expect(stop).toHaveBeenCalledOnce();
28 +});
apps/web/src/speech/browserPcmCapture.ts new
+180
@@ -0,0 +1,180 @@
1 +import type {
2 + PreparedVoiceTranscription,
3 + VoiceRecorder,
4 + VoiceStreamingSession,
5 + VoiceTranscriptionOptions,
6 +} from "@t3tools/client-runtime/voice-input";
7 +import workletUrl from "./pcmWorklet.ts?worker&url";
8 +
9 +type PreparedBrowserTranscription = PreparedVoiceTranscription & {
10 + readonly streaming?: VoiceStreamingSession & { readonly feed: (pcm: Float32Array) => void };
11 +};
12 +
13 +export function createBrowserPcmCapture(
14 + input: {
15 + readonly getMicrophoneId: () => string;
16 + readonly onLevel: (level: number) => void;
17 + readonly onDurationLimit: () => void;
18 + readonly onError: (message: string) => void;
19 + },
20 + prepare: (options: VoiceTranscriptionOptions) => Promise<PreparedBrowserTranscription>,
21 +) {
22 + let stream: MediaStream | undefined;
23 + let recordingUri: string | null = null;
24 + let chunks: Float32Array<ArrayBuffer>[] = [];
25 + let audioContext: AudioContext | undefined;
26 + let worklet: AudioWorkletNode | undefined;
27 + let durationTimer: ReturnType<typeof setTimeout> | undefined;
28 + let signal: AbortSignal | undefined;
29 + let feed: ((pcm: Float32Array) => void) | undefined;
30 + let stopped:
31 + | { readonly resolve: () => void; readonly reject: (error: Error) => void }
32 + | undefined;
33 +
34 + const cleanupCapture = () => {
35 + if (durationTimer) clearTimeout(durationTimer);
36 + durationTimer = undefined;
37 + worklet?.disconnect();
38 + worklet?.port.close();
39 + worklet = undefined;
40 + stream?.getTracks().forEach((track) => track.stop());
41 + stream = undefined;
42 + void audioContext?.close();
43 + audioContext = undefined;
44 + stopped?.resolve();
45 + stopped = undefined;
46 + chunks = [];
47 + feed = undefined;
48 + input.onLevel(0);
49 + };
50 +
51 + const recorder: VoiceRecorder = {
52 + get uri() {
53 + return recordingUri;
54 + },
55 + prepareToRecordAsync: async () => {
56 + const captureSignal = signal;
57 + if (!captureSignal) throw new Error("Voice transcription is not prepared.");
58 + captureSignal.throwIfAborted();
59 + if (recordingUri) URL.revokeObjectURL(recordingUri);
60 + recordingUri = null;
61 + chunks = [];
62 + const microphoneId = input.getMicrophoneId();
63 + const captured = await new Promise<MediaStream>((resolve, reject) => {
64 + const onAbort = () => reject(captureSignal.reason);
65 + const permission = navigator.mediaDevices.getUserMedia({
66 + audio: microphoneId ? { deviceId: { exact: microphoneId } } : true,
67 + });
68 + captureSignal.addEventListener("abort", onAbort, { once: true });
69 + void permission.then(
70 + (media) => {
71 + captureSignal.removeEventListener("abort", onAbort);
72 + if (captureSignal.aborted) media.getTracks().forEach((track) => track.stop());
73 + else resolve(media);
74 + },
75 + (error: unknown) => {
76 + captureSignal.removeEventListener("abort", onAbort);
77 + reject(error);
78 + },
79 + );
80 + });
81 + if (captureSignal.aborted) {
82 + captured.getTracks().forEach((track) => track.stop());
83 + captureSignal.throwIfAborted();
84 + }
85 + stream = captured;
86 + try {
87 + const context = new AudioContext();
88 + audioContext = context;
89 + await context.audioWorklet.addModule(workletUrl);
90 + captureSignal.throwIfAborted();
91 + const node = new AudioWorkletNode(context, "t3-pcm-capture", { channelCount: 1 });
92 + worklet = node;
93 + node.port.onmessage = ({
94 + data,
95 + }: MessageEvent<Float32Array<ArrayBuffer> | number | "stopped" | "limit">) => {
96 + if (captureSignal.aborted) return;
97 + if (data === "stopped") {
98 + stopped?.resolve();
99 + return;
100 + }
101 + if (data === "limit") {
102 + input.onDurationLimit();
103 + return;
104 + }
105 + if (typeof data === "number") {
106 + input.onLevel(data);
107 + return;
108 + }
109 + if (feed) feed(data);
110 + else chunks.push(data);
111 + };
112 + node.onprocessorerror = () => {
113 + stopped?.reject(new Error("Microphone processing failed."));
114 + input.onError("Microphone processing failed.");
115 + };
116 + for (const track of captured.getTracks())
117 + track.addEventListener("ended", () => input.onError("The microphone disconnected."), {
118 + once: true,
119 + });
120 + context.createMediaStreamSource(captured).connect(node);
121 + // The processor emits silence, keeping the graph active without microphone playback.
122 + node.connect(context.destination);
123 + await context.resume();
124 + captureSignal.throwIfAborted();
125 + } catch (error) {
126 + cleanupCapture();
127 + throw error;
128 + }
129 + },
130 + record: ({ forDuration }) => {
131 + // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
132 + worklet?.port.postMessage("start");
133 + durationTimer = setTimeout(input.onDurationLimit, forDuration * 1_000);
134 + },
135 + stop: async () => {
136 + if (!worklet) return;
137 + if (signal?.aborted) {
138 + cleanupCapture();
139 + return;
140 + }
141 + const pending = new Promise<void>((resolve, reject) => {
142 + stopped = { resolve, reject };
143 + });
144 + const timeout = setTimeout(
145 + () => stopped?.reject(new Error("Microphone capture did not stop.")),
146 + 5_000,
147 + );
148 + // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
149 + worklet.port.postMessage("stop");
150 + try {
151 + await pending;
152 + if (!feed && chunks.length) {
153 + const blob = new Blob(chunks, { type: "application/octet-stream" });
154 + recordingUri = URL.createObjectURL(blob);
155 + }
156 + } finally {
157 + clearTimeout(timeout);
158 + cleanupCapture();
159 + }
160 + },
161 + };
162 +
163 + return {
164 + recorder,
165 + cancelRecording: cleanupCapture,
166 + deleteRecording: (uri: string) => {
167 + URL.revokeObjectURL(uri);
168 + if (recordingUri === uri) recordingUri = null;
169 + },
170 + transcriber: {
171 + prepare: async (options: VoiceTranscriptionOptions) => {
172 + const prepared = await prepare(options);
173 + options.signal.throwIfAborted();
174 + signal = options.signal;
175 + feed = prepared.streaming?.feed;
176 + return prepared;
177 + },
178 + },
179 + };
180 +}
apps/web/src/speech/browserVoiceInput.ts new
+89
@@ -0,0 +1,89 @@
1 +import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
2 +import {
3 + getEnvironmentSpeechStatus,
4 + getEnvironmentSpeechStreamUrl,
5 + openSpeechStream,
6 + throwIfVoiceTranscriptionAborted,
7 + transcribeEnvironmentPcm,
8 + VoiceTranscriptionError,
9 + type VoiceTranscriptionOptions,
10 +} from "@t3tools/client-runtime/voice-input";
11 +import type { PreparedConnection } from "@t3tools/client-runtime/connection";
12 +import type { SpeechTranscriptionOptions, SpeechStreamText } from "@t3tools/contracts";
13 +
14 +import { runtime } from "../lib/runtime";
15 +import { createBrowserPcmCapture } from "./browserPcmCapture";
16 +
17 +export function createBrowserVoiceInputPlatform(input: {
18 + readonly prepared: PreparedConnection;
19 + readonly hasAudioConsent: () => boolean;
20 + readonly getTranscriptionOptions: () => SpeechTranscriptionOptions;
21 + readonly getMicrophoneId: () => string;
22 + readonly onLevel: (level: number) => void;
23 + readonly onDurationLimit: () => void;
24 + readonly onText: (text: SpeechStreamText) => void;
25 + readonly onError: (message: string) => void;
26 +}) {
27 + return createBrowserPcmCapture(input, async (options) => {
28 + const recordingOptions = input.getTranscriptionOptions();
29 + const status = await runtime.runPromise(getEnvironmentSpeechStatus(input.prepared), options);
30 + throwIfVoiceTranscriptionAborted(options.signal);
31 + if (!status.supported)
32 + throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
33 + const language = status.languages
34 + ? effectiveSpeechLanguage(
35 + {
36 + languages: status.languages,
37 + supportsLanguageDetection: status.supportsLanguageDetection ?? false,
38 + },
39 + recordingOptions.speechLanguage,
40 + )
41 + : "auto";
42 + const locale =
43 + !status.supportsStreaming &&
44 + status.supportsTranslation &&
45 + recordingOptions.speechTranslateToEnglish
46 + ? "en"
47 + : language;
48 + const transcribeRecording = async (
49 + uri: string,
50 + { signal: transcriptionSignal }: VoiceTranscriptionOptions,
51 + ) => {
52 + try {
53 + const response = await fetch(uri, { signal: transcriptionSignal });
54 + const pcm = new Uint8Array(await response.arrayBuffer());
55 + const result = await runtime.runPromise(
56 + transcribeEnvironmentPcm(input.prepared, pcm, recordingOptions, input.hasAudioConsent),
57 + {
58 + signal: transcriptionSignal,
59 + },
60 + );
61 + throwIfVoiceTranscriptionAborted(transcriptionSignal);
62 + return result.text;
63 + } catch (cause) {
64 + throwIfVoiceTranscriptionAborted(transcriptionSignal);
65 + throw new VoiceTranscriptionError({
66 + code: "transcription-failed",
67 + message: "Voice transcription on this environment failed.",
68 + cause,
69 + });
70 + }
71 + };
72 + if (!status.supportsStreaming && !status.supportsBufferedStreaming) {
73 + return { locale, transcribe: transcribeRecording };
74 + }
75 + const url = await runtime.runPromise(getEnvironmentSpeechStreamUrl(input.prepared), options);
76 + return {
77 + locale,
78 + transcribe: transcribeRecording,
79 + streaming: openSpeechStream({
80 + url,
81 + canSend: input.hasAudioConsent,
82 + options: recordingOptions,
83 + signal: options.signal,
84 + onText: input.onText,
85 + onError: (error) => input.onError(error.message),
86 + }),
87 + };
88 + });
89 +}
apps/web/src/speech/pcmResampler.ts new
+54
@@ -0,0 +1,54 @@
1 +const TARGET_RATE = 16_000;
2 +
3 +/** Low-pass before downsampling, preserving filter and sample phase across capture blocks. */
4 +export class PcmResampler {
5 + private readonly coefficients: Float64Array;
6 + private readonly history: Float32Array;
7 + private position = 0;
8 + private phase = 0;
9 + private readonly ratio: number;
10 +
11 + constructor(sampleRate: number) {
12 + if (sampleRate < TARGET_RATE)
13 + throw new Error("Microphone sample rate must be at least 16 kHz.");
14 + this.ratio = sampleRate / TARGET_RATE;
15 + this.coefficients = new Float64Array(sampleRate === TARGET_RATE ? 1 : 63);
16 + this.history = new Float32Array(this.coefficients.length);
17 + const middle = (this.coefficients.length - 1) / 2;
18 + const cutoff = 7_200 / sampleRate;
19 + let sum = 0;
20 + for (let i = 0; i < this.coefficients.length; i++) {
21 + const x = i - middle;
22 + const sinc = x === 0 ? 2 * cutoff : Math.sin(2 * Math.PI * cutoff * x) / (Math.PI * x);
23 + const window = middle === 0 ? 1 : 0.54 - 0.46 * Math.cos((Math.PI * i) / middle);
24 + this.coefficients[i] = sinc * window;
25 + sum += this.coefficients[i]!;
26 + }
27 + for (let i = 0; i < this.coefficients.length; i++) this.coefficients[i]! /= sum;
28 + }
29 +
30 + push(samples: Float32Array): Float32Array {
31 + const output: number[] = [];
32 + for (const sample of samples) {
33 + this.history[this.position] = sample;
34 + this.phase += 1;
35 + if (this.phase >= this.ratio) {
36 + this.phase -= this.ratio;
37 + let value = 0;
38 + for (let tap = 0; tap < this.coefficients.length; tap++) {
39 + const index = (this.position - tap + this.history.length) % this.history.length;
40 + value += this.history[index]! * this.coefficients[tap]!;
41 + }
42 + output.push(Math.max(-1, Math.min(1, value)));
43 + }
44 + this.position = (this.position + 1) % this.history.length;
45 + }
46 + return Float32Array.from(output);
47 + }
48 +
49 + flush(): Float32Array {
50 + return this.coefficients.length === 1
51 + ? new Float32Array()
52 + : this.push(new Float32Array(this.history.length - 1));
53 + }
54 +}
apps/web/src/speech/pcmWorklet.test.ts new
+64
@@ -0,0 +1,64 @@
1 +import { expect, it, vi } from "vite-plus/test";
2 +
3 +it("meters microphone-only capture like transcription without emitting PCM", async () => {
4 + using globals = new DisposableStack();
5 + const setGlobal = (name: string, value: unknown) => {
6 + const original = Object.getOwnPropertyDescriptor(globalThis, name);
7 + globals.defer(() => {
8 + if (original) Object.defineProperty(globalThis, name, original);
9 + else Reflect.deleteProperty(globalThis, name);
10 + });
11 + Object.defineProperty(globalThis, name, { value, configurable: true, writable: true });
12 + };
13 + class Worklet {
14 + readonly port = {
15 + onmessage: null as ((event: { data: string }) => void) | null,
16 + postMessage: vi.fn<(data: unknown) => void>(),
17 + };
18 + process(_inputs: Float32Array[][]): boolean {
19 + return true;
20 + }
21 + }
22 + let Processor!: new (options?: { processorOptions: { meterOnly: boolean } }) => Worklet;
23 + setGlobal("sampleRate", 48_000);
24 + setGlobal("AudioWorkletProcessor", Worklet);
25 + setGlobal("registerProcessor", (_name: string, processor: typeof Processor) => {
26 + Processor = processor;
27 + });
28 + await import("./pcmWorklet");
29 + const transcription = new Processor();
30 + const microphone = new Processor({ processorOptions: { meterOnly: true } });
31 + transcription.port.onmessage?.({ data: "start" });
32 + microphone.port.onmessage?.({ data: "start" });
33 + const feed = (samples: Float32Array) => {
34 + transcription.process([[samples]]);
35 + microphone.process([[samples]]);
36 + };
37 + const levels = (processor: Worklet) =>
38 + processor.port.postMessage.mock.calls
39 + .map(([data]) => data)
40 + .filter((data) => typeof data === "number");
41 +
42 + feed(new Float32Array(2_399).fill(0.1));
43 + expect(levels(microphone)).toHaveLength(0);
44 + feed(new Float32Array(1).fill(0.1));
45 + expect(levels(microphone)).toHaveLength(1);
46 + feed(new Float32Array(48_000).fill(0.25));
47 + feed(new Float32Array(2_400));
48 +
49 + const measured = levels(microphone);
50 + expect(measured).toEqual(levels(transcription));
51 + expect(measured).toHaveLength(22);
52 + expect(measured[0]).toBeGreaterThan(0);
53 + expect(measured[0]).toBeLessThan(0.4);
54 + expect(measured.at(-1)).toBeGreaterThan(0);
55 + expect(measured.at(-1)).toBeLessThan(measured.at(-2)!);
56 + expect(
57 + transcription.port.postMessage.mock.calls.some(([data]) => data instanceof Float32Array),
58 + ).toBe(true);
59 + expect(microphone.port.postMessage.mock.calls.every(([data]) => typeof data === "number")).toBe(
60 + true,
61 + );
62 + microphone.port.onmessage?.({ data: "stop" });
63 + expect(microphone.port.postMessage).toHaveBeenLastCalledWith("stopped");
64 +});
apps/web/src/speech/pcmWorklet.ts new
+84
@@ -0,0 +1,84 @@
1 +import { PcmResampler } from "./pcmResampler";
2 +
3 +declare const sampleRate: number;
4 +declare class AudioWorkletProcessor {
5 + readonly port: MessagePort;
6 +}
7 +declare function registerProcessor(name: string, processor: typeof AudioWorkletProcessor): void;
8 +
9 +class PcmCaptureProcessor extends AudioWorkletProcessor {
10 + private readonly resampler = new PcmResampler(sampleRate);
11 + private chunk = new Float32Array(8_000);
12 + private length = 0;
13 + private recording = false;
14 + private flushed = false;
15 + private samples = 0;
16 + private levelSamples = 0;
17 + private levelEnergy = 0;
18 + private level = 0;
19 + private readonly meterOnly: boolean;
20 +
21 + constructor(options?: { readonly processorOptions?: { readonly meterOnly?: boolean } }) {
22 + super();
23 + this.meterOnly = options?.processorOptions?.meterOnly === true;
24 + this.port.onmessage = ({ data }) => {
25 + if (data === "start") this.recording = true;
26 + if (data === "stop") {
27 + this.recording = false;
28 + if (!this.meterOnly && !this.flushed) this.append(this.resampler.flush());
29 + this.flushed = true;
30 + if (this.length) this.publish();
31 + // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
32 + this.port.postMessage("stopped");
33 + }
34 + };
35 + }
36 +
37 + private publish() {
38 + const pcm = this.chunk.slice(0, this.length);
39 + this.port.postMessage(pcm, [pcm.buffer]);
40 + this.length = 0;
41 + }
42 +
43 + private append(pcm: Float32Array) {
44 + for (const sample of pcm) {
45 + if (this.samples === 16_000 * 300) {
46 + this.recording = false;
47 + // oxlint-disable-next-line unicorn/require-post-message-target-origin -- AudioWorklet MessagePorts do not accept a target origin
48 + this.port.postMessage("limit");
49 + break;
50 + }
51 + this.chunk[this.length++] = sample;
52 + this.samples++;
53 + if (this.length === this.chunk.length) this.publish();
54 + }
55 + }
56 +
57 + process(inputs: Float32Array[][]): boolean {
58 + if (!this.recording) return true;
59 + const channels = inputs[0];
60 + if (!channels?.[0]) return true;
61 + const mono = new Float32Array(channels[0].length);
62 + for (const channel of channels)
63 + for (let i = 0; i < mono.length; i++) mono[i]! += channel[i]! / channels.length;
64 + // Metering must not wait for the half-second transcription chunks.
65 + for (const sample of mono) {
66 + this.levelEnergy += sample * sample;
67 + this.levelSamples++;
68 + if (this.levelSamples >= sampleRate / 20) {
69 + const target = Math.min(1, Math.sqrt(this.levelEnergy / this.levelSamples) * 4);
70 + // Soften brief peaks without delaying meter updates or transcription.
71 + const timeConstant = target > this.level ? 0.075 : 0.2;
72 + const decay = Math.exp(-this.levelSamples / sampleRate / timeConstant);
73 + this.level = target + (this.level - target) * decay;
74 + this.port.postMessage(this.level, []);
75 + this.levelEnergy = 0;
76 + this.levelSamples = 0;
77 + }
78 + }
79 + if (!this.meterOnly) this.append(this.resampler.push(mono));
80 + return true;
81 + }
82 +}
83 +
84 +registerProcessor("t3-pcm-capture", PcmCaptureProcessor);
apps/web/src/speech/useDictationShortcut.browser.test.tsx new
+82
@@ -0,0 +1,82 @@
1 +import { DEFAULT_RESOLVED_KEYBINDINGS } from "@t3tools/shared/keybindings";
2 +import { expect, it, vi } from "vite-plus/test";
3 +import { render } from "../../test/render";
4 +import { useDictationShortcut } from "./useDictationShortcut";
5 +
6 +vi.mock("../hooks/useSettings", () => ({ useClientSettings: () => "hold" }));
7 +
8 +function Probe({ speech }: { speech: Parameters<typeof useDictationShortcut>[0]["speech"] }) {
9 + useDictationShortcut({
10 + keybindings: DEFAULT_RESOLVED_KEYBINDINGS,
11 + speech,
12 + disabled: false,
13 + terminalOpen: false,
14 + modelPickerOpen: false,
15 + });
16 + return null;
17 +}
18 +
19 +const press = (repeat = false) =>
20 + window.dispatchEvent(
21 + new KeyboardEvent("keydown", {
22 + key: "D",
23 + code: "KeyD",
24 + ctrlKey: !/Mac/i.test(navigator.platform),
25 + metaKey: /Mac/i.test(navigator.platform),
26 + shiftKey: true,
27 + cancelable: true,
28 + repeat,
29 + }),
30 + );
31 +
32 +function actions(available: boolean) {
33 + return {
34 + available,
35 + state: { phase: "idle" as const, error: null, errorAction: null },
36 + start: vi.fn(async () => {}),
37 + stop: vi.fn(async () => {}),
38 + cancel: vi.fn(),
39 + };
40 +}
41 +
42 +it("leaves the browser shortcut alone when dictation is off", async () => {
43 + const speech = actions(false);
44 + await using _screen = await render(<Probe speech={speech} />);
45 + expect(press()).toBe(true);
46 + expect(speech.start).not.toHaveBeenCalled();
47 +});
48 +
49 +it("stops on a new press when the OS omitted keyup, but ignores autorepeat", async () => {
50 + const speech = actions(true);
51 + await using screen = await render(<Probe speech={speech} />);
52 + expect(press()).toBe(false);
53 + expect(speech.start).toHaveBeenCalledOnce();
54 + await screen.rerender(
55 + <Probe speech={{ ...speech, state: { ...speech.state, phase: "recording" } }} />,
56 + );
57 + press(true);
58 + expect(speech.stop).not.toHaveBeenCalled();
59 + press();
60 + expect(speech.stop).toHaveBeenCalledOnce();
61 +});
62 +
63 +it("finishes a released hold after startup even if the chord was pressed again while preparing", async () => {
64 + const speech = actions(true);
65 + let finishStart!: () => void;
66 + const startup = new Promise<void>((resolve) => {
67 + finishStart = resolve;
68 + });
69 + speech.start.mockReturnValue(startup);
70 + await using screen = await render(<Probe speech={speech} />);
71 + press();
72 + await screen.rerender(
73 + <Probe speech={{ ...speech, state: { ...speech.state, phase: "preparing" } }} />,
74 + );
75 + window.dispatchEvent(new KeyboardEvent("keyup", { code: "KeyD" }));
76 + press();
77 + window.dispatchEvent(new KeyboardEvent("keyup", { code: "KeyD" }));
78 + finishStart();
79 + await startup;
80 + expect(speech.start).toHaveBeenCalledOnce();
81 + expect(speech.stop).toHaveBeenCalledOnce();
82 +});
apps/web/src/speech/useDictationShortcut.ts new
+148
@@ -0,0 +1,148 @@
1 +import type { VoiceInputState } from "@t3tools/client-runtime/voice-input";
2 +import type { ResolvedKeybindingsConfig } from "@t3tools/contracts";
3 +import { useEffect, useRef, type RefObject } from "react";
4 +
5 +import { isCommandPaletteOpen } from "../commandPaletteBus";
6 +import { useClientSettings } from "../hooks/useSettings";
7 +import { resolveShortcutCommand, shortcutLabelForCommand } from "../keybindings";
8 +import { getTerminalFocusOwner } from "../lib/terminalFocus";
9 +
10 +const HOLD_THRESHOLD_MS = 300;
11 +
12 +function shouldFinishDictationOnRelease(mode: "auto" | "hold" | "toggle", elapsedMs: number) {
13 + return mode === "hold" || (mode === "auto" && elapsedMs >= HOLD_THRESHOLD_MS);
14 +}
15 +
16 +type SpeechActions = {
17 + available: boolean;
18 + state: VoiceInputState;
19 + start(): Promise<void>;
20 + stop(): Promise<void>;
21 + cancel(): void;
22 +};
23 +
24 +type ShortcutInput = {
25 + keybindings: ResolvedKeybindingsConfig;
26 + speech: SpeechActions;
27 + disabled: boolean;
28 + terminalOpen: boolean;
29 + modelPickerOpen: boolean;
30 + /** When set, the chord targets this editor only while it has focus. */
31 + targetRef?: RefObject<HTMLElement | null>;
32 +};
33 +
34 +// Several editors can bind the chord; the busy one, then the focused one, then the untargeted one wins.
35 +const shortcutTargets = new Map<symbol, () => ShortcutInput>();
36 +
37 +function ownsShortcut(id: symbol) {
38 + const entries = [...shortcutTargets];
39 + const busy = entries.find(([, read]) => {
40 + const phase = read().speech.state.phase;
41 + return phase !== "idle" && phase !== "error";
42 + });
43 + if (busy) return busy[0] === id;
44 + const focused = entries.find(([, read]) =>
45 + read().targetRef?.current?.contains(document.activeElement),
46 + );
47 + if (focused) return focused[0] === id;
48 + return entries.find(([, read]) => !read().targetRef)?.[0] === id;
49 +}
50 +
51 +export function useDictationShortcut(input: ShortcutInput) {
52 + const id = useRef(Symbol("dictation-target"));
53 + const mode = useClientSettings((settings) => settings.voiceShortcutMode);
54 + const current = useRef(input);
55 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed actions to the window keyboard listener and shortcut registry
56 + useEffect(() => {
57 + current.current = input;
58 + });
59 + const generation = useRef(0);
60 + const pressed = useRef<{
61 + generation: number;
62 + code: string;
63 + startedAt: number;
64 + mode: typeof mode;
65 + starting: Promise<void>;
66 + } | null>(null);
67 +
68 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the window keyboard listeners and dictation target registry entry
69 + useEffect(() => {
70 + if (typeof window === "undefined") return;
71 + let disposed = false;
72 + shortcutTargets.set(id.current, () => current.current);
73 + const finish = (event: KeyboardEvent | null) => {
74 + const press = pressed.current;
75 + if (!press || (event && (event.code || event.key) !== press.code)) return;
76 + pressed.current = null;
77 + if (shouldFinishDictationOnRelease(press.mode, performance.now() - press.startedAt)) {
78 + void press.starting.then(() => {
79 + if (!disposed && generation.current === press.generation)
80 + return current.current.speech.stop();
81 + });
82 + }
83 + };
84 +
85 + const onKeyDown = (event: KeyboardEvent) => {
86 + if (!ownsShortcut(id.current)) return;
87 + if (event.isComposing || event.keyCode === 229) return;
88 + const { speech } = current.current;
89 + if (event.defaultPrevented || isCommandPaletteOpen()) return;
90 + if (
91 + event.key === "Escape" &&
92 + (pressed.current !== null ||
93 + (speech.state.phase !== "idle" && speech.state.phase !== "error"))
94 + ) {
95 + event.preventDefault();
96 + event.stopImmediatePropagation();
97 + pressed.current = null;
98 + generation.current += 1;
99 + speech.cancel();
100 + return;
101 + }
102 + const { keybindings, terminalOpen, modelPickerOpen, disabled } = current.current;
103 + const command = resolveShortcutCommand(event, keybindings, {
104 + context: {
105 + terminalFocus: getTerminalFocusOwner() !== null,
106 + terminalOpen,
107 + modelPickerOpen,
108 + },
109 + });
110 + if (command !== "composer.dictation") return;
111 + const canStart = speech.state.phase === "idle" || speech.state.phase === "error";
112 + if (canStart && (!speech.available || disabled)) return;
113 + event.preventDefault();
114 + event.stopPropagation();
115 + if (event.repeat || (!canStart && speech.state.phase !== "recording")) return;
116 + // macOS may omit the previous non-modifier keyup while Command is held.
117 + pressed.current = null;
118 + generation.current += 1;
119 + if (speech.state.phase === "recording") {
120 + void speech.stop();
121 + return;
122 + }
123 + pressed.current = {
124 + generation: generation.current,
125 + code: event.code || event.key,
126 + startedAt: performance.now(),
127 + mode,
128 + starting: speech.start(),
129 + };
130 + };
131 + const onBlur = () => finish(null);
132 + window.addEventListener("keydown", onKeyDown, true);
133 + window.addEventListener("keyup", finish, true);
134 + window.addEventListener("blur", onBlur);
135 + return () => {
136 + disposed = true;
137 + pressed.current = null;
138 + shortcutTargets.delete(id.current);
139 + window.removeEventListener("keydown", onKeyDown, true);
140 + window.removeEventListener("keyup", finish, true);
141 + window.removeEventListener("blur", onBlur);
142 + };
143 + }, [mode]);
144 +
145 + return shortcutLabelForCommand(input.keybindings, "composer.dictation", {
146 + context: { terminalFocus: false, terminalOpen: input.terminalOpen },
147 + });
148 +}
apps/web/src/speech/useDictationSubmissionShortcut.ts new
+33
@@ -0,0 +1,33 @@
1 +import { useEffect, useRef, type RefObject } from "react";
2 +
3 +import { isCommandPaletteOpen } from "../commandPaletteBus";
4 +
5 +/** Keep submission shortcuts reachable when the frozen composer loses focus to the page. */
6 +export function useDictationSubmissionShortcut(input: {
7 + enabled: boolean;
8 + formRef: RefObject<HTMLFormElement | null>;
9 + onKeyDown(event: KeyboardEvent): void;
10 +}) {
11 + const latest = useRef(input);
12 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed inputs to the window keydown listener
13 + useEffect(() => {
14 + latest.current = input;
15 + });
16 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to window keydown while dictation owns submission
17 + useEffect(() => {
18 + if (!input.enabled) return;
19 + const onKeyDown = (event: KeyboardEvent) => {
20 + if (event.defaultPrevented || isCommandPaletteOpen()) return;
21 + const target = event.target;
22 + if (!(target instanceof Element)) return;
23 + if (target !== document.body && target !== document.documentElement) {
24 + if (!latest.current.formRef.current?.contains(target)) return;
25 + if (!target.closest('[data-testid="composer-editor"]')) return;
26 + if (target.closest("textarea, input, button")) return;
27 + }
28 + latest.current.onKeyDown(event);
29 + };
30 + window.addEventListener("keydown", onKeyDown, true);
31 + return () => window.removeEventListener("keydown", onKeyDown, true);
32 + }, [input.enabled]);
33 +}
apps/web/src/speech/useEnvironmentSpeechInput.browser.test.tsx new
+493
@@ -0,0 +1,493 @@
1 +import { useLayoutEffect } from "react";
2 +import { expect, it, vi } from "vite-plus/test";
3 +import * as Effect from "effect/Effect";
4 +import * as Option from "effect/Option";
5 +import type { PreparedConnection } from "@t3tools/client-runtime/connection";
6 +import {
7 + DEFAULT_CLIENT_SETTINGS,
8 + DEFAULT_SERVER_SETTINGS,
9 + type EnvironmentId,
10 + type ProjectId,
11 + type SpeechCustomWords,
12 + type SpeechPostProcessingOptions,
13 + type SpeechTranscriptionOptions,
14 +} from "@t3tools/contracts";
15 +
16 +import { act, render } from "../../test/render";
17 +import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
18 +import { chooseAudioDestination, chooseTranscriptCleanup, bindAudioGrant } from "./voiceConsent";
19 +import { writeBrowserClientSettings } from "../clientPersistenceStorage";
20 +
21 +const mocks = vi.hoisted(() => ({
22 + voice: undefined as unknown as ReturnType<typeof useEnvironmentSpeechInput>,
23 + settings: {} as Record<string, unknown>,
24 + connected: new Set<string>(),
25 + voiceCapable: new Set<string>(),
26 + canDictate: new Set<string>(),
27 + prepared: new Map<string, unknown>(),
28 + primaryEnvironmentId: "primary-env",
29 + statusRequests: [] as string[],
30 + statusUnavailable: false,
31 + modelState: "ready" as "ready" | "missing-model",
32 + download: vi.fn<() => Promise<void>>(),
33 + platforms: [] as string[],
34 + microphoneRequests: [] as string[],
35 + transcribeCalls: [] as string[],
36 + cleanupCalls: [] as string[],
37 + cleanupDraft: vi.fn(),
38 + cleanupOptions: [] as SpeechPostProcessingOptions[],
39 + recordingOptions: [] as SpeechTranscriptionOptions[],
40 + projectWords: {} as Record<string, SpeechCustomWords>,
41 + transcripts: [] as Array<string | Error>,
42 + committed: vi.fn(),
43 + draft: "",
44 +}));
45 +
46 +vi.mock("../state/session", () => ({
47 + useEnvironmentScope: (id: string) => mocks.canDictate.has(id),
48 + usePreparedConnection: (environmentId: string | null) => {
49 + if (!environmentId || !mocks.connected.has(environmentId)) return Option.none();
50 + let prepared = mocks.prepared.get(environmentId);
51 + if (!prepared) {
52 + prepared = { environmentId, label: environmentId };
53 + mocks.prepared.set(environmentId, prepared);
54 + }
55 + return Option.some(prepared);
56 + },
57 +}));
58 +vi.mock("../state/environments", () => ({
59 + usePrimaryEnvironmentId: () => mocks.primaryEnvironmentId,
60 + useEnvironment: (environmentId: string | null) =>
61 + environmentId && mocks.connected.has(environmentId)
62 + ? {
63 + environmentId,
64 + serverConfig: {
65 + environment: {
66 + capabilities: { voiceTranscription: mocks.voiceCapable.has(environmentId) },
67 + },
68 + },
69 + }
70 + : null,
71 +}));
72 +vi.mock("../hooks/useSettings", () => ({
73 + useClientSettingsHydrated: () => true,
74 + useClientSettings: (selector?: (settings: Record<string, unknown>) => unknown) => {
75 + const settings = { ...DEFAULT_CLIENT_SETTINGS, ...mocks.settings };
76 + return selector ? selector(settings) : settings;
77 + },
78 + useEnvironmentSettings: (
79 + environmentId: string,
80 + selector: (settings: Record<string, unknown>) => unknown,
81 + ) =>
82 + selector({
83 + ...DEFAULT_SERVER_SETTINGS,
84 + projectSettingsOverrides: Object.fromEntries(
85 + Object.entries(environmentId === THREAD_ENV ? mocks.projectWords : {}).map(
86 + ([projectId, words]) => [projectId, { speechProjectCustomWords: words }],
87 + ),
88 + ),
89 + }),
90 +}));
91 +vi.mock("../lib/runtime", () => ({ runtime: { runPromise: Effect.runPromise } }));
92 +vi.mock("../components/ui/toast", () => ({ toastManager: { add: vi.fn() } }));
93 +vi.mock("@t3tools/client-runtime/voice-input", async (importOriginal) => ({
94 + ...(await importOriginal<typeof import("@t3tools/client-runtime/voice-input")>()),
95 + getEnvironmentSpeechStatus: (prepared: PreparedConnection) =>
96 + Effect.sync(() => {
97 + mocks.statusRequests.push(prepared.environmentId);
98 + if (mocks.statusUnavailable) throw new Error("Temporarily offline");
99 + return {
100 + supported: true,
101 + state: mocks.modelState,
102 + modelId: "test-model",
103 + model: "Test",
104 + size: 1,
105 + supportsStreaming: false,
106 + supportsTranslation: false,
107 + acceleration: "auto",
108 + modelUnloadTimeout: "min_15",
109 + gpuDevices: [],
110 + };
111 + }),
112 + downloadEnvironmentSpeechModel: () => Effect.promise(mocks.download),
113 + getEnvironmentSpeechModels: () => Effect.succeed({ models: [] }),
114 + postProcessEnvironmentTranscript: (
115 + prepared: PreparedConnection,
116 + transcript: string,
117 + draft: unknown,
118 + options: SpeechPostProcessingOptions,
119 + ) =>
120 + Effect.sync(() => {
121 + mocks.cleanupCalls.push(prepared.environmentId);
122 + mocks.cleanupDraft(draft);
123 + mocks.cleanupOptions.push(options);
124 + return { text: `Cleaned ${transcript}` };
125 + }),
126 +}));
127 +// Capture and the wire live behind the platform; its `prepared` is where audio would go.
128 +vi.mock("./browserVoiceInput", () => ({
129 + createBrowserVoiceInputPlatform: (input: {
130 + prepared: PreparedConnection;
131 + getTranscriptionOptions?: () => SpeechTranscriptionOptions;
132 + }) => {
133 + const environmentId = input.prepared.environmentId;
134 + mocks.platforms.push(environmentId);
135 + return {
136 + recorder: {
137 + uri: `blob:${environmentId}`,
138 + prepareToRecordAsync: async () => {
139 + mocks.microphoneRequests.push(environmentId);
140 + },
141 + record() {},
142 + stop: async () => {},
143 + },
144 + transcriber: {
145 + prepare: async () => {
146 + const options = input.getTranscriptionOptions?.();
147 + if (options) mocks.recordingOptions.push(options);
148 + return {
149 + locale: "en",
150 + transcribe: async () => {
151 + mocks.transcribeCalls.push(environmentId);
152 + const next = mocks.transcripts.shift() ?? "dictated";
153 + if (next instanceof Error) throw next;
154 + return next;
155 + },
156 + };
157 + },
158 + },
159 + cancelRecording() {},
160 + deleteRecording() {},
161 + };
162 + },
163 +}));
164 +
165 +const THREAD_ENV = "thread-env";
166 +
167 +function setup(options: { readonly destination?: string | null } = {}) {
168 + mocks.settings = {};
169 + setDestination(options.destination ?? null);
170 + chooseTranscriptCleanup(false);
171 + mocks.primaryEnvironmentId = "primary-env";
172 + mocks.connected = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
173 + mocks.voiceCapable = new Set(["primary-env", THREAD_ENV, "env-a", "env-b"]);
174 + mocks.canDictate = new Set(mocks.voiceCapable);
175 + mocks.prepared.clear();
176 + mocks.statusRequests = [];
177 + mocks.statusUnavailable = false;
178 + mocks.modelState = "ready";
179 + mocks.download.mockReset();
180 + mocks.platforms = [];
181 + mocks.microphoneRequests = [];
182 + mocks.transcribeCalls = [];
183 + mocks.cleanupCalls = [];
184 + mocks.cleanupDraft.mockClear();
185 + mocks.cleanupOptions = [];
186 + mocks.recordingOptions = [];
187 + mocks.projectWords = {};
188 + mocks.transcripts = [];
189 + mocks.committed.mockClear();
190 + mocks.draft = "";
191 +}
192 +
193 +function setDestination(environmentId: string | null) {
194 + chooseAudioDestination(
195 + environmentId ? { kind: "environment", environmentId: environmentId as EnvironmentId } : null,
196 + );
197 +}
198 +
199 +function Probe(props: {
200 + readonly projectId?: string;
201 + readonly projectName?: string;
202 + readonly cleanup?: boolean;
203 +}) {
204 + const value = useEnvironmentSpeechInput({
205 + environmentId: THREAD_ENV as EnvironmentId,
206 + projectId: props.projectId as ProjectId | undefined,
207 + projectName: props.projectName,
208 + cleanup: props.cleanup,
209 + ownerKey: "draft",
210 + draftText: mocks.draft,
211 + readDraft: () => ({
212 + text: mocks.draft,
213 + selection: { start: mocks.draft.length, end: mocks.draft.length },
214 + }),
215 + commitDraft: mocks.committed,
216 + });
217 + useLayoutEffect(() => {
218 + mocks.voice = value;
219 + });
220 + return null;
221 +}
222 +
223 +function expectNoAudioSent() {
224 + expect(mocks.platforms).toEqual([]);
225 + expect(mocks.microphoneRequests).toEqual([]);
226 + expect(mocks.transcribeCalls).toEqual([]);
227 + expect(mocks.statusRequests).toEqual([]);
228 +}
229 +
230 +it.each([
231 + ["nothing is configured", {}],
232 + // A transcription host picked by an earlier PoC build is not consent.
233 + ["a legacy transcription host is configured", { voiceTranscriptionEnvironmentId: "env-a" }],
234 +])("stays local-only when %s, even with voice-capable environments", async (_, legacy) => {
235 + setup();
236 + mocks.settings = { ...legacy, speechSilenceFiltering: true };
237 + await using _screen = await render(<Probe />);
238 + await act(() => mocks.voice.start());
239 + expectNoAudioSent();
240 + expect(mocks.committed).not.toHaveBeenCalled();
241 + expect(mocks.voice.available).toBe(false);
242 + expect(mocks.voice.transcriptionEnvironmentId).toBeNull();
243 +});
244 +
245 +it.each([true, false])(
246 + "sends audio with silence filtering %s only to the chosen environment",
247 + async (speechSilenceFiltering) => {
248 + setup({ destination: "env-a" });
249 + mocks.settings = { speechSilenceFiltering };
250 + await using _screen = await render(<Probe />);
251 + expect(mocks.voice.transcriptionEnvironmentId).toBe("env-a");
252 + await act(() => mocks.voice.start());
253 + expect(mocks.voice.state.phase).toBe("recording");
254 + await act(() => mocks.voice.stop());
255 + expect(
256 + new Set([...mocks.platforms, ...mocks.microphoneRequests, ...mocks.transcribeCalls]),
257 + ).toEqual(new Set(["env-a"]));
258 + expect(mocks.statusRequests.every((id) => id === "env-a")).toBe(true);
259 + expect(mocks.recordingOptions[0]?.speechSilenceFiltering).toBe(speechSilenceFiltering);
260 + // Cleanup is a separate opt-in and stays off by default.
261 + expect(mocks.cleanupCalls).toEqual([]);
262 + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything());
263 + },
264 +);
265 +
266 +it.each([
267 + ["is disconnected", () => mocks.connected.delete("env-a")],
268 + ["cannot operate", () => mocks.canDictate.delete("env-a")],
269 + ["does not support transcription", () => mocks.voiceCapable.delete("env-a")],
270 +])(
271 + "does not fall back to another environment when the chosen one %s",
272 + async (_, breakDestination) => {
273 + setup({ destination: "env-a" });
274 + breakDestination();
275 + await using _screen = await render(<Probe />);
276 + await act(() => mocks.voice.start());
277 + expectNoAudioSent();
278 + expect(mocks.voice.available).toBe(false);
279 + },
280 +);
281 +
282 +it.each([
283 + ["revoked", null],
284 + ["moved to another environment", "env-b"],
285 + ["renewed for the same environment", "env-a"],
286 +])("abandons an in-progress recording when consent is %s", async (_, next) => {
287 + setup({ destination: "env-a" });
288 + await using screen = await render(<Probe />);
289 + await act(() => mocks.voice.start());
290 + expect(mocks.voice.state.phase).toBe("recording");
291 + setDestination(next);
292 + await screen.rerender(<Probe />);
293 + expect(mocks.voice.state.phase).toBe("idle");
294 + await act(() => mocks.voice.stop());
295 + expect(mocks.transcribeCalls).toEqual([]);
296 + expect(mocks.committed).not.toHaveBeenCalled();
297 +});
298 +
299 +it("retries a failed transcription on the same chosen environment", async () => {
300 + setup({ destination: "env-a" });
301 + mocks.transcripts = [new Error("inference failed"), "second try"];
302 + await using _screen = await render(<Probe />);
303 + await act(() => mocks.voice.start());
304 + await act(() => mocks.voice.stop());
305 + expect(mocks.voice.state).toMatchObject({ phase: "error", errorAction: "retry" });
306 + await act(() => mocks.voice.start());
307 + if (mocks.voice.state.phase === "recording") await act(() => mocks.voice.stop());
308 + expect(mocks.transcribeCalls).toEqual(["env-a", "env-a"]);
309 + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("second try", expect.anything());
310 +});
311 +
312 +it("reopens setup after reconnecting during a download and ignores its stale completion", async () => {
313 + setup({ destination: "env-a" });
314 + mocks.modelState = "missing-model";
315 + let finishDownload!: () => void;
316 + mocks.download.mockReturnValue(
317 + new Promise<void>((resolve) => {
318 + finishDownload = resolve;
319 + }),
320 + );
321 + await using screen = await render(<Probe />);
322 + await act(() => mocks.voice.start());
323 + let done: Promise<void> | undefined;
324 + await act(() => {
325 + done = mocks.voice.setup.download();
326 + });
327 + expect(mocks.voice.setup.downloading).toBe(true);
328 + mocks.connected.delete("env-a");
329 + await screen.rerender(<Probe />);
330 + expect(mocks.voice.setup.open).toBe(false);
331 + mocks.connected.add("env-a");
332 + await screen.rerender(<Probe />);
333 + await act(() => mocks.voice.start());
334 + expect(mocks.voice.setup).toMatchObject({ open: true, downloading: false, step: 0 });
335 + finishDownload();
336 + await act(async () => {
337 + await done;
338 + });
339 + expect(mocks.voice.setup.error).toBeNull();
340 +});
341 +
342 +it("does not resend a failed recording after consent is revoked", async () => {
343 + setup({ destination: "env-a" });
344 + mocks.transcripts = [new Error("inference failed")];
345 + await using screen = await render(<Probe />);
346 + await act(() => mocks.voice.start());
347 + await act(() => mocks.voice.stop());
348 + expect(mocks.voice.state.phase).toBe("error");
349 + setDestination(null);
350 + await screen.rerender(<Probe />);
351 + await act(() => mocks.voice.start());
352 + expect(mocks.transcribeCalls).toEqual(["env-a"]);
353 + expect(mocks.microphoneRequests).toEqual(["env-a"]);
354 + expect(mocks.committed).not.toHaveBeenCalled();
355 +});
356 +
357 +it("revokes across windows even before the storage event and survives stale preference writes", async () => {
358 + setup({ destination: "env-a" });
359 + await using _screen = await render(<Probe />);
360 + await act(() => mocks.voice.start());
361 + expect(mocks.voice.state.phase).toBe("recording");
362 + const other = document.createElement("iframe");
363 + document.body.append(other);
364 + try {
365 + const storage = other.contentWindow!.localStorage;
366 + const key = "t3code:voice-audio-grant:v1";
367 + const grant = JSON.parse(storage.getItem(key)!) as { id: string };
368 + const canSend = bindAudioGrant(grant.id);
369 + expect(canSend()).toBe(true);
370 + storage.removeItem(key);
371 + expect(canSend()).toBe(false);
372 + writeBrowserClientSettings({ ...DEFAULT_CLIENT_SETTINGS, wordWrap: false });
373 + await vi.waitFor(() => expect(mocks.voice.available).toBe(false));
374 + expect(mocks.voice.state.phase).toBe("idle");
375 + await act(() => mocks.voice.stop());
376 + expect(mocks.transcribeCalls).toEqual([]);
377 + setDestination("env-a");
378 + expect(canSend()).toBe(false);
379 + } finally {
380 + other.remove();
381 + }
382 +});
383 +
384 +it("cleans the transcript only after cleanup is separately enabled", async () => {
385 + setup({ destination: "env-a" });
386 + chooseTranscriptCleanup(true);
387 + const nearby = "x".repeat(1_024);
388 + mocks.draft = "unrelated private paragraph " + nearby;
389 + await using screen = await render(<Probe />);
390 + await act(() => mocks.voice.start());
391 + await act(() => mocks.voice.stop());
392 + expect(mocks.cleanupCalls).toHaveLength(1);
393 + expect(mocks.cleanupDraft).toHaveBeenCalledExactlyOnceWith({
394 + text: nearby,
395 + selection: { start: nearby.length, end: nearby.length },
396 + });
397 + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith(
398 + mocks.draft + " Cleaned dictated",
399 + expect.anything(),
400 + );
401 +
402 + await act(() => mocks.voice.start());
403 + chooseTranscriptCleanup(false);
404 + await screen.rerender(<Probe />);
405 + await act(() => mocks.voice.stop());
406 + expect(mocks.cleanupCalls).toHaveLength(1);
407 +});
408 +
409 +// The settings model test must show what the model heard, not what a provider rewrote.
410 +it.each([
411 + { cleanup: false, authorized: true },
412 + { cleanup: undefined, authorized: false },
413 +])(
414 + "keeps a recording without authorized cleanup off the provider: $cleanup / $authorized",
415 + async ({ cleanup, authorized }) => {
416 + setup({ destination: "env-a" });
417 + chooseTranscriptCleanup(true);
418 + mocks.settings.speechCorrectionWord = "sorry";
419 + if (!authorized) mocks.canDictate.delete(THREAD_ENV);
420 + mocks.projectWords = { "proj-1": [{ term: "Acme", aliases: ["ack me"] }] };
421 + await using _screen = await render(
422 + <Probe {...(cleanup === undefined ? {} : { cleanup })} projectId="proj-1" />,
423 + );
424 + await act(() => mocks.voice.start());
425 + await act(() => mocks.voice.stop());
426 + expect(mocks.cleanupCalls).toEqual([]);
427 + expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual(mocks.projectWords["proj-1"]);
428 + expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined();
429 + expect(mocks.transcribeCalls).toEqual(["env-a"]);
430 + expect(mocks.committed).toHaveBeenCalledExactlyOnceWith("dictated", expect.anything());
431 + },
432 +);
433 +
434 +it("sends the project's vocabulary and name as hints, and the cue only with cleanup on", async () => {
435 + setup({ destination: "env-a" });
436 + mocks.settings.speechCustomWords = [{ term: "Effect", aliases: ["a fact"] }];
437 + mocks.settings.speechCorrectionWord = "scratch that";
438 + // Stored on the draft's environment under its project, never read from the audio destination.
439 + mocks.projectWords = { "proj-1": [{ term: "effect", aliases: ["affect"] }] };
440 + await using screen = await render(<Probe projectId="proj-1" projectName="Acme App" />);
441 + await act(() => mocks.voice.start());
442 + await act(() => mocks.voice.stop());
443 + expect(mocks.recordingOptions.at(-1)?.speechCustomWords).toEqual([
444 + { term: "Acme App", aliases: [] },
445 + { term: "effect", aliases: ["affect"] },
446 + ]);
447 + expect(mocks.cleanupCalls).toEqual([]);
448 +
449 + chooseTranscriptCleanup(true);
450 + await screen.rerender(<Probe projectId="proj-1" projectName="Acme App" />);
451 + await act(() => mocks.voice.start());
452 + await act(() => mocks.voice.stop());
453 + expect(mocks.recordingOptions.at(-1)?.speechCustomWords.map(({ term }) => term)).toEqual([
454 + "scratch that",
455 + "Acme App",
456 + "effect",
457 + ]);
458 + expect(mocks.cleanupOptions.at(-1)).toMatchObject({ speechCorrectionWord: "scratch that" });
459 + expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBe("scratch that");
460 + chooseTranscriptCleanup(false);
461 + await act(() => mocks.voice.start());
462 + await act(() => mocks.voice.stop());
463 + expect(mocks.recordingOptions.at(-1)?.speechCorrectionWord).toBeUndefined();
464 + expect(mocks.recordingOptions.at(-1)?.speechCustomWords.map(({ term }) => term)).not.toContain(
465 + "scratch that",
466 + );
467 +});
468 +
469 +it("shares one status probe per destination across every mounted editor", async () => {
470 + setup({ destination: "env-a" });
471 + await using _screen = await render(
472 + <>
473 + <Probe />
474 + <Probe />
475 + <Probe />
476 + </>,
477 + );
478 + expect(mocks.voice.available).toBe(true);
479 + expect(mocks.statusRequests).toEqual(["env-a"]);
480 +});
481 +
482 +it("lets the user retry after a status failure without reselecting the destination", async () => {
483 + setup({ destination: "env-a" });
484 + mocks.statusUnavailable = true;
485 + await using _screen = await render(<Probe />);
486 + expect(mocks.statusRequests).toEqual(["env-a"]);
487 + expect(mocks.voice.available).toBe(true);
488 + expect(mocks.microphoneRequests).toEqual([]);
489 + mocks.statusUnavailable = false;
490 + await act(() => mocks.voice.start());
491 + await act(() => mocks.voice.stop());
492 + expect(mocks.transcribeCalls).toEqual(["env-a"]);
493 +});
apps/web/src/speech/useEnvironmentSpeechInput.ts new
+552
@@ -0,0 +1,552 @@
1 +import {
2 + cancelEnvironmentSpeechModelDownload,
3 + downloadEnvironmentSpeechModel,
4 + getEnvironmentSpeechModels,
5 + getEnvironmentSpeechStatus,
6 + postProcessEnvironmentTranscript,
7 + VoiceInputController,
8 + voiceInputBlocksSubmission,
9 + type VoiceDraftSnapshot,
10 + type VoiceInputControllerDependencies,
11 + type VoiceInputState,
12 +} from "@t3tools/client-runtime/voice-input";
13 +import type { PreparedConnection } from "@t3tools/client-runtime/connection";
14 +import { AuthOrchestrationOperateScope } from "@t3tools/contracts";
15 +import type {
16 + EnvironmentId,
17 + EnvironmentSpeechModel,
18 + EnvironmentSpeechStatus,
19 + ProjectId,
20 + SpeechPostProcessingOptions,
21 + SpeechStreamText,
22 + SpeechTranscriptionOptions,
23 +} from "@t3tools/contracts";
24 +import { resolveProjectSettings } from "@t3tools/shared/projectSettings";
25 +import { resolveSpeechVocabulary } from "@t3tools/shared/speech";
26 +import * as Option from "effect/Option";
27 +import { useEffect, useRef, useState } from "react";
28 +
29 +import { useStableCallback } from "~/hooks/useStableCallback";
30 +import { toastManager } from "../components/ui/toast";
31 +import {
32 + useClientSettings,
33 + useClientSettingsHydrated,
34 + useEnvironmentSettings,
35 +} from "../hooks/useSettings";
36 +import { runtime } from "../lib/runtime";
37 +import { useEnvironment } from "../state/environments";
38 +import { useEnvironmentScope, usePreparedConnection } from "../state/session";
39 +import { createBrowserVoiceInputPlatform } from "./browserVoiceInput";
40 +import { bindAudioGrant, readCleanupGrant, useAudioGrant } from "./voiceConsent";
41 +
42 +const INITIAL_STATE: VoiceInputState = { phase: "idle", error: null, errorAction: null };
43 +const WAITING_STATE: VoiceInputState = { phase: "preparing", error: null, errorAction: null };
44 +const INITIAL_SETUP = {
45 + open: false,
46 + step: 0,
47 + model: null as EnvironmentSpeechModel | null,
48 + downloading: false,
49 + error: null as string | null,
50 +};
51 +
52 +// Every mounted editor shares one status probe per destination; starting a recording refreshes it.
53 +const statusProbes = new WeakMap<PreparedConnection, Promise<EnvironmentSpeechStatus>>();
54 +function probeSpeechStatus(prepared: PreparedConnection, refresh = false) {
55 + const cached = refresh ? undefined : statusProbes.get(prepared);
56 + if (cached) return cached;
57 + const probe = runtime.runPromise(getEnvironmentSpeechStatus(prepared));
58 + statusProbes.set(prepared, probe);
59 + probe.catch(() => {
60 + if (statusProbes.get(prepared) === probe) statusProbes.delete(prepared);
61 + });
62 + return probe;
63 +}
64 +
65 +type DraftInput = {
66 + readonly text: string;
67 + readonly selection: { readonly start: number; readonly end: number };
68 +};
69 +
70 +function cleanupContext({ text, selection }: DraftInput): DraftInput {
71 + const limit = 1_024;
72 + const start = Math.max(0, Math.min(selection.start, text.length));
73 + const end = Math.max(start, Math.min(selection.end, text.length));
74 + const before = text.slice(Math.max(0, start - limit), start);
75 + const selected = text.slice(start, Math.min(end, start + limit));
76 + return {
77 + text: before + selected + text.slice(end, end + limit),
78 + selection: { start: before.length, end: before.length + selected.length },
79 + };
80 +}
81 +
82 +type HookInput = {
83 + /** The draft's own environment. Optional transcript cleanup runs there, never on the audio destination. */
84 + readonly environmentId: EnvironmentId;
85 + /** The draft's project: its vocabulary on that environment and its name become recognition hints. */
86 + readonly projectId?: ProjectId | null | undefined;
87 + readonly projectName?: string | null | undefined;
88 + readonly ownerKey: string;
89 + readonly draftText: string;
90 + readonly disabled?: boolean | undefined;
91 + /** False keeps the transcript off the cleanup provider even when cleanup is on; the model test uses it. */
92 + readonly cleanup?: boolean | undefined;
93 + readonly readDraft: () => DraftInput;
94 + readonly commitDraft: (
95 + text: string,
96 + selection: { readonly start: number; readonly end: number },
97 + ) => void;
98 +};
99 +
100 +/**
101 + * Microphone audio leaves this device only for the environment the person chose in
102 + * Voice settings. Nothing is inferred from the primary or the draft's environment, and
103 + * the recording is abandoned the moment that choice changes or its connection drops.
104 + */
105 +export function useVoiceTranscriptionDestination() {
106 + const hydrated = useClientSettingsHydrated();
107 + const grant = useAudioGrant();
108 + const environmentId = hydrated && grant ? grant.destination.environmentId : null;
109 + const environment = useEnvironment(environmentId);
110 + const connection = Option.getOrNull(usePreparedConnection(environmentId));
111 + const canDictate = useEnvironmentScope(environmentId, AuthOrchestrationOperateScope);
112 + const supported = environment?.serverConfig?.environment.capabilities.voiceTranscription === true;
113 + return {
114 + environmentId,
115 + grantId: grant?.id ?? null,
116 + label: environment?.label ?? null,
117 + prepared: supported && canDictate ? connection : null,
118 + };
119 +}
120 +
121 +export function useEnvironmentSpeechInput(input: HookInput) {
122 + const destination = useVoiceTranscriptionDestination();
123 + const prepared = destination.prepared;
124 + const grantId = destination.grantId;
125 + const originatingEnvironment = useEnvironment(input.environmentId);
126 + const originatingConnection = Option.getOrNull(usePreparedConnection(input.environmentId));
127 + const canCleanUp = useEnvironmentScope(input.environmentId, AuthOrchestrationOperateScope);
128 + const cleanupPrepared =
129 + canCleanUp &&
130 + originatingEnvironment?.serverConfig?.environment.capabilities.voiceTranscription === true
131 + ? originatingConnection
132 + : null;
133 + const clientSettings = useClientSettings();
134 + const projectId = input.projectId ?? null;
135 + const projectWords = useEnvironmentSettings(
136 + input.environmentId,
137 + (settings) => resolveProjectSettings(settings, projectId).settings.speechProjectCustomWords,
138 + );
139 + const [status, setStatus] = useState<{
140 + readonly prepared: PreparedConnection;
141 + readonly value: EnvironmentSpeechStatus | null;
142 + } | null>(null);
143 + const [controllerState, setControllerState] = useState({
144 + prepared,
145 + grantId,
146 + value: INITIAL_STATE,
147 + });
148 + const [queuedStart, setQueuedStart] = useState<PreparedConnection | null>(null);
149 + const [level, setLevel] = useState(0);
150 + const [preview, setPreview] = useState<SpeechStreamText | null>(null);
151 + const [setup, setSetup] = useState(INITIAL_SETUP);
152 + const [previousOwner, setPreviousOwner] = useState({
153 + key: input.ownerKey,
154 + disabled: input.disabled,
155 + });
156 + if (controllerState.prepared !== prepared || controllerState.grantId !== grantId) {
157 + setControllerState({ prepared, grantId, value: INITIAL_STATE });
158 + setSetup(INITIAL_SETUP);
159 + setQueuedStart(null);
160 + setPreview(null);
161 + setLevel(0);
162 + }
163 + if (previousOwner.key !== input.ownerKey || previousOwner.disabled !== input.disabled) {
164 + setPreviousOwner({ key: input.ownerKey, disabled: input.disabled });
165 + setQueuedStart(null);
166 + setSetup(INITIAL_SETUP);
167 + }
168 + const state: VoiceInputState =
169 + queuedStart === prepared && prepared
170 + ? WAITING_STATE
171 + : controllerState.prepared === prepared
172 + ? controllerState.value
173 + : INITIAL_STATE;
174 +
175 + const controllerRef = useRef<VoiceInputController | null>(null);
176 + const startRequestRef = useRef(0);
177 + const pendingSubmissionRef = useRef<{
178 + ownerKey: string;
179 + submit: (text: string) => void;
180 + text?: string;
181 + } | null>(null);
182 + const latestRef = useRef({ input, clientSettings, cleanupPrepared, projectWords });
183 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed props and settings to the asynchronous recording controller
184 + useEffect(() => {
185 + latestRef.current = { input, clientSettings, cleanupPrepared, projectWords };
186 + });
187 + const draftRevisionRef = useRef({ ownerKey: input.ownerKey, text: input.draftText, revision: 0 });
188 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- tracks committed draft revisions so a late transcript cannot overwrite an edited draft
189 + useEffect(() => {
190 + const revision = draftRevisionRef.current;
191 + if (revision.ownerKey !== input.ownerKey || revision.text !== input.draftText) {
192 + draftRevisionRef.current = {
193 + ownerKey: input.ownerKey,
194 + text: input.draftText,
195 + revision: revision.revision + 1,
196 + };
197 + }
198 + }, [input.ownerKey, input.draftText]);
199 +
200 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- owns the recording controller bound to the consented destination; it is disposed with the consent
201 + useEffect(() => {
202 + if (!prepared || !grantId || typeof navigator === "undefined") return;
203 + const hasAudioConsent = bindAudioGrant(grantId);
204 + let disposed = false;
205 + let controller: VoiceInputController;
206 + let recordingPreferences:
207 + | {
208 + readonly cleanup: SpeechPostProcessingOptions | null;
209 + readonly cleanupGrant: string | null;
210 + readonly environmentId: EnvironmentId;
211 + }
212 + | undefined;
213 +
214 + const readDraft = (): VoiceDraftSnapshot => {
215 + const current = latestRef.current.input;
216 + const draft = current.readDraft();
217 + const revision = draftRevisionRef.current;
218 + if (revision.ownerKey !== current.ownerKey || revision.text !== draft.text) {
219 + draftRevisionRef.current = {
220 + ownerKey: current.ownerKey,
221 + text: draft.text,
222 + revision: revision.revision + 1,
223 + };
224 + }
225 + return {
226 + ownerKey: current.ownerKey,
227 + text: draft.text,
228 + selection: draft.selection,
229 + revision: draftRevisionRef.current.revision,
230 + };
231 + };
232 + const platform = createBrowserVoiceInputPlatform({
233 + prepared,
234 + hasAudioConsent,
235 + getTranscriptionOptions: (): SpeechTranscriptionOptions => {
236 + const {
237 + clientSettings: settings,
238 + projectWords: project,
239 + input: current,
240 + } = latestRef.current;
241 + const cleanupGrant = current.cleanup === false ? null : readCleanupGrant();
242 + const cleanupEnabled = cleanupGrant !== null;
243 + const cue =
244 + cleanupEnabled && latestRef.current.cleanupPrepared ? settings.speechCorrectionWord : "";
245 + const speechCustomWords = resolveSpeechVocabulary({
246 + personal: settings.speechCustomWords,
247 + project,
248 + hints: [cue, current.projectName ?? ""],
249 + });
250 + recordingPreferences = {
251 + cleanupGrant,
252 + environmentId: current.environmentId,
253 + cleanup: cleanupEnabled
254 + ? {
255 + speechCustomWords,
256 + speechPostProcessingPrompt: settings.speechPostProcessingPrompt,
257 + ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
258 + }
259 + : null,
260 + };
261 + return {
262 + speechLanguage: settings.speechLanguage,
263 + speechTranslateToEnglish: settings.speechTranslateToEnglish,
264 + speechCustomWords,
265 + speechRemoveFillerWords: settings.speechRemoveFillerWords,
266 + ...(cue.trim() ? { speechCorrectionWord: cue } : {}),
267 + speechSilenceFiltering: settings.speechSilenceFiltering,
268 + };
269 + },
270 + getMicrophoneId: () => latestRef.current.clientSettings.voiceMicrophone,
271 + onLevel: setLevel,
272 + onDurationLimit: () => void controller.stop(),
273 + onText: (text) => {
274 + if (!disposed) setPreview(text);
275 + },
276 + onError: (message) => {
277 + if (!disposed) void controller.interruptRecording(message);
278 + },
279 + });
280 + const postProcess: NonNullable<VoiceInputControllerDependencies["postProcess"]> = (
281 + transcript,
282 + options,
283 + ) => {
284 + const preferences = recordingPreferences;
285 + const { input: current, cleanupPrepared: target } = latestRef.current;
286 + if (
287 + !preferences?.cleanup ||
288 + !hasAudioConsent() ||
289 + preferences.cleanupGrant !== readCleanupGrant() ||
290 + current.cleanup === false ||
291 + current.environmentId !== preferences.environmentId ||
292 + current.ownerKey !== options.draft.ownerKey
293 + )
294 + return null;
295 + if (!target)
296 + return Promise.reject(
297 + new Error("The draft's environment is unavailable for transcript cleanup."),
298 + );
299 + return runtime
300 + .runPromise(
301 + postProcessEnvironmentTranscript(
302 + target,
303 + transcript,
304 + cleanupContext(options.draft),
305 + preferences.cleanup,
306 + () => hasAudioConsent() && preferences.cleanupGrant === readCleanupGrant(),
307 + ),
308 + options,
309 + )
310 + .then((result) => result.text);
311 + };
312 + controller = new VoiceInputController({
313 + retryFailedRecording: true,
314 + recorder: platform.recorder,
315 + getTranscriber: () => platform.transcriber,
316 + requestPermission: async () => ({ granted: true, canAskAgain: true }),
317 + configureRecording: async () => undefined,
318 + releaseRecording: async () => platform.cancelRecording(),
319 + deleteRecording: platform.deleteRecording,
320 + postProcess,
321 + onPostProcessingError: () =>
322 + toastManager.add({
323 + type: "warning",
324 + title: "Transcript cleanup failed",
325 + description: "The original transcript was added.",
326 + }),
327 + readDraft,
328 + commitDraft: (text, selection) => {
329 + if (latestRef.current.input.disabled) return;
330 + latestRef.current.input.commitDraft(text, selection);
331 + if (pendingSubmissionRef.current) pendingSubmissionRef.current.text = text;
332 + },
333 + onStateChange: (value) => {
334 + if (disposed) return;
335 + if (value.phase === "error") pendingSubmissionRef.current = null;
336 + setControllerState({ prepared, grantId, value });
337 + if (value.phase !== "recording" && value.phase !== "transcribing") setPreview(null);
338 + },
339 + });
340 + controllerRef.current = controller;
341 + return () => {
342 + disposed = true;
343 + pendingSubmissionRef.current = null;
344 + startRequestRef.current += 1;
345 + setQueuedStart(null);
346 + controller.dispose();
347 + if (controllerRef.current === controller) controllerRef.current = null;
348 + };
349 + }, [prepared, grantId]);
350 +
351 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- probes the chosen environment's speech status once its connection is available
352 + useEffect(() => {
353 + if (!prepared) return;
354 + let disposed = false;
355 + void probeSpeechStatus(prepared)
356 + .then((value) => {
357 + if (!disposed) setStatus({ prepared, value });
358 + })
359 + .catch(() => {
360 + if (!disposed) setStatus({ prepared, value: null });
361 + });
362 + return () => {
363 + disposed = true;
364 + };
365 + }, [prepared]);
366 + const currentStatus = status?.prepared === prepared ? status.value : null;
367 +
368 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- polls download progress while the setup dialog owns an active model download
369 + useEffect(() => {
370 + if (!setup.open || !setup.downloading || !prepared || !currentStatus?.supported) return;
371 + const modelId = currentStatus.modelId;
372 + let disposed = false;
373 + let refreshing = false;
374 + const timer = window.setInterval(() => {
375 + if (refreshing) return;
376 + refreshing = true;
377 + void runtime
378 + .runPromise(getEnvironmentSpeechModels(prepared))
379 + .then((result) => {
380 + if (disposed) return;
381 + const model = result.models.find((candidate) => candidate.id === modelId) ?? null;
382 + setSetup((current) => ({ ...current, model }));
383 + })
384 + .catch(() => undefined)
385 + .finally(() => {
386 + refreshing = false;
387 + });
388 + }, 1_000);
389 + return () => {
390 + disposed = true;
391 + window.clearInterval(timer);
392 + };
393 + }, [currentStatus, prepared, setup.downloading, setup.open]);
394 +
395 + const previousOwnerRef = useRef(input.ownerKey);
396 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- abandons the recording and any queued submission when the draft's owner changes
397 + useEffect(() => {
398 + if (previousOwnerRef.current === input.ownerKey && !input.disabled) return;
399 + previousOwnerRef.current = input.ownerKey;
400 + pendingSubmissionRef.current = null;
401 + startRequestRef.current += 1;
402 + controllerRef.current?.ownerChanged();
403 + }, [input.ownerKey, input.disabled]);
404 +
405 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- delivers a queued submission only after the transcript has committed to the draft
406 + useEffect(() => {
407 + const pending = pendingSubmissionRef.current;
408 + if (state.phase !== "idle" || !pending) return;
409 + pendingSubmissionRef.current = null;
410 + if (!input.disabled && pending.ownerKey === input.ownerKey && pending.text !== undefined)
411 + pending.submit(pending.text);
412 + }, [state, input.ownerKey, input.disabled]);
413 +
414 + const submitAfterDictation = useStableCallback((submit: (text: string) => void) => {
415 + if (latestRef.current.input.disabled || state.phase === "preparing") return;
416 + const controller = controllerRef.current;
417 + if (!controller || !voiceInputBlocksSubmission(state)) {
418 + submit(latestRef.current.input.readDraft().text);
419 + return;
420 + }
421 + if (pendingSubmissionRef.current) return;
422 + pendingSubmissionRef.current = { ownerKey: latestRef.current.input.ownerKey, submit };
423 + if (controller.currentState.phase === "recording") void controller.stop();
424 + });
425 +
426 + const start = useStableCallback(async () => {
427 + const request = ++startRequestRef.current;
428 + const controller = controllerRef.current;
429 + const owner = latestRef.current.input.ownerKey;
430 + if (latestRef.current.input.disabled || !controller || !prepared) return;
431 + const stillCurrent = () =>
432 + request === startRequestRef.current &&
433 + controllerRef.current === controller &&
434 + latestRef.current.input.ownerKey === owner &&
435 + !latestRef.current.input.disabled;
436 + setQueuedStart(prepared);
437 + let latestStatus: EnvironmentSpeechStatus;
438 + try {
439 + latestStatus = await probeSpeechStatus(prepared, true);
440 + } catch (error) {
441 + if (!stillCurrent()) return;
442 + setQueuedStart(null);
443 + toastManager.add({
444 + type: "error",
445 + title: "Could not start voice input",
446 + description: error instanceof Error ? error.message : String(error),
447 + });
448 + return;
449 + }
450 + if (!stillCurrent()) return;
451 + setQueuedStart(null);
452 + setStatus({ prepared, value: latestStatus });
453 + if (!latestStatus.supported) {
454 + toastManager.add({
455 + type: "error",
456 + title: "Voice input is not available",
457 + description: latestStatus.reason,
458 + });
459 + return;
460 + }
461 + if (latestStatus.state === "missing-model") {
462 + setSetup({ ...INITIAL_SETUP, open: true });
463 + return;
464 + }
465 + setLevel(0);
466 + await controller.start();
467 + });
468 +
469 + const downloadSetupModel = useStableCallback(async () => {
470 + if (!prepared || !currentStatus?.supported || setup.downloading) return;
471 + const request = ++startRequestRef.current;
472 + setSetup((current) => ({ ...current, downloading: true, error: null }));
473 + try {
474 + await runtime.runPromise(downloadEnvironmentSpeechModel(prepared, currentStatus.modelId));
475 + if (request !== startRequestRef.current) return;
476 + const next = await probeSpeechStatus(prepared, true);
477 + if (request !== startRequestRef.current) return;
478 + setStatus({ prepared, value: next });
479 + setSetup((current) => ({
480 + ...current,
481 + downloading: false,
482 + step: next.supported && next.state !== "missing-model" ? 1 : current.step,
483 + error:
484 + next.supported && next.state !== "missing-model"
485 + ? null
486 + : "The model is not ready. Try downloading it again.",
487 + }));
488 + } catch {
489 + if (request !== startRequestRef.current) return;
490 + setSetup((current) => ({
491 + ...current,
492 + downloading: false,
493 + error: "Could not download the model. Check this environment's connection and try again.",
494 + }));
495 + }
496 + });
497 +
498 + const cancelSetupDownload = useStableCallback(async () => {
499 + if (!prepared || !currentStatus?.supported) return;
500 + startRequestRef.current += 1;
501 + setSetup((current) => ({ ...current, open: false, downloading: false }));
502 + try {
503 + await runtime.runPromise(
504 + cancelEnvironmentSpeechModelDownload(prepared, currentStatus.modelId),
505 + );
506 + } catch (error) {
507 + setSetup((current) => ({
508 + ...current,
509 + error: error instanceof Error ? error.message : String(error),
510 + }));
511 + }
512 + });
513 +
514 + return {
515 + available:
516 + prepared !== null &&
517 + status?.prepared === prepared &&
518 + status.value?.supported !== false &&
519 + typeof navigator !== "undefined" &&
520 + Boolean(navigator.mediaDevices?.getUserMedia) &&
521 + typeof AudioWorkletNode !== "undefined",
522 + status: currentStatus,
523 + transcriptionEnvironmentId: destination.environmentId,
524 + transcriptionEnvironmentLabel: destination.label,
525 + setup: {
526 + ...setup,
527 + setOpen: (open: boolean) => setSetup((current) => ({ ...current, open })),
528 + download: downloadSetupModel,
529 + cancelDownload: cancelSetupDownload,
530 + startRecording: async () => {
531 + setSetup((current) => ({ ...current, open: false }));
532 + await start();
533 + },
534 + },
535 + state,
536 + preview: state.phase === "recording" || state.phase === "transcribing" ? preview : null,
537 + level,
538 + preparing: state.phase === "preparing",
539 + blocksSubmission: voiceInputBlocksSubmission(state),
540 + freezesEditor: voiceInputBlocksSubmission(state),
541 + start,
542 + submitAfterDictation,
543 + stop: useStableCallback(() => controllerRef.current?.stop() ?? Promise.resolve()),
544 + cancel: useStableCallback(() => {
545 + pendingSubmissionRef.current = null;
546 + startRequestRef.current += 1;
547 + setQueuedStart(null);
548 + controllerRef.current?.cancel();
549 + }),
550 + skipPostProcessing: useStableCallback(() => controllerRef.current?.skipPostProcessing()),
551 + };
552 +}
apps/web/src/speech/useTextFieldSpeech.tsx new
+131
@@ -0,0 +1,131 @@
1 +import { useAtomValue } from "@effect/atom-react";
2 +import type { EnvironmentId, ProjectId } from "@t3tools/contracts";
3 +import { useEffect, useRef, type ReactNode, type RefObject } from "react";
4 +
5 +import {
6 + ComposerSpeechButton,
7 + ComposerSpeechRecordingPill,
8 +} from "~/components/chat/ComposerSpeechButton";
9 +import { VoiceInputSetup } from "~/components/chat/VoiceInputSetup";
10 +import { primaryServerKeybindingsAtom } from "~/state/server";
11 +import { useDictationShortcut } from "./useDictationShortcut";
12 +import { useEnvironmentSpeechInput } from "./useEnvironmentSpeechInput";
13 +
14 +/** Binds dictation to a plain textarea draft; the shortcut targets it only while it has focus. */
15 +export function useTextFieldSpeech(input: {
16 + readonly environmentId: EnvironmentId;
17 + readonly projectId?: ProjectId | null | undefined;
18 + readonly ownerKey: string;
19 + readonly text: string;
20 + readonly textareaRef: RefObject<HTMLTextAreaElement | null>;
21 + readonly onTextChange: (text: string) => void;
22 + readonly disabled?: boolean | undefined;
23 +}) {
24 + const latest = useRef(input);
25 + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- publishes committed field callbacks to the asynchronous recording controller
26 + useEffect(() => {
27 + latest.current = input;
28 + });
29 + const keybindings = useAtomValue(primaryServerKeybindingsAtom);
30 + const speech = useEnvironmentSpeechInput({
31 + environmentId: input.environmentId,
32 + projectId: input.projectId,
33 + ownerKey: input.ownerKey,
34 + draftText: input.text,
35 + disabled: input.disabled,
36 + readDraft: () => {
37 + const field = latest.current.textareaRef.current;
38 + const cursor = field?.selectionStart ?? latest.current.text.length;
39 + return {
40 + text: field?.value ?? latest.current.text,
41 + selection: { start: cursor, end: field?.selectionEnd ?? cursor },
42 + };
43 + },
44 + commitDraft: (text, selection) => {
45 + latest.current.onTextChange(text);
46 + window.requestAnimationFrame(() => {
47 + const field = latest.current.textareaRef.current;
48 + if (!field) return;
49 + field.focus({ preventScroll: true });
50 + field.setSelectionRange(selection.start, selection.end);
51 + });
52 + },
53 + });
54 + const shortcutLabel = useDictationShortcut({
55 + keybindings,
56 + speech,
57 + disabled: input.disabled ?? false,
58 + terminalOpen: false,
59 + modelPickerOpen: false,
60 + targetRef: input.textareaRef,
61 + });
62 + return { ...speech, shortcutLabel };
63 +}
64 +
65 +export type TextFieldSpeech = ReturnType<typeof useTextFieldSpeech>;
66 +
67 +export function TextFieldSpeechControls({
68 + speech,
69 + disabled = false,
70 +}: {
71 + speech: TextFieldSpeech;
72 + disabled?: boolean;
73 +}) {
74 + return (
75 + <>
76 + {speech.preview ? (
77 + <p role="status" className="w-full text-sm text-muted-foreground">
78 + {speech.preview.committed}
79 + <span className="opacity-70">{speech.preview.tentative}</span>
80 + </p>
81 + ) : null}
82 + <div className="flex min-w-0 items-center gap-1">
83 + <ComposerSpeechRecordingPill
84 + state={speech.state}
85 + progress={null}
86 + level={speech.level}
87 + finishShortcutLabel={speech.shortcutLabel}
88 + cancelShortcutLabel="Esc"
89 + disabled={disabled}
90 + onStart={() => void speech.start()}
91 + onStop={() => void speech.stop()}
92 + onCancel={speech.cancel}
93 + onSkipPostProcessing={speech.skipPostProcessing}
94 + />
95 + {speech.available ? (
96 + <ComposerSpeechButton
97 + state={speech.state}
98 + progress={null}
99 + shortcutLabel={speech.shortcutLabel}
100 + disabled={disabled}
101 + onStart={() => void speech.start()}
102 + onCancel={speech.cancel}
103 + />
104 + ) : null}
105 + </div>
106 + <VoiceInputSetup
107 + environmentId={speech.transcriptionEnvironmentId}
108 + open={speech.setup.open}
109 + step={speech.setup.step}
110 + status={speech.status}
111 + model={speech.setup.model}
112 + downloading={speech.setup.downloading}
113 + error={speech.setup.error}
114 + onOpenChange={speech.setup.setOpen}
115 + onDownload={() => void speech.setup.download()}
116 + onCancelDownload={() => void speech.setup.cancelDownload()}
117 + onStartRecording={() => void speech.setup.startRecording()}
118 + />
119 + </>
120 + );
121 +}
122 +
123 +/** Render-prop form for editors that only sometimes have an environment to dictate on. */
124 +export function SpeechTextField({
125 + children,
126 + ...input
127 +}: Parameters<typeof useTextFieldSpeech>[0] & {
128 + children: (speech: TextFieldSpeech) => ReactNode;
129 +}) {
130 + return children(useTextFieldSpeech(input));
131 +}
apps/web/src/speech/voiceConsent.ts new
+57
@@ -0,0 +1,57 @@
1 +import { VoiceTranscriptionDestination } from "@t3tools/contracts";
2 +import * as Schema from "effect/Schema";
3 +import { randomUUID } from "../lib/utils";
4 +import {
5 + readLocalStorageValue,
6 + useLocalStorage,
7 + writeLocalStorageValue,
8 +} from "../hooks/useLocalStorage";
9 +
10 +// Separate keys: a stale window saving ordinary preferences must not restore either grant.
11 +const AUDIO_KEY = "t3code:voice-audio-grant:v1";
12 +const CLEANUP_KEY = "t3code:voice-cleanup-grant:v1";
13 +const AudioGrant = Schema.NullOr(
14 + Schema.Struct({ id: Schema.String, destination: VoiceTranscriptionDestination }),
15 +);
16 +const CleanupGrant = Schema.NullOr(Schema.String);
17 +const decodeAudioGrant = Schema.decodeSync(Schema.fromJsonString(AudioGrant));
18 +
19 +export function useAudioGrant() {
20 + return useLocalStorage(AUDIO_KEY, null, AudioGrant)[0];
21 +}
22 +
23 +export function chooseAudioDestination(destination: VoiceTranscriptionDestination | null) {
24 + writeLocalStorageValue(
25 + AUDIO_KEY,
26 + destination && { id: randomUUID(), destination },
27 + null,
28 + AudioGrant,
29 + );
30 +}
31 +
32 +// Bind this recording to this act of consent, even if the user later reselects the same destination.
33 +export function bindAudioGrant(id: string) {
34 + try {
35 + const raw = window.localStorage.getItem(AUDIO_KEY);
36 + if (!raw || decodeAudioGrant(raw)?.id !== id) return () => false;
37 + return () => {
38 + try {
39 + return window.localStorage.getItem(AUDIO_KEY) === raw;
40 + } catch {
41 + return false;
42 + }
43 + };
44 + } catch {
45 + return () => false;
46 + }
47 +}
48 +
49 +export function useCleanupGrant() {
50 + return useLocalStorage(CLEANUP_KEY, null, CleanupGrant)[0];
51 +}
52 +export function readCleanupGrant() {
53 + return readLocalStorageValue(CLEANUP_KEY, null, CleanupGrant);
54 +}
55 +export function chooseTranscriptCleanup(enabled: boolean) {
56 + writeLocalStorageValue(CLEANUP_KEY, enabled ? randomUUID() : null, null, CleanupGrant);
57 +}
docs/README.md
+1
@@ -11,6 +11,7 @@
11 - [Project settings](./user/project-settings.md)
12 - [Appearance and themes](./user/appearance.md)
13 - [Keyboard shortcuts](./user/keybindings.md)
14 +- [Voice input](./user/voice-input.md)
15 - [SnapShots](./user/snap-shot.md)
16 - [Visual replies](./user/html-renders.md)
17 - [Import browser sessions](./user/browser-import.md)
docs/internals/voice-input.md
+47 -4
@@ -2,14 +2,55 @@
2
3 Transcription edits a composer draft. It does not submit an agent turn. Audio is
4 temporary client input, and only normal message submission sends the resulting
5 -text. The current implementation transcribes locally on supported iOS devices;
6 -environment-backed transcription is not implemented.
5 +text. Mobile transcribes on the device; web and desktop send audio to one
6 +environment the person chose.
7
8 The [shared controller](../../packages/client-runtime/src/voice-input/controller.ts)
9 owns the operation while the client supplies capture and transcription. Preparation
10 binds the transcriber and resolved locale for the whole recording. Draft ownership,
11 text, and revision are captured before recording and checked before insertion, so
12 -a late transcript cannot overwrite a draft that was edited or replaced.
12 +a late transcript cannot overwrite a draft that was edited or replaced. A
13 +transcriber may expose a streaming session, in which case audio leaves during
14 +capture and `finish` replaces the file-based `transcribe`.
15 +
16 +## Where audio may go
17 +
18 +Only an explicit [audio grant](../../apps/web/src/speech/voiceConsent.ts) authorizes
19 +off-device transcription. It lives outside the preferences snapshot so another
20 +window saving unrelated settings cannot resurrect a revoked grant. Each new
21 +choice gets a distinct grant, even if the destination was previously selected. The
22 +[web hook](../../apps/web/src/speech/useEnvironmentSpeechInput.ts) binds a
23 +recording controller to the prepared connection of that destination and to
24 +nothing else. When the destination changes, disconnects, or stops advertising the
25 +`voiceTranscription` capability, the controller is disposed: a recording in
26 +progress is discarded, a failed recording kept for retry is deleted, and nothing
27 +falls back to another environment. Every sender in the web app, including the
28 +settings page's transcription test, goes through that hook. The microphone loopback
29 +test captures separately for local playback and sends nothing. The transport
30 +refuses redirects so credentials and audio cannot follow one to another host.
31 +
32 +Transcript cleanup requires its own grant, off by default.
33 +It sends text, never audio, to the text generation provider of the draft's
34 +own environment, because that is where the provider credentials live. A later
35 +backend adds its own destination kind, disclosure, and transcriber; capture and
36 +the controller stay as they are.
37 +
38 +## Server side
39 +
40 +The [speech service](../../apps/server/src/speech/SpeechService.ts) runs
41 +transcribe-cpp in a child process so a hung native call can be killed without
42 +taking the server down; see [native.ts](../../apps/server/src/speech/native.ts)
43 +for the packaging constraints (Node SEA, Electron ASAR). One native request runs
44 +at a time. Batch uploads carry their options in the body framing defined in
45 +[shared/speech.ts](../../packages/shared/src/speech.ts); streaming sends them in
46 +the socket's first command. Model files and the selected model, hardware
47 +placement, and unload timeout are environment state in server settings.
48 +
49 +The Silero silence detector is optional. If its download fails, recordings use
50 +raw audio during the retry cooldown so a detector outage cannot block dictation.
51 +Explicitly installing a speech model also retries the detector without waiting.
52 +
53 +## Cancellation and resource ownership
54
55 Cancellation invalidates a result immediately, but resources stay owned until the
56 underlying work settles. Apple's native transcription call cannot be interrupted
@@ -17,4 +58,6 @@ once started. Releasing the session or deleting its recording when the abort sig
58 fires would race that work. The [transcription contract](../../packages/client-runtime/src/voice-input/transcription.ts)
59 therefore requires implementations to settle only after their work has stopped;
60 the [Apple binding](../../apps/mobile/src/native/voiceTranscription.ios.ts) checks
20 -cancellation between native calls and discards late results.
61 +cancellation between native calls and discards late results. On the server,
62 +interrupting a batch request kills the worker process instead; the next request
63 +reloads the model.
docs/user/composer.md
+6
@@ -184,6 +184,12 @@ Stashes containing uploaded files must be restored in their original environment
184 Those files are retained for 24 hours. After an upload expires, restore the prompt
185 and use **Attach again** or remove the missing file before sending.
186
187 +## Voice input on web and desktop
188 +
189 +Use the composer's microphone or `mod+shift+d` to dictate into your draft. Audio
190 +is transcribed on an environment you choose in **Settings → Voice**; nothing is
191 +sent until you choose one. See [Voice input](./voice-input.md).
192 +
193 ## Voice input on mobile
194
195 On supported iPhones with iOS 26 or later and Android devices with Android 12 or
docs/user/keybindings.md
+5
@@ -5,6 +5,11 @@ also lists the command IDs and defaults available in your version.
5
6 ## Composer controls
7
8 +`mod+shift+d` starts dictation in the focused editor, or the composer, and finishes it on the next press;
9 +`Esc` discards the recording. Change the chord under **Composer: Start or Finish
10 +Dictation**, and the hold or toggle behavior in **Settings → Voice**. See
11 +[Voice input](./voice-input.md).
12 +
13 In **Settings → Chat → Send shortcut**, choose whether Enter sends, requires
14 `mod+Enter` for multiline prompts, or always requires `mod+Enter`. `Shift+Enter`
15 inserts a new line. This applies to the web and desktop composer at desktop widths.
docs/user/voice-input.md new
+111
@@ -0,0 +1,111 @@
1 +# Voice input
2 +
3 +Dictate into the composer instead of typing. The transcript lands in your draft
4 +for review, and only sending the message sends anything to an agent. Dictation
5 +also works in citation comments, diff and file annotations, and pull request
6 +comments, replies, and review summaries.
7 +
8 +On iPhone and Android, voice input transcribes on the device. See
9 +[Voice input on mobile](./composer.md#voice-input-on-mobile).
10 +
11 +## Choose where audio goes
12 +
13 +On web and desktop, transcription runs on one of your T3 Code environments.
14 +Nothing is sent until you choose one: open **Settings → Voice** and set
15 +**Send microphone audio to** the environment that should transcribe for you.
16 +Microphone audio recorded on this device is sent to that environment over your
17 +authenticated connection, transcribed there, and discarded. The choice is saved
18 +on this device only, so every computer and browser you use asks for it once.
19 +Set it back to **Off** at any time; a recording in progress is discarded.
20 +
21 +If the chosen environment goes offline, voice input waits for it. T3 Code never
22 +substitutes another environment.
23 +
24 +## Set up a model
25 +
26 +The first time you dictate, T3 Code offers to download the recommended English
27 +model to the chosen environment. In **Settings → Voice → Transcription**, select
28 +the languages you dictate in to compare models that support all of them.
29 +Accuracy shows how often the model got words or characters wrong in each
30 +language; speed is the time to transcribe 30 seconds on a reference GPU. Choose
31 +**All languages** to browse the full catalog. Filtering does not switch your model.
32 +
33 +**Advanced** controls GPU or CPU use and when an idle model is unloaded. With hardware set to
34 +**Auto**, a recording that fails on the GPU is retried once on that same
35 +environment's CPU; a GPU you picked yourself reports the error instead. Nothing
36 +is ever retried on another environment. Models marked **Live preview** show
37 +text while you speak. Other models can process long recordings in chunks at pauses
38 +while you record; models with built-in long-form support transcribe when you stop.
39 +
40 +## Dictate
41 +
42 +Press the microphone in the composer, or `mod+shift+d` (`mod` is Command on
43 +macOS and Ctrl elsewhere). Press again, or the check mark, to finish; `Esc`
44 +discards the recording and leaves your draft unchanged. Recordings last up to
45 +five minutes. Text is inserted at the cursor or replaces the selection you had
46 +when recording started, and the editor is read-only while dictation runs.
47 +
48 +Pressing Send while dictating finishes the recording and sends the draft once
49 +the transcript has been inserted. Cancelling also cancels that queued send. In
50 +the other editors, Save, Comment, Reply, or Submit review works the same way,
51 +and the shortcut targets the editor that has focus, or the composer when none
52 +does.
53 +
54 +**Dictation shortcut** in Voice settings picks how the keybinding behaves:
55 +**Auto** tap-toggles or records while held, **Hold** records only while the keys
56 +are down, and **Toggle** always needs a second press. Change the chord itself in
57 +[Keybindings settings](./keybindings.md).
58 +
59 +## Improve recognition
60 +
61 +- **Language** picks what you speak. **Auto** lets a multilingual model detect
62 + it. Models that translate offer **Translate to English**.
63 +- **Dictionary** holds names and terms the model should get right. Add an alias
64 + under a term when transcription keeps producing a particular misspelling.
65 + Your dictionary is saved on this device. The name of the project you dictate
66 + in is sent along as a hint.
67 +- **Project dictionary** is vocabulary saved with a project on its environment,
68 + so everyone who dictates in that project gets the same spellings. Pick the
69 + project at the top of Settings to edit it; with an environment selected, the
70 + entries apply to every project there. Project spellings win when they overlap
71 + your personal dictionary. Each recording carries up to 100 entries from the
72 + two combined.
73 +- **Silence filtering** is on by default. Your chosen environment skips non-speech
74 + audio before transcription. Turn it off to keep those sections. Filtering happens
75 + after upload: it does not grant permission to send audio or reduce how much is sent.
76 +- **Remove filler words** drops hesitation sounds from finished transcripts.
77 +
78 +Language and these preferences travel with each recording to the chosen
79 +environment; nothing else about the project is sent.
80 +
81 +## Clean up transcripts with a provider
82 +
83 +**Transcript cleanup** is a separate choice and is off by default. When on, the
84 +finished transcript and up to 1,024 characters each from before, inside, and
85 +after your selection are sent to the
86 +text generation provider configured on the draft's environment, which fixes
87 +punctuation, numbers, and spoken corrections such as "make it 42, sorry, 24".
88 +Depending on the provider, that can involve an external service. Audio is never
89 +sent to it. Use the built-in instructions or write your own, press **Skip**
90 +while cleanup runs to insert the raw transcript, and if cleanup fails the raw
91 +transcript is inserted.
92 +
93 +Cleanup uses the environment's text generation model unless you pick a
94 +**Cleanup model** for that environment, for example a cheaper or faster one.
95 +
96 +If cleanup misses the way you correct yourself, set a **Correction cue**: the
97 +word or phrase you say to flag a correction, such as "scratch that". Cleanup
98 +applies the correction only where the context shows one and drops the cue, and
99 +the model hears the cue as a dictionary hint.
100 +
101 +## Test and troubleshoot
102 +
103 +In **Settings → Voice**, **Test microphone** beside the microphone selector
104 +records up to ten seconds and plays it back in your browser;
105 +the audio stays on this device and is never transcribed, so silent playback
106 +points at the microphone. **Test model** on the selected model shows a transcript using the
107 +same environment, language, and dictionary as the composer, but skips transcript
108 +cleanup even when it is on, so you see what the model heard. Browser recording
109 +needs HTTPS or localhost and microphone permission; for a remote server over
110 +plain HTTP, use the desktop app. If the selected microphone is missing, choose
111 +another or the system default.
packages/client-runtime/src/voice-input/controller.test.ts
+15
@@ -231,6 +231,21 @@ describe("environment voice input", () => {
231 });
232
233 describe("resolveTranscriptCommit", () => {
234 + it.each([
235 + ["Hello", "world", "Hello world"],
236 + ["Hello", "世界", "Hello世界"],
237 + ["你好", "世界", "你好世界"],
238 + ])(
239 + "preserves script boundaries with an autodetected language: %s + %s",
240 + (text, spoken, expected) => {
241 + const captured = draft({ text, selection: { start: text.length, end: text.length } });
242 + expect(resolveTranscriptCommit(captured, captured, spoken, "auto")).toMatchObject({
243 + kind: "commit",
244 + text: expected,
245 + });
246 + },
247 + );
248 +
249 it("replaces the recorded UTF-16 selection around emoji and composer tokens", () => {
250 const text = "Fix 🧪 then $review please";
251 const tokenStart = text.indexOf("$review");
packages/client-runtime/src/voice-input/controller.ts
+5 -2
@@ -11,7 +11,7 @@ const isVoiceTranscriptionError = Schema.is(VoiceTranscriptionError);
11
12 export const VOICE_RECORDING_LIMIT_SECONDS = 5 * 60;
13
14 -const MICROPHONE_PERMISSION_ERROR =
14 +export const MICROPHONE_PERMISSION_ERROR =
15 "Microphone access is required for voice input. Allow access in your browser or system settings and try again.";
16
17 function isMicrophonePermissionError(error: unknown): boolean {
@@ -131,14 +131,17 @@ export function resolveTranscriptCommit(
131 const normalizedLocale = locale.replaceAll("_", "-").toLowerCase();
132 const usesEnglishSpacing = normalizedLocale === "en" || normalizedLocale.startsWith("en-");
133 let insertion = replacement;
134 - if (isEmptySelection && usesEnglishSpacing) {
134 + const autoSpacing = normalizedLocale === "auto";
135 + if (isEmptySelection && (usesEnglishSpacing || autoSpacing)) {
136 const left = captured.text[captured.selection.start - 1];
137 const right = captured.text[captured.selection.start];
138 const leftNeedsBoundary =
139 + (!autoSpacing || /^[\p{Script=Latin}\p{N}]/u.test(replacement)) &&
140 left !== undefined &&
141 /[A-Za-z0-9.!?,:;)\]}'"]/.test(left) &&
142 (right === undefined || /\s/.test(right));
143 const rightNeedsBoundary =
144 + (!autoSpacing || /[\p{Script=Latin}\p{N}]$/u.test(replacement)) &&
145 right !== undefined &&
146 /[A-Za-z0-9([{'"]/.test(right) &&
147 (left === undefined || /\s/.test(left));
packages/client-runtime/src/voice-input/environment.ts new
+208
@@ -0,0 +1,208 @@
1 +import { encodeSpeechPcmRequest } from "@t3tools/shared/speech";
2 +import * as Effect from "effect/Effect";
3 +import * as Redacted from "effect/Redacted";
4 +import { SPEECH_STREAM_PATH } from "@t3tools/contracts";
5 +import type {
6 + SpeechAcceleration,
7 + SpeechModelUnloadTimeout,
8 + SpeechTranscriptionOptions,
9 + SpeechPostProcessingOptions,
10 +} from "@t3tools/contracts";
11 +
12 +import * as RemoteEnvironmentAuthorization from "../authorization/service.ts";
13 +import type { PreparedConnection } from "../connection/model.ts";
14 +import * as ManagedRelay from "../relay/managedRelay.ts";
15 +import {
16 + makeEnvironmentHttpApiGroupClient,
17 + makeEnvironmentHttpApiUrlBuilder,
18 +} from "../rpc/http.ts";
19 +import { executeAuthenticatedEnvironmentHttpRequest } from "../state/environmentHttpAuth.ts";
20 +
21 +const INFERENCE_OR_DOWNLOAD_TIMEOUT_MS = 30 * 60_000;
22 +type EnvironmentApiClient<G extends "voice" | "auth"> = Effect.Success<
23 + ReturnType<typeof makeEnvironmentHttpApiGroupClient<G>>
24 +>;
25 +
26 +const request = Effect.fn("clientRuntime.voiceInput.environmentRequest")(function* <
27 + A,
28 + E,
29 + G extends "voice" | "auth",
30 +>(input: {
31 + readonly group: G;
32 + readonly prepared: PreparedConnection;
33 + readonly method: "GET" | "POST" | "DELETE";
34 + readonly timeoutMs?: number;
35 + readonly canSend?: () => boolean;
36 + readonly path: (urls: ReturnType<typeof makeEnvironmentHttpApiUrlBuilder>[G]) => string;
37 + readonly run: (input: {
38 + readonly client: EnvironmentApiClient<G>;
39 + readonly headers: { readonly authorization?: string; readonly dpop?: string };
40 + }) => Effect.Effect<A, E>;
41 +}) {
42 + return yield* executeAuthenticatedEnvironmentHttpRequest({
43 + group: input.group,
44 + prepared: input.prepared,
45 + signer: yield* Effect.serviceOption(ManagedRelay.ManagedRelayDpopSigner),
46 + remoteAuthorization: yield* Effect.serviceOption(
47 + RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization,
48 + ),
49 + method: input.method,
50 + url: input.path,
51 + timeoutMs: input.timeoutMs ?? 30_000,
52 + // Audio and credentials never follow a redirect to another host.
53 + requestInit: { redirect: "error" },
54 + request: (args) =>
55 + Effect.suspend(() => (input.canSend?.() === false ? Effect.interrupt : input.run(args))),
56 + });
57 +});
58 +
59 +export const getEnvironmentSpeechStatus = (prepared: PreparedConnection) =>
60 + request({
61 + group: "voice",
62 + prepared,
63 + method: "GET",
64 + path: (urls) => urls.status(),
65 + run: ({ client, headers }) => client.status({ headers }),
66 + });
67 +
68 +export const getEnvironmentSpeechStreamUrl = (prepared: PreparedConnection) => {
69 + let endpoint = prepared.httpBaseUrl;
70 + return request({
71 + group: "auth",
72 + prepared,
73 + method: "POST",
74 + path: (urls) => {
75 + endpoint = urls.webSocketTicketFor({ params: { use: "media" } });
76 + return endpoint;
77 + },
78 + run: ({ client, headers }) => client.webSocketTicketFor({ headers, params: { use: "media" } }),
79 + }).pipe(
80 + Effect.map(({ ticket }) => {
81 + const url = new URL(SPEECH_STREAM_PATH, endpoint);
82 + url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
83 + url.searchParams.set("wsTicket", Redacted.value(ticket));
84 + return url.toString();
85 + }),
86 + );
87 +};
88 +
89 +export const getEnvironmentSpeechModels = (
90 + prepared: PreparedConnection,
91 + includeBenchmarks = false,
92 +) => {
93 + const query = includeBenchmarks ? { benchmarks: "1" as const } : {};
94 + return request({
95 + group: "voice",
96 + prepared,
97 + method: "GET",
98 + path: (urls) => urls.models({ query }),
99 + run: ({ client, headers }) => client.models({ headers, query }),
100 + });
101 +};
102 +
103 +export const downloadEnvironmentSpeechModel = (prepared: PreparedConnection, modelId: string) =>
104 + request({
105 + group: "voice",
106 + prepared,
107 + method: "POST",
108 + path: (urls) => urls.downloadModel(),
109 + timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
110 + run: ({ client, headers }) => client.downloadModel({ headers, payload: { modelId } }),
111 + });
112 +
113 +export const selectEnvironmentSpeechModel = (prepared: PreparedConnection, modelId: string) =>
114 + request({
115 + group: "voice",
116 + prepared,
117 + method: "POST",
118 + path: (urls) => urls.selectModel(),
119 + run: ({ client, headers }) => client.selectModel({ headers, payload: { modelId } }),
120 + });
121 +
122 +export const cancelEnvironmentSpeechModelDownload = (
123 + prepared: PreparedConnection,
124 + modelId: string,
125 +) =>
126 + request({
127 + group: "voice",
128 + prepared,
129 + method: "POST",
130 + path: (urls) => urls.cancelModelDownload(),
131 + run: ({ client, headers }) => client.cancelModelDownload({ headers, payload: { modelId } }),
132 + });
133 +
134 +export const transcribeEnvironmentPcm = (
135 + prepared: PreparedConnection,
136 + pcm: Uint8Array,
137 + options: SpeechTranscriptionOptions,
138 + canSend: () => boolean,
139 +) =>
140 + request({
141 + group: "voice",
142 + prepared,
143 + method: "POST",
144 + path: (urls) => urls.transcribe(),
145 + canSend,
146 + timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
147 + run: ({ client, headers }) =>
148 + client.transcribe({
149 + headers,
150 + payload: encodeSpeechPcmRequest(pcm, options),
151 + }),
152 + });
153 +
154 +/** Sends the transcript and draft context to a provider on `prepared`; a separate opt-in. */
155 +export const postProcessEnvironmentTranscript = (
156 + prepared: PreparedConnection,
157 + transcript: string,
158 + draft: {
159 + readonly text: string;
160 + readonly selection: { readonly start: number; readonly end: number };
161 + },
162 + options: SpeechPostProcessingOptions,
163 + canSend: () => boolean,
164 +) =>
165 + request({
166 + group: "voice",
167 + prepared,
168 + method: "POST",
169 + path: (urls) => urls.postProcess(),
170 + canSend,
171 + timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
172 + run: ({ client, headers }) =>
173 + client.postProcess({ headers, payload: { transcript, draft, options } }),
174 + });
175 +
176 +export const updateEnvironmentSpeechAcceleration = (
177 + prepared: PreparedConnection,
178 + acceleration: SpeechAcceleration,
179 +) =>
180 + request({
181 + group: "voice",
182 + prepared,
183 + method: "POST",
184 + path: (urls) => urls.updateAcceleration(),
185 + run: ({ client, headers }) => client.updateAcceleration({ headers, payload: { acceleration } }),
186 + });
187 +
188 +export const updateEnvironmentSpeechModelUnloadTimeout = (
189 + prepared: PreparedConnection,
190 + timeout: SpeechModelUnloadTimeout,
191 +) =>
192 + request({
193 + group: "voice",
194 + prepared,
195 + method: "POST",
196 + path: (urls) => urls.updateModelUnloadTimeout(),
197 + run: ({ client, headers }) =>
198 + client.updateModelUnloadTimeout({ headers, payload: { timeout } }),
199 + });
200 +
201 +export const removeEnvironmentSpeechModel = (prepared: PreparedConnection, modelId: string) =>
202 + request({
203 + group: "voice",
204 + prepared,
205 + method: "POST",
206 + path: (urls) => urls.removeModel(),
207 + run: ({ client, headers }) => client.removeModel({ headers, payload: { modelId } }),
208 + });
packages/client-runtime/src/voice-input/index.ts
+15
@@ -1,6 +1,7 @@
1 export {
2 VoiceInputController,
3 VOICE_RECORDING_LIMIT_SECONDS,
4 + MICROPHONE_PERMISSION_ERROR,
5 voiceInputBlocksSubmission,
6 type VoiceDraftSnapshot,
7 type VoiceInputControllerDependencies,
@@ -9,6 +10,20 @@ export {
10 type VoiceRecorder,
11 type VoiceRecorderStatus,
12 } from "./controller.ts";
13 +export {
14 + cancelEnvironmentSpeechModelDownload,
15 + downloadEnvironmentSpeechModel,
16 + getEnvironmentSpeechModels,
17 + getEnvironmentSpeechStatus,
18 + getEnvironmentSpeechStreamUrl,
19 + postProcessEnvironmentTranscript,
20 + removeEnvironmentSpeechModel,
21 + selectEnvironmentSpeechModel,
22 + transcribeEnvironmentPcm,
23 + updateEnvironmentSpeechAcceleration,
24 + updateEnvironmentSpeechModelUnloadTimeout,
25 +} from "./environment.ts";
26 +export { openSpeechStream } from "./stream.ts";
27 export {
28 VoiceTranscriptionError,
29 throwIfVoiceTranscriptionAborted,
packages/client-runtime/src/voice-input/stream.test.ts new
+108
@@ -0,0 +1,108 @@
1 +import { DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS } from "@t3tools/contracts";
2 +import { expect, it, vi } from "vite-plus/test";
3 +import { SPEECH_STREAM_MAX_QUEUED_BYTES, type SpeechStreamEvent } from "@t3tools/contracts";
4 +import { openSpeechStream } from "./stream.ts";
5 +
6 +class TestSocket extends EventTarget {
7 + sent: (string | Uint8Array)[] = [];
8 + closed = false;
9 + send(data: string | Uint8Array) {
10 + this.sent.push(data);
11 + }
12 + close() {
13 + this.closed = true;
14 + this.dispatchEvent(new Event("close"));
15 + }
16 + receive(event: SpeechStreamEvent) {
17 + this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(event) }));
18 + }
19 +}
20 +
21 +function connect(bufferedAudio?: Float32Array, canSend = () => true) {
22 + const socket = new TestSocket();
23 + const abort = new AbortController();
24 + const onText = vi.fn();
25 + const onError = vi.fn();
26 + const stream = openSpeechStream({
27 + url: "wss://environment.test/ws/voice",
28 + options: DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
29 + signal: abort.signal,
30 + canSend,
31 + onText,
32 + onError,
33 + createSocket: () => socket as unknown as WebSocket,
34 + });
35 + socket.dispatchEvent(new Event("open"));
36 + expect(JSON.parse(socket.sent.shift() as string)).toEqual({
37 + type: "start",
38 + options: DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
39 + });
40 + if (bufferedAudio) {
41 + stream.feed(bufferedAudio);
42 + expect(socket.sent).toHaveLength(0);
43 + }
44 + socket.receive({ type: "ready" });
45 + return { socket, abort, onText, onError, stream };
46 +}
47 +
48 +it("drains ordered PCM before finishing and replaces tentative hypotheses", async () => {
49 + const { socket, stream, onText } = connect(new Float32Array([0.1]));
50 + stream.feed(new Float32Array([0.2]));
51 + const final = stream.finish();
52 + expect(socket.sent).toHaveLength(2);
53 + socket.receive({ type: "update", revision: 1, text: { committed: "hello ", tentative: "were" } });
54 + expect(socket.sent).toHaveLength(2);
55 + expect(new Float32Array((socket.sent[1] as Uint8Array).slice().buffer)[0]).toBeCloseTo(0.2);
56 + socket.receive({
57 + type: "update",
58 + revision: 2,
59 + text: { committed: "hello ", tentative: "world" },
60 + });
61 + expect(socket.sent[2]).toBe('{"type":"finish"}');
62 + expect(onText.mock.calls.map(([text]) => text)).toEqual([
63 + { committed: "hello ", tentative: "were" },
64 + { committed: "hello ", tentative: "world" },
65 + ]);
66 + socket.receive({ type: "finished", text: "hello world" });
67 + await expect(final).resolves.toBe("hello world");
68 + expect(socket.closed).toBe(true);
69 +});
70 +
71 +it("rejects recordings beyond the duration limit", async () => {
72 + const { socket, stream, onError } = connect();
73 + stream.feed(new Float32Array(SPEECH_STREAM_MAX_QUEUED_BYTES / 4 + 1));
74 + await expect(stream.finish()).rejects.toThrow("limited to five minutes");
75 + expect(onError).toHaveBeenCalledOnce();
76 + expect(socket.sent).toHaveLength(0);
77 + expect(socket.closed).toBe(true);
78 +});
79 +
80 +it.each(["cancel", "revoke"])("does not send queued frames after %s", async (action) => {
81 + let allowed = true;
82 + const { socket, stream, abort, onText } = connect(undefined, () => allowed);
83 + stream.feed(new Float32Array([0.1]));
84 + stream.feed(new Float32Array([0.2]));
85 + stream.feed(new Float32Array([0.3]));
86 + if (action === "cancel") abort.abort();
87 + else allowed = false;
88 + socket.receive({ type: "update", revision: 1, text: { committed: "late", tentative: "" } });
89 + expect(socket.sent).toHaveLength(2);
90 + expect(onText).not.toHaveBeenCalled();
91 + await expect(stream.finish()).rejects.toThrow(action === "cancel" ? "cancelled" : "revoked");
92 +});
93 +
94 +it("fails on disconnect and times out stalled inference", async () => {
95 + vi.useFakeTimers();
96 + try {
97 + const first = connect();
98 + first.socket.close();
99 + await expect(first.stream.finish()).rejects.toThrow("connection closed");
100 + const second = connect();
101 + second.stream.feed(new Float32Array([0.1]));
102 + await vi.advanceTimersByTimeAsync(120_000);
103 + await expect(second.stream.finish()).rejects.toThrow("stopped responding");
104 + expect(second.socket.closed).toBe(true);
105 + } finally {
106 + vi.useRealTimers();
107 + }
108 +});
packages/client-runtime/src/voice-input/stream.ts new
+172
@@ -0,0 +1,172 @@
1 +// @effect-diagnostics globalTimers:off - imperative WebSocket callbacks own and clear their response deadline.
2 +import {
3 + SPEECH_STREAM_MAX_CHUNK_BYTES,
4 + SPEECH_STREAM_MAX_QUEUED_BYTES,
5 + SpeechStreamEvent,
6 + type SpeechStreamText,
7 + type SpeechTranscriptionOptions,
8 +} from "@t3tools/contracts";
9 +import * as Schema from "effect/Schema";
10 +
11 +const decodeEvent = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechStreamEvent));
12 +
13 +/** Audio stays bounded while the environment acknowledges a small send window. */
14 +export function openSpeechStream(input: {
15 + readonly url: string;
16 + readonly options: SpeechTranscriptionOptions;
17 + readonly signal: AbortSignal;
18 + readonly canSend: () => boolean;
19 + readonly onText: (text: SpeechStreamText) => void;
20 + readonly onError: (error: Error) => void;
21 + readonly createSocket?: (url: string) => WebSocket;
22 +}) {
23 + input.signal.throwIfAborted();
24 + const socket = (input.createSocket ?? ((url) => new WebSocket(url)))(input.url);
25 + let resolveFinal!: (text: string) => void;
26 + let rejectFinal!: (error: Error) => void;
27 + const final = new Promise<string>((resolve, reject) => {
28 + resolveFinal = resolve;
29 + rejectFinal = reject;
30 + });
31 + // A capture failure can precede stop(), when nobody is awaiting the final result yet.
32 + void final.catch(() => undefined);
33 + let queue: Uint8Array<ArrayBuffer>[] = [];
34 + let queuedBytes = 0;
35 + const inFlight: number[] = [];
36 + let inFlightBytes = 0;
37 + let finishing = false;
38 + let finishSent = false;
39 + let closed = false;
40 + let isReady = false;
41 + let revision = -1;
42 + let timer: ReturnType<typeof setTimeout>;
43 + const cleanup = () => {
44 + clearTimeout(timer);
45 + input.signal.removeEventListener("abort", abort);
46 + queue = [];
47 + queuedBytes = 0;
48 + };
49 + const fail = (error: Error, notify = true) => {
50 + if (closed) return;
51 + closed = true;
52 + cleanup();
53 + rejectFinal(error);
54 + socket.close();
55 + if (notify) input.onError(error);
56 + };
57 + const armTimeout = () => {
58 + clearTimeout(timer);
59 + timer = setTimeout(() => fail(new Error("Live transcription stopped responding.")), 120_000);
60 + };
61 + const abort = () => fail(new Error("Voice transcription was cancelled."), false);
62 + const pump = () => {
63 + if (!isReady || closed || finishSent) return;
64 + if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
65 + while (inFlight.length < 2 && queue.length) {
66 + const chunk = queue.shift()!;
67 + inFlight.push(chunk.byteLength);
68 + inFlightBytes += chunk.byteLength;
69 + queuedBytes -= chunk.byteLength;
70 + socket.send(chunk);
71 + armTimeout();
72 + }
73 + if (finishing && !queue.length && !inFlight.length) {
74 + finishSent = true;
75 + socket.send(JSON.stringify({ type: "finish" }));
76 + armTimeout();
77 + }
78 + };
79 + socket.addEventListener("open", () => {
80 + if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
81 + if (!closed) socket.send(JSON.stringify({ type: "start", options: input.options }));
82 + });
83 + socket.addEventListener("message", (event) => {
84 + if (closed) return;
85 + if (!input.canSend()) return fail(new Error("Voice consent was revoked."));
86 + try {
87 + const message = decodeEvent(event.data);
88 + switch (message.type) {
89 + case "ready":
90 + if (isReady) throw new Error("Unexpected speech stream response.");
91 + isReady = true;
92 + clearTimeout(timer);
93 + pump();
94 + break;
95 + case "update":
96 + if (!inFlight.length || message.revision < revision)
97 + throw new Error("Unexpected speech stream response.");
98 + revision = message.revision;
99 + inFlightBytes -= inFlight.shift()!;
100 + clearTimeout(timer);
101 + if (message.text) input.onText(message.text);
102 + pump();
103 + if (inFlight.length) armTimeout();
104 + break;
105 + case "finished":
106 + if (!finishSent) throw new Error("Unexpected speech stream response.");
107 + closed = true;
108 + cleanup();
109 + resolveFinal(message.text);
110 + socket.close();
111 + break;
112 + case "error":
113 + fail(new Error(message.message));
114 + }
115 + } catch (error) {
116 + fail(error instanceof Error ? error : new Error("Invalid speech stream response."));
117 + }
118 + });
119 + socket.addEventListener("close", () =>
120 + fail(new Error("The voice connection closed. Please record again.")),
121 + );
122 + socket.addEventListener("error", () =>
123 + fail(new Error("Could not connect to live transcription.")),
124 + );
125 + input.signal.addEventListener("abort", abort, { once: true });
126 + armTimeout();
127 + if (input.signal.aborted) abort();
128 + return {
129 + feed: (pcm: Float32Array) => {
130 + if (closed || finishing) return;
131 + const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
132 + if (queuedBytes + inFlightBytes + bytes.byteLength > SPEECH_STREAM_MAX_QUEUED_BYTES) {
133 + fail(new Error("Voice recordings are limited to five minutes."));
134 + return;
135 + }
136 + let offset = 0;
137 + while (offset < bytes.length) {
138 + const tail = queue.at(-1);
139 + const available = tail ? SPEECH_STREAM_MAX_CHUNK_BYTES - tail.byteLength : 0;
140 + if (tail && available > 0) {
141 + const appended = Math.min(available, bytes.length - offset);
142 + const merged = new Uint8Array(tail.byteLength + appended);
143 + merged.set(tail);
144 + merged.set(bytes.subarray(offset, offset + appended), tail.byteLength);
145 + queue[queue.length - 1] = merged;
146 + queuedBytes += appended;
147 + offset += appended;
148 + continue;
149 + }
150 + const chunk = bytes.slice(offset, offset + SPEECH_STREAM_MAX_CHUNK_BYTES);
151 + queue.push(chunk);
152 + queuedBytes += chunk.byteLength;
153 + offset += chunk.byteLength;
154 + }
155 + try {
156 + pump();
157 + } catch {
158 + fail(new Error("Could not send microphone audio."));
159 + }
160 + },
161 + finish: () => {
162 + finishing = true;
163 + try {
164 + pump();
165 + } catch {
166 + fail(new Error("Could not finish live transcription."));
167 + }
168 + return final;
169 + },
170 + cancel: abort,
171 + };
172 +}
packages/contracts/src/keybindings.ts
+1
@@ -90,6 +90,7 @@ export const STATIC_KEYBINDING_COMMANDS = [
90 "composer.sendAlternate",
91 "composer.sendBackground",
92 "composer.sendAndNewThread",
93 + "composer.dictation",
94 "composer.host",
95 "composer.cycleHost",
96 "composer.effort",
packages/contracts/src/settings.test.ts
+4
@@ -501,6 +501,10 @@ describe("voice settings", () => {
501 voiceTranscriptionDestination: { kind: "environment", environmentId: "env-1" },
502 speechPostProcessingEnabled: true,
503 };
504 + expect(decodeClientSettings({}).speechSilenceFiltering).toBe(true);
505 + expect(decodeClientSettingsPatch({ speechSilenceFiltering: false })).toEqual({
506 + speechSilenceFiltering: false,
507 + });
508 for (const key of Object.keys(imported)) {
509 expect(decodeClientSettings(imported)).not.toHaveProperty(key);
510 expect(decodeClientSettingsPatch(imported)).not.toHaveProperty(key);
packages/contracts/src/settings.ts
+2
@@ -532,6 +532,7 @@ export const ClientSettingsSchema = Schema.Struct({
532 speechLanguage: SpeechLanguage.pipe(Schema.withDecodingDefault(Effect.succeed("auto"))),
533 speechTranslateToEnglish: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(false))),
534 speechCustomWords: SpeechCustomWords.pipe(Schema.withDecodingDefault(Effect.succeed([]))),
535 + speechSilenceFiltering: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
536 speechRemoveFillerWords: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
537 speechPostProcessingPrompt: SpeechPostProcessingPromptSettings.pipe(
538 Schema.withDecodingDefault(
@@ -1892,6 +1893,7 @@ export const ClientSettingsPatch = Schema.Struct({
1893 speechLanguage: Schema.optionalKey(SpeechLanguage),
1894 speechTranslateToEnglish: Schema.optionalKey(Schema.Boolean),
1895 speechCustomWords: Schema.optionalKey(SpeechCustomWords),
1896 + speechSilenceFiltering: Schema.optionalKey(Schema.Boolean),
1897 speechRemoveFillerWords: Schema.optionalKey(Schema.Boolean),
1898 speechPostProcessingPrompt: Schema.optionalKey(SpeechPostProcessingPromptSettings),
1899 speechCorrectionWord: Schema.optionalKey(SpeechCorrectionCue),
packages/contracts/src/speech.ts
+10 -3
@@ -1,3 +1,4 @@
1 +import * as Effect from "effect/Effect";
2 import * as Schema from "effect/Schema";
3
4 export const SpeechModelId = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(160));
@@ -101,6 +102,7 @@ export const EnvironmentSpeechStatus = Schema.Union([
102 model: Schema.String,
103 size: Schema.Finite,
104 supportsStreaming: Schema.Boolean,
105 + supportsBufferedStreaming: Schema.optionalKey(Schema.Boolean),
106 supportsTranslation: Schema.Boolean,
107 languages: Schema.optionalKey(Schema.Array(Schema.String)),
108 supportsLanguageDetection: Schema.optionalKey(Schema.Boolean),
@@ -126,12 +128,18 @@ export const SpeechPostProcessingPromptSettings = Schema.Struct({
128 });
129 export type SpeechPostProcessingPromptSettings = typeof SpeechPostProcessingPromptSettings.Type;
130
131 +/** A word the person says to flag a spoken correction, such as "sorry" or "err". */
132 +export const SpeechCorrectionCue = Schema.String.check(Schema.isMaxLength(50));
133 +
134 /** Recognition preferences that travel with every recording. */
135 export const SpeechTranscriptionOptions = Schema.Struct({
136 speechLanguage: SpeechLanguage,
137 speechTranslateToEnglish: Schema.Boolean,
138 speechCustomWords: SpeechCustomWords,
139 speechRemoveFillerWords: Schema.Boolean,
140 + /** Preserve this cue for separately consented transcript cleanup. */
141 + speechCorrectionWord: Schema.optionalKey(SpeechCorrectionCue),
142 + speechSilenceFiltering: Schema.Boolean.pipe(Schema.withDecodingDefault(Effect.succeed(true))),
143 });
144 export type SpeechTranscriptionOptions = typeof SpeechTranscriptionOptions.Type;
145 export const DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS: SpeechTranscriptionOptions = {
@@ -139,12 +147,10 @@ export const DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS: SpeechTranscriptionOptions =
147 speechTranslateToEnglish: false,
148 speechCustomWords: [],
149 speechRemoveFillerWords: true,
150 + speechSilenceFiltering: true,
151 };
152 export const SPEECH_MAX_OPTIONS_BYTES = 128_000;
153
145 -/** A word the person says to flag a spoken correction, such as "sorry" or "err". */
146 -export const SpeechCorrectionCue = Schema.String.check(Schema.isMaxLength(50));
147 -
154 export const SpeechPostProcessingOptions = Schema.Struct({
155 speechCustomWords: SpeechCustomWords,
156 speechPostProcessingPrompt: SpeechPostProcessingPromptSettings,
@@ -175,6 +181,7 @@ Preserve the original language, wording, word order, answers, numbers, and negat
181 export const SPEECH_STREAM_PATH = "/ws/voice";
182 const SPEECH_SAMPLE_RATE = 16_000;
183 export const SPEECH_STREAM_MAX_CHUNK_BYTES = SPEECH_SAMPLE_RATE * 4;
184 +export const SPEECH_STREAM_MAX_QUEUED_BYTES = SPEECH_SAMPLE_RATE * 4 * 5 * 60;
185
186 export const SpeechStreamCommand = Schema.Union([
187 Schema.Struct({ type: Schema.Literal("start"), options: SpeechTranscriptionOptions }),
packages/shared/src/keybindings.ts
+1
@@ -159,6 +159,7 @@ export const DEFAULT_KEYBINDINGS: ReadonlyArray<KeybindingRule> = [
159 { key: "mod+t", command: "rightPanel.new", when: "rightPanelOpen && !terminalFocus" },
160 { key: "mod+d", command: "terminal.split", when: "terminalFocus" },
161 { key: "mod+shift+d", command: "terminal.splitVertical", when: "terminalFocus" },
162 + { key: "mod+shift+d", command: "composer.dictation", when: "!terminalFocus" },
163 { key: "mod+n", command: "terminal.new", when: "terminalFocus" },
164 { key: "mod+w", command: "terminal.close", when: "terminalFocus" },
165 { key: "mod+w", command: "rightPanel.close", when: "!terminalFocus" },
packages/shared/src/speech.test.ts
+20
@@ -17,6 +17,26 @@ describe("speech PCM request framing", () => {
17 expect(decodeSpeechPcmRequest(pool.subarray(3))).toEqual({ pcm, options });
18 });
19
20 + it("accepts recordings from clients that predate silence filtering", () => {
21 + const metadata = new TextEncoder().encode(
22 + JSON.stringify({
23 + speechLanguage: "fr",
24 + speechTranslateToEnglish: false,
25 + speechCustomWords: [],
26 + speechRemoveFillerWords: false,
27 + }),
28 + );
29 + const body = new Uint8Array(4 + metadata.length + 4);
30 + new DataView(body.buffer).setUint32(0, metadata.length, true);
31 + body.set(metadata, 4);
32 + expect(decodeSpeechPcmRequest(body).options).toEqual({
33 + ...DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
34 + speechLanguage: "fr",
35 + speechRemoveFillerWords: false,
36 + speechSilenceFiltering: true,
37 + });
38 + });
39 +
40 it("rejects truncated frames, oversized option lengths and missing audio", () => {
41 expect(() => decodeSpeechPcmRequest(new Uint8Array(3))).toThrow();
42 const oversized = new Uint8Array(5);