@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 179 lines 6.24 KB
Raw
1 // @effect-diagnostics globalTimers:off - imperative WebSocket callbacks own and clear their response deadline.
2 import { msg } from "gt-i18n";
3 import { translatableError } from "../errors/translatable.ts";
4 import {
5 SPEECH_STREAM_MAX_CHUNK_BYTES,
6 SPEECH_STREAM_MAX_QUEUED_BYTES,
7 SpeechStreamEvent,
8 type SpeechStreamText,
9 type SpeechTranscriptionOptions,
10 } from "@t3tools/contracts";
11 import * as Schema from "effect/Schema";
12
13 const decodeEvent = Schema.decodeUnknownSync(Schema.fromJsonString(SpeechStreamEvent));
14
15 /** Audio stays bounded while the environment acknowledges a small send window. */
16 export function openSpeechStream(input: {
17 readonly url: string;
18 readonly options: SpeechTranscriptionOptions;
19 readonly signal: AbortSignal;
20 readonly canSend: () => boolean;
21 readonly onText: (text: SpeechStreamText) => void;
22 readonly onError: (error: Error) => void;
23 readonly createSocket?: (url: string) => WebSocket;
24 }) {
25 input.signal.throwIfAborted();
26 const socket = (input.createSocket ?? ((url) => new WebSocket(url)))(input.url);
27 let resolveFinal!: (text: string) => void;
28 let rejectFinal!: (error: Error) => void;
29 const final = new Promise<string>((resolve, reject) => {
30 resolveFinal = resolve;
31 rejectFinal = reject;
32 });
33 // A capture failure can precede stop(), when nobody is awaiting the final result yet.
34 void final.catch(() => undefined);
35 let queue: Uint8Array<ArrayBuffer>[] = [];
36 let queuedBytes = 0;
37 const inFlight: number[] = [];
38 let inFlightBytes = 0;
39 let finishing = false;
40 let finishSent = false;
41 let closed = false;
42 let isReady = false;
43 let revision = -1;
44 let timer: ReturnType<typeof setTimeout>;
45 const cleanup = () => {
46 clearTimeout(timer);
47 input.signal.removeEventListener("abort", abort);
48 queue = [];
49 queuedBytes = 0;
50 };
51 const fail = (error: Error, notify = true) => {
52 if (closed) return;
53 closed = true;
54 cleanup();
55 rejectFinal(error);
56 socket.close();
57 if (notify) input.onError(error);
58 };
59 const armTimeout = () => {
60 clearTimeout(timer);
61 timer = setTimeout(
62 () => fail(translatableError(msg("Live transcription stopped responding."))),
63 120_000,
64 );
65 };
66 const abort = () => fail(translatableError(msg("Voice transcription was cancelled.")), false);
67 const pump = () => {
68 if (!isReady || closed || finishSent) return;
69 if (!input.canSend()) return fail(translatableError(msg("Voice consent was revoked.")));
70 while (inFlight.length < 2 && queue.length) {
71 const chunk = queue.shift()!;
72 inFlight.push(chunk.byteLength);
73 inFlightBytes += chunk.byteLength;
74 queuedBytes -= chunk.byteLength;
75 socket.send(chunk);
76 armTimeout();
77 }
78 if (finishing && !queue.length && !inFlight.length) {
79 finishSent = true;
80 socket.send(JSON.stringify({ type: "finish" }));
81 armTimeout();
82 }
83 };
84 socket.addEventListener("open", () => {
85 if (!input.canSend()) return fail(translatableError(msg("Voice consent was revoked.")));
86 if (!closed) socket.send(JSON.stringify({ type: "start", options: input.options }));
87 });
88 socket.addEventListener("message", (event) => {
89 if (closed) return;
90 if (!input.canSend()) return fail(translatableError(msg("Voice consent was revoked.")));
91 try {
92 const message = decodeEvent(event.data);
93 switch (message.type) {
94 case "ready":
95 if (isReady) throw translatableError(msg("Unexpected speech stream response."));
96 isReady = true;
97 clearTimeout(timer);
98 pump();
99 break;
100 case "update":
101 if (!inFlight.length || message.revision < revision)
102 throw translatableError(msg("Unexpected speech stream response."));
103 revision = message.revision;
104 inFlightBytes -= inFlight.shift()!;
105 clearTimeout(timer);
106 if (message.text) input.onText(message.text);
107 pump();
108 if (inFlight.length) armTimeout();
109 break;
110 case "finished":
111 if (!finishSent) throw translatableError(msg("Unexpected speech stream response."));
112 closed = true;
113 cleanup();
114 resolveFinal(message.text);
115 socket.close();
116 break;
117 case "error":
118 fail(new Error(message.message));
119 }
120 } catch (error) {
121 fail(
122 error instanceof Error ? error : translatableError(msg("Invalid speech stream response.")),
123 );
124 }
125 });
126 socket.addEventListener("close", () =>
127 fail(translatableError(msg("The voice connection closed. Please record again."))),
128 );
129 socket.addEventListener("error", () =>
130 fail(translatableError(msg("Could not connect to live transcription."))),
131 );
132 input.signal.addEventListener("abort", abort, { once: true });
133 armTimeout();
134 if (input.signal.aborted) abort();
135 return {
136 feed: (pcm: Float32Array) => {
137 if (closed || finishing) return;
138 const bytes = new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength);
139 if (queuedBytes + inFlightBytes + bytes.byteLength > SPEECH_STREAM_MAX_QUEUED_BYTES) {
140 fail(translatableError(msg("Voice recordings are limited to five minutes.")));
141 return;
142 }
143 let offset = 0;
144 while (offset < bytes.length) {
145 const tail = queue.at(-1);
146 const available = tail ? SPEECH_STREAM_MAX_CHUNK_BYTES - tail.byteLength : 0;
147 if (tail && available > 0) {
148 const appended = Math.min(available, bytes.length - offset);
149 const merged = new Uint8Array(tail.byteLength + appended);
150 merged.set(tail);
151 merged.set(bytes.subarray(offset, offset + appended), tail.byteLength);
152 queue[queue.length - 1] = merged;
153 queuedBytes += appended;
154 offset += appended;
155 continue;
156 }
157 const chunk = bytes.slice(offset, offset + SPEECH_STREAM_MAX_CHUNK_BYTES);
158 queue.push(chunk);
159 queuedBytes += chunk.byteLength;
160 offset += chunk.byteLength;
161 }
162 try {
163 pump();
164 } catch {
165 fail(translatableError(msg("Could not send microphone audio.")));
166 }
167 },
168 finish: () => {
169 finishing = true;
170 try {
171 pump();
172 } catch {
173 fail(translatableError(msg("Could not finish live transcription.")));
174 }
175 return final;
176 },
177 cancel: abort,
178 };
179 }