@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 03:37 UTC

main
ts 458 lines 16.3 KB
Raw
1 // @effect-diagnostics nodeBuiltinImport:off globalTimers:off - native inference and IPC cleanup need killable Promise boundaries outside the Effect runtime.
2 import * as NodeChildProcess from "node:child_process";
3 import * as NodeSea from "node:sea";
4 import type { SpeechStreamText } from "@t3tools/contracts";
5 import { createSpeechAudio } from "./audio.ts";
6
7 // Inline the small child entry so the packaged CLI does not need a separate worker artifact.
8 const entry = `
9 const createSpeechAudio = ${createSpeechAudio.toString()};
10 let speech;
11 let model;
12 let vadModel;
13 let vad;
14 let session;
15 let stream;
16 let audio;
17 let options;
18 let nativeStreaming;
19 let useVad;
20 let scored;
21 let revision;
22 let texts;
23 let heardAudio;
24 const warnVad = () => console.warn("Silero silence filtering unavailable; preserving raw recording and using bounded ASR chunks.");
25 const resetAsr = () => {
26 if (stream) stream.reset();
27 if (session) session.dispose();
28 stream = session = undefined;
29 };
30 const newAudio = (filterSilence) => createSpeechAudio({
31 frameSamples: 512, filterSilence, nativeStreaming,
32 longForm: model.supports("long_form"), maxAudioMs: model.capabilities.maxAudioMs,
33 });
34 const openAsr = async () => {
35 if (nativeStreaming) {
36 session = model.createSession();
37 stream = await session.stream(options);
38 }
39 };
40 const begin = async (message, live) => {
41 resetAsr();
42 options = message.options;
43 nativeStreaming = live && model.capabilities.supportsStreaming;
44 texts = [];
45 scored = revision = 0;
46 heardAudio = false;
47 useVad = Boolean(message.vadPath) && (message.filterSilence ||
48 (!nativeStreaming && !model.supports("long_form")));
49 if (useVad) {
50 try {
51 if (!vadModel) {
52 vadModel = await speech.TranscribeModel.load(message.vadPath, { backend: "cpu" });
53 if (vadModel.vadInfo.sampleRate !== 16000 || vadModel.vadInfo.frameSamples !== 512)
54 throw new Error("Unsupported Silero model frames.");
55 vad = vadModel.createVadSession({ nThreads: 1 });
56 }
57 await vad.streamReset();
58 } catch {
59 vad?.dispose();
60 vadModel?.dispose();
61 vad = vadModel = undefined;
62 useVad = false;
63 warnVad();
64 }
65 }
66 audio = newAudio(useVad && message.filterSilence);
67 await openAsr();
68 };
69 const transcribe = async (pcm) => {
70 try {
71 const result = await model.transcribe(pcm, options);
72 texts.push(result.text);
73 } catch (error) {
74 if ((error?.constructor?.name !== "OutputTruncated" &&
75 error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error;
76 const middle = Math.floor(pcm.length / 2);
77 await transcribe(pcm.subarray(0, middle));
78 await transcribe(pcm.subarray(middle));
79 }
80 };
81 const decode = async (final) => {
82 let preview = null;
83 for (const pcm of audio.take(final)) {
84 if (!pcm.length) continue;
85 heardAudio = true;
86 if (nativeStreaming) {
87 const update = await stream.feed(pcm);
88 if (update.committedChanged || update.tentativeChanged) {
89 const text = stream.text;
90 preview = { committed: text.committed, tentative: text.tentative };
91 }
92 } else {
93 await transcribe(pcm);
94 preview = { committed: texts.join(" ").trim(), tentative: "" };
95 }
96 }
97 return preview;
98 };
99 const feed = async (pcm, final = false) => {
100 if (pcm) audio.append(pcm);
101 try {
102 if (useVad) {
103 if (pcm) {
104 const result = await vad.streamFeed(pcm);
105 audio.score(result.firstFrame, result.probs);
106 if (result.firstFrame + result.probs.length !== Math.floor(audio.samples() / 512))
107 throw new Error("Incomplete Silero frames.");
108 }
109 if (final) {
110 const result = await vad.streamFlush();
111 audio.score(result.firstFrame, result.probs);
112 if (result.firstFrame + result.probs.length !== Math.ceil(audio.samples() / 512))
113 throw new Error("Incomplete Silero tail.");
114 }
115 } else {
116 const count = final ? Math.ceil(audio.samples() / 512) : Math.floor(audio.samples() / 512);
117 audio.score(scored, new Array(count - scored).fill(1));
118 scored = count;
119 }
120 } catch {
121 // Replay on optional VAD failure, including audio it had withheld; ASR errors stay separate.
122 const raw = audio.audio();
123 useVad = false;
124 warnVad();
125 resetAsr();
126 texts = [];
127 heardAudio = false;
128 audio = newAudio(false);
129 audio.append(raw);
130 scored = final ? Math.ceil(raw.length / 512) : Math.floor(raw.length / 512);
131 audio.score(0, new Array(scored).fill(1));
132 await openAsr();
133 }
134 return await decode(final);
135 };
136 process.on("message", async (message) => {
137 try {
138 if (message.kind === "load") {
139 speech = await loadSpeechModule(message.moduleUrl);
140 const { TranscribeModel, getAvailableBackends } = speech;
141 const device = message.acceleration.startsWith("gpu:")
142 ? getAvailableBackends().find((item) =>
143 (item.deviceType === "gpu" || item.deviceType === "igpu") &&
144 JSON.stringify([item.kind, item.deviceId ?? item.name]) === message.acceleration.slice(4))
145 : undefined;
146 if (message.acceleration.startsWith("gpu:") && !device)
147 throw new Error("The selected speech GPU is unavailable.");
148 model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration });
149 process.send({ type: "t3-speech-reply", ok: true, backend: model.backend,
150 languages: model.capabilities.languages,
151 supportsStreaming: model.capabilities.supportsStreaming,
152 supportsLongForm: model.supports("long_form"),
153 supportsTranslation: model.capabilities.supportsTranslate,
154 supportsVocabulary: model.supports("vocabulary") });
155 } else if (message.kind === "begin") {
156 await begin(message, true);
157 process.send({ type: "t3-speech-reply", ok: true });
158 } else if (message.kind === "feed") {
159 const preview = await feed(message.pcm);
160 process.send({ type: "t3-speech-reply", ok: true, revision: ++revision, preview });
161 } else if (message.kind === "finish") {
162 await feed(undefined, true);
163 if (nativeStreaming && heardAudio) await stream.finalize();
164 const text = nativeStreaming ? (heardAudio ? stream.text.full : "") : texts.join(" ").trim();
165 resetAsr();
166 audio = undefined;
167 process.send({ type: "t3-speech-reply", ok: true, text });
168 } else if (message.kind === "reset") {
169 resetAsr();
170 if (vad) await vad.streamReset();
171 audio = undefined;
172 process.send({ type: "t3-speech-reply", ok: true });
173 } else if (message.kind === "transcribe") {
174 await begin(message, false);
175 await feed(message.pcm, true);
176 const text = texts.join(" ").trim();
177 audio = undefined;
178 process.send({ type: "t3-speech-reply", ok: true, text });
179 }
180 } catch (error) {
181 process.send({ type: "t3-speech-reply", ok: false,
182 error: error instanceof Error ? error.stack ?? error.message : String(error) });
183 }
184 });
185 // Exit teardown under in-flight native inference segfaults, so terminate without it.
186 process.on("disconnect", () => process.kill(process.pid, "SIGKILL"));
187 `;
188
189 type Reply = {
190 readonly type: "t3-speech-reply";
191 readonly ok: boolean;
192 readonly text?: string;
193 readonly error?: string;
194 readonly supportsStreaming?: boolean;
195 readonly supportsLongForm?: boolean;
196 readonly supportsTranslation?: boolean;
197 readonly supportsVocabulary?: boolean;
198 readonly backend?: string;
199 readonly languages?: unknown;
200 readonly revision?: number;
201 readonly preview?: SpeechStreamText | null;
202 };
203
204 // transcribe-cpp has an import-only export. Load its disk entry with require so
205 // Node SEA can reach it, including its native dependencies.
206 const loaderEntry = `
207 const { createRequire, findPackageJSON } = await import("node:module");
208 const { dirname, join } = await import("node:path");
209 const loadSpeechModule = async (moduleUrl) => {
210 let speech;
211 if (moduleUrl) speech = await import(moduleUrl);
212 else {
213 const packagePath = findPackageJSON("transcribe-cpp", baseUrl);
214 if (!packagePath) throw new Error("The native speech package is unavailable.");
215 speech = createRequire(packagePath)(join(dirname(packagePath), "dist/index.js"));
216 }
217 if (process.versions.electron && !process.env.TRANSCRIBE_LIBRARY) {
218 const directory = speech.artifactDir();
219 const unpacked = directory.replace(/\\.asar([\\\\/])/g, ".asar.unpacked$1");
220 // Koffi opens native libraries outside Electron's ASAR-aware filesystem.
221 if (unpacked !== directory) process.env.TRANSCRIBE_LIBRARY = join(unpacked,
222 process.platform === "win32" ? "transcribe.dll"
223 : process.platform === "darwin" ? "libtranscribe.dylib" : "libtranscribe.so");
224 }
225 return speech;
226 };
227 `;
228
229 const discoveryEntry = `
230 const { getAvailableBackends } = await loadSpeechModule(moduleUrl);
231 process.send(getAvailableBackends().filter((device) =>
232 device.deviceType === "gpu" || device.deviceType === "igpu").map((device) => ({
233 id: JSON.stringify([device.kind, device.deviceId ?? device.name]),
234 name: device.description || device.name,
235 })));
236 `;
237
238 export async function runNativeSpeechWorker(mode: "discover" | "model", moduleUrl?: string) {
239 if (!process.send) throw new Error("Speech workers require an IPC channel.");
240 const run = new Function(
241 "baseUrl",
242 "moduleUrl",
243 `return (async () => {${loaderEntry}${mode === "discover" ? discoveryEntry : entry}})();`,
244 ) as (baseUrl: string, moduleUrl?: string) => Promise<void>;
245 await run(import.meta.url, moduleUrl);
246 }
247
248 function speechWorkerArgs(mode: "discover" | "model", moduleUrl?: string) {
249 if (NodeSea.isSea()) return ["__speech-worker", mode, ...(moduleUrl ? [moduleUrl] : [])];
250 // Normal Node and Electron hosts can evaluate the same entry without a sibling artifact.
251 const loader = `const baseUrl = ${JSON.stringify(import.meta.url)};
252 const moduleUrl = process.argv[1];${loaderEntry}`;
253 return [
254 "--input-type=module",
255 "-e",
256 loader + (mode === "discover" ? discoveryEntry : entry),
257 ...(moduleUrl ? [moduleUrl] : []),
258 ];
259 }
260
261 export async function listNativeSpeechGpuDevices(moduleUrl?: string) {
262 const child = NodeChildProcess.spawn(process.execPath, speechWorkerArgs("discover", moduleUrl), {
263 stdio: ["ignore", "ignore", "inherit", "ipc"],
264 env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" },
265 });
266 return await new Promise<{ id: string; name: string }[]>((resolve, reject) => {
267 const timer = setTimeout(() => {
268 child.kill();
269 reject(new Error("Speech device discovery timed out."));
270 }, 10_000);
271 child.on("message", (devices: unknown) => {
272 if (
273 Array.isArray(devices) &&
274 devices.every(
275 (device: unknown) =>
276 typeof device === "object" &&
277 device !== null &&
278 "id" in device &&
279 typeof device.id === "string" &&
280 "name" in device &&
281 typeof device.name === "string",
282 )
283 ) {
284 clearTimeout(timer);
285 resolve(devices);
286 child.kill();
287 }
288 });
289 child.once("error", (error) => {
290 clearTimeout(timer);
291 child.kill();
292 reject(error);
293 });
294 child.once("exit", (code) => {
295 clearTimeout(timer);
296 reject(new Error(`Speech device discovery exited (${code}).`));
297 });
298 });
299 }
300
301 /** Loads a model in an isolated process. "auto" retries on the CPU when GPU setup fails. */
302 export async function loadNativeSpeechModel(
303 path: string,
304 signal: AbortSignal,
305 moduleUrl?: string,
306 acceleration = "auto",
307 ) {
308 try {
309 return await loadNativeSpeechModelOnce(path, signal, moduleUrl, acceleration);
310 } catch (error) {
311 if (acceleration !== "auto" || signal.aborted) throw error;
312 return await loadNativeSpeechModelOnce(path, signal, moduleUrl, "cpu");
313 }
314 }
315
316 async function loadNativeSpeechModelOnce(
317 path: string,
318 signal: AbortSignal,
319 moduleUrl?: string,
320 acceleration = "auto",
321 ) {
322 signal.throwIfAborted();
323 const child = NodeChildProcess.spawn(process.execPath, speechWorkerArgs("model"), {
324 stdio: ["ignore", "ignore", "inherit", "ipc"],
325 serialization: "advanced",
326 env: {
327 ...process.env,
328 ELECTRON_RUN_AS_NODE: "1",
329 // Backend registration can initialize drivers even when model placement is CPU.
330 ...(acceleration === "cpu" ? { TRANSCRIBE_BACKENDS: "cpu" } : {}),
331 },
332 });
333 let stopped = false;
334 let pending: ReturnType<typeof Promise.withResolvers<Reply>> | undefined;
335 const exited = Promise.withResolvers<void>();
336 const fail = (error: Error) => {
337 stopped = true;
338 pending?.reject(error);
339 pending = undefined;
340 };
341 child.on("error", (error) => {
342 fail(error);
343 exited.resolve();
344 });
345 child.on("exit", (code, exitSignal) => {
346 fail(new Error(`Speech process exited (${exitSignal ?? code}).`));
347 signal.removeEventListener("abort", stop);
348 exited.resolve();
349 });
350 child.on("message", (message: unknown) => {
351 if (
352 typeof message !== "object" ||
353 message === null ||
354 !("type" in message) ||
355 message.type !== "t3-speech-reply"
356 )
357 return;
358 const reply = message as Reply;
359 const request = pending;
360 pending = undefined;
361 if (reply.ok) request?.resolve(reply);
362 else request?.reject(new Error(reply.error ?? "Native speech failed."));
363 });
364 function stop() {
365 fail(new Error("Speech process stopped."));
366 // Never call native dispose while inference is active. The OS owns cleanup of the isolated process.
367 child.kill("SIGKILL");
368 }
369 signal.addEventListener("abort", stop, { once: true });
370 if (signal.aborted) stop();
371 const send = (message: object) => {
372 if (stopped) return Promise.reject(new Error("Speech process stopped."));
373 if (pending) return Promise.reject(new Error("Speech process is busy."));
374 const request = Promise.withResolvers<Reply>();
375 pending = request;
376 child.send(message, (error) => {
377 if (error) {
378 request.reject(error);
379 if (pending === request) pending = undefined;
380 }
381 });
382 return request.promise;
383 };
384 const dispose = async () => {
385 stop();
386 await exited.promise;
387 };
388 let loaded: Reply;
389 try {
390 loaded = await send({ kind: "load", path, moduleUrl, acceleration });
391 } catch (error) {
392 await dispose();
393 throw error;
394 }
395 return {
396 backend: loaded.backend ?? "unknown",
397 languages: Array.isArray(loaded.languages)
398 ? loaded.languages.filter((entry): entry is string => typeof entry === "string")
399 : [],
400 supportsStreaming: loaded.supportsStreaming === true,
401 supportsLongForm: loaded.supportsLongForm === true,
402 supportsTranslation: loaded.supportsTranslation === true,
403 supportsVocabulary: loaded.supportsVocabulary === true,
404 begin: async (options: NativeSpeechOptions, filterSilence: boolean, vadPath?: string) => {
405 await send({ kind: "begin", options, filterSilence, vadPath });
406 },
407 feed: async (pcm: Float32Array) => {
408 const reply = await send({ kind: "feed", pcm });
409 if (typeof reply.revision !== "number" || reply.preview === undefined)
410 throw new Error("Invalid speech stream response.");
411 return { revision: reply.revision, text: reply.preview };
412 },
413 finish: async () => {
414 const reply = await send({ kind: "finish" });
415 if (typeof reply.text !== "string") throw new Error("Invalid speech stream response.");
416 return reply.text;
417 },
418 reset: async () => {
419 const inFlight = pending?.promise;
420 if (inFlight) {
421 let timeout: ReturnType<typeof setTimeout> | undefined;
422 try {
423 await Promise.race([
424 inFlight.catch(() => undefined),
425 new Promise<never>((_, reject) => {
426 timeout = setTimeout(
427 () => reject(new Error("Speech stream reset timed out.")),
428 1_000,
429 );
430 }),
431 ]);
432 } finally {
433 clearTimeout(timeout);
434 }
435 }
436 await send({ kind: "reset" });
437 },
438 transcribe: async (
439 pcm: Float32Array,
440 options: NativeSpeechOptions,
441 filterSilence: boolean,
442 vadPath?: string,
443 ) => {
444 const result = await send({ kind: "transcribe", pcm, options, filterSilence, vadPath });
445 if (typeof result.text !== "string") throw new Error("Invalid speech process response.");
446 return { text: result.text };
447 },
448 dispose,
449 };
450 }
451
452 type NativeSpeechOptions = {
453 readonly timestamps: "none";
454 readonly task?: "translate";
455 readonly targetLanguage?: "en";
456 readonly language?: string;
457 readonly vocabulary?: readonly string[];
458 };