The best way to control Claude Code, Codex and more from any computer or phone
Mirrored from https://github.com/Camo67/t3code.git
Synced Nov 21, 03:37 UTC
| 1 | // @effect-diagnostics nodeBuiltinImport:off globalTimers:off - native inference and IPC cleanup need killable Promise boundaries outside the Effect runtime. |
| 2 | import * as NodeChildProcess from "node:child_process"; |
| 3 | import * as NodeSea from "node:sea"; |
| 4 | import type { SpeechStreamText } from "@t3tools/contracts"; |
| 5 | import { createSpeechAudio } from "./audio.ts"; |
| 6 | |
| 7 | // Inline the small child entry so the packaged CLI does not need a separate worker artifact. |
| 8 | const entry = ` |
| 9 | const createSpeechAudio = ${createSpeechAudio.toString()}; |
| 10 | let speech; |
| 11 | let model; |
| 12 | let vadModel; |
| 13 | let vad; |
| 14 | let session; |
| 15 | let stream; |
| 16 | let audio; |
| 17 | let options; |
| 18 | let nativeStreaming; |
| 19 | let useVad; |
| 20 | let scored; |
| 21 | let revision; |
| 22 | let texts; |
| 23 | let heardAudio; |
| 24 | const warnVad = () => console.warn("Silero silence filtering unavailable; preserving raw recording and using bounded ASR chunks."); |
| 25 | const resetAsr = () => { |
| 26 | if (stream) stream.reset(); |
| 27 | if (session) session.dispose(); |
| 28 | stream = session = undefined; |
| 29 | }; |
| 30 | const newAudio = (filterSilence) => createSpeechAudio({ |
| 31 | frameSamples: 512, filterSilence, nativeStreaming, |
| 32 | longForm: model.supports("long_form"), maxAudioMs: model.capabilities.maxAudioMs, |
| 33 | }); |
| 34 | const openAsr = async () => { |
| 35 | if (nativeStreaming) { |
| 36 | session = model.createSession(); |
| 37 | stream = await session.stream(options); |
| 38 | } |
| 39 | }; |
| 40 | const begin = async (message, live) => { |
| 41 | resetAsr(); |
| 42 | options = message.options; |
| 43 | nativeStreaming = live && model.capabilities.supportsStreaming; |
| 44 | texts = []; |
| 45 | scored = revision = 0; |
| 46 | heardAudio = false; |
| 47 | useVad = Boolean(message.vadPath) && (message.filterSilence || |
| 48 | (!nativeStreaming && !model.supports("long_form"))); |
| 49 | if (useVad) { |
| 50 | try { |
| 51 | if (!vadModel) { |
| 52 | vadModel = await speech.TranscribeModel.load(message.vadPath, { backend: "cpu" }); |
| 53 | if (vadModel.vadInfo.sampleRate !== 16000 || vadModel.vadInfo.frameSamples !== 512) |
| 54 | throw new Error("Unsupported Silero model frames."); |
| 55 | vad = vadModel.createVadSession({ nThreads: 1 }); |
| 56 | } |
| 57 | await vad.streamReset(); |
| 58 | } catch { |
| 59 | vad?.dispose(); |
| 60 | vadModel?.dispose(); |
| 61 | vad = vadModel = undefined; |
| 62 | useVad = false; |
| 63 | warnVad(); |
| 64 | } |
| 65 | } |
| 66 | audio = newAudio(useVad && message.filterSilence); |
| 67 | await openAsr(); |
| 68 | }; |
| 69 | const transcribe = async (pcm) => { |
| 70 | try { |
| 71 | const result = await model.transcribe(pcm, options); |
| 72 | texts.push(result.text); |
| 73 | } catch (error) { |
| 74 | if ((error?.constructor?.name !== "OutputTruncated" && |
| 75 | error?.constructor?.name !== "InputTooLong") || pcm.length <= 4000) throw error; |
| 76 | const middle = Math.floor(pcm.length / 2); |
| 77 | await transcribe(pcm.subarray(0, middle)); |
| 78 | await transcribe(pcm.subarray(middle)); |
| 79 | } |
| 80 | }; |
| 81 | const decode = async (final) => { |
| 82 | let preview = null; |
| 83 | for (const pcm of audio.take(final)) { |
| 84 | if (!pcm.length) continue; |
| 85 | heardAudio = true; |
| 86 | if (nativeStreaming) { |
| 87 | const update = await stream.feed(pcm); |
| 88 | if (update.committedChanged || update.tentativeChanged) { |
| 89 | const text = stream.text; |
| 90 | preview = { committed: text.committed, tentative: text.tentative }; |
| 91 | } |
| 92 | } else { |
| 93 | await transcribe(pcm); |
| 94 | preview = { committed: texts.join(" ").trim(), tentative: "" }; |
| 95 | } |
| 96 | } |
| 97 | return preview; |
| 98 | }; |
| 99 | const feed = async (pcm, final = false) => { |
| 100 | if (pcm) audio.append(pcm); |
| 101 | try { |
| 102 | if (useVad) { |
| 103 | if (pcm) { |
| 104 | const result = await vad.streamFeed(pcm); |
| 105 | audio.score(result.firstFrame, result.probs); |
| 106 | if (result.firstFrame + result.probs.length !== Math.floor(audio.samples() / 512)) |
| 107 | throw new Error("Incomplete Silero frames."); |
| 108 | } |
| 109 | if (final) { |
| 110 | const result = await vad.streamFlush(); |
| 111 | audio.score(result.firstFrame, result.probs); |
| 112 | if (result.firstFrame + result.probs.length !== Math.ceil(audio.samples() / 512)) |
| 113 | throw new Error("Incomplete Silero tail."); |
| 114 | } |
| 115 | } else { |
| 116 | const count = final ? Math.ceil(audio.samples() / 512) : Math.floor(audio.samples() / 512); |
| 117 | audio.score(scored, new Array(count - scored).fill(1)); |
| 118 | scored = count; |
| 119 | } |
| 120 | } catch { |
| 121 | // Replay on optional VAD failure, including audio it had withheld; ASR errors stay separate. |
| 122 | const raw = audio.audio(); |
| 123 | useVad = false; |
| 124 | warnVad(); |
| 125 | resetAsr(); |
| 126 | texts = []; |
| 127 | heardAudio = false; |
| 128 | audio = newAudio(false); |
| 129 | audio.append(raw); |
| 130 | scored = final ? Math.ceil(raw.length / 512) : Math.floor(raw.length / 512); |
| 131 | audio.score(0, new Array(scored).fill(1)); |
| 132 | await openAsr(); |
| 133 | } |
| 134 | return await decode(final); |
| 135 | }; |
| 136 | process.on("message", async (message) => { |
| 137 | try { |
| 138 | if (message.kind === "load") { |
| 139 | speech = await loadSpeechModule(message.moduleUrl); |
| 140 | const { TranscribeModel, getAvailableBackends } = speech; |
| 141 | const device = message.acceleration.startsWith("gpu:") |
| 142 | ? getAvailableBackends().find((item) => |
| 143 | (item.deviceType === "gpu" || item.deviceType === "igpu") && |
| 144 | JSON.stringify([item.kind, item.deviceId ?? item.name]) === message.acceleration.slice(4)) |
| 145 | : undefined; |
| 146 | if (message.acceleration.startsWith("gpu:") && !device) |
| 147 | throw new Error("The selected speech GPU is unavailable."); |
| 148 | model = await TranscribeModel.load(message.path, device ? { device } : { backend: message.acceleration }); |
| 149 | process.send({ type: "t3-speech-reply", ok: true, backend: model.backend, |
| 150 | languages: model.capabilities.languages, |
| 151 | supportsStreaming: model.capabilities.supportsStreaming, |
| 152 | supportsLongForm: model.supports("long_form"), |
| 153 | supportsTranslation: model.capabilities.supportsTranslate, |
| 154 | supportsVocabulary: model.supports("vocabulary") }); |
| 155 | } else if (message.kind === "begin") { |
| 156 | await begin(message, true); |
| 157 | process.send({ type: "t3-speech-reply", ok: true }); |
| 158 | } else if (message.kind === "feed") { |
| 159 | const preview = await feed(message.pcm); |
| 160 | process.send({ type: "t3-speech-reply", ok: true, revision: ++revision, preview }); |
| 161 | } else if (message.kind === "finish") { |
| 162 | await feed(undefined, true); |
| 163 | if (nativeStreaming && heardAudio) await stream.finalize(); |
| 164 | const text = nativeStreaming ? (heardAudio ? stream.text.full : "") : texts.join(" ").trim(); |
| 165 | resetAsr(); |
| 166 | audio = undefined; |
| 167 | process.send({ type: "t3-speech-reply", ok: true, text }); |
| 168 | } else if (message.kind === "reset") { |
| 169 | resetAsr(); |
| 170 | if (vad) await vad.streamReset(); |
| 171 | audio = undefined; |
| 172 | process.send({ type: "t3-speech-reply", ok: true }); |
| 173 | } else if (message.kind === "transcribe") { |
| 174 | await begin(message, false); |
| 175 | await feed(message.pcm, true); |
| 176 | const text = texts.join(" ").trim(); |
| 177 | audio = undefined; |
| 178 | process.send({ type: "t3-speech-reply", ok: true, text }); |
| 179 | } |
| 180 | } catch (error) { |
| 181 | process.send({ type: "t3-speech-reply", ok: false, |
| 182 | error: error instanceof Error ? error.stack ?? error.message : String(error) }); |
| 183 | } |
| 184 | }); |
| 185 | // Exit teardown under in-flight native inference segfaults, so terminate without it. |
| 186 | process.on("disconnect", () => process.kill(process.pid, "SIGKILL")); |
| 187 | `; |
| 188 | |
| 189 | type Reply = { |
| 190 | readonly type: "t3-speech-reply"; |
| 191 | readonly ok: boolean; |
| 192 | readonly text?: string; |
| 193 | readonly error?: string; |
| 194 | readonly supportsStreaming?: boolean; |
| 195 | readonly supportsLongForm?: boolean; |
| 196 | readonly supportsTranslation?: boolean; |
| 197 | readonly supportsVocabulary?: boolean; |
| 198 | readonly backend?: string; |
| 199 | readonly languages?: unknown; |
| 200 | readonly revision?: number; |
| 201 | readonly preview?: SpeechStreamText | null; |
| 202 | }; |
| 203 | |
| 204 | // transcribe-cpp has an import-only export. Load its disk entry with require so |
| 205 | // Node SEA can reach it, including its native dependencies. |
| 206 | const loaderEntry = ` |
| 207 | const { createRequire, findPackageJSON } = await import("node:module"); |
| 208 | const { dirname, join } = await import("node:path"); |
| 209 | const loadSpeechModule = async (moduleUrl) => { |
| 210 | let speech; |
| 211 | if (moduleUrl) speech = await import(moduleUrl); |
| 212 | else { |
| 213 | const packagePath = findPackageJSON("transcribe-cpp", baseUrl); |
| 214 | if (!packagePath) throw new Error("The native speech package is unavailable."); |
| 215 | speech = createRequire(packagePath)(join(dirname(packagePath), "dist/index.js")); |
| 216 | } |
| 217 | if (process.versions.electron && !process.env.TRANSCRIBE_LIBRARY) { |
| 218 | const directory = speech.artifactDir(); |
| 219 | const unpacked = directory.replace(/\\.asar([\\\\/])/g, ".asar.unpacked$1"); |
| 220 | // Koffi opens native libraries outside Electron's ASAR-aware filesystem. |
| 221 | if (unpacked !== directory) process.env.TRANSCRIBE_LIBRARY = join(unpacked, |
| 222 | process.platform === "win32" ? "transcribe.dll" |
| 223 | : process.platform === "darwin" ? "libtranscribe.dylib" : "libtranscribe.so"); |
| 224 | } |
| 225 | return speech; |
| 226 | }; |
| 227 | `; |
| 228 | |
| 229 | const discoveryEntry = ` |
| 230 | const { getAvailableBackends } = await loadSpeechModule(moduleUrl); |
| 231 | process.send(getAvailableBackends().filter((device) => |
| 232 | device.deviceType === "gpu" || device.deviceType === "igpu").map((device) => ({ |
| 233 | id: JSON.stringify([device.kind, device.deviceId ?? device.name]), |
| 234 | name: device.description || device.name, |
| 235 | }))); |
| 236 | `; |
| 237 | |
| 238 | export async function runNativeSpeechWorker(mode: "discover" | "model", moduleUrl?: string) { |
| 239 | if (!process.send) throw new Error("Speech workers require an IPC channel."); |
| 240 | const run = new Function( |
| 241 | "baseUrl", |
| 242 | "moduleUrl", |
| 243 | `return (async () => {${loaderEntry}${mode === "discover" ? discoveryEntry : entry}})();`, |
| 244 | ) as (baseUrl: string, moduleUrl?: string) => Promise<void>; |
| 245 | await run(import.meta.url, moduleUrl); |
| 246 | } |
| 247 | |
| 248 | function speechWorkerArgs(mode: "discover" | "model", moduleUrl?: string) { |
| 249 | if (NodeSea.isSea()) return ["__speech-worker", mode, ...(moduleUrl ? [moduleUrl] : [])]; |
| 250 | // Normal Node and Electron hosts can evaluate the same entry without a sibling artifact. |
| 251 | const loader = `const baseUrl = ${JSON.stringify(import.meta.url)}; |
| 252 | const moduleUrl = process.argv[1];${loaderEntry}`; |
| 253 | return [ |
| 254 | "--input-type=module", |
| 255 | "-e", |
| 256 | loader + (mode === "discover" ? discoveryEntry : entry), |
| 257 | ...(moduleUrl ? [moduleUrl] : []), |
| 258 | ]; |
| 259 | } |
| 260 | |
| 261 | export async function listNativeSpeechGpuDevices(moduleUrl?: string) { |
| 262 | const child = NodeChildProcess.spawn(process.execPath, speechWorkerArgs("discover", moduleUrl), { |
| 263 | stdio: ["ignore", "ignore", "inherit", "ipc"], |
| 264 | env: { ...process.env, ELECTRON_RUN_AS_NODE: "1" }, |
| 265 | }); |
| 266 | return await new Promise<{ id: string; name: string }[]>((resolve, reject) => { |
| 267 | const timer = setTimeout(() => { |
| 268 | child.kill(); |
| 269 | reject(new Error("Speech device discovery timed out.")); |
| 270 | }, 10_000); |
| 271 | child.on("message", (devices: unknown) => { |
| 272 | if ( |
| 273 | Array.isArray(devices) && |
| 274 | devices.every( |
| 275 | (device: unknown) => |
| 276 | typeof device === "object" && |
| 277 | device !== null && |
| 278 | "id" in device && |
| 279 | typeof device.id === "string" && |
| 280 | "name" in device && |
| 281 | typeof device.name === "string", |
| 282 | ) |
| 283 | ) { |
| 284 | clearTimeout(timer); |
| 285 | resolve(devices); |
| 286 | child.kill(); |
| 287 | } |
| 288 | }); |
| 289 | child.once("error", (error) => { |
| 290 | clearTimeout(timer); |
| 291 | child.kill(); |
| 292 | reject(error); |
| 293 | }); |
| 294 | child.once("exit", (code) => { |
| 295 | clearTimeout(timer); |
| 296 | reject(new Error(`Speech device discovery exited (${code}).`)); |
| 297 | }); |
| 298 | }); |
| 299 | } |
| 300 | |
| 301 | /** Loads a model in an isolated process. "auto" retries on the CPU when GPU setup fails. */ |
| 302 | export async function loadNativeSpeechModel( |
| 303 | path: string, |
| 304 | signal: AbortSignal, |
| 305 | moduleUrl?: string, |
| 306 | acceleration = "auto", |
| 307 | ) { |
| 308 | try { |
| 309 | return await loadNativeSpeechModelOnce(path, signal, moduleUrl, acceleration); |
| 310 | } catch (error) { |
| 311 | if (acceleration !== "auto" || signal.aborted) throw error; |
| 312 | return await loadNativeSpeechModelOnce(path, signal, moduleUrl, "cpu"); |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | async function loadNativeSpeechModelOnce( |
| 317 | path: string, |
| 318 | signal: AbortSignal, |
| 319 | moduleUrl?: string, |
| 320 | acceleration = "auto", |
| 321 | ) { |
| 322 | signal.throwIfAborted(); |
| 323 | const child = NodeChildProcess.spawn(process.execPath, speechWorkerArgs("model"), { |
| 324 | stdio: ["ignore", "ignore", "inherit", "ipc"], |
| 325 | serialization: "advanced", |
| 326 | env: { |
| 327 | ...process.env, |
| 328 | ELECTRON_RUN_AS_NODE: "1", |
| 329 | // Backend registration can initialize drivers even when model placement is CPU. |
| 330 | ...(acceleration === "cpu" ? { TRANSCRIBE_BACKENDS: "cpu" } : {}), |
| 331 | }, |
| 332 | }); |
| 333 | let stopped = false; |
| 334 | let pending: ReturnType<typeof Promise.withResolvers<Reply>> | undefined; |
| 335 | const exited = Promise.withResolvers<void>(); |
| 336 | const fail = (error: Error) => { |
| 337 | stopped = true; |
| 338 | pending?.reject(error); |
| 339 | pending = undefined; |
| 340 | }; |
| 341 | child.on("error", (error) => { |
| 342 | fail(error); |
| 343 | exited.resolve(); |
| 344 | }); |
| 345 | child.on("exit", (code, exitSignal) => { |
| 346 | fail(new Error(`Speech process exited (${exitSignal ?? code}).`)); |
| 347 | signal.removeEventListener("abort", stop); |
| 348 | exited.resolve(); |
| 349 | }); |
| 350 | child.on("message", (message: unknown) => { |
| 351 | if ( |
| 352 | typeof message !== "object" || |
| 353 | message === null || |
| 354 | !("type" in message) || |
| 355 | message.type !== "t3-speech-reply" |
| 356 | ) |
| 357 | return; |
| 358 | const reply = message as Reply; |
| 359 | const request = pending; |
| 360 | pending = undefined; |
| 361 | if (reply.ok) request?.resolve(reply); |
| 362 | else request?.reject(new Error(reply.error ?? "Native speech failed.")); |
| 363 | }); |
| 364 | function stop() { |
| 365 | fail(new Error("Speech process stopped.")); |
| 366 | // Never call native dispose while inference is active. The OS owns cleanup of the isolated process. |
| 367 | child.kill("SIGKILL"); |
| 368 | } |
| 369 | signal.addEventListener("abort", stop, { once: true }); |
| 370 | if (signal.aborted) stop(); |
| 371 | const send = (message: object) => { |
| 372 | if (stopped) return Promise.reject(new Error("Speech process stopped.")); |
| 373 | if (pending) return Promise.reject(new Error("Speech process is busy.")); |
| 374 | const request = Promise.withResolvers<Reply>(); |
| 375 | pending = request; |
| 376 | child.send(message, (error) => { |
| 377 | if (error) { |
| 378 | request.reject(error); |
| 379 | if (pending === request) pending = undefined; |
| 380 | } |
| 381 | }); |
| 382 | return request.promise; |
| 383 | }; |
| 384 | const dispose = async () => { |
| 385 | stop(); |
| 386 | await exited.promise; |
| 387 | }; |
| 388 | let loaded: Reply; |
| 389 | try { |
| 390 | loaded = await send({ kind: "load", path, moduleUrl, acceleration }); |
| 391 | } catch (error) { |
| 392 | await dispose(); |
| 393 | throw error; |
| 394 | } |
| 395 | return { |
| 396 | backend: loaded.backend ?? "unknown", |
| 397 | languages: Array.isArray(loaded.languages) |
| 398 | ? loaded.languages.filter((entry): entry is string => typeof entry === "string") |
| 399 | : [], |
| 400 | supportsStreaming: loaded.supportsStreaming === true, |
| 401 | supportsLongForm: loaded.supportsLongForm === true, |
| 402 | supportsTranslation: loaded.supportsTranslation === true, |
| 403 | supportsVocabulary: loaded.supportsVocabulary === true, |
| 404 | begin: async (options: NativeSpeechOptions, filterSilence: boolean, vadPath?: string) => { |
| 405 | await send({ kind: "begin", options, filterSilence, vadPath }); |
| 406 | }, |
| 407 | feed: async (pcm: Float32Array) => { |
| 408 | const reply = await send({ kind: "feed", pcm }); |
| 409 | if (typeof reply.revision !== "number" || reply.preview === undefined) |
| 410 | throw new Error("Invalid speech stream response."); |
| 411 | return { revision: reply.revision, text: reply.preview }; |
| 412 | }, |
| 413 | finish: async () => { |
| 414 | const reply = await send({ kind: "finish" }); |
| 415 | if (typeof reply.text !== "string") throw new Error("Invalid speech stream response."); |
| 416 | return reply.text; |
| 417 | }, |
| 418 | reset: async () => { |
| 419 | const inFlight = pending?.promise; |
| 420 | if (inFlight) { |
| 421 | let timeout: ReturnType<typeof setTimeout> | undefined; |
| 422 | try { |
| 423 | await Promise.race([ |
| 424 | inFlight.catch(() => undefined), |
| 425 | new Promise<never>((_, reject) => { |
| 426 | timeout = setTimeout( |
| 427 | () => reject(new Error("Speech stream reset timed out.")), |
| 428 | 1_000, |
| 429 | ); |
| 430 | }), |
| 431 | ]); |
| 432 | } finally { |
| 433 | clearTimeout(timeout); |
| 434 | } |
| 435 | } |
| 436 | await send({ kind: "reset" }); |
| 437 | }, |
| 438 | transcribe: async ( |
| 439 | pcm: Float32Array, |
| 440 | options: NativeSpeechOptions, |
| 441 | filterSilence: boolean, |
| 442 | vadPath?: string, |
| 443 | ) => { |
| 444 | const result = await send({ kind: "transcribe", pcm, options, filterSilence, vadPath }); |
| 445 | if (typeof result.text !== "string") throw new Error("Invalid speech process response."); |
| 446 | return { text: result.text }; |
| 447 | }, |
| 448 | dispose, |
| 449 | }; |
| 450 | } |
| 451 | |
| 452 | type NativeSpeechOptions = { |
| 453 | readonly timestamps: "none"; |
| 454 | readonly task?: "translate"; |
| 455 | readonly targetLanguage?: "en"; |
| 456 | readonly language?: string; |
| 457 | readonly vocabulary?: readonly string[]; |
| 458 | }; |