@devries-cameron20 / t3code Mirror · read-only
Fork 0

The best way to control Claude Code, Codex and more from any computer or phone

Mirrored from https://github.com/Camo67/t3code.git

Synced Nov 21, 04:45 UTC

main
ts 191 lines 7.33 KB
Raw
1 import type { DesktopAppleSpeechResult } from "@t3tools/contracts";
2 import * as HostProcess from "@t3tools/shared/HostProcess";
3 import * as Context from "effect/Context";
4 import * as Duration from "effect/Duration";
5 import * as Effect from "effect/Effect";
6 import * as FileSystem from "effect/FileSystem";
7 import * as Layer from "effect/Layer";
8 import * as Schema from "effect/Schema";
9 import * as Semaphore from "effect/Semaphore";
10 import * as Stream from "effect/Stream";
11 import { ChildProcess, ChildProcessSpawner } from "effect/process";
12
13 import * as DesktopEnvironment from "../app/DesktopEnvironment.ts";
14
15 const SAMPLE_RATE = 16_000;
16 const MAX_PCM_BYTES = SAMPLE_RATE * Float32Array.BYTES_PER_ELEMENT * 5 * 60;
17 // Preparing a language for the first time downloads Apple's model.
18 const PREPARE_TIMEOUT = Duration.minutes(10);
19 const TRANSCRIBE_TIMEOUT = Duration.minutes(5);
20
21 const decodeOutput = Schema.decodeUnknownEffect(
22 Schema.fromJsonString(Schema.Struct({ value: Schema.String })),
23 );
24
25 /** 16 kHz mono Float32 PCM to a 16-bit WAV that AVAudioFile reads. */
26 function encodeSpeechWav(pcmBytes: Uint8Array): Uint8Array {
27 // Float32Array needs a 4-byte-aligned offset; copy a view that is not (Buffer.slice is a view).
28 const aligned =
29 pcmBytes.byteOffset % Float32Array.BYTES_PER_ELEMENT === 0
30 ? pcmBytes
31 : new Uint8Array(pcmBytes);
32 const samples = new Float32Array(
33 aligned.buffer,
34 aligned.byteOffset,
35 aligned.byteLength / Float32Array.BYTES_PER_ELEMENT,
36 );
37 const wav = new Uint8Array(44 + samples.length * 2);
38 const view = new DataView(wav.buffer);
39 const ascii = (offset: number, text: string) => {
40 for (let index = 0; index < text.length; index++)
41 view.setUint8(offset + index, text.charCodeAt(index));
42 };
43 ascii(0, "RIFF");
44 view.setUint32(4, 36 + samples.length * 2, true);
45 ascii(8, "WAVE");
46 ascii(12, "fmt ");
47 view.setUint32(16, 16, true);
48 view.setUint16(20, 1, true);
49 view.setUint16(22, 1, true);
50 view.setUint32(24, SAMPLE_RATE, true);
51 view.setUint32(28, SAMPLE_RATE * 2, true);
52 view.setUint16(32, 2, true);
53 view.setUint16(34, 16, true);
54 ascii(36, "data");
55 view.setUint32(40, samples.length * 2, true);
56 samples.forEach((sample, index) => {
57 const clamped = Number.isFinite(sample) ? Math.max(-1, Math.min(1, sample)) : 0;
58 view.setInt16(44 + index * 2, Math.round(clamped * 0x7fff), true);
59 });
60 return wav;
61 }
62
63 export class AppleSpeech extends Context.Service<
64 AppleSpeech,
65 {
66 readonly supportedLocales: Effect.Effect<readonly string[]>;
67 readonly prepare: (locale: string) => Effect.Effect<DesktopAppleSpeechResult>;
68 readonly transcribe: (
69 locale: string,
70 pcm: Uint8Array,
71 ) => Effect.Effect<DesktopAppleSpeechResult>;
72 }
73 >()("@t3tools/desktop/speech/AppleSpeech") {}
74
75 const unavailable: DesktopAppleSpeechResult = { ok: false, code: "unavailable" };
76 const failed: DesktopAppleSpeechResult = { ok: false, code: "failed" };
77
78 export const layer = Layer.effect(
79 AppleSpeech,
80 Effect.gen(function* () {
81 const environment = yield* DesktopEnvironment.DesktopEnvironment;
82 const fileSystem = yield* FileSystem.FileSystem;
83 const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
84 const hostEnvironment = yield* HostProcess.Environment;
85 const kernelRelease = yield* HostProcess.KernelRelease;
86
87 let helper: string | undefined;
88 // macOS 26 is Darwin 25, the helper's minimum deployment target.
89 if (environment.platform === "darwin" && Number.parseInt(kernelRelease, 10) >= 25) {
90 const candidate = environment.isPackaged
91 ? environment.path.join(environment.resourcesPath, "apple-speech", "t3-apple-speech")
92 : environment.path.join(
93 environment.rootDir,
94 "native",
95 "apple-speech",
96 "build",
97 environment.processArch,
98 "t3-apple-speech",
99 );
100 if (yield* fileSystem.exists(candidate).pipe(Effect.orElseSucceed(() => false)))
101 helper = candidate;
102 }
103
104 const run = (args: readonly string[], timeout: Duration.Input) =>
105 Effect.scoped(
106 Effect.gen(function* () {
107 if (helper === undefined) return unavailable;
108 const handle = yield* spawner.spawn(
109 ChildProcess.make(helper, args, { stdin: "ignore", env: hostEnvironment }),
110 );
111 const [stdout, stderr, exitCode] = yield* Effect.all(
112 [
113 handle.stdout.pipe(Stream.decodeText(), Stream.mkString),
114 handle.stderr.pipe(Stream.decodeText(), Stream.mkString),
115 handle.exitCode,
116 ],
117 { concurrency: "unbounded" },
118 );
119 switch (Number(exitCode)) {
120 case 0:
121 return { ok: true, value: (yield* decodeOutput(stdout)).value } as const;
122 case 2:
123 return { ok: false, code: "unsupported-locale" } as const;
124 case 3:
125 return unavailable;
126 default:
127 yield* Effect.logWarning("Apple Speech helper failed", { exitCode, stderr });
128 return failed;
129 }
130 }),
131 ).pipe(
132 // Leaving the scope kills a helper that outlives its timeout.
133 Effect.timeoutOrElse({ duration: timeout, orElse: () => Effect.succeed(failed) }),
134 Effect.catch((cause) =>
135 Effect.logWarning("Apple Speech helper did not run", { cause }).pipe(Effect.as(failed)),
136 ),
137 );
138
139 // Only a definite answer is kept; a slow or failed probe is asked again next time.
140 let locales: readonly string[] | undefined;
141 const supportedLocales = Effect.suspend(() =>
142 locales === undefined
143 ? run(["available"], Duration.seconds(10)).pipe(
144 Effect.map((result) => {
145 if (result.ok) locales = result.value.split(",").filter(Boolean);
146 else if (result.code === "unavailable") locales = [];
147 return locales ?? [];
148 }),
149 )
150 : Effect.succeed(locales),
151 );
152
153 const pending = new Map<string, Effect.Effect<DesktopAppleSpeechResult>>();
154 const preparePermit = yield* Semaphore.make(1);
155 const getPendingPrepare = Effect.fn("AppleSpeech.getPendingPrepare")(function* (
156 locale: string,
157 ) {
158 const existing = pending.get(locale);
159 if (existing) return existing;
160 const preparation = yield* Effect.cached(
161 run(["prepare", locale], PREPARE_TIMEOUT).pipe(
162 Effect.ensuring(Effect.sync(() => pending.delete(locale))),
163 ),
164 );
165 pending.set(locale, preparation);
166 return preparation;
167 }, preparePermit.withPermits(1));
168
169 return AppleSpeech.of({
170 supportedLocales,
171 prepare: Effect.fn("AppleSpeech.prepare")(function* (locale) {
172 return yield* yield* getPendingPrepare(locale);
173 }),
174 transcribe: Effect.fn("AppleSpeech.transcribe")(function* (locale, pcm) {
175 if (
176 pcm.byteLength === 0 ||
177 pcm.byteLength > MAX_PCM_BYTES ||
178 pcm.byteLength % Float32Array.BYTES_PER_ELEMENT !== 0
179 )
180 return failed;
181 return yield* Effect.scoped(
182 Effect.gen(function* () {
183 const file = yield* fileSystem.makeTempFileScoped({ suffix: ".wav" });
184 yield* fileSystem.writeFile(file, encodeSpeechWav(pcm));
185 return yield* run(["transcribe", locale, file], TRANSCRIBE_TIMEOUT);
186 }),
187 ).pipe(Effect.orElseSucceed(() => failed));
188 }),
189 });
190 }),
191 );