The best way to control Claude Code, Codex and more from any computer or phone

main
ts 333 lines 10.3 KB
Raw
1 import { encodeSpeechPcmRequest } from "@t3tools/shared/speech";
2 import * as Effect from "effect/Effect";
3 import * as Redacted from "effect/Redacted";
4 import * as Schema from "effect/Schema";
5 import { CloudSpeechError, SPEECH_STREAM_PATH } from "@t3tools/contracts";
6 import type {
7 CloudSpeechModelId,
8 CloudSpeechProvider,
9 EnvironmentSpeechModelDownload,
10 SettingsSecretValue,
11 SpeechAcceleration,
12 SpeechModelUnloadTimeout,
13 SpeechTranscriptionOptions,
14 SpeechPostProcessingOptions,
15 } from "@t3tools/contracts";
16
17 import * as RemoteEnvironmentAuthorization from "../authorization/service.ts";
18 import type { PreparedConnection } from "../connection/model.ts";
19 import * as ManagedRelay from "../relay/managedRelay.ts";
20 import {
21 makeEnvironmentHttpApiGroupClient,
22 makeEnvironmentHttpApiUrlBuilder,
23 } from "../rpc/http.ts";
24 import { executeAuthenticatedEnvironmentHttpRequest } from "../state/environmentHttpAuth.ts";
25 import { VoiceTranscriptionError } from "./transcription.ts";
26
27 const INFERENCE_OR_DOWNLOAD_TIMEOUT_MS = 30 * 60_000;
28 const isCloudSpeechError = Schema.is(CloudSpeechError);
29 type EnvironmentApiClient<G extends "voice" | "auth"> = Effect.Success<
30 ReturnType<typeof makeEnvironmentHttpApiGroupClient<G>>
31 >;
32
33 const request = Effect.fn("clientRuntime.voiceInput.environmentRequest")(function* <
34 A,
35 E,
36 G extends "voice" | "auth",
37 >(input: {
38 readonly group: G;
39 readonly prepared: PreparedConnection;
40 readonly method: "GET" | "POST" | "DELETE";
41 readonly timeoutMs?: number;
42 readonly canSend?: () => boolean;
43 readonly path: (urls: ReturnType<typeof makeEnvironmentHttpApiUrlBuilder>[G]) => string;
44 readonly run: (input: {
45 readonly client: EnvironmentApiClient<G>;
46 readonly headers: { readonly authorization?: string; readonly dpop?: string };
47 }) => Effect.Effect<A, E>;
48 }) {
49 return yield* executeAuthenticatedEnvironmentHttpRequest({
50 group: input.group,
51 prepared: input.prepared,
52 signer: yield* Effect.serviceOption(ManagedRelay.ManagedRelayDpopSigner),
53 remoteAuthorization: yield* Effect.serviceOption(
54 RemoteEnvironmentAuthorization.RemoteEnvironmentAuthorization,
55 ),
56 method: input.method,
57 url: input.path,
58 timeoutMs: input.timeoutMs ?? 30_000,
59 // Audio and credentials never follow a redirect to another host.
60 requestInit: { redirect: "error" },
61 request: (args) =>
62 Effect.suspend(() => (input.canSend?.() === false ? Effect.interrupt : input.run(args))),
63 });
64 });
65
66 export const getEnvironmentSpeechStatus = (prepared: PreparedConnection) =>
67 request({
68 group: "voice",
69 prepared,
70 method: "GET",
71 path: (urls) => urls.status(),
72 run: ({ client, headers }) => client.status({ headers }),
73 });
74
75 export const getCloudSpeechKeys = (prepared: PreparedConnection) =>
76 request({
77 group: "voice",
78 prepared,
79 method: "GET",
80 path: (urls) => urls.cloudKeys(),
81 run: ({ client, headers }) => client.cloudKeys({ headers }),
82 });
83
84 export const getCloudSpeechModels = (prepared: PreparedConnection) =>
85 request({
86 group: "voice",
87 prepared,
88 method: "GET",
89 path: (urls) => urls.cloudModels(),
90 run: ({ client, headers }) => client.cloudModels({ headers }),
91 });
92
93 export const requireCloudSpeechKey = (
94 prepared: PreparedConnection,
95 provider: CloudSpeechProvider,
96 ) =>
97 getCloudSpeechKeys(prepared).pipe(
98 Effect.flatMap((keys) =>
99 keys[provider]
100 ? Effect.void
101 : Effect.fail(
102 new VoiceTranscriptionError({
103 code: "unavailable",
104 message: new CloudSpeechError({ provider, reason: "missing-key" }).message,
105 }),
106 ),
107 ),
108 );
109
110 export const setCloudSpeechKey = (
111 prepared: PreparedConnection,
112 provider: CloudSpeechProvider,
113 key: SettingsSecretValue | null,
114 ) =>
115 request({
116 group: "voice",
117 prepared,
118 method: "POST",
119 path: (urls) => urls.setCloudKey({ params: { provider } }),
120 run: ({ client, headers }) =>
121 client.setCloudKey({
122 headers,
123 params: { provider },
124 payload: { key: typeof key === "string" ? Redacted.make(key) : key },
125 }),
126 });
127
128 export const transcribeCloudPcm = (
129 prepared: PreparedConnection,
130 provider: CloudSpeechProvider,
131 pcm: Uint8Array,
132 options: SpeechTranscriptionOptions,
133 canSend: () => boolean,
134 model?: CloudSpeechModelId,
135 ) => {
136 const query = model === undefined ? {} : { model };
137 return request({
138 group: "voice",
139 prepared,
140 method: "POST",
141 canSend,
142 timeoutMs: 130_000,
143 path: (urls) => urls.transcribeCloud({ params: { provider }, query }),
144 run: ({ client, headers }) =>
145 client.transcribeCloud({
146 headers,
147 params: { provider },
148 query,
149 payload: encodeSpeechPcmRequest(pcm, options),
150 }),
151 }).pipe(
152 Effect.mapError((error) =>
153 error._tag === "RemoteEnvironmentAuthFetchError" && isCloudSpeechError(error.cause)
154 ? new VoiceTranscriptionError({
155 code: "transcription-failed",
156 message: error.cause.message,
157 })
158 : error,
159 ),
160 );
161 };
162
163 export const getEnvironmentSpeechStreamUrl = (prepared: PreparedConnection) => {
164 let endpoint = prepared.httpBaseUrl;
165 return request({
166 group: "auth",
167 prepared,
168 method: "POST",
169 path: (urls) => {
170 endpoint = urls.webSocketTicketFor({ params: { use: "media" } });
171 return endpoint;
172 },
173 run: ({ client, headers }) => client.webSocketTicketFor({ headers, params: { use: "media" } }),
174 }).pipe(
175 Effect.map(({ ticket }) => {
176 const url = new URL(SPEECH_STREAM_PATH, endpoint);
177 url.protocol = url.protocol === "https:" ? "wss:" : "ws:";
178 url.searchParams.set("wsTicket", Redacted.value(ticket));
179 return url.toString();
180 }),
181 );
182 };
183
184 export const getEnvironmentSpeechModels = (
185 prepared: PreparedConnection,
186 includeBenchmarks = false,
187 ) => {
188 const query = includeBenchmarks ? { benchmarks: "1" as const } : {};
189 return request({
190 group: "voice",
191 prepared,
192 method: "GET",
193 path: (urls) => urls.models({ query }),
194 run: ({ client, headers }) => client.models({ headers, query }),
195 });
196 };
197
198 export const downloadEnvironmentSpeechModel = (prepared: PreparedConnection, modelId: string) =>
199 request({
200 group: "voice",
201 prepared,
202 method: "POST",
203 path: (urls) => urls.downloadModel(),
204 timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
205 run: ({ client, headers }) => client.downloadModel({ headers, payload: { modelId } }),
206 });
207
208 export const selectEnvironmentSpeechModel = (prepared: PreparedConnection, modelId: string) =>
209 request({
210 group: "voice",
211 prepared,
212 method: "POST",
213 path: (urls) => urls.selectModel(),
214 run: ({ client, headers }) => client.selectModel({ headers, payload: { modelId } }),
215 });
216
217 export function getSpeechModelDownloadOutcome(
218 download: EnvironmentSpeechModelDownload,
219 modelId: string,
220 ) {
221 if (download.cancelled) return "incomplete";
222 const model = download.models.find(({ id }) => id === modelId);
223 if (model?.state !== "installed") return "incomplete";
224 return model.active ? "already-selected" : "needs-selection";
225 }
226
227 /** Downloads a model and makes it the environment's transcription model on completion. */
228 export const installEnvironmentSpeechModel = Effect.fn(
229 "clientRuntime.voiceInput.installEnvironmentSpeechModel",
230 )(function* (prepared: PreparedConnection, modelId: string) {
231 const downloaded = yield* downloadEnvironmentSpeechModel(prepared, modelId);
232 if (getSpeechModelDownloadOutcome(downloaded, modelId) !== "needs-selection") return downloaded;
233 return yield* selectEnvironmentSpeechModel(prepared, modelId);
234 });
235
236 export const cancelEnvironmentSpeechModelDownload = (
237 prepared: PreparedConnection,
238 modelId: string,
239 ) =>
240 request({
241 group: "voice",
242 prepared,
243 method: "POST",
244 path: (urls) => urls.cancelModelDownload(),
245 run: ({ client, headers }) => client.cancelModelDownload({ headers, payload: { modelId } }),
246 });
247
248 export const transcribeEnvironmentPcm = (
249 prepared: PreparedConnection,
250 pcm: Uint8Array,
251 options: SpeechTranscriptionOptions,
252 canSend: () => boolean,
253 ) =>
254 request({
255 group: "voice",
256 prepared,
257 method: "POST",
258 path: (urls) => urls.transcribe(),
259 canSend,
260 timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
261 run: ({ client, headers }) =>
262 client.transcribe({
263 headers,
264 payload: encodeSpeechPcmRequest(pcm, options),
265 }),
266 });
267
268 /** Sends the transcript and draft context to a provider on `prepared`; a separate opt-in. */
269 export const postProcessEnvironmentTranscript = (
270 prepared: PreparedConnection,
271 transcript: string,
272 draft: {
273 readonly text: string;
274 readonly selection: { readonly start: number; readonly end: number };
275 },
276 options: SpeechPostProcessingOptions,
277 canSend: () => boolean,
278 ) =>
279 request({
280 group: "voice",
281 prepared,
282 method: "POST",
283 path: (urls) => urls.postProcess(),
284 canSend,
285 timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
286 run: ({ client, headers }) =>
287 client.postProcess({ headers, payload: { transcript, draft, options } }),
288 });
289
290 export const updateEnvironmentSpeechAcceleration = (
291 prepared: PreparedConnection,
292 acceleration: SpeechAcceleration,
293 ) =>
294 request({
295 group: "voice",
296 prepared,
297 method: "POST",
298 path: (urls) => urls.updateAcceleration(),
299 run: ({ client, headers }) => client.updateAcceleration({ headers, payload: { acceleration } }),
300 });
301
302 export const updateEnvironmentSpeechModelUnloadTimeout = (
303 prepared: PreparedConnection,
304 timeout: SpeechModelUnloadTimeout,
305 ) =>
306 request({
307 group: "voice",
308 prepared,
309 method: "POST",
310 path: (urls) => urls.updateModelUnloadTimeout(),
311 run: ({ client, headers }) =>
312 client.updateModelUnloadTimeout({ headers, payload: { timeout } }),
313 });
314
315 export const linkEnvironmentSpeechModels = (prepared: PreparedConnection, path: string) =>
316 request({
317 group: "voice",
318 prepared,
319 method: "POST",
320 path: (urls) => urls.linkModels(),
321 // Hashing several multi-gigabyte files takes longer than an ordinary request.
322 timeoutMs: INFERENCE_OR_DOWNLOAD_TIMEOUT_MS,
323 run: ({ client, headers }) => client.linkModels({ headers, payload: { path } }),
324 });
325
326 export const removeEnvironmentSpeechModel = (prepared: PreparedConnection, modelId: string) =>
327 request({
328 group: "voice",
329 prepared,
330 method: "POST",
331 path: (urls) => urls.removeModel(),
332 run: ({ client, headers }) => client.removeModel({ headers, payload: { modelId } }),
333 });