The best way to control Claude Code, Codex and more from any computer or phone

main
ts 267 lines 9.31 KB
Raw
1 import { msg } from "gt-react-native";
2
3 import { m } from "../../i18n/messages";
4 import {
5 AuthOrchestrationOperateScope,
6 type CloudSpeechModelId,
7 type CloudSpeechProvider,
8 type EnvironmentId,
9 DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
10 type VoiceTranscriptionDestination,
11 } from "@t3tools/contracts";
12 import {
13 getEnvironmentSpeechModels,
14 getCloudSpeechKeys,
15 getCloudSpeechModels,
16 requireCloudSpeechKey,
17 transcribeCloudPcm,
18 getEnvironmentSpeechStatus,
19 transcribeEnvironmentPcm,
20 VoiceTranscriptionError,
21 } from "@t3tools/client-runtime/voice-input";
22 import { FetchHttpClient, type HttpClient } from "effect/http";
23 import type { PreparedConnection } from "@t3tools/client-runtime/connection";
24 import { effectiveSpeechLanguage } from "@t3tools/shared/speech";
25 import * as Effect from "effect/Effect";
26 import * as Option from "effect/Option";
27 import { Atom, AsyncResult, AtomRegistry } from "effect/reactivity";
28 import { requireOptionalNativeModule } from "expo";
29 import { fetch as expoFetch } from "expo/fetch";
30
31 import { appAtomRegistry } from "../../state/atom-registry";
32 import { environmentPresentations } from "../../state/presentation";
33 import { environmentSession, sessionHasScope } from "../../state/session";
34 import { connectionAtomRuntime } from "../../connection/runtime";
35 import { uuidv4 } from "../../lib/uuid";
36 import { createEnvironmentTranscriber } from "./environmentTranscriber";
37 import { readVoiceGrant, writeVoiceGrant, type VoiceGrant } from "./voiceGrantStorage";
38
39 const storedGrant = Atom.make(Effect.promise(readVoiceGrant)).pipe(Atom.keepAlive);
40 const choice = Atom.make<VoiceGrant | null | undefined>(undefined).pipe(Atom.keepAlive);
41 export const voiceGrantAtom = Atom.make((get) => {
42 const chosen = get(choice);
43 if (chosen !== undefined) return chosen;
44 return AsyncResult.getOrElse(get(storedGrant), () => null);
45 });
46
47 let choiceRevision = 0;
48
49 type CloudDestination = Extract<VoiceTranscriptionDestination, { kind: "cloud" }>;
50 /** OpenRouter has no default model, so a cloud choice without one is no choice at all. */
51 function cloudVoiceDestination(
52 environmentId: EnvironmentId,
53 provider: CloudSpeechProvider,
54 model?: CloudSpeechModelId,
55 ): CloudDestination | null {
56 if (provider === "openrouter")
57 return model ? { kind: "cloud", environmentId, provider, model } : null;
58 return { kind: "cloud", environmentId, provider };
59 }
60
61 export function chooseVoiceDestination(
62 environmentId: EnvironmentId | null,
63 provider?: CloudSpeechProvider,
64 model?: CloudSpeechModelId,
65 ) {
66 choiceRevision++;
67 // A failed change must not leave the old destination armed in this session.
68 appAtomRegistry.set(choice, null);
69 const destination: VoiceTranscriptionDestination | null =
70 environmentId === null
71 ? null
72 : provider
73 ? cloudVoiceDestination(environmentId, provider, model)
74 : { kind: "environment", environmentId };
75 const grant: VoiceGrant | null = destination && { id: uuidv4(), destination };
76 writeVoiceGrant(grant);
77 appAtomRegistry.set(choice, grant);
78 }
79
80 export function pendingCloudVoiceChoice(
81 id: EnvironmentId,
82 provider: CloudSpeechProvider,
83 model?: CloudSpeechModelId,
84 ) {
85 const revision = ++choiceRevision;
86 return () => {
87 if (revision === choiceRevision) chooseVoiceDestination(id, provider, model);
88 };
89 }
90
91 const native = requireOptionalNativeModule<{
92 decodeRecording?: (uri: string) => Promise<Uint8Array>;
93 }>("T3VoiceTranscription");
94 export const hasEnvironmentVoiceCapture = typeof native?.decodeRecording === "function";
95
96 const voiceConnections = (capability: "voiceTranscription" | "cloudVoiceTranscription") =>
97 Atom.family((id: EnvironmentId) =>
98 Atom.make((get) => {
99 const environment = get(environmentPresentations.presentationAtom(id));
100 if (
101 !hasEnvironmentVoiceCapture ||
102 environment?.connection.phase !== "connected" ||
103 !environment.serverConfig?.environment.capabilities[capability] ||
104 !sessionHasScope(
105 get(environmentSession.sessionStateAtom(id)),
106 AuthOrchestrationOperateScope,
107 )
108 )
109 return null;
110 return Option.getOrNull(get(environmentSession.preparedConnectionValueAtom(id)));
111 }),
112 );
113 export const voiceConnectionAtom = voiceConnections("voiceTranscription");
114 export const cloudVoiceConnectionAtom = voiceConnections("cloudVoiceTranscription");
115
116 // React Native's XMLHttpRequest fetch follows redirects. Expo's native transport honors redirect: "error".
117 export function runSpeech<A, E>(
118 effect: Effect.Effect<A, E, HttpClient.HttpClient>,
119 signal?: AbortSignal,
120 canSend: () => boolean = () => true,
121 ) {
122 return Effect.runPromise(
123 AtomRegistry.getResult(appAtomRegistry, connectionAtomRuntime).pipe(
124 Effect.flatMap((context) =>
125 effect.pipe(
126 Effect.provideService(FetchHttpClient.Fetch, async (url, options) => {
127 if (signal?.aborted || !canSend())
128 throw new VoiceTranscriptionError({
129 code: "cancelled",
130 message: m(msg("Your transcription choice changed.")),
131 });
132 return expoFetch(url, { ...options, redirect: "error" });
133 }),
134 Effect.provide(context),
135 ),
136 ),
137 ),
138 { signal },
139 );
140 }
141 function environmentTranscriber(
142 prepared: PreparedConnection,
143 canSend: () => boolean,
144 cloud?: CloudDestination,
145 ) {
146 const provider = cloud?.provider;
147 const model = cloud?.provider === "openrouter" ? cloud.model : undefined;
148 return createEnvironmentTranscriber({
149 canSend,
150 prepare: async (signal) => {
151 if (provider) {
152 await runSpeech(requireCloudSpeechKey(prepared, provider), signal, canSend);
153 return "auto";
154 }
155 const status = await runSpeech(getEnvironmentSpeechStatus(prepared), signal, canSend);
156 if (!status.supported)
157 throw new VoiceTranscriptionError({ code: "unavailable", message: status.reason });
158 if (status.state === "missing-model")
159 throw new VoiceTranscriptionError({
160 code: "unavailable",
161 message: m(
162 msg(
163 "Install a transcription model for this environment in Settings → Voice, then try again.",
164 ),
165 ),
166 });
167 return status.languages
168 ? effectiveSpeechLanguage(
169 {
170 languages: status.languages,
171 supportsLanguageDetection: status.supportsLanguageDetection ?? false,
172 },
173 "auto",
174 )
175 : "auto";
176 },
177 decode: (uri) => {
178 if (!native?.decodeRecording)
179 throw new VoiceTranscriptionError({
180 code: "unavailable",
181 message: m(msg("Update T3 Code to transcribe on an environment.")),
182 });
183 return native.decodeRecording(uri);
184 },
185 upload: async (pcm, signal) =>
186 (
187 await runSpeech(
188 provider
189 ? transcribeCloudPcm(
190 prepared,
191 provider,
192 pcm,
193 DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
194 canSend,
195 model,
196 )
197 : transcribeEnvironmentPcm(
198 prepared,
199 pcm,
200 DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS,
201 canSend,
202 ),
203 signal,
204 canSend,
205 )
206 ).text,
207 });
208 }
209
210 const speechAtom = <A, E>(
211 id: EnvironmentId,
212 read: (prepared: PreparedConnection) => Effect.Effect<A, E, HttpClient.HttpClient>,
213 connectionAtom = voiceConnectionAtom,
214 ) =>
215 Atom.make((get) => {
216 const prepared = get(connectionAtom(id));
217 return prepared
218 ? Effect.tryPromise({
219 try: (signal) => runSpeech(read(prepared), signal),
220 catch: (cause) =>
221 new VoiceTranscriptionError({
222 code: "unavailable",
223 message: cause instanceof Error ? cause.message : String(cause),
224 }),
225 })
226 : Effect.succeed(null);
227 });
228
229 export const voiceSpeechStatusAtom = Atom.family((id: EnvironmentId) =>
230 speechAtom(id, getEnvironmentSpeechStatus),
231 );
232 export const cloudVoiceKeysAtom = Atom.family((id: EnvironmentId) =>
233 speechAtom(id, getCloudSpeechKeys, cloudVoiceConnectionAtom),
234 );
235 export const cloudVoiceModelsAtom = Atom.family((id: EnvironmentId | null) =>
236 id === null
237 ? Atom.make(Effect.succeed(null))
238 : speechAtom(id, getCloudSpeechModels, cloudVoiceConnectionAtom),
239 );
240 export const voiceSpeechModelsAtom = Atom.family((id: EnvironmentId) =>
241 speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared)),
242 );
243 // Fetched once for the model sheet; download progress polls the lighter list above.
244 export const voiceSpeechBenchmarksAtom = Atom.family((id: EnvironmentId) =>
245 speechAtom(id, (prepared) => getEnvironmentSpeechModels(prepared, true)),
246 );
247
248 // null selects on-device; a grant with no usable transcriber must never fall back.
249 export const voiceRouteAtom = Atom.make((get) => {
250 const grant = get(voiceGrantAtom);
251 if (!grant) return null;
252 const cloud = grant.destination.kind === "cloud" ? grant.destination : undefined;
253 const connectionAtom = (cloud ? cloudVoiceConnectionAtom : voiceConnectionAtom)(
254 grant.destination.environmentId,
255 );
256 const connection = get(connectionAtom);
257 const registry = get.registry;
258 return {
259 transcriber:
260 connection &&
261 environmentTranscriber(
262 connection,
263 () => registry.get(voiceGrantAtom) === grant && registry.get(connectionAtom) === connection,
264 cloud,
265 ),
266 };
267 });