@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
copilot/assess-inference-engine-complexity
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
ts 351 lines 11.3 KB
Raw
1 import OndeInferenceModule from "./OndeInferenceModule";
2 import type {
3 ChatMessage,
4 ChatRole,
5 EngineInfo,
6 GgufModelConfig,
7 InferenceResult,
8 SamplingConfig,
9 StreamChunk,
10 ToolCallInfo,
11 } from "./types";
12 import { OndeError } from "./types";
13
14 // Re-export all types
15 export type {
16 ChatMessage,
17 ChatRole,
18 EngineInfo,
19 EngineStatus,
20 GgufModelConfig,
21 InferenceResult,
22 SamplingConfig,
23 StreamChunk,
24 ToolCallInfo,
25 } from "./types";
26 export { OndeError } from "./types";
27
28 // ── JSON helpers ─────────────────────────────────────────────────────────────
29
30 // Convert SamplingConfig from camelCase TS to snake_case Rust
31 function serializeSampling(config: SamplingConfig): string {
32 return JSON.stringify({
33 temperature: config.temperature,
34 top_p: config.topP,
35 top_k: config.topK,
36 min_p: config.minP,
37 max_tokens: config.maxTokens,
38 frequency_penalty: config.frequencyPenalty,
39 presence_penalty: config.presencePenalty,
40 });
41 }
42
43 // Convert GgufModelConfig from camelCase TS to snake_case Rust
44 function serializeModelConfig(config: GgufModelConfig): string {
45 return JSON.stringify({
46 model_id: config.modelId,
47 files: config.files,
48 tok_model_id: config.tokModelId,
49 display_name: config.displayName,
50 approx_memory: config.approxMemory,
51 chat_template: config.chatTemplate,
52 });
53 }
54
55 // Parse a JSON result string, throwing OndeError if it contains an error field
56 function parseResult<T>(json: string): T {
57 const parsed = JSON.parse(json);
58 if (parsed.error) {
59 throw new OndeError(parsed.error);
60 }
61 return parsed as T;
62 }
63
64 // Parse snake_case JSON from Rust into camelCase TS types
65 function parseInferenceResult(json: string): InferenceResult {
66 const raw = parseResult<{
67 text: string;
68 duration_secs: number;
69 duration_display: string;
70 finish_reason?: string;
71 tool_calls?: Array<{
72 id: string;
73 function_name: string;
74 arguments: string;
75 }>;
76 }>(json);
77 return {
78 text: raw.text,
79 durationSecs: raw.duration_secs,
80 durationDisplay: raw.duration_display,
81 finishReason: raw.finish_reason,
82 toolCalls: (raw.tool_calls ?? []).map((tc) => ({
83 id: tc.id,
84 functionName: tc.function_name,
85 arguments: tc.arguments,
86 })),
87 };
88 }
89
90 function parseEngineInfo(json: string): EngineInfo {
91 const raw = parseResult<{
92 status: string;
93 model_name?: string;
94 approx_memory?: string;
95 history_length: number;
96 }>(json);
97 return {
98 status: raw.status as EngineInfo["status"],
99 modelName: raw.model_name,
100 approxMemory: raw.approx_memory,
101 historyLength: raw.history_length,
102 };
103 }
104
105 function parseModelConfig(json: string): GgufModelConfig {
106 const raw = JSON.parse(json);
107 return {
108 modelId: raw.model_id,
109 files: raw.files,
110 tokModelId: raw.tok_model_id,
111 displayName: raw.display_name,
112 approxMemory: raw.approx_memory,
113 chatTemplate: raw.chat_template,
114 };
115 }
116
117 function parseSamplingConfig(json: string): SamplingConfig {
118 const raw = JSON.parse(json);
119 return {
120 temperature: raw.temperature,
121 topP: raw.top_p,
122 topK: raw.top_k,
123 minP: raw.min_p,
124 maxTokens: raw.max_tokens,
125 frequencyPenalty: raw.frequency_penalty,
126 presencePenalty: raw.presence_penalty,
127 };
128 }
129
130 // ── OndeChatEngine ───────────────────────────────────────────────────────────
131
132 /**
133 * On-device LLM chat inference engine for React Native.
134 *
135 * The engine is backed by a native Rust inference engine (mistral.rs)
136 * with Metal acceleration on iOS and CPU inference on Android.
137 *
138 * @example
139 * ```typescript
140 * import { OndeChatEngine } from "@ondeinference/react-native";
141 *
142 * // Load the platform-appropriate default model
143 * const elapsed = await OndeChatEngine.loadDefaultModel("You are helpful.");
144 * console.log(`Model loaded in ${elapsed}s`);
145 *
146 * // Multi-turn chat
147 * const result = await OndeChatEngine.sendMessage("Hello!");
148 * console.log(result.text);
149 *
150 * // Cleanup
151 * await OndeChatEngine.unloadModel();
152 * ```
153 */
154 export const OndeChatEngine = {
155 // ── Model lifecycle ──────────────────────────────────────────────────
156
157 /**
158 * Load the platform-appropriate default model.
159 * - iOS → Qwen 2.5 1.5B (~941 MB, Metal)
160 * - Android → Qwen 2.5 1.5B (~941 MB, CPU)
161 * @returns Wall-clock loading time in seconds.
162 */
163 async loadDefaultModel(
164 systemPrompt?: string,
165 sampling?: SamplingConfig,
166 ): Promise<number> {
167 const json = await OndeInferenceModule.loadDefaultModel(
168 systemPrompt ?? null,
169 sampling ? serializeSampling(sampling) : null,
170 );
171 const result = parseResult<{ elapsed_secs: number }>(json);
172 return result.elapsed_secs;
173 },
174
175 /**
176 * Load the model assigned to this app via the Onde dashboard.
177 *
178 * Register your app at https://ondeinference.com to get an app ID
179 * and secret. The dashboard lets you assign a model — the SDK
180 * fetches it automatically. If no model is assigned, the platform
181 * default is loaded instead.
182 *
183 * @param appId - Your Onde app ID from ondeinference.com
184 * @param appSecret - Your Onde app secret from ondeinference.com
185 * @returns Wall-clock loading time in seconds.
186 */
187 async loadAssignedModel(
188 appId: string,
189 appSecret: string,
190 systemPrompt?: string,
191 sampling?: SamplingConfig,
192 ): Promise<number> {
193 const json = await OndeInferenceModule.loadAssignedModel(
194 appId,
195 appSecret,
196 systemPrompt ?? null,
197 sampling ? serializeSampling(sampling) : null,
198 );
199 const result = parseResult<{ elapsed_secs: number }>(json);
200 return result.elapsed_secs;
201 },
202
203 /**
204 * Load a specific GGUF model.
205 * @returns Wall-clock loading time in seconds.
206 */
207 async loadModel(
208 config: GgufModelConfig,
209 systemPrompt?: string,
210 sampling?: SamplingConfig,
211 ): Promise<number> {
212 const json = await OndeInferenceModule.loadModel(
213 serializeModelConfig(config),
214 systemPrompt ?? null,
215 sampling ? serializeSampling(sampling) : null,
216 );
217 const result = parseResult<{ elapsed_secs: number }>(json);
218 return result.elapsed_secs;
219 },
220
221 /** Unload the current model, freeing all memory. */
222 async unloadModel(): Promise<string | null> {
223 const json = await OndeInferenceModule.unloadModel();
224 const result = parseResult<{ model_name: string | null }>(json);
225 return result.model_name;
226 },
227
228 /** Check whether a model is currently loaded. */
229 isLoaded(): boolean {
230 return OndeInferenceModule.isLoaded();
231 },
232
233 /** Get a snapshot of the engine's current state. */
234 async info(): Promise<EngineInfo> {
235 const json = await OndeInferenceModule.info();
236 return parseEngineInfo(json);
237 },
238
239 // ── System prompt ────────────────────────────────────────────────────
240
241 /** Set or replace the system prompt. */
242 setSystemPrompt(prompt: string): void {
243 OndeInferenceModule.setSystemPrompt(prompt);
244 },
245
246 /** Clear the system prompt. */
247 clearSystemPrompt(): void {
248 OndeInferenceModule.clearSystemPrompt();
249 },
250
251 // ── Sampling ─────────────────────────────────────────────────────────
252
253 /** Replace the sampling configuration. */
254 setSampling(sampling: SamplingConfig): void {
255 OndeInferenceModule.setSampling(serializeSampling(sampling));
256 },
257
258 // ── History ──────────────────────────────────────────────────────────
259
260 /** Get the full conversation history. */
261 async history(): Promise<ChatMessage[]> {
262 const json = await OndeInferenceModule.history();
263 return JSON.parse(json) as ChatMessage[];
264 },
265
266 /** Clear the conversation history. Returns the number of removed turns. */
267 clearHistory(): number {
268 return OndeInferenceModule.clearHistory();
269 },
270
271 /** Append a message to history without running inference. */
272 pushHistory(message: ChatMessage): void {
273 OndeInferenceModule.pushHistory(JSON.stringify(message));
274 },
275
276 // ── Inference ────────────────────────────────────────────────────────
277
278 /**
279 * Send a user message and receive a complete assistant reply.
280 * The user message and assistant reply are automatically appended
281 * to the conversation history.
282 */
283 async sendMessage(message: string): Promise<InferenceResult> {
284 const json = await OndeInferenceModule.sendMessage(message);
285 return parseInferenceResult(json);
286 },
287
288 /**
289 * Run inference on an explicit list of messages WITHOUT modifying
290 * the engine's internal history. Useful for one-shot prompts.
291 */
292 async generate(
293 messages: ChatMessage[],
294 sampling?: SamplingConfig,
295 ): Promise<InferenceResult> {
296 const json = await OndeInferenceModule.generate(
297 JSON.stringify(messages),
298 sampling ? serializeSampling(sampling) : null,
299 );
300 return parseInferenceResult(json);
301 },
302 };
303
304 // ── Free functions ─────────────────────────────────────────────────────────
305
306 /** Return the platform-appropriate default GGUF model config. */
307 export function defaultModelConfig(): GgufModelConfig {
308 return parseModelConfig(OndeInferenceModule.defaultModelConfig());
309 }
310
311 /** Return the Qwen 2.5 1.5B config (~941 MB). */
312 export function qwen251_5bConfig(): GgufModelConfig {
313 return parseModelConfig(OndeInferenceModule.qwen251_5bConfig());
314 }
315
316 /** Return the Qwen 2.5 3B config (~1.93 GB). */
317 export function qwen253bConfig(): GgufModelConfig {
318 return parseModelConfig(OndeInferenceModule.qwen253bConfig());
319 }
320
321 /** Return default sampling parameters (temp=0.7, top_p=0.95, max_tokens=512). */
322 export function defaultSamplingConfig(): SamplingConfig {
323 return parseSamplingConfig(OndeInferenceModule.defaultSamplingConfig());
324 }
325
326 /** Return deterministic (greedy) sampling parameters (temp=0.0). */
327 export function deterministicSamplingConfig(): SamplingConfig {
328 return parseSamplingConfig(OndeInferenceModule.deterministicSamplingConfig());
329 }
330
331 /** Return conservative mobile sampling parameters (max_tokens=128). */
332 export function mobileSamplingConfig(): SamplingConfig {
333 return parseSamplingConfig(OndeInferenceModule.mobileSamplingConfig());
334 }
335
336 // ── Message constructors ───────────────────────────────────────────────────
337
338 /** Create a system message. */
339 export function systemMessage(content: string): ChatMessage {
340 return { role: "system", content };
341 }
342
343 /** Create a user message. */
344 export function userMessage(content: string): ChatMessage {
345 return { role: "user", content };
346 }
347
348 /** Create an assistant message. */
349 export function assistantMessage(content: string): ChatMessage {
350 return { role: "assistant", content };
351 }