@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
development
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
ts 471 lines 14.9 KB
Raw
1 // Copyright 2026 Splitfire AB (Onde Inference). All rights reserved.
2 // SPDX-License-Identifier: MIT OR Apache-2.0
3
4 import OndeInferenceModule from "./OndeInferenceModule";
5 import type {
6 ChatMessage,
7 ChatRole,
8 EngineInfo,
9 GgufModelConfig,
10 InferenceResult,
11 SamplingConfig,
12 StreamChunk,
13 ToolCallInfo,
14 UqffModelConfig,
15 } from "./types";
16 import { OndeError } from "./types";
17
18 // Re-export all types
19 export type {
20 ChatMessage,
21 ChatRole,
22 EngineInfo,
23 EngineStatus,
24 GgufModelConfig,
25 InferenceResult,
26 SamplingConfig,
27 StreamChunk,
28 ToolCallInfo,
29 UqffModelConfig,
30 } from "./types";
31 export { OndeError } from "./types";
32
33 // ── JSON helpers ─────────────────────────────────────────────────────────────
34
35 // Convert SamplingConfig from camelCase TS to snake_case Rust
36 function serializeSampling(config: SamplingConfig): string {
37 return JSON.stringify({
38 temperature: config.temperature,
39 top_p: config.topP,
40 top_k: config.topK,
41 min_p: config.minP,
42 max_tokens: config.maxTokens,
43 frequency_penalty: config.frequencyPenalty,
44 presence_penalty: config.presencePenalty,
45 });
46 }
47
48 // Convert GgufModelConfig from camelCase TS to snake_case Rust
49 function serializeModelConfig(config: GgufModelConfig): string {
50 return JSON.stringify({
51 model_id: config.modelId,
52 files: config.files,
53 tok_model_id: config.tokModelId,
54 display_name: config.displayName,
55 approx_memory: config.approxMemory,
56 chat_template: config.chatTemplate,
57 });
58 }
59
60 // Convert UqffModelConfig from camelCase TS to snake_case Rust
61 function serializeUqffModelConfig(config: UqffModelConfig): string {
62 return JSON.stringify({
63 model_id: config.modelId,
64 files: config.files,
65 display_name: config.displayName,
66 approx_memory: config.approxMemory,
67 chat_template: config.chatTemplate,
68 });
69 }
70
71 // Parse a JSON result string, throwing OndeError if it contains an error field
72 function parseResult<T>(json: string): T {
73 const parsed = JSON.parse(json);
74 if (parsed.error) {
75 throw new OndeError(parsed.error);
76 }
77 return parsed as T;
78 }
79
80 // Parse snake_case JSON from Rust into camelCase TS types
81 function parseInferenceResult(json: string): InferenceResult {
82 const raw = parseResult<{
83 text: string;
84 duration_secs: number;
85 duration_display: string;
86 finish_reason?: string;
87 tool_calls?: Array<{
88 id: string;
89 function_name: string;
90 arguments: string;
91 }>;
92 }>(json);
93 return {
94 text: raw.text,
95 durationSecs: raw.duration_secs,
96 durationDisplay: raw.duration_display,
97 finishReason: raw.finish_reason,
98 toolCalls: (raw.tool_calls ?? []).map((tc) => ({
99 id: tc.id,
100 functionName: tc.function_name,
101 arguments: tc.arguments,
102 })),
103 };
104 }
105
106 function parseEngineInfo(json: string): EngineInfo {
107 const raw = parseResult<{
108 status: string;
109 model_name?: string;
110 approx_memory?: string;
111 history_length: number;
112 }>(json);
113 return {
114 status: raw.status as EngineInfo["status"],
115 modelName: raw.model_name,
116 approxMemory: raw.approx_memory,
117 historyLength: raw.history_length,
118 };
119 }
120
121 function parseModelConfig(json: string): GgufModelConfig {
122 const raw = JSON.parse(json);
123 return {
124 modelId: raw.model_id,
125 files: raw.files,
126 tokModelId: raw.tok_model_id,
127 displayName: raw.display_name,
128 approxMemory: raw.approx_memory,
129 chatTemplate: raw.chat_template,
130 };
131 }
132
133 function parseUqffModelConfig(json: string): UqffModelConfig {
134 const raw = parseResult<{
135 model_id: string;
136 files: string[];
137 display_name: string;
138 approx_memory: string;
139 chat_template?: string;
140 }>(json);
141 return {
142 modelId: raw.model_id,
143 files: raw.files,
144 displayName: raw.display_name,
145 approxMemory: raw.approx_memory,
146 chatTemplate: raw.chat_template,
147 };
148 }
149
150 function parseSamplingConfig(json: string): SamplingConfig {
151 const raw = JSON.parse(json);
152 return {
153 temperature: raw.temperature,
154 topP: raw.top_p,
155 topK: raw.top_k,
156 minP: raw.min_p,
157 maxTokens: raw.max_tokens,
158 frequencyPenalty: raw.frequency_penalty,
159 presencePenalty: raw.presence_penalty,
160 };
161 }
162
163 // ── OndeChatEngine ───────────────────────────────────────────────────────────
164
165 /**
166 * On-device LLM chat inference engine for React Native.
167 *
168 * The engine is backed by a native Rust inference engine (mistral.rs)
169 * with Metal acceleration on iOS and CPU inference on Android.
170 *
171 * @example
172 * ```typescript
173 * import { OndeChatEngine } from "@ondeinference/react-native";
174 *
175 * // Load the platform-appropriate default model
176 * const elapsed = await OndeChatEngine.loadDefaultModel("You are helpful.");
177 * console.log(`Model loaded in ${elapsed}s`);
178 *
179 * // Multi-turn chat
180 * const result = await OndeChatEngine.sendMessage("Hello!");
181 * console.log(result.text);
182 *
183 * // Cleanup
184 * await OndeChatEngine.unloadModel();
185 * ```
186 */
187 export const OndeChatEngine = {
188 // ── Model lifecycle ──────────────────────────────────────────────────
189
190 /**
191 * Load the platform-appropriate default model.
192 * - iOS → Qwen 2.5 1.5B (~941 MB, Metal)
193 * - Android → Qwen 2.5 1.5B (~941 MB, CPU)
194 * @returns Wall-clock loading time in seconds.
195 */
196 async loadDefaultModel(
197 systemPrompt?: string,
198 sampling?: SamplingConfig,
199 ): Promise<number> {
200 const json = await OndeInferenceModule.loadDefaultModel(
201 systemPrompt ?? null,
202 sampling ? serializeSampling(sampling) : null,
203 );
204 const result = parseResult<{ elapsed_secs: number }>(json);
205 return result.elapsed_secs;
206 },
207
208 /**
209 * Load the model assigned to this app via the Onde dashboard.
210 *
211 * Register your app at https://ondeinference.com to get an app ID
212 * and secret. The dashboard lets you assign a model — the SDK
213 * fetches it automatically. If no model is assigned, the platform
214 * default is loaded instead.
215 *
216 * @param appId - Your Onde app ID from ondeinference.com
217 * @param appSecret - Your Onde app secret from ondeinference.com
218 * @returns Wall-clock loading time in seconds.
219 */
220 async loadAssignedModel(
221 appId: string,
222 appSecret: string,
223 systemPrompt?: string,
224 sampling?: SamplingConfig,
225 ): Promise<number> {
226 const json = await OndeInferenceModule.loadAssignedModel(
227 appId,
228 appSecret,
229 systemPrompt ?? null,
230 sampling ? serializeSampling(sampling) : null,
231 );
232 const result = parseResult<{ elapsed_secs: number }>(json);
233 return result.elapsed_secs;
234 },
235
236 /**
237 * Load a specific GGUF model.
238 * @returns Wall-clock loading time in seconds.
239 */
240 async loadModel(
241 config: GgufModelConfig,
242 systemPrompt?: string,
243 sampling?: SamplingConfig,
244 ): Promise<number> {
245 const json = await OndeInferenceModule.loadModel(
246 serializeModelConfig(config),
247 systemPrompt ?? null,
248 sampling ? serializeSampling(sampling) : null,
249 );
250 const result = parseResult<{ elapsed_secs: number }>(json);
251 return result.elapsed_secs;
252 },
253
254 /**
255 * Load a specific UQFF model.
256 *
257 * `config.modelId` is the base repo/local directory used for tokenizer and
258 * config resolution. `config.files` accepts the first shard, explicit
259 * shards, or mistral.rs shorthands such as `"q4k"` and `"4"`.
260 *
261 * @returns Wall-clock loading time in seconds.
262 */
263 async loadUqffModel(
264 config: UqffModelConfig,
265 systemPrompt?: string,
266 sampling?: SamplingConfig,
267 ): Promise<number> {
268 const json = await OndeInferenceModule.loadUqffModel(
269 serializeUqffModelConfig(config),
270 systemPrompt ?? null,
271 sampling ? serializeSampling(sampling) : null,
272 );
273 const result = parseResult<{ elapsed_secs: number }>(json);
274 return result.elapsed_secs;
275 },
276
277 /** Unload the current model, freeing all memory. */
278 async unloadModel(): Promise<string | null> {
279 const json = await OndeInferenceModule.unloadModel();
280 const result = parseResult<{ model_name: string | null }>(json);
281 return result.model_name;
282 },
283
284 /** Check whether a model is currently loaded. */
285 isLoaded(): boolean {
286 return OndeInferenceModule.isLoaded();
287 },
288
289 /** Get a snapshot of the engine's current state. */
290 async info(): Promise<EngineInfo> {
291 const json = await OndeInferenceModule.info();
292 return parseEngineInfo(json);
293 },
294
295 // ── System prompt ────────────────────────────────────────────────────
296
297 /** Set or replace the system prompt. */
298 setSystemPrompt(prompt: string): void {
299 OndeInferenceModule.setSystemPrompt(prompt);
300 },
301
302 /** Clear the system prompt. */
303 clearSystemPrompt(): void {
304 OndeInferenceModule.clearSystemPrompt();
305 },
306
307 // ── Sampling ─────────────────────────────────────────────────────────
308
309 /** Replace the sampling configuration. */
310 setSampling(sampling: SamplingConfig): void {
311 OndeInferenceModule.setSampling(serializeSampling(sampling));
312 },
313
314 // ── History ──────────────────────────────────────────────────────────
315
316 /** Get the full conversation history. */
317 async history(): Promise<ChatMessage[]> {
318 const json = await OndeInferenceModule.history();
319 return JSON.parse(json) as ChatMessage[];
320 },
321
322 /** Clear the conversation history. Returns the number of removed turns. */
323 clearHistory(): number {
324 return OndeInferenceModule.clearHistory();
325 },
326
327 /** Append a message to history without running inference. */
328 pushHistory(message: ChatMessage): void {
329 OndeInferenceModule.pushHistory(JSON.stringify(message));
330 },
331
332 // ── Inference ────────────────────────────────────────────────────────
333
334 /**
335 * Send a user message and receive a complete assistant reply.
336 * The user message and assistant reply are automatically appended
337 * to the conversation history.
338 */
339 async sendMessage(message: string): Promise<InferenceResult> {
340 const json = await OndeInferenceModule.sendMessage(message);
341 return parseInferenceResult(json);
342 },
343
344 /**
345 * Run inference on an explicit list of messages WITHOUT modifying
346 * the engine's internal history. Useful for one-shot prompts.
347 */
348 async generate(
349 messages: ChatMessage[],
350 sampling?: SamplingConfig,
351 ): Promise<InferenceResult> {
352 const json = await OndeInferenceModule.generate(
353 JSON.stringify(messages),
354 sampling ? serializeSampling(sampling) : null,
355 );
356 return parseInferenceResult(json);
357 },
358 };
359
360 // ── Free functions ─────────────────────────────────────────────────────────
361
362 /** Return the platform-appropriate default GGUF model config. */
363 export function defaultModelConfig(): GgufModelConfig {
364 return parseModelConfig(OndeInferenceModule.defaultModelConfig());
365 }
366
367 /** Create a generic UQFF model config. */
368 export function uqffModelConfig(
369 modelId: string,
370 files: string[],
371 displayName: string,
372 approxMemory: string,
373 chatTemplate?: string,
374 ): UqffModelConfig {
375 return parseUqffModelConfig(
376 OndeInferenceModule.uqffModelConfig(
377 modelId,
378 JSON.stringify(files),
379 displayName,
380 approxMemory,
381 chatTemplate ?? null,
382 ),
383 );
384 }
385
386 /** Return the Qwen 2.5 1.5B config (~941 MB). */
387 export function qwen251_5bConfig(): GgufModelConfig {
388 return parseModelConfig(OndeInferenceModule.qwen251_5bConfig());
389 }
390
391 /** Return the Qwen 2.5 3B config (~1.93 GB). */
392 export function qwen253bConfig(): GgufModelConfig {
393 return parseModelConfig(OndeInferenceModule.qwen253bConfig());
394 }
395
396 /** Return the Qwen 3 0.6B config (~0.5 GB). */
397 export function qwen3_0_6bConfig(): GgufModelConfig {
398 return parseModelConfig(OndeInferenceModule.qwen3_0_6bConfig());
399 }
400
401 /** Return the Qwen 3 1.7B config (~1.3 GB). */
402 export function qwen3_1_7bConfig(): GgufModelConfig {
403 return parseModelConfig(OndeInferenceModule.qwen3_1_7bConfig());
404 }
405
406 /** Return the Qwen 3 4B config (~2.5 GB). */
407 export function qwen3_4bConfig(): GgufModelConfig {
408 return parseModelConfig(OndeInferenceModule.qwen3_4bConfig());
409 }
410
411 /** Return the Qwen 3 8B config (~5 GB). */
412 export function qwen3_8bConfig(): GgufModelConfig {
413 return parseModelConfig(OndeInferenceModule.qwen3_8bConfig());
414 }
415
416 /** Return the Qwen 3 14B config (~8.4 GB). */
417 export function qwen3_14bConfig(): GgufModelConfig {
418 return parseModelConfig(OndeInferenceModule.qwen3_14bConfig());
419 }
420
421 /** Return the Qwen 3 32B config (~19.8 GB). */
422 export function qwen3_32bConfig(): GgufModelConfig {
423 return parseModelConfig(OndeInferenceModule.qwen3_32bConfig());
424 }
425
426 /** Return the non-thinking Qwen 3 4B Instruct 2507 config (~2.5 GB). */
427 export function qwen3_4bInstruct2507Config(): GgufModelConfig {
428 return parseModelConfig(OndeInferenceModule.qwen3_4bInstruct2507Config());
429 }
430
431 /** Return the Qwen 3 4B Thinking 2507 config (~2.5 GB). */
432 export function qwen3_4bThinking2507Config(): GgufModelConfig {
433 return parseModelConfig(OndeInferenceModule.qwen3_4bThinking2507Config());
434 }
435
436 /** Return the Qwen 3 30B-A3B Instruct 2507 MoE config (~18.6 GB). */
437 export function qwen3_30bA3bInstruct2507Config(): GgufModelConfig {
438 return parseModelConfig(OndeInferenceModule.qwen3_30bA3bInstruct2507Config());
439 }
440
441 /** Return default sampling parameters (temp=0.7, top_p=0.95, max_tokens=512). */
442 export function defaultSamplingConfig(): SamplingConfig {
443 return parseSamplingConfig(OndeInferenceModule.defaultSamplingConfig());
444 }
445
446 /** Return deterministic (greedy) sampling parameters (temp=0.0). */
447 export function deterministicSamplingConfig(): SamplingConfig {
448 return parseSamplingConfig(OndeInferenceModule.deterministicSamplingConfig());
449 }
450
451 /** Return conservative mobile sampling parameters (max_tokens=128). */
452 export function mobileSamplingConfig(): SamplingConfig {
453 return parseSamplingConfig(OndeInferenceModule.mobileSamplingConfig());
454 }
455
456 // ── Message constructors ───────────────────────────────────────────────────
457
458 /** Create a system message. */
459 export function systemMessage(content: string): ChatMessage {
460 return { role: "system", content };
461 }
462
463 /** Create a user message. */
464 export function userMessage(content: string): ChatMessage {
465 return { role: "user", content };
466 }
467
468 /** Create an assistant message. */
469 export function assistantMessage(content: string): ChatMessage {
470 return { role: "assistant", content };
471 }