@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
feature/diffusion-vision
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
ts 399 lines 13 KB
Raw
1 // Copyright 2026 Splitfire AB (Onde Inference). All rights reserved.
2 // SPDX-License-Identifier: MIT OR Apache-2.0
3
4 import OndeInferenceModule from "./OndeInferenceModule";
5 import type {
6 ChatMessage,
7 ChatRole,
8 EngineInfo,
9 GgufModelConfig,
10 InferenceResult,
11 SamplingConfig,
12 StreamChunk,
13 ToolCallInfo,
14 } from "./types";
15 import { OndeError } from "./types";
16
17 // Re-export all types
18 export type {
19 ChatMessage,
20 ChatRole,
21 EngineInfo,
22 EngineStatus,
23 GgufModelConfig,
24 InferenceResult,
25 SamplingConfig,
26 StreamChunk,
27 ToolCallInfo,
28 } from "./types";
29 export { OndeError } from "./types";
30
31 // ── JSON helpers ─────────────────────────────────────────────────────────────
32
33 // Convert SamplingConfig from camelCase TS to snake_case Rust
34 function serializeSampling(config: SamplingConfig): string {
35 return JSON.stringify({
36 temperature: config.temperature,
37 top_p: config.topP,
38 top_k: config.topK,
39 min_p: config.minP,
40 max_tokens: config.maxTokens,
41 frequency_penalty: config.frequencyPenalty,
42 presence_penalty: config.presencePenalty,
43 });
44 }
45
46 // Convert GgufModelConfig from camelCase TS to snake_case Rust
47 function serializeModelConfig(config: GgufModelConfig): string {
48 return JSON.stringify({
49 model_id: config.modelId,
50 files: config.files,
51 tok_model_id: config.tokModelId,
52 display_name: config.displayName,
53 approx_memory: config.approxMemory,
54 chat_template: config.chatTemplate,
55 });
56 }
57
58 // Parse a JSON result string, throwing OndeError if it contains an error field
59 function parseResult<T>(json: string): T {
60 const parsed = JSON.parse(json);
61 if (parsed.error) {
62 throw new OndeError(parsed.error);
63 }
64 return parsed as T;
65 }
66
67 // Parse snake_case JSON from Rust into camelCase TS types
68 function parseInferenceResult(json: string): InferenceResult {
69 const raw = parseResult<{
70 text: string;
71 duration_secs: number;
72 duration_display: string;
73 finish_reason?: string;
74 tool_calls?: Array<{
75 id: string;
76 function_name: string;
77 arguments: string;
78 }>;
79 }>(json);
80 return {
81 text: raw.text,
82 durationSecs: raw.duration_secs,
83 durationDisplay: raw.duration_display,
84 finishReason: raw.finish_reason,
85 toolCalls: (raw.tool_calls ?? []).map((tc) => ({
86 id: tc.id,
87 functionName: tc.function_name,
88 arguments: tc.arguments,
89 })),
90 };
91 }
92
93 function parseEngineInfo(json: string): EngineInfo {
94 const raw = parseResult<{
95 status: string;
96 model_name?: string;
97 approx_memory?: string;
98 history_length: number;
99 }>(json);
100 return {
101 status: raw.status as EngineInfo["status"],
102 modelName: raw.model_name,
103 approxMemory: raw.approx_memory,
104 historyLength: raw.history_length,
105 };
106 }
107
108 function parseModelConfig(json: string): GgufModelConfig {
109 const raw = JSON.parse(json);
110 return {
111 modelId: raw.model_id,
112 files: raw.files,
113 tokModelId: raw.tok_model_id,
114 displayName: raw.display_name,
115 approxMemory: raw.approx_memory,
116 chatTemplate: raw.chat_template,
117 };
118 }
119
120 function parseSamplingConfig(json: string): SamplingConfig {
121 const raw = JSON.parse(json);
122 return {
123 temperature: raw.temperature,
124 topP: raw.top_p,
125 topK: raw.top_k,
126 minP: raw.min_p,
127 maxTokens: raw.max_tokens,
128 frequencyPenalty: raw.frequency_penalty,
129 presencePenalty: raw.presence_penalty,
130 };
131 }
132
133 // ── OndeChatEngine ───────────────────────────────────────────────────────────
134
135 /**
136 * On-device LLM chat inference engine for React Native.
137 *
138 * The engine is backed by a native Rust inference engine (mistral.rs)
139 * with Metal acceleration on iOS and CPU inference on Android.
140 *
141 * @example
142 * ```typescript
143 * import { OndeChatEngine } from "@ondeinference/react-native";
144 *
145 * // Load the platform-appropriate default model
146 * const elapsed = await OndeChatEngine.loadDefaultModel("You are helpful.");
147 * console.log(`Model loaded in ${elapsed}s`);
148 *
149 * // Multi-turn chat
150 * const result = await OndeChatEngine.sendMessage("Hello!");
151 * console.log(result.text);
152 *
153 * // Cleanup
154 * await OndeChatEngine.unloadModel();
155 * ```
156 */
157 export const OndeChatEngine = {
158 // ── Model lifecycle ──────────────────────────────────────────────────
159
160 /**
161 * Load the platform-appropriate default model.
162 * - iOS → Qwen 2.5 1.5B (~941 MB, Metal)
163 * - Android → Qwen 2.5 1.5B (~941 MB, CPU)
164 * @returns Wall-clock loading time in seconds.
165 */
166 async loadDefaultModel(
167 systemPrompt?: string,
168 sampling?: SamplingConfig,
169 ): Promise<number> {
170 const json = await OndeInferenceModule.loadDefaultModel(
171 systemPrompt ?? null,
172 sampling ? serializeSampling(sampling) : null,
173 );
174 const result = parseResult<{ elapsed_secs: number }>(json);
175 return result.elapsed_secs;
176 },
177
178 /**
179 * Load the model assigned to this app via the Onde dashboard.
180 *
181 * Register your app at https://ondeinference.com to get an app ID
182 * and secret. The dashboard lets you assign a model — the SDK
183 * fetches it automatically. If no model is assigned, the platform
184 * default is loaded instead.
185 *
186 * @param appId - Your Onde app ID from ondeinference.com
187 * @param appSecret - Your Onde app secret from ondeinference.com
188 * @returns Wall-clock loading time in seconds.
189 */
190 async loadAssignedModel(
191 appId: string,
192 appSecret: string,
193 systemPrompt?: string,
194 sampling?: SamplingConfig,
195 ): Promise<number> {
196 const json = await OndeInferenceModule.loadAssignedModel(
197 appId,
198 appSecret,
199 systemPrompt ?? null,
200 sampling ? serializeSampling(sampling) : null,
201 );
202 const result = parseResult<{ elapsed_secs: number }>(json);
203 return result.elapsed_secs;
204 },
205
206 /**
207 * Load a specific GGUF model.
208 * @returns Wall-clock loading time in seconds.
209 */
210 async loadModel(
211 config: GgufModelConfig,
212 systemPrompt?: string,
213 sampling?: SamplingConfig,
214 ): Promise<number> {
215 const json = await OndeInferenceModule.loadModel(
216 serializeModelConfig(config),
217 systemPrompt ?? null,
218 sampling ? serializeSampling(sampling) : null,
219 );
220 const result = parseResult<{ elapsed_secs: number }>(json);
221 return result.elapsed_secs;
222 },
223
224 /** Unload the current model, freeing all memory. */
225 async unloadModel(): Promise<string | null> {
226 const json = await OndeInferenceModule.unloadModel();
227 const result = parseResult<{ model_name: string | null }>(json);
228 return result.model_name;
229 },
230
231 /** Check whether a model is currently loaded. */
232 isLoaded(): boolean {
233 return OndeInferenceModule.isLoaded();
234 },
235
236 /** Get a snapshot of the engine's current state. */
237 async info(): Promise<EngineInfo> {
238 const json = await OndeInferenceModule.info();
239 return parseEngineInfo(json);
240 },
241
242 // ── System prompt ────────────────────────────────────────────────────
243
244 /** Set or replace the system prompt. */
245 setSystemPrompt(prompt: string): void {
246 OndeInferenceModule.setSystemPrompt(prompt);
247 },
248
249 /** Clear the system prompt. */
250 clearSystemPrompt(): void {
251 OndeInferenceModule.clearSystemPrompt();
252 },
253
254 // ── Sampling ─────────────────────────────────────────────────────────
255
256 /** Replace the sampling configuration. */
257 setSampling(sampling: SamplingConfig): void {
258 OndeInferenceModule.setSampling(serializeSampling(sampling));
259 },
260
261 // ── History ──────────────────────────────────────────────────────────
262
263 /** Get the full conversation history. */
264 async history(): Promise<ChatMessage[]> {
265 const json = await OndeInferenceModule.history();
266 return JSON.parse(json) as ChatMessage[];
267 },
268
269 /** Clear the conversation history. Returns the number of removed turns. */
270 clearHistory(): number {
271 return OndeInferenceModule.clearHistory();
272 },
273
274 /** Append a message to history without running inference. */
275 pushHistory(message: ChatMessage): void {
276 OndeInferenceModule.pushHistory(JSON.stringify(message));
277 },
278
279 // ── Inference ────────────────────────────────────────────────────────
280
281 /**
282 * Send a user message and receive a complete assistant reply.
283 * The user message and assistant reply are automatically appended
284 * to the conversation history.
285 */
286 async sendMessage(message: string): Promise<InferenceResult> {
287 const json = await OndeInferenceModule.sendMessage(message);
288 return parseInferenceResult(json);
289 },
290
291 /**
292 * Run inference on an explicit list of messages WITHOUT modifying
293 * the engine's internal history. Useful for one-shot prompts.
294 */
295 async generate(
296 messages: ChatMessage[],
297 sampling?: SamplingConfig,
298 ): Promise<InferenceResult> {
299 const json = await OndeInferenceModule.generate(
300 JSON.stringify(messages),
301 sampling ? serializeSampling(sampling) : null,
302 );
303 return parseInferenceResult(json);
304 },
305 };
306
307 // ── Free functions ─────────────────────────────────────────────────────────
308
309 /** Return the platform-appropriate default GGUF model config. */
310 export function defaultModelConfig(): GgufModelConfig {
311 return parseModelConfig(OndeInferenceModule.defaultModelConfig());
312 }
313
314 /** Return the Qwen 2.5 1.5B config (~941 MB). */
315 export function qwen251_5bConfig(): GgufModelConfig {
316 return parseModelConfig(OndeInferenceModule.qwen251_5bConfig());
317 }
318
319 /** Return the Qwen 2.5 3B config (~1.93 GB). */
320 export function qwen253bConfig(): GgufModelConfig {
321 return parseModelConfig(OndeInferenceModule.qwen253bConfig());
322 }
323
324 /** Return the Qwen 3 0.6B config (~0.5 GB). */
325 export function qwen3_0_6bConfig(): GgufModelConfig {
326 return parseModelConfig(OndeInferenceModule.qwen3_0_6bConfig());
327 }
328
329 /** Return the Qwen 3 1.7B config (~1.3 GB). */
330 export function qwen3_1_7bConfig(): GgufModelConfig {
331 return parseModelConfig(OndeInferenceModule.qwen3_1_7bConfig());
332 }
333
334 /** Return the Qwen 3 4B config (~2.5 GB). */
335 export function qwen3_4bConfig(): GgufModelConfig {
336 return parseModelConfig(OndeInferenceModule.qwen3_4bConfig());
337 }
338
339 /** Return the Qwen 3 8B config (~5 GB). */
340 export function qwen3_8bConfig(): GgufModelConfig {
341 return parseModelConfig(OndeInferenceModule.qwen3_8bConfig());
342 }
343
344 /** Return the Qwen 3 14B config (~8.4 GB). */
345 export function qwen3_14bConfig(): GgufModelConfig {
346 return parseModelConfig(OndeInferenceModule.qwen3_14bConfig());
347 }
348
349 /** Return the Qwen 3 32B config (~19.8 GB). */
350 export function qwen3_32bConfig(): GgufModelConfig {
351 return parseModelConfig(OndeInferenceModule.qwen3_32bConfig());
352 }
353
354 /** Return the non-thinking Qwen 3 4B Instruct 2507 config (~2.5 GB). */
355 export function qwen3_4bInstruct2507Config(): GgufModelConfig {
356 return parseModelConfig(OndeInferenceModule.qwen3_4bInstruct2507Config());
357 }
358
359 /** Return the Qwen 3 4B Thinking 2507 config (~2.5 GB). */
360 export function qwen3_4bThinking2507Config(): GgufModelConfig {
361 return parseModelConfig(OndeInferenceModule.qwen3_4bThinking2507Config());
362 }
363
364 /** Return the Qwen 3 30B-A3B Instruct 2507 MoE config (~18.6 GB). */
365 export function qwen3_30bA3bInstruct2507Config(): GgufModelConfig {
366 return parseModelConfig(OndeInferenceModule.qwen3_30bA3bInstruct2507Config());
367 }
368
369 /** Return default sampling parameters (temp=0.7, top_p=0.95, max_tokens=512). */
370 export function defaultSamplingConfig(): SamplingConfig {
371 return parseSamplingConfig(OndeInferenceModule.defaultSamplingConfig());
372 }
373
374 /** Return deterministic (greedy) sampling parameters (temp=0.0). */
375 export function deterministicSamplingConfig(): SamplingConfig {
376 return parseSamplingConfig(OndeInferenceModule.deterministicSamplingConfig());
377 }
378
379 /** Return conservative mobile sampling parameters (max_tokens=128). */
380 export function mobileSamplingConfig(): SamplingConfig {
381 return parseSamplingConfig(OndeInferenceModule.mobileSamplingConfig());
382 }
383
384 // ── Message constructors ───────────────────────────────────────────────────
385
386 /** Create a system message. */
387 export function systemMessage(content: string): ChatMessage {
388 return { role: "system", content };
389 }
390
391 /** Create a user message. */
392 export function userMessage(content: string): ChatMessage {
393 return { role: "user", content };
394 }
395
396 /** Create an assistant message. */
397 export function assistantMessage(content: string): ChatMessage {
398 return { role: "assistant", content };
399 }