@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
main
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
dart 325 lines 12.4 KB
Raw
1 // Copyright 2026 Splitfire AB (Onde Inference). All rights reserved.
2 // SPDX-License-Identifier: MIT OR Apache-2.0
3
4 // ignore_for_file: prefer_const_constructors
5
6 import 'dart:async';
7 import 'dart:io' show Platform;
8
9 import 'package:flutter/services.dart';
10 import 'package:flutter_rust_bridge/flutter_rust_bridge_for_generated.dart'
11 show ExternalLibrary;
12
13 import 'frb_generated.dart/frb_generated.dart' as frb;
14 import 'frb_generated.dart/api.dart' as api;
15 import 'types.dart';
16
17 export 'frb_generated.dart/frb_generated.dart' show RustLib;
18 export 'frb_generated.dart/api.dart'
19 show
20 OndeChatEngine,
21 OndeError,
22 OndeError_NoModelLoaded,
23 OndeError_AlreadyLoaded,
24 OndeError_ModelBuild,
25 OndeError_Inference,
26 OndeError_Cancelled,
27 OndeError_Other,
28 configureCacheDir;
29
30 // ---------------------------------------------------------------------------
31 // OndeChatEngine convenience wrapper
32 // ---------------------------------------------------------------------------
33
34 /// High-level wrapper around the FRB-generated [api.OndeChatEngine] opaque type.
35 ///
36 /// Provides:
37 /// - [OndeChatEngineX.loadDefaultModel]: convenience method not present in generated code
38 /// - [OndeChatEngineX.clearHistoryCount]: returns `int` instead of `BigInt`
39 /// - [OndeInference]: static initialisation plus config and sampling helpers
40 ///
41 /// ## Lifecycle
42 ///
43 /// ```dart
44 /// // 1. Initialise once at app startup (idempotent).
45 /// await OndeInference.init();
46 ///
47 /// // 2. Create a native engine instance (sync, no Future).
48 /// final engine = OndeChatEngine();
49 ///
50 /// // 3a. Load your assigned model (recommended for production).
51 /// await engine.loadAssignedModel(
52 /// appId: 'your-app-id', // from ondeinference.com
53 /// appSecret: 'your-app-secret',
54 /// systemPrompt: 'You are a helpful assistant.',
55 /// );
56 ///
57 /// // 3b. Or load the platform default (for prototyping).
58 /// // await engine.loadDefaultModel(
59 /// // systemPrompt: 'You are a helpful assistant.',
60 /// // );
61 ///
62 /// // 4. Chat.
63 /// final result = await engine.sendMessage(message: 'Hello!');
64 /// print(result.text);
65 ///
66 /// // 5. Streaming.
67 /// await for (final chunk in engine.streamMessage(message: 'Tell me a story.')) {
68 /// stdout.write(chunk.delta);
69 /// if (chunk.done) break;
70 /// }
71 ///
72 /// // 6. Clean up.
73 /// await engine.unloadModel();
74 /// ```
75 ///
76 /// ## Thread safety
77 ///
78 /// [OndeChatEngine] is backed by an `Arc`-wrapped Rust object and is safe to
79 /// reference from multiple Dart isolates. Concurrent inference calls are
80 /// serialised internally.
81 extension OndeChatEngineX on api.OndeChatEngine {
82 // -------------------------------------------------------------------------
83 // loadDefaultModel: implemented in Dart, not in the generated bridge.
84 // -------------------------------------------------------------------------
85
86 /// Loads the platform-appropriate default model.
87 ///
88 /// On iOS, tvOS, and Android the Qwen 2.5 1.5B (Q4_K_M, ~941 MB) model is
89 /// used. On macOS, Windows, and Linux the Qwen 2.5 3B (Q4_K_M, ~1.93 GB)
90 /// model is used.
91 ///
92 /// Delegates to [loadGgufModel] with [OndeInference.defaultModelConfig].
93 ///
94 /// [systemPrompt] replaces the engine's system prompt before loading.
95 /// [sampling] overrides the default sampling parameters.
96 ///
97 /// Returns the wall-clock seconds taken to load the model.
98 ///
99 /// Throws [OndeError] if the model cannot be downloaded or loaded.
100 Future<double> loadDefaultModel({
101 String? systemPrompt,
102 SamplingConfig? sampling,
103 }) => loadGgufModel(
104 config: api.defaultModelConfig(),
105 systemPrompt: systemPrompt,
106 sampling: sampling,
107 );
108
109 // -------------------------------------------------------------------------
110 // clearHistoryCount: int-typed convenience over clearHistory() -> BigInt.
111 // -------------------------------------------------------------------------
112
113 /// Clears the conversation history and returns the count as a plain [int].
114 ///
115 /// The generated [clearHistory] returns `Future<BigInt>`; this helper
116 /// downcasts it so callers never need to handle [BigInt] directly.
117 Future<int> clearHistoryCount() async => (await clearHistory()).toInt();
118 }
119
120 // ---------------------------------------------------------------------------
121 // OndeInference static SDK helpers
122 // ---------------------------------------------------------------------------
123
124 /// Static helper namespace for Onde SDK initialisation and configuration.
125 ///
126 /// ```dart
127 /// // Initialise before creating any OndeChatEngine.
128 /// await OndeInference.init();
129 ///
130 /// // Obtain a model config.
131 /// final config = OndeInference.defaultModelConfig();
132 ///
133 /// // Obtain sampling parameters.
134 /// final sampling = OndeInference.deterministicSamplingConfig();
135 /// ```
136 abstract final class OndeInference {
137 // -------------------------------------------------------------------------
138 // Initialisation
139 // -------------------------------------------------------------------------
140
141 /// Initialises the Rust shared library.
142 ///
143 /// Must be called before creating any [OndeChatEngine]. Subsequent calls
144 /// are safe no-ops. The library is only initialised once per process.
145 ///
146 /// Call this in `main()` or in a Flutter `initState` override before any
147 /// user interaction that could trigger model loading.
148 ///
149 /// On macOS the Rust static library is force-loaded into the
150 /// `onde_inference.framework` CocoaPods dynamic framework, so we must
151 /// `dlopen` that framework by name.
152 ///
153 /// On iOS the pod is statically linked into the Runner executable
154 /// (no separate `.framework` file exists at runtime), so we use
155 /// `DynamicLibrary.process()` (`RTLD_DEFAULT`) which searches the
156 /// main executable and all loaded images.
157 ///
158 /// On Android, Linux, and Windows, the default FRB stem-based loader
159 /// finds the `.so` or `.dll` automatically. No override needed.
160 static Future<void> init() async {
161 ExternalLibrary? lib;
162 if (Platform.isMacOS) {
163 // With CocoaPods + use_frameworks!, the pod is a dynamic framework.
164 // DynamicLibrary.process() cannot resolve symbols inside it.
165 lib = ExternalLibrary.open('onde_inference.framework/onde_inference');
166 } else if (Platform.isIOS) {
167 // On iOS, CocoaPods statically links the pod into the Runner binary
168 // through -force_load, so the symbols live in the main executable.
169 lib = ExternalLibrary.process(iKnowHowToUseIt: true);
170 }
171 await frb.RustLib.init(externalLibrary: lib);
172 }
173
174 // -------------------------------------------------------------------------
175 // Sandbox / cache setup
176 // -------------------------------------------------------------------------
177
178 /// Method channel shared with the native iOS/macOS plugin.
179 static const _channel = MethodChannel('com.ondeinference.onde_inference');
180
181 /// Configures the HuggingFace model cache for sandboxed platforms.
182 ///
183 /// On **iOS and macOS** this first tries to resolve the App Group shared
184 /// container (`group.com.ondeinference.apps`) via a native method channel
185 /// so all Onde-powered apps share downloaded models. If the App Group is
186 /// unavailable (entitlement missing, group not registered), it falls back
187 /// to [fallbackDir] if provided.
188 ///
189 /// On **Android** there is no App Group equivalent. Pass [fallbackDir]
190 /// (e.g. from `getApplicationSupportDirectory().path`).
191 ///
192 /// On **desktop Linux / Windows** this is a no-op. The default
193 /// `~/.cache/huggingface` works without intervention.
194 ///
195 /// Call **once** at startup, after [init] and before any model load:
196 ///
197 /// ```dart
198 /// await OndeInference.init();
199 /// await OndeInference.setupCacheDir();
200 /// // or with explicit fallback:
201 /// final dir = await getApplicationSupportDirectory();
202 /// await OndeInference.setupCacheDir(fallbackDir: dir.path);
203 /// ```
204 static Future<void> setupCacheDir({String? fallbackDir}) async {
205 // 1. On Apple platforms, try the shared App Group container first.
206 if (Platform.isIOS || Platform.isMacOS) {
207 try {
208 final String? groupPath = await _channel.invokeMethod<String>(
209 'getAppGroupContainerPath',
210 );
211 if (groupPath != null && groupPath.isNotEmpty) {
212 api.configureCacheDir(appDataDir: groupPath);
213 return;
214 }
215 } catch (_) {
216 // Method channel not available (e.g. plugin not registered).
217 // Fall through to fallbackDir.
218 }
219 }
220
221 // 2. Otherwise use the caller-provided directory in the app sandbox.
222 if (fallbackDir != null && fallbackDir.isNotEmpty) {
223 api.configureCacheDir(appDataDir: fallbackDir);
224 }
225 }
226
227 // -------------------------------------------------------------------------
228 // Model config factories
229 // -------------------------------------------------------------------------
230
231 /// Platform-appropriate default model config.
232 ///
233 /// Selects Qwen 2.5 1.5B on iOS / tvOS / Android and Qwen 2.5 3B on
234 /// macOS / Windows / Linux. Delegates to the Rust `default_model_config`
235 /// free function so the platform check runs natively.
236 static GgufModelConfig defaultModelConfig() => api.defaultModelConfig();
237
238 /// Builds a configuration for a pre-quantised UQFF model.
239 ///
240 /// [modelId] identifies the base repository or local model directory used
241 /// for tokenizer and configuration resolution. For sharded models, pass the
242 /// first shard in [files]; mistral.rs discovers sibling shards.
243 static UqffModelConfig uqffModelConfig({
244 required String modelId,
245 required List<String> files,
246 required String displayName,
247 required String approxMemory,
248 String? chatTemplate,
249 }) => api.uqffModelConfig(
250 modelId: modelId,
251 files: files,
252 displayName: displayName,
253 approxMemory: approxMemory,
254 chatTemplate: chatTemplate,
255 );
256
257 /// Qwen 2.5 1.5B Instruct (GGUF Q4_K_M, ~941 MB).
258 ///
259 /// Suitable for iOS, tvOS, and Android where available memory is limited.
260 static GgufModelConfig qwen2515bConfig() => api.qwen2515BConfig();
261
262 /// Qwen 2.5 3B Instruct (GGUF Q4_K_M, ~1.93 GB).
263 ///
264 /// Suitable for macOS, Windows, and Linux desktop deployments.
265 static GgufModelConfig qwen253bConfig() => api.qwen253BConfig();
266
267 /// Qwen 2.5 Coder 1.5B Instruct (GGUF Q4_K_M, ~941 MB).
268 ///
269 /// Optimised for code-generation tasks on memory-constrained devices.
270 static GgufModelConfig qwen25Coder15bConfig() => api.qwen25Coder15BConfig();
271
272 /// Qwen 2.5 Coder 3B Instruct (GGUF Q4_K_M, ~1.93 GB).
273 ///
274 /// Optimised for code-generation tasks on macOS and Linux desktop.
275 static GgufModelConfig qwen25Coder3bConfig() => api.qwen25Coder3BConfig();
276
277 /// Qwen 3 0.6B (GGUF Q4_K_M, ~0.5 GB).
278 static GgufModelConfig qwen306bConfig() => api.qwen306BConfig();
279
280 /// Qwen 3 1.7B (GGUF Q4_K_M, ~1.3 GB).
281 static GgufModelConfig qwen317bConfig() => api.qwen317BConfig();
282
283 /// Qwen 3 4B (GGUF Q4_K_M, ~2.5 GB).
284 static GgufModelConfig qwen34bConfig() => api.qwen34BConfig();
285
286 /// Qwen 3 8B (GGUF Q4_K_M, ~5 GB).
287 static GgufModelConfig qwen38bConfig() => api.qwen38BConfig();
288
289 /// Qwen 3 14B (GGUF Q4_K_M, ~8.4 GB).
290 static GgufModelConfig qwen314bConfig() => api.qwen314BConfig();
291
292 /// Qwen 3 32B (GGUF Q4_K_M, ~19.8 GB).
293 static GgufModelConfig qwen332bConfig() => api.qwen332BConfig();
294
295 /// Qwen 3 4B Instruct 2507, the non-thinking checkpoint (~2.5 GB).
296 static GgufModelConfig qwen34bInstruct2507Config() =>
297 api.qwen34BInstruct2507Config();
298
299 /// Qwen 3 4B Thinking 2507 (~2.5 GB).
300 static GgufModelConfig qwen34bThinking2507Config() =>
301 api.qwen34BThinking2507Config();
302
303 /// Qwen 3 30B-A3B Instruct 2507 mixture-of-experts (~18.6 GB).
304 static GgufModelConfig qwen330bA3bInstruct2507Config() =>
305 api.qwen330BA3BInstruct2507Config();
306
307 // -------------------------------------------------------------------------
308 // Sampling config factories
309 // -------------------------------------------------------------------------
310
311 /// Balanced defaults: `temperature=0.7`, `topP=0.95`, `maxTokens=512`.
312 ///
313 /// Good for general-purpose conversational chat.
314 static SamplingConfig defaultSamplingConfig() => api.defaultSamplingConfig();
315
316 /// Greedy / fully deterministic decoding: `temperature=0.0`, `maxTokens=512`.
317 ///
318 /// Use for coding, fact-retrieval, or reproducibility-sensitive tasks.
319 static SamplingConfig deterministicSamplingConfig() =>
320 api.deterministicSamplingConfig();
321
322 /// Low-latency preset for memory-constrained mobile:
323 /// `temperature=0.7`, `topP=0.95`, `maxTokens=128`.
324 static SamplingConfig mobileSamplingConfig() => api.mobileSamplingConfig();
325 }