@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
copilot/assess-inference-engine-complexity
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
rs 572 lines 19.7 KB
Raw
1 //! Flutter Rust Bridge v2 API for the Onde Inference Dart SDK.
2 //!
3 //! This module declares mirror types (re-declared plain Rust structs/enums
4 //! without UniFFI derives) and an `OndeChatEngine` opaque wrapper so that
5 //! `flutter_rust_bridge_codegen generate` can produce clean Dart bindings.
6 //!
7 //! All `onde::inference` types are imported with an `Onde*` alias to avoid
8 //! name collisions with the bridge mirrors defined below.
9
10 use crate::frb_generated::StreamSink;
11 use flutter_rust_bridge::frb;
12 use onde::inference::{
13 ChatEngine, ChatMessage as OndeChatMessage, ChatRole as OndeChatRole,
14 EngineInfo as OndeEngineInfo, EngineStatus as OndeEngineStatus,
15 GgufModelConfig as OndeGgufModelConfig, InferenceError, InferenceResult as OndeInferenceResult,
16 SamplingConfig as OndeSamplingConfig, StreamChunk as OndeStreamChunk,
17 };
18
19 // ── Mirror: ChatRole ──────────────────────────────────────────────────────────
20
21 /// Role of a participant in a chat conversation.
22 #[derive(Debug, Clone, PartialEq, Eq)]
23 pub enum ChatRole {
24 System,
25 User,
26 Assistant,
27 }
28
29 impl From<OndeChatRole> for ChatRole {
30 fn from(r: OndeChatRole) -> Self {
31 match r {
32 OndeChatRole::System => ChatRole::System,
33 OndeChatRole::User => ChatRole::User,
34 OndeChatRole::Assistant => ChatRole::Assistant,
35 }
36 }
37 }
38
39 impl From<ChatRole> for OndeChatRole {
40 fn from(r: ChatRole) -> Self {
41 match r {
42 ChatRole::System => OndeChatRole::System,
43 ChatRole::User => OndeChatRole::User,
44 ChatRole::Assistant => OndeChatRole::Assistant,
45 }
46 }
47 }
48
49 // ── Mirror: ChatMessage ───────────────────────────────────────────────────────
50
51 /// A single message in a conversation.
52 #[derive(Debug, Clone)]
53 pub struct ChatMessage {
54 pub role: ChatRole,
55 pub content: String,
56 }
57
58 impl From<OndeChatMessage> for ChatMessage {
59 fn from(m: OndeChatMessage) -> Self {
60 Self {
61 role: m.role.into(),
62 content: m.content,
63 }
64 }
65 }
66
67 impl From<ChatMessage> for OndeChatMessage {
68 fn from(m: ChatMessage) -> Self {
69 Self {
70 role: m.role.into(),
71 content: m.content,
72 }
73 }
74 }
75
76 // ── Mirror: SamplingConfig ────────────────────────────────────────────────────
77
78 /// Sampling parameters for text generation. All fields are optional — `None`
79 /// means "use the engine default".
80 #[derive(Debug, Clone)]
81 pub struct SamplingConfig {
82 pub temperature: Option<f64>,
83 pub top_p: Option<f64>,
84 pub top_k: Option<u64>,
85 pub min_p: Option<f64>,
86 pub max_tokens: Option<u64>,
87 pub frequency_penalty: Option<f32>,
88 pub presence_penalty: Option<f32>,
89 }
90
91 impl From<OndeSamplingConfig> for SamplingConfig {
92 fn from(s: OndeSamplingConfig) -> Self {
93 Self {
94 temperature: s.temperature,
95 top_p: s.top_p,
96 top_k: s.top_k,
97 min_p: s.min_p,
98 max_tokens: s.max_tokens,
99 frequency_penalty: s.frequency_penalty,
100 presence_penalty: s.presence_penalty,
101 }
102 }
103 }
104
105 impl From<SamplingConfig> for OndeSamplingConfig {
106 fn from(s: SamplingConfig) -> Self {
107 Self {
108 temperature: s.temperature,
109 top_p: s.top_p,
110 top_k: s.top_k,
111 min_p: s.min_p,
112 max_tokens: s.max_tokens,
113 frequency_penalty: s.frequency_penalty,
114 presence_penalty: s.presence_penalty,
115 }
116 }
117 }
118
119 // ── Mirror: GgufModelConfig ───────────────────────────────────────────────────
120
121 /// Configuration for loading a pre-quantised GGUF model.
122 #[derive(Debug, Clone)]
123 pub struct GgufModelConfig {
124 /// HuggingFace repository ID, e.g. `"bartowski/Qwen2.5-1.5B-Instruct-GGUF"`.
125 pub model_id: String,
126 /// GGUF filename(s) within the repository.
127 pub files: Vec<String>,
128 /// Optional explicit tokenizer model ID (required on Android).
129 pub tok_model_id: Option<String>,
130 /// Human-friendly display name, e.g. `"Qwen 2.5 1.5B"`.
131 pub display_name: String,
132 /// Approximate memory footprint, e.g. `"~941 MB (GGUF Q4_K_M)"`.
133 pub approx_memory: String,
134 }
135
136 impl From<OndeGgufModelConfig> for GgufModelConfig {
137 fn from(c: OndeGgufModelConfig) -> Self {
138 Self {
139 model_id: c.model_id,
140 files: c.files,
141 tok_model_id: c.tok_model_id,
142 display_name: c.display_name,
143 approx_memory: c.approx_memory,
144 }
145 }
146 }
147
148 impl From<GgufModelConfig> for OndeGgufModelConfig {
149 fn from(c: GgufModelConfig) -> Self {
150 Self {
151 model_id: c.model_id,
152 files: c.files,
153 tok_model_id: c.tok_model_id,
154 display_name: c.display_name,
155 approx_memory: c.approx_memory,
156 }
157 }
158 }
159
160 // ── Mirror: InferenceResult ───────────────────────────────────────────────────
161
162 /// The result of a completed inference request.
163 #[derive(Debug, Clone)]
164 pub struct InferenceResult {
165 /// The generated assistant reply text.
166 pub text: String,
167 /// Wall-clock inference duration in seconds.
168 pub duration_secs: f64,
169 /// Human-readable duration string, e.g. `"4.5s"` or `"2m 3.1s"`.
170 pub duration_display: String,
171 /// Finish reason reported by the model, e.g. `"stop"` or `"length"`.
172 pub finish_reason: String,
173 }
174
175 impl From<OndeInferenceResult> for InferenceResult {
176 fn from(r: OndeInferenceResult) -> Self {
177 Self {
178 text: r.text,
179 duration_secs: r.duration_secs,
180 duration_display: r.duration_display,
181 finish_reason: r.finish_reason,
182 }
183 }
184 }
185
186 // ── Mirror: StreamChunk ───────────────────────────────────────────────────────
187
188 /// A single streaming token chunk emitted during inference.
189 #[derive(Debug, Clone)]
190 pub struct StreamChunk {
191 /// The token text delta (may be empty on the final chunk).
192 pub delta: String,
193 /// `true` when this is the last chunk in the stream.
194 pub done: bool,
195 /// Finish reason (set only on the final chunk).
196 pub finish_reason: Option<String>,
197 }
198
199 impl From<OndeStreamChunk> for StreamChunk {
200 fn from(c: OndeStreamChunk) -> Self {
201 Self {
202 delta: c.delta,
203 done: c.done,
204 finish_reason: c.finish_reason,
205 }
206 }
207 }
208
209 // ── Mirror: EngineStatus ──────────────────────────────────────────────────────
210
211 /// Lifecycle status of the inference engine.
212 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
213 pub enum EngineStatus {
214 Unloaded,
215 Loading,
216 Ready,
217 Generating,
218 Error,
219 }
220
221 impl From<OndeEngineStatus> for EngineStatus {
222 fn from(s: OndeEngineStatus) -> Self {
223 match s {
224 OndeEngineStatus::Unloaded => EngineStatus::Unloaded,
225 OndeEngineStatus::Loading => EngineStatus::Loading,
226 OndeEngineStatus::Ready => EngineStatus::Ready,
227 OndeEngineStatus::Generating => EngineStatus::Generating,
228 OndeEngineStatus::Error => EngineStatus::Error,
229 }
230 }
231 }
232
233 // ── Mirror: EngineInfo ────────────────────────────────────────────────────────
234
235 /// A point-in-time snapshot of the engine's state.
236 #[derive(Debug, Clone)]
237 pub struct EngineInfo {
238 pub status: EngineStatus,
239 /// Display name of the currently loaded model, if any.
240 pub model_name: Option<String>,
241 /// Approximate memory footprint of the loaded model, if any.
242 pub approx_memory: Option<String>,
243 /// Number of conversation turns in the current history.
244 pub history_length: u64,
245 }
246
247 impl From<OndeEngineInfo> for EngineInfo {
248 fn from(i: OndeEngineInfo) -> Self {
249 Self {
250 status: i.status.into(),
251 model_name: i.model_name,
252 approx_memory: i.approx_memory,
253 history_length: i.history_length,
254 }
255 }
256 }
257
258 // ── Error type ────────────────────────────────────────────────────────────────
259
260 /// Bridge error type mapping from [`onde::inference::InferenceError`].
261 #[derive(Debug, thiserror::Error)]
262 pub enum OndeError {
263 #[error("No model loaded — call load_gguf_model first")]
264 NoModelLoaded,
265
266 #[error("Model already loaded: {model_name}")]
267 AlreadyLoaded { model_name: String },
268
269 #[error("Failed to build model: {reason}")]
270 ModelBuild { reason: String },
271
272 #[error("Inference failed: {reason}")]
273 Inference { reason: String },
274
275 #[error("Operation was cancelled")]
276 Cancelled,
277
278 #[error("{reason}")]
279 Other { reason: String },
280 }
281
282 impl From<InferenceError> for OndeError {
283 fn from(e: InferenceError) -> Self {
284 match e {
285 InferenceError::NoModelLoaded => OndeError::NoModelLoaded,
286 InferenceError::AlreadyLoaded { model_name } => OndeError::AlreadyLoaded { model_name },
287 InferenceError::ModelBuild { reason } => OndeError::ModelBuild { reason },
288 InferenceError::Inference { reason } => OndeError::Inference { reason },
289 InferenceError::Cancelled => OndeError::Cancelled,
290 InferenceError::Other { reason } => OndeError::Other { reason },
291 }
292 }
293 }
294
295 // ── OndeChatEngine ────────────────────────────────────────────────────────────
296
297 /// Opaque wrapper around `onde::inference::ChatEngine`.
298 ///
299 /// Dart/Flutter receives this as an opaque `RustOpaque<OndeChatEngine>` handle.
300 /// All methods delegate directly to the inner `ChatEngine`.
301 pub struct OndeChatEngine {
302 inner: ChatEngine,
303 }
304
305 impl OndeChatEngine {
306 // ── Construction ──────────────────────────────────────────────────────
307
308 /// Create a new engine with no model loaded.
309 #[frb(sync)]
310 pub fn new() -> OndeChatEngine {
311 OndeChatEngine {
312 inner: ChatEngine::new(),
313 }
314 }
315
316 // ── Model lifecycle ───────────────────────────────────────────────────
317
318 /// Load a GGUF model into the engine.
319 ///
320 /// Returns the load duration in seconds on success.
321 pub async fn load_gguf_model(
322 &self,
323 config: GgufModelConfig,
324 system_prompt: Option<String>,
325 sampling: Option<SamplingConfig>,
326 ) -> Result<f64, OndeError> {
327 let elapsed = self
328 .inner
329 .load_gguf_model(config.into(), system_prompt, sampling.map(Into::into))
330 .await
331 .map_err(OndeError::from)?;
332 Ok(elapsed.as_secs_f64())
333 }
334
335 /// Unload the current model and release its memory.
336 ///
337 /// Returns the display name of the model that was unloaded, or `None`
338 /// if no model was loaded.
339 pub async fn unload_model(&self) -> Option<String> {
340 self.inner.unload_model().await
341 }
342
343 /// Returns `true` if a model is currently loaded and ready.
344 pub async fn is_loaded(&self) -> bool {
345 self.inner.is_loaded().await
346 }
347
348 /// Returns a point-in-time snapshot of the engine's state.
349 pub async fn info(&self) -> EngineInfo {
350 self.inner.info().await.into()
351 }
352
353 // ── Configuration ─────────────────────────────────────────────────────
354
355 /// Set or replace the system prompt.
356 ///
357 /// The prompt is prepended to every inference request and is **not**
358 /// stored in the conversation history.
359 pub async fn set_system_prompt(&self, prompt: String) {
360 self.inner.set_system_prompt(prompt).await;
361 }
362
363 /// Clear the current system prompt.
364 pub async fn clear_system_prompt(&self) {
365 self.inner.clear_system_prompt().await;
366 }
367
368 /// Replace the sampling configuration used for all subsequent inference.
369 pub async fn set_sampling(&self, sampling: SamplingConfig) {
370 self.inner.set_sampling(sampling.into()).await;
371 }
372
373 // ── Conversation history ──────────────────────────────────────────────
374
375 /// Returns a clone of the full conversation history.
376 pub async fn history(&self) -> Vec<ChatMessage> {
377 self.inner
378 .history()
379 .await
380 .into_iter()
381 .map(ChatMessage::from)
382 .collect()
383 }
384
385 /// Clear the conversation history while keeping the model loaded.
386 ///
387 /// Returns the number of turns that were removed.
388 pub async fn clear_history(&self) -> u64 {
389 self.inner.clear_history().await as u64
390 }
391
392 /// Append a message to history without running inference.
393 ///
394 /// Useful for restoring a saved conversation or injecting context.
395 pub async fn push_history(&self, message: ChatMessage) {
396 self.inner.push_history(message.into()).await;
397 }
398
399 // ── Non-streaming inference ───────────────────────────────────────────
400
401 /// Send a user message and receive a complete assistant reply.
402 ///
403 /// Both the user message and assistant reply are appended to the
404 /// conversation history on success.
405 pub async fn send_message(&self, message: String) -> Result<InferenceResult, OndeError> {
406 self.inner
407 .send_message(message)
408 .await
409 .map(InferenceResult::from)
410 .map_err(OndeError::from)
411 }
412
413 /// One-shot generation from an explicit message list.
414 ///
415 /// Does **not** modify the conversation history.
416 pub async fn generate(
417 &self,
418 messages: Vec<ChatMessage>,
419 sampling: Option<SamplingConfig>,
420 ) -> Result<InferenceResult, OndeError> {
421 let onde_messages: Vec<OndeChatMessage> =
422 messages.into_iter().map(OndeChatMessage::from).collect();
423 self.inner
424 .generate(onde_messages, sampling.map(Into::into))
425 .await
426 .map(InferenceResult::from)
427 .map_err(OndeError::from)
428 }
429
430 // ── Streaming inference ───────────────────────────────────────────────
431
432 /// Stream a reply token-by-token.
433 ///
434 /// Each [`StreamChunk`] carries a `delta` string. The final chunk has
435 /// `done == true`. The `sink` is closed automatically when the stream
436 /// ends or if the receiver is dropped.
437 pub async fn stream_message(
438 &self,
439 message: String,
440 sink: StreamSink<StreamChunk>,
441 ) -> Result<(), OndeError> {
442 let mut rx = self
443 .inner
444 .stream_message(message)
445 .await
446 .map_err(OndeError::from)?;
447
448 while let Some(chunk) = rx.recv().await {
449 let bridge_chunk = StreamChunk::from(chunk);
450 let done = bridge_chunk.done;
451 let _ = sink.add(bridge_chunk);
452 if done {
453 break;
454 }
455 }
456
457 Ok(())
458 }
459 }
460
461 // ── Free functions ────────────────────────────────────────────────────────────
462
463 /// Return the platform-appropriate default `GgufModelConfig`.
464 ///
465 /// Selects the 1.5B model on iOS / tvOS / Android and the 3B model on
466 /// macOS / Windows / Linux.
467 #[frb(sync)]
468 pub fn default_model_config() -> GgufModelConfig {
469 OndeGgufModelConfig::platform_default().into()
470 }
471
472 /// `GgufModelConfig` for Qwen 2.5 1.5B Instruct Q4_K_M (~941 MB).
473 #[frb(sync)]
474 pub fn qwen25_1_5b_config() -> GgufModelConfig {
475 OndeGgufModelConfig::qwen25_1_5b().into()
476 }
477
478 /// `GgufModelConfig` for Qwen 2.5 3B Instruct Q4_K_M (~1.93 GB).
479 #[frb(sync)]
480 pub fn qwen25_3b_config() -> GgufModelConfig {
481 OndeGgufModelConfig::qwen25_3b().into()
482 }
483
484 /// `GgufModelConfig` for Qwen 2.5 Coder 1.5B Instruct Q4_K_M (~941 MB).
485 #[frb(sync)]
486 pub fn qwen25_coder_1_5b_config() -> GgufModelConfig {
487 OndeGgufModelConfig::qwen25_coder_1_5b().into()
488 }
489
490 /// `GgufModelConfig` for Qwen 2.5 Coder 3B Instruct Q4_K_M (~1.93 GB).
491 #[frb(sync)]
492 pub fn qwen25_coder_3b_config() -> GgufModelConfig {
493 OndeGgufModelConfig::qwen25_coder_3b().into()
494 }
495
496 /// Default sampling config: `temperature=0.7`, `top_p=0.95`, `max_tokens=512`.
497 #[frb(sync)]
498 pub fn default_sampling_config() -> SamplingConfig {
499 OndeSamplingConfig::default().into()
500 }
501
502 /// Deterministic sampling config: `temperature=0.0`, greedy, `max_tokens=512`.
503 #[frb(sync)]
504 pub fn deterministic_sampling_config() -> SamplingConfig {
505 OndeSamplingConfig::deterministic().into()
506 }
507
508 /// Mobile sampling config: `temperature=0.7`, `top_p=0.95`, `max_tokens=128`.
509 #[frb(sync)]
510 pub fn mobile_sampling_config() -> SamplingConfig {
511 OndeSamplingConfig::mobile().into()
512 }
513
514 /// Seed the HuggingFace cache environment for sandboxed platforms.
515 ///
516 /// On iOS (and Android), the default `~/.cache/huggingface/hub` path is outside
517 /// the app sandbox and writes fail with `Operation not permitted (os error 1)`.
518 ///
519 /// Call this **once** at startup — before any `load_gguf_model` or
520 /// `download_model` — with the app's writable data directory.
521 ///
522 /// # Dart usage
523 ///
524 /// ```dart
525 /// import 'package:path_provider/path_provider.dart';
526 ///
527 /// final dir = await getApplicationSupportDirectory();
528 /// configureCacheDir(appDataDir: dir.path);
529 /// ```
530 ///
531 /// Internally this:
532 /// 1. Creates `<app_data_dir>/huggingface/hub/` if it doesn't exist.
533 /// 2. Sets the `HF_HOME` and `HF_HUB_CACHE` environment variables.
534 /// 3. Seeds `mistralrs_core::GLOBAL_HF_CACHE` so the model loader never
535 /// falls back to `Cache::default()`.
536 #[frb(sync)]
537 pub fn configure_cache_dir(app_data_dir: String) -> Result<(), OndeError> {
538 use std::fs;
539 use std::path::PathBuf;
540
541 let data_dir = PathBuf::from(&app_data_dir);
542 let hf_home = data_dir.join("models");
543 let hf_hub_cache = hf_home.join("hub");
544
545 fs::create_dir_all(&hf_hub_cache).map_err(|e| OndeError::Other {
546 reason: format!(
547 "Cannot create HF cache dir {}: {}",
548 hf_hub_cache.display(),
549 e
550 ),
551 })?;
552
553 // Set env vars BEFORE any hf-hub or mistralrs call. Both libraries
554 // read these to resolve the cache directory — no need to touch
555 // GLOBAL_HF_CACHE directly (that's seeded lazily by load_gguf_model
556 // on Android/iOS when it reads HF_HOME back).
557 std::env::set_var("HF_HOME", &hf_home);
558 std::env::set_var("HF_HUB_CACHE", &hf_hub_cache);
559
560 // Pre-create tmp dir — iOS sandbox also restricts the system TMPDIR.
561 let tmp_dir = data_dir.join("tmp");
562 let _ = fs::create_dir_all(&tmp_dir);
563 std::env::set_var("TMPDIR", &tmp_dir);
564
565 log::info!(
566 "HF cache configured: HF_HOME={}, HF_HUB_CACHE={}",
567 hf_home.display(),
568 hf_hub_cache.display(),
569 );
570
571 Ok(())
572 }