@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
copilot/fix-onde-npm-sdk-v1-1-2-release
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
rs 621 lines 21.2 KB
Raw
1 //! Flutter Rust Bridge v2 API for the Onde Inference Dart SDK.
2 //!
3 //! This module declares mirror types (re-declared plain Rust structs/enums
4 //! without UniFFI derives) and an `OndeChatEngine` opaque wrapper so that
5 //! `flutter_rust_bridge_codegen generate` can produce clean Dart bindings.
6 //!
7 //! All `onde::inference` types are imported with an `Onde*` alias to avoid
8 //! name collisions with the bridge mirrors defined below.
9
10 use crate::frb_generated::StreamSink;
11 use flutter_rust_bridge::frb;
12 use onde::inference::{
13 ChatEngine, ChatMessage as OndeChatMessage, ChatRole as OndeChatRole,
14 EngineInfo as OndeEngineInfo, EngineStatus as OndeEngineStatus,
15 GgufModelConfig as OndeGgufModelConfig, InferenceError, InferenceResult as OndeInferenceResult,
16 SamplingConfig as OndeSamplingConfig, StreamChunk as OndeStreamChunk,
17 ToolCallInfo as OndeToolCallInfo,
18 };
19
20 // ── Mirror: ChatRole ──────────────────────────────────────────────────────────
21
22 /// Role of a participant in a chat conversation.
23 #[derive(Debug, Clone, PartialEq, Eq)]
24 pub enum ChatRole {
25 System,
26 User,
27 Assistant,
28 }
29
30 impl From<OndeChatRole> for ChatRole {
31 fn from(r: OndeChatRole) -> Self {
32 match r {
33 OndeChatRole::System => ChatRole::System,
34 OndeChatRole::User => ChatRole::User,
35 OndeChatRole::Assistant => ChatRole::Assistant,
36 }
37 }
38 }
39
40 impl From<ChatRole> for OndeChatRole {
41 fn from(r: ChatRole) -> Self {
42 match r {
43 ChatRole::System => OndeChatRole::System,
44 ChatRole::User => OndeChatRole::User,
45 ChatRole::Assistant => OndeChatRole::Assistant,
46 }
47 }
48 }
49
50 // ── Mirror: ChatMessage ───────────────────────────────────────────────────────
51
52 /// A single message in a conversation.
53 #[derive(Debug, Clone)]
54 pub struct ChatMessage {
55 pub role: ChatRole,
56 pub content: String,
57 }
58
59 impl From<OndeChatMessage> for ChatMessage {
60 fn from(m: OndeChatMessage) -> Self {
61 Self {
62 role: m.role.into(),
63 content: m.content,
64 }
65 }
66 }
67
68 impl From<ChatMessage> for OndeChatMessage {
69 fn from(m: ChatMessage) -> Self {
70 Self {
71 role: m.role.into(),
72 content: m.content,
73 }
74 }
75 }
76
77 // ── Mirror: SamplingConfig ────────────────────────────────────────────────────
78
79 /// Sampling parameters for text generation. All fields are optional — `None`
80 /// means "use the engine default".
81 #[derive(Debug, Clone)]
82 pub struct SamplingConfig {
83 pub temperature: Option<f64>,
84 pub top_p: Option<f64>,
85 pub top_k: Option<u64>,
86 pub min_p: Option<f64>,
87 pub max_tokens: Option<u64>,
88 pub frequency_penalty: Option<f32>,
89 pub presence_penalty: Option<f32>,
90 }
91
92 impl From<OndeSamplingConfig> for SamplingConfig {
93 fn from(s: OndeSamplingConfig) -> Self {
94 Self {
95 temperature: s.temperature,
96 top_p: s.top_p,
97 top_k: s.top_k,
98 min_p: s.min_p,
99 max_tokens: s.max_tokens,
100 frequency_penalty: s.frequency_penalty,
101 presence_penalty: s.presence_penalty,
102 }
103 }
104 }
105
106 impl From<SamplingConfig> for OndeSamplingConfig {
107 fn from(s: SamplingConfig) -> Self {
108 Self {
109 temperature: s.temperature,
110 top_p: s.top_p,
111 top_k: s.top_k,
112 min_p: s.min_p,
113 max_tokens: s.max_tokens,
114 frequency_penalty: s.frequency_penalty,
115 presence_penalty: s.presence_penalty,
116 }
117 }
118 }
119
120 // ── Mirror: GgufModelConfig ───────────────────────────────────────────────────
121
122 /// Configuration for loading a pre-quantised GGUF model.
123 #[derive(Debug, Clone)]
124 pub struct GgufModelConfig {
125 /// HuggingFace repository ID, e.g. `"bartowski/Qwen2.5-1.5B-Instruct-GGUF"`.
126 pub model_id: String,
127 /// GGUF filename(s) within the repository.
128 pub files: Vec<String>,
129 /// Optional explicit tokenizer model ID (required on Android).
130 pub tok_model_id: Option<String>,
131 /// Human-friendly display name, e.g. `"Qwen 2.5 1.5B"`.
132 pub display_name: String,
133 /// Approximate memory footprint, e.g. `"~941 MB (GGUF Q4_K_M)"`.
134 pub approx_memory: String,
135 }
136
137 impl From<OndeGgufModelConfig> for GgufModelConfig {
138 fn from(c: OndeGgufModelConfig) -> Self {
139 Self {
140 model_id: c.model_id,
141 files: c.files,
142 tok_model_id: c.tok_model_id,
143 display_name: c.display_name,
144 approx_memory: c.approx_memory,
145 }
146 }
147 }
148
149 impl From<GgufModelConfig> for OndeGgufModelConfig {
150 fn from(c: GgufModelConfig) -> Self {
151 Self {
152 model_id: c.model_id,
153 files: c.files,
154 tok_model_id: c.tok_model_id,
155 display_name: c.display_name,
156 approx_memory: c.approx_memory,
157 chat_template: None,
158 }
159 }
160 }
161
162 // ── Mirror: ToolCallInfo ────────────────────────────────────────────────────
163
164 /// Structured tool-call request emitted by the model.
165 #[derive(Debug, Clone)]
166 pub struct ToolCallInfo {
167 pub id: String,
168 pub function_name: String,
169 pub arguments: String,
170 }
171
172 impl From<OndeToolCallInfo> for ToolCallInfo {
173 fn from(t: OndeToolCallInfo) -> Self {
174 Self {
175 id: t.id,
176 function_name: t.function_name,
177 arguments: t.arguments,
178 }
179 }
180 }
181
182 // ── Mirror: InferenceResult ───────────────────────────────────────────────────
183
184 /// The result of a completed inference request.
185 #[derive(Debug, Clone)]
186 pub struct InferenceResult {
187 /// The generated assistant reply text.
188 pub text: String,
189 /// Wall-clock inference duration in seconds.
190 pub duration_secs: f64,
191 /// Human-readable duration string, e.g. `"4.5s"` or `"2m 3.1s"`.
192 pub duration_display: String,
193 /// Finish reason reported by the model, e.g. `"stop"` or `"length"`.
194 pub finish_reason: String,
195 /// Tool calls requested by the model (empty if none).
196 pub tool_calls: Vec<ToolCallInfo>,
197 }
198
199 impl From<OndeInferenceResult> for InferenceResult {
200 fn from(r: OndeInferenceResult) -> Self {
201 Self {
202 text: r.text,
203 duration_secs: r.duration_secs,
204 duration_display: r.duration_display,
205 finish_reason: r.finish_reason,
206 tool_calls: r.tool_calls.into_iter().map(ToolCallInfo::from).collect(),
207 }
208 }
209 }
210
211 // ── Mirror: StreamChunk ───────────────────────────────────────────────────────
212
213 /// A single streaming token chunk emitted during inference.
214 #[derive(Debug, Clone)]
215 pub struct StreamChunk {
216 /// The token text delta (may be empty on the final chunk).
217 pub delta: String,
218 /// `true` when this is the last chunk in the stream.
219 pub done: bool,
220 /// Finish reason (set only on the final chunk).
221 pub finish_reason: Option<String>,
222 }
223
224 impl From<OndeStreamChunk> for StreamChunk {
225 fn from(c: OndeStreamChunk) -> Self {
226 Self {
227 delta: c.delta,
228 done: c.done,
229 finish_reason: c.finish_reason,
230 }
231 }
232 }
233
234 // ── Mirror: EngineStatus ──────────────────────────────────────────────────────
235
236 /// Lifecycle status of the inference engine.
237 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
238 pub enum EngineStatus {
239 Unloaded,
240 Loading,
241 Ready,
242 Generating,
243 Error,
244 }
245
246 impl From<OndeEngineStatus> for EngineStatus {
247 fn from(s: OndeEngineStatus) -> Self {
248 match s {
249 OndeEngineStatus::Unloaded => EngineStatus::Unloaded,
250 OndeEngineStatus::Loading => EngineStatus::Loading,
251 OndeEngineStatus::Ready => EngineStatus::Ready,
252 OndeEngineStatus::Generating => EngineStatus::Generating,
253 OndeEngineStatus::Error => EngineStatus::Error,
254 }
255 }
256 }
257
258 // ── Mirror: EngineInfo ────────────────────────────────────────────────────────
259
260 /// A point-in-time snapshot of the engine's state.
261 #[derive(Debug, Clone)]
262 pub struct EngineInfo {
263 pub status: EngineStatus,
264 /// Display name of the currently loaded model, if any.
265 pub model_name: Option<String>,
266 /// Approximate memory footprint of the loaded model, if any.
267 pub approx_memory: Option<String>,
268 /// Number of conversation turns in the current history.
269 pub history_length: u64,
270 }
271
272 impl From<OndeEngineInfo> for EngineInfo {
273 fn from(i: OndeEngineInfo) -> Self {
274 Self {
275 status: i.status.into(),
276 model_name: i.model_name,
277 approx_memory: i.approx_memory,
278 history_length: i.history_length,
279 }
280 }
281 }
282
283 // ── Error type ────────────────────────────────────────────────────────────────
284
285 /// Bridge error type mapping from [`onde::inference::InferenceError`].
286 #[derive(Debug, thiserror::Error)]
287 pub enum OndeError {
288 #[error("No model loaded — call load_gguf_model first")]
289 NoModelLoaded,
290
291 #[error("Model already loaded: {model_name}")]
292 AlreadyLoaded { model_name: String },
293
294 #[error("Failed to build model: {reason}")]
295 ModelBuild { reason: String },
296
297 #[error("Inference failed: {reason}")]
298 Inference { reason: String },
299
300 #[error("Operation was cancelled")]
301 Cancelled,
302
303 #[error("{reason}")]
304 Other { reason: String },
305 }
306
307 impl From<InferenceError> for OndeError {
308 fn from(e: InferenceError) -> Self {
309 match e {
310 InferenceError::NoModelLoaded => OndeError::NoModelLoaded,
311 InferenceError::AlreadyLoaded { model_name } => OndeError::AlreadyLoaded { model_name },
312 InferenceError::ModelBuild { reason } => OndeError::ModelBuild { reason },
313 InferenceError::Inference { reason } => OndeError::Inference { reason },
314 InferenceError::Cancelled => OndeError::Cancelled,
315 InferenceError::Other { reason } => OndeError::Other { reason },
316 }
317 }
318 }
319
320 // ── OndeChatEngine ────────────────────────────────────────────────────────────
321
322 /// Opaque wrapper around `onde::inference::ChatEngine`.
323 ///
324 /// Dart/Flutter receives this as an opaque `RustOpaque<OndeChatEngine>` handle.
325 /// All methods delegate directly to the inner `ChatEngine`.
326 pub struct OndeChatEngine {
327 inner: ChatEngine,
328 }
329
330 impl OndeChatEngine {
331 // ── Construction ──────────────────────────────────────────────────────
332
333 /// Create a new engine with no model loaded.
334 #[frb(sync)]
335 pub fn new() -> OndeChatEngine {
336 OndeChatEngine {
337 inner: ChatEngine::new(),
338 }
339 }
340
341 // ── Model lifecycle ───────────────────────────────────────────────────
342
343 /// Load a GGUF model into the engine.
344 ///
345 /// Returns the load duration in seconds on success.
346 pub async fn load_gguf_model(
347 &self,
348 config: GgufModelConfig,
349 system_prompt: Option<String>,
350 sampling: Option<SamplingConfig>,
351 ) -> Result<f64, OndeError> {
352 let elapsed = self
353 .inner
354 .load_gguf_model(config.into(), system_prompt, sampling.map(Into::into))
355 .await
356 .map_err(OndeError::from)?;
357 Ok(elapsed.as_secs_f64())
358 }
359
360 /// Load the model assigned to this Onde app from the dashboard.
361 ///
362 /// Returns the wall-clock seconds taken to load the model.
363 pub async fn load_assigned_model(
364 &self,
365 app_id: String,
366 app_secret: String,
367 system_prompt: Option<String>,
368 sampling: Option<SamplingConfig>,
369 ) -> Result<f64, OndeError> {
370 let elapsed = self
371 .inner
372 .load_assigned_model(
373 smbcloud_gresiq_sdk::Environment::Production,
374 &app_id,
375 &app_secret,
376 system_prompt,
377 sampling.map(Into::into),
378 )
379 .await
380 .map_err(OndeError::from)?;
381 Ok(elapsed.as_secs_f64())
382 }
383
384 /// Unload the current model and release its memory.
385 ///
386 /// Returns the display name of the model that was unloaded, or `None`
387 /// if no model was loaded.
388 pub async fn unload_model(&self) -> Option<String> {
389 self.inner.unload_model().await
390 }
391
392 /// Returns `true` if a model is currently loaded and ready.
393 pub async fn is_loaded(&self) -> bool {
394 self.inner.is_loaded().await
395 }
396
397 /// Returns a point-in-time snapshot of the engine's state.
398 pub async fn info(&self) -> EngineInfo {
399 self.inner.info().await.into()
400 }
401
402 // ── Configuration ─────────────────────────────────────────────────────
403
404 /// Set or replace the system prompt.
405 ///
406 /// The prompt is prepended to every inference request and is **not**
407 /// stored in the conversation history.
408 pub async fn set_system_prompt(&self, prompt: String) {
409 self.inner.set_system_prompt(prompt).await;
410 }
411
412 /// Clear the current system prompt.
413 pub async fn clear_system_prompt(&self) {
414 self.inner.clear_system_prompt().await;
415 }
416
417 /// Replace the sampling configuration used for all subsequent inference.
418 pub async fn set_sampling(&self, sampling: SamplingConfig) {
419 self.inner.set_sampling(sampling.into()).await;
420 }
421
422 // ── Conversation history ──────────────────────────────────────────────
423
424 /// Returns a clone of the full conversation history.
425 pub async fn history(&self) -> Vec<ChatMessage> {
426 self.inner
427 .history()
428 .await
429 .into_iter()
430 .map(ChatMessage::from)
431 .collect()
432 }
433
434 /// Clear the conversation history while keeping the model loaded.
435 ///
436 /// Returns the number of turns that were removed.
437 pub async fn clear_history(&self) -> u64 {
438 self.inner.clear_history().await as u64
439 }
440
441 /// Append a message to history without running inference.
442 ///
443 /// Useful for restoring a saved conversation or injecting context.
444 pub async fn push_history(&self, message: ChatMessage) {
445 self.inner.push_history(message.into()).await;
446 }
447
448 // ── Non-streaming inference ───────────────────────────────────────────
449
450 /// Send a user message and receive a complete assistant reply.
451 ///
452 /// Both the user message and assistant reply are appended to the
453 /// conversation history on success.
454 pub async fn send_message(&self, message: String) -> Result<InferenceResult, OndeError> {
455 self.inner
456 .send_message(message)
457 .await
458 .map(InferenceResult::from)
459 .map_err(OndeError::from)
460 }
461
462 /// One-shot generation from an explicit message list.
463 ///
464 /// Does **not** modify the conversation history.
465 pub async fn generate(
466 &self,
467 messages: Vec<ChatMessage>,
468 sampling: Option<SamplingConfig>,
469 ) -> Result<InferenceResult, OndeError> {
470 let onde_messages: Vec<OndeChatMessage> =
471 messages.into_iter().map(OndeChatMessage::from).collect();
472 self.inner
473 .generate(onde_messages, sampling.map(Into::into))
474 .await
475 .map(InferenceResult::from)
476 .map_err(OndeError::from)
477 }
478
479 // ── Streaming inference ───────────────────────────────────────────────
480
481 /// Stream a reply token-by-token.
482 ///
483 /// Each [`StreamChunk`] carries a `delta` string. The final chunk has
484 /// `done == true`. The `sink` is closed automatically when the stream
485 /// ends or if the receiver is dropped.
486 pub async fn stream_message(
487 &self,
488 message: String,
489 sink: StreamSink<StreamChunk>,
490 ) -> Result<(), OndeError> {
491 let mut rx = self
492 .inner
493 .stream_message(message)
494 .await
495 .map_err(OndeError::from)?;
496
497 while let Some(chunk) = rx.recv().await {
498 let bridge_chunk = StreamChunk::from(chunk);
499 let done = bridge_chunk.done;
500 let _ = sink.add(bridge_chunk);
501 if done {
502 break;
503 }
504 }
505
506 Ok(())
507 }
508 }
509
510 // ── Free functions ────────────────────────────────────────────────────────────
511
512 /// Return the platform-appropriate default `GgufModelConfig`.
513 ///
514 /// Selects the 1.5B model on iOS / tvOS / Android and the 3B model on
515 /// macOS / Windows / Linux.
516 #[frb(sync)]
517 pub fn default_model_config() -> GgufModelConfig {
518 OndeGgufModelConfig::platform_default().into()
519 }
520
521 /// `GgufModelConfig` for Qwen 2.5 1.5B Instruct Q4_K_M (~941 MB).
522 #[frb(sync)]
523 pub fn qwen25_1_5b_config() -> GgufModelConfig {
524 OndeGgufModelConfig::qwen25_1_5b().into()
525 }
526
527 /// `GgufModelConfig` for Qwen 2.5 3B Instruct Q4_K_M (~1.93 GB).
528 #[frb(sync)]
529 pub fn qwen25_3b_config() -> GgufModelConfig {
530 OndeGgufModelConfig::qwen25_3b().into()
531 }
532
533 /// `GgufModelConfig` for Qwen 2.5 Coder 1.5B Instruct Q4_K_M (~941 MB).
534 #[frb(sync)]
535 pub fn qwen25_coder_1_5b_config() -> GgufModelConfig {
536 OndeGgufModelConfig::qwen25_coder_1_5b().into()
537 }
538
539 /// `GgufModelConfig` for Qwen 2.5 Coder 3B Instruct Q4_K_M (~1.93 GB).
540 #[frb(sync)]
541 pub fn qwen25_coder_3b_config() -> GgufModelConfig {
542 OndeGgufModelConfig::qwen25_coder_3b().into()
543 }
544
545 /// Default sampling config: `temperature=0.7`, `top_p=0.95`, `max_tokens=512`.
546 #[frb(sync)]
547 pub fn default_sampling_config() -> SamplingConfig {
548 OndeSamplingConfig::default().into()
549 }
550
551 /// Deterministic sampling config: `temperature=0.0`, greedy, `max_tokens=512`.
552 #[frb(sync)]
553 pub fn deterministic_sampling_config() -> SamplingConfig {
554 OndeSamplingConfig::deterministic().into()
555 }
556
557 /// Mobile sampling config: `temperature=0.7`, `top_p=0.95`, `max_tokens=128`.
558 #[frb(sync)]
559 pub fn mobile_sampling_config() -> SamplingConfig {
560 OndeSamplingConfig::mobile().into()
561 }
562
563 /// Seed the HuggingFace cache environment for sandboxed platforms.
564 ///
565 /// On iOS (and Android), the default `~/.cache/huggingface/hub` path is outside
566 /// the app sandbox and writes fail with `Operation not permitted (os error 1)`.
567 ///
568 /// Call this **once** at startup — before any `load_gguf_model` or
569 /// `download_model` — with the app's writable data directory.
570 ///
571 /// # Dart usage
572 ///
573 /// ```dart
574 /// import 'package:path_provider/path_provider.dart';
575 ///
576 /// final dir = await getApplicationSupportDirectory();
577 /// configureCacheDir(appDataDir: dir.path);
578 /// ```
579 ///
580 /// Internally this:
581 /// 1. Creates `<app_data_dir>/huggingface/hub/` if it doesn't exist.
582 /// 2. Sets the `HF_HOME` and `HF_HUB_CACHE` environment variables.
583 /// 3. Seeds `mistralrs_core::GLOBAL_HF_CACHE` so the model loader never
584 /// falls back to `Cache::default()`.
585 #[frb(sync)]
586 pub fn configure_cache_dir(app_data_dir: String) -> Result<(), OndeError> {
587 use std::fs;
588 use std::path::PathBuf;
589
590 let data_dir = PathBuf::from(&app_data_dir);
591 let hf_home = data_dir.join("models");
592 let hf_hub_cache = hf_home.join("hub");
593
594 fs::create_dir_all(&hf_hub_cache).map_err(|e| OndeError::Other {
595 reason: format!(
596 "Cannot create HF cache dir {}: {}",
597 hf_hub_cache.display(),
598 e
599 ),
600 })?;
601
602 // Set env vars BEFORE any hf-hub or mistralrs call. Both libraries
603 // read these to resolve the cache directory — no need to touch
604 // GLOBAL_HF_CACHE directly (that's seeded lazily by load_gguf_model
605 // on Android/iOS when it reads HF_HOME back).
606 std::env::set_var("HF_HOME", &hf_home);
607 std::env::set_var("HF_HUB_CACHE", &hf_hub_cache);
608
609 // Pre-create tmp dir — iOS sandbox also restricts the system TMPDIR.
610 let tmp_dir = data_dir.join("tmp");
611 let _ = fs::create_dir_all(&tmp_dir);
612 std::env::set_var("TMPDIR", &tmp_dir);
613
614 log::info!(
615 "HF cache configured: HF_HOME={}, HF_HUB_CACHE={}",
616 hf_home.display(),
617 hf_hub_cache.display(),
618 );
619
620 Ok(())
621 }