@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
main
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
rs 757 lines 25.4 KB
Raw
1 // Copyright 2026 Splitfire AB (Onde Inference). All rights reserved.
2 // SPDX-License-Identifier: MIT OR Apache-2.0
3
4 //! Flutter Rust Bridge v2 API for the Onde Inference Dart SDK.
5 //!
6 //! This module declares mirror types (re-declared plain Rust structs/enums
7 //! without UniFFI derives) and an `OndeChatEngine` opaque wrapper so that
8 //! `flutter_rust_bridge_codegen generate` can produce clean Dart bindings.
9 //!
10 //! All `onde::inference` types are imported with an `Onde*` alias to avoid
11 //! name collisions with the bridge mirrors defined below.
12
13 use crate::frb_generated::StreamSink;
14 use flutter_rust_bridge::frb;
15 use onde::inference::{
16 ChatEngine, ChatMessage as OndeChatMessage, ChatRole as OndeChatRole,
17 EngineInfo as OndeEngineInfo, EngineStatus as OndeEngineStatus,
18 GgufModelConfig as OndeGgufModelConfig, InferenceError, InferenceResult as OndeInferenceResult,
19 SamplingConfig as OndeSamplingConfig, StreamChunk as OndeStreamChunk,
20 ToolCallInfo as OndeToolCallInfo, UqffModelConfig as OndeUqffModelConfig,
21 };
22
23 // ── Mirror: ChatRole ──────────────────────────────────────────────────────────
24
25 /// Role of a participant in a chat conversation.
26 #[derive(Debug, Clone, PartialEq, Eq)]
27 pub enum ChatRole {
28 System,
29 User,
30 Assistant,
31 }
32
33 impl From<OndeChatRole> for ChatRole {
34 fn from(r: OndeChatRole) -> Self {
35 match r {
36 OndeChatRole::System => ChatRole::System,
37 OndeChatRole::User => ChatRole::User,
38 OndeChatRole::Assistant => ChatRole::Assistant,
39 }
40 }
41 }
42
43 impl From<ChatRole> for OndeChatRole {
44 fn from(r: ChatRole) -> Self {
45 match r {
46 ChatRole::System => OndeChatRole::System,
47 ChatRole::User => OndeChatRole::User,
48 ChatRole::Assistant => OndeChatRole::Assistant,
49 }
50 }
51 }
52
53 // ── Mirror: ChatMessage ───────────────────────────────────────────────────────
54
55 /// A single message in a conversation.
56 #[derive(Debug, Clone)]
57 pub struct ChatMessage {
58 pub role: ChatRole,
59 pub content: String,
60 }
61
62 impl From<OndeChatMessage> for ChatMessage {
63 fn from(m: OndeChatMessage) -> Self {
64 Self {
65 role: m.role.into(),
66 content: m.content,
67 }
68 }
69 }
70
71 impl From<ChatMessage> for OndeChatMessage {
72 fn from(m: ChatMessage) -> Self {
73 Self {
74 role: m.role.into(),
75 content: m.content,
76 }
77 }
78 }
79
80 // ── Mirror: SamplingConfig ────────────────────────────────────────────────────
81
82 /// Sampling parameters for text generation. All fields are optional — `None`
83 /// means "use the engine default".
84 #[derive(Debug, Clone)]
85 pub struct SamplingConfig {
86 pub temperature: Option<f64>,
87 pub top_p: Option<f64>,
88 pub top_k: Option<u64>,
89 pub min_p: Option<f64>,
90 pub max_tokens: Option<u64>,
91 pub frequency_penalty: Option<f32>,
92 pub presence_penalty: Option<f32>,
93 }
94
95 impl From<OndeSamplingConfig> for SamplingConfig {
96 fn from(s: OndeSamplingConfig) -> Self {
97 Self {
98 temperature: s.temperature,
99 top_p: s.top_p,
100 top_k: s.top_k,
101 min_p: s.min_p,
102 max_tokens: s.max_tokens,
103 frequency_penalty: s.frequency_penalty,
104 presence_penalty: s.presence_penalty,
105 }
106 }
107 }
108
109 impl From<SamplingConfig> for OndeSamplingConfig {
110 fn from(s: SamplingConfig) -> Self {
111 Self {
112 temperature: s.temperature,
113 top_p: s.top_p,
114 top_k: s.top_k,
115 min_p: s.min_p,
116 max_tokens: s.max_tokens,
117 frequency_penalty: s.frequency_penalty,
118 presence_penalty: s.presence_penalty,
119 }
120 }
121 }
122
123 // ── Mirror: GgufModelConfig ───────────────────────────────────────────────────
124
125 /// Configuration for loading a pre-quantised GGUF model.
126 #[derive(Debug, Clone)]
127 pub struct GgufModelConfig {
128 /// HuggingFace repository ID, e.g. `"bartowski/Qwen2.5-1.5B-Instruct-GGUF"`.
129 pub model_id: String,
130 /// GGUF filename(s) within the repository.
131 pub files: Vec<String>,
132 /// Optional explicit tokenizer model ID (required on Android).
133 pub tok_model_id: Option<String>,
134 /// Human-friendly display name, e.g. `"Qwen 2.5 1.5B"`.
135 pub display_name: String,
136 /// Approximate memory footprint, e.g. `"~941 MB (GGUF Q4_K_M)"`.
137 pub approx_memory: String,
138 }
139
140 impl From<OndeGgufModelConfig> for GgufModelConfig {
141 fn from(c: OndeGgufModelConfig) -> Self {
142 Self {
143 model_id: c.model_id,
144 files: c.files,
145 tok_model_id: c.tok_model_id,
146 display_name: c.display_name,
147 approx_memory: c.approx_memory,
148 }
149 }
150 }
151
152 impl From<GgufModelConfig> for OndeGgufModelConfig {
153 fn from(c: GgufModelConfig) -> Self {
154 Self {
155 model_id: c.model_id,
156 files: c.files,
157 tok_model_id: c.tok_model_id,
158 display_name: c.display_name,
159 approx_memory: c.approx_memory,
160 chat_template: None,
161 }
162 }
163 }
164
165 // ── Mirror: UqffModelConfig ──────────────────────────────────────────────────
166
167 /// Configuration for loading a pre-quantised UQFF model.
168 #[derive(Debug, Clone)]
169 pub struct UqffModelConfig {
170 /// Base HuggingFace repository ID or local model directory.
171 pub model_id: String,
172 /// UQFF shard filename(s), first shard, or shorthand such as `q4k` / `4`.
173 pub files: Vec<String>,
174 /// Human-friendly display name.
175 pub display_name: String,
176 /// Approximate memory footprint.
177 pub approx_memory: String,
178 /// Optional Jinja chat template override.
179 pub chat_template: Option<String>,
180 }
181
182 impl From<OndeUqffModelConfig> for UqffModelConfig {
183 fn from(c: OndeUqffModelConfig) -> Self {
184 Self {
185 model_id: c.model_id,
186 files: c.files,
187 display_name: c.display_name,
188 approx_memory: c.approx_memory,
189 chat_template: c.chat_template,
190 }
191 }
192 }
193
194 impl From<UqffModelConfig> for OndeUqffModelConfig {
195 fn from(c: UqffModelConfig) -> Self {
196 Self {
197 model_id: c.model_id,
198 files: c.files,
199 display_name: c.display_name,
200 approx_memory: c.approx_memory,
201 chat_template: c.chat_template,
202 }
203 }
204 }
205
206 // ── Mirror: ToolCallInfo ────────────────────────────────────────────────────
207
208 /// Structured tool-call request emitted by the model.
209 #[derive(Debug, Clone)]
210 pub struct ToolCallInfo {
211 pub id: String,
212 pub function_name: String,
213 pub arguments: String,
214 }
215
216 impl From<OndeToolCallInfo> for ToolCallInfo {
217 fn from(t: OndeToolCallInfo) -> Self {
218 Self {
219 id: t.id,
220 function_name: t.function_name,
221 arguments: t.arguments,
222 }
223 }
224 }
225
226 // ── Mirror: InferenceResult ───────────────────────────────────────────────────
227
228 /// The result of a completed inference request.
229 #[derive(Debug, Clone)]
230 pub struct InferenceResult {
231 /// The generated assistant reply text.
232 pub text: String,
233 /// Wall-clock inference duration in seconds.
234 pub duration_secs: f64,
235 /// Human-readable duration string, e.g. `"4.5s"` or `"2m 3.1s"`.
236 pub duration_display: String,
237 /// Finish reason reported by the model, e.g. `"stop"` or `"length"`.
238 pub finish_reason: String,
239 /// Tool calls requested by the model (empty if none).
240 pub tool_calls: Vec<ToolCallInfo>,
241 }
242
243 impl From<OndeInferenceResult> for InferenceResult {
244 fn from(r: OndeInferenceResult) -> Self {
245 Self {
246 text: r.text,
247 duration_secs: r.duration_secs,
248 duration_display: r.duration_display,
249 finish_reason: r.finish_reason,
250 tool_calls: r.tool_calls.into_iter().map(ToolCallInfo::from).collect(),
251 }
252 }
253 }
254
255 // ── Mirror: StreamChunk ───────────────────────────────────────────────────────
256
257 /// A single streaming token chunk emitted during inference.
258 #[derive(Debug, Clone)]
259 pub struct StreamChunk {
260 /// The token text delta (may be empty on the final chunk).
261 pub delta: String,
262 /// `true` when this is the last chunk in the stream.
263 pub done: bool,
264 /// Finish reason (set only on the final chunk).
265 pub finish_reason: Option<String>,
266 }
267
268 impl From<OndeStreamChunk> for StreamChunk {
269 fn from(c: OndeStreamChunk) -> Self {
270 Self {
271 delta: c.delta,
272 done: c.done,
273 finish_reason: c.finish_reason,
274 }
275 }
276 }
277
278 // ── Mirror: EngineStatus ──────────────────────────────────────────────────────
279
280 /// Lifecycle status of the inference engine.
281 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
282 pub enum EngineStatus {
283 Unloaded,
284 Loading,
285 Ready,
286 Generating,
287 Error,
288 }
289
290 impl From<OndeEngineStatus> for EngineStatus {
291 fn from(s: OndeEngineStatus) -> Self {
292 match s {
293 OndeEngineStatus::Unloaded => EngineStatus::Unloaded,
294 OndeEngineStatus::Loading => EngineStatus::Loading,
295 OndeEngineStatus::Ready => EngineStatus::Ready,
296 OndeEngineStatus::Generating => EngineStatus::Generating,
297 OndeEngineStatus::Error => EngineStatus::Error,
298 }
299 }
300 }
301
302 // ── Mirror: EngineInfo ────────────────────────────────────────────────────────
303
304 /// A point-in-time snapshot of the engine's state.
305 #[derive(Debug, Clone)]
306 pub struct EngineInfo {
307 pub status: EngineStatus,
308 /// Display name of the currently loaded model, if any.
309 pub model_name: Option<String>,
310 /// Approximate memory footprint of the loaded model, if any.
311 pub approx_memory: Option<String>,
312 /// Number of conversation turns in the current history.
313 pub history_length: u64,
314 }
315
316 impl From<OndeEngineInfo> for EngineInfo {
317 fn from(i: OndeEngineInfo) -> Self {
318 Self {
319 status: i.status.into(),
320 model_name: i.model_name,
321 approx_memory: i.approx_memory,
322 history_length: i.history_length,
323 }
324 }
325 }
326
327 // ── Error type ────────────────────────────────────────────────────────────────
328
329 /// Bridge error type mapping from [`onde::inference::InferenceError`].
330 #[derive(Debug, thiserror::Error)]
331 pub enum OndeError {
332 #[error("No model loaded — call load_gguf_model first")]
333 NoModelLoaded,
334
335 #[error("Model already loaded: {model_name}")]
336 AlreadyLoaded { model_name: String },
337
338 #[error("Failed to build model: {reason}")]
339 ModelBuild { reason: String },
340
341 #[error("Inference failed: {reason}")]
342 Inference { reason: String },
343
344 #[error("Operation was cancelled")]
345 Cancelled,
346
347 #[error("{reason}")]
348 Other { reason: String },
349 }
350
351 impl From<InferenceError> for OndeError {
352 fn from(e: InferenceError) -> Self {
353 match e {
354 InferenceError::NoModelLoaded => OndeError::NoModelLoaded,
355 InferenceError::AlreadyLoaded { model_name } => OndeError::AlreadyLoaded { model_name },
356 InferenceError::ModelBuild { reason } => OndeError::ModelBuild { reason },
357 InferenceError::Inference { reason } => OndeError::Inference { reason },
358 InferenceError::Cancelled => OndeError::Cancelled,
359 InferenceError::Other { reason } => OndeError::Other { reason },
360 }
361 }
362 }
363
364 // ── OndeChatEngine ────────────────────────────────────────────────────────────
365
366 /// Opaque wrapper around `onde::inference::ChatEngine`.
367 ///
368 /// Dart/Flutter receives this as an opaque `RustOpaque<OndeChatEngine>` handle.
369 /// All methods delegate directly to the inner `ChatEngine`.
370 pub struct OndeChatEngine {
371 inner: ChatEngine,
372 }
373
374 impl OndeChatEngine {
375 // ── Construction ──────────────────────────────────────────────────────
376
377 /// Create a new engine with no model loaded.
378 #[frb(sync)]
379 pub fn new() -> OndeChatEngine {
380 OndeChatEngine {
381 inner: ChatEngine::new(),
382 }
383 }
384
385 // ── Model lifecycle ───────────────────────────────────────────────────
386
387 /// Load a GGUF model into the engine.
388 ///
389 /// Returns the load duration in seconds on success.
390 pub async fn load_gguf_model(
391 &self,
392 config: GgufModelConfig,
393 system_prompt: Option<String>,
394 sampling: Option<SamplingConfig>,
395 ) -> Result<f64, OndeError> {
396 let elapsed = self
397 .inner
398 .load_gguf_model(config.into(), system_prompt, sampling.map(Into::into))
399 .await
400 .map_err(OndeError::from)?;
401 Ok(elapsed.as_secs_f64())
402 }
403
404 /// Load a UQFF model into the engine.
405 ///
406 /// Returns the load duration in seconds on success.
407 pub async fn load_uqff_model(
408 &self,
409 config: UqffModelConfig,
410 system_prompt: Option<String>,
411 sampling: Option<SamplingConfig>,
412 ) -> Result<f64, OndeError> {
413 let elapsed = self
414 .inner
415 .load_uqff_model(config.into(), system_prompt, sampling.map(Into::into))
416 .await
417 .map_err(OndeError::from)?;
418 Ok(elapsed.as_secs_f64())
419 }
420
421 /// Load the model assigned to this Onde app from the dashboard.
422 ///
423 /// Returns the wall-clock seconds taken to load the model.
424 pub async fn load_assigned_model(
425 &self,
426 app_id: String,
427 app_secret: String,
428 system_prompt: Option<String>,
429 sampling: Option<SamplingConfig>,
430 ) -> Result<f64, OndeError> {
431 let elapsed = self
432 .inner
433 .load_assigned_model(
434 smbcloud_gresiq_sdk::Environment::Production,
435 &app_id,
436 &app_secret,
437 system_prompt,
438 sampling.map(Into::into),
439 )
440 .await
441 .map_err(OndeError::from)?;
442 Ok(elapsed.as_secs_f64())
443 }
444
445 /// Unload the current model and release its memory.
446 ///
447 /// Returns the display name of the model that was unloaded, or `None`
448 /// if no model was loaded.
449 pub async fn unload_model(&self) -> Option<String> {
450 self.inner.unload_model().await
451 }
452
453 /// Returns `true` if a model is currently loaded and ready.
454 pub async fn is_loaded(&self) -> bool {
455 self.inner.is_loaded().await
456 }
457
458 /// Returns a point-in-time snapshot of the engine's state.
459 pub async fn info(&self) -> EngineInfo {
460 self.inner.info().await.into()
461 }
462
463 // ── Configuration ─────────────────────────────────────────────────────
464
465 /// Set or replace the system prompt.
466 ///
467 /// The prompt is prepended to every inference request and is **not**
468 /// stored in the conversation history.
469 pub async fn set_system_prompt(&self, prompt: String) {
470 self.inner.set_system_prompt(prompt).await;
471 }
472
473 /// Clear the current system prompt.
474 pub async fn clear_system_prompt(&self) {
475 self.inner.clear_system_prompt().await;
476 }
477
478 /// Replace the sampling configuration used for all subsequent inference.
479 pub async fn set_sampling(&self, sampling: SamplingConfig) {
480 self.inner.set_sampling(sampling.into()).await;
481 }
482
483 // ── Conversation history ──────────────────────────────────────────────
484
485 /// Returns a clone of the full conversation history.
486 pub async fn history(&self) -> Vec<ChatMessage> {
487 self.inner
488 .history()
489 .await
490 .into_iter()
491 .map(ChatMessage::from)
492 .collect()
493 }
494
495 /// Clear the conversation history while keeping the model loaded.
496 ///
497 /// Returns the number of turns that were removed.
498 pub async fn clear_history(&self) -> u64 {
499 self.inner.clear_history().await as u64
500 }
501
502 /// Append a message to history without running inference.
503 ///
504 /// Useful for restoring a saved conversation or injecting context.
505 pub async fn push_history(&self, message: ChatMessage) {
506 self.inner.push_history(message.into()).await;
507 }
508
509 // ── Non-streaming inference ───────────────────────────────────────────
510
511 /// Send a user message and receive a complete assistant reply.
512 ///
513 /// Both the user message and assistant reply are appended to the
514 /// conversation history on success.
515 pub async fn send_message(&self, message: String) -> Result<InferenceResult, OndeError> {
516 self.inner
517 .send_message(message)
518 .await
519 .map(InferenceResult::from)
520 .map_err(OndeError::from)
521 }
522
523 /// One-shot generation from an explicit message list.
524 ///
525 /// Does **not** modify the conversation history.
526 pub async fn generate(
527 &self,
528 messages: Vec<ChatMessage>,
529 sampling: Option<SamplingConfig>,
530 ) -> Result<InferenceResult, OndeError> {
531 let onde_messages: Vec<OndeChatMessage> =
532 messages.into_iter().map(OndeChatMessage::from).collect();
533 self.inner
534 .generate(onde_messages, sampling.map(Into::into))
535 .await
536 .map(InferenceResult::from)
537 .map_err(OndeError::from)
538 }
539
540 // ── Streaming inference ───────────────────────────────────────────────
541
542 /// Stream a reply token-by-token.
543 ///
544 /// Each [`StreamChunk`] carries a `delta` string. The final chunk has
545 /// `done == true`. The `sink` is closed automatically when the stream
546 /// ends or if the receiver is dropped.
547 pub async fn stream_message(
548 &self,
549 message: String,
550 sink: StreamSink<StreamChunk>,
551 ) -> Result<(), OndeError> {
552 let mut rx = self
553 .inner
554 .stream_message(message)
555 .await
556 .map_err(OndeError::from)?;
557
558 while let Some(chunk) = rx.recv().await {
559 let bridge_chunk = StreamChunk::from(chunk);
560 let done = bridge_chunk.done;
561 let _ = sink.add(bridge_chunk);
562 if done {
563 break;
564 }
565 }
566
567 Ok(())
568 }
569 }
570
571 // ── Free functions ────────────────────────────────────────────────────────────
572
573 /// Return the platform-appropriate default `GgufModelConfig`.
574 ///
575 /// Selects the 1.5B model on iOS / tvOS / Android and the 3B model on
576 /// macOS / Windows / Linux.
577 #[frb(sync)]
578 pub fn default_model_config() -> GgufModelConfig {
579 OndeGgufModelConfig::platform_default().into()
580 }
581
582 /// Build a generic `UqffModelConfig`.
583 #[frb(sync)]
584 pub fn uqff_model_config(
585 model_id: String,
586 files: Vec<String>,
587 display_name: String,
588 approx_memory: String,
589 chat_template: Option<String>,
590 ) -> UqffModelConfig {
591 OndeUqffModelConfig {
592 model_id,
593 files,
594 display_name,
595 approx_memory,
596 chat_template,
597 }
598 .into()
599 }
600
601 /// `GgufModelConfig` for Qwen 2.5 1.5B Instruct Q4_K_M (~941 MB).
602 #[frb(sync)]
603 pub fn qwen25_1_5b_config() -> GgufModelConfig {
604 OndeGgufModelConfig::qwen25_1_5b().into()
605 }
606
607 /// `GgufModelConfig` for Qwen 2.5 3B Instruct Q4_K_M (~1.93 GB).
608 #[frb(sync)]
609 pub fn qwen25_3b_config() -> GgufModelConfig {
610 OndeGgufModelConfig::qwen25_3b().into()
611 }
612
613 /// `GgufModelConfig` for Qwen 2.5 Coder 1.5B Instruct Q4_K_M (~941 MB).
614 #[frb(sync)]
615 pub fn qwen25_coder_1_5b_config() -> GgufModelConfig {
616 OndeGgufModelConfig::qwen25_coder_1_5b().into()
617 }
618
619 /// `GgufModelConfig` for Qwen 2.5 Coder 3B Instruct Q4_K_M (~1.93 GB).
620 #[frb(sync)]
621 pub fn qwen25_coder_3b_config() -> GgufModelConfig {
622 OndeGgufModelConfig::qwen25_coder_3b().into()
623 }
624
625 // ── Qwen 3 family ────────────────────────────────────────────────────────────
626
627 /// `GgufModelConfig` for Qwen 3 0.6B Q4_K_M (~0.5 GB).
628 #[frb(sync)]
629 pub fn qwen3_0_6b_config() -> GgufModelConfig {
630 OndeGgufModelConfig::qwen3_0_6b().into()
631 }
632
633 /// `GgufModelConfig` for Qwen 3 1.7B Q4_K_M (~1.3 GB).
634 #[frb(sync)]
635 pub fn qwen3_1_7b_config() -> GgufModelConfig {
636 OndeGgufModelConfig::qwen3_1_7b().into()
637 }
638
639 /// `GgufModelConfig` for Qwen 3 4B Q4_K_M (~2.7 GB).
640 #[frb(sync)]
641 pub fn qwen3_4b_config() -> GgufModelConfig {
642 OndeGgufModelConfig::qwen3_4b().into()
643 }
644
645 /// `GgufModelConfig` for Qwen 3 8B Q4_K_M (~5 GB).
646 #[frb(sync)]
647 pub fn qwen3_8b_config() -> GgufModelConfig {
648 OndeGgufModelConfig::qwen3_8b().into()
649 }
650
651 /// `GgufModelConfig` for Qwen 3 14B Q4_K_M (~8.4 GB).
652 #[frb(sync)]
653 pub fn qwen3_14b_config() -> GgufModelConfig {
654 OndeGgufModelConfig::qwen3_14b().into()
655 }
656
657 /// `GgufModelConfig` for Qwen 3 32B Q4_K_M (~19.8 GB).
658 #[frb(sync)]
659 pub fn qwen3_32b_config() -> GgufModelConfig {
660 OndeGgufModelConfig::qwen3_32b().into()
661 }
662
663 /// `GgufModelConfig` for Qwen 3 4B Instruct 2507 Q4_K_M (~2.5 GB).
664 #[frb(sync)]
665 pub fn qwen3_4b_instruct_2507_config() -> GgufModelConfig {
666 OndeGgufModelConfig::qwen3_4b_instruct_2507().into()
667 }
668
669 /// `GgufModelConfig` for Qwen 3 4B Thinking 2507 Q4_K_M (~2.5 GB).
670 #[frb(sync)]
671 pub fn qwen3_4b_thinking_2507_config() -> GgufModelConfig {
672 OndeGgufModelConfig::qwen3_4b_thinking_2507().into()
673 }
674
675 /// `GgufModelConfig` for Qwen 3 30B-A3B Instruct 2507 (MoE) Q4_K_M (~18.6 GB).
676 #[frb(sync)]
677 pub fn qwen3_30b_a3b_instruct_2507_config() -> GgufModelConfig {
678 OndeGgufModelConfig::qwen3_30b_a3b_instruct_2507().into()
679 }
680
681 /// Default sampling config: `temperature=0.7`, `top_p=0.95`, `max_tokens=512`.
682 #[frb(sync)]
683 pub fn default_sampling_config() -> SamplingConfig {
684 OndeSamplingConfig::default().into()
685 }
686
687 /// Deterministic sampling config: `temperature=0.0`, greedy, `max_tokens=512`.
688 #[frb(sync)]
689 pub fn deterministic_sampling_config() -> SamplingConfig {
690 OndeSamplingConfig::deterministic().into()
691 }
692
693 /// Mobile sampling config: `temperature=0.7`, `top_p=0.95`, `max_tokens=128`.
694 #[frb(sync)]
695 pub fn mobile_sampling_config() -> SamplingConfig {
696 OndeSamplingConfig::mobile().into()
697 }
698
699 /// Seed the HuggingFace cache environment for sandboxed platforms.
700 ///
701 /// On iOS (and Android), the default `~/.cache/huggingface/hub` path is outside
702 /// the app sandbox and writes fail with `Operation not permitted (os error 1)`.
703 ///
704 /// Call this **once** at startup — before any `load_gguf_model` or
705 /// `download_model` — with the app's writable data directory.
706 ///
707 /// # Dart usage
708 ///
709 /// ```dart
710 /// import 'package:path_provider/path_provider.dart';
711 ///
712 /// final dir = await getApplicationSupportDirectory();
713 /// configureCacheDir(appDataDir: dir.path);
714 /// ```
715 ///
716 /// Internally this:
717 /// 1. Creates `<app_data_dir>/huggingface/hub/` if it doesn't exist.
718 /// 2. Sets the `HF_HOME` and `HF_HUB_CACHE` environment variables.
719 /// 3. Seeds `mistralrs_core::GLOBAL_HF_CACHE` so the model loader never
720 /// falls back to `Cache::default()`.
721 #[frb(sync)]
722 pub fn configure_cache_dir(app_data_dir: String) -> Result<(), OndeError> {
723 use std::fs;
724 use std::path::PathBuf;
725
726 let data_dir = PathBuf::from(&app_data_dir);
727 let hf_home = data_dir.join("models");
728 let hf_hub_cache = hf_home.join("hub");
729
730 fs::create_dir_all(&hf_hub_cache).map_err(|e| OndeError::Other {
731 reason: format!(
732 "Cannot create HF cache dir {}: {}",
733 hf_hub_cache.display(),
734 e
735 ),
736 })?;
737
738 // Set env vars BEFORE any hf-hub or mistralrs call. Both libraries
739 // read these to resolve the cache directory — no need to touch
740 // GLOBAL_HF_CACHE directly (that's seeded lazily by load_gguf_model
741 // on Android/iOS when it reads HF_HOME back).
742 std::env::set_var("HF_HOME", &hf_home);
743 std::env::set_var("HF_HUB_CACHE", &hf_hub_cache);
744
745 // Pre-create tmp dir — iOS sandbox also restricts the system TMPDIR.
746 let tmp_dir = data_dir.join("tmp");
747 let _ = fs::create_dir_all(&tmp_dir);
748 std::env::set_var("TMPDIR", &tmp_dir);
749
750 log::info!(
751 "HF cache configured: HF_HOME={}, HF_HUB_CACHE={}",
752 hf_home.display(),
753 hf_hub_cache.display(),
754 );
755
756 Ok(())
757 }