|
1
|
//! # Onde |
|
2
|
//! |
|
3
|
//! **On-device chat inference for cross-platform apps.** |
|
4
|
//! |
|
5
|
//! Run LLM chat locally — no cloud, no latency, no data leaving the device. |
|
6
|
//! |
|
7
|
//! Onde wraps [mistral.rs](https://github.com/EricLBuehler/mistral.rs) with a |
|
8
|
//! unified API for model discovery, HuggingFace Hub downloads, cache |
|
9
|
//! management, and GPU acceleration across every platform. |
|
10
|
//! |
|
11
|
//! Built by [Onde Inference](https://ondeinference.com) |
|
12
|
//! |
|
13
|
//! ## Modules |
|
14
|
//! |
|
15
|
//! - [`hf_cache`] — HuggingFace Hub cache inspection, repair, and model |
|
16
|
//! download with a framework-agnostic progress-callback API. |
|
17
|
//! - [`inference`] — Chat inference engine, UniFFI FFI wrapper, model metadata, |
|
18
|
//! and HuggingFace token resolution. |
|
19
|
//! |
|
20
|
//! ## Re-exports |
|
21
|
//! |
|
22
|
//! `mistralrs`, `hf_hub`, and `mistralrs_core` are re-exported so that apps |
|
23
|
//! depending on `onde` do not need their own direct dependency on those crates. |
|
24
|
//! Access them as `onde::mistralrs`, `onde::hf_hub`, and `onde::mistralrs_core`. |
|
25
|
//! |
|
26
|
//! ## Example |
|
27
|
//! |
|
28
|
//! ```rust,ignore |
|
29
|
//! use onde::inference::ChatEngine; |
|
30
|
//! use onde::inference::GgufModelConfig; |
|
31
|
//! |
|
32
|
//! let engine = ChatEngine::new(); |
|
33
|
//! engine |
|
34
|
//! .load_gguf_model( |
|
35
|
//! GgufModelConfig::platform_default(), |
|
36
|
//! Some("You are a helpful assistant.".into()), |
|
37
|
//! None, |
|
38
|
//! ) |
|
39
|
//! .await?; |
|
40
|
//! |
|
41
|
//! let result = engine.send_message("Hello!").await?; |
|
42
|
//! println!("{}", result.text); |
|
43
|
//! ``` |
|
44
|
|
|
45
|
pub mod hf_cache; |
|
46
|
|
|
47
|
pub mod inference; |
|
48
|
pub mod pulse; |
|
49
|
|
|
50
|
static PANIC_HOOK_ONCE: std::sync::Once = std::sync::Once::new(); |
|
51
|
|
|
52
|
pub(crate) fn install_panic_hook_once() { |
|
53
|
PANIC_HOOK_ONCE.call_once(|| { |
|
54
|
std::panic::set_hook(Box::new(|panic_info| { |
|
55
|
let location = panic_info |
|
56
|
.location() |
|
57
|
.map(|loc| format!("{}:{}:{}", loc.file(), loc.line(), loc.column())) |
|
58
|
.unwrap_or_else(|| "<unknown>".to_string()); |
|
59
|
|
|
60
|
let message = if let Some(msg) = panic_info.payload().downcast_ref::<&str>() { |
|
61
|
(*msg).to_string() |
|
62
|
} else if let Some(msg) = panic_info.payload().downcast_ref::<String>() { |
|
63
|
msg.clone() |
|
64
|
} else { |
|
65
|
"<non-string panic payload>".to_string() |
|
66
|
}; |
|
67
|
|
|
68
|
log::error!("Rust panic at {}: {}", location, message); |
|
69
|
})); |
|
70
|
}); |
|
71
|
} |
|
72
|
|
|
73
|
uniffi::setup_scaffolding!(); |
|
74
|
|
|
75
|
// Re-export mistralrs for every platform that onde supports. |
|
76
|
// Apps use `onde::mistralrs::Model` etc. instead of declaring a direct dep. |
|
77
|
#[cfg(any( |
|
78
|
target_os = "macos", |
|
79
|
target_os = "ios", |
|
80
|
target_os = "tvos", |
|
81
|
target_os = "visionos", |
|
82
|
target_os = "watchos", |
|
83
|
target_os = "windows", |
|
84
|
target_os = "linux", |
|
85
|
target_os = "android" |
|
86
|
))] |
|
87
|
pub use mistralrs; |
|
88
|
|
|
89
|
// Re-exports needed for the GLOBAL_HF_CACHE workaround on sandboxed platforms. |
|
90
|
// On iOS/tvOS `~/.cache` is outside the container; on Android `dirs::home_dir()` |
|
91
|
// panics. All three need `hf_hub::Cache` + `mistralrs_core::GLOBAL_HF_CACHE`. |
|
92
|
#[cfg(any(target_os = "android", target_os = "ios", target_os = "tvos"))] |
|
93
|
pub use hf_hub; |
|
94
|
#[cfg(any(target_os = "android", target_os = "ios", target_os = "tvos"))] |
|
95
|
pub use mistralrs_core; |