|
1
|
//! On-device LLM inference powered by [mistral.rs](https://github.com/EricLBuehler/mistral.rs). |
|
2
|
//! |
|
3
|
//! This module provides a high-level, framework-agnostic API for running |
|
4
|
//! LLM chat inference on-device across macOS, iOS, Windows, Linux, and Android. |
|
5
|
//! |
|
6
|
//! # Modules |
|
7
|
//! |
|
8
|
//! - [`engine`] — The main [`ChatEngine`](engine::ChatEngine) that wraps mistral.rs |
|
9
|
//! with conversation history, sampling config, and model lifecycle management. |
|
10
|
//! - [`types`] — Shared types ([`ChatMessage`](types::ChatMessage), |
|
11
|
//! [`SamplingConfig`](types::SamplingConfig), [`InferenceResult`](types::InferenceResult), etc.) |
|
12
|
//! used by the engine and any UI layer. |
|
13
|
//! - [`models`] — Model ID constants and rich metadata for all supported models. |
|
14
|
//! - [`token`] — HuggingFace token resolution (build-time literal or on-disk cache). |
|
15
|
//! |
|
16
|
//! # Quick Start |
|
17
|
//! |
|
18
|
//! ```rust,ignore |
|
19
|
//! use onde::inference::engine::ChatEngine; |
|
20
|
//! use onde::inference::types::*; |
|
21
|
//! |
|
22
|
//! // Create engine and load the platform-appropriate default model. |
|
23
|
//! let engine = ChatEngine::new(); |
|
24
|
//! engine.load_gguf_model( |
|
25
|
//! GgufModelConfig::platform_default(), |
|
26
|
//! Some("You are a helpful assistant.".into()), |
|
27
|
//! None, // platform-aware sampling defaults |
|
28
|
//! ).await?; |
|
29
|
//! |
|
30
|
//! // Multi-turn chat (history is managed automatically). |
|
31
|
//! let reply = engine.send_message("What is Rust's ownership model?").await?; |
|
32
|
//! println!("{}", reply.text); |
|
33
|
//! |
|
34
|
//! let follow_up = engine.send_message("Can you give me an example?").await?; |
|
35
|
//! println!("{}", follow_up.text); |
|
36
|
//! |
|
37
|
//! // One-shot generation (does NOT modify conversation history). |
|
38
|
//! let enhanced = engine.generate( |
|
39
|
//! vec![ChatMessage::user("Expand this into a detailed prompt: a cat in space")], |
|
40
|
//! Some(SamplingConfig::deterministic()), |
|
41
|
//! ).await?; |
|
42
|
//! println!("{}", enhanced.text); |
|
43
|
//! |
|
44
|
//! // Streaming inference. |
|
45
|
//! let mut rx = engine.stream_message("Tell me a story.").await?; |
|
46
|
//! while let Some(chunk) = rx.recv().await { |
|
47
|
//! if !chunk.delta.is_empty() { |
|
48
|
//! print!("{}", chunk.delta); |
|
49
|
//! } |
|
50
|
//! if chunk.done { |
|
51
|
//! break; |
|
52
|
//! } |
|
53
|
//! } |
|
54
|
//! |
|
55
|
//! // Cleanup. |
|
56
|
//! engine.unload_model().await; |
|
57
|
//! ``` |
|
58
|
|
|
59
|
pub mod engine; |
|
60
|
pub mod ffi; |
|
61
|
pub mod models; |
|
62
|
pub mod token; |
|
63
|
pub mod types; |
|
64
|
|
|
65
|
// Re-export the most commonly used items at the `inference::` level for |
|
66
|
// ergonomic imports like `use onde::inference::{ChatEngine, ChatMessage, ...}`. |
|
67
|
pub use engine::ChatEngine; |
|
68
|
pub use ffi::{OndeChatEngine, StreamChunkListener}; |
|
69
|
pub use types::{ |
|
70
|
ChatMessage, |
|
71
|
ChatRole, |
|
72
|
EngineInfo, |
|
73
|
EngineStatus, |
|
74
|
GgufModelConfig, |
|
75
|
InferenceError, |
|
76
|
InferenceResult, |
|
77
|
IsqModelConfig, |
|
78
|
SamplingConfig, |
|
79
|
StreamChunk, |
|
80
|
// Tool calling |
|
81
|
ToolAwareResult, |
|
82
|
ToolCallInfo, |
|
83
|
ToolCallRequest, |
|
84
|
ToolDefinition, |
|
85
|
ToolResult, |
|
86
|
}; |