| 1 | //! Chat feature module: fully on-device LLM inference for Siti AI. |
| 2 | //! |
| 3 | //! Each Tauri command lives in its own file (`command_*.rs`). Shared types, |
| 4 | //! the cached engine instance, model/sampling configuration, and helpers are |
| 5 | //! kept here in `mod.rs` so every command file can `use super::*`. |
| 6 | //! |
| 7 | //! Model loading, inference, and history management are delegated to |
| 8 | //! [`onde::inference::ChatEngine`], the shared on-device inference engine |
| 9 | //! from the `onde` crate. Siti runs the engine **fully offline**: it loads |
| 10 | //! the platform-default GGUF model directly (no operator-assigned model, no |
| 11 | //! app credentials) and pulse telemetry is disabled during app setup. |
| 12 | |
| 13 | // ── Command submodules (one Tauri command per file) ────────────────────────── |
| 14 | |
| 15 | pub mod command_clear_history; |
| 16 | pub mod command_get_history; |
| 17 | pub mod command_get_status; |
| 18 | pub mod command_list_models; |
| 19 | pub mod command_load_model; |
| 20 | pub mod command_send_message; |
| 21 | pub mod command_set_model; |
| 22 | pub mod command_unload_model; |
| 23 | |
| 24 | // ── Re-exports so lib.rs can pull in commands with a flat path ─────────────── |
| 25 | |
| 26 | pub use command_clear_history::chat_clear_history; |
| 27 | pub use command_get_history::chat_get_history; |
| 28 | pub use command_get_status::chat_get_status; |
| 29 | pub use command_list_models::chat_list_models; |
| 30 | pub use command_load_model::chat_load_model; |
| 31 | pub use command_send_message::chat_send_message; |
| 32 | pub use command_set_model::chat_set_model; |
| 33 | pub use command_unload_model::chat_unload_model; |
| 34 | |
| 35 | // ── Imports shared with submodules via `super::` ───────────────────────────── |
| 36 | |
| 37 | use { |
| 38 | crate::constants::ChatStatus, |
| 39 | serde::{Deserialize, Serialize}, |
| 40 | }; |
| 41 | |
| 42 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 43 | use { |
| 44 | crate::events::{EVENT_CHAT_REPLY, EVENT_CHAT_STATUS_CHANGED}, |
| 45 | log::error, |
| 46 | once_cell::sync::Lazy, |
| 47 | onde::inference::{ChatEngine, GgufModelConfig, SamplingConfig}, |
| 48 | tauri::{AppHandle, Emitter}, |
| 49 | }; |
| 50 | |
| 51 | // ── Response / payload types ───────────────────────────────────────────────── |
| 52 | |
| 53 | /// Payload emitted with the `chat_status_changed` event. |
| 54 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 55 | pub struct ChatStatusPayload { |
| 56 | pub status: ChatStatus, |
| 57 | pub model_name: Option<String>, |
| 58 | pub error: Option<String>, |
| 59 | } |
| 60 | |
| 61 | /// A single message in the conversation, returned to the frontend. |
| 62 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 63 | pub struct ChatMessagePayload { |
| 64 | pub role: String, |
| 65 | pub content: String, |
| 66 | } |
| 67 | |
| 68 | /// Event payload emitted as `chat_reply` once inference completes. |
| 69 | /// |
| 70 | /// Using an event instead of a blocking command return prevents the WebView |
| 71 | /// from garbage-collecting the JS invoke-callback before the slow on-device |
| 72 | /// inference finishes. |
| 73 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 74 | pub struct ChatReplyPayload { |
| 75 | /// The assistant's reply text, or `None` on error. |
| 76 | pub reply: Option<String>, |
| 77 | /// Human-readable inference duration, or `None` on error. |
| 78 | pub duration: Option<String>, |
| 79 | /// Error message if inference failed, or `None` on success. |
| 80 | pub error: Option<String>, |
| 81 | } |
| 82 | |
| 83 | /// Response returned by `chat_get_status`. |
| 84 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 85 | pub struct ChatStatusResponse { |
| 86 | pub status: ChatStatus, |
| 87 | pub model_name: Option<String>, |
| 88 | pub approx_memory: Option<String>, |
| 89 | pub history_length: usize, |
| 90 | } |
| 91 | |
| 92 | /// A selectable on-device model, returned by `chat_list_models` and rendered |
| 93 | /// in the Settings model dropdown. |
| 94 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 95 | pub struct ModelInfo { |
| 96 | /// HuggingFace repo id; the value passed back to `chat_set_model`. |
| 97 | pub id: String, |
| 98 | /// Human-friendly display name, e.g. `"Qwen 2.5 1.5B (GGUF)"`. |
| 99 | pub name: String, |
| 100 | /// Publisher / organisation, e.g. `"Qwen / Alibaba"`. |
| 101 | pub org: String, |
| 102 | /// Short description of the model's purpose and footprint. |
| 103 | pub description: String, |
| 104 | /// Approximate memory footprint, e.g. `"~941 MB (GGUF Q4_K_M)"`. |
| 105 | pub approx_memory: String, |
| 106 | /// Approximate on-disk download size in bytes (for display). |
| 107 | pub size_bytes: Option<u64>, |
| 108 | /// Whether this model is the one currently selected for Siti. |
| 109 | pub is_selected: bool, |
| 110 | } |
| 111 | |
| 112 | // ── Shared ChatEngine instance ─────────────────────────────────────────────── |
| 113 | // |
| 114 | // `ChatEngine` from `onde` handles all model lifecycle, history, and inference. |
| 115 | // It is `Send + Sync` and manages its own internal mutex. |
| 116 | |
| 117 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 118 | pub(crate) static ENGINE: Lazy<ChatEngine> = Lazy::new(ChatEngine::new); |
| 119 | |
| 120 | // ── Model selection & configuration ────────────────────────────────────────── |
| 121 | // |
| 122 | // The user can switch models from Settings (`chat_set_model`). The chosen |
| 123 | // HuggingFace repo id is held in `SELECTED_MODEL` and drives both the model |
| 124 | // config and the sampling config. When nothing has been chosen yet, Siti |
| 125 | // falls back to a general-purpose chat model appropriate for the platform: |
| 126 | // - iOS / Android → Qwen 2.5 1.5B (~941 MB, fits mobile memory budgets) |
| 127 | // - macOS → Qwen 2.5 3B (~1.93 GB, more headroom on desktop) |
| 128 | // |
| 129 | // `config_for_model_id` handles per-platform details (e.g. `tok_model_id` on |
| 130 | // Android) via the onde `GgufModelConfig` constructors. |
| 131 | |
| 132 | /// The model id currently selected for Siti. Defaults to the platform- |
| 133 | /// appropriate general chat model; updated by `chat_set_model`. |
| 134 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 135 | pub(crate) static SELECTED_MODEL: Lazy<std::sync::Mutex<String>> = |
| 136 | Lazy::new(|| std::sync::Mutex::new(siti_default_config().model_id)); |
| 137 | |
| 138 | /// Siti's default general-purpose chat model for the current platform. |
| 139 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 140 | pub(crate) fn siti_default_config() -> GgufModelConfig { |
| 141 | if cfg!(any(target_os = "ios", target_os = "android")) { |
| 142 | GgufModelConfig::qwen25_1_5b() |
| 143 | } else { |
| 144 | GgufModelConfig::qwen25_3b() |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | /// Map a HuggingFace repo id to a loadable [`GgufModelConfig`], or `None` if |
| 149 | /// the id is not a model the onde engine knows how to load. |
| 150 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 151 | pub(crate) fn config_for_model_id(id: &str) -> Option<GgufModelConfig> { |
| 152 | use crate::inference::models as m; |
| 153 | let cfg = if id == m::BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF { |
| 154 | GgufModelConfig::qwen25_0_5b() |
| 155 | } else if id == m::BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF { |
| 156 | GgufModelConfig::qwen25_1_5b() |
| 157 | } else if id == m::BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF { |
| 158 | GgufModelConfig::qwen25_3b() |
| 159 | } else if id == m::BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF { |
| 160 | GgufModelConfig::qwen25_coder_1_5b() |
| 161 | } else if id == m::BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF { |
| 162 | GgufModelConfig::qwen25_coder_3b() |
| 163 | } else if id == m::BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF { |
| 164 | GgufModelConfig::qwen25_coder_7b() |
| 165 | } else if id == m::BARTOWSKI_QWEN3_1_7B_GGUF { |
| 166 | GgufModelConfig::qwen3_1_7b() |
| 167 | } else if id == m::BARTOWSKI_QWEN3_4B_GGUF { |
| 168 | GgufModelConfig::qwen3_4b() |
| 169 | } else if id == m::BARTOWSKI_QWEN3_8B_GGUF { |
| 170 | GgufModelConfig::qwen3_8b() |
| 171 | } else if id == m::BARTOWSKI_QWEN3_14B_GGUF { |
| 172 | GgufModelConfig::qwen3_14b() |
| 173 | } else if id == m::THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF { |
| 174 | GgufModelConfig::deepseek_coder_6_7b() |
| 175 | } else { |
| 176 | return None; |
| 177 | }; |
| 178 | Some(cfg) |
| 179 | } |
| 180 | |
| 181 | /// Return the config for the currently selected model (falling back to the |
| 182 | /// platform default if the selection is somehow unknown). |
| 183 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 184 | pub(crate) fn model_config() -> GgufModelConfig { |
| 185 | let id = SELECTED_MODEL.lock().map(|g| g.clone()).unwrap_or_default(); |
| 186 | config_for_model_id(&id).unwrap_or_else(siti_default_config) |
| 187 | } |
| 188 | |
| 189 | /// Return the sampling config for the currently selected model. |
| 190 | /// |
| 191 | /// iOS and Android use conservative settings to bound KV cache memory and |
| 192 | /// keep worst-case latency reasonable on slower SoCs. Qwen 3 models get a |
| 193 | /// larger `max_tokens` budget because their `<think>…</think>` block can |
| 194 | /// consume hundreds of tokens before the visible reply begins, and too small |
| 195 | /// a budget yields empty replies. |
| 196 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 197 | pub(crate) fn sampling_config() -> SamplingConfig { |
| 198 | let id = SELECTED_MODEL.lock().map(|g| g.clone()).unwrap_or_default(); |
| 199 | let is_qwen3 = id.contains("Qwen3") || id.contains("Qwen_Qwen3"); |
| 200 | let mobile = cfg!(any(target_os = "ios", target_os = "android")); |
| 201 | |
| 202 | let max_tokens = match (mobile, is_qwen3) { |
| 203 | (true, true) => 1024, |
| 204 | (true, false) => 256, |
| 205 | (false, true) => 4096, |
| 206 | (false, false) => 512, |
| 207 | }; |
| 208 | |
| 209 | if mobile { |
| 210 | SamplingConfig { |
| 211 | temperature: Some(0.7), |
| 212 | top_p: Some(0.95), |
| 213 | max_tokens: Some(max_tokens), |
| 214 | ..SamplingConfig::mobile() |
| 215 | } |
| 216 | } else { |
| 217 | SamplingConfig { |
| 218 | temperature: Some(0.7), |
| 219 | top_p: Some(0.95), |
| 220 | max_tokens: Some(max_tokens), |
| 221 | ..SamplingConfig::default() |
| 222 | } |
| 223 | } |
| 224 | } |
| 225 | |
| 226 | /// Default system prompt for Siti, the private on-device personal assistant. |
| 227 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 228 | pub(crate) const CHAT_SYSTEM_PROMPT: &str = r#"You are Siti, a friendly and capable personal assistant that runs entirely on the user's device. Nothing the user says ever leaves the device. |
| 229 | |
| 230 | You help with everyday things: answering questions, brainstorming, drafting and editing text, summarising, explaining ideas in plain terms, and planning. |
| 231 | |
| 232 | Keep replies clear and concise, and prefer plain language. If a request is ambiguous, ask one short question instead of guessing. Use a short list only when it genuinely helps, and go easy on emoji. |
| 233 | |
| 234 | You run locally, so you may not know about very recent events. When you are not sure, say so instead of inventing an answer."#; |
| 235 | |
| 236 | // ── Helpers ────────────────────────────────────────────────────────────────── |
| 237 | |
| 238 | /// Format a `Duration` as `Xm Ys` or just `Ys` when under a minute. |
| 239 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 240 | pub(crate) fn fmt_duration(d: std::time::Duration) -> String { |
| 241 | let total_secs = d.as_secs_f64(); |
| 242 | let mins = (total_secs / 60.0).floor() as u64; |
| 243 | let secs = total_secs - (mins as f64 * 60.0); |
| 244 | if mins > 0 { |
| 245 | format!("{}m {:.1}s", mins, secs) |
| 246 | } else { |
| 247 | format!("{:.1}s", secs) |
| 248 | } |
| 249 | } |
| 250 | |
| 251 | /// Emit a chat status event to the frontend. |
| 252 | #[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))] |
| 253 | pub(crate) fn emit_chat_status( |
| 254 | app: &AppHandle, |
| 255 | status: ChatStatus, |
| 256 | model_name: Option<&str>, |
| 257 | error_msg: Option<&str>, |
| 258 | ) { |
| 259 | let payload = ChatStatusPayload { |
| 260 | status, |
| 261 | model_name: model_name.map(|s| s.to_string()), |
| 262 | error: error_msg.map(|s| s.to_string()), |
| 263 | }; |
| 264 | if let Err(e) = app.emit(EVENT_CHAT_STATUS_CHANGED, &payload) { |
| 265 | error!("Failed to emit chat status event: {:?}", e); |
| 266 | } |
| 267 | } |