| 1 | //! Chat feature module: fully on-device LLM inference for Siti AI. |
| 2 | //! |
| 3 | //! Each Tauri command lives in its own file (`command_*.rs`). Shared types, |
| 4 | //! the cached engine instance, model/sampling configuration, and helpers are |
| 5 | //! kept here in `mod.rs` so every command file can `use super::*`. |
| 6 | //! |
| 7 | //! Model loading, inference, history and streaming are delegated to |
| 8 | //! [`ed_agent::Ed`], the agent SDK over the shared on-device inference engine |
| 9 | //! from the `onde` crate. Ed lives in a managed [`EdState`] and reports to the |
| 10 | //! webview through `ed-agent-tauri`'s events. Siti runs the engine **fully |
| 11 | //! offline**: it loads |
| 12 | //! the platform-default GGUF model directly (no operator-assigned model, no |
| 13 | //! app credentials) and pulse telemetry is disabled during app setup. |
| 14 | |
| 15 | // ── Command submodules (one Tauri command per file) ────────────────────────── |
| 16 | |
| 17 | pub mod command_download_progress; |
| 18 | pub mod command_list_models; |
| 19 | pub mod command_load_model; |
| 20 | pub mod command_remove_model; |
| 21 | pub mod command_send_message; |
| 22 | pub mod command_set_model; |
| 23 | pub mod command_unload_model; |
| 24 | |
| 25 | // ── Re-exports so lib.rs can pull in commands with a flat path ─────────────── |
| 26 | |
| 27 | pub use command_download_progress::chat_download_progress; |
| 28 | pub use command_list_models::chat_list_models; |
| 29 | pub use command_load_model::chat_load_model; |
| 30 | pub use command_remove_model::chat_remove_model; |
| 31 | pub use command_send_message::chat_send_message; |
| 32 | pub use command_set_model::chat_set_model; |
| 33 | pub use command_unload_model::chat_unload_model; |
| 34 | |
| 35 | // ── Imports shared with submodules via `super::` ───────────────────────────── |
| 36 | |
| 37 | use serde::{Deserialize, Serialize}; |
| 38 | |
| 39 | #[cfg(any( |
| 40 | target_os = "macos", |
| 41 | target_os = "ios", |
| 42 | target_os = "android", |
| 43 | target_os = "windows" |
| 44 | ))] |
| 45 | use { |
| 46 | ed_agent::{Ed, GgufModelConfig, InferenceError, SamplingConfig, UqffModelConfig}, |
| 47 | ed_agent_tauri::TauriSink, |
| 48 | once_cell::sync::Lazy, |
| 49 | }; |
| 50 | |
| 51 | // `IsqModelConfig` and the ISQ load path only exist on macOS (Metal). Gemma is |
| 52 | // offered exclusively there — see `ResolvedModel` below for the reasoning. |
| 53 | #[cfg(target_os = "macos")] |
| 54 | use onde::inference::IsqModelConfig; |
| 55 | |
| 56 | // ── Response / payload types ───────────────────────────────────────────────── |
| 57 | |
| 58 | /// A selectable on-device model, returned by `chat_list_models` and rendered |
| 59 | /// in the Settings model dropdown. |
| 60 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 61 | pub struct ModelInfo { |
| 62 | /// HuggingFace repo id; the value passed back to `chat_set_model`. |
| 63 | pub id: String, |
| 64 | /// Human-friendly display name, e.g. `"Qwen 2.5 1.5B (GGUF)"`. |
| 65 | pub name: String, |
| 66 | /// Publisher / organisation, e.g. `"Qwen / Alibaba"`. |
| 67 | pub org: String, |
| 68 | /// Short description of the model's purpose and footprint. |
| 69 | pub description: String, |
| 70 | /// Approximate memory footprint, e.g. `"~941 MB (GGUF Q4_K_M)"`. |
| 71 | pub approx_memory: String, |
| 72 | /// Approximate on-disk download size in bytes (for display). |
| 73 | pub size_bytes: Option<u64>, |
| 74 | /// Whether the model's weights are already downloaded to the local cache. |
| 75 | pub is_downloaded: bool, |
| 76 | /// Whether this model is the one currently selected for Siti. |
| 77 | pub is_selected: bool, |
| 78 | } |
| 79 | |
| 80 | // ── The agent ──────────────────────────────────────────────────────────────── |
| 81 | // |
| 82 | // One `Ed<TauriSink>` lives in managed `EdState` (see `lib.rs`). Ed owns the |
| 83 | // `ChatEngine`, reports load transitions as `chat_status_changed`, streams |
| 84 | // tokens as `chat_text_delta` and delivers replies as `chat_reply`. Commands |
| 85 | // reach it through `State<EdState>`. |
| 86 | |
| 87 | // ── Model selection & configuration ────────────────────────────────────────── |
| 88 | // |
| 89 | // The user can switch models from Settings (`chat_set_model`). The chosen |
| 90 | // HuggingFace repo id is held in `SELECTED_MODEL` and drives both the model |
| 91 | // config and the sampling config. When nothing has been chosen yet, Siti |
| 92 | // falls back to a general-purpose chat model appropriate for the platform: |
| 93 | // - iOS / Android → Qwen 2.5 1.5B (~941 MB, fits mobile memory budgets) |
| 94 | // - macOS/Windows → Qwen 2.5 3B (~1.93 GB, more headroom on desktop) |
| 95 | // |
| 96 | // `config_for_model_id` handles per-platform details (e.g. `tok_model_id` on |
| 97 | // mobile) via the onde `GgufModelConfig` constructors. |
| 98 | |
| 99 | /// The model id currently selected for Siti. Defaults to the platform- |
| 100 | /// appropriate general chat model; updated by `chat_set_model`. |
| 101 | #[cfg(any( |
| 102 | target_os = "macos", |
| 103 | target_os = "ios", |
| 104 | target_os = "android", |
| 105 | target_os = "windows" |
| 106 | ))] |
| 107 | pub(crate) static SELECTED_MODEL: Lazy<std::sync::Mutex<String>> = |
| 108 | Lazy::new(|| std::sync::Mutex::new(siti_default_config().model_id)); |
| 109 | |
| 110 | /// Siti's default general-purpose chat model for the current platform. |
| 111 | #[cfg(any( |
| 112 | target_os = "macos", |
| 113 | target_os = "ios", |
| 114 | target_os = "android", |
| 115 | target_os = "windows" |
| 116 | ))] |
| 117 | pub(crate) fn siti_default_config() -> GgufModelConfig { |
| 118 | if cfg!(any(target_os = "ios", target_os = "android")) { |
| 119 | GgufModelConfig::qwen25_1_5b() |
| 120 | } else { |
| 121 | GgufModelConfig::qwen25_3b() |
| 122 | } |
| 123 | } |
| 124 | |
| 125 | /// Map a HuggingFace repo id to a loadable [`GgufModelConfig`], or `None` if |
| 126 | /// the id is not a model the onde engine knows how to load. |
| 127 | #[cfg(any( |
| 128 | target_os = "macos", |
| 129 | target_os = "ios", |
| 130 | target_os = "android", |
| 131 | target_os = "windows" |
| 132 | ))] |
| 133 | pub(crate) fn config_for_model_id(id: &str) -> Option<GgufModelConfig> { |
| 134 | use crate::inference::models as m; |
| 135 | let cfg = if id == m::BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF { |
| 136 | GgufModelConfig::qwen25_0_5b() |
| 137 | } else if id == m::BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF { |
| 138 | GgufModelConfig::qwen25_1_5b() |
| 139 | } else if id == m::BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF { |
| 140 | GgufModelConfig::qwen25_3b() |
| 141 | } else if id == m::BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF { |
| 142 | GgufModelConfig::qwen25_coder_1_5b() |
| 143 | } else if id == m::BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF { |
| 144 | GgufModelConfig::qwen25_coder_3b() |
| 145 | } else if id == m::BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF { |
| 146 | GgufModelConfig::qwen25_coder_7b() |
| 147 | } else if id == m::BARTOWSKI_QWEN3_0_6B_GGUF { |
| 148 | GgufModelConfig::qwen3_0_6b() |
| 149 | } else if id == m::BARTOWSKI_QWEN3_1_7B_GGUF { |
| 150 | GgufModelConfig::qwen3_1_7b() |
| 151 | } else if id == m::BARTOWSKI_QWEN3_4B_GGUF { |
| 152 | GgufModelConfig::qwen3_4b() |
| 153 | } else if id == m::BARTOWSKI_QWEN3_8B_GGUF { |
| 154 | GgufModelConfig::qwen3_8b() |
| 155 | } else if id == m::BARTOWSKI_QWEN3_14B_GGUF { |
| 156 | GgufModelConfig::qwen3_14b() |
| 157 | } else if id == m::BARTOWSKI_QWEN3_32B_GGUF { |
| 158 | GgufModelConfig::qwen3_32b() |
| 159 | } else if id == m::BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF { |
| 160 | GgufModelConfig::qwen3_4b_instruct_2507() |
| 161 | } else if id == m::BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF { |
| 162 | GgufModelConfig::qwen3_4b_thinking_2507() |
| 163 | } else if id == m::BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF { |
| 164 | GgufModelConfig::qwen3_30b_a3b_instruct_2507() |
| 165 | } else if id == m::THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF { |
| 166 | GgufModelConfig::deepseek_coder_6_7b() |
| 167 | } else { |
| 168 | return None; |
| 169 | }; |
| 170 | Some(cfg) |
| 171 | } |
| 172 | |
| 173 | // ── Qwen 3 (UQFF pre-quantised path) ───────────────────────────────────────── |
| 174 | // |
| 175 | // UQFF (Universal Quantized File Format) stores pre-quantised weights and loads |
| 176 | // directly through mistral.rs's `UqffTextModelBuilder`, avoiding the ISQ path's |
| 177 | // full-precision download + in-memory quantisation spike. onde exposes it via |
| 178 | // `ChatEngine::load_uqff_model` on every platform its engine builds for — the |
| 179 | // `q4k` shards are a GGML-family type that dequantises on CPU as well as Metal |
| 180 | // — so, unlike Gemma ISQ, these are offered everywhere Siti does inference |
| 181 | // (macOS/iOS/Android/Windows). Keep these `cfg`s in step with the rest of the |
| 182 | // module: a narrower gate here breaks the Windows build, because |
| 183 | // `resolve_model_id` and `chat_list_models` reference the table unconditionally. |
| 184 | // |
| 185 | // We ship the `mistralrs-community` Qwen 3 UQFF repos that use a *text* |
| 186 | // architecture the UQFF text loader (`UqffTextModelBuilder` / |
| 187 | // `NormalLoaderType`) actually supports: `Qwen3ForCausalLM` (dense) and |
| 188 | // `Qwen3MoeForCausalLM` (the 30B-A3B mixture-of-experts family, including its |
| 189 | // Coder variant). Deliberately excluded, even though the repos exist under |
| 190 | // `mistralrs-community`: `Qwen3.5`/`Qwen3.6` (`Qwen3_5(Moe)ForConditionalGeneration`) |
| 191 | // and `Qwen3-VL` (`Qwen3VL(Moe)ForConditionalGeneration`) route through |
| 192 | // mistral.rs's *multimodal* loader, which the text-only UQFF builder can't |
| 193 | // reach; `Qwen3-Embedding` is non-generative; `Qwen3Guard-Gen` emits safety |
| 194 | // labels rather than a chat reply; and `-Base` variants are not |
| 195 | // instruction-tuned, so they're a poor fit for Siti's chat UI even though |
| 196 | // they do carry a chat template and would technically load. |
| 197 | // |
| 198 | // Each repo is self-contained (base `config.json` + tokenizer + |
| 199 | // `residual.safetensors` + the `q4k` shard), ungated, and in current |
| 200 | // (post-1.0) UQFF format, so they load and generate. Only the `q4k-0.uqff` |
| 201 | // shard (plus the small residual) is downloaded per model, so |
| 202 | // `expected_size_bytes` counts just those, matching the on-disk footprint. |
| 203 | |
| 204 | /// Static metadata for a Qwen 3 UQFF model offered in Siti's model list. |
| 205 | #[cfg(any( |
| 206 | target_os = "macos", |
| 207 | target_os = "ios", |
| 208 | target_os = "android", |
| 209 | target_os = "windows" |
| 210 | ))] |
| 211 | pub(crate) struct UqffModelEntry { |
| 212 | /// HuggingFace repo id; also the value passed back to `chat_set_model`. |
| 213 | pub id: &'static str, |
| 214 | /// Short label shown in the Settings dropdown, e.g. `"Qwen 3 1.7B (UQFF)"`. |
| 215 | pub name: &'static str, |
| 216 | /// Display name reported by the engine while loaded/loading. |
| 217 | pub display_name: &'static str, |
| 218 | /// Approximate runtime memory footprint, e.g. `"~1.5 GB (UQFF Q4K)"`. |
| 219 | pub approx_memory: &'static str, |
| 220 | /// Short description of the model's purpose and footprint. |
| 221 | pub description: &'static str, |
| 222 | /// Approximate on-disk download size (the `q4k` shard + `residual`) in bytes. |
| 223 | pub expected_size_bytes: u64, |
| 224 | /// Restrict this entry to macOS/Windows. Set for ~40+ GB downloads that |
| 225 | /// cannot realistically fit an iOS/Android device's storage or memory |
| 226 | /// budget; `chat_list_models` filters these out on mobile platforms. |
| 227 | pub desktop_only: bool, |
| 228 | } |
| 229 | |
| 230 | /// The Qwen 3 UQFF models Siti can load. Sizes are the measured |
| 231 | /// `q4k-0.uqff` + `residual.safetensors` totals of each `mistralrs-community` |
| 232 | /// repo (small config/tokenizer files add a negligible remainder). |
| 233 | #[cfg(any( |
| 234 | target_os = "macos", |
| 235 | target_os = "ios", |
| 236 | target_os = "android", |
| 237 | target_os = "windows" |
| 238 | ))] |
| 239 | pub(crate) const UQFF_MODELS: &[UqffModelEntry] = &[ |
| 240 | UqffModelEntry { |
| 241 | id: "mistralrs-community/Qwen3-0.6B-UQFF", |
| 242 | name: "Qwen 3 0.6B (UQFF)", |
| 243 | display_name: "Qwen 3 0.6B (UQFF Q4K)", |
| 244 | approx_memory: "~0.6 GB (UQFF Q4K)", |
| 245 | description: "Qwen 3 0.6B, pre-quantised to 4-bit (UQFF Q4K). Loads directly with no \ |
| 246 | on-device quantisation step; the lightest UQFF option.", |
| 247 | expected_size_bytes: 646_635_509, |
| 248 | desktop_only: false, |
| 249 | }, |
| 250 | UqffModelEntry { |
| 251 | id: "mistralrs-community/Qwen3-1.7B-UQFF", |
| 252 | name: "Qwen 3 1.7B (UQFF)", |
| 253 | display_name: "Qwen 3 1.7B (UQFF Q4K)", |
| 254 | approx_memory: "~1.5 GB (UQFF Q4K)", |
| 255 | description: "Qwen 3 1.7B, pre-quantised to 4-bit (UQFF Q4K). Loads directly with no \ |
| 256 | on-device quantisation step.", |
| 257 | expected_size_bytes: 1_590_429_781, |
| 258 | desktop_only: false, |
| 259 | }, |
| 260 | UqffModelEntry { |
| 261 | id: "mistralrs-community/Qwen3-4B-UQFF", |
| 262 | name: "Qwen 3 4B (UQFF)", |
| 263 | display_name: "Qwen 3 4B (UQFF Q4K)", |
| 264 | approx_memory: "~2.9 GB (UQFF Q4K)", |
| 265 | description: "Qwen 3 4B, pre-quantised to 4-bit (UQFF Q4K). Loads directly with no \ |
| 266 | on-device quantisation step.", |
| 267 | expected_size_bytes: 3_040_959_629, |
| 268 | desktop_only: false, |
| 269 | }, |
| 270 | UqffModelEntry { |
| 271 | id: "mistralrs-community/Qwen3-4B-Instruct-2507-UQFF", |
| 272 | name: "Qwen 3 4B Instruct 2507 (UQFF)", |
| 273 | display_name: "Qwen 3 4B Instruct 2507 (UQFF Q4K)", |
| 274 | approx_memory: "~2.9 GB (UQFF Q4K)", |
| 275 | description: "Qwen 3 4B Instruct (2507 refresh), pre-quantised to 4-bit (UQFF Q4K). \ |
| 276 | A non-thinking instruct model that loads directly.", |
| 277 | expected_size_bytes: 3_040_959_629, |
| 278 | desktop_only: false, |
| 279 | }, |
| 280 | UqffModelEntry { |
| 281 | id: "mistralrs-community/Qwen3-4B-Thinking-2507-UQFF", |
| 282 | name: "Qwen 3 4B Thinking 2507 (UQFF)", |
| 283 | display_name: "Qwen 3 4B Thinking 2507 (UQFF Q4K)", |
| 284 | approx_memory: "~2.9 GB (UQFF Q4K)", |
| 285 | description: "Qwen 3 4B Thinking (2507 refresh), pre-quantised to 4-bit (UQFF Q4K). \ |
| 286 | Emits a reasoning block before its reply; loads directly.", |
| 287 | expected_size_bytes: 3_040_959_629, |
| 288 | desktop_only: false, |
| 289 | }, |
| 290 | UqffModelEntry { |
| 291 | id: "mistralrs-community/Qwen3-4B-SafeRL-UQFF", |
| 292 | name: "Qwen 3 4B SafeRL (UQFF)", |
| 293 | display_name: "Qwen 3 4B SafeRL (UQFF Q4K)", |
| 294 | approx_memory: "~3.0 GB (UQFF Q4K)", |
| 295 | description: "Qwen 3 4B, safety-RL-tuned variant, pre-quantised to 4-bit (UQFF Q4K). \ |
| 296 | Loads directly with no on-device quantisation step.", |
| 297 | expected_size_bytes: 3_040_959_629, |
| 298 | desktop_only: false, |
| 299 | }, |
| 300 | UqffModelEntry { |
| 301 | id: "mistralrs-community/Qwen3-8B-UQFF", |
| 302 | name: "Qwen 3 8B (UQFF)", |
| 303 | display_name: "Qwen 3 8B (UQFF Q4K)", |
| 304 | approx_memory: "~5.2 GB (UQFF Q4K)", |
| 305 | description: "Qwen 3 8B, pre-quantised to 4-bit (UQFF Q4K). Loads directly; needs \ |
| 306 | roomier memory (12+ GB recommended).", |
| 307 | expected_size_bytes: 5_502_458_509, |
| 308 | desktop_only: false, |
| 309 | }, |
| 310 | UqffModelEntry { |
| 311 | id: "mistralrs-community/Qwen3-14B-UQFF", |
| 312 | name: "Qwen 3 14B (UQFF)", |
| 313 | display_name: "Qwen 3 14B (UQFF Q4K)", |
| 314 | approx_memory: "~9.0 GB (UQFF Q4K)", |
| 315 | description: "Qwen 3 14B, pre-quantised to 4-bit (UQFF Q4K). The largest UQFF option; \ |
| 316 | desktop-class memory (16+ GB) recommended.", |
| 317 | expected_size_bytes: 9_426_174_577, |
| 318 | desktop_only: false, |
| 319 | }, |
| 320 | UqffModelEntry { |
| 321 | id: "mistralrs-community/Qwen3-32B-UQFF", |
| 322 | name: "Qwen 3 32B (UQFF)", |
| 323 | display_name: "Qwen 3 32B (UQFF Q4K)", |
| 324 | approx_memory: "~19.5 GB (UQFF Q4K)", |
| 325 | description: "Qwen 3 32B, pre-quantised to 4-bit (UQFF Q4K). Desktop-class memory \ |
| 326 | (32+ GB) recommended.", |
| 327 | expected_size_bytes: 19_548_142_977, |
| 328 | desktop_only: false, |
| 329 | }, |
| 330 | UqffModelEntry { |
| 331 | id: "mistralrs-community/Qwen3-30B-A3B-UQFF", |
| 332 | name: "Qwen 3 30B-A3B (UQFF)", |
| 333 | display_name: "Qwen 3 30B-A3B (UQFF Q4K)", |
| 334 | approx_memory: "~17.6 GB (UQFF Q4K)", |
| 335 | description: "Qwen 3 30B-A3B, a mixture-of-experts model (3B active params), \ |
| 336 | pre-quantised to 4-bit (UQFF Q4K). Desktop-class memory (24+ GB) \ |
| 337 | recommended despite the small active-parameter count.", |
| 338 | expected_size_bytes: 17_640_189_769, |
| 339 | desktop_only: false, |
| 340 | }, |
| 341 | UqffModelEntry { |
| 342 | id: "mistralrs-community/Qwen3-30B-A3B-Instruct-2507-UQFF", |
| 343 | name: "Qwen 3 30B-A3B Instruct 2507 (UQFF)", |
| 344 | display_name: "Qwen 3 30B-A3B Instruct 2507 (UQFF Q4K)", |
| 345 | approx_memory: "~17.6 GB (UQFF Q4K)", |
| 346 | description: "Qwen 3 30B-A3B Instruct (2507 refresh), mixture-of-experts (3B active \ |
| 347 | params), pre-quantised to 4-bit (UQFF Q4K). A non-thinking instruct \ |
| 348 | model; desktop-class memory (24+ GB) recommended.", |
| 349 | expected_size_bytes: 17_640_189_769, |
| 350 | desktop_only: false, |
| 351 | }, |
| 352 | UqffModelEntry { |
| 353 | id: "mistralrs-community/Qwen3-30B-A3B-Thinking-2507-UQFF", |
| 354 | name: "Qwen 3 30B-A3B Thinking 2507 (UQFF)", |
| 355 | display_name: "Qwen 3 30B-A3B Thinking 2507 (UQFF Q4K)", |
| 356 | approx_memory: "~17.6 GB (UQFF Q4K)", |
| 357 | description: "Qwen 3 30B-A3B Thinking (2507 refresh), mixture-of-experts (3B active \ |
| 358 | params), pre-quantised to 4-bit (UQFF Q4K). Emits a reasoning block \ |
| 359 | before its reply; desktop-class memory (24+ GB) recommended.", |
| 360 | expected_size_bytes: 17_640_189_769, |
| 361 | desktop_only: false, |
| 362 | }, |
| 363 | UqffModelEntry { |
| 364 | id: "mistralrs-community/Qwen3-Coder-30B-A3B-Instruct-UQFF", |
| 365 | name: "Qwen 3 Coder 30B-A3B Instruct (UQFF)", |
| 366 | display_name: "Qwen 3 Coder 30B-A3B Instruct (UQFF Q4K)", |
| 367 | approx_memory: "~17.6 GB (UQFF Q4K)", |
| 368 | description: "Qwen 3 Coder 30B-A3B Instruct, mixture-of-experts (3B active params) \ |
| 369 | coding model, pre-quantised to 4-bit (UQFF Q4K). Desktop-class memory \ |
| 370 | (24+ GB) recommended.", |
| 371 | expected_size_bytes: 17_640_189_769, |
| 372 | desktop_only: false, |
| 373 | }, |
| 374 | // The following three repos are ~45 GB downloads (q4k shard + residual) — |
| 375 | // impractical for iOS/Android storage and memory budgets, so they're |
| 376 | // restricted to macOS/Windows via `desktop_only`. |
| 377 | UqffModelEntry { |
| 378 | id: "mistralrs-community/Qwen3-Coder-Next-UQFF", |
| 379 | name: "Qwen 3 Coder Next (UQFF)", |
| 380 | display_name: "Qwen 3 Coder Next (UQFF Q4K)", |
| 381 | approx_memory: "~45.3 GB (UQFF Q4K)", |
| 382 | description: "Qwen 3 Coder Next, pre-quantised to 4-bit (UQFF Q4K). The largest \ |
| 383 | coding UQFF option; desktop-class memory (64+ GB) recommended. \ |
| 384 | macOS/Windows only.", |
| 385 | expected_size_bytes: 45_338_887_469, |
| 386 | desktop_only: true, |
| 387 | }, |
| 388 | UqffModelEntry { |
| 389 | id: "mistralrs-community/Qwen3-Next-80B-A3B-Instruct-UQFF", |
| 390 | name: "Qwen 3 Next 80B-A3B Instruct (UQFF)", |
| 391 | display_name: "Qwen 3 Next 80B-A3B Instruct (UQFF Q4K)", |
| 392 | approx_memory: "~45.3 GB (UQFF Q4K)", |
| 393 | description: "Qwen 3 Next 80B-A3B Instruct, mixture-of-experts (3B active params), \ |
| 394 | pre-quantised to 4-bit (UQFF Q4K). A non-thinking instruct model; \ |
| 395 | desktop-class memory (64+ GB) recommended. macOS/Windows only.", |
| 396 | expected_size_bytes: 45_338_887_469, |
| 397 | desktop_only: true, |
| 398 | }, |
| 399 | UqffModelEntry { |
| 400 | id: "mistralrs-community/Qwen3-Next-80B-A3B-Thinking-UQFF", |
| 401 | name: "Qwen 3 Next 80B-A3B Thinking (UQFF)", |
| 402 | display_name: "Qwen 3 Next 80B-A3B Thinking (UQFF Q4K)", |
| 403 | approx_memory: "~45.3 GB (UQFF Q4K)", |
| 404 | description: "Qwen 3 Next 80B-A3B Thinking, mixture-of-experts (3B active params), \ |
| 405 | pre-quantised to 4-bit (UQFF Q4K). Emits a reasoning block before its \ |
| 406 | reply; desktop-class memory (64+ GB) recommended. macOS/Windows only.", |
| 407 | expected_size_bytes: 45_338_887_469, |
| 408 | desktop_only: true, |
| 409 | }, |
| 410 | ]; |
| 411 | |
| 412 | /// Map a HuggingFace repo id to a loadable [`UqffModelConfig`], or `None` if the |
| 413 | /// id is not one of the Qwen 3 UQFF models Siti offers. |
| 414 | /// |
| 415 | /// Every `mistralrs-community` Qwen 3 UQFF repo names its 4-bit shard |
| 416 | /// `q4k-0.uqff`; passing that single shard is enough for mistral.rs to resolve |
| 417 | /// the base config, tokenizer, and residual weights. |
| 418 | #[cfg(any( |
| 419 | target_os = "macos", |
| 420 | target_os = "ios", |
| 421 | target_os = "android", |
| 422 | target_os = "windows" |
| 423 | ))] |
| 424 | pub(crate) fn uqff_config_for_model_id(id: &str) -> Option<UqffModelConfig> { |
| 425 | let entry = UQFF_MODELS.iter().find(|m| m.id == id)?; |
| 426 | Some(UqffModelConfig { |
| 427 | model_id: entry.id.to_string(), |
| 428 | files: vec!["q4k-0.uqff".to_string()], |
| 429 | display_name: entry.display_name.to_string(), |
| 430 | approx_memory: entry.approx_memory.to_string(), |
| 431 | chat_template: None, |
| 432 | }) |
| 433 | } |
| 434 | |
| 435 | // ── Gemma (ISQ safetensors path, macOS only) ───────────────────────────────── |
| 436 | // |
| 437 | // Gemma cannot be loaded through the GGUF path Siti uses for every other model: |
| 438 | // mistral.rs's GGUF loader (`GGUFArchitecture`) only recognises Llama / Qwen2 / |
| 439 | // Qwen3 / Mistral3 / etc. — it has **no Gemma architecture**, so a Gemma GGUF |
| 440 | // file fails to load with "Unknown GGUF architecture `gemma2`". mistral.rs does |
| 441 | // support Gemma, but only via its plain safetensors loader, which onde exposes |
| 442 | // through the ISQ (in-situ quantise) path (`load_isq_model`). |
| 443 | // |
| 444 | // That path is Metal-only, so Gemma is offered on **macOS only** — never on |
| 445 | // iOS/Android, whose builds have no `IsqModelConfig` load path and far tighter |
| 446 | // memory budgets than an ISQ load (full bf16 weights downloaded, then quantised |
| 447 | // in memory) allows. |
| 448 | |
| 449 | /// HuggingFace repo id for the Gemma 2 2B Instruct model, loaded via ISQ. |
| 450 | /// |
| 451 | /// The `unsloth` re-upload is used instead of `google/gemma-2-2b-it` because the |
| 452 | /// official repo is gated (requires accepting Google's licence + an HF token), |
| 453 | /// which Siti's credential-free offline download cannot satisfy. This mirror |
| 454 | /// ships the same `Gemma2ForCausalLM` bf16 weights, ungated. |
| 455 | #[cfg(target_os = "macos")] |
| 456 | pub(crate) const GEMMA2_2B_IT_ISQ_ID: &str = "unsloth/gemma-2-2b-it"; |
| 457 | |
| 458 | /// ISQ config for Gemma 2 2B Instruct (4-bit, Metal). macOS only. |
| 459 | #[cfg(target_os = "macos")] |
| 460 | pub(crate) fn gemma2_2b_isq_config() -> IsqModelConfig { |
| 461 | IsqModelConfig { |
| 462 | model_id: GEMMA2_2B_IT_ISQ_ID.to_string(), |
| 463 | isq_bits: 4, |
| 464 | display_name: "Gemma 2 2B (ISQ 4-bit)".to_string(), |
| 465 | approx_memory: "~1.6 GB (ISQ Q4K, Metal)".to_string(), |
| 466 | } |
| 467 | } |
| 468 | |
| 469 | /// A model selection resolved to the concrete engine config and load path it |
| 470 | /// needs. Most models are GGUF; Qwen 3 UQFF models use the UQFF path (all |
| 471 | /// platforms), and Gemma is the sole ISQ model (macOS only). |
| 472 | #[cfg(any( |
| 473 | target_os = "macos", |
| 474 | target_os = "ios", |
| 475 | target_os = "android", |
| 476 | target_os = "windows" |
| 477 | ))] |
| 478 | pub(crate) enum ResolvedModel { |
| 479 | Gguf(GgufModelConfig), |
| 480 | Uqff(UqffModelConfig), |
| 481 | #[cfg(target_os = "macos")] |
| 482 | Isq(IsqModelConfig), |
| 483 | } |
| 484 | |
| 485 | #[cfg(any( |
| 486 | target_os = "macos", |
| 487 | target_os = "ios", |
| 488 | target_os = "android", |
| 489 | target_os = "windows" |
| 490 | ))] |
| 491 | impl ResolvedModel { |
| 492 | /// Human-friendly display name for the resolved model. |
| 493 | pub(crate) fn display_name(&self) -> String { |
| 494 | match self { |
| 495 | ResolvedModel::Gguf(c) => c.display_name.clone(), |
| 496 | ResolvedModel::Uqff(c) => c.display_name.clone(), |
| 497 | #[cfg(target_os = "macos")] |
| 498 | ResolvedModel::Isq(c) => c.display_name.clone(), |
| 499 | } |
| 500 | } |
| 501 | |
| 502 | /// Load the resolved model into `ed`, dispatching to the GGUF, UQFF, or ISQ |
| 503 | /// load path as appropriate. Ed reports `Loading` then `Ready` or `Error`. |
| 504 | pub(crate) async fn load( |
| 505 | self, |
| 506 | ed: &Ed<TauriSink>, |
| 507 | system_prompt: Option<String>, |
| 508 | sampling: Option<SamplingConfig>, |
| 509 | ) -> Result<std::time::Duration, InferenceError> { |
| 510 | match self { |
| 511 | ResolvedModel::Gguf(c) => ed.load(c, system_prompt, sampling).await, |
| 512 | ResolvedModel::Uqff(c) => { |
| 513 | let name = c.display_name.clone(); |
| 514 | ed.load_with(&name, |engine| { |
| 515 | engine.load_uqff_model(c, system_prompt, sampling) |
| 516 | }) |
| 517 | .await |
| 518 | } |
| 519 | #[cfg(target_os = "macos")] |
| 520 | ResolvedModel::Isq(c) => { |
| 521 | let name = c.display_name.clone(); |
| 522 | ed.load_with(&name, |engine| { |
| 523 | engine.load_isq_model(c, system_prompt, sampling) |
| 524 | }) |
| 525 | .await |
| 526 | } |
| 527 | } |
| 528 | } |
| 529 | } |
| 530 | |
| 531 | /// Resolve a HuggingFace repo id to a loadable [`ResolvedModel`], or `None` if |
| 532 | /// the id is not a model Siti can load on this platform. |
| 533 | #[cfg(any( |
| 534 | target_os = "macos", |
| 535 | target_os = "ios", |
| 536 | target_os = "android", |
| 537 | target_os = "windows" |
| 538 | ))] |
| 539 | pub(crate) fn resolve_model_id(id: &str) -> Option<ResolvedModel> { |
| 540 | #[cfg(target_os = "macos")] |
| 541 | if id == GEMMA2_2B_IT_ISQ_ID { |
| 542 | return Some(ResolvedModel::Isq(gemma2_2b_isq_config())); |
| 543 | } |
| 544 | if let Some(cfg) = uqff_config_for_model_id(id) { |
| 545 | return Some(ResolvedModel::Uqff(cfg)); |
| 546 | } |
| 547 | config_for_model_id(id).map(ResolvedModel::Gguf) |
| 548 | } |
| 549 | |
| 550 | /// Return the resolved model (GGUF or ISQ) for the current selection, falling |
| 551 | /// back to the platform-default GGUF model if the selection is unknown. |
| 552 | #[cfg(any( |
| 553 | target_os = "macos", |
| 554 | target_os = "ios", |
| 555 | target_os = "android", |
| 556 | target_os = "windows" |
| 557 | ))] |
| 558 | pub(crate) fn resolved_model_config() -> ResolvedModel { |
| 559 | let id = SELECTED_MODEL.lock().map(|g| g.clone()).unwrap_or_default(); |
| 560 | resolve_model_id(&id).unwrap_or_else(|| ResolvedModel::Gguf(siti_default_config())) |
| 561 | } |
| 562 | |
| 563 | // ── Download-status detection ──────────────────────────────────────────────── |
| 564 | |
| 565 | /// Fraction of `expected_size_bytes` that must be present on disk for a model |
| 566 | /// to count as fully downloaded (mirrors onde's own completeness threshold). |
| 567 | #[cfg(any( |
| 568 | target_os = "macos", |
| 569 | target_os = "ios", |
| 570 | target_os = "android", |
| 571 | target_os = "windows" |
| 572 | ))] |
| 573 | const DOWNLOAD_COMPLETE_THRESHOLD: f64 = 0.99; |
| 574 | |
| 575 | /// Sum the byte size of every real file directly and recursively under `path`. |
| 576 | /// |
| 577 | /// Only used against the HF cache's `blobs/` directory, which holds the actual |
| 578 | /// downloaded files (not the `snapshots/` symlink/hard-link views), so nothing |
| 579 | /// is double-counted. |
| 580 | /// |
| 581 | /// `include_incomplete` decides how an in-flight download is treated. hf-hub |
| 582 | /// streams each blob into a sibling `<sha>.part` file and only renames it to the |
| 583 | /// final blob name once the transfer completes, holding a `<sha>.lock` alongside |
| 584 | /// it meanwhile. Those two answer different questions: |
| 585 | /// |
| 586 | /// - **Completeness** (`false`) must ignore them, or a download interrupted far |
| 587 | /// enough into the last file reads as finished — for a two-file UQFF model, |
| 588 | /// one finished shard plus a nearly-full `.part` clears the 99% threshold |
| 589 | /// while the model is still unloadable. |
| 590 | /// - **Progress** (`true`) wants exactly the opposite: the `.part` file *is* the |
| 591 | /// live byte count, and excluding it would peg the UI at 0% for the entire |
| 592 | /// download. |
| 593 | #[cfg(any( |
| 594 | target_os = "macos", |
| 595 | target_os = "ios", |
| 596 | target_os = "android", |
| 597 | target_os = "windows" |
| 598 | ))] |
| 599 | fn blobs_dir_size(path: &std::path::Path, include_incomplete: bool) -> u64 { |
| 600 | let mut total = 0; |
| 601 | if let Ok(entries) = std::fs::read_dir(path) { |
| 602 | for entry in entries.flatten() { |
| 603 | let p = entry.path(); |
| 604 | if !include_incomplete |
| 605 | && matches!( |
| 606 | p.extension().and_then(|e| e.to_str()), |
| 607 | Some("part") | Some("lock") |
| 608 | ) |
| 609 | { |
| 610 | continue; |
| 611 | } |
| 612 | match entry.metadata() { |
| 613 | Ok(md) if md.is_dir() => total += blobs_dir_size(&p, include_incomplete), |
| 614 | Ok(md) => total += md.len(), |
| 615 | Err(_) => {} |
| 616 | } |
| 617 | } |
| 618 | } |
| 619 | total |
| 620 | } |
| 621 | |
| 622 | /// Whether `id`'s weights are already downloaded to the local HF cache. |
| 623 | /// |
| 624 | /// A model counts as downloaded when its cache `blobs/` directory holds at |
| 625 | /// least [`DOWNLOAD_COMPLETE_THRESHOLD`] of the model's expected size, which |
| 626 | /// excludes partial/interrupted downloads. |
| 627 | #[cfg(any( |
| 628 | target_os = "macos", |
| 629 | target_os = "ios", |
| 630 | target_os = "android", |
| 631 | target_os = "windows" |
| 632 | ))] |
| 633 | pub(crate) fn is_model_downloaded(id: &str, expected_size_bytes: u64) -> bool { |
| 634 | if expected_size_bytes == 0 { |
| 635 | return false; |
| 636 | } |
| 637 | match onde::hf_cache::model_cache_path(id) { |
| 638 | Some(root) => { |
| 639 | let blobs = root.join("blobs"); |
| 640 | // Completed blobs only — a `.part` still being streamed must not |
| 641 | // count towards "downloaded". |
| 642 | blobs_dir_size(&blobs, false) as f64 |
| 643 | >= expected_size_bytes as f64 * DOWNLOAD_COMPLETE_THRESHOLD |
| 644 | } |
| 645 | None => false, |
| 646 | } |
| 647 | } |
| 648 | |
| 649 | /// Bytes of `id`'s weights currently on disk, **including** the partially |
| 650 | /// written `.part` file of an in-flight download. |
| 651 | /// |
| 652 | /// Drives the download progress indicator in Settings. mistral.rs owns the |
| 653 | /// actual transfer and only reports progress to an `indicatif` bar that goes to |
| 654 | /// the log, so the cache directory is the one place the UI can observe it. |
| 655 | #[cfg(any( |
| 656 | target_os = "macos", |
| 657 | target_os = "ios", |
| 658 | target_os = "android", |
| 659 | target_os = "windows" |
| 660 | ))] |
| 661 | pub(crate) fn model_downloaded_bytes(id: &str) -> u64 { |
| 662 | match onde::hf_cache::model_cache_path(id) { |
| 663 | Some(root) => blobs_dir_size(&root.join("blobs"), true), |
| 664 | None => 0, |
| 665 | } |
| 666 | } |
| 667 | |
| 668 | /// Return the sampling config for the currently selected model. |
| 669 | /// |
| 670 | /// iOS and Android use conservative settings to bound KV cache memory and |
| 671 | /// keep worst-case latency reasonable on slower SoCs. Qwen 3 models get a |
| 672 | /// larger `max_tokens` budget because their `<think>…</think>` block can |
| 673 | /// consume hundreds of tokens before the visible reply begins, and too small |
| 674 | /// a budget yields empty replies. |
| 675 | #[cfg(any( |
| 676 | target_os = "macos", |
| 677 | target_os = "ios", |
| 678 | target_os = "android", |
| 679 | target_os = "windows" |
| 680 | ))] |
| 681 | pub(crate) fn sampling_config() -> SamplingConfig { |
| 682 | let id = SELECTED_MODEL.lock().map(|g| g.clone()).unwrap_or_default(); |
| 683 | let is_qwen3 = id.contains("Qwen3") || id.contains("Qwen_Qwen3"); |
| 684 | let mobile = cfg!(any(target_os = "ios", target_os = "android")); |
| 685 | |
| 686 | let max_tokens = match (mobile, is_qwen3) { |
| 687 | (true, true) => 1024, |
| 688 | (true, false) => 256, |
| 689 | (false, true) => 4096, |
| 690 | (false, false) => 512, |
| 691 | }; |
| 692 | |
| 693 | if mobile { |
| 694 | SamplingConfig { |
| 695 | temperature: Some(0.7), |
| 696 | top_p: Some(0.95), |
| 697 | max_tokens: Some(max_tokens), |
| 698 | ..SamplingConfig::mobile() |
| 699 | } |
| 700 | } else { |
| 701 | SamplingConfig { |
| 702 | temperature: Some(0.7), |
| 703 | top_p: Some(0.95), |
| 704 | max_tokens: Some(max_tokens), |
| 705 | ..SamplingConfig::default() |
| 706 | } |
| 707 | } |
| 708 | } |
| 709 | |
| 710 | /// Default system prompt for Siti, the private on-device personal assistant. |
| 711 | #[cfg(any( |
| 712 | target_os = "macos", |
| 713 | target_os = "ios", |
| 714 | target_os = "android", |
| 715 | target_os = "windows" |
| 716 | ))] |
| 717 | pub(crate) const CHAT_SYSTEM_PROMPT: &str = r#"You are Siti, a friendly and capable personal assistant that runs entirely on the user's device. Nothing the user says ever leaves the device. |
| 718 | |
| 719 | You help with everyday things: answering questions, brainstorming, drafting and editing text, summarising, explaining ideas in plain terms, and planning. |
| 720 | |
| 721 | Keep replies clear and concise, and prefer plain language. If a request is ambiguous, ask one short question instead of guessing. Use a short list only when it genuinely helps, and go easy on emoji. |
| 722 | |
| 723 | You run locally, so you may not know about very recent events. When you are not sure, say so instead of inventing an answer."#; |
| 724 | |
| 725 | // ── Helpers ────────────────────────────────────────────────────────────────── |
| 726 | |
| 727 | /// Format a `Duration` as `Xm Ys` or just `Ys` when under a minute. |
| 728 | #[cfg(any( |
| 729 | target_os = "macos", |
| 730 | target_os = "ios", |
| 731 | target_os = "android", |
| 732 | target_os = "windows" |
| 733 | ))] |
| 734 | pub(crate) fn fmt_duration(d: std::time::Duration) -> String { |
| 735 | let total_secs = d.as_secs_f64(); |
| 736 | let mins = (total_secs / 60.0).floor() as u64; |
| 737 | let secs = total_secs - (mins as f64 * 60.0); |
| 738 | if mins > 0 { |
| 739 | format!("{}m {:.1}s", mins, secs) |
| 740 | } else { |
| 741 | format!("{:.1}s", secs) |
| 742 | } |
| 743 | } |
| 744 | |
| 745 | // ── Tests ──────────────────────────────────────────────────────────────────── |
| 746 | |
| 747 | #[cfg(all( |
| 748 | test, |
| 749 | any( |
| 750 | target_os = "macos", |
| 751 | target_os = "ios", |
| 752 | target_os = "android", |
| 753 | target_os = "windows" |
| 754 | ) |
| 755 | ))] |
| 756 | mod tests { |
| 757 | use super::*; |
| 758 | |
| 759 | /// A two-file model whose download died partway through the second file: |
| 760 | /// one finished blob, plus the `.part`/`.lock` pair hf-hub leaves behind |
| 761 | /// while streaming the next. Sized so the in-flight bytes land *above* the |
| 762 | /// completeness threshold and the finished bytes alone land below it, which |
| 763 | /// is precisely the case that used to report an unloadable model as ready. |
| 764 | const FIXTURE_EXPECTED_TOTAL: u64 = 2000; |
| 765 | |
| 766 | fn blobs_fixture(name: &str) -> std::path::PathBuf { |
| 767 | let dir = std::env::temp_dir().join(format!("siti-blobs-{name}")); |
| 768 | let _ = std::fs::remove_dir_all(&dir); |
| 769 | std::fs::create_dir_all(&dir).expect("create fixture dir"); |
| 770 | std::fs::write(dir.join("aaaa"), vec![0u8; 1000]).expect("write blob"); |
| 771 | std::fs::write(dir.join("bbbb.part"), vec![0u8; 990]).expect("write part"); |
| 772 | std::fs::write(dir.join("bbbb.lock"), vec![0u8; 10]).expect("write lock"); |
| 773 | dir |
| 774 | } |
| 775 | |
| 776 | #[test] |
| 777 | fn blobs_dir_size_excludes_incomplete_downloads() { |
| 778 | let dir = blobs_fixture("complete"); |
| 779 | assert_eq!(blobs_dir_size(&dir, false), 1000); |
| 780 | let _ = std::fs::remove_dir_all(&dir); |
| 781 | } |
| 782 | |
| 783 | #[test] |
| 784 | fn blobs_dir_size_counts_partial_for_progress() { |
| 785 | let dir = blobs_fixture("progress"); |
| 786 | assert_eq!(blobs_dir_size(&dir, true), 2000); |
| 787 | let _ = std::fs::remove_dir_all(&dir); |
| 788 | } |
| 789 | |
| 790 | /// The regression this guards: one finished blob plus a nearly-complete |
| 791 | /// `.part` clears the 99% threshold, so counting in-flight bytes towards |
| 792 | /// completeness reports an unloadable model as downloaded. |
| 793 | #[test] |
| 794 | fn partial_download_does_not_read_as_complete() { |
| 795 | let dir = blobs_fixture("threshold"); |
| 796 | let cutoff = FIXTURE_EXPECTED_TOTAL as f64 * DOWNLOAD_COMPLETE_THRESHOLD; |
| 797 | assert!( |
| 798 | blobs_dir_size(&dir, true) as f64 >= cutoff, |
| 799 | "fixture must cross the threshold when in-flight bytes are counted" |
| 800 | ); |
| 801 | assert!( |
| 802 | (blobs_dir_size(&dir, false) as f64) < cutoff, |
| 803 | "completed bytes alone must stay below the threshold" |
| 804 | ); |
| 805 | let _ = std::fs::remove_dir_all(&dir); |
| 806 | } |
| 807 | } |