6 files changed
+133
-27
src-tauri/src/chat/command_list_models.rs
+22
@@ -58,6 +58,28 @@ pub async fn chat_list_models() -> Vec<ModelInfo> {
58
})
59
.collect();
60
61
+ // Gemma is not in onde's GGUF catalogue: mistral.rs's GGUF loader has no
62
+ // Gemma architecture, so Gemma can only load via the ISQ (safetensors)
63
+ // path, which is Metal-only. Offer it on macOS exclusively.
64
+ #[cfg(target_os = "macos")]
65
+ {
66
+ use super::{gemma2_2b_isq_config, GEMMA2_2B_IT_ISQ_ID};
67
+ let cfg = gemma2_2b_isq_config();
68
+ models.push(ModelInfo {
69
+ id: GEMMA2_2B_IT_ISQ_ID.to_string(),
70
+ name: "Gemma 2 2B (ISQ)".to_string(),
71
+ org: "Google".to_string(),
72
+ description: sanitize_description(
73
+ "Google's Gemma 2 2B Instruct, quantised on-device to 4-bit (~1.6 GB). \
74
+ Loaded from safetensors via ISQ; requires 8+ GB RAM.",
75
+ ),
76
+ approx_memory: cfg.approx_memory,
77
+ // Full bf16 safetensors download before in-situ quantisation (~5.2 GB).
78
+ size_bytes: Some(5_228_717_512),
79
+ is_selected: GEMMA2_2B_IT_ISQ_ID == selected,
80
+ });
81
+ }
82
+
83
// Smallest (most device-friendly) first.
84
models.sort_by_key(|m| m.size_bytes.unwrap_or(u64::MAX));
85
models
src-tauri/src/chat/command_load_model.rs
+5
-6
@@ -10,7 +10,7 @@ use tauri::AppHandle;
10
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
11
use {
12
super::{
13
- emit_chat_status, fmt_duration, model_config, sampling_config, CHAT_SYSTEM_PROMPT, ENGINE,
13
+ emit_chat_status, fmt_duration, resolved_model_config, sampling_config, CHAT_SYSTEM_PROMPT,
14
},
15
crate::constants::ChatStatus,
16
log::{error, info},
@@ -20,14 +20,13 @@ use {
20
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
21
#[tauri::command]
22
pub async fn chat_load_model(app: AppHandle) -> Result<String, String> {
23
- let config = model_config();
24
- let display_name = config.display_name.clone();
23
+ let config = resolved_model_config();
24
+ let display_name = config.display_name();
25
info!("chat_load_model: loading on-device model {}", display_name);
26
emit_chat_status(&app, ChatStatus::Loading, Some(&display_name), None);
27
28
- let elapsed = ENGINE
29
- .load_gguf_model(
30
- config,
28
+ let elapsed = config
29
+ .load(
30
Some(CHAT_SYSTEM_PROMPT.to_string()),
31
Some(sampling_config()),
32
)
src-tauri/src/chat/command_send_message.rs
+4
-5
@@ -34,14 +34,13 @@ pub async fn chat_send_message(app: AppHandle, message: String) -> Result<(), St
34
if !ENGINE.is_loaded().await {
35
info!("No chat model loaded; lazy-loading default model.");
36
37
- let config = model_config();
38
- let display_name = config.display_name.clone();
37
+ let config = resolved_model_config();
38
+ let display_name = config.display_name();
39
40
emit_chat_status(&app, ChatStatus::Loading, Some(&display_name), None);
41
42
- ENGINE
43
- .load_gguf_model(
44
- config,
42
+ config
43
+ .load(
44
Some(CHAT_SYSTEM_PROMPT.to_string()),
45
Some(sampling_config()),
46
)
src-tauri/src/chat/command_set_model.rs
+10
-9
@@ -12,8 +12,8 @@ use tauri::AppHandle;
12
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
13
use {
14
super::{
15
- config_for_model_id, emit_chat_status, fmt_duration, sampling_config, CHAT_SYSTEM_PROMPT,
16
- ENGINE, SELECTED_MODEL,
15
+ emit_chat_status, fmt_duration, resolve_model_id, resolved_model_config, sampling_config,
16
+ CHAT_SYSTEM_PROMPT, ENGINE, SELECTED_MODEL,
17
},
18
crate::constants::ChatStatus,
19
log::{error, info},
@@ -26,11 +26,11 @@ use {
26
#[tauri::command]
27
pub async fn chat_set_model(app: AppHandle, model_id: String) -> Result<(), String> {
28
// ── 1. Validate & resolve the requested model ────────────────────────
29
- let config = config_for_model_id(&model_id)
30
- .ok_or_else(|| format!("Unknown or unsupported model id: {model_id}"))?;
31
- let display_name = config.display_name.clone();
29
+ let display_name = resolve_model_id(&model_id)
30
+ .ok_or_else(|| format!("Unknown or unsupported model id: {model_id}"))?
31
+ .display_name();
32
33
- // ── 2. Persist the selection so model_config()/sampling_config() agree ─
33
+ // ── 2. Persist the selection so resolved_model_config()/sampling_config() agree ─
34
// NB: never hold the std Mutex guard across an `.await` (it would make this
35
// command future `!Send`), so resolve the loaded-state first.
36
let already_loaded = ENGINE.is_loaded().await;
@@ -49,14 +49,15 @@ pub async fn chat_set_model(app: AppHandle, model_id: String) -> Result<(), Stri
49
emit_chat_status(&app, ChatStatus::Loading, Some(&display_name), None);
50
51
// ── 3. Swap models off-thread; report progress via events ────────────
52
+ // Re-resolve from the (now-persisted) selection inside the task so the
53
+ // GGUF/ISQ dispatch matches exactly what `chat_send_message` would load.
54
tokio::task::spawn(async move {
55
if ENGINE.is_loaded().await {
56
ENGINE.unload_model().await;
57
}
58
57
- let result = ENGINE
58
- .load_gguf_model(
59
- config,
59
+ let result = resolved_model_config()
60
+ .load(
61
Some(CHAT_SYSTEM_PROMPT.to_string()),
62
Some(sampling_config()),
63
)
src-tauri/src/chat/command_unload_model.rs
+2
-2
@@ -4,7 +4,7 @@ use {log::debug, tauri::AppHandle};
4
5
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
6
use {
7
- super::{emit_chat_status, model_config, ENGINE},
7
+ super::{emit_chat_status, resolved_model_config, ENGINE},
8
crate::constants::ChatStatus,
9
log::info,
10
};
@@ -14,7 +14,7 @@ use {
14
#[tauri::command]
15
pub async fn chat_unload_model(app: AppHandle) -> Result<String, String> {
16
if ENGINE.is_loaded().await {
17
- let display_name = model_config().display_name;
17
+ let display_name = resolved_model_config().display_name();
18
ENGINE.unload_model().await;
19
info!("Chat model unloaded: {}", display_name);
20
emit_chat_status(&app, ChatStatus::Unloaded, None, None);
src-tauri/src/chat/mod.rs
+90
-5
@@ -44,10 +44,15 @@ use {
44
crate::events::{EVENT_CHAT_REPLY, EVENT_CHAT_STATUS_CHANGED},
45
log::error,
46
once_cell::sync::Lazy,
47
- onde::inference::{ChatEngine, GgufModelConfig, SamplingConfig},
47
+ onde::inference::{ChatEngine, GgufModelConfig, InferenceError, SamplingConfig},
48
tauri::{AppHandle, Emitter},
49
};
50
51
+// `IsqModelConfig` and the ISQ load path only exist on macOS (Metal). Gemma is
52
+// offered exclusively there — see `ResolvedModel` below for the reasoning.
53
+#[cfg(target_os = "macos")]
54
+use onde::inference::IsqModelConfig;
55
+
56
// ── Response / payload types ─────────────────────────────────────────────────
57
58
/// Payload emitted with the `chat_status_changed` event.
@@ -188,12 +193,92 @@ pub(crate) fn config_for_model_id(id: &str) -> Option<GgufModelConfig> {
193
Some(cfg)
194
}
195
191
-/// Return the config for the currently selected model (falling back to the
192
-/// platform default if the selection is somehow unknown).
196
+// ── Gemma (ISQ safetensors path, macOS only) ─────────────────────────────────
197
+//
198
+// Gemma cannot be loaded through the GGUF path Siti uses for every other model:
199
+// mistral.rs's GGUF loader (`GGUFArchitecture`) only recognises Llama / Qwen2 /
200
+// Qwen3 / Mistral3 / etc. — it has **no Gemma architecture**, so a Gemma GGUF
201
+// file fails to load with "Unknown GGUF architecture `gemma2`". mistral.rs does
202
+// support Gemma, but only via its plain safetensors loader, which onde exposes
203
+// through the ISQ (in-situ quantise) path (`load_isq_model`).
204
+//
205
+// That path is Metal-only, so Gemma is offered on **macOS only** — never on
206
+// iOS/Android, whose builds have no `IsqModelConfig` load path and far tighter
207
+// memory budgets than an ISQ load (full bf16 weights downloaded, then quantised
208
+// in memory) allows.
209
+
210
+/// HuggingFace repo id for the Gemma 2 2B Instruct model, loaded via ISQ.
211
+///
212
+/// The `unsloth` re-upload is used instead of `google/gemma-2-2b-it` because the
213
+/// official repo is gated (requires accepting Google's licence + an HF token),
214
+/// which Siti's credential-free offline download cannot satisfy. This mirror
215
+/// ships the same `Gemma2ForCausalLM` bf16 weights, ungated.
216
+#[cfg(target_os = "macos")]
217
+pub(crate) const GEMMA2_2B_IT_ISQ_ID: &str = "unsloth/gemma-2-2b-it";
218
+
219
+/// ISQ config for Gemma 2 2B Instruct (4-bit, Metal). macOS only.
220
+#[cfg(target_os = "macos")]
221
+pub(crate) fn gemma2_2b_isq_config() -> IsqModelConfig {
222
+ IsqModelConfig {
223
+ model_id: GEMMA2_2B_IT_ISQ_ID.to_string(),
224
+ isq_bits: 4,
225
+ display_name: "Gemma 2 2B (ISQ 4-bit)".to_string(),
226
+ approx_memory: "~1.6 GB (ISQ Q4K, Metal)".to_string(),
227
+ }
228
+}
229
+
230
+/// A model selection resolved to the concrete engine config and load path it
231
+/// needs. Most models are GGUF; Gemma is the sole ISQ model (macOS only).
232
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
233
+pub(crate) enum ResolvedModel {
234
+ Gguf(GgufModelConfig),
235
+ #[cfg(target_os = "macos")]
236
+ Isq(IsqModelConfig),
237
+}
238
+
239
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
240
+impl ResolvedModel {
241
+ /// Human-friendly display name for the resolved model.
242
+ pub(crate) fn display_name(&self) -> String {
243
+ match self {
244
+ ResolvedModel::Gguf(c) => c.display_name.clone(),
245
+ #[cfg(target_os = "macos")]
246
+ ResolvedModel::Isq(c) => c.display_name.clone(),
247
+ }
248
+ }
249
+
250
+ /// Load the resolved model into the shared [`ENGINE`], dispatching to the
251
+ /// GGUF or ISQ load path as appropriate.
252
+ pub(crate) async fn load(
253
+ self,
254
+ system_prompt: Option<String>,
255
+ sampling: Option<SamplingConfig>,
256
+ ) -> Result<std::time::Duration, InferenceError> {
257
+ match self {
258
+ ResolvedModel::Gguf(c) => ENGINE.load_gguf_model(c, system_prompt, sampling).await,
259
+ #[cfg(target_os = "macos")]
260
+ ResolvedModel::Isq(c) => ENGINE.load_isq_model(c, system_prompt, sampling).await,
261
+ }
262
+ }
263
+}
264
+
265
+/// Resolve a HuggingFace repo id to a loadable [`ResolvedModel`], or `None` if
266
+/// the id is not a model Siti can load on this platform.
267
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
268
+pub(crate) fn resolve_model_id(id: &str) -> Option<ResolvedModel> {
269
+ #[cfg(target_os = "macos")]
270
+ if id == GEMMA2_2B_IT_ISQ_ID {
271
+ return Some(ResolvedModel::Isq(gemma2_2b_isq_config()));
272
+ }
273
+ config_for_model_id(id).map(ResolvedModel::Gguf)
274
+}
275
+
276
+/// Return the resolved model (GGUF or ISQ) for the current selection, falling
277
+/// back to the platform-default GGUF model if the selection is unknown.
278
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
194
-pub(crate) fn model_config() -> GgufModelConfig {
279
+pub(crate) fn resolved_model_config() -> ResolvedModel {
280
let id = SELECTED_MODEL.lock().map(|g| g.clone()).unwrap_or_default();
196
- config_for_model_id(&id).unwrap_or_else(siti_default_config)
281
+ resolve_model_id(&id).unwrap_or_else(|| ResolvedModel::Gguf(siti_default_config()))
282
}
283
284
/// Return the sampling config for the currently selected model.