4 files changed
+157
-7
src-tauri/Cargo.lock
-2
@@ -4584,8 +4584,6 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
4584
[[package]]
4585
name = "onde"
4586
version = "1.2.2"
4587
-source = "registry+https://github.com/rust-lang/crates.io-index"
4588
-checksum = "8d84f62e53e6ce8efce62baac8e769d08f83ff19ed6cd9d1a91eeb769fedd618"
4587
dependencies = [
4588
"anyhow",
4589
"cc",
src-tauri/Cargo.toml
+6
-2
@@ -65,5 +65,9 @@ strip = false
65
# ── Local Onde SDK development ─────────────────────────────────────────────────
66
# Uncomment to build Siti against a local checkout of the Onde engine (sibling
67
# directory) instead of the published crate. Keep this commented on `main`.
68
-# [patch.crates-io]
69
-# onde = { path = "../../onde" }
68
+#
69
+# Enabled on this branch: the UQFF models added to Siti (Qwen 3 family) require
70
+# onde's `load_uqff_model` API, which is not in the published `onde 1.2.2` crate
71
+# yet. Revert to the crates.io version (or bump `onde`) once that release ships.
72
+[patch.crates-io]
73
+onde = { path = "../../onde" }
src-tauri/src/chat/command_list_models.rs
+19
@@ -59,6 +59,25 @@ pub async fn chat_list_models() -> Vec<ModelInfo> {
59
})
60
.collect();
61
62
+ // Qwen 3 UQFF models: pre-quantised (Q4K) safetensors loaded via onde's
63
+ // UQFF path on every inference platform. Not in onde's GGUF catalogue, so
64
+ // they're appended here from Siti's own `UQFF_MODELS` table.
65
+ {
66
+ use super::UQFF_MODELS;
67
+ for entry in UQFF_MODELS {
68
+ models.push(ModelInfo {
69
+ id: entry.id.to_string(),
70
+ name: entry.name.to_string(),
71
+ org: "Qwen / Alibaba".to_string(),
72
+ description: sanitize_description(entry.description),
73
+ approx_memory: entry.approx_memory.to_string(),
74
+ size_bytes: Some(entry.expected_size_bytes),
75
+ is_downloaded: is_model_downloaded(entry.id, entry.expected_size_bytes),
76
+ is_selected: entry.id == selected,
77
+ });
78
+ }
79
+ }
80
+
81
// Gemma is not in onde's GGUF catalogue: mistral.rs's GGUF loader has no
82
// Gemma architecture, so Gemma can only load via the ISQ (safetensors)
83
// path, which is Metal-only. Offer it on macOS exclusively.
src-tauri/src/chat/mod.rs
+132
-3
@@ -46,7 +46,9 @@ use {
46
crate::events::{EVENT_CHAT_REPLY, EVENT_CHAT_STATUS_CHANGED},
47
log::error,
48
once_cell::sync::Lazy,
49
- onde::inference::{ChatEngine, GgufModelConfig, InferenceError, SamplingConfig},
49
+ onde::inference::{
50
+ ChatEngine, GgufModelConfig, InferenceError, SamplingConfig, UqffModelConfig,
51
+ },
52
tauri::{AppHandle, Emitter},
53
};
54
@@ -197,6 +199,126 @@ pub(crate) fn config_for_model_id(id: &str) -> Option<GgufModelConfig> {
199
Some(cfg)
200
}
201
202
+// ── Qwen 3 (UQFF pre-quantised path) ─────────────────────────────────────────
203
+//
204
+// UQFF (Universal Quantized File Format) stores pre-quantised weights and loads
205
+// directly through mistral.rs's `UqffTextModelBuilder`, avoiding the ISQ path's
206
+// full-precision download + in-memory quantisation spike. onde exposes it via
207
+// `ChatEngine::load_uqff_model`, which is available on every inference platform
208
+// (macOS/iOS/Android), so — unlike Gemma ISQ — these are offered on all three.
209
+//
210
+// We ship the `mistralrs-community` Qwen 3 UQFF repos: each is self-contained
211
+// (base `config.json` + tokenizer + `residual.safetensors` + the `q4k` shard),
212
+// ungated, and in current (post-1.0) UQFF format. `Qwen3ForCausalLM` is a text
213
+// architecture the UQFF text loader supports, so they load and generate. Only
214
+// the `q4k-0.uqff` shard (plus the small residual) is downloaded per model, so
215
+// `expected_size_bytes` counts just those, matching the on-disk footprint.
216
+
217
+/// Static metadata for a Qwen 3 UQFF model offered in Siti's model list.
218
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
219
+pub(crate) struct UqffModelEntry {
220
+ /// HuggingFace repo id; also the value passed back to `chat_set_model`.
221
+ pub id: &'static str,
222
+ /// Short label shown in the Settings dropdown, e.g. `"Qwen 3 1.7B (UQFF)"`.
223
+ pub name: &'static str,
224
+ /// Display name reported by the engine while loaded/loading.
225
+ pub display_name: &'static str,
226
+ /// Approximate runtime memory footprint, e.g. `"~1.5 GB (UQFF Q4K)"`.
227
+ pub approx_memory: &'static str,
228
+ /// Short description of the model's purpose and footprint.
229
+ pub description: &'static str,
230
+ /// Approximate on-disk download size (the `q4k` shard + `residual`) in bytes.
231
+ pub expected_size_bytes: u64,
232
+}
233
+
234
+/// The Qwen 3 UQFF models Siti can load. Sizes are the measured
235
+/// `q4k-0.uqff` + `residual.safetensors` totals of each `mistralrs-community`
236
+/// repo (small config/tokenizer files add a negligible remainder).
237
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
238
+pub(crate) const UQFF_MODELS: &[UqffModelEntry] = &[
239
+ UqffModelEntry {
240
+ id: "mistralrs-community/Qwen3-0.6B-UQFF",
241
+ name: "Qwen 3 0.6B (UQFF)",
242
+ display_name: "Qwen 3 0.6B (UQFF Q4K)",
243
+ approx_memory: "~0.6 GB (UQFF Q4K)",
244
+ description: "Qwen 3 0.6B, pre-quantised to 4-bit (UQFF Q4K). Loads directly with no \
245
+ on-device quantisation step; the lightest UQFF option.",
246
+ expected_size_bytes: 646_635_509,
247
+ },
248
+ UqffModelEntry {
249
+ id: "mistralrs-community/Qwen3-1.7B-UQFF",
250
+ name: "Qwen 3 1.7B (UQFF)",
251
+ display_name: "Qwen 3 1.7B (UQFF Q4K)",
252
+ approx_memory: "~1.5 GB (UQFF Q4K)",
253
+ description: "Qwen 3 1.7B, pre-quantised to 4-bit (UQFF Q4K). Loads directly with no \
254
+ on-device quantisation step.",
255
+ expected_size_bytes: 1_590_429_781,
256
+ },
257
+ UqffModelEntry {
258
+ id: "mistralrs-community/Qwen3-4B-UQFF",
259
+ name: "Qwen 3 4B (UQFF)",
260
+ display_name: "Qwen 3 4B (UQFF Q4K)",
261
+ approx_memory: "~2.9 GB (UQFF Q4K)",
262
+ description: "Qwen 3 4B, pre-quantised to 4-bit (UQFF Q4K). Loads directly with no \
263
+ on-device quantisation step.",
264
+ expected_size_bytes: 3_040_959_629,
265
+ },
266
+ UqffModelEntry {
267
+ id: "mistralrs-community/Qwen3-4B-Instruct-2507-UQFF",
268
+ name: "Qwen 3 4B Instruct 2507 (UQFF)",
269
+ display_name: "Qwen 3 4B Instruct 2507 (UQFF Q4K)",
270
+ approx_memory: "~2.9 GB (UQFF Q4K)",
271
+ description: "Qwen 3 4B Instruct (2507 refresh), pre-quantised to 4-bit (UQFF Q4K). \
272
+ A non-thinking instruct model that loads directly.",
273
+ expected_size_bytes: 3_040_959_629,
274
+ },
275
+ UqffModelEntry {
276
+ id: "mistralrs-community/Qwen3-4B-Thinking-2507-UQFF",
277
+ name: "Qwen 3 4B Thinking 2507 (UQFF)",
278
+ display_name: "Qwen 3 4B Thinking 2507 (UQFF Q4K)",
279
+ approx_memory: "~2.9 GB (UQFF Q4K)",
280
+ description: "Qwen 3 4B Thinking (2507 refresh), pre-quantised to 4-bit (UQFF Q4K). \
281
+ Emits a reasoning block before its reply; loads directly.",
282
+ expected_size_bytes: 3_040_959_629,
283
+ },
284
+ UqffModelEntry {
285
+ id: "mistralrs-community/Qwen3-8B-UQFF",
286
+ name: "Qwen 3 8B (UQFF)",
287
+ display_name: "Qwen 3 8B (UQFF Q4K)",
288
+ approx_memory: "~5.2 GB (UQFF Q4K)",
289
+ description: "Qwen 3 8B, pre-quantised to 4-bit (UQFF Q4K). Loads directly; needs \
290
+ roomier memory (12+ GB recommended).",
291
+ expected_size_bytes: 5_502_458_509,
292
+ },
293
+ UqffModelEntry {
294
+ id: "mistralrs-community/Qwen3-14B-UQFF",
295
+ name: "Qwen 3 14B (UQFF)",
296
+ display_name: "Qwen 3 14B (UQFF Q4K)",
297
+ approx_memory: "~9.0 GB (UQFF Q4K)",
298
+ description: "Qwen 3 14B, pre-quantised to 4-bit (UQFF Q4K). The largest UQFF option; \
299
+ desktop-class memory (16+ GB) recommended.",
300
+ expected_size_bytes: 9_426_174_577,
301
+ },
302
+];
303
+
304
+/// Map a HuggingFace repo id to a loadable [`UqffModelConfig`], or `None` if the
305
+/// id is not one of the Qwen 3 UQFF models Siti offers.
306
+///
307
+/// Every `mistralrs-community` Qwen 3 UQFF repo names its 4-bit shard
308
+/// `q4k-0.uqff`; passing that single shard is enough for mistral.rs to resolve
309
+/// the base config, tokenizer, and residual weights.
310
+#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
311
+pub(crate) fn uqff_config_for_model_id(id: &str) -> Option<UqffModelConfig> {
312
+ let entry = UQFF_MODELS.iter().find(|m| m.id == id)?;
313
+ Some(UqffModelConfig {
314
+ model_id: entry.id.to_string(),
315
+ files: vec!["q4k-0.uqff".to_string()],
316
+ display_name: entry.display_name.to_string(),
317
+ approx_memory: entry.approx_memory.to_string(),
318
+ chat_template: None,
319
+ })
320
+}
321
+
322
// ── Gemma (ISQ safetensors path, macOS only) ─────────────────────────────────
323
//
324
// Gemma cannot be loaded through the GGUF path Siti uses for every other model:
@@ -232,10 +354,12 @@ pub(crate) fn gemma2_2b_isq_config() -> IsqModelConfig {
354
}
355
356
/// A model selection resolved to the concrete engine config and load path it
235
-/// needs. Most models are GGUF; Gemma is the sole ISQ model (macOS only).
357
+/// needs. Most models are GGUF; Qwen 3 UQFF models use the UQFF path (all
358
+/// platforms), and Gemma is the sole ISQ model (macOS only).
359
#[cfg(any(target_os = "macos", target_os = "ios", target_os = "android"))]
360
pub(crate) enum ResolvedModel {
361
Gguf(GgufModelConfig),
362
+ Uqff(UqffModelConfig),
363
#[cfg(target_os = "macos")]
364
Isq(IsqModelConfig),
365
}
@@ -246,13 +370,14 @@ impl ResolvedModel {
370
pub(crate) fn display_name(&self) -> String {
371
match self {
372
ResolvedModel::Gguf(c) => c.display_name.clone(),
373
+ ResolvedModel::Uqff(c) => c.display_name.clone(),
374
#[cfg(target_os = "macos")]
375
ResolvedModel::Isq(c) => c.display_name.clone(),
376
}
377
}
378
379
/// Load the resolved model into the shared [`ENGINE`], dispatching to the
255
- /// GGUF or ISQ load path as appropriate.
380
+ /// GGUF, UQFF, or ISQ load path as appropriate.
381
pub(crate) async fn load(
382
self,
383
system_prompt: Option<String>,
@@ -260,6 +385,7 @@ impl ResolvedModel {
385
) -> Result<std::time::Duration, InferenceError> {
386
match self {
387
ResolvedModel::Gguf(c) => ENGINE.load_gguf_model(c, system_prompt, sampling).await,
388
+ ResolvedModel::Uqff(c) => ENGINE.load_uqff_model(c, system_prompt, sampling).await,
389
#[cfg(target_os = "macos")]
390
ResolvedModel::Isq(c) => ENGINE.load_isq_model(c, system_prompt, sampling).await,
391
}
@@ -274,6 +400,9 @@ pub(crate) fn resolve_model_id(id: &str) -> Option<ResolvedModel> {
400
if id == GEMMA2_2B_IT_ISQ_ID {
401
return Some(ResolvedModel::Isq(gemma2_2b_isq_config()));
402
}
403
+ if let Some(cfg) = uqff_config_for_model_id(id) {
404
+ return Some(ResolvedModel::Uqff(cfg));
405
+ }
406
config_for_model_id(id).map(ResolvedModel::Gguf)
407
}
408