Add UQFF model format
Seto Elkahfi committed
Aug 10, 2026 at 22:09 UTC
5a6615af4324286d68c1cf1ca7f08c7e465c7375
4 files changed
+24
-17
README.md
+2
-2
@@ -44,7 +44,7 @@ Onde is already shipping in real apps on the App Store and Google Play. Chat run
44
45
## Model formats
46
47
-Onde can load GGUF models and UQFF models through the same chat engine. UQFF is mistral.rs' native pre-quantized format: pass the base repository or local model directory for tokenizer/config resolution, plus the first UQFF shard or shorthand.
47
+Onde can load GGUF models and UQFF models through the same chat engine. UQFF is mistral.rs' native pre-quantized format. Point `model_id` at the UQFF export — the repository or local directory holding the shards, `residual.safetensors`, `config.json`, and the tokenizer — and name the first shard (or a shorthand such as `q4k`) in `files`. Use the UQFF repository, not the original unquantized one; the export is self-contained and everything is resolved relative to it.
48
49
```rust
50
use onde::inference::{ChatEngine, UqffModelConfig};
@@ -53,7 +53,7 @@ let engine = ChatEngine::new();
53
engine
54
.load_uqff_model(
55
UqffModelConfig {
56
- model_id: "google/gemma-4-E4B-it".into(),
56
+ model_id: "mistralrs-community/gemma-4-E4B-it-UQFF".into(),
57
files: vec!["q4k-0.uqff".into()],
58
display_name: "Gemma 4 E4B (UQFF Q4K)".into(),
59
approx_memory: "~2.5 GB (UQFF Q4K)".into(),
src/inference/engine.rs
+1
-1
@@ -2611,7 +2611,7 @@ mod tests {
2611
let result = engine
2612
.load_uqff_model(
2613
UqffModelConfig {
2614
- model_id: "google/gemma-4-E4B-it".into(),
2614
+ model_id: "mistralrs-community/gemma-4-E4B-it-UQFF".into(),
2615
files: Vec::new(),
2616
display_name: "Gemma 4 E4B (UQFF Q4K)".into(),
2617
approx_memory: "~2.5 GB (UQFF Q4K)".into(),
src/inference/ffi.rs
+8
-6
@@ -122,9 +122,9 @@ impl OndeChatEngine {
122
/// Load a UQFF model into the engine.
123
///
124
/// UQFF files are pre-quantised and load directly without runtime
125
- /// conversion. `config.model_id` is required for tokenizer/base model
126
- /// resolution, and `config.files` should contain the first UQFF shard or
127
- /// explicit shard filenames.
125
+ /// conversion. `config.model_id` names the UQFF export (not the original
126
+ /// unquantised repo), and `config.files` should contain the first UQFF
127
+ /// shard or explicit shard filenames.
128
///
129
/// Returns the wall-clock loading time in seconds.
130
pub async fn load_uqff_model(
@@ -331,9 +331,11 @@ pub fn default_model_config() -> GgufModelConfig {
331
332
/// Create a generic UQFF model configuration.
333
///
334
-/// `model_id` is the base HuggingFace repo or local model directory used for
335
-/// tokenizer/config resolution. `files` accepts a first shard (`q4k-0.uqff`),
336
-/// explicit shards, or mistral.rs shorthands such as `q4k` and `4`.
334
+/// `model_id` is the HuggingFace repo or local directory of the UQFF export
335
+/// (the shards plus `residual.safetensors`, `config.json`, and the tokenizer),
336
+/// not the original unquantised repo. `files` accepts a first shard
337
+/// (`q4k-0.uqff`), explicit shards, or mistral.rs shorthands such as `q4k`
338
+/// and `4`.
339
#[uniffi::export]
340
pub fn uqff_model_config(
341
model_id: String,
src/inference/types.rs
+13
-8
@@ -227,14 +227,19 @@ pub struct GgufModelConfig {
227
/// `UqffTextModelBuilder`.
228
///
229
/// UQFF files store pre-quantised weights and load directly
230
-/// without the runtime memory spike of in-situ quantisation. `model_id` is the
231
-/// base HuggingFace repository or local model directory used for tokenizer and
232
-/// config resolution, while `files` identifies the first UQFF shard or explicit
233
-/// UQFF shard files.
230
+/// without the runtime memory spike of in-situ quantisation. `model_id` points
231
+/// at the UQFF *export* — the directory or repository holding the shards
232
+/// alongside `residual.safetensors`, `config.json`, and the tokenizer — and
233
+/// `files` identifies the first UQFF shard or explicit UQFF shard files.
234
+///
235
+/// Note that `model_id` is **not** the original unquantised model repository:
236
+/// mistral.rs resolves the shards, the residual weights, and the JSON assets by
237
+/// sibling lookup next to `model_id`, so pointing it at the source repo fails
238
+/// with a missing-file error. A UQFF export is self-contained by design.
239
#[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
240
pub struct UqffModelConfig {
236
- /// HuggingFace base model repository ID or local model directory, e.g.
237
- /// `"google/gemma-4-E4B-it"`.
241
+ /// HuggingFace repository ID or local directory of the UQFF export, e.g.
242
+ /// `"mistralrs-community/gemma-4-E4B-it-UQFF"`.
243
pub model_id: String,
244
/// UQFF filename(s) within the repository. For sharded UQFFs, passing the
245
/// first shard, e.g. `["q4k-0.uqff"]`, is enough for mistral.rs to discover
@@ -555,14 +560,14 @@ mod tests {
560
#[test]
561
fn uqff_model_config_accepts_first_shard() {
562
let config = UqffModelConfig {
558
- model_id: "google/gemma-4-E4B-it".into(),
563
+ model_id: "mistralrs-community/gemma-4-E4B-it-UQFF".into(),
564
files: vec!["q4k-0.uqff".into()],
565
display_name: "Gemma 4 E4B (UQFF Q4K)".into(),
566
approx_memory: "~2.5 GB (UQFF Q4K)".into(),
567
chat_template: None,
568
};
569
565
- assert_eq!(config.model_id, "google/gemma-4-E4B-it");
570
+ assert_eq!(config.model_id, "mistralrs-community/gemma-4-E4B-it-UQFF");
571
assert_eq!(config.files, vec!["q4k-0.uqff"]);
572
}
573