@ondeinference / cli / commits / 6046f4e

Fix Qwen3 GGUF export and fine-tune NaN so fine-tuned models run

The fine-tune, quantize, and run pipeline produced GGUFs that loaded but emitted garbage, and the fine-tuner corrupted its own weights to NaN. Both are fixed and verified end to end. gguf.rs: - Write tensor dims reversed (innermost first). candle reverses them on read, so the straight order broke the token embedding. - Parse head_dim from config and emit attention.key_length and value_length. Qwen3-0.6B uses head_dim 128, not hidden_size/num_heads (64), and the loader otherwise assumes the latter. - Write tokenizer.ggml.token_type as an INT32 array; mistral.rs reads it with to_i32() and rejects a UINT32 array. - Choose the architecture from model_type so a Qwen3 model routes to mistral.rs's qwen3 loader instead of being garbled through the qwen2 one. finetune.rs: - Zero non-finite gradient elements before clipping, and skip a step when the global norm is still non-finite. A single NaN element was poisoning AdamW and corrupting every weight after the first step. Added two ignored integration tests: base Qwen3-0.6B exports and replies coherently in mistral.rs, and the full fine-tune, merge, export, run path yields a runnable model. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Seto Elkahfi committed Jun 27, 2026 at 00:31 UTC 6046f4e00715cb7da0b0b58171e1ccd4238548c9

2 files changed +343 -31

src/finetune.rs

+37 -14

@@ -287,39 +287,50 @@ fn run_finetune(

287 // We avoid `backward_step` so we can clip gradients between the

288 // backward pass and the AdamW update. This preserves AdamW's

289 // momentum / velocity state (unlike manually applying lr * grad).

290 - let grads = loss.backward().context("backward pass")?;

291 -

292 - // Compute global gradient L2 norm across all LoRA variables.

290 + let mut grads = loss.backward().context("backward pass")?;

291 +

292 + // Sanitize gradients before clipping. Backprop through the deep

293 + // stack can produce non-finite (NaN/Inf) gradient *elements* even

294 + // when the forward loss is finite; a single such element makes the

295 + // whole tensor's L2 norm non-finite, which would defeat norm-based

296 + // clipping (NaN > threshold is false → unclipped) and poison the

297 + // AdamW moment estimates, corrupting every weight from that step on.

298 + // Replace non-finite elements with zero so the remaining finite

299 + // gradient still trains those parameters this step.

300 let mut total_norm_sq: f64 = 0.0;

301 for var in &lora_vars {

302 if let Some(grad) = grads.get(var.as_tensor()) {

296 - let norm_sq = grad

303 + let clean = sanitize_grad(grad).context("sanitizing gradient")?;

304 + let norm_sq = clean

305 .sqr()

306 .and_then(|t| t.sum_all())

307 .and_then(|t| t.to_scalar::<f32>())

308 .unwrap_or(0.0) as f64;

309 total_norm_sq += norm_sq;

310 + grads.insert(var.as_tensor(), clean);

311 }

312 }

313 let total_norm = total_norm_sq.sqrt();

314

315 + // Nothing finite to learn from this step — skip the update so AdamW

316 + // state stays clean.

317 + if total_norm == 0.0 || !total_norm.is_finite() {

318 + continue;

319 + }

320 +

321 // If the global norm exceeds the threshold, scale every gradient

322 // down and insert the clipped version back into the GradStore.

308 - let grads = if total_norm > max_grad_norm {

323 + if total_norm > max_grad_norm {

324 let clip_coef = max_grad_norm / (total_norm + 1e-6);

310 - let mut clipped_grads = grads;

325 for var in &lora_vars {

312 - if let Some(grad) = clipped_grads.get(var.as_tensor()) {

326 + if let Some(grad) = grads.get(var.as_tensor()) {

327 let clipped = (grad * clip_coef).context("clipping gradient")?;

314 - clipped_grads.insert(var.as_tensor(), clipped);

328 + grads.insert(var.as_tensor(), clipped);

329 }

330 }

317 - clipped_grads

318 - } else {

319 - grads

320 - };

331 + }

332

322 - // AdamW step with (possibly clipped) gradients — preserves

333 + // AdamW step with sanitized, clipped gradients — preserves

334 // first-moment and second-moment estimates.

335 optimizer.step(&grads).context("optimizer step")?;

336

@@ -382,6 +393,17 @@ struct IndexJson {

393 // Private: tensor helpers

394 // ---------------------------------------------------------------------------

395

396 +/// Replace every non-finite element (NaN, ±Inf) of a gradient tensor with zero.

397 +///

398 +/// Finite values satisfy `|x| <= 1e30` (well below `f32::MAX ≈ 3.4e38`); NaN

399 +/// fails the comparison (NaN compares false) and ±Inf exceeds the bound, so

400 +/// both map to zero while genuine gradients pass through unchanged.

401 +fn sanitize_grad(grad: &Tensor) -> candle_core::Result<Tensor> {

402 + let finite = grad.abs()?.le(1e30f64)?;

403 + let zeros = grad.zeros_like()?;

404 + finite.where_cond(grad, &zeros)

405 +}

406 +

407 /// RMS layer normalisation: `x / rms(x) * weight`.

408 ///

409 /// `weight` has shape `[hidden_size]`; `x` has shape `[batch, seq, hidden_size]`.

@@ -644,7 +666,8 @@ impl TransformerLayer {

666 let scale = 1.0 / (self.head_dim as f64).sqrt();

667 let attn_weights = (q.matmul(&k.transpose(2, 3)?)? * scale)?;

668

647 - // Causal mask: add -1e9 to positions the query must not attend to.

669 + // Causal mask: add a large negative bias to positions the query must

670 + // not attend to.

671 let device = x.device();

672 let tril =

673 Tensor::tril2(seq, DType::F32, device)?.broadcast_as((b, self.num_heads, seq, seq))?;

src/gguf.rs

+306 -17

@@ -73,6 +73,11 @@ struct ModelConfig {

73 num_hidden_layers: usize,

74 num_attention_heads: usize,

75 num_key_value_heads: Option<usize>,

76 + /// Per-head attention dimension. Decoupled from `hidden_size / num_attention_heads`

77 + /// in Qwen3 (e.g. Qwen3-0.6B: hidden_size 1024, 16 heads, but head_dim 128).

78 + /// Absent in Qwen2/2.5 configs, where it equals `hidden_size / num_attention_heads`.

79 + #[serde(default)]

80 + head_dim: Option<usize>,

81 intermediate_size: usize,

82 vocab_size: usize,

83 #[serde(default = "default_rms_norm_eps")]

@@ -175,6 +180,7 @@ const GGUF_TYPE_Q8_0: u32 = 8;

180

181 // GGUF metadata value type IDs

182 const GGUF_META_U32: u32 = 4;

183 +const GGUF_META_INT32: u32 = 5;

184 const GGUF_META_F32: u32 = 6;

185 const GGUF_META_STRING: u32 = 8;

186 const GGUF_META_ARRAY: u32 = 9;

@@ -229,13 +235,16 @@ fn write_kv_string_array(w: &mut impl Write, key: &str, vals: &[String]) -> std:

235 Ok(())

236 }

237

232 -fn write_kv_u32_array(w: &mut impl Write, key: &str, vals: &[u32]) -> std::io::Result<()> {

238 +/// Write an array of signed 32-bit integers. GGUF declares `token_type` as an

239 +/// INT32 array; llama.cpp / mistral.rs read it with `to_i32()` and reject a

240 +/// UINT32-typed array, so the element type tag must be INT32 (not U32).

241 +fn write_kv_i32_array(w: &mut impl Write, key: &str, vals: &[i32]) -> std::io::Result<()> {

242 write_gguf_string(w, key)?;

243 write_u32(w, GGUF_META_ARRAY)?;

235 - write_u32(w, GGUF_META_U32)?;

244 + write_u32(w, GGUF_META_INT32)?;

245 write_u64(w, vals.len() as u64)?;

246 for &v in vals {

238 - write_u32(w, v)?;

247 + w.write_all(&v.to_le_bytes())?;

248 }

249 Ok(())

250 }

@@ -424,8 +433,8 @@ fn kv_string_array_size(key: &str, vals: &[String]) -> u64 {

433 size

434 }

435

427 -/// Compute the byte size of a KV u32-array entry.

428 -fn kv_u32_array_size(key: &str, vals: &[u32]) -> u64 {

436 +/// Compute the byte size of a KV i32-array entry (same layout as u32).

437 +fn kv_i32_array_size(key: &str, vals: &[i32]) -> u64 {

438 gguf_string_size(key) + 4 + 4 + 8 + (vals.len() as u64 * 4)

439 }

440

@@ -563,8 +572,8 @@ fn run_gguf_export(

572 .map(|at| (at.id, true))

573 .collect();

574

566 - // Token types: 1 = normal, 3 = control (special)

567 - let token_types: Vec<u32> = (0..vocab_size as u32)

575 + // Token types: 1 = normal, 3 = control (special). Stored as INT32 in GGUF.

576 + let token_types: Vec<i32> = (0..vocab_size as u32)

577 .map(|id| if special_set.contains_key(&id) { 3 } else { 1 })

578 .collect();

579

@@ -604,7 +613,12 @@ fn run_gguf_export(

613 let shape = tensor_view.shape();

614 let ndim = shape.len();

615 let num_elements: usize = shape.iter().product();

607 - let dims: Vec<u64> = shape.iter().map(|&d| d as u64).collect();

616 + // GGUF stores dimensions innermost-first (llama.cpp convention); the

617 + // reader reverses them back to logical order. safetensors shapes are

618 + // outermost-first, so write them reversed. For a Linear weight stored

619 + // as [out, in] this yields ne = [in, out], matching what mistral.rs /

620 + // candle expect after their `dimensions.reverse()`.

621 + let dims: Vec<u64> = shape.iter().rev().map(|&d| d as u64).collect();

622 let gguf_type = choose_gguf_type(&gguf_name, ndim, &config.dtype);

623 let data_size = tensor_data_size(num_elements, gguf_type);

624

@@ -650,7 +664,12 @@ fn run_gguf_export(

664 let num_layers = model_cfg.num_hidden_layers;

665 let num_heads = model_cfg.num_attention_heads;

666 let num_kv_heads_usize = model_cfg.num_key_value_heads.unwrap_or(num_heads);

653 - let head_dim = model_cfg.hidden_size / num_heads;

667 + // Qwen3 decouples head_dim from hidden_size / num_heads; honour the explicit

668 + // config value when present so synthesized bias tensors and the GGUF

669 + // key_length/value_length metadata match the real attention dimension.

670 + let head_dim = model_cfg

671 + .head_dim

672 + .unwrap_or(model_cfg.hidden_size / num_heads);

673

674 for i in 0..num_layers {

675 let bias_specs: &[(&str, usize)] = &[

@@ -709,14 +728,19 @@ fn run_gguf_export(

728

729 // GGUF architecture string.

730 //

712 - // In the llama.cpp / mistralrs GGUF ecosystem, Qwen3 is treated as a

713 - // Qwen2-variant (same tensor layout, plus optional QK-norm). Bartowski's

714 - // pre-built Qwen3 GGUFs all declare `general.architecture = "qwen2"`.

715 - // Using "qwen3" would cause loaders to hang or reject the file.

716 - let arch = "qwen2";

731 + // Qwen3 has the same tensor layout as Qwen2 plus mandatory QK-norm

732 + // (`attn_q_norm` / `attn_k_norm`). mistral.rs ships dedicated loaders for

733 + // each — `quantized_qwen` for `qwen2`, `quantized_qwen3` for `qwen3` — so

734 + // declare the architecture that matches the source `model_type`. Routing a

735 + // Qwen3 model through the qwen2 loader loads but produces garbage output.

736 + let arch = if model_cfg.model_type.starts_with("qwen3") {

737 + "qwen3"

738 + } else {

739 + "qwen2"

740 + };

741

742 // Count metadata KV entries

719 - let num_metadata: u64 = 18;

743 + let num_metadata: u64 = 20;

744

745 // Pre-compute header + metadata + tensor info byte size

746 let mut header_size: u64 = 0;

@@ -732,12 +756,14 @@ fn run_gguf_export(

756 header_size += kv_u32_size(&format!("{arch}.feed_forward_length"));

757 header_size += kv_u32_size(&format!("{arch}.attention.head_count"));

758 header_size += kv_u32_size(&format!("{arch}.attention.head_count_kv"));

759 + header_size += kv_u32_size(&format!("{arch}.attention.key_length"));

760 + header_size += kv_u32_size(&format!("{arch}.attention.value_length"));

761 header_size += kv_f32_size(&format!("{arch}.attention.layer_norm_rms_epsilon"));

762 header_size += kv_f32_size(&format!("{arch}.rope.freq_base"));

763 header_size += kv_u32_size(&format!("{arch}.context_length"));

764 header_size += kv_string_size("tokenizer.ggml.model", "gpt2");

765 header_size += kv_string_array_size("tokenizer.ggml.tokens", &tokens);

740 - header_size += kv_u32_array_size("tokenizer.ggml.token_type", &token_types);

766 + header_size += kv_i32_array_size("tokenizer.ggml.token_type", &token_types);

767 header_size += kv_string_array_size("tokenizer.ggml.merges", &merges);

768 header_size += kv_u32_size("tokenizer.ggml.bos_token_id");

769 header_size += kv_u32_size("tokenizer.ggml.eos_token_id");

@@ -800,6 +826,19 @@ fn run_gguf_export(

826 &format!("{arch}.attention.head_count_kv"),

827 num_kv_heads,

828 )?;

829 + // Per-head dimension. mistral.rs / llama.cpp set head_dim = key_length and

830 + // fall back to embedding_length / head_count when these keys are absent —

831 + // which is wrong for Qwen3 (head_dim is decoupled), so always emit them.

832 + write_kv_u32(

833 + &mut w,

834 + &format!("{arch}.attention.key_length"),

835 + head_dim as u32,

836 + )?;

837 + write_kv_u32(

838 + &mut w,

839 + &format!("{arch}.attention.value_length"),

840 + head_dim as u32,

841 + )?;

842 write_kv_f32(

843 &mut w,

844 &format!("{arch}.attention.layer_norm_rms_epsilon"),

@@ -817,7 +856,7 @@ fn run_gguf_export(

856 )?;

857 write_kv_string(&mut w, "tokenizer.ggml.model", "gpt2")?;

858 write_kv_string_array(&mut w, "tokenizer.ggml.tokens", &tokens)?;

820 - write_kv_u32_array(&mut w, "tokenizer.ggml.token_type", &token_types)?;

859 + write_kv_i32_array(&mut w, "tokenizer.ggml.token_type", &token_types)?;

860 write_kv_string_array(&mut w, "tokenizer.ggml.merges", &merges)?;

861 write_kv_u32(&mut w, "tokenizer.ggml.bos_token_id", bos_id)?;

862 write_kv_u32(&mut w, "tokenizer.ggml.eos_token_id", eos_id)?;

@@ -899,3 +938,253 @@ fn run_gguf_export(

938

939 Ok(())

940 }

941 +

942 +#[cfg(test)]

943 +mod tests {

944 + use super::*;

945 +

946 + /// Resolve the locally cached merged Qwen3-0.6B model directory, if present.

947 + /// Returns `None` so the heavy integration test can be skipped on machines

948 + /// that have not run a fine-tune.

949 + fn merged_qwen3_dir() -> Option<PathBuf> {

950 + let dir = dirs::home_dir()?

951 + .join("Library/Group Containers/group.com.ondeinference.apps")

952 + .join("models/hub/models--Qwen--Qwen3-0.6B/snapshots/merged-model");

953 + if dir.join("config.json").exists() && dir.join("model.safetensors").exists() {

954 + Some(dir)

955 + } else {

956 + None

957 + }

958 + }

959 +

960 + /// Resolve the cached base Qwen3-0.6B snapshot (the hex commit dir), if present.

961 + fn base_qwen3_dir() -> Option<PathBuf> {

962 + let snapshots = dirs::home_dir()?

963 + .join("Library/Group Containers/group.com.ondeinference.apps")

964 + .join("models/hub/models--Qwen--Qwen3-0.6B/snapshots");

965 + let entry = std::fs::read_dir(&snapshots).ok()?.flatten().find(|e| {

966 + let n = e.file_name();

967 + let n = n.to_string_lossy();

968 + n.len() >= 7 && n.chars().all(|c| c.is_ascii_hexdigit())

969 + })?;

970 + let dir = entry.path();

971 + (dir.join("config.json").exists() && dir.join("model.safetensors").exists()).then_some(dir)

972 + }

973 +

974 + /// Load a GGUF and generate a reply via the same `onde::mistralrs` engine the

975 + /// Onde SDK uses. Returns the assistant text. Panics on model/inference error.

976 + fn run_gguf_inference(gguf_path: &std::path::Path, user: &str) -> String {

977 + use onde::mistralrs::{

978 + GgufModelBuilder, RequestBuilder, Response, TextMessageRole, TokenSource,

979 + };

980 + let rt = tokio::runtime::Runtime::new().unwrap();

981 + let path = gguf_path.to_path_buf();

982 + let user = user.to_string();

983 + rt.block_on(async move {

984 + let file_name = path.file_name().unwrap().to_string_lossy().to_string();

985 + let model_root = path.parent().unwrap().to_string_lossy().to_string();

986 + let model = GgufModelBuilder::new(&model_root, vec![file_name])

987 + .with_token_source(TokenSource::None)

988 + .build()

989 + .await

990 + .expect("exported GGUF must load in mistral.rs");

991 + let req = RequestBuilder::new()

992 + .set_sampler_max_len(512)

993 + .add_message(TextMessageRole::User, &user);

994 + let mut stream = model

995 + .stream_chat_request(req)

996 + .await

997 + .expect("inference request must start");

998 + let mut out = String::new();

999 + while let Some(resp) = stream.next().await {

1000 + match resp {

1001 + Response::Chunk(chunk) => {

1002 + if let Some(choice) = chunk.choices.first()

1003 + && let Some(delta) = &choice.delta.content

1004 + {

1005 + out.push_str(delta);

1006 + }

1007 + }

1008 + Response::Done(_) => break,

1009 + Response::ModelError(msg, _) => panic!("model error: {msg}"),

1010 + Response::InternalError(e) => panic!("internal error: {e}"),

1011 + Response::ValidationError(e) => panic!("validation error: {e}"),

1012 + _ => {}

1013 + }

1014 + }

1015 + out

1016 + })

1017 + }

1018 +

1019 + /// Full v1 pipeline regression guard: fine-tune Qwen3-0.6B → merge the LoRA

1020 + /// adapter → export GGUF → load and run in mistral.rs. Proves the whole

1021 + /// fine-tune → quantize → run path produces a valid, runnable model.

1022 + ///

1023 + /// Ignored by default (trains + loads ~1 GB on Metal/CPU); run with

1024 + /// `cargo test gguf::tests::finetune_merge_export_run -- --ignored --nocapture`.

1025 + #[test]

1026 + #[ignore = "heavy: fine-tunes, merges, exports ~1GB GGUF and runs it"]

1027 + fn finetune_merge_export_run() {

1028 + let Some(base_dir) = base_qwen3_dir() else {

1029 + eprintln!("base Qwen3-0.6B not found locally; skipping");

1030 + return;

1031 + };

1032 + let data_path = dirs::home_dir()

1033 + .unwrap()

1034 + .join(".onde/datasets/Qwen/Qwen3-0.6B/train.jsonl");

1035 + if !data_path.exists() {

1036 + eprintln!("training data {} not found; skipping", data_path.display());

1037 + return;

1038 + }

1039 +

1040 + let work = std::env::temp_dir().join("onde-e2e-test");

1041 + let _ = std::fs::remove_dir_all(&work);

1042 + std::fs::create_dir_all(&work).unwrap();

1043 +

1044 + // 1. Fine-tune.

1045 + let lora_dir = work.join("lora");

1046 + let (ft_tx, mut ft_rx) = tokio::sync::mpsc::unbounded_channel();

1047 + crate::finetune::start_finetune(

1048 + crate::finetune::FineTuneConfig {

1049 + model_dir: base_dir.clone(),

1050 + data_path,

1051 + output_dir: lora_dir.clone(),

1052 + lora_rank: 8,

1053 + lora_alpha: 16.0,

1054 + learning_rate: std::env::var("ONDE_TEST_LR")

1055 + .ok()

1056 + .and_then(|v| v.parse().ok())

1057 + .unwrap_or(1e-4),

1058 + epochs: 2,

1059 + max_seq_len: 512,

1060 + },

1061 + ft_tx,

1062 + );

1063 + let mut adapter_path = None;

1064 + let mut last_loss = f32::NAN;

1065 + let rt = tokio::runtime::Runtime::new().unwrap();

1066 + rt.block_on(async {

1067 + while let Some(p) = ft_rx.recv().await {

1068 + match p {

1069 + crate::finetune::FineTuneProgress::Training { loss, .. } => last_loss = loss,

1070 + crate::finetune::FineTuneProgress::Done { adapter_path: a } => {

1071 + adapter_path = Some(a);

1072 + break;

1073 + }

1074 + crate::finetune::FineTuneProgress::Failed(e) => panic!("fine-tune failed: {e}"),

1075 + _ => {}

1076 + }

1077 + }

1078 + });

1079 + let adapter_path = adapter_path.expect("fine-tune must produce an adapter");

1080 + eprintln!("[e2e] fine-tune done, last loss {last_loss}");

1081 + assert!(last_loss.is_finite(), "training loss is NaN/Inf — training diverged");

1082 +

1083 + // 2. Merge.

1084 + let merged_dir = work.join("merged");

1085 + let (mg_tx, mut mg_rx) = tokio::sync::mpsc::unbounded_channel();

1086 + crate::merge::start_merge(

1087 + crate::merge::MergeConfig {

1088 + base_dir,

1089 + adapter_path,

1090 + output_dir: merged_dir.clone(),

1091 + },

1092 + mg_tx,

1093 + );

1094 + let mut merged_ok = false;

1095 + rt.block_on(async {

1096 + while let Some(p) = mg_rx.recv().await {

1097 + match p {

1098 + crate::merge::MergeProgress::Done { .. } => {

1099 + merged_ok = true;

1100 + break;

1101 + }

1102 + crate::merge::MergeProgress::Failed(e) => panic!("merge failed: {e}"),

1103 + _ => {}

1104 + }

1105 + }

1106 + });

1107 + assert!(merged_ok, "merge did not complete");

1108 +

1109 + // 3. Export GGUF (the dtype the CLI ships).

1110 + let gguf_path = work.join("finetuned-q8_0.gguf");

1111 + let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();

1112 + run_gguf_export(

1113 + &GgufConfig {

1114 + model_dir: merged_dir,

1115 + output_path: gguf_path.clone(),

1116 + dtype: GgufDtype::Q8_0,

1117 + },

1118 + &tx,

1119 + )

1120 + .expect("GGUF export should succeed");

1121 +

1122 + // 4. Load + run in mistral.rs. Query in the training distribution

1123 + // (the dataset is a Javanese-language assistant) so a lightly

1124 + // fine-tuned model has something to say.

1125 + let reply = run_gguf_inference(&gguf_path, "Apa bahasa Jawa dari \"terima kasih\"?");

1126 + eprintln!("[e2e] fine-tuned GGUF reply: {reply:?}");

1127 + assert!(

1128 + !reply.trim().is_empty(),

1129 + "fine-tuned GGUF produced an empty reply — not runnable"

1130 + );

1131 + }

1132 +

1133 + /// End-to-end check: export a Qwen3 safetensors model to GGUF, then load and

1134 + /// run it through the same `onde::mistralrs` engine the Onde SDK uses.

1135 + ///

1136 + /// This is the regression guard for the Qwen3 `head_dim` fix: with the wrong

1137 + /// per-head dimension the GGUF loads with a reshape mismatch and inference

1138 + /// fails. Ignored by default (downloads/loads ~1 GB, needs Metal/CPU);

1139 + /// run with `cargo test gguf -- --ignored --nocapture`.

1140 + #[test]

1141 + #[ignore = "heavy: exports ~1GB GGUF and loads it in mistral.rs"]

1142 + fn exported_qwen3_gguf_is_runnable() {

1143 + // Allow overriding the source model dir and dtype to compare

1144 + // base-vs-merged and F16-vs-Q8_0 without changing code.

1145 + let model_dir = match std::env::var("ONDE_TEST_MODEL_DIR") {

1146 + Ok(d) => PathBuf::from(d),

1147 + // Default to the known-good base snapshot so this isolates the

1148 + // exporter; `finetune_merge_export_run` covers the fine-tuned path.

1149 + Err(_) => match base_qwen3_dir().or_else(merged_qwen3_dir) {

1150 + Some(d) => d,

1151 + None => {

1152 + eprintln!("Qwen3-0.6B model not found locally; skipping");

1153 + return;

1154 + }

1155 + },

1156 + };

1157 + let dtype = match std::env::var("ONDE_TEST_DTYPE").as_deref() {

1158 + Ok("f16") => GgufDtype::F16,

1159 + _ => GgufDtype::Q8_0,

1160 + };

1161 + eprintln!("[test] model_dir={} dtype={}", model_dir.display(),

1162 + if matches!(dtype, GgufDtype::F16) { "f16" } else { "q8_0" });

1163 +

1164 + let out_dir = std::env::temp_dir().join("onde-gguf-test");

1165 + std::fs::create_dir_all(&out_dir).unwrap();

1166 + let output_path = out_dir.join("qwen3-0.6b-test.gguf");

1167 + let _ = std::fs::remove_file(&output_path);

1168 +

1169 + // 1. Export GGUF.

1170 + let (tx, _rx) = tokio::sync::mpsc::unbounded_channel();

1171 + run_gguf_export(

1172 + &GgufConfig {

1173 + model_dir,

1174 + output_path: output_path.clone(),

1175 + dtype,

1176 + },

1177 + &tx,

1178 + )

1179 + .expect("GGUF export should succeed");

1180 + assert!(output_path.exists(), "GGUF file was not written");

1181 +

1182 + // 2. Load + run it through mistral.rs (the Onde SDK's engine).

1183 + let reply = run_gguf_inference(&output_path, "Say hello in one short sentence.");