@ondeinference / cli / commits / 7bbcad3

Add Qwen3 GGUF support and synthesize missing bias tensors

- Support Qwen3 chat templates and metadata in GGUF export - Synthesize zero-filled Q/K/V bias tensors for Qwen3 models - Display fine-tune option only for safetensors models - Improve chat model loading logs and error handling

keypair34 committed Apr 29, 2026 at 22:58 UTC 7bbcad392efa4f5804b7f19af4cea9a7448bd488

4 files changed +193 -46

src/app.rs

+17 -6

@@ -326,6 +326,9 @@ pub struct App {

326 // adapters discovered on the model detail screen

327 pub adapter_list: Vec<AdapterEntry>,

328 pub adapter_cursor: usize,

329 + /// `true` when the model on the detail screen has `config.json` (safetensors)

330 + /// and can therefore be fine-tuned. GGUF-only models set this to `false`.

331 + pub model_detail_can_finetune: bool,

332 // GGUF detail + upload

333 pub selected_gguf: Option<AdapterEntry>,

334 pub upload_running: bool,

@@ -407,6 +410,7 @@ impl App {

410 merged_model_dir: None,

411 adapter_list: Vec::new(),

412 adapter_cursor: 0,

413 + model_detail_can_finetune: false,

414 selected_gguf: None,

415 upload_running: false,

416 upload_progress: None,

@@ -1248,17 +1252,28 @@ fn handle_key_downloads(

1252 }

1253 (Enter, _) if !app.downloads.is_empty() => {

1254 // Scan for existing adapters before entering the detail screen

1255 + let can_finetune;

1256 if let Some(model) = app.downloads.get(app.downloads_cursor) {

1257 let resolved = resolve_hf_cache_path(&model.model_id);

1258 + can_finetune = !resolved.is_empty()

1259 + && std::path::Path::new(&resolved).join("config.json").exists();

1260 app.adapter_list = scan_adapters(&resolved);

1261 app.adapter_cursor = 0;

1262 + } else {

1263 + can_finetune = false;

1264 }

1265 + app.model_detail_can_finetune = can_finetune;

1266 app.screen = Screen::ModelDetail;

1267 if app.adapter_list.is_empty() {

1268 app.status = Status::neutral("Model details.");

1269 } else {

1270 + let ft_hint = if can_finetune {

1271 + " f · fine-tune new"

1272 + } else {

1273 + ""

1274 + };

1275 app.status = Status::success(format!(

1261 - "{} adapter{} found. Enter · merge & export f · fine-tune new",

1276 + "{} adapter{} found. Enter · merge & export{ft_hint}",

1277 app.adapter_list.len(),

1278 if app.adapter_list.len() == 1 { "" } else { "s" }

1279 ));

@@ -1385,15 +1400,11 @@ fn handle_key_model_detail(

1400 app.status = Status::error("No adapter selected.");

1401 }

1402 }

1388 - (Char('f'), KeyModifiers::NONE) => {

1403 + (Char('f'), KeyModifiers::NONE) if app.model_detail_can_finetune => {

1404 if let Some(model) = app.downloads.get(app.downloads_cursor) {

1405 let resolved = resolve_hf_cache_path(&model.model_id);

1406 if resolved.is_empty() {

1407 app.status = Status::error("Model not downloaded locally.");

1393 - } else if !std::path::Path::new(&resolved).join("config.json").exists() {

1394 - app.status = Status::error(

1395 - "Fine-tuning requires a safetensors model (GGUF not supported).",

1396 - );

1408 } else {

1409 app.finetune_model_id = model.model_id.clone();

1410 app.finetune_model_dir = resolved;

src/chat.rs

+7 -1

@@ -168,13 +168,19 @@ pub async fn start_chat(

168 };

169

170 // ── 3. Build and load the model ──────────────────────────────────────────

171 + log::info!("[chat] loading GGUF from dir={model_dir:?} file={file_name:?}");

172 let model = match GgufModelBuilder::new(&model_dir, vec![file_name.clone()])

173 .with_token_source(TokenSource::None)

174 + .with_logging()

175 .build()

176 .await

177 {

176 - Ok(m) => m,

178 + Ok(m) => {

179 + log::info!("[chat] model loaded successfully");

180 + m

181 + }

182 Err(e) => {

183 + log::error!("[chat] model load failed: {e}");

184 let _ = progress_tx.send(ChatProgress::Error(format!(

185 "Failed to load model \"{file_name}\": {e}"

186 )));

src/gguf.rs

+159 -32

@@ -83,6 +83,11 @@ struct ModelConfig {

83 max_position_embeddings: usize,

84 #[serde(default)]

85 tie_word_embeddings: bool,

86 + /// HuggingFace model type, e.g. "qwen2" or "qwen3".

87 + /// Kept for future architecture-specific logic.

88 + #[allow(dead_code)]

89 + #[serde(default = "default_model_type")]

90 + model_type: String,

91 }

93 fn default_rms_norm_eps() -> f64 {

@@ -94,6 +99,9 @@ fn default_rope_theta() -> f64 {

99 fn default_max_position_embeddings() -> usize {

100 32768

101 }

102 +fn default_model_type() -> String {

103 + "qwen2".to_string()

104 +}

105

106 #[derive(serde::Deserialize)]

107 struct TokenizerJson {

@@ -149,6 +157,17 @@ const GGUF_MAGIC: u32 = 0x46554747;

157 const GGUF_VERSION: u32 = 3;

158 const ALIGNMENT: u64 = 32;

159

160 +/// Standard Qwen ChatML template used as a fallback when `tokenizer_config.json`

161 +/// does not provide one. Matches the template shipped in Bartowski's Qwen GGUFs.

162 +const QWEN_CHATML_TEMPLATE: &str = concat!(

163 + "{%- for message in messages %}",

164 + "{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}",

165 + "{%- endfor %}",

166 + "{%- if add_generation_prompt %}",

167 + "{{'<|im_start|>assistant\n'}}",

168 + "{%- endif %}",

169 +);

170 +

171 // GGUF tensor type IDs

172 const GGUF_TYPE_F32: u32 = 0;

173 const GGUF_TYPE_F16: u32 = 1;

@@ -266,6 +285,11 @@ fn map_tensor_name(hf_name: &str) -> Option<String> {

285 "self_attn.k_proj.weight" => "attn_k.weight",

286 "self_attn.v_proj.weight" => "attn_v.weight",

287 "self_attn.o_proj.weight" => "attn_output.weight",

288 + // Bias tensors (present in Qwen2/2.5, absent in Qwen3)

289 + "self_attn.q_proj.bias" => "attn_q.bias",

290 + "self_attn.k_proj.bias" => "attn_k.bias",

291 + "self_attn.v_proj.bias" => "attn_v.bias",

292 + // QK-Norm (Qwen3)

293 "self_attn.q_norm.weight" => "attn_q_norm.weight",

294 "self_attn.k_norm.weight" => "attn_k_norm.weight",

295 "mlp.gate_proj.weight" => "ffn_gate.weight",

@@ -459,6 +483,47 @@ fn run_gguf_export(

483 let tok: TokenizerJson =

484 serde_json::from_str(&tok_text).context("failed to parse tokenizer.json")?;

485

486 + // Read the chat template from tokenizer_config.json.

487 + // This is needed by mistral.rs / llama.cpp to format multi-turn conversations.

488 + //

489 + // The field can be either:

490 + // - a plain string (Qwen2/2.5)

491 + // - an array of {"name": "...", "template": "..."} objects (Qwen3)

492 + //

493 + // When it's an array, pick the "default" entry, or fall back to the first.

494 + // If the file is missing or the field is absent, use the standard Qwen

495 + // ChatML template so the GGUF is always loadable for chat.

496 + let chat_template: String = {

497 + let tok_config_path = config.model_dir.join("tokenizer_config.json");

498 + let from_file = std::fs::read_to_string(&tok_config_path)

499 + .ok()

500 + .and_then(|text| {

501 + let v: serde_json::Value = serde_json::from_str(&text).ok()?;

502 + let ct = v.get("chat_template")?;

503 +

504 + // Plain string

505 + if let Some(s) = ct.as_str() {

506 + return Some(s.to_string());

507 + }

508 +

509 + // Array of objects — pick "default", else first entry

510 + if let Some(arr) = ct.as_array() {

511 + let default = arr.iter().find(|entry| {

512 + entry.get("name").and_then(|n| n.as_str()) == Some("default")

513 + });

514 + let chosen = default.or_else(|| arr.first());

515 + return chosen

516 + .and_then(|entry| entry.get("template"))

517 + .and_then(|t| t.as_str())

518 + .map(String::from);

519 + }

520 +

521 + None

522 + });

523 +

524 + from_file.unwrap_or_else(|| QWEN_CHATML_TEMPLATE.to_string())

525 + };

526 +

527 // Build ordered token list sorted by ID

528 let vocab_size = model_cfg.vocab_size;

529 let mut id_to_token: Vec<(u32, String)> =

@@ -576,6 +641,47 @@ fn run_gguf_export(

641 });

642 }

643

644 + // Synthesize zero-filled Q/K/V bias tensors for layers that lack them.

645 + //

646 + // mistralrs expects `blk.{i}.attn_q.bias` etc. for the qwen2 architecture.

647 + // Qwen2/2.5 have these in the safetensors; Qwen3 does not. Bartowski's

648 + // llama.cpp-converted Qwen3 GGUFs include zero-filled bias tensors.

649 + // We do the same here so the GGUF loads correctly for both model families.

650 + let num_layers = model_cfg.num_hidden_layers;

651 + let num_heads = model_cfg.num_attention_heads;

652 + let num_kv_heads_usize = model_cfg.num_key_value_heads.unwrap_or(num_heads);

653 + let head_dim = model_cfg.hidden_size / num_heads;

654 +

655 + for i in 0..num_layers {

656 + let bias_specs: &[(&str, usize)] = &[

657 + (&format!("blk.{i}.attn_q.bias"), num_heads * head_dim),

658 + (

659 + &format!("blk.{i}.attn_k.bias"),

660 + num_kv_heads_usize * head_dim,

661 + ),

662 + (

663 + &format!("blk.{i}.attn_v.bias"),

664 + num_kv_heads_usize * head_dim,

665 + ),

666 + ];

667 + for (gguf_name, num_elements) in bias_specs {

668 + if !entries.iter().any(|e| e.gguf_name == *gguf_name) {

669 + // 1-D bias → always F32

670 + let data_size = tensor_data_size(*num_elements, GGUF_TYPE_F32);

671 + entries.push(TensorEntry {

672 + gguf_name: gguf_name.to_string(),

673 + hf_name: String::new(), // sentinel: no safetensors source

674 + dims: vec![*num_elements as u64],

675 + ndim: 1,

676 + num_elements: *num_elements,

677 + gguf_type: GGUF_TYPE_F32,

678 + data_size,

679 + offset: 0,

680 + });

681 + }

682 + }

683 + }

684 +

685 // Compute offsets (relative to start of data section, 32-byte aligned)

686 let mut data_offset: u64 = 0;

687 for entry in &mut entries {

@@ -601,8 +707,16 @@ fn run_gguf_export(

707 GgufDtype::Q8_0 => 7,

708 };

709

710 + // GGUF architecture string.

711 + //

712 + // In the llama.cpp / mistralrs GGUF ecosystem, Qwen3 is treated as a

713 + // Qwen2-variant (same tensor layout, plus optional QK-norm). Bartowski's

714 + // pre-built Qwen3 GGUFs all declare `general.architecture = "qwen2"`.

715 + // Using "qwen3" would cause loaders to hang or reject the file.

716 + let arch = "qwen2";

717 +

718 // Count metadata KV entries

605 - let num_metadata: u64 = 17; // all the keys listed below

719 + let num_metadata: u64 = 18;

720

721 // Pre-compute header + metadata + tensor info byte size

722 let mut header_size: u64 = 0;

@@ -610,23 +724,24 @@ fn run_gguf_export(

724 header_size += 4 + 4 + 8 + 8;

725

726 // Metadata sizes

613 - header_size += kv_string_size("general.architecture", "qwen2");

727 + header_size += kv_string_size("general.architecture", arch);

728 header_size += kv_string_size("general.name", "onde-finetuned");

729 header_size += kv_u32_size("general.file_type");

616 - header_size += kv_u32_size("qwen2.block_count");

617 - header_size += kv_u32_size("qwen2.embedding_length");

618 - header_size += kv_u32_size("qwen2.feed_forward_length");

619 - header_size += kv_u32_size("qwen2.attention.head_count");

620 - header_size += kv_u32_size("qwen2.attention.head_count_kv");

621 - header_size += kv_f32_size("qwen2.attention.layer_norm_rms_epsilon");

622 - header_size += kv_f32_size("qwen2.rope.freq_base");

623 - header_size += kv_u32_size("qwen2.context_length");

730 + header_size += kv_u32_size(&format!("{arch}.block_count"));

731 + header_size += kv_u32_size(&format!("{arch}.embedding_length"));

732 + header_size += kv_u32_size(&format!("{arch}.feed_forward_length"));

733 + header_size += kv_u32_size(&format!("{arch}.attention.head_count"));

734 + header_size += kv_u32_size(&format!("{arch}.attention.head_count_kv"));

735 + header_size += kv_f32_size(&format!("{arch}.attention.layer_norm_rms_epsilon"));

736 + header_size += kv_f32_size(&format!("{arch}.rope.freq_base"));

737 + header_size += kv_u32_size(&format!("{arch}.context_length"));

738 header_size += kv_string_size("tokenizer.ggml.model", "gpt2");

739 header_size += kv_string_array_size("tokenizer.ggml.tokens", &tokens);

740 header_size += kv_u32_array_size("tokenizer.ggml.token_type", &token_types);

741 header_size += kv_string_array_size("tokenizer.ggml.merges", &merges);

742 header_size += kv_u32_size("tokenizer.ggml.bos_token_id");

743 header_size += kv_u32_size("tokenizer.ggml.eos_token_id");

744 + header_size += kv_string_size("tokenizer.chat_template", &chat_template);

745

746 // Tensor info sizes

747 for entry in &entries {

@@ -657,39 +772,47 @@ fn run_gguf_export(

772 write_u64(&mut w, num_metadata)?;

773

774 // Metadata KV pairs

660 - write_kv_string(&mut w, "general.architecture", "qwen2")?;

775 + write_kv_string(&mut w, "general.architecture", arch)?;

776 write_kv_string(&mut w, "general.name", "onde-finetuned")?;

777 write_kv_u32(&mut w, "general.file_type", file_type_val)?;

778 write_kv_u32(

779 &mut w,

665 - "qwen2.block_count",

780 + &format!("{arch}.block_count"),

781 model_cfg.num_hidden_layers as u32,

782 )?;

783 write_kv_u32(

784 &mut w,

670 - "qwen2.embedding_length",

785 + &format!("{arch}.embedding_length"),

786 model_cfg.hidden_size as u32,

787 )?;

788 write_kv_u32(

789 &mut w,

675 - "qwen2.feed_forward_length",

790 + &format!("{arch}.feed_forward_length"),

791 model_cfg.intermediate_size as u32,

792 )?;

793 write_kv_u32(

794 &mut w,

680 - "qwen2.attention.head_count",

795 + &format!("{arch}.attention.head_count"),

796 model_cfg.num_attention_heads as u32,

797 )?;

683 - write_kv_u32(&mut w, "qwen2.attention.head_count_kv", num_kv_heads)?;

798 + write_kv_u32(

799 + &mut w,

800 + &format!("{arch}.attention.head_count_kv"),

801 + num_kv_heads,

802 + )?;

803 write_kv_f32(

804 &mut w,

686 - "qwen2.attention.layer_norm_rms_epsilon",

805 + &format!("{arch}.attention.layer_norm_rms_epsilon"),

806 model_cfg.rms_norm_eps as f32,

807 )?;

689 - write_kv_f32(&mut w, "qwen2.rope.freq_base", model_cfg.rope_theta as f32)?;

808 + write_kv_f32(

809 + &mut w,

810 + &format!("{arch}.rope.freq_base"),

811 + model_cfg.rope_theta as f32,

812 + )?;

813 write_kv_u32(

814 &mut w,

692 - "qwen2.context_length",

815 + &format!("{arch}.context_length"),

816 model_cfg.max_position_embeddings as u32,

817 )?;

818 write_kv_string(&mut w, "tokenizer.ggml.model", "gpt2")?;

@@ -698,6 +821,7 @@ fn run_gguf_export(

821 write_kv_string_array(&mut w, "tokenizer.ggml.merges", &merges)?;

822 write_kv_u32(&mut w, "tokenizer.ggml.bos_token_id", bos_id)?;

823 write_kv_u32(&mut w, "tokenizer.ggml.eos_token_id", eos_id)?;

824 + write_kv_string(&mut w, "tokenizer.chat_template", &chat_template)?;

825

826 // Tensor info entries

827 for entry in &entries {

@@ -726,19 +850,22 @@ fn run_gguf_export(

850 name: entry.gguf_name.clone(),

851 });

852

729 - // Load tensor from safetensors and convert to f32 on CPU

730 - let tensor = safetensors

731 - .load(&entry.hf_name, &Device::Cpu)

732 - .with_context(|| format!("failed to load tensor {}", entry.hf_name))?;

733 - let tensor_f32 = tensor

734 - .to_dtype(candle_core::DType::F32)

735 - .with_context(|| format!("failed to convert {} to f32", entry.hf_name))?;

736 - let data = tensor_f32

737 - .flatten_all()

738 - .context("failed to flatten tensor")?;

739 - let data = data

740 - .to_vec1::<f32>()

741 - .context("failed to extract f32 data from tensor")?;

853 + // Load tensor data. Synthetic entries (empty hf_name) get zeros.

854 + let data: Vec<f32> = if entry.hf_name.is_empty() {

855 + vec![0.0f32; entry.num_elements]

856 + } else {

857 + let tensor = safetensors

858 + .load(&entry.hf_name, &Device::Cpu)

859 + .with_context(|| format!("failed to load tensor {}", entry.hf_name))?;

860 + let tensor_f32 = tensor

861 + .to_dtype(candle_core::DType::F32)

862 + .with_context(|| format!("failed to convert {} to f32", entry.hf_name))?;

863 + let flat = tensor_f32

864 + .flatten_all()

865 + .context("failed to flatten tensor")?;

866 + flat.to_vec1::<f32>()

867 + .context("failed to extract f32 data from tensor")?

868 + };

869

870 match entry.gguf_type {

871 GGUF_TYPE_F32 => write_tensor_f32(&mut w, &data)?,

src/ui.rs

+10 -7

@@ -1227,11 +1227,12 @@ fn render_model_detail(frame: &mut Frame, app: &App, area: Rect) {

1227 /// Render the selectable artifact list inside the Model Detail screen.

1228 fn render_model_detail_adapters(frame: &mut Frame, app: &App, area: Rect) {

1229 if app.adapter_list.is_empty() {

1230 - frame.render_widget(

1231 - Paragraph::new(" No artifacts found. Press f to fine-tune.")

1232 - .style(Style::new().fg(C_MUTED)),

1233 - area,

1234 - );

1230 + let msg = if app.model_detail_can_finetune {

1231 + " No artifacts found. Press f to fine-tune."

1232 + } else {

1233 + " No artifacts found. (GGUF models cannot be fine-tuned.)"

1234 + };

1235 + frame.render_widget(Paragraph::new(msg).style(Style::new().fg(C_MUTED)), area);

1236 return;

1237 }

1238

@@ -2576,8 +2577,10 @@ fn render_footer(frame: &mut Frame, app: &App, area: Rect) {

2577 Style::new().fg(C_MUTED),

2578 ));

2579 }

2579 - keys.push(Span::styled("f", Style::new().fg(C_NEON)));

2580 - keys.push(Span::styled(" · fine-tune ", Style::new().fg(C_MUTED)));

2580 + if app.model_detail_can_finetune {

2581 + keys.push(Span::styled("f", Style::new().fg(C_NEON)));

2582 + keys.push(Span::styled(" · fine-tune ", Style::new().fg(C_MUTED)));

2583 + }

2584 keys.push(Span::styled("Esc", Style::new().fg(C_NEON)));

2585 keys.push(Span::styled(" · back ", Style::new().fg(C_MUTED)));

2586 keys.push(Span::styled("Ctrl+C", Style::new().fg(C_NEON)));