Add Qwen 3
keypair34 committed
Apr 14, 2026 at 19:03 UTC
765a9024cddbd19c1853c12783e9a292c2dfaeaa
2 files changed
+36
src/inference/engine.rs
+15
@@ -1161,6 +1161,21 @@ impl GgufModelConfig {
1161
}
1162
}
1163
1164
+ /// Qwen 3 4B Instruct (GGUF Q4_K_M) — ~2.7 GB.
1165
+ ///
1166
+ /// Full OpenAI-compatible tool calling with extended thinking mode.
1167
+ /// Always load with `max_tokens ≥ 4096`; the `<think>…</think>` block can
1168
+ /// consume 300–400 tokens before the real response begins.
1169
+ pub fn qwen3_4b() -> Self {
1170
+ Self {
1171
+ model_id: super::models::BARTOWSKI_QWEN3_4B_GGUF.into(),
1172
+ files: vec![super::models::QWEN3_4B_GGUF_FILE.into()],
1173
+ tok_model_id: None,
1174
+ display_name: "Qwen 3 4B (Q4_K_M)".into(),
1175
+ approx_memory: "~2.7 GB".into(),
1176
+ }
1177
+ }
1178
+
1179
/// Return the platform-appropriate default **coding** model config.
1180
///
1181
/// - iOS / Android → Qwen 2.5 Coder 1.5B (~941 MB, fits mobile memory budgets)
src/inference/models.rs
+21
@@ -63,12 +63,24 @@ pub const QWEN25_3B_GGUF_FILE: &str = "Qwen2.5-3B-Instruct-Q4_K_M.gguf";
63
/// Base model repo used for the HF tokenizer (tokenizer.json + tokenizer_config.json).
64
pub const QWEN25_3B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-3B-Instruct";
65
66
+/// Pre-quantized Qwen 3 4B Instruct (GGUF Q4_K_M) — full OpenAI-compatible tool calling (~2.7 GB).
67
+///
68
+/// Qwen 3 uses an extended thinking mode (`<think>…</think>`) that significantly improves
69
+/// reasoning and tool-use accuracy. Load with `max_tokens ≥ 4096` to avoid empty replies caused
70
+/// by the model exhausting its token budget on thinking before producing a response.
71
+///
72
+/// Recommended model for siGit Code (coding agent with tool calling on macOS/Linux/Windows).
73
+pub const BARTOWSKI_QWEN3_4B_GGUF: &str = "bartowski/Qwen_Qwen3-4B-GGUF";
74
+/// The specific GGUF filename to download from the bartowski Qwen 3 4B repo.
75
+pub const QWEN3_4B_GGUF_FILE: &str = "Qwen_Qwen3-4B-Q4_K_M.gguf";
76
+
77
/// All model IDs that the Onde inference engine supports.
78
/// Used by `list_local_hf_models` to filter the HuggingFace cache
79
/// to only show models that can actually be used for generation.
80
pub const SUPPORTED_MODELS: &[&str] = &[
81
BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,
82
BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,
83
+ BARTOWSKI_QWEN3_4B_GGUF,
84
];
85
86
/// Rich metadata for a supported model, used by the frontend to display
@@ -117,4 +129,13 @@ pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[
129
// Exact file size from HuggingFace API siblings[].size.
130
expected_size_bytes: 1_929_903_264,
131
},
132
+ SupportedModelInfo {
133
+ id: BARTOWSKI_QWEN3_4B_GGUF,
134
+ name: "Qwen 3 4B (GGUF)",
135
+ org: "Qwen / Alibaba",
136
+ description: "Full tool-calling support with extended reasoning mode (~2.7 GB). \
137
+ Recommended for siGit Code on macOS, Linux, and Windows.",
138
+ // Qwen_Qwen3-4B-Q4_K_M.gguf from bartowski repo.
139
+ expected_size_bytes: 2_596_306_912,
140
+ },
141
];