Add Qwen 3 8B (GGUF Q4_K_M) model support
- Add constants for Qwen 3 8B repo and filename - Register in SUPPORTED_MODELS and SUPPORTED_MODEL_INFO - Add GgufModelConfig::qwen3_8b() constructor
keypair34 committed
Apr 25, 2026 at 09:36 UTC
1f0f3f1c4c6bba1b7c7b471a3ff666e42d7172b1
2 files changed
+38
-9
src/inference/engine.rs
+22
-9
@@ -1547,19 +1547,18 @@ fn attach_tools(mut req: RequestBuilder, tools: &[ToolDefinition]) -> RequestBui
1547
let mistral_tools: Vec<MistralTool> = tools
1548
.iter()
1549
.map(|td| {
1550
- let params: HashMap<String, serde_json::Value> = match serde_json::from_str(
1551
- &td.parameters_schema,
1552
- ) {
1553
- Ok(p) => p,
1554
- Err(err) => {
1555
- log::warn!(
1550
+ let params: HashMap<String, serde_json::Value> =
1551
+ match serde_json::from_str(&td.parameters_schema) {
1552
+ Ok(p) => p,
1553
+ Err(err) => {
1554
+ log::warn!(
1555
"tool '{}': malformed parameters_schema JSON ({}), using empty params",
1556
td.name,
1557
err
1558
);
1560
- HashMap::new()
1561
- }
1562
- };
1559
+ HashMap::new()
1560
+ }
1561
+ };
1562
MistralTool {
1563
tp: ToolType::Function,
1564
function: Function {
@@ -1941,6 +1940,20 @@ impl GgufModelConfig {
1940
}
1941
}
1942
1943
+ /// Qwen 3 8B Instruct (GGUF Q4_K_M) — ~5 GB.
1944
+ ///
1945
+ /// Strong tool-calling model with extended thinking. Best balance of
1946
+ /// quality and memory for macOS with 24+ GB RAM.
1947
+ pub fn qwen3_8b() -> Self {
1948
+ Self {
1949
+ model_id: super::models::BARTOWSKI_QWEN3_8B_GGUF.into(),
1950
+ files: vec![super::models::QWEN3_8B_GGUF_FILE.into()],
1951
+ tok_model_id: None,
1952
+ display_name: "Qwen 3 8B (Q4_K_M)".into(),
1953
+ approx_memory: "~5 GB".into(),
1954
+ }
1955
+ }
1956
+
1957
/// Return the platform-appropriate default **coding** model config.
1958
///
1959
/// - iOS / Android → Qwen 2.5 Coder 1.5B (~941 MB, fits mobile memory budgets)
src/inference/models.rs
+16
@@ -74,6 +74,13 @@ pub const BARTOWSKI_QWEN3_4B_GGUF: &str = "bartowski/Qwen_Qwen3-4B-GGUF";
74
/// The specific GGUF filename to download from the bartowski Qwen 3 4B repo.
75
pub const QWEN3_4B_GGUF_FILE: &str = "Qwen_Qwen3-4B-Q4_K_M.gguf";
76
77
+/// Pre-quantized Qwen 3 8B (GGUF Q4_K_M) — strong tool-calling model (~5 GB).
78
+///
79
+/// Best balance of quality and memory for macOS with 24+ GB RAM.
80
+/// Full tool calling and extended thinking mode support.
81
+pub const BARTOWSKI_QWEN3_8B_GGUF: &str = "bartowski/Qwen_Qwen3-8B-GGUF";
82
+pub const QWEN3_8B_GGUF_FILE: &str = "Qwen_Qwen3-8B-Q4_K_M.gguf";
83
+
84
/// Pre-quantized Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling model (~1.3 GB).
85
///
86
/// Smallest Qwen 3 variant with tool calling support. Suitable for mobile devices
@@ -89,6 +96,7 @@ pub const SUPPORTED_MODELS: &[&str] = &[
96
BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,
97
BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,
98
BARTOWSKI_QWEN3_4B_GGUF,
99
+ BARTOWSKI_QWEN3_8B_GGUF,
100
BARTOWSKI_QWEN3_1_7B_GGUF,
101
];
102
@@ -147,6 +155,14 @@ pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[
155
// Qwen_Qwen3-4B-Q4_K_M.gguf from bartowski repo.
156
expected_size_bytes: 2_596_306_912,
157
},
158
+ SupportedModelInfo {
159
+ id: BARTOWSKI_QWEN3_8B_GGUF,
160
+ name: "Qwen 3 8B (GGUF)",
161
+ org: "Qwen / Alibaba",
162
+ description: "Strong tool-calling model with extended thinking (~5 GB). \
163
+ Best balance of quality and memory for macOS with 24+ GB RAM.",
164
+ expected_size_bytes: 5_131_567_104,
165
+ },
166
SupportedModelInfo {
167
id: BARTOWSKI_QWEN3_1_7B_GGUF,
168
name: "Qwen 3 1.7B (GGUF)",