@ondeinference / onde / commits / dfa9c0f

Add full Qwen 3 family GGUF support across core and SDKs

Extends Qwen 3 coverage from the four partially-wired sizes (1.7B/4B/8B/14B) to the complete on-device, open-weight Qwen 3 GGUF lineup, and wires every variant through all binding layers. Models added: - Dense: Qwen 3 0.6B and 32B (rounding out 0.6B/1.7B/4B/8B/14B/32B) - Latest "2507" updates: Qwen 3 4B Instruct 2507 (non-thinking), 4B Thinking 2507, and the 30B-A3B Instruct 2507 mixture-of-experts (loads via the qwen3moe GGUF architecture) Wiring completed for all nine Qwen 3 variants: - models.rs: repo/file/tokenizer constants, SUPPORTED_MODELS, SUPPORTED_MODEL_INFO, and tok_model_id_for_repo entries - engine.rs: GgufModelConfig constructors, now Android-tokenizer-aware (the previous 1.7B/4B/8B/14B constructors passed tok_model_id: None even on Android, which breaks the candle GGUF backend) - ffi.rs: #[uniffi::export] config functions (reaches Swift/Kotlin/Python) - Kotlin OndeModels convenience wrappers, Dart FRB bridge, and the React Native C-ABI exports Notes: - The 3.5/3.6/3.7 point releases are excluded: they are either huge MoE (235B-A22B / 397B-A17B) or closed-weights, API-only checkpoints, none of which are loadable on-device through this GGUF SDK. - expected_size_bytes for the new variants are approximate; the HuggingFace API was unreachable from the build environment, so exact siblings[].size values should be backfilled when network access is available. - Added consistency tests: every supported model has display metadata, and every Qwen 3 repo resolves to an Android tokenizer. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DkBxc4UETJyXpFUDnBSn3E

Claude committed Jun 22, 2026 at 22:30 UTC dfa9c0f37506913fc1c41607399a08def3868153

7 files changed +562 -41

.agents/AGENTS.md

+18 -4

@@ -147,10 +147,15 @@ All model constants live in `src/inference/models.rs`. When adding a new model:

161

@@ -374,6 +379,15 @@ await engine.unloadModel()

379 | `defaultModelConfig()` | `GgufModelConfig` | Platform-aware Coder default (1.5B on iOS/tvOS/Android, 3B on desktop) |

380 | `qwen251_5bConfig()` | `GgufModelConfig` | Forces Qwen 2.5 1.5B regardless of platform |

381 | `qwen253bConfig()` | `GgufModelConfig` | Forces Qwen 2.5 3B regardless of platform |

382 +| `qwen306bConfig()` | `GgufModelConfig` | Qwen 3 0.6B (~0.5 GB) |

383 +| `qwen317bConfig()` | `GgufModelConfig` | Qwen 3 1.7B (~1.3 GB) |

384 +| `qwen34bConfig()` | `GgufModelConfig` | Qwen 3 4B (~2.7 GB) |

385 +| `qwen38bConfig()` | `GgufModelConfig` | Qwen 3 8B (~5 GB) |

386 +| `qwen314bConfig()` | `GgufModelConfig` | Qwen 3 14B (~8.4 GB) |

387 +| `qwen332bConfig()` | `GgufModelConfig` | Qwen 3 32B (~19.8 GB) |

388 +| `qwen34bInstruct2507Config()` | `GgufModelConfig` | Qwen 3 4B Instruct 2507 — latest non-thinking 4B (~2.5 GB) |

389 +| `qwen34bThinking2507Config()` | `GgufModelConfig` | Qwen 3 4B Thinking 2507 — latest reasoning 4B (~2.5 GB) |

390 +| `qwen330bA3bInstruct2507Config()` | `GgufModelConfig` | Qwen 3 30B-A3B Instruct 2507 MoE (~18.6 GB) |

391 | `defaultSamplingConfig()` | `SamplingConfig` | temp=0.7, top_p=0.95, max_tokens=512 |

392 | `deterministicSamplingConfig()` | `SamplingConfig` | temp=0.0, greedy |

393 | `mobileSamplingConfig()` | `SamplingConfig` | temp=0.7, max_tokens=128 |

sdk/dart/rust/src/api.rs

+56

@@ -542,6 +542,62 @@ pub fn qwen25_coder_3b_config() -> GgufModelConfig {

542 OndeGgufModelConfig::qwen25_coder_3b().into()

543 }

544

545 +// ── Qwen 3 family ────────────────────────────────────────────────────────────

546 +

547 +/// `GgufModelConfig` for Qwen 3 0.6B Q4_K_M (~0.5 GB).

548 +#[frb(sync)]

549 +pub fn qwen3_0_6b_config() -> GgufModelConfig {

550 + OndeGgufModelConfig::qwen3_0_6b().into()

551 +}

552 +

553 +/// `GgufModelConfig` for Qwen 3 1.7B Q4_K_M (~1.3 GB).

554 +#[frb(sync)]

555 +pub fn qwen3_1_7b_config() -> GgufModelConfig {

556 + OndeGgufModelConfig::qwen3_1_7b().into()

557 +}

558 +

559 +/// `GgufModelConfig` for Qwen 3 4B Q4_K_M (~2.7 GB).

560 +#[frb(sync)]

561 +pub fn qwen3_4b_config() -> GgufModelConfig {

562 + OndeGgufModelConfig::qwen3_4b().into()

563 +}

564 +

565 +/// `GgufModelConfig` for Qwen 3 8B Q4_K_M (~5 GB).

566 +#[frb(sync)]

567 +pub fn qwen3_8b_config() -> GgufModelConfig {

568 + OndeGgufModelConfig::qwen3_8b().into()

569 +}

570 +

571 +/// `GgufModelConfig` for Qwen 3 14B Q4_K_M (~8.4 GB).

572 +#[frb(sync)]

573 +pub fn qwen3_14b_config() -> GgufModelConfig {

574 + OndeGgufModelConfig::qwen3_14b().into()

575 +}

576 +

577 +/// `GgufModelConfig` for Qwen 3 32B Q4_K_M (~19.8 GB).

578 +#[frb(sync)]

579 +pub fn qwen3_32b_config() -> GgufModelConfig {

580 + OndeGgufModelConfig::qwen3_32b().into()

581 +}

582 +

583 +/// `GgufModelConfig` for Qwen 3 4B Instruct 2507 Q4_K_M (~2.5 GB).

584 +#[frb(sync)]

585 +pub fn qwen3_4b_instruct_2507_config() -> GgufModelConfig {

586 + OndeGgufModelConfig::qwen3_4b_instruct_2507().into()

587 +}

588 +

589 +/// `GgufModelConfig` for Qwen 3 4B Thinking 2507 Q4_K_M (~2.5 GB).

590 +#[frb(sync)]

591 +pub fn qwen3_4b_thinking_2507_config() -> GgufModelConfig {

592 + OndeGgufModelConfig::qwen3_4b_thinking_2507().into()

593 +}

594 +

595 +/// `GgufModelConfig` for Qwen 3 30B-A3B Instruct 2507 (MoE) Q4_K_M (~18.6 GB).

596 +#[frb(sync)]

597 +pub fn qwen3_30b_a3b_instruct_2507_config() -> GgufModelConfig {

598 + OndeGgufModelConfig::qwen3_30b_a3b_instruct_2507().into()

599 +}

600 +

601 /// Default sampling config: `temperature=0.7`, `top_p=0.95`, `max_tokens=512`.

602 #[frb(sync)]

603 pub fn default_sampling_config() -> SamplingConfig {

sdk/kotlin/lib/src/shared/kotlin/com/ondeinference/onde/Convenience.kt

+20

@@ -25,6 +25,26 @@ object OndeModels {

25 fun qwen25_1_5b(): GgufModelConfig = uniffi.onde.qwen2515bConfig()

26 /** Qwen 2.5 3B Instruct GGUF Q4_K_M (~1.93 GB). */

27 fun qwen25_3b(): GgufModelConfig = uniffi.onde.qwen253bConfig()

28 +

29 + // ── Qwen 3 family ────────────────────────────────────────────────────────

30 + /** Qwen 3 0.6B GGUF Q4_K_M (~0.5 GB) — smallest Qwen 3 variant. */

31 + fun qwen3_0_6b(): GgufModelConfig = uniffi.onde.qwen306bConfig()

32 + /** Qwen 3 1.7B GGUF Q4_K_M (~1.3 GB). */

33 + fun qwen3_1_7b(): GgufModelConfig = uniffi.onde.qwen317bConfig()

34 + /** Qwen 3 4B GGUF Q4_K_M (~2.7 GB). */

35 + fun qwen3_4b(): GgufModelConfig = uniffi.onde.qwen34bConfig()

36 + /** Qwen 3 8B GGUF Q4_K_M (~5 GB). */

37 + fun qwen3_8b(): GgufModelConfig = uniffi.onde.qwen38bConfig()

38 + /** Qwen 3 14B GGUF Q4_K_M (~8.4 GB). */

39 + fun qwen3_14b(): GgufModelConfig = uniffi.onde.qwen314bConfig()

40 + /** Qwen 3 32B GGUF Q4_K_M (~19.8 GB) — largest dense Qwen 3. */

41 + fun qwen3_32b(): GgufModelConfig = uniffi.onde.qwen332bConfig()

42 + /** Qwen 3 4B Instruct 2507 GGUF Q4_K_M (~2.5 GB) — latest non-thinking 4B. */

43 + fun qwen3_4b_instruct_2507(): GgufModelConfig = uniffi.onde.qwen34bInstruct2507Config()

44 + /** Qwen 3 4B Thinking 2507 GGUF Q4_K_M (~2.5 GB) — latest reasoning 4B. */

45 + fun qwen3_4b_thinking_2507(): GgufModelConfig = uniffi.onde.qwen34bThinking2507Config()

46 + /** Qwen 3 30B-A3B Instruct 2507 GGUF Q4_K_M (~18.6 GB) — flagship MoE. */

47 + fun qwen3_30b_a3b_instruct_2507(): GgufModelConfig = uniffi.onde.qwen330bA3bInstruct2507Config()

48 }

50 /**

sdk/react-native/rust/src/lib.rs

+56

@@ -470,6 +470,62 @@ pub extern "C" fn onde_qwen25_3b_config() -> *mut c_char {

470 to_json_cstring(&GgufModelConfig::qwen25_3b())

471 }

472

473 +// ── Qwen 3 family ────────────────────────────────────────────────────────────

474 +

475 +/// Return the Qwen 3 0.6B GGUF model config as JSON.

476 +#[no_mangle]

477 +pub extern "C" fn onde_qwen3_0_6b_config() -> *mut c_char {

478 + to_json_cstring(&GgufModelConfig::qwen3_0_6b())

479 +}

480 +

481 +/// Return the Qwen 3 1.7B GGUF model config as JSON.

482 +#[no_mangle]

483 +pub extern "C" fn onde_qwen3_1_7b_config() -> *mut c_char {

484 + to_json_cstring(&GgufModelConfig::qwen3_1_7b())

485 +}

486 +

487 +/// Return the Qwen 3 4B GGUF model config as JSON.

488 +#[no_mangle]

489 +pub extern "C" fn onde_qwen3_4b_config() -> *mut c_char {

490 + to_json_cstring(&GgufModelConfig::qwen3_4b())

491 +}

492 +

493 +/// Return the Qwen 3 8B GGUF model config as JSON.

494 +#[no_mangle]

495 +pub extern "C" fn onde_qwen3_8b_config() -> *mut c_char {

496 + to_json_cstring(&GgufModelConfig::qwen3_8b())

497 +}

498 +

499 +/// Return the Qwen 3 14B GGUF model config as JSON.

500 +#[no_mangle]

501 +pub extern "C" fn onde_qwen3_14b_config() -> *mut c_char {

502 + to_json_cstring(&GgufModelConfig::qwen3_14b())

503 +}

504 +

505 +/// Return the Qwen 3 32B GGUF model config as JSON.

506 +#[no_mangle]

507 +pub extern "C" fn onde_qwen3_32b_config() -> *mut c_char {

508 + to_json_cstring(&GgufModelConfig::qwen3_32b())

509 +}

510 +

511 +/// Return the Qwen 3 4B Instruct 2507 GGUF model config as JSON.

512 +#[no_mangle]

513 +pub extern "C" fn onde_qwen3_4b_instruct_2507_config() -> *mut c_char {

514 + to_json_cstring(&GgufModelConfig::qwen3_4b_instruct_2507())

515 +}

516 +

517 +/// Return the Qwen 3 4B Thinking 2507 GGUF model config as JSON.

518 +#[no_mangle]

519 +pub extern "C" fn onde_qwen3_4b_thinking_2507_config() -> *mut c_char {

520 + to_json_cstring(&GgufModelConfig::qwen3_4b_thinking_2507())

521 +}

522 +

523 +/// Return the Qwen 3 30B-A3B Instruct 2507 (MoE) GGUF model config as JSON.

524 +#[no_mangle]

525 +pub extern "C" fn onde_qwen3_30b_a3b_instruct_2507_config() -> *mut c_char {

526 + to_json_cstring(&GgufModelConfig::qwen3_30b_a3b_instruct_2507())

527 +}

528 +

529 // ── Sampling presets ─────────────────────────────────────────────────────────

530

531 /// Return the default sampling config as JSON.

src/inference/engine.rs

+165 -20

@@ -2039,7 +2039,43 @@ impl GgufModelConfig {

2039 }

2040 }

2041

2042 - /// Qwen 3 4B Instruct (GGUF Q4_K_M) — ~2.7 GB.

2042 + /// Qwen 3 0.6B (GGUF Q4_K_M) — smallest Qwen 3 variant, ~0.5 GB.

2043 + ///

2044 + /// Lightest tool-capable Qwen 3 model; fits tvOS and the most

2045 + /// memory-constrained mobile devices. Extended thinking mode — load with

2046 + /// `max_tokens ≥ 4096`.

2047 + pub fn qwen3_0_6b() -> Self {

2048 + Self {

2049 + model_id: super::models::BARTOWSKI_QWEN3_0_6B_GGUF.into(),

2050 + files: vec![super::models::QWEN3_0_6B_GGUF_FILE.into()],

2051 + tok_model_id: if cfg!(target_os = "android") {

2052 + Some(super::models::QWEN3_0_6B_TOK_MODEL_ID.into())

2053 + } else {

2054 + None

2055 + },

2056 + display_name: "Qwen 3 0.6B (Q4_K_M)".into(),

2057 + approx_memory: "~0.5 GB".into(),

2058 + chat_template: None,

2059 + }

2060 + }

2061 +

2062 + /// Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling, ~1.3 GB.

2063 + pub fn qwen3_1_7b() -> Self {

2064 + Self {

2065 + model_id: super::models::BARTOWSKI_QWEN3_1_7B_GGUF.into(),

2066 + files: vec![super::models::QWEN3_1_7B_GGUF_FILE.into()],

2067 + tok_model_id: if cfg!(target_os = "android") {

2068 + Some(super::models::QWEN3_1_7B_TOK_MODEL_ID.into())

2069 + } else {

2070 + None

2071 + },

2072 + display_name: "Qwen 3 1.7B (Q4_K_M)".into(),

2073 + approx_memory: "~1.3 GB".into(),

2074 + chat_template: None,

2075 + }

2076 + }

2077 +

2078 + /// Qwen 3 4B (GGUF Q4_K_M) — ~2.7 GB.

2079 ///

2080 /// Full OpenAI-compatible tool calling with extended thinking mode.

2081 /// Always load with `max_tokens ≥ 4096`; the `<think>…</think>` block can

@@ -2048,49 +2084,127 @@ impl GgufModelConfig {

2084 Self {

2085 model_id: super::models::BARTOWSKI_QWEN3_4B_GGUF.into(),

2086 files: vec![super::models::QWEN3_4B_GGUF_FILE.into()],

2051 - tok_model_id: None,

2087 + tok_model_id: if cfg!(target_os = "android") {

2088 + Some(super::models::QWEN3_4B_TOK_MODEL_ID.into())

2089 + } else {

2090 + None

2091 + },

2092 display_name: "Qwen 3 4B (Q4_K_M)".into(),

2093 approx_memory: "~2.7 GB".into(),

2094 chat_template: None,

2095 }

2096 }

2097

2058 - /// Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling, ~1.3 GB.

2059 - pub fn qwen3_1_7b() -> Self {

2098 + /// Strong tool-calling model with extended thinking. Best balance of

2099 + /// quality and memory for macOS with 24+ GB RAM.

2100 + pub fn qwen3_8b() -> Self {

2101 Self {

2061 - model_id: super::models::BARTOWSKI_QWEN3_1_7B_GGUF.into(),

2062 - files: vec![super::models::QWEN3_1_7B_GGUF_FILE.into()],

2063 - tok_model_id: None,

2064 - display_name: "Qwen 3 1.7B (Q4_K_M)".into(),

2065 - approx_memory: "~1.3 GB".into(),

2102 + model_id: super::models::BARTOWSKI_QWEN3_8B_GGUF.into(),

2103 + files: vec![super::models::QWEN3_8B_GGUF_FILE.into()],

2104 + tok_model_id: if cfg!(target_os = "android") {

2105 + Some(super::models::QWEN3_8B_TOK_MODEL_ID.into())

2106 + } else {

2107 + None

2108 + },

2109 + display_name: "Qwen 3 8B (Q4_K_M)".into(),

2110 + approx_memory: "~5 GB".into(),

2111 chat_template: None,

2112 }

2113 }

2114

2070 - /// Qwen 3 14B Instruct (GGUF Q4_K_M) — ~8.4 GB.

2115 + /// Qwen 3 14B (GGUF Q4_K_M) — ~8.4 GB.

2116 ///

2072 - /// Strongest reasoning and tool-calling model with extended thinking.

2117 + /// Strong reasoning and tool-calling model with extended thinking.

2118 /// Best all-around model for macOS with 16+ GB RAM.

2119 pub fn qwen3_14b() -> Self {

2120 Self {

2121 model_id: super::models::BARTOWSKI_QWEN3_14B_GGUF.into(),

2122 files: vec![super::models::QWEN3_14B_GGUF_FILE.into()],

2078 - tok_model_id: None,

2123 + tok_model_id: if cfg!(target_os = "android") {

2124 + Some(super::models::QWEN3_14B_TOK_MODEL_ID.into())

2125 + } else {

2126 + None

2127 + },

2128 display_name: "Qwen 3 14B (Q4_K_M)".into(),

2129 approx_memory: "~8.4 GB".into(),

2130 chat_template: None,

2131 }

2132 }

2133

2085 - /// Strong tool-calling model with extended thinking. Best balance of

2086 - /// quality and memory for macOS with 24+ GB RAM.

2087 - pub fn qwen3_8b() -> Self {

2134 + /// Qwen 3 32B (GGUF Q4_K_M) — largest dense Qwen 3, ~19.8 GB.

2135 + ///

2136 + /// Highest-quality dense Qwen 3 variant. Requires a high-memory desktop

2137 + /// (32+ GB RAM / unified memory). Extended thinking and full tool calling.

2138 + pub fn qwen3_32b() -> Self {

2139 Self {

2089 - model_id: super::models::BARTOWSKI_QWEN3_8B_GGUF.into(),

2090 - files: vec![super::models::QWEN3_8B_GGUF_FILE.into()],

2091 - tok_model_id: None,

2092 - display_name: "Qwen 3 8B (Q4_K_M)".into(),

2093 - approx_memory: "~5 GB".into(),

2140 + model_id: super::models::BARTOWSKI_QWEN3_32B_GGUF.into(),

2141 + files: vec![super::models::QWEN3_32B_GGUF_FILE.into()],

2142 + tok_model_id: if cfg!(target_os = "android") {

2143 + Some(super::models::QWEN3_32B_TOK_MODEL_ID.into())

2144 + } else {

2145 + None

2146 + },

2147 + display_name: "Qwen 3 32B (Q4_K_M)".into(),

2148 + approx_memory: "~19.8 GB".into(),

2149 + chat_template: None,

2150 + }

2151 + }

2152 +

2153 + /// Qwen 3 4B Instruct 2507 (GGUF Q4_K_M) — latest non-thinking 4B, ~2.5 GB.

2154 + ///

2155 + /// Updated instruction-tuned checkpoint that does *not* emit a `<think>`

2156 + /// block, making it faster and more predictable for chat and tool use.

2157 + /// Recommended general-purpose Qwen 3 model.

2158 + pub fn qwen3_4b_instruct_2507() -> Self {

2159 + Self {

2160 + model_id: super::models::BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF.into(),

2161 + files: vec![super::models::QWEN3_4B_INSTRUCT_2507_GGUF_FILE.into()],

2162 + tok_model_id: if cfg!(target_os = "android") {

2163 + Some(super::models::QWEN3_4B_INSTRUCT_2507_TOK_MODEL_ID.into())

2164 + } else {

2165 + None

2166 + },

2167 + display_name: "Qwen 3 4B Instruct 2507 (Q4_K_M)".into(),

2168 + approx_memory: "~2.5 GB".into(),

2169 + chat_template: None,

2170 + }

2171 + }

2172 +

2173 + /// Qwen 3 4B Thinking 2507 (GGUF Q4_K_M) — latest reasoning 4B, ~2.5 GB.

2174 + ///

2175 + /// Updated thinking-only checkpoint with stronger reasoning and tool-use

2176 + /// accuracy. Always emits a `<think>` block — load with `max_tokens ≥ 4096`.

2177 + pub fn qwen3_4b_thinking_2507() -> Self {

2178 + Self {

2179 + model_id: super::models::BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF.into(),

2180 + files: vec![super::models::QWEN3_4B_THINKING_2507_GGUF_FILE.into()],

2181 + tok_model_id: if cfg!(target_os = "android") {

2182 + Some(super::models::QWEN3_4B_THINKING_2507_TOK_MODEL_ID.into())

2183 + } else {

2184 + None

2185 + },

2186 + display_name: "Qwen 3 4B Thinking 2507 (Q4_K_M)".into(),

2187 + approx_memory: "~2.5 GB".into(),

2188 + chat_template: None,

2189 + }

2190 + }

2191 +

2192 + /// Qwen 3 30B-A3B Instruct 2507 (GGUF Q4_K_M) — flagship MoE, ~18.6 GB.

2193 + ///

2194 + /// Mixture-of-experts model: 30B total parameters, ~3B active per token, so

2195 + /// inference is far cheaper than a 30B dense model while quality rivals it.

2196 + /// Loads via the `qwen3moe` GGUF architecture. Requires 32+ GB RAM.

2197 + pub fn qwen3_30b_a3b_instruct_2507() -> Self {

2198 + Self {

2199 + model_id: super::models::BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF.into(),

2200 + files: vec![super::models::QWEN3_30B_A3B_INSTRUCT_2507_GGUF_FILE.into()],

2201 + tok_model_id: if cfg!(target_os = "android") {

2202 + Some(super::models::QWEN3_30B_A3B_INSTRUCT_2507_TOK_MODEL_ID.into())

2203 + } else {

2204 + None

2205 + },

2206 + display_name: "Qwen 3 30B-A3B Instruct 2507 (Q4_K_M)".into(),

2207 + approx_memory: "~18.6 GB".into(),

2208 chat_template: None,

2209 }

2210 }

@@ -2172,6 +2286,37 @@ mod tests {

2286 assert_eq!(cfg.files.len(), 1);

2287 }

2288

2289 + #[test]

2290 + fn gguf_model_config_qwen3_variants() {

2291 + // Every Qwen 3 constructor must produce a valid, single-file config

2292 + // whose repo ID is registered in the supported-models list.

2293 + let configs = [

2294 + GgufModelConfig::qwen3_0_6b(),

2295 + GgufModelConfig::qwen3_1_7b(),

2296 + GgufModelConfig::qwen3_4b(),

2297 + GgufModelConfig::qwen3_8b(),

2298 + GgufModelConfig::qwen3_14b(),

2299 + GgufModelConfig::qwen3_32b(),

2300 + GgufModelConfig::qwen3_4b_instruct_2507(),

2301 + GgufModelConfig::qwen3_4b_thinking_2507(),

2302 + GgufModelConfig::qwen3_30b_a3b_instruct_2507(),

2303 + ];

2304 + for cfg in &configs {

2305 + assert!(

2306 + cfg.model_id.contains("Qwen3"),

2307 + "unexpected id: {}",

2308 + cfg.model_id

2309 + );

2310 + assert_eq!(cfg.files.len(), 1);

2311 + assert!(cfg.files[0].ends_with("-Q4_K_M.gguf"));

2312 + assert!(

2313 + super::super::models::SUPPORTED_MODELS.contains(&cfg.model_id.as_str()),

2314 + "{} missing from SUPPORTED_MODELS",

2315 + cfg.model_id

2316 + );

2317 + }

2318 + }

2319 +

2320 #[test]

2321 fn gguf_model_config_platform_default() {

2322 let cfg = GgufModelConfig::platform_default();

src/inference/ffi.rs

+60

@@ -326,6 +326,66 @@ pub fn qwen25_3b_config() -> GgufModelConfig {

326 GgufModelConfig::qwen25_3b()

327 }

328

329 +// ── Qwen 3 family ────────────────────────────────────────────────────────────

330 +

331 +/// Return the Qwen 3 0.6B GGUF model configuration (~0.5 GB).

332 +#[uniffi::export]

333 +pub fn qwen3_0_6b_config() -> GgufModelConfig {

334 + GgufModelConfig::qwen3_0_6b()

335 +}

336 +

337 +/// Return the Qwen 3 1.7B GGUF model configuration (~1.3 GB).

338 +#[uniffi::export]

339 +pub fn qwen3_1_7b_config() -> GgufModelConfig {

340 + GgufModelConfig::qwen3_1_7b()

341 +}

342 +

343 +/// Return the Qwen 3 4B GGUF model configuration (~2.7 GB).

344 +#[uniffi::export]

345 +pub fn qwen3_4b_config() -> GgufModelConfig {

346 + GgufModelConfig::qwen3_4b()

347 +}

348 +

349 +/// Return the Qwen 3 8B GGUF model configuration (~5 GB).

350 +#[uniffi::export]

351 +pub fn qwen3_8b_config() -> GgufModelConfig {

352 + GgufModelConfig::qwen3_8b()

353 +}

354 +

355 +/// Return the Qwen 3 14B GGUF model configuration (~8.4 GB).

356 +#[uniffi::export]

357 +pub fn qwen3_14b_config() -> GgufModelConfig {

358 + GgufModelConfig::qwen3_14b()

359 +}

360 +

361 +/// Return the Qwen 3 32B GGUF model configuration (~19.8 GB).

362 +#[uniffi::export]

363 +pub fn qwen3_32b_config() -> GgufModelConfig {

364 + GgufModelConfig::qwen3_32b()

365 +}

366 +

367 +/// Return the Qwen 3 4B Instruct 2507 GGUF model configuration (~2.5 GB).

368 +///

369 +/// Latest non-thinking 4B checkpoint — recommended general-purpose Qwen 3 model.

370 +#[uniffi::export]

371 +pub fn qwen3_4b_instruct_2507_config() -> GgufModelConfig {

372 + GgufModelConfig::qwen3_4b_instruct_2507()

373 +}

374 +

375 +/// Return the Qwen 3 4B Thinking 2507 GGUF model configuration (~2.5 GB).

376 +///

377 +/// Latest reasoning-focused 4B checkpoint; load with `max_tokens ≥ 4096`.

378 +#[uniffi::export]

379 +pub fn qwen3_4b_thinking_2507_config() -> GgufModelConfig {

380 + GgufModelConfig::qwen3_4b_thinking_2507()

381 +}

382 +

383 +/// Return the Qwen 3 30B-A3B Instruct 2507 (MoE) GGUF model configuration (~18.6 GB).

384 +#[uniffi::export]

385 +pub fn qwen3_30b_a3b_instruct_2507_config() -> GgufModelConfig {

386 + GgufModelConfig::qwen3_30b_a3b_instruct_2507()

387 +}

388 +

389 /// Return default sampling parameters for creative chat.

390 #[uniffi::export]

391 pub fn default_sampling_config() -> SamplingConfig {

src/inference/models.rs

+187 -17

@@ -83,16 +83,47 @@ pub const QWEN25_3B_GGUF_FILE: &str = "Qwen2.5-3B-Instruct-Q4_K_M.gguf";

83 /// Base model repo used for the HF tokenizer (tokenizer.json + tokenizer_config.json).

84 pub const QWEN25_3B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-3B-Instruct";

86 -/// Pre-quantized Qwen 3 4B Instruct (GGUF Q4_K_M) — full OpenAI-compatible tool calling (~2.7 GB).

86 +// ── Qwen 3 family (GGUF Q4_K_M) ──────────────────────────────────────────────

87 +//

88 +// The Qwen 3 line uses the `qwen3` GGUF architecture (and `qwen3moe` for the

89 +// 30B-A3B mixture-of-experts variant), both supported by mistral.rs's quantized

90 +// loader. Every Qwen 3 model is a hybrid reasoner with an extended thinking mode

91 +// (`<think>…</think>`); always load with `max_tokens ≥ 4096` so the thinking

92 +// block does not exhaust the token budget before the real reply.

93 +//

94 +// bartowski's repos embed the tokenizer and chat template inside the GGUF, so on

95 +// iOS/macOS no separate tokenizer download is needed. On Android the candle GGUF

96 +// backend cannot parse the embedded tokenizer, so each model also declares a

97 +// `TOK_MODEL_ID` pointing at the official Qwen repo for a standalone tokenizer.

98 +

99 +/// Pre-quantized Qwen 3 0.6B (GGUF Q4_K_M) — smallest Qwen 3 variant (~0.5 GB).

100 ///

88 -/// Qwen 3 uses an extended thinking mode (`<think>…</think>`) that significantly improves

89 -/// reasoning and tool-use accuracy. Load with `max_tokens ≥ 4096` to avoid empty replies caused

90 -/// by the model exhausting its token budget on thinking before producing a response.

101 +/// Lightest tool-capable Qwen 3 model. Suitable for tvOS and the most

102 +/// memory-constrained mobile devices where even the 1.7B is too large.

103 +pub const BARTOWSKI_QWEN3_0_6B_GGUF: &str = "bartowski/Qwen_Qwen3-0.6B-GGUF";

104 +/// The specific GGUF filename for the bartowski Qwen 3 0.6B repo.

105 +pub const QWEN3_0_6B_GGUF_FILE: &str = "Qwen_Qwen3-0.6B-Q4_K_M.gguf";

106 +/// Base model repo used for the HF tokenizer on Android.

107 +pub const QWEN3_0_6B_TOK_MODEL_ID: &str = "Qwen/Qwen3-0.6B";

108 +

109 +/// Pre-quantized Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling model (~1.3 GB).

110 +///

111 +/// Smallest Qwen 3 variant with comfortable tool calling. Suitable for mobile

112 +/// devices where the 4B model would be too large.

113 +pub const BARTOWSKI_QWEN3_1_7B_GGUF: &str = "bartowski/Qwen_Qwen3-1.7B-GGUF";

114 +/// The specific GGUF filename for the Qwen3 1.7B repo.

115 +pub const QWEN3_1_7B_GGUF_FILE: &str = "Qwen_Qwen3-1.7B-Q4_K_M.gguf";

116 +/// Base model repo used for the HF tokenizer on Android.

117 +pub const QWEN3_1_7B_TOK_MODEL_ID: &str = "Qwen/Qwen3-1.7B";

118 +

119 +/// Pre-quantized Qwen 3 4B (GGUF Q4_K_M) — full OpenAI-compatible tool calling (~2.7 GB).

120 ///

121 /// Recommended model for siGit Code (coding agent with tool calling on macOS/Linux/Windows).

122 pub const BARTOWSKI_QWEN3_4B_GGUF: &str = "bartowski/Qwen_Qwen3-4B-GGUF";

123 /// The specific GGUF filename to download from the bartowski Qwen 3 4B repo.

124 pub const QWEN3_4B_GGUF_FILE: &str = "Qwen_Qwen3-4B-Q4_K_M.gguf";

125 +/// Base model repo used for the HF tokenizer on Android.

126 +pub const QWEN3_4B_TOK_MODEL_ID: &str = "Qwen/Qwen3-4B";

127

128 /// Pre-quantized Qwen 3 8B (GGUF Q4_K_M) — strong tool-calling model (~5 GB).

129 ///

@@ -100,22 +131,70 @@ pub const QWEN3_4B_GGUF_FILE: &str = "Qwen_Qwen3-4B-Q4_K_M.gguf";

131 /// Full tool calling and extended thinking mode support.

132 pub const BARTOWSKI_QWEN3_8B_GGUF: &str = "bartowski/Qwen_Qwen3-8B-GGUF";

133 pub const QWEN3_8B_GGUF_FILE: &str = "Qwen_Qwen3-8B-Q4_K_M.gguf";

134 +/// Base model repo used for the HF tokenizer on Android.

135 +pub const QWEN3_8B_TOK_MODEL_ID: &str = "Qwen/Qwen3-8B";

136

137 /// Pre-quantized Qwen 3 14B (GGUF Q4_K_M) — strong reasoning and tool-calling model (~8.4 GB).

138 ///

106 -/// Qwen 3 uses extended thinking mode (`<think>…</think>`) for improved reasoning.

139 /// Best all-around model for macOS with 16+ GB RAM. Full tool calling support.

140 pub const BARTOWSKI_QWEN3_14B_GGUF: &str = "bartowski/Qwen_Qwen3-14B-GGUF";

141 /// The specific GGUF filename.

142 pub const QWEN3_14B_GGUF_FILE: &str = "Qwen_Qwen3-14B-Q4_K_M.gguf";

143 +/// Base model repo used for the HF tokenizer on Android.

144 +pub const QWEN3_14B_TOK_MODEL_ID: &str = "Qwen/Qwen3-14B";

145

112 -/// Pre-quantized Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling model (~1.3 GB).

146 +/// Pre-quantized Qwen 3 32B (GGUF Q4_K_M) — largest dense Qwen 3 model (~19.8 GB).

147 ///

114 -/// Smallest Qwen 3 variant with tool calling support. Suitable for mobile devices

115 -/// where the 4B model would be too large.

116 -pub const BARTOWSKI_QWEN3_1_7B_GGUF: &str = "bartowski/Qwen_Qwen3-1.7B-GGUF";

117 -/// The specific GGUF filename for the Qwen3 1.7B repo.

118 -pub const QWEN3_1_7B_GGUF_FILE: &str = "Qwen_Qwen3-1.7B-Q4_K_M.gguf";

148 +/// Highest-quality dense Qwen 3 variant. Requires a high-memory desktop

149 +/// (32+ GB RAM / unified memory). Full tool calling and extended thinking.

150 +pub const BARTOWSKI_QWEN3_32B_GGUF: &str = "bartowski/Qwen_Qwen3-32B-GGUF";

151 +/// The specific GGUF filename for the bartowski Qwen 3 32B repo.

152 +pub const QWEN3_32B_GGUF_FILE: &str = "Qwen_Qwen3-32B-Q4_K_M.gguf";

153 +/// Base model repo used for the HF tokenizer on Android.

154 +pub const QWEN3_32B_TOK_MODEL_ID: &str = "Qwen/Qwen3-32B";

155 +

156 +// ── Qwen 3 "2507" updated releases ───────────────────────────────────────────

157 +//

158 +// The July/August 2025 refresh split Qwen 3 into dedicated non-thinking

159 +// (`-Instruct-2507`) and thinking-only (`-Thinking-2507`) checkpoints with

160 +// markedly improved instruction following, tool use, and long-context quality.

161 +// These are the latest open Qwen 3 weights available as on-device GGUF.

162 +

163 +/// Pre-quantized Qwen 3 4B Instruct 2507 (GGUF Q4_K_M) — latest non-thinking 4B (~2.5 GB).

164 +///

165 +/// Updated instruction-tuned checkpoint. Unlike the base 4B it does *not* emit a

166 +/// `<think>` block, so it is faster and more predictable for chat and tool use.

167 +pub const BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF: &str =

168 + "bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF";

169 +/// The specific GGUF filename for the bartowski Qwen 3 4B Instruct 2507 repo.

170 +pub const QWEN3_4B_INSTRUCT_2507_GGUF_FILE: &str = "Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf";

171 +/// Base model repo used for the HF tokenizer on Android.

172 +pub const QWEN3_4B_INSTRUCT_2507_TOK_MODEL_ID: &str = "Qwen/Qwen3-4B-Instruct-2507";

173 +

174 +/// Pre-quantized Qwen 3 4B Thinking 2507 (GGUF Q4_K_M) — latest reasoning 4B (~2.5 GB).

175 +///

176 +/// Updated thinking-only checkpoint with stronger reasoning and tool-use accuracy.

177 +/// Always emits a `<think>` block — load with `max_tokens ≥ 4096`.

178 +pub const BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF: &str =

179 + "bartowski/Qwen_Qwen3-4B-Thinking-2507-GGUF";

180 +/// The specific GGUF filename for the bartowski Qwen 3 4B Thinking 2507 repo.

181 +pub const QWEN3_4B_THINKING_2507_GGUF_FILE: &str = "Qwen_Qwen3-4B-Thinking-2507-Q4_K_M.gguf";

182 +/// Base model repo used for the HF tokenizer on Android.

183 +pub const QWEN3_4B_THINKING_2507_TOK_MODEL_ID: &str = "Qwen/Qwen3-4B-Thinking-2507";

184 +

185 +/// Pre-quantized Qwen 3 30B-A3B Instruct 2507 (GGUF Q4_K_M) — flagship MoE (~18.6 GB).

186 +///

187 +/// Mixture-of-experts model: 30B total parameters but only ~3B active per token,

188 +/// so inference is far cheaper than a 30B dense model while quality rivals it.

189 +/// Loads via the `qwen3moe` GGUF architecture in mistral.rs. Requires a

190 +/// high-memory desktop (32+ GB RAM / unified memory).

191 +pub const BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF: &str =

192 + "bartowski/Qwen_Qwen3-30B-A3B-Instruct-2507-GGUF";

193 +/// The specific GGUF filename for the bartowski Qwen 3 30B-A3B Instruct 2507 repo.

194 +pub const QWEN3_30B_A3B_INSTRUCT_2507_GGUF_FILE: &str =

195 + "Qwen_Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf";

196 +/// Base model repo used for the HF tokenizer on Android.

197 +pub const QWEN3_30B_A3B_INSTRUCT_2507_TOK_MODEL_ID: &str = "Qwen/Qwen3-30B-A3B-Instruct-2507";

198

199 /// DeepSeek Coder v1 6.7B Instruct (GGUF Q4_K_M) — dedicated code generation model (~3.8 GB).

200 ///

@@ -136,10 +215,15 @@ pub const SUPPORTED_MODELS: &[&str] = &[

215 BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF,

216 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,

217 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,

218 + BARTOWSKI_QWEN3_0_6B_GGUF,

219 + BARTOWSKI_QWEN3_1_7B_GGUF,

220 BARTOWSKI_QWEN3_4B_GGUF,

221 BARTOWSKI_QWEN3_8B_GGUF,

222 BARTOWSKI_QWEN3_14B_GGUF,

142 - BARTOWSKI_QWEN3_1_7B_GGUF,

223 + BARTOWSKI_QWEN3_32B_GGUF,

224 + BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF,

225 + BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF,

226 + BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF,

227 BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF,

228 THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF,

229 ];

@@ -197,6 +281,23 @@ pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[

281 // Exact file size from HuggingFace API siblings[].size.

282 expected_size_bytes: 1_929_903_264,

283 },

284 + SupportedModelInfo {

285 + id: BARTOWSKI_QWEN3_0_6B_GGUF,

286 + name: "Qwen 3 0.6B (GGUF)",

287 + org: "Qwen / Alibaba",

288 + description: "Smallest Qwen 3 variant with tool calling (~0.5 GB). \

289 + Suitable for tvOS and the most memory-constrained mobile devices.",

290 + // Approximate Q4_K_M size; HF API unreachable at authoring time.

291 + expected_size_bytes: 483_000_000,

292 + },

293 + SupportedModelInfo {

294 + id: BARTOWSKI_QWEN3_1_7B_GGUF,

295 + name: "Qwen 3 1.7B (GGUF)",

296 + org: "Qwen / Alibaba",

297 + description: "Lightweight tool-calling model for mobile (~1.3 GB). \

298 + Smallest Qwen 3 variant with comfortable tool calling support.",

299 + expected_size_bytes: 1_282_439_584,

300 + },

301 SupportedModelInfo {

302 id: BARTOWSKI_QWEN3_4B_GGUF,

303 name: "Qwen 3 4B (GGUF)",

@@ -225,12 +326,40 @@ pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[

326 expected_size_bytes: 9_001_753_632,

327 },

328 SupportedModelInfo {

228 - id: BARTOWSKI_QWEN3_1_7B_GGUF,

229 - name: "Qwen 3 1.7B (GGUF)",

329 + id: BARTOWSKI_QWEN3_32B_GGUF,

330 + name: "Qwen 3 32B (GGUF)",

331 org: "Qwen / Alibaba",

231 - description: "Lightweight tool-calling model for mobile (~1.3 GB). \

232 - Smallest Qwen 3 variant with tool calling support.",

233 - expected_size_bytes: 1_282_439_584,

332 + description: "Largest dense Qwen 3 model with extended thinking (~19.8 GB). \

333 + Highest-quality dense variant; requires 32+ GB RAM.",

334 + // Approximate Q4_K_M size; HF API unreachable at authoring time.

335 + expected_size_bytes: 21_260_000_000,

336 + },

337 + SupportedModelInfo {

338 + id: BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF,

339 + name: "Qwen 3 4B Instruct 2507 (GGUF)",

340 + org: "Qwen / Alibaba",

341 + description: "Latest non-thinking 4B checkpoint — faster, predictable chat and \

342 + tool use (~2.5 GB). Recommended general-purpose Qwen 3 model.",

343 + // Approximate Q4_K_M size; HF API unreachable at authoring time.

344 + expected_size_bytes: 2_500_000_000,

345 + },

346 + SupportedModelInfo {

347 + id: BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF,

348 + name: "Qwen 3 4B Thinking 2507 (GGUF)",

349 + org: "Qwen / Alibaba",

350 + description: "Latest reasoning-focused 4B checkpoint with extended thinking (~2.5 GB). \

351 + Stronger reasoning and tool-use accuracy; load with max_tokens ≥ 4096.",

352 + // Approximate Q4_K_M size; HF API unreachable at authoring time.

353 + expected_size_bytes: 2_500_000_000,

354 + },

355 + SupportedModelInfo {

356 + id: BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF,

357 + name: "Qwen 3 30B-A3B Instruct 2507 (GGUF)",

358 + org: "Qwen / Alibaba",

359 + description: "Flagship mixture-of-experts model: 30B total / ~3B active (~18.6 GB). \

360 + Near-dense quality at far lower inference cost; requires 32+ GB RAM.",

361 + // Approximate Q4_K_M size; HF API unreachable at authoring time.

362 + expected_size_bytes: 19_971_600_000,

363 },

364 SupportedModelInfo {

365 id: BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF,

@@ -264,7 +393,48 @@ pub fn tok_model_id_for_repo(hf_repo_id: &str) -> Option<&'static str> {

393 BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF => Some(QWEN25_CODER_1_5B_TOK_MODEL_ID),

394 BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF => Some(QWEN25_CODER_3B_TOK_MODEL_ID),

395 BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF => Some(QWEN25_CODER_7B_TOK_MODEL_ID),

396 + BARTOWSKI_QWEN3_0_6B_GGUF => Some(QWEN3_0_6B_TOK_MODEL_ID),

397 + BARTOWSKI_QWEN3_1_7B_GGUF => Some(QWEN3_1_7B_TOK_MODEL_ID),

398 + BARTOWSKI_QWEN3_4B_GGUF => Some(QWEN3_4B_TOK_MODEL_ID),

399 + BARTOWSKI_QWEN3_8B_GGUF => Some(QWEN3_8B_TOK_MODEL_ID),

400 + BARTOWSKI_QWEN3_14B_GGUF => Some(QWEN3_14B_TOK_MODEL_ID),

401 + BARTOWSKI_QWEN3_32B_GGUF => Some(QWEN3_32B_TOK_MODEL_ID),

402 + BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF => Some(QWEN3_4B_INSTRUCT_2507_TOK_MODEL_ID),

403 + BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF => Some(QWEN3_4B_THINKING_2507_TOK_MODEL_ID),

404 + BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF => {

405 + Some(QWEN3_30B_A3B_INSTRUCT_2507_TOK_MODEL_ID)

406 + }

407 THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF => Some(DEEPSEEK_CODER_6_7B_TOK_MODEL_ID),

408 _ => None,

409 }

410 }

411 +

412 +#[cfg(test)]

413 +mod tests {

414 + use super::*;

415 +

416 + /// Every supported model must have a corresponding display-metadata entry

417 + /// so the model-list UI can render it.

418 + #[test]

419 + fn every_supported_model_has_info() {

420 + for id in SUPPORTED_MODELS {

421 + assert!(

422 + SUPPORTED_MODEL_INFO.iter().any(|info| info.id == *id),

423 + "{id} is in SUPPORTED_MODELS but missing from SUPPORTED_MODEL_INFO"

424 + );

425 + }

426 + }

427 +

428 + /// Every Qwen 3 repo must resolve to an Android tokenizer ID so loading

429 + /// works on the candle GGUF backend.

430 + #[test]

431 + fn qwen3_repos_have_android_tokenizer() {

432 + let qwen3 = SUPPORTED_MODELS.iter().filter(|id| id.contains("Qwen3"));

433 + for id in qwen3 {

434 + assert!(

435 + tok_model_id_for_repo(id).is_some(),

436 + "{id} has no Android tokenizer mapping"

437 + );

438 + }

439 + }

440 +}