@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
feature/diffusion-vision
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
rs 515 lines 26.2 KB
Raw
1 // Copyright 2026 Splitfire AB (Onde Inference). All rights reserved.
2 // SPDX-License-Identifier: MIT OR Apache-2.0
3
4 pub const BFL_FLUX_1_SCHNELL: &str = "black-forest-labs/FLUX.1-schnell";
5 pub const GOOGLE_GEMMA_3N_E2B_IT: &str = "google/gemma-3n-E2B-it";
6 pub const GOOGLE_GEMMA_3N_E4B_IT: &str = "google/gemma-3n-E4B-it";
7 pub const MICROSOFT_PHI_35_MINI_INSTRUCT: &str = "microsoft/Phi-3.5-mini-instruct";
8
9 /// Pre-quantized Qwen 2.5 1.5B Instruct (GGUF Q4_K_M) — lightest mobile option (~941 MB).
10 /// Fits comfortably on both iOS (iPhone 16e, 8 GB RAM) and Android memory-constrained devices.
11 /// Note: the 3B variant (~1.93 GB) caused OOM on iPhone 16e because iOS gives apps only ~2-3 GB;
12 /// the 1.5B variant at ~941 MB leaves comfortable headroom for KV cache, activations, and the app.
13 /// bartowski's GGUF embeds the full tokenizer and Qwen2.5 chat template, so on iOS/macOS no
14 /// separate tok_model_id download is needed. On Android an explicit tok_model_id is required.
15 pub const BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF: &str = "bartowski/Qwen2.5-1.5B-Instruct-GGUF";
16 /// The specific GGUF filename to download from the bartowski 1.5B repo.
17 pub const QWEN25_1_5B_GGUF_FILE: &str = "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf";
18 /// Base model repo used for the HF tokenizer on Android.
19 /// On Android, GGUF loading requires an explicit tokenizer source; on iOS/macOS the tokenizer
20 /// embedded in the GGUF file is used instead to avoid an extra network download.
21 pub const QWEN25_1_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-1.5B-Instruct";
22
23 /// Pre-quantized Qwen 2.5 0.5B Instruct (GGUF Q4_K_M), smallest model (~380 MB).
24 /// Used on tvOS (Apple TV 4K) where the hard memory limit is 2 GB.
25 pub const BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF: &str = "bartowski/Qwen2.5-0.5B-Instruct-GGUF";
26 /// The specific GGUF filename to download from the bartowski 0.5B repo.
27 pub const QWEN25_0_5B_GGUF_FILE: &str = "Qwen2.5-0.5B-Instruct-Q4_K_M.gguf";
28 /// Base model repo used for the HF tokenizer on Android.
29 pub const QWEN25_0_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-0.5B-Instruct";
30
31 /// Pre-quantized Qwen 2.5 Coder 1.5B Instruct (GGUF Q4_K_M) — dedicated coding model (~941 MB).
32 ///
33 /// Uses the `qwen2` GGUF architecture, identical to Qwen2.5-1.5B-Instruct, so it loads through
34 /// the existing `quantized_qwen.rs` path in mistral.rs without any code changes.
35 ///
36 /// Strongly preferred over the general-purpose 1.5B for coding tasks: trained on 5.5T tokens of
37 /// code and math data, with fill-in-the-middle (FIM) and repo-level code understanding.
38 /// Same memory footprint as the general 1.5B (~941 MB) but dramatically better at code.
39 ///
40 /// bartowski's GGUF embeds the full tokenizer and chat template, so on iOS/macOS no separate
41 /// tok_model_id download is needed. On Android an explicit tok_model_id is required.
42 pub const BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF: &str =
43 "bartowski/Qwen2.5-Coder-1.5B-Instruct-GGUF";
44 /// The specific GGUF filename to download from the bartowski Coder 1.5B repo.
45 pub const QWEN25_CODER_1_5B_GGUF_FILE: &str = "Qwen2.5-Coder-1.5B-Instruct-Q4_K_M.gguf";
46 /// Base model repo used for the HF tokenizer on Android.
47 pub const QWEN25_CODER_1_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-Coder-1.5B-Instruct";
48
49 /// HuggingFace repo for the full-precision Qwen 2.5 Coder 7B Instruct model.
50 ///
51 /// Used by ISQ pipelines (`TextModelBuilder`) which download the safetensors weights
52 /// directly and quantise them in-situ on the device. Unlike the bartowski GGUF
53 /// variants, this repo ships the original BF16 weights; mistral.rs handles
54 /// quantisation to Q4K or Q8_0 at load time via the `--isq` flag.
55 ///
56 /// Requires ~8 GB RAM during the load phase (4-bit); ~12 GB for 8-bit.
57 /// Metal-accelerated on macOS; CPU fallback available but very slow.
58 pub const QWEN25_CODER_7B_INSTRUCT: &str = "Qwen/Qwen2.5-Coder-7B-Instruct";
59
60 /// Pre-quantized Qwen 2.5 Coder 3B Instruct (GGUF Q4_K_M) — best coding quality on macOS (~1.93 GB).
61 ///
62 /// Same `qwen2` architecture as the 3B general model; ideal for macOS desktops where the extra
63 /// quality headroom over the 1.5B is worthwhile. Not recommended for iOS (OOM risk).
64 pub const BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF: &str =
65 "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF";
66 /// The specific GGUF filename to download from the bartowski Coder 3B repo.
67 pub const QWEN25_CODER_3B_GGUF_FILE: &str = "Qwen2.5-Coder-3B-Instruct-Q4_K_M.gguf";
68 /// Base model repo used for the HF tokenizer on Android.
69 pub const QWEN25_CODER_3B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-Coder-3B-Instruct";
70
71 /// Pre-quantized Qwen 2.5 Coder 7B Instruct (GGUF Q4_K_M) — strong coding model with tool calling (~4.4 GB).
72 ///
73 /// Uses the `qwen2` GGUF architecture. Trained on 5.5T tokens of code/math data with
74 /// fill-in-the-middle and repo-level code understanding. Supports function/tool calling
75 /// via the Qwen2.5 chat template. Requires ~8 GB RAM.
76 pub const BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF: &str =
77 "bartowski/Qwen2.5-Coder-7B-Instruct-GGUF";
78 /// The specific GGUF filename to download from the bartowski Coder 7B repo.
79 pub const QWEN25_CODER_7B_GGUF_FILE: &str = "Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf";
80 /// Base model repo used for the HF tokenizer on Android.
81 pub const QWEN25_CODER_7B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-Coder-7B-Instruct";
82
83 /// Pre-quantized Qwen 2.5 3B Instruct (GGUF Q4_K_M) — balanced option (~1.93 GB).
84 /// Ideal for macOS desktops and Android devices with sufficient RAM.
85 /// Not recommended as default on iOS: the 3B variant caused OOM on iPhone 16e (8 GB RAM)
86 /// because iOS gives apps only ~2-3 GB; use the 1.5B variant for iOS instead.
87 /// No in-situ quantization needed; loads directly at quantized size.
88 pub const BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF: &str = "bartowski/Qwen2.5-3B-Instruct-GGUF";
89 /// The specific GGUF filename to download from the bartowski 3B repo.
90 pub const QWEN25_3B_GGUF_FILE: &str = "Qwen2.5-3B-Instruct-Q4_K_M.gguf";
91 /// Base model repo used for the HF tokenizer (tokenizer.json + tokenizer_config.json).
92 pub const QWEN25_3B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-3B-Instruct";
93
94 // ── Qwen 3 family (GGUF Q4_K_M) ──────────────────────────────────────────────
95 //
96 // The Qwen 3 line uses the `qwen3` GGUF architecture (and `qwen3moe` for the
97 // 30B-A3B mixture-of-experts variant), both supported by mistral.rs's quantized
98 // loader. Every Qwen 3 model is a hybrid reasoner with an extended thinking mode
99 // (`<think>…</think>`); always load with `max_tokens ≥ 4096` so the thinking
100 // block does not exhaust the token budget before the real reply.
101 //
102 // bartowski's repos embed the tokenizer and chat template inside the GGUF, so on
103 // iOS/macOS no separate tokenizer download is needed. On Android the candle GGUF
104 // backend cannot parse the embedded tokenizer, so each model also declares a
105 // `TOK_MODEL_ID` pointing at the official Qwen repo for a standalone tokenizer.
106
107 /// Pre-quantized Qwen 3 0.6B (GGUF Q4_K_M) — smallest Qwen 3 variant (~0.5 GB).
108 ///
109 /// Lightest tool-capable Qwen 3 model. Suitable for tvOS and the most
110 /// memory-constrained mobile devices where even the 1.7B is too large.
111 pub const BARTOWSKI_QWEN3_0_6B_GGUF: &str = "bartowski/Qwen_Qwen3-0.6B-GGUF";
112 /// The specific GGUF filename for the bartowski Qwen 3 0.6B repo.
113 pub const QWEN3_0_6B_GGUF_FILE: &str = "Qwen_Qwen3-0.6B-Q4_K_M.gguf";
114 /// Base model repo used for the HF tokenizer on Android.
115 pub const QWEN3_0_6B_TOK_MODEL_ID: &str = "Qwen/Qwen3-0.6B";
116
117 /// Pre-quantized Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling model (~1.3 GB).
118 ///
119 /// Smallest Qwen 3 variant with comfortable tool calling. Suitable for mobile
120 /// devices where the 4B model would be too large.
121 pub const BARTOWSKI_QWEN3_1_7B_GGUF: &str = "bartowski/Qwen_Qwen3-1.7B-GGUF";
122 /// The specific GGUF filename for the Qwen3 1.7B repo.
123 pub const QWEN3_1_7B_GGUF_FILE: &str = "Qwen_Qwen3-1.7B-Q4_K_M.gguf";
124 /// Base model repo used for the HF tokenizer on Android.
125 pub const QWEN3_1_7B_TOK_MODEL_ID: &str = "Qwen/Qwen3-1.7B";
126
127 /// Pre-quantized Qwen 3 4B (GGUF Q4_K_M) — full OpenAI-compatible tool calling (~2.7 GB).
128 ///
129 /// Recommended model for siGit Code (coding agent with tool calling on macOS/Linux/Windows).
130 pub const BARTOWSKI_QWEN3_4B_GGUF: &str = "bartowski/Qwen_Qwen3-4B-GGUF";
131 /// The specific GGUF filename to download from the bartowski Qwen 3 4B repo.
132 pub const QWEN3_4B_GGUF_FILE: &str = "Qwen_Qwen3-4B-Q4_K_M.gguf";
133 /// Base model repo used for the HF tokenizer on Android.
134 pub const QWEN3_4B_TOK_MODEL_ID: &str = "Qwen/Qwen3-4B";
135
136 /// Pre-quantized Qwen 3 8B (GGUF Q4_K_M) — strong tool-calling model (~5 GB).
137 ///
138 /// Best balance of quality and memory for macOS with 24+ GB RAM.
139 /// Full tool calling and extended thinking mode support.
140 pub const BARTOWSKI_QWEN3_8B_GGUF: &str = "bartowski/Qwen_Qwen3-8B-GGUF";
141 pub const QWEN3_8B_GGUF_FILE: &str = "Qwen_Qwen3-8B-Q4_K_M.gguf";
142 /// Base model repo used for the HF tokenizer on Android.
143 pub const QWEN3_8B_TOK_MODEL_ID: &str = "Qwen/Qwen3-8B";
144
145 /// Pre-quantized Qwen 3 14B (GGUF Q4_K_M) — strong reasoning and tool-calling model (~8.4 GB).
146 ///
147 /// Best all-around model for macOS with 16+ GB RAM. Full tool calling support.
148 pub const BARTOWSKI_QWEN3_14B_GGUF: &str = "bartowski/Qwen_Qwen3-14B-GGUF";
149 /// The specific GGUF filename.
150 pub const QWEN3_14B_GGUF_FILE: &str = "Qwen_Qwen3-14B-Q4_K_M.gguf";
151 /// Base model repo used for the HF tokenizer on Android.
152 pub const QWEN3_14B_TOK_MODEL_ID: &str = "Qwen/Qwen3-14B";
153
154 /// Pre-quantized Qwen 3 32B (GGUF Q4_K_M) — largest dense Qwen 3 model (~19.8 GB).
155 ///
156 /// Highest-quality dense Qwen 3 variant. Requires a high-memory desktop
157 /// (32+ GB RAM / unified memory). Full tool calling and extended thinking.
158 pub const BARTOWSKI_QWEN3_32B_GGUF: &str = "bartowski/Qwen_Qwen3-32B-GGUF";
159 /// The specific GGUF filename for the bartowski Qwen 3 32B repo.
160 pub const QWEN3_32B_GGUF_FILE: &str = "Qwen_Qwen3-32B-Q4_K_M.gguf";
161 /// Base model repo used for the HF tokenizer on Android.
162 pub const QWEN3_32B_TOK_MODEL_ID: &str = "Qwen/Qwen3-32B";
163
164 // ── Qwen 3 "2507" updated releases ───────────────────────────────────────────
165 //
166 // The July/August 2025 refresh split Qwen 3 into dedicated non-thinking
167 // (`-Instruct-2507`) and thinking-only (`-Thinking-2507`) checkpoints with
168 // markedly improved instruction following, tool use, and long-context quality.
169 // These are the latest open Qwen 3 weights available as on-device GGUF.
170
171 /// Pre-quantized Qwen 3 4B Instruct 2507 (GGUF Q4_K_M) — latest non-thinking 4B (~2.5 GB).
172 ///
173 /// Updated instruction-tuned checkpoint. Unlike the base 4B it does *not* emit a
174 /// `<think>` block, so it is faster and more predictable for chat and tool use.
175 pub const BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF: &str =
176 "bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF";
177 /// The specific GGUF filename for the bartowski Qwen 3 4B Instruct 2507 repo.
178 pub const QWEN3_4B_INSTRUCT_2507_GGUF_FILE: &str = "Qwen_Qwen3-4B-Instruct-2507-Q4_K_M.gguf";
179 /// Base model repo used for the HF tokenizer on Android.
180 pub const QWEN3_4B_INSTRUCT_2507_TOK_MODEL_ID: &str = "Qwen/Qwen3-4B-Instruct-2507";
181
182 /// Pre-quantized Qwen 3 4B Thinking 2507 (GGUF Q4_K_M) — latest reasoning 4B (~2.5 GB).
183 ///
184 /// Updated thinking-only checkpoint with stronger reasoning and tool-use accuracy.
185 /// Always emits a `<think>` block — load with `max_tokens ≥ 4096`.
186 pub const BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF: &str =
187 "bartowski/Qwen_Qwen3-4B-Thinking-2507-GGUF";
188 /// The specific GGUF filename for the bartowski Qwen 3 4B Thinking 2507 repo.
189 pub const QWEN3_4B_THINKING_2507_GGUF_FILE: &str = "Qwen_Qwen3-4B-Thinking-2507-Q4_K_M.gguf";
190 /// Base model repo used for the HF tokenizer on Android.
191 pub const QWEN3_4B_THINKING_2507_TOK_MODEL_ID: &str = "Qwen/Qwen3-4B-Thinking-2507";
192
193 /// Pre-quantized Qwen 3 30B-A3B Instruct 2507 (GGUF Q4_K_M) — flagship MoE (~18.6 GB).
194 ///
195 /// Mixture-of-experts model: 30B total parameters but only ~3B active per token,
196 /// so inference is far cheaper than a 30B dense model while quality rivals it.
197 /// Loads via the `qwen3moe` GGUF architecture in mistral.rs. Requires a
198 /// high-memory desktop (32+ GB RAM / unified memory).
199 pub const BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF: &str =
200 "bartowski/Qwen_Qwen3-30B-A3B-Instruct-2507-GGUF";
201 /// The specific GGUF filename for the bartowski Qwen 3 30B-A3B Instruct 2507 repo.
202 pub const QWEN3_30B_A3B_INSTRUCT_2507_GGUF_FILE: &str =
203 "Qwen_Qwen3-30B-A3B-Instruct-2507-Q4_K_M.gguf";
204 /// Base model repo used for the HF tokenizer on Android.
205 pub const QWEN3_30B_A3B_INSTRUCT_2507_TOK_MODEL_ID: &str = "Qwen/Qwen3-30B-A3B-Instruct-2507";
206
207 /// DeepSeek Coder v1 6.7B Instruct (GGUF Q4_K_M) — dedicated code generation model (~3.8 GB).
208 ///
209 /// Uses the `llama` GGUF architecture, so it loads through the existing
210 /// `quantized_llama.rs` path in mistral.rs without any code changes.
211 /// Strong coding performance. Requires ~8 GB RAM.
212 pub const THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF: &str =
213 "TheBloke/deepseek-coder-6.7B-instruct-GGUF";
214 /// The specific GGUF filename.
215 pub const DEEPSEEK_CODER_6_7B_GGUF_FILE: &str = "deepseek-coder-6.7b-instruct.Q4_K_M.gguf";
216 /// Base model repo used for the HF tokenizer on Android.
217 pub const DEEPSEEK_CODER_6_7B_TOK_MODEL_ID: &str = "deepseek-ai/deepseek-coder-6.7b-instruct";
218
219 /// All model IDs that the Onde inference engine supports.
220 /// Used by `list_local_hf_models` to filter the HuggingFace cache
221 /// to only show models that can actually be used for generation.
222 pub const SUPPORTED_MODELS: &[&str] = &[
223 BFL_FLUX_1_SCHNELL,
224 GOOGLE_GEMMA_3N_E2B_IT,
225 GOOGLE_GEMMA_3N_E4B_IT,
226 MICROSOFT_PHI_35_MINI_INSTRUCT,
227 BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF,
228 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,
229 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,
230 BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF,
231 BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF,
232 BARTOWSKI_QWEN3_0_6B_GGUF,
233 BARTOWSKI_QWEN3_1_7B_GGUF,
234 BARTOWSKI_QWEN3_4B_GGUF,
235 BARTOWSKI_QWEN3_8B_GGUF,
236 BARTOWSKI_QWEN3_14B_GGUF,
237 BARTOWSKI_QWEN3_32B_GGUF,
238 BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF,
239 BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF,
240 BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF,
241 BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF,
242 THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF,
243 ];
244
245 /// Rich metadata for a supported model, used by the frontend to display
246 /// unavailable models that can be downloaded.
247 pub struct SupportedModelInfo {
248 /// The full HuggingFace model identifier, e.g. "black-forest-labs/FLUX.1-schnell".
249 pub id: &'static str,
250 /// Human-friendly display name for the model.
251 pub name: &'static str,
252 /// Organisation or publisher display name.
253 pub org: &'static str,
254 /// Short description of the model's purpose / capabilities.
255 pub description: &'static str,
256 /// Approximate total size in bytes when fully downloaded.
257 /// Used by the progress monitor to estimate download percentage.
258 ///
259 /// These values are computed by summing every file in the model's
260 /// HuggingFace repository (via the `/api/models/{id}?blobs=true`
261 /// endpoint's `siblings[].size` fields).
262 pub expected_size_bytes: u64,
263 }
264
265 /// Complete list of supported models with display metadata.
266 ///
267 /// When adding a new model, add its constant ID to [`SUPPORTED_MODELS`] **and**
268 /// a corresponding entry here so the frontend can show it in the model list UI.
269 pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[
270 SupportedModelInfo {
271 id: BFL_FLUX_1_SCHNELL,
272 name: "FLUX.1 Schnell",
273 org: "Black Forest Labs",
274 description: "Fast, high-quality image generation model optimised for speed.",
275 // Only counting flux1-schnell.safetensors (23.78 GB) — the single-file
276 // model weight that mistral.rs actually downloads for FluxOffloaded.
277 expected_size_bytes: 23_782_506_688,
278 },
279 SupportedModelInfo {
280 id: GOOGLE_GEMMA_3N_E2B_IT,
281 name: "Gemma 3n E2B",
282 org: "Google",
283 description: "Compact prompt enhancer model — lightweight and fast on macOS.",
284 // Sum of all repo files: 3 safetensor shards (3.08 + 4.98 + 2.82 GB)
285 // + tokenizer.json (33 MB) + tokenizer.model (4.7 MB) + configs, etc.
286 // Total ≈ 10.92 GB
287 expected_size_bytes: 10_922_539_343,
288 },
289 SupportedModelInfo {
290 id: GOOGLE_GEMMA_3N_E4B_IT,
291 name: "Gemma 3n E4B",
292 org: "Google",
293 description:
294 "Balanced prompt enhancer model — good quality-to-size ratio for macOS desktop.",
295 // Sum of all repo files: 4 safetensor shards (3.08 + 4.97 + 4.99 + 2.66 GB)
296 // + tokenizer.json (33 MB) + tokenizer.model (4.7 MB) + configs, etc.
297 // Total ≈ 15.74 GB
298 expected_size_bytes: 15_743_647_548,
299 },
300 SupportedModelInfo {
301 id: MICROSOFT_PHI_35_MINI_INSTRUCT,
302 name: "Phi-3.5-mini",
303 org: "Microsoft",
304 description:
305 "Strongest prompt enhancer quality, larger memory footprint (~2.8 GB with Q4K).",
306 // Sum of all repo files: 2 safetensor shards (4.97 + 2.67 GB)
307 // + tokenizer.json (1.8 MB) + tokenizer.model (500 KB) + configs, etc.
308 // Total ≈ 7.64 GB
309 expected_size_bytes: 7_644_702_484,
310 },
311 SupportedModelInfo {
312 id: BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF,
313 name: "Qwen 2.5 Coder 1.5B (GGUF)",
314 org: "Qwen / Alibaba",
315 description:
316 "Dedicated coding model — same size as the general 1.5B (~941 MB) but trained \
317 on 5.5T tokens of code and math. Strongly preferred for sigit. \
318 Ideal for iOS & Android.",
319 // Qwen2.5-Coder-1.5B-Instruct-Q4_K_M.gguf from bartowski repo.
320 expected_size_bytes: 986_048_768,
321 },
322 SupportedModelInfo {
323 id: BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF,
324 name: "Qwen 2.5 Coder 3B (GGUF)",
325 org: "Qwen / Alibaba",
326 description: "Best on-device coding quality for macOS desktop (~1.93 GB). \
327 Trained on 5.5T tokens of code and math data. Not recommended for iOS.",
328 // Qwen2.5-Coder-3B-Instruct-Q4_K_M.gguf from bartowski repo.
329 expected_size_bytes: 1_929_903_264,
330 },
331 SupportedModelInfo {
332 id: BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF,
333 name: "Qwen 2.5 0.5B (GGUF)",
334 org: "Qwen / Alibaba",
335 description: "Smallest pre-quantized chat model, used on tvOS Apple TV (~380 MB).",
336 expected_size_bytes: 397_808_192,
337 },
338 SupportedModelInfo {
339 id: BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,
340 name: "Qwen 2.5 1.5B (GGUF)",
341 org: "Qwen / Alibaba",
342 description: "Lightest pre-quantized chat model — ideal for iOS & Android (~941 MB). \
343 Fits comfortably within iOS memory limits (iPhone 16e, 8 GB RAM).",
344 // Qwen2.5-1.5B-Instruct-Q4_K_M.gguf from bartowski repo.
345 // Exact file size from HuggingFace API siblings[].size.
346 expected_size_bytes: 986_048_768,
347 },
348 SupportedModelInfo {
349 id: BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,
350 name: "Qwen 2.5 3B (GGUF)",
351 org: "Qwen / Alibaba",
352 description:
353 "Pre-quantized chat model for macOS & Android — balanced quality and size (~1.93 GB). \
354 Not recommended as default on iOS due to memory constraints.",
355 // Qwen2.5-3B-Instruct-Q4_K_M.gguf from bartowski repo.
356 // Exact file size from HuggingFace API siblings[].size.
357 expected_size_bytes: 1_929_903_264,
358 },
359 SupportedModelInfo {
360 id: BARTOWSKI_QWEN3_0_6B_GGUF,
361 name: "Qwen 3 0.6B (GGUF)",
362 org: "Qwen / Alibaba",
363 description: "Smallest Qwen 3 variant with tool calling (~0.5 GB). \
364 Suitable for tvOS and the most memory-constrained mobile devices.",
365 // Exact Q4_K_M file size from HuggingFace API siblings[].size.
366 expected_size_bytes: 484_220_320,
367 },
368 SupportedModelInfo {
369 id: BARTOWSKI_QWEN3_1_7B_GGUF,
370 name: "Qwen 3 1.7B (GGUF)",
371 org: "Qwen / Alibaba",
372 description: "Lightweight tool-calling model for mobile (~1.3 GB). \
373 Smallest Qwen 3 variant with comfortable tool calling support.",
374 expected_size_bytes: 1_282_439_584,
375 },
376 SupportedModelInfo {
377 id: BARTOWSKI_QWEN3_4B_GGUF,
378 name: "Qwen 3 4B (GGUF)",
379 org: "Qwen / Alibaba",
380 description: "Full tool-calling support with extended reasoning mode (~2.7 GB). \
381 Recommended for siGit Code on macOS, Linux, and Windows.",
382 // Qwen_Qwen3-4B-Q4_K_M.gguf from bartowski repo.
383 expected_size_bytes: 2_497_280_960,
384 },
385 SupportedModelInfo {
386 id: BARTOWSKI_QWEN3_8B_GGUF,
387 name: "Qwen 3 8B (GGUF)",
388 org: "Qwen / Alibaba",
389 description: "Strong tool-calling model with extended thinking (~5 GB). \
390 Best balance of quality and memory for macOS with 24+ GB RAM.",
391 expected_size_bytes: 5_027_784_224,
392 },
393 SupportedModelInfo {
394 id: BARTOWSKI_QWEN3_14B_GGUF,
395 name: "Qwen 3 14B (GGUF)",
396 org: "Qwen / Alibaba",
397 description: "Strong reasoning and tool-calling model with extended thinking (~8.4 GB). \
398 Best all-around model for macOS with 16+ GB RAM.",
399 // Qwen_Qwen3-14B-Q4_K_M.gguf from bartowski repo.
400 // Exact file size from HuggingFace API siblings[].size.
401 expected_size_bytes: 9_001_753_632,
402 },
403 SupportedModelInfo {
404 id: BARTOWSKI_QWEN3_32B_GGUF,
405 name: "Qwen 3 32B (GGUF)",
406 org: "Qwen / Alibaba",
407 description: "Largest dense Qwen 3 model with extended thinking (~19.8 GB). \
408 Highest-quality dense variant; requires 32+ GB RAM.",
409 // Exact Q4_K_M file size from HuggingFace API siblings[].size.
410 expected_size_bytes: 19_762_149_696,
411 },
412 SupportedModelInfo {
413 id: BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF,
414 name: "Qwen 3 4B Instruct 2507 (GGUF)",
415 org: "Qwen / Alibaba",
416 description: "Latest non-thinking 4B checkpoint — faster, predictable chat and \
417 tool use (~2.5 GB). Recommended general-purpose Qwen 3 model.",
418 // Exact Q4_K_M file size from HuggingFace API siblings[].size.
419 expected_size_bytes: 2_497_280_736,
420 },
421 SupportedModelInfo {
422 id: BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF,
423 name: "Qwen 3 4B Thinking 2507 (GGUF)",
424 org: "Qwen / Alibaba",
425 description: "Latest reasoning-focused 4B checkpoint with extended thinking (~2.5 GB). \
426 Stronger reasoning and tool-use accuracy; load with max_tokens ≥ 4096.",
427 // Exact Q4_K_M file size from HuggingFace API siblings[].size.
428 expected_size_bytes: 2_497_280_736,
429 },
430 SupportedModelInfo {
431 id: BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF,
432 name: "Qwen 3 30B-A3B Instruct 2507 (GGUF)",
433 org: "Qwen / Alibaba",
434 description: "Flagship mixture-of-experts model: 30B total / ~3B active (~18.6 GB). \
435 Near-dense quality at far lower inference cost; requires 32+ GB RAM.",
436 // Exact Q4_K_M file size from HuggingFace API siblings[].size.
437 expected_size_bytes: 18_632_183_808,
438 },
439 SupportedModelInfo {
440 id: BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF,
441 name: "Qwen 2.5 Coder 7B (GGUF)",
442 org: "Qwen / Alibaba",
443 description: "Strong coding model with tool calling support (~4.4 GB). \
444 Trained on 5.5T code tokens. Requires 8+ GB RAM.",
445 expected_size_bytes: 4_683_074_336,
446 },
447 SupportedModelInfo {
448 id: THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF,
449 name: "DeepSeek Coder 6.7B (GGUF)",
450 org: "DeepSeek AI",
451 description: "Strong code generation model using the llama architecture (~3.8 GB). \
452 Requires 8+ GB RAM. Not recommended for mobile devices.",
453 expected_size_bytes: 4_083_015_904,
454 },
455 ];
456
457 /// Return the explicit tokenizer model ID required on Android for `hf_repo_id`.
458 ///
459 /// The candle GGUF backend cannot parse the tokenizer embedded inside GGUF
460 /// files; an explicit `tok_model_id` triggers a separate tokenizer download
461 /// from the base model repo. Returns `None` on iOS and macOS where the
462 /// embedded tokenizer is used automatically.
463 pub fn tok_model_id_for_repo(hf_repo_id: &str) -> Option<&'static str> {
464 match hf_repo_id {
465 BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF => Some(QWEN25_0_5B_TOK_MODEL_ID),
466 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF => Some(QWEN25_1_5B_TOK_MODEL_ID),
467 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF => Some(QWEN25_3B_TOK_MODEL_ID),
468 BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF => Some(QWEN25_CODER_1_5B_TOK_MODEL_ID),
469 BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF => Some(QWEN25_CODER_3B_TOK_MODEL_ID),
470 BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF => Some(QWEN25_CODER_7B_TOK_MODEL_ID),
471 BARTOWSKI_QWEN3_0_6B_GGUF => Some(QWEN3_0_6B_TOK_MODEL_ID),
472 BARTOWSKI_QWEN3_1_7B_GGUF => Some(QWEN3_1_7B_TOK_MODEL_ID),
473 BARTOWSKI_QWEN3_4B_GGUF => Some(QWEN3_4B_TOK_MODEL_ID),
474 BARTOWSKI_QWEN3_8B_GGUF => Some(QWEN3_8B_TOK_MODEL_ID),
475 BARTOWSKI_QWEN3_14B_GGUF => Some(QWEN3_14B_TOK_MODEL_ID),
476 BARTOWSKI_QWEN3_32B_GGUF => Some(QWEN3_32B_TOK_MODEL_ID),
477 BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF => Some(QWEN3_4B_INSTRUCT_2507_TOK_MODEL_ID),
478 BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF => Some(QWEN3_4B_THINKING_2507_TOK_MODEL_ID),
479 BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF => {
480 Some(QWEN3_30B_A3B_INSTRUCT_2507_TOK_MODEL_ID)
481 }
482 THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF => Some(DEEPSEEK_CODER_6_7B_TOK_MODEL_ID),
483 _ => None,
484 }
485 }
486
487 #[cfg(test)]
488 mod tests {
489 use super::*;
490
491 /// Every supported model must have a corresponding display-metadata entry
492 /// so the model-list UI can render it.
493 #[test]
494 fn every_supported_model_has_info() {
495 for id in SUPPORTED_MODELS {
496 assert!(
497 SUPPORTED_MODEL_INFO.iter().any(|info| info.id == *id),
498 "{id} is in SUPPORTED_MODELS but missing from SUPPORTED_MODEL_INFO"
499 );
500 }
501 }
502
503 /// Every Qwen 3 repo must resolve to an Android tokenizer ID so loading
504 /// works on the candle GGUF backend.
505 #[test]
506 fn qwen3_repos_have_android_tokenizer() {
507 let qwen3 = SUPPORTED_MODELS.iter().filter(|id| id.contains("Qwen3"));
508 for id in qwen3 {
509 assert!(
510 tok_model_id_for_repo(id).is_some(),
511 "{id} has no Android tokenizer mapping"
512 );
513 }
514 }
515 }