@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
copilot/fix-onde-npm-sdk-v1-1-2-release
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
rs 270 lines 14.3 KB
Raw
1 /// Pre-quantized Qwen 2.5 1.5B Instruct (GGUF Q4_K_M) — lightest mobile option (~941 MB).
2 /// Fits comfortably on both iOS (iPhone 16e, 8 GB RAM) and Android memory-constrained devices.
3 /// Note: the 3B variant (~1.93 GB) caused OOM on iPhone 16e because iOS gives apps only ~2-3 GB;
4 /// the 1.5B variant at ~941 MB leaves comfortable headroom for KV cache, activations, and the app.
5 /// bartowski's GGUF embeds the full tokenizer and Qwen2.5 chat template, so on iOS/macOS no
6 /// separate tok_model_id download is needed. On Android an explicit tok_model_id is required.
7 pub const BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF: &str = "bartowski/Qwen2.5-1.5B-Instruct-GGUF";
8 /// The specific GGUF filename to download from the bartowski 1.5B repo.
9 pub const QWEN25_1_5B_GGUF_FILE: &str = "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf";
10 /// Base model repo used for the HF tokenizer on Android.
11 /// On Android, GGUF loading requires an explicit tokenizer source; on iOS/macOS the tokenizer
12 /// embedded in the GGUF file is used instead to avoid an extra network download.
13 pub const QWEN25_1_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-1.5B-Instruct";
14
15 /// Pre-quantized Qwen 2.5 0.5B Instruct (GGUF Q4_K_M), smallest model (~380 MB).
16 /// Used on tvOS (Apple TV 4K) where the hard memory limit is 2 GB.
17 pub const BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF: &str = "bartowski/Qwen2.5-0.5B-Instruct-GGUF";
18 /// The specific GGUF filename to download from the bartowski 0.5B repo.
19 pub const QWEN25_0_5B_GGUF_FILE: &str = "Qwen2.5-0.5B-Instruct-Q4_K_M.gguf";
20 /// Base model repo used for the HF tokenizer on Android.
21 pub const QWEN25_0_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-0.5B-Instruct";
22
23 /// Pre-quantized Qwen 2.5 Coder 1.5B Instruct (GGUF Q4_K_M) — dedicated coding model (~941 MB).
24 ///
25 /// Uses the `qwen2` GGUF architecture, identical to Qwen2.5-1.5B-Instruct, so it loads through
26 /// the existing `quantized_qwen.rs` path in mistral.rs without any code changes.
27 ///
28 /// Strongly preferred over the general-purpose 1.5B for coding tasks: trained on 5.5T tokens of
29 /// code and math data, with fill-in-the-middle (FIM) and repo-level code understanding.
30 /// Same memory footprint as the general 1.5B (~941 MB) but dramatically better at code.
31 ///
32 /// bartowski's GGUF embeds the full tokenizer and chat template, so on iOS/macOS no separate
33 /// tok_model_id download is needed. On Android an explicit tok_model_id is required.
34 pub const BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF: &str =
35 "bartowski/Qwen2.5-Coder-1.5B-Instruct-GGUF";
36 /// The specific GGUF filename to download from the bartowski Coder 1.5B repo.
37 pub const QWEN25_CODER_1_5B_GGUF_FILE: &str = "Qwen2.5-Coder-1.5B-Instruct-Q4_K_M.gguf";
38 /// Base model repo used for the HF tokenizer on Android.
39 pub const QWEN25_CODER_1_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-Coder-1.5B-Instruct";
40
41 /// HuggingFace repo for the full-precision Qwen 2.5 Coder 7B Instruct model.
42 ///
43 /// Used by ISQ pipelines (`TextModelBuilder`) which download the safetensors weights
44 /// directly and quantise them in-situ on the device. Unlike the bartowski GGUF
45 /// variants, this repo ships the original BF16 weights; mistral.rs handles
46 /// quantisation to Q4K or Q8_0 at load time via the `--isq` flag.
47 ///
48 /// Requires ~8 GB RAM during the load phase (4-bit); ~12 GB for 8-bit.
49 /// Metal-accelerated on macOS; CPU fallback available but very slow.
50 pub const QWEN25_CODER_7B_INSTRUCT: &str = "Qwen/Qwen2.5-Coder-7B-Instruct";
51
52 /// Pre-quantized Qwen 2.5 Coder 3B Instruct (GGUF Q4_K_M) — best coding quality on macOS (~1.93 GB).
53 ///
54 /// Same `qwen2` architecture as the 3B general model; ideal for macOS desktops where the extra
55 /// quality headroom over the 1.5B is worthwhile. Not recommended for iOS (OOM risk).
56 pub const BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF: &str =
57 "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF";
58 /// The specific GGUF filename to download from the bartowski Coder 3B repo.
59 pub const QWEN25_CODER_3B_GGUF_FILE: &str = "Qwen2.5-Coder-3B-Instruct-Q4_K_M.gguf";
60 /// Base model repo used for the HF tokenizer on Android.
61 pub const QWEN25_CODER_3B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-Coder-3B-Instruct";
62
63 /// Pre-quantized Qwen 2.5 Coder 7B Instruct (GGUF Q4_K_M) — strong coding model with tool calling (~4.4 GB).
64 ///
65 /// Uses the `qwen2` GGUF architecture. Trained on 5.5T tokens of code/math data with
66 /// fill-in-the-middle and repo-level code understanding. Supports function/tool calling
67 /// via the Qwen2.5 chat template. Requires ~8 GB RAM.
68 pub const BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF: &str =
69 "bartowski/Qwen2.5-Coder-7B-Instruct-GGUF";
70 /// The specific GGUF filename to download from the bartowski Coder 7B repo.
71 pub const QWEN25_CODER_7B_GGUF_FILE: &str = "Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf";
72 /// Base model repo used for the HF tokenizer on Android.
73 pub const QWEN25_CODER_7B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-Coder-7B-Instruct";
74
75 /// Pre-quantized Qwen 2.5 3B Instruct (GGUF Q4_K_M) — balanced option (~1.93 GB).
76 /// Ideal for macOS desktops and Android devices with sufficient RAM.
77 /// Not recommended as default on iOS: the 3B variant caused OOM on iPhone 16e (8 GB RAM)
78 /// because iOS gives apps only ~2-3 GB; use the 1.5B variant for iOS instead.
79 /// No in-situ quantization needed; loads directly at quantized size.
80 pub const BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF: &str = "bartowski/Qwen2.5-3B-Instruct-GGUF";
81 /// The specific GGUF filename to download from the bartowski 3B repo.
82 pub const QWEN25_3B_GGUF_FILE: &str = "Qwen2.5-3B-Instruct-Q4_K_M.gguf";
83 /// Base model repo used for the HF tokenizer (tokenizer.json + tokenizer_config.json).
84 pub const QWEN25_3B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-3B-Instruct";
85
86 /// Pre-quantized Qwen 3 4B Instruct (GGUF Q4_K_M) — full OpenAI-compatible tool calling (~2.7 GB).
87 ///
88 /// Qwen 3 uses an extended thinking mode (`<think>…</think>`) that significantly improves
89 /// reasoning and tool-use accuracy. Load with `max_tokens ≥ 4096` to avoid empty replies caused
90 /// by the model exhausting its token budget on thinking before producing a response.
91 ///
92 /// Recommended model for siGit Code (coding agent with tool calling on macOS/Linux/Windows).
93 pub const BARTOWSKI_QWEN3_4B_GGUF: &str = "bartowski/Qwen_Qwen3-4B-GGUF";
94 /// The specific GGUF filename to download from the bartowski Qwen 3 4B repo.
95 pub const QWEN3_4B_GGUF_FILE: &str = "Qwen_Qwen3-4B-Q4_K_M.gguf";
96
97 /// Pre-quantized Qwen 3 8B (GGUF Q4_K_M) — strong tool-calling model (~5 GB).
98 ///
99 /// Best balance of quality and memory for macOS with 24+ GB RAM.
100 /// Full tool calling and extended thinking mode support.
101 pub const BARTOWSKI_QWEN3_8B_GGUF: &str = "bartowski/Qwen_Qwen3-8B-GGUF";
102 pub const QWEN3_8B_GGUF_FILE: &str = "Qwen_Qwen3-8B-Q4_K_M.gguf";
103
104 /// Pre-quantized Qwen 3 14B (GGUF Q4_K_M) — strong reasoning and tool-calling model (~8.4 GB).
105 ///
106 /// Qwen 3 uses extended thinking mode (`<think>…</think>`) for improved reasoning.
107 /// Best all-around model for macOS with 16+ GB RAM. Full tool calling support.
108 pub const BARTOWSKI_QWEN3_14B_GGUF: &str = "bartowski/Qwen_Qwen3-14B-GGUF";
109 /// The specific GGUF filename.
110 pub const QWEN3_14B_GGUF_FILE: &str = "Qwen_Qwen3-14B-Q4_K_M.gguf";
111
112 /// Pre-quantized Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling model (~1.3 GB).
113 ///
114 /// Smallest Qwen 3 variant with tool calling support. Suitable for mobile devices
115 /// where the 4B model would be too large.
116 pub const BARTOWSKI_QWEN3_1_7B_GGUF: &str = "bartowski/Qwen_Qwen3-1.7B-GGUF";
117 /// The specific GGUF filename for the Qwen3 1.7B repo.
118 pub const QWEN3_1_7B_GGUF_FILE: &str = "Qwen_Qwen3-1.7B-Q4_K_M.gguf";
119
120 /// DeepSeek Coder v1 6.7B Instruct (GGUF Q4_K_M) — dedicated code generation model (~3.8 GB).
121 ///
122 /// Uses the `llama` GGUF architecture, so it loads through the existing
123 /// `quantized_llama.rs` path in mistral.rs without any code changes.
124 /// Strong coding performance. Requires ~8 GB RAM.
125 pub const THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF: &str =
126 "TheBloke/deepseek-coder-6.7B-instruct-GGUF";
127 /// The specific GGUF filename.
128 pub const DEEPSEEK_CODER_6_7B_GGUF_FILE: &str = "deepseek-coder-6.7b-instruct.Q4_K_M.gguf";
129 /// Base model repo used for the HF tokenizer on Android.
130 pub const DEEPSEEK_CODER_6_7B_TOK_MODEL_ID: &str = "deepseek-ai/deepseek-coder-6.7b-instruct";
131
132 /// All model IDs that the Onde inference engine supports.
133 /// Used by `list_local_hf_models` to filter the HuggingFace cache
134 /// to only show models that can actually be used for generation.
135 pub const SUPPORTED_MODELS: &[&str] = &[
136 BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF,
137 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,
138 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,
139 BARTOWSKI_QWEN3_4B_GGUF,
140 BARTOWSKI_QWEN3_8B_GGUF,
141 BARTOWSKI_QWEN3_14B_GGUF,
142 BARTOWSKI_QWEN3_1_7B_GGUF,
143 BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF,
144 THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF,
145 ];
146
147 /// Rich metadata for a supported model, used by the frontend to display
148 /// unavailable models that can be downloaded.
149 pub struct SupportedModelInfo {
150 /// The full HuggingFace model identifier, e.g. "bartowski/Qwen2.5-1.5B-Instruct-GGUF".
151 pub id: &'static str,
152 /// Human-friendly display name for the model.
153 pub name: &'static str,
154 /// Organisation or publisher display name.
155 pub org: &'static str,
156 /// Short description of the model's purpose / capabilities.
157 pub description: &'static str,
158 /// Approximate total size in bytes when fully downloaded.
159 /// Used by the progress monitor to estimate download percentage.
160 ///
161 /// These values are computed by summing every file in the model's
162 /// HuggingFace repository (via the `/api/models/{id}?blobs=true`
163 /// endpoint's `siblings[].size` fields).
164 pub expected_size_bytes: u64,
165 }
166
167 /// Complete list of supported models with display metadata.
168 ///
169 /// When adding a new model, add its constant ID to [`SUPPORTED_MODELS`] **and**
170 /// a corresponding entry here so the frontend can show it in the model list UI.
171 pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[
172 SupportedModelInfo {
173 id: BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF,
174 name: "Qwen 2.5 0.5B (GGUF)",
175 org: "Qwen / Alibaba",
176 description: "Smallest pre-quantized chat model, used on tvOS Apple TV (~380 MB).",
177 expected_size_bytes: 397_808_192,
178 },
179 SupportedModelInfo {
180 id: BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,
181 name: "Qwen 2.5 1.5B (GGUF)",
182 org: "Qwen / Alibaba",
183 description: "Lightest pre-quantized chat model — ideal for iOS & Android (~941 MB). \
184 Fits comfortably within iOS memory limits (iPhone 16e, 8 GB RAM).",
185 // Qwen2.5-1.5B-Instruct-Q4_K_M.gguf from bartowski repo.
186 // Exact file size from HuggingFace API siblings[].size.
187 expected_size_bytes: 986_048_768,
188 },
189 SupportedModelInfo {
190 id: BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,
191 name: "Qwen 2.5 3B (GGUF)",
192 org: "Qwen / Alibaba",
193 description:
194 "Pre-quantized chat model for macOS & Android — balanced quality and size (~1.93 GB). \
195 Not recommended as default on iOS due to memory constraints.",
196 // Qwen2.5-3B-Instruct-Q4_K_M.gguf from bartowski repo.
197 // Exact file size from HuggingFace API siblings[].size.
198 expected_size_bytes: 1_929_903_264,
199 },
200 SupportedModelInfo {
201 id: BARTOWSKI_QWEN3_4B_GGUF,
202 name: "Qwen 3 4B (GGUF)",
203 org: "Qwen / Alibaba",
204 description: "Full tool-calling support with extended reasoning mode (~2.7 GB). \
205 Recommended for siGit Code on macOS, Linux, and Windows.",
206 // Qwen_Qwen3-4B-Q4_K_M.gguf from bartowski repo.
207 expected_size_bytes: 2_596_306_912,
208 },
209 SupportedModelInfo {
210 id: BARTOWSKI_QWEN3_8B_GGUF,
211 name: "Qwen 3 8B (GGUF)",
212 org: "Qwen / Alibaba",
213 description: "Strong tool-calling model with extended thinking (~5 GB). \
214 Best balance of quality and memory for macOS with 24+ GB RAM.",
215 expected_size_bytes: 5_131_567_104,
216 },
217 SupportedModelInfo {
218 id: BARTOWSKI_QWEN3_14B_GGUF,
219 name: "Qwen 3 14B (GGUF)",
220 org: "Qwen / Alibaba",
221 description: "Strong reasoning and tool-calling model with extended thinking (~8.4 GB). \
222 Best all-around model for macOS with 16+ GB RAM.",
223 // Qwen_Qwen3-14B-Q4_K_M.gguf from bartowski repo.
224 // Exact file size from HuggingFace API siblings[].size.
225 expected_size_bytes: 9_001_753_632,
226 },
227 SupportedModelInfo {
228 id: BARTOWSKI_QWEN3_1_7B_GGUF,
229 name: "Qwen 3 1.7B (GGUF)",
230 org: "Qwen / Alibaba",
231 description: "Lightweight tool-calling model for mobile (~1.3 GB). \
232 Smallest Qwen 3 variant with tool calling support.",
233 expected_size_bytes: 1_282_439_584,
234 },
235 SupportedModelInfo {
236 id: BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF,
237 name: "Qwen 2.5 Coder 7B (GGUF)",
238 org: "Qwen / Alibaba",
239 description: "Strong coding model with tool calling support (~4.4 GB). \
240 Trained on 5.5T code tokens. Requires 8+ GB RAM.",
241 expected_size_bytes: 4_683_074_336,
242 },
243 SupportedModelInfo {
244 id: THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF,
245 name: "DeepSeek Coder 6.7B (GGUF)",
246 org: "DeepSeek AI",
247 description: "Strong code generation model using the llama architecture (~3.8 GB). \
248 Requires 8+ GB RAM. Not recommended for mobile devices.",
249 expected_size_bytes: 4_083_015_904,
250 },
251 ];
252
253 /// Return the explicit tokenizer model ID required on Android for `hf_repo_id`.
254 ///
255 /// The candle GGUF backend cannot parse the tokenizer embedded inside GGUF
256 /// files; an explicit `tok_model_id` triggers a separate tokenizer download
257 /// from the base model repo. Returns `None` on iOS and macOS where the
258 /// embedded tokenizer is used automatically.
259 pub fn tok_model_id_for_repo(hf_repo_id: &str) -> Option<&'static str> {
260 match hf_repo_id {
261 BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF => Some(QWEN25_0_5B_TOK_MODEL_ID),
262 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF => Some(QWEN25_1_5B_TOK_MODEL_ID),
263 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF => Some(QWEN25_3B_TOK_MODEL_ID),
264 BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF => Some(QWEN25_CODER_1_5B_TOK_MODEL_ID),
265 BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF => Some(QWEN25_CODER_3B_TOK_MODEL_ID),
266 BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF => Some(QWEN25_CODER_7B_TOK_MODEL_ID),
267 THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF => Some(DEEPSEEK_CODER_6_7B_TOK_MODEL_ID),
268 _ => None,
269 }
270 }