@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
copilot/assess-inference-engine-complexity
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
rs 253 lines 13.4 KB
Raw
1 /// Pre-quantized Qwen 2.5 1.5B Instruct (GGUF Q4_K_M) — lightest mobile option (~941 MB).
2 /// Fits comfortably on both iOS (iPhone 16e, 8 GB RAM) and Android memory-constrained devices.
3 /// Note: the 3B variant (~1.93 GB) caused OOM on iPhone 16e because iOS gives apps only ~2-3 GB;
4 /// the 1.5B variant at ~941 MB leaves comfortable headroom for KV cache, activations, and the app.
5 /// bartowski's GGUF embeds the full tokenizer and Qwen2.5 chat template, so on iOS/macOS no
6 /// separate tok_model_id download is needed. On Android an explicit tok_model_id is required.
7 pub const BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF: &str = "bartowski/Qwen2.5-1.5B-Instruct-GGUF";
8 /// The specific GGUF filename to download from the bartowski 1.5B repo.
9 pub const QWEN25_1_5B_GGUF_FILE: &str = "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf";
10 /// Base model repo used for the HF tokenizer on Android.
11 /// On Android, GGUF loading requires an explicit tokenizer source; on iOS/macOS the tokenizer
12 /// embedded in the GGUF file is used instead to avoid an extra network download.
13 pub const QWEN25_1_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-1.5B-Instruct";
14
15 /// Pre-quantized Qwen 2.5 Coder 1.5B Instruct (GGUF Q4_K_M) — dedicated coding model (~941 MB).
16 ///
17 /// Uses the `qwen2` GGUF architecture, identical to Qwen2.5-1.5B-Instruct, so it loads through
18 /// the existing `quantized_qwen.rs` path in mistral.rs without any code changes.
19 ///
20 /// Strongly preferred over the general-purpose 1.5B for coding tasks: trained on 5.5T tokens of
21 /// code and math data, with fill-in-the-middle (FIM) and repo-level code understanding.
22 /// Same memory footprint as the general 1.5B (~941 MB) but dramatically better at code.
23 ///
24 /// bartowski's GGUF embeds the full tokenizer and chat template, so on iOS/macOS no separate
25 /// tok_model_id download is needed. On Android an explicit tok_model_id is required.
26 pub const BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF: &str =
27 "bartowski/Qwen2.5-Coder-1.5B-Instruct-GGUF";
28 /// The specific GGUF filename to download from the bartowski Coder 1.5B repo.
29 pub const QWEN25_CODER_1_5B_GGUF_FILE: &str = "Qwen2.5-Coder-1.5B-Instruct-Q4_K_M.gguf";
30 /// Base model repo used for the HF tokenizer on Android.
31 pub const QWEN25_CODER_1_5B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-Coder-1.5B-Instruct";
32
33 /// HuggingFace repo for the full-precision Qwen 2.5 Coder 7B Instruct model.
34 ///
35 /// Used by ISQ pipelines (`TextModelBuilder`) which download the safetensors weights
36 /// directly and quantise them in-situ on the device. Unlike the bartowski GGUF
37 /// variants, this repo ships the original BF16 weights; mistral.rs handles
38 /// quantisation to Q4K or Q8_0 at load time via the `--isq` flag.
39 ///
40 /// Requires ~8 GB RAM during the load phase (4-bit); ~12 GB for 8-bit.
41 /// Metal-accelerated on macOS; CPU fallback available but very slow.
42 pub const QWEN25_CODER_7B_INSTRUCT: &str = "Qwen/Qwen2.5-Coder-7B-Instruct";
43
44 /// Pre-quantized Qwen 2.5 Coder 3B Instruct (GGUF Q4_K_M) — best coding quality on macOS (~1.93 GB).
45 ///
46 /// Same `qwen2` architecture as the 3B general model; ideal for macOS desktops where the extra
47 /// quality headroom over the 1.5B is worthwhile. Not recommended for iOS (OOM risk).
48 pub const BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF: &str =
49 "bartowski/Qwen2.5-Coder-3B-Instruct-GGUF";
50 /// The specific GGUF filename to download from the bartowski Coder 3B repo.
51 pub const QWEN25_CODER_3B_GGUF_FILE: &str = "Qwen2.5-Coder-3B-Instruct-Q4_K_M.gguf";
52 /// Base model repo used for the HF tokenizer on Android.
53 pub const QWEN25_CODER_3B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-Coder-3B-Instruct";
54
55 /// Pre-quantized Qwen 2.5 Coder 7B Instruct (GGUF Q4_K_M) — strong coding model with tool calling (~4.4 GB).
56 ///
57 /// Uses the `qwen2` GGUF architecture. Trained on 5.5T tokens of code/math data with
58 /// fill-in-the-middle and repo-level code understanding. Supports function/tool calling
59 /// via the Qwen2.5 chat template. Requires ~8 GB RAM.
60 pub const BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF: &str =
61 "bartowski/Qwen2.5-Coder-7B-Instruct-GGUF";
62 /// The specific GGUF filename to download from the bartowski Coder 7B repo.
63 pub const QWEN25_CODER_7B_GGUF_FILE: &str = "Qwen2.5-Coder-7B-Instruct-Q4_K_M.gguf";
64 /// Base model repo used for the HF tokenizer on Android.
65 pub const QWEN25_CODER_7B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-Coder-7B-Instruct";
66
67 /// Pre-quantized Qwen 2.5 3B Instruct (GGUF Q4_K_M) — balanced option (~1.93 GB).
68 /// Ideal for macOS desktops and Android devices with sufficient RAM.
69 /// Not recommended as default on iOS: the 3B variant caused OOM on iPhone 16e (8 GB RAM)
70 /// because iOS gives apps only ~2-3 GB; use the 1.5B variant for iOS instead.
71 /// No in-situ quantization needed; loads directly at quantized size.
72 pub const BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF: &str = "bartowski/Qwen2.5-3B-Instruct-GGUF";
73 /// The specific GGUF filename to download from the bartowski 3B repo.
74 pub const QWEN25_3B_GGUF_FILE: &str = "Qwen2.5-3B-Instruct-Q4_K_M.gguf";
75 /// Base model repo used for the HF tokenizer (tokenizer.json + tokenizer_config.json).
76 pub const QWEN25_3B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-3B-Instruct";
77
78 /// Pre-quantized Qwen 3 4B Instruct (GGUF Q4_K_M) — full OpenAI-compatible tool calling (~2.7 GB).
79 ///
80 /// Qwen 3 uses an extended thinking mode (`<think>…</think>`) that significantly improves
81 /// reasoning and tool-use accuracy. Load with `max_tokens ≥ 4096` to avoid empty replies caused
82 /// by the model exhausting its token budget on thinking before producing a response.
83 ///
84 /// Recommended model for siGit Code (coding agent with tool calling on macOS/Linux/Windows).
85 pub const BARTOWSKI_QWEN3_4B_GGUF: &str = "bartowski/Qwen_Qwen3-4B-GGUF";
86 /// The specific GGUF filename to download from the bartowski Qwen 3 4B repo.
87 pub const QWEN3_4B_GGUF_FILE: &str = "Qwen_Qwen3-4B-Q4_K_M.gguf";
88
89 /// Pre-quantized Qwen 3 8B (GGUF Q4_K_M) — strong tool-calling model (~5 GB).
90 ///
91 /// Best balance of quality and memory for macOS with 24+ GB RAM.
92 /// Full tool calling and extended thinking mode support.
93 pub const BARTOWSKI_QWEN3_8B_GGUF: &str = "bartowski/Qwen_Qwen3-8B-GGUF";
94 pub const QWEN3_8B_GGUF_FILE: &str = "Qwen_Qwen3-8B-Q4_K_M.gguf";
95
96 /// Pre-quantized Qwen 3 14B (GGUF Q4_K_M) — strong reasoning and tool-calling model (~8.4 GB).
97 ///
98 /// Qwen 3 uses extended thinking mode (`<think>…</think>`) for improved reasoning.
99 /// Best all-around model for macOS with 16+ GB RAM. Full tool calling support.
100 pub const BARTOWSKI_QWEN3_14B_GGUF: &str = "bartowski/Qwen_Qwen3-14B-GGUF";
101 /// The specific GGUF filename.
102 pub const QWEN3_14B_GGUF_FILE: &str = "Qwen_Qwen3-14B-Q4_K_M.gguf";
103
104 /// Pre-quantized Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling model (~1.3 GB).
105 ///
106 /// Smallest Qwen 3 variant with tool calling support. Suitable for mobile devices
107 /// where the 4B model would be too large.
108 pub const BARTOWSKI_QWEN3_1_7B_GGUF: &str = "bartowski/Qwen_Qwen3-1.7B-GGUF";
109 /// The specific GGUF filename for the Qwen3 1.7B repo.
110 pub const QWEN3_1_7B_GGUF_FILE: &str = "Qwen_Qwen3-1.7B-Q4_K_M.gguf";
111
112 /// DeepSeek Coder v1 6.7B Instruct (GGUF Q4_K_M) — dedicated code generation model (~3.8 GB).
113 ///
114 /// Uses the `llama` GGUF architecture, so it loads through the existing
115 /// `quantized_llama.rs` path in mistral.rs without any code changes.
116 /// Strong coding performance. Requires ~8 GB RAM.
117 pub const THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF: &str =
118 "TheBloke/deepseek-coder-6.7B-instruct-GGUF";
119 /// The specific GGUF filename.
120 pub const DEEPSEEK_CODER_6_7B_GGUF_FILE: &str = "deepseek-coder-6.7b-instruct.Q4_K_M.gguf";
121 /// Base model repo used for the HF tokenizer on Android.
122 pub const DEEPSEEK_CODER_6_7B_TOK_MODEL_ID: &str = "deepseek-ai/deepseek-coder-6.7b-instruct";
123
124 /// All model IDs that the Onde inference engine supports.
125 /// Used by `list_local_hf_models` to filter the HuggingFace cache
126 /// to only show models that can actually be used for generation.
127 pub const SUPPORTED_MODELS: &[&str] = &[
128 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,
129 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,
130 BARTOWSKI_QWEN3_4B_GGUF,
131 BARTOWSKI_QWEN3_8B_GGUF,
132 BARTOWSKI_QWEN3_14B_GGUF,
133 BARTOWSKI_QWEN3_1_7B_GGUF,
134 BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF,
135 THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF,
136 ];
137
138 /// Rich metadata for a supported model, used by the frontend to display
139 /// unavailable models that can be downloaded.
140 pub struct SupportedModelInfo {
141 /// The full HuggingFace model identifier, e.g. "bartowski/Qwen2.5-1.5B-Instruct-GGUF".
142 pub id: &'static str,
143 /// Human-friendly display name for the model.
144 pub name: &'static str,
145 /// Organisation or publisher display name.
146 pub org: &'static str,
147 /// Short description of the model's purpose / capabilities.
148 pub description: &'static str,
149 /// Approximate total size in bytes when fully downloaded.
150 /// Used by the progress monitor to estimate download percentage.
151 ///
152 /// These values are computed by summing every file in the model's
153 /// HuggingFace repository (via the `/api/models/{id}?blobs=true`
154 /// endpoint's `siblings[].size` fields).
155 pub expected_size_bytes: u64,
156 }
157
158 /// Complete list of supported models with display metadata.
159 ///
160 /// When adding a new model, add its constant ID to [`SUPPORTED_MODELS`] **and**
161 /// a corresponding entry here so the frontend can show it in the model list UI.
162 pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[
163 SupportedModelInfo {
164 id: BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,
165 name: "Qwen 2.5 1.5B (GGUF)",
166 org: "Qwen / Alibaba",
167 description: "Lightest pre-quantized chat model — ideal for iOS & Android (~941 MB). \
168 Fits comfortably within iOS memory limits (iPhone 16e, 8 GB RAM).",
169 // Qwen2.5-1.5B-Instruct-Q4_K_M.gguf from bartowski repo.
170 // Exact file size from HuggingFace API siblings[].size.
171 expected_size_bytes: 986_048_768,
172 },
173 SupportedModelInfo {
174 id: BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,
175 name: "Qwen 2.5 3B (GGUF)",
176 org: "Qwen / Alibaba",
177 description:
178 "Pre-quantized chat model for macOS & Android — balanced quality and size (~1.93 GB). \
179 Not recommended as default on iOS due to memory constraints.",
180 // Qwen2.5-3B-Instruct-Q4_K_M.gguf from bartowski repo.
181 // Exact file size from HuggingFace API siblings[].size.
182 expected_size_bytes: 1_929_903_264,
183 },
184 SupportedModelInfo {
185 id: BARTOWSKI_QWEN3_4B_GGUF,
186 name: "Qwen 3 4B (GGUF)",
187 org: "Qwen / Alibaba",
188 description: "Full tool-calling support with extended reasoning mode (~2.7 GB). \
189 Recommended for siGit Code on macOS, Linux, and Windows.",
190 // Qwen_Qwen3-4B-Q4_K_M.gguf from bartowski repo.
191 expected_size_bytes: 2_596_306_912,
192 },
193 SupportedModelInfo {
194 id: BARTOWSKI_QWEN3_8B_GGUF,
195 name: "Qwen 3 8B (GGUF)",
196 org: "Qwen / Alibaba",
197 description: "Strong tool-calling model with extended thinking (~5 GB). \
198 Best balance of quality and memory for macOS with 24+ GB RAM.",
199 expected_size_bytes: 5_131_567_104,
200 },
201 SupportedModelInfo {
202 id: BARTOWSKI_QWEN3_14B_GGUF,
203 name: "Qwen 3 14B (GGUF)",
204 org: "Qwen / Alibaba",
205 description: "Strong reasoning and tool-calling model with extended thinking (~8.4 GB). \
206 Best all-around model for macOS with 16+ GB RAM.",
207 // Qwen_Qwen3-14B-Q4_K_M.gguf from bartowski repo.
208 // Exact file size from HuggingFace API siblings[].size.
209 expected_size_bytes: 9_001_753_632,
210 },
211 SupportedModelInfo {
212 id: BARTOWSKI_QWEN3_1_7B_GGUF,
213 name: "Qwen 3 1.7B (GGUF)",
214 org: "Qwen / Alibaba",
215 description: "Lightweight tool-calling model for mobile (~1.3 GB). \
216 Smallest Qwen 3 variant with tool calling support.",
217 expected_size_bytes: 1_282_439_584,
218 },
219 SupportedModelInfo {
220 id: BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF,
221 name: "Qwen 2.5 Coder 7B (GGUF)",
222 org: "Qwen / Alibaba",
223 description: "Strong coding model with tool calling support (~4.4 GB). \
224 Trained on 5.5T code tokens. Requires 8+ GB RAM.",
225 expected_size_bytes: 4_683_074_336,
226 },
227 SupportedModelInfo {
228 id: THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF,
229 name: "DeepSeek Coder 6.7B (GGUF)",
230 org: "DeepSeek AI",
231 description: "Strong code generation model using the llama architecture (~3.8 GB). \
232 Requires 8+ GB RAM. Not recommended for mobile devices.",
233 expected_size_bytes: 4_083_015_904,
234 },
235 ];
236
237 /// Return the explicit tokenizer model ID required on Android for `hf_repo_id`.
238 ///
239 /// The candle GGUF backend cannot parse the tokenizer embedded inside GGUF
240 /// files; an explicit `tok_model_id` triggers a separate tokenizer download
241 /// from the base model repo. Returns `None` on iOS and macOS where the
242 /// embedded tokenizer is used automatically.
243 pub fn tok_model_id_for_repo(hf_repo_id: &str) -> Option<&'static str> {
244 match hf_repo_id {
245 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF => Some(QWEN25_1_5B_TOK_MODEL_ID),
246 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF => Some(QWEN25_3B_TOK_MODEL_ID),
247 BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF => Some(QWEN25_CODER_1_5B_TOK_MODEL_ID),
248 BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF => Some(QWEN25_CODER_3B_TOK_MODEL_ID),
249 BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF => Some(QWEN25_CODER_7B_TOK_MODEL_ID),
250 THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF => Some(DEEPSEEK_CODER_6_7B_TOK_MODEL_ID),
251 _ => None,
252 }
253 }