@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
development
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
rs 3,100 lines 115 KB
Raw
1 // Copyright 2026 Splitfire AB (Onde Inference). All rights reserved.
2 // SPDX-License-Identifier: MIT OR Apache-2.0
3
4 //! On-device LLM chat inference engine powered by [mistral.rs](https://github.com/EricLBuehler/mistral.rs).
5 //!
6 //! `ChatEngine` provides a high-level, framework-agnostic API for:
7 //!
8 //! - Loading GGUF-quantized models (the primary format for mobile/desktop)
9 //! - Multi-turn chat with conversation history management
10 //! - Both blocking (non-streaming) and streaming inference
11 //! - Model lifecycle management (load / unload / status)
12 //!
13 //! # Architecture
14 //!
15 //! ```text
16 //! ┌──────────────────────────────────────────┐
17 //! │ App / UniFFI binding / test │
18 //! └──────────────┬───────────────────────────┘
19 //! │ (framework-agnostic API)
20 //! ▼
21 //! ┌──────────────────────────────────────────┐
22 //! │ ChatEngine │
23 //! │ ┌────────────────────────────────────┐ │
24 //! │ │ Mutex<Option<LoadedModel>> │ │
25 //! │ │ · Arc<mistralrs::Model> │ │
26 //! │ │ · config (GgufModelConfig) │ │
27 //! │ │ · history (Vec<ChatMessage>) │ │
28 //! │ │ · sampling (SamplingConfig) │ │
29 //! │ └────────────────────────────────────┘ │
30 //! └──────────────┬───────────────────────────┘
31 //! │ (delegates to)
32 //! ▼
33 //! ┌──────────────────────────────────────────┐
34 //! │ mistralrs::Model │
35 //! │ (wraps Arc<MistralRs> engine thread) │
36 //! └──────────────────────────────────────────┘
37 //! ```
38 //!
39 //! # Example
40 //!
41 //! ```rust,ignore
42 //! use onde::inference::engine::ChatEngine;
43 //! use onde::inference::types::*;
44 //!
45 //! let engine = ChatEngine::new();
46 //!
47 //! let config = GgufModelConfig {
48 //! model_id: "bartowski/Qwen2.5-1.5B-Instruct-GGUF".into(),
49 //! files: vec!["Qwen2.5-1.5B-Instruct-Q4_K_M.gguf".into()],
50 //! tok_model_id: None,
51 //! display_name: "Qwen 2.5 1.5B".into(),
52 //! approx_memory: "~941 MB".into(),
53 //! chat_template: None,
54 //! };
55 //!
56 //! engine.load_gguf_model(config, None).await?;
57 //! engine.set_system_prompt("You are a helpful assistant.");
58 //!
59 //! let result = engine.send_message("Hello!").await?;
60 //! println!("{}", result.text);
61 //! ```
62
63 #[cfg(any(
64 target_os = "macos",
65 target_os = "ios",
66 target_os = "tvos",
67 target_os = "visionos",
68 target_os = "watchos",
69 target_os = "windows",
70 target_os = "linux",
71 target_os = "android"
72 ))]
73 use std::sync::Arc;
74
75 #[cfg(any(
76 target_os = "macos",
77 target_os = "ios",
78 target_os = "tvos",
79 target_os = "visionos",
80 target_os = "watchos",
81 target_os = "windows",
82 target_os = "linux",
83 target_os = "android"
84 ))]
85 use tokio::sync::Mutex;
86
87 use super::types::*;
88
89 // ── Platform-gated mistralrs imports ─────────────────────────────────────────
90
91 #[cfg(any(
92 target_os = "macos",
93 target_os = "ios",
94 target_os = "tvos",
95 target_os = "visionos",
96 target_os = "watchos",
97 target_os = "windows",
98 target_os = "linux",
99 target_os = "android"
100 ))]
101 use mistralrs::{
102 CalledFunction, Function, GgufModelBuilder, Model, RequestBuilder, TextMessageRole,
103 Tool as MistralTool, ToolCallResponse, ToolCallType, ToolChoice, ToolType,
104 UqffTextModelBuilder,
105 };
106
107 #[cfg(any(
108 target_os = "macos",
109 target_os = "ios",
110 target_os = "tvos",
111 target_os = "visionos",
112 target_os = "watchos",
113 target_os = "windows",
114 target_os = "linux",
115 target_os = "android"
116 ))]
117 use std::collections::{HashMap, HashSet};
118 #[cfg(any(
119 target_os = "macos",
120 target_os = "ios",
121 target_os = "tvos",
122 target_os = "visionos",
123 target_os = "watchos",
124 target_os = "windows",
125 target_os = "linux",
126 target_os = "android"
127 ))]
128 use std::path::PathBuf;
129
130 // ISQ types are only used by load_isq_model, which is macOS-only.
131 #[cfg(target_os = "macos")]
132 use mistralrs::{IsqBits, TextModelBuilder};
133
134 // ── Internals ────────────────────────────────────────────────────────────────
135
136 /// How a model was loaded — carries the data needed for status reporting.
137 #[cfg(any(
138 target_os = "macos",
139 target_os = "ios",
140 target_os = "tvos",
141 target_os = "visionos",
142 target_os = "watchos",
143 target_os = "windows",
144 target_os = "linux",
145 target_os = "android"
146 ))]
147 enum LoadedModelConfig {
148 Gguf(GgufModelConfig),
149 Uqff(UqffModelConfig),
150 #[cfg(target_os = "macos")]
151 Isq(IsqModelConfig),
152 }
153
154 #[cfg(any(
155 target_os = "macos",
156 target_os = "ios",
157 target_os = "tvos",
158 target_os = "visionos",
159 target_os = "watchos",
160 target_os = "windows",
161 target_os = "linux",
162 target_os = "android"
163 ))]
164 impl LoadedModelConfig {
165 fn display_name(&self) -> &str {
166 match self {
167 LoadedModelConfig::Gguf(c) => &c.display_name,
168 LoadedModelConfig::Uqff(c) => &c.display_name,
169 #[cfg(target_os = "macos")]
170 LoadedModelConfig::Isq(c) => &c.display_name,
171 }
172 }
173
174 fn approx_memory(&self) -> &str {
175 match self {
176 LoadedModelConfig::Gguf(c) => &c.approx_memory,
177 LoadedModelConfig::Uqff(c) => &c.approx_memory,
178 #[cfg(target_os = "macos")]
179 LoadedModelConfig::Isq(c) => &c.approx_memory,
180 }
181 }
182 }
183
184 /// State of a loaded model held inside the engine's mutex.
185 #[cfg(any(
186 target_os = "macos",
187 target_os = "ios",
188 target_os = "tvos",
189 target_os = "visionos",
190 target_os = "watchos",
191 target_os = "windows",
192 target_os = "linux",
193 target_os = "android"
194 ))]
195 struct LoadedModel {
196 /// The mistral.rs model handle. `Model` wraps `Arc<MistralRs>` so
197 /// cloning is a cheap pointer copy — safe to snapshot before inference
198 /// while releasing the mutex.
199 model: Arc<Model>,
200 /// Configuration used to load this model (kept for status reporting).
201 config: LoadedModelConfig,
202 /// Conversation history (system prompt is stored separately).
203 history: Vec<HistoryEntry>,
204 /// System prompt prepended to every request.
205 system_prompt: Option<String>,
206 /// Sampling parameters for generation.
207 sampling: SamplingConfig,
208 }
209
210 /// An entry in the conversation history (internal only).
211 /// Supports tool-related messages alongside regular text.
212 #[cfg(any(
213 target_os = "macos",
214 target_os = "ios",
215 target_os = "tvos",
216 target_os = "visionos",
217 target_os = "watchos",
218 target_os = "windows",
219 target_os = "linux",
220 target_os = "android"
221 ))]
222 #[derive(Debug, Clone)]
223 enum HistoryEntry {
224 /// A regular text message (user, assistant, system).
225 Text(ChatMessage),
226 /// An assistant response that includes tool calls.
227 AssistantToolCall {
228 content: String,
229 tool_calls: Vec<ToolCallRequest>,
230 },
231 /// A tool execution result.
232 ToolResult {
233 tool_call_id: String,
234 content: String,
235 },
236 }
237
238 // ═════════════════════════════════════════════════════════════════════════════
239 // ChatEngine — the public API
240 // ═════════════════════════════════════════════════════════════════════════════
241
242 /// A reusable on-device LLM chat inference engine.
243 ///
244 /// Thread-safe (`Send + Sync`) — safe to store in a `once_cell::sync::Lazy`,
245 /// `tokio::sync::OnceCell`, or shared application state.
246 ///
247 /// All mutating operations acquire an internal `tokio::sync::Mutex`. The
248 /// mutex is released *before* the actual (potentially slow) model inference
249 /// runs, so other tasks can still query status or history while generation
250 /// is in progress.
251 #[cfg(any(
252 target_os = "macos",
253 target_os = "ios",
254 target_os = "tvos",
255 target_os = "visionos",
256 target_os = "watchos",
257 target_os = "windows",
258 target_os = "linux",
259 target_os = "android"
260 ))]
261 pub struct ChatEngine {
262 inner: Mutex<Option<LoadedModel>>,
263 pulse: std::sync::OnceLock<Option<crate::pulse::PulseClient>>,
264 /// The Onde app UUID this engine reports telemetry for. Threaded into every
265 /// pulse event so the document store associates rows with the right app.
266 /// `None` for open-ended SDK consumers (UniFFI `OndeChatEngine`).
267 onde_app_id: Option<String>,
268 /// Device-declared ISO 3166-1 alpha-2 country for the public Pulse globe
269 /// (spec 0008). `None` writes no geography.
270 pulse_country: Option<String>,
271 }
272
273 #[cfg(any(
274 target_os = "macos",
275 target_os = "ios",
276 target_os = "tvos",
277 target_os = "visionos",
278 target_os = "watchos",
279 target_os = "windows",
280 target_os = "linux",
281 target_os = "android"
282 ))]
283 impl ChatEngine {
284 // ── Construction ─────────────────────────────────────────────────────
285
286 /// Create a new engine with no model loaded and no Onde app association.
287 /// Pulse telemetry is still enabled, but events carry no `onde_app_id`.
288 pub fn new() -> Self {
289 Self::with_app_id(None)
290 }
291
292 /// Create a new engine that reports telemetry for a specific Onde app.
293 /// The app UUID is threaded into every pulse event so the document store
294 /// associates rows with the right app. Consumer apps that know their ID
295 /// call `ChatEngine::with_app_id(Some(smbcloud::onde_app_id()))`; the
296 /// UniFFI `OndeChatEngine` keeps calling `new()` (→ `None`).
297 pub fn with_app_id(onde_app_id: Option<String>) -> Self {
298 crate::install_panic_hook_once();
299 Self {
300 inner: Mutex::new(None),
301 pulse: std::sync::OnceLock::new(),
302 onde_app_id,
303 pulse_country: None,
304 }
305 }
306
307 /// Declare the device's country for Pulse geography. Pass the ISO 3166-1
308 /// alpha-2 region from the OS locale or time zone (for example `"SE"`),
309 /// never a full locale string or anything derived from the network.
310 ///
311 /// Only the country code is sent, as one observation per edge per UTC
312 /// day, and only when dual-write and an Onde app id are both enabled.
313 /// UniFFI hosts can set `ONDE_PULSE_COUNTRY` instead.
314 pub fn with_pulse_country(mut self, country_code: Option<String>) -> Self {
315 self.pulse_country = country_code;
316 self
317 }
318
319 /// Lazily initialise the pulse telemetry client.
320 ///
321 /// The underlying GresIQ SDK creates a `reqwest::Client`, which needs
322 /// a Tokio runtime. `ChatEngine::new()` is called from a sync UniFFI
323 /// constructor where no runtime exists yet, so we defer creation to the
324 /// first async call site (e.g. `load_gguf_model` or `send_message`).
325 fn pulse(&self) -> Option<&crate::pulse::PulseClient> {
326 self.pulse
327 .get_or_init(|| {
328 let environment = Self::pulse_environment();
329 let edge_id =
330 std::env::var("ONDE_EDGE_ID").unwrap_or_else(|_| "onde-unknown".to_string());
331 // Prefer the constructor-supplied app id; fall back to the
332 // ONDE_APP_ID env var so UniFFI consumers can opt in without a
333 // new constructor signature.
334 let onde_app_id = self.onde_app_id.clone().or_else(|| {
335 std::env::var("ONDE_APP_ID")
336 .ok()
337 .filter(|id| !id.is_empty())
338 });
339 let country_code = self.pulse_country.clone().or_else(|| {
340 std::env::var("ONDE_PULSE_COUNTRY")
341 .ok()
342 .filter(|code| !code.is_empty())
343 });
344 let client =
345 crate::pulse::PulseClient::new(environment, edge_id, onde_app_id, country_code);
346
347 match &client {
348 Some(_) => log::info!(
349 "ChatEngine: pulse telemetry enabled (environment={environment})"
350 ),
351 None if crate::pulse::PulseClient::disabled_by_env() => {
352 log::info!("ChatEngine: pulse telemetry disabled by ONDE_DISABLE_PULSE")
353 }
354 None => log::info!(
355 "ChatEngine: pulse telemetry disabled \
356 (GRESIQ_API_KEY / GRESIQ_API_SECRET not embedded at SDK build time)"
357 ),
358 }
359
360 client
361 })
362 .as_ref()
363 }
364
365 /// Resolve the pulse environment from the `GRESIQ_ENVIRONMENT` env var.
366 /// Defaults to `Production` when the var is absent or unrecognised.
367 fn pulse_environment() -> smbcloud_gresiq_sdk::Environment {
368 match std::env::var("GRESIQ_ENVIRONMENT").as_deref() {
369 Ok("dev") => smbcloud_gresiq_sdk::Environment::Dev,
370 _ => smbcloud_gresiq_sdk::Environment::Production,
371 }
372 }
373
374 // ── Model lifecycle ──────────────────────────────────────────────────
375
376 /// Load a GGUF model into the engine.
377 ///
378 /// If a model is already loaded it will be unloaded first (the previous
379 /// `Model` is dropped, which terminates its engine thread).
380 ///
381 /// # Arguments
382 ///
383 /// * `config` — Which model to load (repo ID, filename, etc.).
384 /// * `system_prompt` — Optional system prompt to prepend to every request.
385 /// * `sampling` — Sampling parameters; pass `None` for platform-aware
386 /// defaults (mobile gets [`SamplingConfig::mobile`],
387 /// desktop gets [`SamplingConfig::default`]).
388 ///
389 /// # Errors
390 ///
391 /// Returns [`InferenceError::ModelBuild`] if the model fails to download
392 /// or load.
393 pub async fn load_gguf_model(
394 &self,
395 config: GgufModelConfig,
396 system_prompt: Option<String>,
397 sampling: Option<SamplingConfig>,
398 ) -> Result<std::time::Duration, InferenceError> {
399 use log::info;
400 use std::time::Instant;
401
402 info!(
403 "ChatEngine: loading GGUF model {} (files: {:?})",
404 config.model_id, config.files
405 );
406
407 // ── Sandboxed platforms: seed GLOBAL_HF_CACHE ────────────────────
408 //
409 // On sandboxed platforms the default `~/.cache/huggingface/hub` path
410 // is either inaccessible or non-existent:
411 // - Android: `dirs::home_dir()` returns `None` → `Cache::default()` panics.
412 // - iOS/tvOS: `~/.cache` is outside the app container → os error 1.
413 //
414 // The `get_paths_gguf!` macro in mistralrs falls back to
415 // `Cache::default()` when `GLOBAL_HF_CACHE` (a OnceLock) is empty.
416 //
417 // `HF_HOME` must be set by the host app (via `configure_cache_dir`
418 // or `download_model(app_data_dir:)`) before any model load.
419 // `get_or_init` is a no-op if already seeded — safe to call repeatedly.
420 #[cfg(any(
421 target_os = "android",
422 target_os = "ios",
423 target_os = "tvos",
424 target_os = "visionos",
425 target_os = "watchos"
426 ))]
427 {
428 let hf_home = std::env::var("HF_HOME")
429 .map(std::path::PathBuf::from)
430 .map_err(|_| InferenceError::ModelBuild {
431 reason: "HF_HOME is not set — cannot initialise HF cache. \
432 On iOS/tvOS/Android, call configure_cache_dir() or \
433 download_model(app_data_dir:) before load_gguf_model()."
434 .to_string(),
435 })?;
436 let hf_hub_cache = hf_home.join("hub");
437 if let Err(e) = std::fs::create_dir_all(&hf_hub_cache) {
438 log::warn!(
439 "ChatEngine: could not create HF hub cache dir {}: {}",
440 hf_hub_cache.display(),
441 e
442 );
443 }
444 mistralrs_core::GLOBAL_HF_CACHE
445 .get_or_init(|| hf_hub::Cache::new(hf_hub_cache.clone()));
446 std::env::set_var("HF_HUB_CACHE", &hf_hub_cache);
447 log::debug!(
448 "ChatEngine: GLOBAL_HF_CACHE seeded at {}",
449 hf_hub_cache.display()
450 );
451 }
452
453 // Clean up stale HF cache artefacts before loading.
454 crate::hf_cache::clean_stale_lock_files(&config.model_id);
455 crate::hf_cache::repair_hf_cache_symlinks(&config.model_id);
456
457 // If the blob was downloaded but the process was killed before hf-hub
458 // created the snapshot entry, hf-hub will re-download. Create the
459 // missing snapshot hard links so the cache hit succeeds.
460 let file_refs: Vec<&str> = config.files.iter().map(|s| s.as_str()).collect();
461 crate::hf_cache::ensure_snapshot_entries(&config.model_id, &file_refs);
462
463 let start = Instant::now();
464
465 let mut builder = GgufModelBuilder::new(&config.model_id, config.files.clone())
466 .with_token_source(super::token::hf_token_source())
467 .with_logging();
468
469 // tvOS: Apple TV 4K has only 4 GB RAM. Reduce memory footprint by
470 // limiting concurrent sequences, disabling the prefix cache,
471 // disabling the KV cache, and capping context length at 512 tokens
472 // so the auto device mapper allocates far less memory.
473 #[cfg(target_os = "tvos")]
474 {
475 use mistralrs::DeviceMapSetting;
476 use mistralrs_core::AutoDeviceMapParams;
477
478 builder = builder
479 .with_max_num_seqs(1)
480 .with_prefix_cache_n(None)
481 .with_no_kv_cache()
482 .with_device_mapping(DeviceMapSetting::Auto(AutoDeviceMapParams::Text {
483 max_seq_len: 512,
484 max_batch_size: 1,
485 }));
486 }
487
488 // On Android the GGUF embedded tokenizer is not supported by the
489 // candle backend — an explicit tok_model_id is required.
490 if let Some(ref tok_id) = config.tok_model_id {
491 builder = builder.with_tok_model_id(tok_id);
492 }
493
494 // Some older GGUF files (e.g. TheBloke) do not embed a chat template.
495 if let Some(ref template) = config.chat_template {
496 builder = builder.with_chat_template(template);
497 }
498
499 let model = builder
500 .build()
501 .await
502 .map_err(|e| InferenceError::ModelBuild {
503 reason: format!("Failed to build {} model: {}", config.display_name, e),
504 })?;
505
506 let elapsed = start.elapsed();
507
508 let sampling = sampling.unwrap_or_else(|| {
509 if config.uses_extended_thinking() {
510 SamplingConfig::reasoning()
511 } else if cfg!(any(
512 target_os = "ios",
513 target_os = "tvos",
514 target_os = "visionos",
515 target_os = "watchos",
516 target_os = "android"
517 )) {
518 SamplingConfig::mobile()
519 } else {
520 SamplingConfig::default()
521 }
522 });
523
524 info!(
525 "ChatEngine: model {} loaded in {} (sampling: temp={:?}, max_tokens={:?})",
526 config.display_name,
527 format_duration(elapsed),
528 sampling.temperature,
529 sampling.max_tokens,
530 );
531
532 // Capture before config is moved into LoadedModel.
533 let pulse_model_id = config.model_id.clone();
534 let pulse_model_name = config.display_name.clone();
535
536 // Swap the new model in and take the old one out — all under the lock
537 // so the engine is never seen as None by a concurrent prompt. Then
538 // drop the old weights *outside* the lock so we don't block inference
539 // while the allocator frees potentially several GB of tensors.
540 let old_model = {
541 let mut guard = self.inner.lock().await;
542 guard.replace(LoadedModel {
543 model: Arc::new(model),
544 config: LoadedModelConfig::Gguf(config),
545 history: Vec::new(),
546 system_prompt,
547 sampling,
548 })
549 };
550 drop(old_model); // free old weights outside the lock
551
552 if let Some(pulse) = self.pulse() {
553 pulse
554 .record_model_loaded(pulse_model_id, pulse_model_name, elapsed.as_millis() as u64)
555 .await;
556 }
557
558 Ok(elapsed)
559 }
560
561 /// Load a UQFF model into the engine.
562 ///
563 /// UQFF stores pre-quantised weights and loads directly
564 /// without runtime conversion. `config.model_id` is still required for
565 /// tokenizer and base model config resolution; `config.files` contains the
566 /// first shard or explicit UQFF shard filenames.
567 ///
568 /// If a model is already loaded it will be unloaded first.
569 ///
570 /// # Errors
571 ///
572 /// Returns [`InferenceError::ModelBuild`] if the model fails to download
573 /// or load.
574 pub async fn load_uqff_model(
575 &self,
576 config: UqffModelConfig,
577 system_prompt: Option<String>,
578 sampling: Option<SamplingConfig>,
579 ) -> Result<std::time::Duration, InferenceError> {
580 use log::info;
581 use std::time::Instant;
582
583 if config.files.is_empty() {
584 return Err(InferenceError::ModelBuild {
585 reason: "UQFF model config must include at least one UQFF file or shorthand."
586 .to_string(),
587 });
588 }
589
590 info!(
591 "ChatEngine: loading UQFF model {} (files: {:?})",
592 config.model_id, config.files
593 );
594
595 #[cfg(any(
596 target_os = "android",
597 target_os = "ios",
598 target_os = "tvos",
599 target_os = "visionos",
600 target_os = "watchos"
601 ))]
602 {
603 let hf_home = std::env::var("HF_HOME")
604 .map(std::path::PathBuf::from)
605 .map_err(|_| InferenceError::ModelBuild {
606 reason: "HF_HOME is not set — cannot initialise HF cache. \
607 On iOS/tvOS/Android, call configure_cache_dir() or \
608 download_model(app_data_dir:) before load_uqff_model()."
609 .to_string(),
610 })?;
611 let hf_hub_cache = hf_home.join("hub");
612 if let Err(e) = std::fs::create_dir_all(&hf_hub_cache) {
613 log::warn!(
614 "ChatEngine: could not create HF hub cache dir {}: {}",
615 hf_hub_cache.display(),
616 e
617 );
618 }
619 mistralrs_core::GLOBAL_HF_CACHE
620 .get_or_init(|| hf_hub::Cache::new(hf_hub_cache.clone()));
621 std::env::set_var("HF_HUB_CACHE", &hf_hub_cache);
622 log::debug!(
623 "ChatEngine: GLOBAL_HF_CACHE seeded at {}",
624 hf_hub_cache.display()
625 );
626 }
627
628 crate::hf_cache::clean_stale_lock_files(&config.model_id);
629 crate::hf_cache::repair_hf_cache_symlinks(&config.model_id);
630
631 let start = Instant::now();
632 let uqff_files: Vec<PathBuf> = config.files.iter().map(PathBuf::from).collect();
633
634 let mut builder = UqffTextModelBuilder::new(&config.model_id, uqff_files)
635 .into_inner()
636 .with_token_source(super::token::hf_token_source())
637 .with_logging();
638
639 #[cfg(target_os = "tvos")]
640 {
641 use mistralrs::DeviceMapSetting;
642 use mistralrs_core::AutoDeviceMapParams;
643
644 builder = builder
645 .with_max_num_seqs(1)
646 .with_prefix_cache_n(None)
647 .with_no_kv_cache()
648 .with_device_mapping(DeviceMapSetting::Auto(AutoDeviceMapParams::Text {
649 max_seq_len: 512,
650 max_batch_size: 1,
651 }));
652 }
653
654 if let Some(ref template) = config.chat_template {
655 builder = builder.with_chat_template(template);
656 }
657
658 let model = builder
659 .build()
660 .await
661 .map_err(|e| InferenceError::ModelBuild {
662 reason: format!("Failed to build UQFF model {}: {}", config.display_name, e),
663 })?;
664
665 let elapsed = start.elapsed();
666
667 let sampling = sampling.unwrap_or_else(|| {
668 if cfg!(any(
669 target_os = "ios",
670 target_os = "tvos",
671 target_os = "visionos",
672 target_os = "watchos",
673 target_os = "android"
674 )) {
675 SamplingConfig::mobile()
676 } else {
677 SamplingConfig::default()
678 }
679 });
680
681 info!(
682 "ChatEngine: UQFF model {} loaded in {} (sampling: temp={:?}, max_tokens={:?})",
683 config.display_name,
684 format_duration(elapsed),
685 sampling.temperature,
686 sampling.max_tokens,
687 );
688
689 let pulse_model_id = config.model_id.clone();
690 let pulse_model_name = config.display_name.clone();
691
692 let old_model = {
693 let mut guard = self.inner.lock().await;
694 guard.replace(LoadedModel {
695 model: Arc::new(model),
696 config: LoadedModelConfig::Uqff(config),
697 history: Vec::new(),
698 system_prompt,
699 sampling,
700 })
701 };
702 drop(old_model);
703
704 if let Some(pulse) = self.pulse() {
705 pulse
706 .record_model_loaded(pulse_model_id, pulse_model_name, elapsed.as_millis() as u64)
707 .await;
708 }
709
710 Ok(elapsed)
711 }
712
713 /// Fetch the model assigned to this Onde app from the SDK model-config
714 /// endpoint and load it.
715 ///
716 /// Authenticates using the app's own `onde_app_id` + `onde_app_secret`
717 /// (the SDK credentials shown in the ondeinference.com dashboard).
718 /// No end-user JWT is required — model assignment is an operator-level
719 /// configuration that is independent of which user is currently signed in.
720 ///
721 /// **Fallback behaviour:**
722 /// - HTTP 404 (no model assigned) → loads [`GgufModelConfig::platform_default()`].
723 /// - Missing `hf_repo_id` or `gguf_file` in response → loads platform default.
724 /// - Any HTTP or network error → returns [`InferenceError::ModelBuild`] so
725 /// the caller can apply its own fallback.
726 pub async fn load_assigned_model(
727 &self,
728 environment: smbcloud_gresiq_sdk::Environment,
729 onde_app_id: &str,
730 onde_app_secret: &str,
731 system_prompt: Option<String>,
732 sampling: Option<SamplingConfig>,
733 ) -> Result<std::time::Duration, InferenceError> {
734 use log::{info, warn};
735
736 #[derive(serde::Deserialize)]
737 struct ModelConfigResponse {
738 hf_repo_id: Option<String>,
739 gguf_file: Option<String>,
740 name: Option<String>,
741 approx_size_bytes: Option<i64>,
742 }
743
744 let url = format!(
745 "{}://{}/v1/client/onde_sdk/model_config?app_id={}&app_secret={}",
746 environment.api_protocol(),
747 environment.api_host(),
748 onde_app_id,
749 onde_app_secret,
750 );
751
752 let response = reqwest::Client::new()
753 .get(&url)
754 .header("Content-Type", "application/json")
755 .send()
756 .await
757 .map_err(|e| InferenceError::ModelBuild {
758 reason: format!("SDK model_config request failed: {e}"),
759 })?;
760
761 if response.status().as_u16() == 404 {
762 warn!(
763 "ChatEngine: no model assigned to Onde app {onde_app_id}; loading platform default."
764 );
765 return self
766 .load_gguf_model(GgufModelConfig::platform_default(), system_prompt, sampling)
767 .await;
768 }
769
770 if !response.status().is_success() {
771 return Err(InferenceError::ModelBuild {
772 reason: format!(
773 "SDK model_config returned HTTP {}",
774 response.status().as_u16()
775 ),
776 });
777 }
778
779 let resp: ModelConfigResponse =
780 response
781 .json()
782 .await
783 .map_err(|e| InferenceError::ModelBuild {
784 reason: format!("Failed to parse model_config response: {e}"),
785 })?;
786
787 let hf_repo_id = resp.hf_repo_id.as_deref().unwrap_or_default();
788 let gguf_file = resp.gguf_file.as_deref().unwrap_or_default();
789
790 if hf_repo_id.is_empty() || gguf_file.is_empty() {
791 warn!(
792 "ChatEngine: model_config response missing hf_repo_id or gguf_file; loading platform default."
793 );
794 return self
795 .load_gguf_model(GgufModelConfig::platform_default(), system_prompt, sampling)
796 .await;
797 }
798
799 #[cfg(target_os = "android")]
800 let tok_model_id = super::models::tok_model_id_for_repo(hf_repo_id).map(|s| s.to_string());
801 #[cfg(not(target_os = "android"))]
802 let tok_model_id: Option<String> = None;
803
804 let approx_memory = resp
805 .approx_size_bytes
806 .map(|b| {
807 let gb = b as f64 / 1_073_741_824.0;
808 if gb >= 1.0 {
809 format!("~{:.2} GB", gb)
810 } else {
811 format!("~{} MB", b / 1_048_576)
812 }
813 })
814 .unwrap_or_else(|| "—".to_string());
815
816 info!(
817 "ChatEngine: resolved SDK model assignment → {} / {} ({})",
818 hf_repo_id, gguf_file, approx_memory
819 );
820
821 let config = GgufModelConfig {
822 model_id: hf_repo_id.to_string(),
823 files: vec![gguf_file.to_string()],
824 tok_model_id,
825 display_name: resp.name.unwrap_or_else(|| hf_repo_id.to_string()),
826 approx_memory,
827 chat_template: None,
828 };
829
830 self.load_gguf_model(config, system_prompt, sampling).await
831 }
832
833 /// Load an ISQ (in-situ quantised) model into the engine.
834 ///
835 /// Unlike [`load_gguf_model`], this downloads the full-precision safetensors
836 /// from HuggingFace and quantises the weights in-situ on Metal (macOS only).
837 /// The quantisation happens once at load time; subsequent inference uses the
838 /// compressed weights.
839 ///
840 /// # macOS only
841 ///
842 /// ISQ with Metal requires the `metal` feature on mistral.rs and is therefore
843 /// restricted to macOS at the Rust level. Other platforms should continue to
844 /// use [`load_gguf_model`] with pre-quantised GGUF files.
845 ///
846 /// # Arguments
847 ///
848 /// * `config` — Which model to load and with how many ISQ bits.
849 /// * `system_prompt` — Optional system prompt prepended to every request.
850 /// * `sampling` — Sampling parameters; `None` uses [`SamplingConfig::default`].
851 ///
852 /// # Errors
853 ///
854 /// Returns [`InferenceError::ModelBuild`] if the model fails to download
855 /// or quantise.
856 #[cfg(target_os = "macos")]
857 pub async fn load_isq_model(
858 &self,
859 config: IsqModelConfig,
860 system_prompt: Option<String>,
861 sampling: Option<SamplingConfig>,
862 ) -> Result<std::time::Duration, InferenceError> {
863 use log::info;
864 use std::time::Instant;
865
866 info!(
867 "ChatEngine: loading ISQ model {} (bits={})",
868 config.model_id, config.isq_bits
869 );
870
871 // Clean up stale HF cache artefacts before loading.
872 crate::hf_cache::clean_stale_lock_files(&config.model_id);
873 crate::hf_cache::repair_hf_cache_symlinks(&config.model_id);
874
875 let start = Instant::now();
876
877 // Choose ISQ bit width.
878 let isq_bits = match config.isq_bits {
879 8 => IsqBits::Eight,
880 _ => IsqBits::Four, // default to 4-bit
881 };
882
883 let model = TextModelBuilder::new(&config.model_id)
884 .with_token_source(super::token::hf_token_source())
885 .with_auto_isq(isq_bits)
886 .with_logging()
887 .build()
888 .await
889 .map_err(|e| InferenceError::ModelBuild {
890 reason: format!("Failed to build ISQ model {}: {}", config.display_name, e),
891 })?;
892
893 let elapsed = start.elapsed();
894
895 let sampling = sampling.unwrap_or_default();
896
897 info!(
898 "ChatEngine: ISQ model {} loaded in {} (sampling: temp={:?}, max_tokens={:?})",
899 config.display_name,
900 format_duration(elapsed),
901 sampling.temperature,
902 sampling.max_tokens,
903 );
904
905 let mut guard = self.inner.lock().await;
906 *guard = Some(LoadedModel {
907 model: Arc::new(model),
908 config: LoadedModelConfig::Isq(config),
909 history: Vec::new(),
910 system_prompt,
911 sampling,
912 });
913
914 Ok(elapsed)
915 }
916
917 /// Unload the current model, freeing all memory.
918 ///
919 /// Dropping the `Model` sends a `Terminate` message to the mistral.rs
920 /// engine thread, which tears down the KV cache, activations, and model
921 /// weights.
922 ///
923 /// Returns the display name of the model that was unloaded, or `None`
924 /// if no model was loaded.
925 pub async fn unload_model(&self) -> Option<String> {
926 let mut guard = self.inner.lock().await;
927 if let Some(loaded) = guard.take() {
928 let name = loaded.config.display_name().to_string();
929 log::info!("ChatEngine: unloading model {}", name);
930 // `loaded` is dropped here → Model → Arc<MistralRs> (last ref)
931 // → MistralRs::drop() → engine thread termination.
932 Some(name)
933 } else {
934 log::debug!("ChatEngine: unload_model called but no model was loaded.");
935 None
936 }
937 }
938
939 /// Check whether a model is currently loaded.
940 pub async fn is_loaded(&self) -> bool {
941 self.inner.lock().await.is_some()
942 }
943
944 /// Get a snapshot of the engine's current state.
945 pub async fn info(&self) -> EngineInfo {
946 let guard = self.inner.lock().await;
947 match guard.as_ref() {
948 Some(loaded) => EngineInfo {
949 status: EngineStatus::Ready,
950 model_name: Some(loaded.config.display_name().to_string()),
951 approx_memory: Some(loaded.config.approx_memory().to_string()),
952 history_length: loaded.history.len() as u64,
953 },
954 None => EngineInfo {
955 status: EngineStatus::Unloaded,
956 model_name: None,
957 approx_memory: None,
958 history_length: 0u64,
959 },
960 }
961 }
962
963 // ── System prompt ────────────────────────────────────────────────────
964
965 /// Set or replace the system prompt.
966 ///
967 /// The system prompt is prepended to every inference request (it is NOT
968 /// stored in the conversation history).
969 pub async fn set_system_prompt(&self, prompt: impl Into<String>) {
970 if let Some(loaded) = self.inner.lock().await.as_mut() {
971 loaded.system_prompt = Some(prompt.into());
972 }
973 }
974
975 /// Clear the system prompt.
976 pub async fn clear_system_prompt(&self) {
977 if let Some(loaded) = self.inner.lock().await.as_mut() {
978 loaded.system_prompt = None;
979 }
980 }
981
982 // ── Sampling configuration ───────────────────────────────────────────
983
984 /// Replace the sampling configuration.
985 pub async fn set_sampling(&self, sampling: SamplingConfig) {
986 if let Some(loaded) = self.inner.lock().await.as_mut() {
987 loaded.sampling = sampling;
988 }
989 }
990
991 // ── Conversation history ─────────────────────────────────────────────
992
993 /// Get a clone of the full conversation history.
994 ///
995 /// Tool-related history entries are converted back to plain
996 /// [`ChatMessage`] values (assistant tool-call entries become assistant
997 /// messages with just their text content; tool results are omitted).
998 pub async fn history(&self) -> Vec<ChatMessage> {
999 let guard = self.inner.lock().await;
1000 match guard.as_ref() {
1001 Some(loaded) => loaded
1002 .history
1003 .iter()
1004 .filter_map(|entry| match entry {
1005 HistoryEntry::Text(msg) => Some(msg.clone()),
1006 HistoryEntry::AssistantToolCall { content, .. } => {
1007 Some(ChatMessage::assistant(content))
1008 }
1009 HistoryEntry::ToolResult { .. } => None,
1010 })
1011 .collect(),
1012 None => Vec::new(),
1013 }
1014 }
1015
1016 /// Clear the conversation history but keep the model loaded.
1017 ///
1018 /// Returns the number of turns that were removed.
1019 pub async fn clear_history(&self) -> usize {
1020 let mut guard = self.inner.lock().await;
1021 match guard.as_mut() {
1022 Some(loaded) => {
1023 let count = loaded.history.len();
1024 loaded.history.clear();
1025 log::info!("ChatEngine: cleared {} history turns.", count);
1026 count
1027 }
1028 None => 0,
1029 }
1030 }
1031
1032 /// Append a message to history without running inference.
1033 ///
1034 /// Useful for restoring a saved conversation or injecting context.
1035 pub async fn push_history(&self, message: ChatMessage) {
1036 if let Some(loaded) = self.inner.lock().await.as_mut() {
1037 loaded.history.push(HistoryEntry::Text(message));
1038 }
1039 }
1040
1041 // ── Non-streaming inference ──────────────────────────────────────────
1042
1043 /// Send a user message and receive a complete assistant reply.
1044 ///
1045 /// The user message and assistant reply are automatically appended to
1046 /// the conversation history on success.
1047 ///
1048 /// # Errors
1049 ///
1050 /// - [`InferenceError::NoModelLoaded`] if no model is loaded.
1051 /// - [`InferenceError::Inference`] if the model fails to generate.
1052 pub async fn send_message(
1053 &self,
1054 user_message: impl Into<String>,
1055 ) -> Result<InferenceResult, InferenceError> {
1056 let user_message = user_message.into();
1057
1058 // ── 1. Snapshot model handle + build request, then release lock ──
1059 let (model, request, pulse_model_id) = {
1060 let guard = self.inner.lock().await;
1061 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1062 let request = self::build_request(loaded, &user_message);
1063 let pulse_model_id = match &loaded.config {
1064 LoadedModelConfig::Gguf(c) => c.model_id.clone(),
1065 LoadedModelConfig::Uqff(c) => c.model_id.clone(),
1066 #[cfg(target_os = "macos")]
1067 LoadedModelConfig::Isq(c) => c.model_id.clone(),
1068 };
1069 (loaded.model.clone(), request, pulse_model_id)
1070 }; // ← mutex released before inference
1071
1072 log::info!(
1073 "ChatEngine: inference START — message: \"{}\"",
1074 truncate_for_log(&user_message, 100)
1075 );
1076
1077 // ── 2. Run inference (potentially slow — mutex is NOT held) ──────
1078 let start = std::time::Instant::now();
1079 let response =
1080 model
1081 .send_chat_request(request)
1082 .await
1083 .map_err(|e| InferenceError::Inference {
1084 reason: e.to_string(),
1085 })?;
1086 let elapsed = start.elapsed();
1087
1088 let reply = response.choices[0]
1089 .message
1090 .content
1091 .as_ref()
1092 .map(|c| c.trim().to_string())
1093 .unwrap_or_else(|| "(empty response)".to_string());
1094
1095 let finish_reason = response.choices[0].finish_reason.clone();
1096
1097 log::info!(
1098 "ChatEngine: inference END — {} — reply: \"{}\"",
1099 format_duration(elapsed),
1100 truncate_for_log(&reply, 100)
1101 );
1102
1103 // ── 3. Persist turns to history (brief lock re-acquisition) ──────
1104 {
1105 let mut guard = self.inner.lock().await;
1106 if let Some(loaded) = guard.as_mut() {
1107 loaded
1108 .history
1109 .push(HistoryEntry::Text(ChatMessage::user(user_message)));
1110 loaded
1111 .history
1112 .push(HistoryEntry::Text(ChatMessage::assistant(reply.clone())));
1113 }
1114 }
1115
1116 if let Some(pulse) = self.pulse() {
1117 pulse.record_inference(
1118 pulse_model_id,
1119 crate::pulse::next_request_id(),
1120 elapsed.as_millis() as u64,
1121 // A blocking request returns the whole completion at once,
1122 // so there is no first-token moment to measure here.
1123 None,
1124 "success".to_string(),
1125 );
1126 }
1127
1128 Ok(InferenceResult {
1129 text: reply,
1130 duration_secs: elapsed.as_secs_f64(),
1131 duration_display: format_duration(elapsed),
1132 finish_reason,
1133 tool_calls: vec![],
1134 })
1135 }
1136
1137 /// Run inference on an explicit list of messages WITHOUT modifying the
1138 /// engine's internal history.
1139 ///
1140 /// This is useful for one-shot prompts (e.g. prompt enhancement) where
1141 /// you don't want side-effects on the main conversation.
1142 ///
1143 /// The system prompt set on the engine is still prepended.
1144 pub async fn generate(
1145 &self,
1146 messages: Vec<ChatMessage>,
1147 sampling: Option<SamplingConfig>,
1148 ) -> Result<InferenceResult, InferenceError> {
1149 let (model, request) = {
1150 let guard = self.inner.lock().await;
1151 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1152
1153 let sampling = sampling.as_ref().unwrap_or(&loaded.sampling);
1154 let mut req = RequestBuilder::new();
1155
1156 // Apply sampling parameters.
1157 req = apply_sampling(req, sampling);
1158
1159 // System prompt.
1160 if let Some(ref sp) = loaded.system_prompt {
1161 req = req.add_message(TextMessageRole::System, sp);
1162 }
1163
1164 // Provided messages.
1165 for msg in &messages {
1166 req = req.add_message(chat_role_to_mistral(&msg.role), &msg.content);
1167 }
1168
1169 (loaded.model.clone(), req)
1170 };
1171
1172 let start = std::time::Instant::now();
1173 let response =
1174 model
1175 .send_chat_request(request)
1176 .await
1177 .map_err(|e| InferenceError::Inference {
1178 reason: e.to_string(),
1179 })?;
1180 let elapsed = start.elapsed();
1181
1182 let reply = response.choices[0]
1183 .message
1184 .content
1185 .as_ref()
1186 .map(|c| c.trim().to_string())
1187 .unwrap_or_else(|| "(empty response)".to_string());
1188
1189 let finish_reason = response.choices[0].finish_reason.clone();
1190
1191 Ok(InferenceResult {
1192 text: reply,
1193 duration_secs: elapsed.as_secs_f64(),
1194 duration_display: format_duration(elapsed),
1195 finish_reason,
1196 tool_calls: vec![],
1197 })
1198 }
1199
1200 // ── Streaming inference ──────────────────────────────────────────────
1201
1202 /// Send a user message and receive an async stream of token chunks.
1203 ///
1204 /// The user message and full assembled reply are automatically appended
1205 /// to the conversation history once the stream finishes.
1206 ///
1207 /// # Returns
1208 ///
1209 /// A `tokio::sync::mpsc::Receiver<StreamChunk>`. The caller should
1210 /// receive from it until `chunk.done == true` or the channel closes.
1211 ///
1212 /// # Errors
1213 ///
1214 /// Returns immediately with [`InferenceError::NoModelLoaded`] if no
1215 /// model is loaded. Inference errors are delivered as the final chunk
1216 /// with `done = true` and an empty delta.
1217 pub async fn stream_message(
1218 &self,
1219 user_message: impl Into<String>,
1220 ) -> Result<tokio::sync::mpsc::Receiver<StreamChunk>, InferenceError> {
1221 let user_message = user_message.into();
1222
1223 let (model, request, pulse_model_id) = {
1224 let guard = self.inner.lock().await;
1225 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1226 let request = self::build_request(loaded, &user_message);
1227 let pulse_model_id = match &loaded.config {
1228 LoadedModelConfig::Gguf(c) => c.model_id.clone(),
1229 LoadedModelConfig::Uqff(c) => c.model_id.clone(),
1230 #[cfg(target_os = "macos")]
1231 LoadedModelConfig::Isq(c) => c.model_id.clone(),
1232 };
1233 (loaded.model.clone(), request, pulse_model_id)
1234 };
1235
1236 let (tx, rx) = tokio::sync::mpsc::channel::<StreamChunk>(64);
1237
1238 // We need a reference to `self` inside the spawned task so we can
1239 // update history when the stream completes. Since ChatEngine is
1240 // behind a shared reference (callers hold &self or Arc<ChatEngine>),
1241 // we borrow the inner Mutex via a raw pointer. This is safe because
1242 // ChatEngine's lifetime exceeds the spawned task (it's typically in
1243 // a Lazy static or shared application state).
1244 //
1245 // Alternative: require ChatEngine to always be wrapped in Arc and
1246 // accept `self: Arc<Self>`. We avoid that to keep the API simple.
1247 let inner_ptr = &self.inner as *const Mutex<Option<LoadedModel>>;
1248 // SAFETY: ChatEngine is stored in a Lazy static or equivalent with
1249 // 'static lifetime. The spawned task cannot outlive the engine.
1250 let inner_ref: &'static Mutex<Option<LoadedModel>> = unsafe { &*inner_ptr };
1251
1252 let user_msg_clone = user_message.clone();
1253 // Cloning the pulse client is a pointer bump (Arc-backed reqwest
1254 // client), so the task owns one rather than borrowing the engine.
1255 let pulse = self.pulse().cloned();
1256
1257 tokio::task::spawn(async move {
1258 // `model` is an `Arc<Model>` moved into this task, so it's owned.
1259 // `stream_chat_request` borrows `&Model` — the borrow is scoped
1260 // to this async block and does NOT need to be `'static`.
1261 let started = std::time::Instant::now();
1262 let stream_result = model.stream_chat_request(request).await;
1263
1264 match stream_result {
1265 Ok(mut stream) => {
1266 let mut assembled = String::new();
1267 let mut last_finish_reason = None;
1268 let mut ttft_ms = None;
1269 let mut status = "success";
1270
1271 // `Stream::next()` is an inherent async method on
1272 // mistralrs::Stream that returns `Option<Response>`.
1273 // No `futures::StreamExt` import needed.
1274 while let Some(response) = stream.next().await {
1275 match response {
1276 mistralrs::Response::Chunk(chunk) => {
1277 if let Some(choice) = chunk.choices.first() {
1278 if let Some(ref text) = choice.delta.content {
1279 ttft_ms.get_or_insert_with(|| {
1280 started.elapsed().as_millis() as u64
1281 });
1282 assembled.push_str(text);
1283 let sent = tx
1284 .send(StreamChunk {
1285 delta: text.clone(),
1286 done: false,
1287 finish_reason: None,
1288 })
1289 .await;
1290 // The receiver is gone, so nobody is
1291 // reading the rest. Stop generating and
1292 // keep only what the caller received.
1293 if sent.is_err() {
1294 status = "cancelled";
1295 break;
1296 }
1297 }
1298 if let Some(ref reason) = choice.finish_reason {
1299 last_finish_reason = Some(reason.clone());
1300 }
1301 }
1302 }
1303 mistralrs::Response::Done(_) => {
1304 // Non-streaming response arrived on a streaming
1305 // channel — should not happen, but handle gracefully.
1306 break;
1307 }
1308 mistralrs::Response::InternalError(e) => {
1309 log::error!("ChatEngine stream internal error: {}", e);
1310 status = "error";
1311 break;
1312 }
1313 mistralrs::Response::ValidationError(e) => {
1314 log::error!("ChatEngine stream validation error: {}", e);
1315 status = "error";
1316 break;
1317 }
1318 mistralrs::Response::ModelError(msg, _) => {
1319 log::error!("ChatEngine stream model error: {}", msg);
1320 status = "error";
1321 break;
1322 }
1323 _ => {
1324 // Completion / Image / Speech variants — not expected
1325 // for chat streaming.
1326 break;
1327 }
1328 }
1329 }
1330
1331 // Persist turns to history.
1332 {
1333 let mut guard = inner_ref.lock().await;
1334 if let Some(loaded) = guard.as_mut() {
1335 loaded
1336 .history
1337 .push(HistoryEntry::Text(ChatMessage::user(user_msg_clone)));
1338 loaded
1339 .history
1340 .push(HistoryEntry::Text(ChatMessage::assistant(assembled.trim())));
1341 }
1342 }
1343
1344 // Streamed inferences were invisible to pulse until now;
1345 // only the blocking path reported. This is also the one
1346 // path that can supply a real TTFT.
1347 if let Some(pulse) = pulse {
1348 pulse.record_inference(
1349 pulse_model_id,
1350 crate::pulse::next_request_id(),
1351 started.elapsed().as_millis() as u64,
1352 ttft_ms,
1353 status.to_string(),
1354 );
1355 }
1356
1357 // Send the final "done" chunk.
1358 let _ = tx
1359 .send(StreamChunk {
1360 delta: String::new(),
1361 done: true,
1362 finish_reason: last_finish_reason,
1363 })
1364 .await;
1365 }
1366 Err(e) => {
1367 log::error!("ChatEngine: stream_chat_request failed: {}", e);
1368 if let Some(pulse) = pulse {
1369 pulse.record_inference(
1370 pulse_model_id,
1371 crate::pulse::next_request_id(),
1372 started.elapsed().as_millis() as u64,
1373 None,
1374 "error".to_string(),
1375 );
1376 }
1377 let _ = tx
1378 .send(StreamChunk {
1379 delta: String::new(),
1380 done: true,
1381 finish_reason: Some(format!("error: {}", e)),
1382 })
1383 .await;
1384 }
1385 }
1386 });
1387
1388 Ok(rx)
1389 }
1390
1391 // ── Tool-aware inference (Rust-only) ─────────────────────────────────
1392
1393 /// Whether the currently loaded model is verified for structured tool calls.
1394 ///
1395 /// Returns [`Unsupported`](super::types::ToolCallingSupport::Unsupported) when no model
1396 /// is loaded: an engine with nothing in it cannot produce tool calls, and
1397 /// reporting `Unknown` there would conflate an empty engine with a model
1398 /// repository Onde has not catalogued.
1399 pub async fn tool_calling_support(&self) -> super::types::ToolCallingSupport {
1400 use super::types::ToolCallingSupport;
1401
1402 let guard = self.inner.lock().await;
1403 let Some(loaded) = guard.as_ref() else {
1404 return ToolCallingSupport::Unsupported;
1405 };
1406 let model_id = match &loaded.config {
1407 LoadedModelConfig::Gguf(config) => &config.model_id,
1408 LoadedModelConfig::Uqff(config) => &config.model_id,
1409 #[cfg(target_os = "macos")]
1410 LoadedModelConfig::Isq(config) => &config.model_id,
1411 };
1412 super::models::tool_calling_support(model_id)
1413 }
1414
1415 /// Send a user message with tool definitions available. Non-streaming.
1416 ///
1417 /// If the model decides to call tools, [`ToolAwareResult::tool_calls`]
1418 /// will be non-empty and `finish_reason` will typically be `"tool_calls"`.
1419 ///
1420 /// The user message and response (including any tool calls) are
1421 /// automatically added to conversation history.
1422 pub async fn send_message_with_tools(
1423 &self,
1424 user_message: impl Into<String>,
1425 tools: &[ToolDefinition],
1426 ) -> Result<ToolAwareResult, InferenceError> {
1427 let user_message = user_message.into();
1428
1429 let (model, request) = {
1430 let guard = self.inner.lock().await;
1431 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1432 let request = build_request_with_tools(loaded, &user_message, tools);
1433 (loaded.model.clone(), request)
1434 };
1435
1436 log::info!(
1437 "ChatEngine: tool inference START — message: \"{}\"",
1438 truncate_for_log(&user_message, 100)
1439 );
1440
1441 let start = std::time::Instant::now();
1442 let response =
1443 model
1444 .send_chat_request(request)
1445 .await
1446 .map_err(|e| InferenceError::Inference {
1447 reason: e.to_string(),
1448 })?;
1449 let elapsed = start.elapsed();
1450
1451 let choice = &response.choices[0];
1452 let reply = choice
1453 .message
1454 .content
1455 .as_ref()
1456 .map(|c| c.trim().to_string())
1457 .unwrap_or_default();
1458 let finish_reason = choice.finish_reason.clone();
1459 let tool_calls = parse_tool_calls(choice);
1460
1461 log::info!(
1462 "ChatEngine: tool inference END — {} — tool_calls: {} — reply: \"{}\"",
1463 format_duration(elapsed),
1464 tool_calls.len(),
1465 truncate_for_log(&reply, 100)
1466 );
1467
1468 // Persist to history.
1469 {
1470 let mut guard = self.inner.lock().await;
1471 if let Some(loaded) = guard.as_mut() {
1472 loaded
1473 .history
1474 .push(HistoryEntry::Text(ChatMessage::user(&user_message)));
1475 if tool_calls.is_empty() {
1476 loaded
1477 .history
1478 .push(HistoryEntry::Text(ChatMessage::assistant(&reply)));
1479 } else {
1480 loaded.history.push(HistoryEntry::AssistantToolCall {
1481 content: reply.clone(),
1482 tool_calls: tool_calls.clone(),
1483 });
1484 }
1485 }
1486 }
1487
1488 Ok(ToolAwareResult {
1489 text: reply,
1490 tool_calls,
1491 duration_secs: elapsed.as_secs_f64(),
1492 duration_display: format_duration(elapsed),
1493 finish_reason,
1494 })
1495 }
1496
1497 /// Send tool execution results back to the model. Non-streaming.
1498 ///
1499 /// Call this after executing the tools from a previous
1500 /// [`send_message_with_tools`](Self::send_message_with_tools) response.
1501 /// Pass the same (or updated) tool definitions if the model should be
1502 /// allowed to make further tool calls.
1503 pub async fn send_tool_results(
1504 &self,
1505 results: Vec<ToolResult>,
1506 tools: Option<&[ToolDefinition]>,
1507 ) -> Result<ToolAwareResult, InferenceError> {
1508 let (model, request) = {
1509 let guard = self.inner.lock().await;
1510 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1511 let request = build_tool_results_request(loaded, &results, tools);
1512 (loaded.model.clone(), request)
1513 };
1514
1515 log::info!(
1516 "ChatEngine: tool results inference START — {} results",
1517 results.len()
1518 );
1519
1520 let start = std::time::Instant::now();
1521 let response =
1522 model
1523 .send_chat_request(request)
1524 .await
1525 .map_err(|e| InferenceError::Inference {
1526 reason: e.to_string(),
1527 })?;
1528 let elapsed = start.elapsed();
1529
1530 let choice = &response.choices[0];
1531 let reply = choice
1532 .message
1533 .content
1534 .as_ref()
1535 .map(|c| c.trim().to_string())
1536 .unwrap_or_default();
1537 let finish_reason = choice.finish_reason.clone();
1538 let tool_calls = parse_tool_calls(choice);
1539
1540 log::info!(
1541 "ChatEngine: tool results inference END — {} — tool_calls: {} — reply: \"{}\"",
1542 format_duration(elapsed),
1543 tool_calls.len(),
1544 truncate_for_log(&reply, 100)
1545 );
1546
1547 // Persist to history.
1548 {
1549 let mut guard = self.inner.lock().await;
1550 if let Some(loaded) = guard.as_mut() {
1551 for result in &results {
1552 loaded.history.push(HistoryEntry::ToolResult {
1553 tool_call_id: result.tool_call_id.clone(),
1554 content: result.content.clone(),
1555 });
1556 }
1557 if tool_calls.is_empty() {
1558 loaded
1559 .history
1560 .push(HistoryEntry::Text(ChatMessage::assistant(&reply)));
1561 } else {
1562 loaded.history.push(HistoryEntry::AssistantToolCall {
1563 content: reply.clone(),
1564 tool_calls: tool_calls.clone(),
1565 });
1566 }
1567 }
1568 }
1569
1570 Ok(ToolAwareResult {
1571 text: reply,
1572 tool_calls,
1573 duration_secs: elapsed.as_secs_f64(),
1574 duration_display: format_duration(elapsed),
1575 finish_reason,
1576 })
1577 }
1578
1579 /// Run the loaded model on the caller's conversation, with `tools` offered.
1580 /// Non-streaming.
1581 ///
1582 /// Unlike [`send_message_with_tools`](Self::send_message_with_tools), this
1583 /// neither reads nor changes the engine's history, and the engine's system
1584 /// prompt is not added: `turns` is the whole conversation, system message
1585 /// included. Hosts that keep their own conversation (an ACP agent persisting
1586 /// sessions, for example) call this every turn with the full list, assistant
1587 /// tool calls and tool results included.
1588 ///
1589 /// `sampling` overrides the engine's sampling for this call only.
1590 pub async fn complete(
1591 &self,
1592 turns: Vec<ChatTurn>,
1593 tools: &[ToolDefinition],
1594 sampling: Option<SamplingConfig>,
1595 ) -> Result<CompletionResult, InferenceError> {
1596 let (model, request) = {
1597 let guard = self.inner.lock().await;
1598 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1599 let sampling = sampling.as_ref().unwrap_or(&loaded.sampling);
1600 (
1601 loaded.model.clone(),
1602 build_turns_request(&turns, tools, sampling),
1603 )
1604 };
1605
1606 log::info!(
1607 "ChatEngine: complete START — {} turns, {} tools",
1608 turns.len(),
1609 tools.len()
1610 );
1611 let start = std::time::Instant::now();
1612 let response =
1613 model
1614 .send_chat_request(request)
1615 .await
1616 .map_err(|e| InferenceError::Inference {
1617 reason: e.to_string(),
1618 })?;
1619 let elapsed = start.elapsed();
1620
1621 let choice = response
1622 .choices
1623 .first()
1624 .ok_or_else(|| InferenceError::Inference {
1625 reason: "the model returned no choices".into(),
1626 })?;
1627 let content = choice.message.content.clone().unwrap_or_default();
1628 let (text, inline_reasoning) = split_think(&content);
1629 let reasoning = choice
1630 .message
1631 .reasoning_content
1632 .clone()
1633 .filter(|r| !r.trim().is_empty())
1634 .or(inline_reasoning);
1635 let tool_calls = parse_tool_calls(choice);
1636
1637 log::info!(
1638 "ChatEngine: complete END — {} — tool_calls: {} — reply: \"{}\"",
1639 format_duration(elapsed),
1640 tool_calls.len(),
1641 truncate_for_log(&text, 100)
1642 );
1643
1644 Ok(CompletionResult {
1645 text,
1646 reasoning,
1647 tool_calls,
1648 finish_reason: choice.finish_reason.clone(),
1649 prompt_tokens: response.usage.prompt_tokens,
1650 completion_tokens: response.usage.completion_tokens,
1651 duration_secs: elapsed.as_secs_f64(),
1652 })
1653 }
1654
1655 /// Append tool results to history without starting another inference round.
1656 ///
1657 /// Agent hosts use this when a turn is cancelled after the assistant has
1658 /// requested tools. Recording a result for every request keeps the history
1659 /// structurally valid for the next turn.
1660 ///
1661 /// Only results answering a call the assistant actually requested, and not
1662 /// already recorded, are appended. Anything else is dropped with a warning:
1663 /// a `ToolResult` with no matching `AssistantToolCall` ahead of it builds a
1664 /// message sequence the model cannot interpret, and silently writing one
1665 /// corrupts every subsequent turn rather than the current one. Returns how
1666 /// many results were appended.
1667 pub async fn record_tool_results(&self, results: Vec<ToolResult>) -> usize {
1668 let mut guard = self.inner.lock().await;
1669 let Some(loaded) = guard.as_mut() else {
1670 return 0;
1671 };
1672
1673 // Walk back over results already recorded for this round to find the
1674 // assistant turn they answer. A `Text` entry means the round is closed.
1675 let mut requested: Vec<String> = Vec::new();
1676 let mut seen: HashSet<String> = HashSet::new();
1677 for entry in loaded.history.iter().rev() {
1678 match entry {
1679 HistoryEntry::ToolResult { tool_call_id, .. } => {
1680 seen.insert(tool_call_id.clone());
1681 }
1682 HistoryEntry::AssistantToolCall { tool_calls, .. } => {
1683 requested = tool_calls.iter().map(|call| call.id.clone()).collect();
1684 break;
1685 }
1686 HistoryEntry::Text(_) => break,
1687 }
1688 }
1689
1690 if requested.is_empty() {
1691 log::warn!(
1692 "ChatEngine: record_tool_results found no assistant tool call to answer — dropping {} result(s)",
1693 results.len()
1694 );
1695 return 0;
1696 }
1697
1698 let mut appended = 0;
1699 for result in results {
1700 if !requested.contains(&result.tool_call_id) {
1701 log::warn!(
1702 "ChatEngine: dropping tool result for unrequested call {:?}",
1703 result.tool_call_id
1704 );
1705 continue;
1706 }
1707 if !seen.insert(result.tool_call_id.clone()) {
1708 log::warn!(
1709 "ChatEngine: dropping duplicate tool result for call {:?}",
1710 result.tool_call_id
1711 );
1712 continue;
1713 }
1714 loaded.history.push(HistoryEntry::ToolResult {
1715 tool_call_id: result.tool_call_id,
1716 content: result.content,
1717 });
1718 appended += 1;
1719 }
1720 appended
1721 }
1722
1723 /// Stream tool execution results back to the model.
1724 ///
1725 /// Similar to [`send_tool_results`](Self::send_tool_results) but returns
1726 /// a streaming receiver. Tool calls in the streaming response are NOT
1727 /// parsed — this is intended for the final text response after all tool
1728 /// rounds complete.
1729 pub async fn stream_tool_results(
1730 &self,
1731 results: Vec<ToolResult>,
1732 tools: Option<Vec<ToolDefinition>>,
1733 ) -> Result<tokio::sync::mpsc::Receiver<StreamChunk>, InferenceError> {
1734 let (model, request) = {
1735 let guard = self.inner.lock().await;
1736 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1737 let request = build_tool_results_request(loaded, &results, tools.as_deref());
1738 (loaded.model.clone(), request)
1739 };
1740
1741 let (tx, rx) = tokio::sync::mpsc::channel::<StreamChunk>(64);
1742
1743 let inner_ptr = &self.inner as *const Mutex<Option<LoadedModel>>;
1744 // SAFETY: ChatEngine is stored in a Lazy static or equivalent with
1745 // 'static lifetime. The spawned task cannot outlive the engine.
1746 let inner_ref: &'static Mutex<Option<LoadedModel>> = unsafe { &*inner_ptr };
1747
1748 let results_clone = results;
1749
1750 tokio::task::spawn(async move {
1751 let stream_result = model.stream_chat_request(request).await;
1752
1753 match stream_result {
1754 Ok(mut stream) => {
1755 let mut assembled = String::new();
1756 let mut last_finish_reason = None;
1757
1758 while let Some(response) = stream.next().await {
1759 match response {
1760 mistralrs::Response::Chunk(chunk) => {
1761 if let Some(choice) = chunk.choices.first() {
1762 if let Some(ref text) = choice.delta.content {
1763 assembled.push_str(text);
1764 let _ = tx
1765 .send(StreamChunk {
1766 delta: text.clone(),
1767 done: false,
1768 finish_reason: None,
1769 })
1770 .await;
1771 }
1772 if let Some(ref reason) = choice.finish_reason {
1773 last_finish_reason = Some(reason.clone());
1774 }
1775 }
1776 }
1777 mistralrs::Response::Done(_) => break,
1778 mistralrs::Response::InternalError(e) => {
1779 log::error!("stream_tool_results internal error: {}", e);
1780 break;
1781 }
1782 mistralrs::Response::ValidationError(e) => {
1783 log::error!("stream_tool_results validation error: {}", e);
1784 break;
1785 }
1786 mistralrs::Response::ModelError(msg, _) => {
1787 log::error!("stream_tool_results model error: {}", msg);
1788 break;
1789 }
1790 _ => break,
1791 }
1792 }
1793
1794 // Persist to history.
1795 {
1796 let mut guard = inner_ref.lock().await;
1797 if let Some(loaded) = guard.as_mut() {
1798 for result in &results_clone {
1799 loaded.history.push(HistoryEntry::ToolResult {
1800 tool_call_id: result.tool_call_id.clone(),
1801 content: result.content.clone(),
1802 });
1803 }
1804 loaded
1805 .history
1806 .push(HistoryEntry::Text(ChatMessage::assistant(assembled.trim())));
1807 }
1808 }
1809
1810 let _ = tx
1811 .send(StreamChunk {
1812 delta: String::new(),
1813 done: true,
1814 finish_reason: last_finish_reason,
1815 })
1816 .await;
1817 }
1818 Err(e) => {
1819 log::error!("stream_tool_results failed: {}", e);
1820 let _ = tx
1821 .send(StreamChunk {
1822 delta: String::new(),
1823 done: true,
1824 finish_reason: Some(format!("error: {}", e)),
1825 })
1826 .await;
1827 }
1828 }
1829 });
1830
1831 Ok(rx)
1832 }
1833 }
1834
1835 // ═════════════════════════════════════════════════════════════════════════════
1836 // Default impl
1837 // ═════════════════════════════════════════════════════════════════════════════
1838
1839 #[cfg(any(
1840 target_os = "macos",
1841 target_os = "ios",
1842 target_os = "tvos",
1843 target_os = "visionos",
1844 target_os = "watchos",
1845 target_os = "windows",
1846 target_os = "linux",
1847 target_os = "android"
1848 ))]
1849 impl Default for ChatEngine {
1850 fn default() -> Self {
1851 Self::new()
1852 }
1853 }
1854
1855 // ═════════════════════════════════════════════════════════════════════════════
1856 // Internal helpers (platform-gated)
1857 // ═════════════════════════════════════════════════════════════════════════════
1858
1859 /// Build a `RequestBuilder` from the current engine state and a new user message.
1860 #[cfg(any(
1861 target_os = "macos",
1862 target_os = "ios",
1863 target_os = "tvos",
1864 target_os = "visionos",
1865 target_os = "watchos",
1866 target_os = "windows",
1867 target_os = "linux",
1868 target_os = "android"
1869 ))]
1870 fn build_request(loaded: &LoadedModel, user_message: &str) -> RequestBuilder {
1871 let mut req = RequestBuilder::new();
1872
1873 // Apply sampling.
1874 req = apply_sampling(req, &loaded.sampling);
1875
1876 // System prompt.
1877 if let Some(ref sp) = loaded.system_prompt {
1878 req = req.add_message(TextMessageRole::System, sp);
1879 }
1880
1881 // Replay conversation history for multi-turn context.
1882 for entry in &loaded.history {
1883 match entry {
1884 HistoryEntry::Text(msg) => {
1885 req = req.add_message(chat_role_to_mistral(&msg.role), &msg.content);
1886 }
1887 HistoryEntry::AssistantToolCall { content, .. } => {
1888 // When replaying without tools, just use the text content.
1889 req = req.add_message(TextMessageRole::Assistant, content);
1890 }
1891 HistoryEntry::ToolResult { .. } => {
1892 // Skip tool results when not in tool mode.
1893 }
1894 }
1895 }
1896
1897 // Append the current user message.
1898 req = req.add_message(TextMessageRole::User, user_message);
1899
1900 req
1901 }
1902
1903 /// Build a request with tool support, properly replaying tool-related history.
1904 #[cfg(any(
1905 target_os = "macos",
1906 target_os = "ios",
1907 target_os = "tvos",
1908 target_os = "visionos",
1909 target_os = "watchos",
1910 target_os = "windows",
1911 target_os = "linux",
1912 target_os = "android"
1913 ))]
1914 fn build_request_with_tools(
1915 loaded: &LoadedModel,
1916 user_message: &str,
1917 tools: &[ToolDefinition],
1918 ) -> RequestBuilder {
1919 let mut req = RequestBuilder::new();
1920 req = apply_sampling(req, &loaded.sampling);
1921
1922 if let Some(ref sp) = loaded.system_prompt {
1923 req = req.add_message(TextMessageRole::System, sp);
1924 }
1925
1926 req = replay_history_with_tools(req, &loaded.history);
1927
1928 req = req.add_message(TextMessageRole::User, user_message);
1929
1930 // Add tool definitions.
1931 req = attach_tools(req, tools);
1932
1933 req
1934 }
1935
1936 /// Build a request for sending tool results back to the model.
1937 #[cfg(any(
1938 target_os = "macos",
1939 target_os = "ios",
1940 target_os = "tvos",
1941 target_os = "visionos",
1942 target_os = "watchos",
1943 target_os = "windows",
1944 target_os = "linux",
1945 target_os = "android"
1946 ))]
1947 fn build_tool_results_request(
1948 loaded: &LoadedModel,
1949 results: &[ToolResult],
1950 tools: Option<&[ToolDefinition]>,
1951 ) -> RequestBuilder {
1952 let mut req = RequestBuilder::new();
1953 req = apply_sampling(req, &loaded.sampling);
1954
1955 if let Some(ref sp) = loaded.system_prompt {
1956 req = req.add_message(TextMessageRole::System, sp);
1957 }
1958
1959 // Replay full history (including tool call entries).
1960 req = replay_history_with_tools(req, &loaded.history);
1961
1962 // Add new tool results.
1963 for result in results {
1964 req = req.add_tool_message(&result.content, &result.tool_call_id);
1965 }
1966
1967 // Optionally add tools for further rounds.
1968 if let Some(tool_defs) = tools {
1969 req = attach_tools(req, tool_defs);
1970 }
1971
1972 req
1973 }
1974
1975 /// Replay conversation history entries onto a [`RequestBuilder`], preserving
1976 /// tool call and tool result messages.
1977 #[cfg(any(
1978 target_os = "macos",
1979 target_os = "ios",
1980 target_os = "tvos",
1981 target_os = "visionos",
1982 target_os = "watchos",
1983 target_os = "windows",
1984 target_os = "linux",
1985 target_os = "android"
1986 ))]
1987 fn replay_history_with_tools(mut req: RequestBuilder, history: &[HistoryEntry]) -> RequestBuilder {
1988 for entry in history {
1989 match entry {
1990 HistoryEntry::Text(msg) => {
1991 req = req.add_message(chat_role_to_mistral(&msg.role), &msg.content);
1992 }
1993 HistoryEntry::AssistantToolCall {
1994 content,
1995 tool_calls,
1996 } => {
1997 let mistral_tcs: Vec<ToolCallResponse> = tool_calls
1998 .iter()
1999 .enumerate()
2000 .map(|(i, tc)| ToolCallResponse {
2001 index: i,
2002 id: tc.id.clone(),
2003 tp: ToolCallType::Function,
2004 function: CalledFunction {
2005 name: tc.function_name.clone(),
2006 arguments: tc.arguments.clone(),
2007 },
2008 })
2009 .collect();
2010 req = req.add_message_with_tool_call(
2011 TextMessageRole::Assistant,
2012 content.clone(),
2013 mistral_tcs,
2014 );
2015 }
2016 HistoryEntry::ToolResult {
2017 tool_call_id,
2018 content,
2019 } => {
2020 req = req.add_tool_message(content, tool_call_id);
2021 }
2022 }
2023 }
2024 req
2025 }
2026
2027 #[cfg(any(
2028 target_os = "macos",
2029 target_os = "ios",
2030 target_os = "tvos",
2031 target_os = "visionos",
2032 target_os = "watchos",
2033 target_os = "windows",
2034 target_os = "linux",
2035 target_os = "android"
2036 ))]
2037 /// Build a request from a caller-supplied conversation for [`ChatEngine::complete`].
2038 fn build_turns_request(
2039 turns: &[ChatTurn],
2040 tools: &[ToolDefinition],
2041 sampling: &SamplingConfig,
2042 ) -> RequestBuilder {
2043 let req = apply_sampling(RequestBuilder::new(), sampling);
2044 let history: Vec<HistoryEntry> = turns.iter().cloned().map(HistoryEntry::from).collect();
2045 let req = replay_history_with_tools(req, &history);
2046 attach_tools(req, tools)
2047 }
2048
2049 #[cfg(any(
2050 target_os = "macos",
2051 target_os = "ios",
2052 target_os = "tvos",
2053 target_os = "visionos",
2054 target_os = "watchos",
2055 target_os = "windows",
2056 target_os = "linux",
2057 target_os = "android"
2058 ))]
2059 impl From<ChatTurn> for HistoryEntry {
2060 fn from(turn: ChatTurn) -> Self {
2061 match turn {
2062 ChatTurn::System(content) => HistoryEntry::Text(ChatMessage::system(content)),
2063 ChatTurn::User(content) => HistoryEntry::Text(ChatMessage::user(content)),
2064 ChatTurn::Assistant {
2065 content,
2066 tool_calls,
2067 } if tool_calls.is_empty() => HistoryEntry::Text(ChatMessage::assistant(content)),
2068 ChatTurn::Assistant {
2069 content,
2070 tool_calls,
2071 } => HistoryEntry::AssistantToolCall {
2072 content,
2073 tool_calls,
2074 },
2075 ChatTurn::Tool {
2076 tool_call_id,
2077 content,
2078 } => HistoryEntry::ToolResult {
2079 tool_call_id,
2080 content,
2081 },
2082 }
2083 }
2084 }
2085
2086 /// Split a leading `<think>…</think>` block (Qwen3 thinking models) off a
2087 /// reply: `(reply, reasoning)`. A reply without one comes back trimmed, with
2088 /// no reasoning. An unclosed block is all reasoning: the model ran out of
2089 /// tokens while thinking.
2090 fn split_think(content: &str) -> (String, Option<String>) {
2091 let trimmed = content.trim_start();
2092 let Some(rest) = trimmed.strip_prefix("<think>") else {
2093 return (content.trim().to_string(), None);
2094 };
2095 let (reasoning, reply) = match rest.split_once("</think>") {
2096 Some((reasoning, reply)) => (reasoning, reply),
2097 None => (rest, ""),
2098 };
2099 let reasoning = reasoning.trim();
2100 (
2101 reply.trim().to_string(),
2102 (!reasoning.is_empty()).then(|| reasoning.to_string()),
2103 )
2104 }
2105
2106 /// Attach [`ToolDefinition`] slices as mistralrs [`Tool`] values and set
2107 /// [`ToolChoice::Auto`].
2108 #[cfg(any(
2109 target_os = "macos",
2110 target_os = "ios",
2111 target_os = "tvos",
2112 target_os = "visionos",
2113 target_os = "watchos",
2114 target_os = "windows",
2115 target_os = "linux",
2116 target_os = "android"
2117 ))]
2118 fn attach_tools(mut req: RequestBuilder, tools: &[ToolDefinition]) -> RequestBuilder {
2119 if !tools.is_empty() {
2120 let mistral_tools: Vec<MistralTool> = tools
2121 .iter()
2122 .map(|td| {
2123 let params: HashMap<String, serde_json::Value> =
2124 match serde_json::from_str(&td.parameters_schema) {
2125 Ok(p) => p,
2126 Err(err) => {
2127 log::warn!(
2128 "tool '{}': malformed parameters_schema JSON ({}), using empty params",
2129 td.name,
2130 err
2131 );
2132 HashMap::new()
2133 }
2134 };
2135 MistralTool {
2136 tp: ToolType::Function,
2137 function: Function {
2138 description: Some(td.description.clone()),
2139 name: td.name.clone(),
2140 parameters: Some(params),
2141 strict: Some(true),
2142 },
2143 }
2144 })
2145 .collect();
2146 req = req.set_tools(mistral_tools);
2147 req = req.set_tool_choice(ToolChoice::Auto);
2148 }
2149 req
2150 }
2151
2152 /// Parse tool calls from a mistralrs response choice.
2153 #[cfg(any(
2154 target_os = "macos",
2155 target_os = "ios",
2156 target_os = "tvos",
2157 target_os = "visionos",
2158 target_os = "watchos",
2159 target_os = "windows",
2160 target_os = "linux",
2161 target_os = "android"
2162 ))]
2163 fn parse_tool_calls(choice: &mistralrs::Choice) -> Vec<ToolCallRequest> {
2164 choice
2165 .message
2166 .tool_calls
2167 .as_ref()
2168 .map(|tcs| {
2169 tcs.iter()
2170 .enumerate()
2171 .map(|(i, tc)| {
2172 let id = if tc.id.is_empty() {
2173 log::warn!("tool call at index {i} has empty id — generating fallback");
2174 format!("call_{i}")
2175 } else {
2176 tc.id.clone()
2177 };
2178 if serde_json::from_str::<serde_json::Value>(&tc.function.arguments).is_err() {
2179 log::warn!(
2180 "tool call '{}' has malformed arguments JSON: {}",
2181 tc.function.name,
2182 tc.function.arguments.chars().take(200).collect::<String>()
2183 );
2184 }
2185 ToolCallRequest {
2186 id,
2187 function_name: tc.function.name.clone(),
2188 arguments: tc.function.arguments.clone(),
2189 }
2190 })
2191 .collect()
2192 })
2193 .unwrap_or_default()
2194 }
2195
2196 /// Apply a [`SamplingConfig`] to a [`RequestBuilder`].
2197 #[cfg(any(
2198 target_os = "macos",
2199 target_os = "ios",
2200 target_os = "tvos",
2201 target_os = "visionos",
2202 target_os = "watchos",
2203 target_os = "windows",
2204 target_os = "linux",
2205 target_os = "android"
2206 ))]
2207 fn apply_sampling(mut req: RequestBuilder, sampling: &SamplingConfig) -> RequestBuilder {
2208 if let Some(temp) = sampling.temperature {
2209 req = req.set_sampler_temperature(temp);
2210 }
2211 if let Some(top_p) = sampling.top_p {
2212 req = req.set_sampler_topp(top_p);
2213 }
2214 if let Some(top_k) = sampling.top_k {
2215 req = req.set_sampler_topk(top_k as usize);
2216 }
2217 if let Some(min_p) = sampling.min_p {
2218 req = req.set_sampler_minp(min_p);
2219 }
2220 if let Some(max_tokens) = sampling.max_tokens {
2221 req = req.set_sampler_max_len(max_tokens as usize);
2222 }
2223 if let Some(freq) = sampling.frequency_penalty {
2224 req = req.set_sampler_frequency_penalty(freq);
2225 }
2226 if let Some(pres) = sampling.presence_penalty {
2227 req = req.set_sampler_presence_penalty(pres);
2228 }
2229 req
2230 }
2231
2232 /// Convert a [`ChatRole`] to a [`TextMessageRole`].
2233 #[cfg(any(
2234 target_os = "macos",
2235 target_os = "ios",
2236 target_os = "tvos",
2237 target_os = "visionos",
2238 target_os = "watchos",
2239 target_os = "windows",
2240 target_os = "linux",
2241 target_os = "android"
2242 ))]
2243 fn chat_role_to_mistral(role: &ChatRole) -> TextMessageRole {
2244 match role {
2245 ChatRole::System => TextMessageRole::System,
2246 ChatRole::User => TextMessageRole::User,
2247 ChatRole::Assistant => TextMessageRole::Assistant,
2248 }
2249 }
2250
2251 /// Truncate a string for log output, appending `"..."` if truncated.
2252 fn truncate_for_log(s: &str, max_len: usize) -> String {
2253 if s.len() > max_len {
2254 format!("{}...", &s[..s.floor_char_boundary(max_len)])
2255 } else {
2256 s.to_string()
2257 }
2258 }
2259
2260 // ═════════════════════════════════════════════════════════════════════════════
2261 // Unsupported-platform stub
2262 // ═════════════════════════════════════════════════════════════════════════════
2263
2264 /// Stub `ChatEngine` for platforms where mistral.rs is not available.
2265 ///
2266 /// All methods return errors or empty values so that code using the engine
2267 /// can compile on any target without `#[cfg]` at every call site.
2268 #[cfg(not(any(
2269 target_os = "macos",
2270 target_os = "ios",
2271 target_os = "tvos",
2272 target_os = "visionos",
2273 target_os = "watchos",
2274 target_os = "windows",
2275 target_os = "linux",
2276 target_os = "android"
2277 )))]
2278 pub struct ChatEngine;
2279
2280 #[cfg(not(any(
2281 target_os = "macos",
2282 target_os = "ios",
2283 target_os = "tvos",
2284 target_os = "visionos",
2285 target_os = "watchos",
2286 target_os = "windows",
2287 target_os = "linux",
2288 target_os = "android"
2289 )))]
2290 impl ChatEngine {
2291 pub fn new() -> Self {
2292 Self
2293 }
2294
2295 /// Stub mirror of the real `with_app_id`. The unsupported-platform engine
2296 /// ignores telemetry, so the app id is dropped.
2297 pub fn with_app_id(_onde_app_id: Option<String>) -> Self {
2298 Self
2299 }
2300
2301 /// Stub mirror of the real `with_pulse_country`; no telemetry is sent.
2302 pub fn with_pulse_country(self, _country_code: Option<String>) -> Self {
2303 self
2304 }
2305
2306 pub async fn load_gguf_model(
2307 &self,
2308 _config: GgufModelConfig,
2309 _system_prompt: Option<String>,
2310 _sampling: Option<SamplingConfig>,
2311 ) -> Result<std::time::Duration, InferenceError> {
2312 Err(InferenceError::Other {
2313 reason: "LLM inference is not supported on this platform.".into(),
2314 })
2315 }
2316
2317 pub async fn load_uqff_model(
2318 &self,
2319 _config: UqffModelConfig,
2320 _system_prompt: Option<String>,
2321 _sampling: Option<SamplingConfig>,
2322 ) -> Result<std::time::Duration, InferenceError> {
2323 Err(InferenceError::Other {
2324 reason: "LLM inference is not supported on this platform.".into(),
2325 })
2326 }
2327
2328 pub async fn unload_model(&self) -> Option<String> {
2329 None
2330 }
2331
2332 pub async fn is_loaded(&self) -> bool {
2333 false
2334 }
2335
2336 pub async fn info(&self) -> EngineInfo {
2337 EngineInfo {
2338 status: EngineStatus::Unloaded,
2339 model_name: None,
2340 approx_memory: None,
2341 history_length: 0u64,
2342 }
2343 }
2344
2345 pub async fn set_system_prompt(&self, _prompt: impl Into<String>) {}
2346
2347 pub async fn clear_system_prompt(&self) {}
2348
2349 pub async fn set_sampling(&self, _sampling: SamplingConfig) {}
2350
2351 pub async fn history(&self) -> Vec<ChatMessage> {
2352 Vec::new()
2353 }
2354
2355 pub async fn clear_history(&self) -> usize {
2356 0
2357 }
2358
2359 pub async fn push_history(&self, _message: ChatMessage) {}
2360
2361 pub async fn tool_calling_support(&self) -> super::types::ToolCallingSupport {
2362 // Inference cannot run on this platform at all, so tool calling is not
2363 // merely unverified — it is unavailable.
2364 super::types::ToolCallingSupport::Unsupported
2365 }
2366
2367 pub async fn record_tool_results(&self, _results: Vec<ToolResult>) -> usize {
2368 0
2369 }
2370
2371 pub async fn send_message(
2372 &self,
2373 _user_message: impl Into<String>,
2374 ) -> Result<InferenceResult, InferenceError> {
2375 Err(InferenceError::Other {
2376 reason: "LLM inference is not supported on this platform.".into(),
2377 })
2378 }
2379
2380 pub async fn generate(
2381 &self,
2382 _messages: Vec<ChatMessage>,
2383 _sampling: Option<SamplingConfig>,
2384 ) -> Result<InferenceResult, InferenceError> {
2385 Err(InferenceError::Other {
2386 reason: "LLM inference is not supported on this platform.".into(),
2387 })
2388 }
2389
2390 pub async fn stream_message(
2391 &self,
2392 _user_message: impl Into<String>,
2393 ) -> Result<tokio::sync::mpsc::Receiver<StreamChunk>, InferenceError> {
2394 Err(InferenceError::Other {
2395 reason: "LLM inference is not supported on this platform.".into(),
2396 })
2397 }
2398
2399 pub async fn send_message_with_tools(
2400 &self,
2401 _user_message: &str,
2402 _tools: &[ToolDefinition],
2403 ) -> Result<InferenceResult, InferenceError> {
2404 Err(InferenceError::Other {
2405 reason: "LLM inference is not supported on this platform.".into(),
2406 })
2407 }
2408
2409 pub async fn send_tool_results(
2410 &self,
2411 _results: Vec<ToolResult>,
2412 _next_tools: Option<&[ToolDefinition]>,
2413 ) -> Result<InferenceResult, InferenceError> {
2414 Err(InferenceError::Other {
2415 reason: "LLM inference is not supported on this platform.".into(),
2416 })
2417 }
2418
2419 pub async fn complete(
2420 &self,
2421 _turns: Vec<ChatTurn>,
2422 _tools: &[ToolDefinition],
2423 _sampling: Option<SamplingConfig>,
2424 ) -> Result<CompletionResult, InferenceError> {
2425 Err(InferenceError::Other {
2426 reason: "LLM inference is not supported on this platform.".into(),
2427 })
2428 }
2429 }
2430
2431 #[cfg(not(any(
2432 target_os = "macos",
2433 target_os = "ios",
2434 target_os = "tvos",
2435 target_os = "visionos",
2436 target_os = "watchos",
2437 target_os = "windows",
2438 target_os = "linux",
2439 target_os = "android"
2440 )))]
2441 impl Default for ChatEngine {
2442 fn default() -> Self {
2443 Self::new()
2444 }
2445 }
2446
2447 // ═════════════════════════════════════════════════════════════════════════════
2448 // Prebuilt model configs (convenience constructors)
2449 // ═════════════════════════════════════════════════════════════════════════════
2450
2451 /// DeepSeek Coder v1 Jinja chat template.
2452 ///
2453 /// TheBloke's GGUF files do not embed a chat template — without an explicit
2454 /// template, mistral.rs rejects chat requests with "model does not have a
2455 /// chat template". This template reproduces the official DeepSeek Coder v1
2456 /// `### Instruction: / ### Response:` format.
2457 const DEEPSEEK_CODER_CHAT_TEMPLATE: &str = include_str!("deepseek_coder_chat_template.txt");
2458
2459 /// Convenience constructors for common GGUF model configurations.
2460 ///
2461 /// These mirror the constants in [`super::models`] but return a ready-to-use
2462 /// [`GgufModelConfig`].
2463 impl GgufModelConfig {
2464 /// Whether the model's default chat template emits an extended reasoning
2465 /// block that needs a larger output budget than the general presets.
2466 fn uses_extended_thinking(&self) -> bool {
2467 matches!(
2468 self.model_id.as_str(),
2469 super::models::BARTOWSKI_QWEN3_0_6B_GGUF
2470 | super::models::BARTOWSKI_QWEN3_1_7B_GGUF
2471 | super::models::BARTOWSKI_QWEN3_4B_GGUF
2472 | super::models::BARTOWSKI_QWEN3_8B_GGUF
2473 | super::models::BARTOWSKI_QWEN3_14B_GGUF
2474 | super::models::BARTOWSKI_QWEN3_32B_GGUF
2475 | super::models::BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF
2476 )
2477 }
2478
2479 /// Resolve a model-management ID (a Hugging Face repository from
2480 /// [`SUPPORTED_MODELS`](super::models::SUPPORTED_MODELS), such as
2481 /// `bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF`) to the same configuration
2482 /// used by the inference engine. Keeping this mapping in one place prevents
2483 /// download and load support from drifting apart. `None` for an ID Onde
2484 /// doesn't support.
2485 pub fn from_supported_model_id(model_id: &str) -> Option<Self> {
2486 match model_id {
2487 super::models::BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF => Some(Self::qwen25_0_5b()),
2488 super::models::BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF => Some(Self::qwen25_1_5b()),
2489 super::models::BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF => Some(Self::qwen25_3b()),
2490 super::models::BARTOWSKI_QWEN3_0_6B_GGUF => Some(Self::qwen3_0_6b()),
2491 super::models::BARTOWSKI_QWEN3_1_7B_GGUF => Some(Self::qwen3_1_7b()),
2492 super::models::BARTOWSKI_QWEN3_4B_GGUF => Some(Self::qwen3_4b()),
2493 super::models::BARTOWSKI_QWEN3_8B_GGUF => Some(Self::qwen3_8b()),
2494 super::models::BARTOWSKI_QWEN3_14B_GGUF => Some(Self::qwen3_14b()),
2495 super::models::BARTOWSKI_QWEN3_32B_GGUF => Some(Self::qwen3_32b()),
2496 super::models::BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF => {
2497 Some(Self::qwen3_4b_instruct_2507())
2498 }
2499 super::models::BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF => {
2500 Some(Self::qwen3_4b_thinking_2507())
2501 }
2502 super::models::BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF => {
2503 Some(Self::qwen3_30b_a3b_instruct_2507())
2504 }
2505 super::models::BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF => Some(Self::qwen25_coder_7b()),
2506 super::models::THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF => {
2507 Some(Self::deepseek_coder_6_7b())
2508 }
2509 _ => None,
2510 }
2511 }
2512
2513 /// Qwen 2.5 0.5B Instruct (GGUF Q4_K_M), ~380 MB.
2514 ///
2515 /// Smallest option. Used on tvOS (Apple TV) where the memory limit is 2 GB.
2516 pub fn qwen25_0_5b() -> Self {
2517 Self {
2518 model_id: super::models::BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF.into(),
2519 files: vec![super::models::QWEN25_0_5B_GGUF_FILE.into()],
2520 tok_model_id: if cfg!(target_os = "android") {
2521 Some(super::models::QWEN25_0_5B_TOK_MODEL_ID.into())
2522 } else {
2523 None
2524 },
2525 display_name: "Qwen 2.5 0.5B".into(),
2526 approx_memory: "~380 MB (GGUF Q4_K_M)".into(),
2527 chat_template: None,
2528 }
2529 }
2530
2531 /// Qwen 2.5 1.5B Instruct (GGUF Q4_K_M) — ~941 MB.
2532 ///
2533 /// Lightest option, ideal for iOS and memory-constrained Android devices.
2534 pub fn qwen25_1_5b() -> Self {
2535 Self {
2536 model_id: super::models::BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF.into(),
2537 files: vec![super::models::QWEN25_1_5B_GGUF_FILE.into()],
2538 tok_model_id: if cfg!(target_os = "android") {
2539 Some(super::models::QWEN25_1_5B_TOK_MODEL_ID.into())
2540 } else {
2541 None
2542 },
2543 display_name: "Qwen 2.5 1.5B".into(),
2544 approx_memory: "~941 MB (GGUF Q4_K_M)".into(),
2545 chat_template: None,
2546 }
2547 }
2548
2549 /// Qwen 2.5 3B Instruct (GGUF Q4_K_M) — ~1.93 GB.
2550 ///
2551 /// Balanced quality/size for macOS desktops and high-RAM Android devices.
2552 /// Not recommended for iOS (OOM on 8 GB devices).
2553 pub fn qwen25_3b() -> Self {
2554 Self {
2555 model_id: super::models::BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF.into(),
2556 files: vec![super::models::QWEN25_3B_GGUF_FILE.into()],
2557 tok_model_id: if cfg!(target_os = "android") {
2558 Some(super::models::QWEN25_3B_TOK_MODEL_ID.into())
2559 } else {
2560 None
2561 },
2562 display_name: "Qwen 2.5 3B".into(),
2563 approx_memory: "~1.93 GB (GGUF Q4_K_M)".into(),
2564 chat_template: None,
2565 }
2566 }
2567
2568 /// Qwen 2.5 Coder 1.5B Instruct (GGUF Q4_K_M) — ~941 MB.
2569 ///
2570 /// Dedicated coding model fine-tuned on 5.5T tokens of code and math data.
2571 /// Same `qwen2` GGUF architecture as the general 1.5B — loads via the
2572 /// existing `quantized_qwen.rs` path without any mistral.rs changes.
2573 /// Same memory footprint as the general 1.5B but dramatically better at
2574 /// code generation, explanation, and debugging.
2575 ///
2576 /// Preferred default for iOS, Android, and any coding-focused deployment.
2577 pub fn qwen25_coder_1_5b() -> Self {
2578 Self {
2579 model_id: super::models::BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF.into(),
2580 files: vec![super::models::QWEN25_CODER_1_5B_GGUF_FILE.into()],
2581 tok_model_id: if cfg!(target_os = "android") {
2582 Some(super::models::QWEN25_CODER_1_5B_TOK_MODEL_ID.into())
2583 } else {
2584 None
2585 },
2586 display_name: "Qwen 2.5 Coder 1.5B".into(),
2587 approx_memory: "~941 MB (GGUF Q4_K_M)".into(),
2588 chat_template: None,
2589 }
2590 }
2591
2592 /// Qwen 2.5 Coder 3B Instruct (GGUF Q4_K_M) — ~1.93 GB.
2593 ///
2594 /// Best on-device coding quality for macOS desktop. Fine-tuned on 5.5T
2595 /// tokens of code and math. Same `qwen2` architecture as the general 3B.
2596 /// Not recommended for iOS due to memory constraints.
2597 pub fn qwen25_coder_3b() -> Self {
2598 Self {
2599 model_id: super::models::BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF.into(),
2600 files: vec![super::models::QWEN25_CODER_3B_GGUF_FILE.into()],
2601 tok_model_id: if cfg!(target_os = "android") {
2602 Some(super::models::QWEN25_CODER_3B_TOK_MODEL_ID.into())
2603 } else {
2604 None
2605 },
2606 display_name: "Qwen 2.5 Coder 3B".into(),
2607 approx_memory: "~1.93 GB (GGUF Q4_K_M)".into(),
2608 chat_template: None,
2609 }
2610 }
2611
2612 /// Qwen 2.5 Coder 7B Instruct (GGUF Q4_K_M) — ~4.4 GB.
2613 ///
2614 /// Strong coding model with tool/function calling support via the Qwen2.5
2615 /// chat template. Uses the `qwen2` architecture. Requires 8+ GB RAM.
2616 pub fn qwen25_coder_7b() -> Self {
2617 Self {
2618 model_id: super::models::BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF.into(),
2619 files: vec![super::models::QWEN25_CODER_7B_GGUF_FILE.into()],
2620 tok_model_id: if cfg!(target_os = "android") {
2621 Some(super::models::QWEN25_CODER_7B_TOK_MODEL_ID.into())
2622 } else {
2623 None
2624 },
2625 display_name: "Qwen 2.5 Coder 7B (Q4_K_M)".into(),
2626 approx_memory: "~4.4 GB".into(),
2627 chat_template: None,
2628 }
2629 }
2630
2631 /// Qwen 3 0.6B (GGUF Q4_K_M) — smallest Qwen 3 variant, ~0.5 GB.
2632 ///
2633 /// Lightest tool-capable Qwen 3 model; fits tvOS and the most
2634 /// memory-constrained mobile devices. Extended thinking mode — load with
2635 /// `max_tokens ≥ 4096`.
2636 pub fn qwen3_0_6b() -> Self {
2637 Self {
2638 model_id: super::models::BARTOWSKI_QWEN3_0_6B_GGUF.into(),
2639 files: vec![super::models::QWEN3_0_6B_GGUF_FILE.into()],
2640 tok_model_id: if cfg!(target_os = "android") {
2641 Some(super::models::QWEN3_0_6B_TOK_MODEL_ID.into())
2642 } else {
2643 None
2644 },
2645 display_name: "Qwen 3 0.6B (Q4_K_M)".into(),
2646 approx_memory: "~0.5 GB".into(),
2647 chat_template: None,
2648 }
2649 }
2650
2651 /// Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling, ~1.3 GB.
2652 pub fn qwen3_1_7b() -> Self {
2653 Self {
2654 model_id: super::models::BARTOWSKI_QWEN3_1_7B_GGUF.into(),
2655 files: vec![super::models::QWEN3_1_7B_GGUF_FILE.into()],
2656 tok_model_id: if cfg!(target_os = "android") {
2657 Some(super::models::QWEN3_1_7B_TOK_MODEL_ID.into())
2658 } else {
2659 None
2660 },
2661 display_name: "Qwen 3 1.7B (Q4_K_M)".into(),
2662 approx_memory: "~1.3 GB".into(),
2663 chat_template: None,
2664 }
2665 }
2666
2667 /// Qwen 3 4B (GGUF Q4_K_M) — ~2.7 GB.
2668 ///
2669 /// Full OpenAI-compatible tool calling with extended thinking mode.
2670 /// Always load with `max_tokens ≥ 4096`; the `<think>…</think>` block can
2671 /// consume 300–400 tokens before the real response begins.
2672 pub fn qwen3_4b() -> Self {
2673 Self {
2674 model_id: super::models::BARTOWSKI_QWEN3_4B_GGUF.into(),
2675 files: vec![super::models::QWEN3_4B_GGUF_FILE.into()],
2676 tok_model_id: if cfg!(target_os = "android") {
2677 Some(super::models::QWEN3_4B_TOK_MODEL_ID.into())
2678 } else {
2679 None
2680 },
2681 display_name: "Qwen 3 4B (Q4_K_M)".into(),
2682 approx_memory: "~2.7 GB".into(),
2683 chat_template: None,
2684 }
2685 }
2686
2687 /// Strong tool-calling model with extended thinking. Best balance of
2688 /// quality and memory for macOS with 24+ GB RAM.
2689 pub fn qwen3_8b() -> Self {
2690 Self {
2691 model_id: super::models::BARTOWSKI_QWEN3_8B_GGUF.into(),
2692 files: vec![super::models::QWEN3_8B_GGUF_FILE.into()],
2693 tok_model_id: if cfg!(target_os = "android") {
2694 Some(super::models::QWEN3_8B_TOK_MODEL_ID.into())
2695 } else {
2696 None
2697 },
2698 display_name: "Qwen 3 8B (Q4_K_M)".into(),
2699 approx_memory: "~5 GB".into(),
2700 chat_template: None,
2701 }
2702 }
2703
2704 /// Qwen 3 14B (GGUF Q4_K_M) — ~8.4 GB.
2705 ///
2706 /// Strong reasoning and tool-calling model with extended thinking.
2707 /// Best all-around model for macOS with 16+ GB RAM.
2708 pub fn qwen3_14b() -> Self {
2709 Self {
2710 model_id: super::models::BARTOWSKI_QWEN3_14B_GGUF.into(),
2711 files: vec![super::models::QWEN3_14B_GGUF_FILE.into()],
2712 tok_model_id: if cfg!(target_os = "android") {
2713 Some(super::models::QWEN3_14B_TOK_MODEL_ID.into())
2714 } else {
2715 None
2716 },
2717 display_name: "Qwen 3 14B (Q4_K_M)".into(),
2718 approx_memory: "~8.4 GB".into(),
2719 chat_template: None,
2720 }
2721 }
2722
2723 /// Qwen 3 32B (GGUF Q4_K_M) — largest dense Qwen 3, ~19.8 GB.
2724 ///
2725 /// Highest-quality dense Qwen 3 variant. Requires a high-memory desktop
2726 /// (32+ GB RAM / unified memory). Extended thinking and full tool calling.
2727 pub fn qwen3_32b() -> Self {
2728 Self {
2729 model_id: super::models::BARTOWSKI_QWEN3_32B_GGUF.into(),
2730 files: vec![super::models::QWEN3_32B_GGUF_FILE.into()],
2731 tok_model_id: if cfg!(target_os = "android") {
2732 Some(super::models::QWEN3_32B_TOK_MODEL_ID.into())
2733 } else {
2734 None
2735 },
2736 display_name: "Qwen 3 32B (Q4_K_M)".into(),
2737 approx_memory: "~19.8 GB".into(),
2738 chat_template: None,
2739 }
2740 }
2741
2742 /// Qwen 3 4B Instruct 2507 (GGUF Q4_K_M) — latest non-thinking 4B, ~2.5 GB.
2743 ///
2744 /// Updated instruction-tuned checkpoint that does *not* emit a `<think>`
2745 /// block, making it faster and more predictable for chat and tool use.
2746 /// Recommended general-purpose Qwen 3 model.
2747 pub fn qwen3_4b_instruct_2507() -> Self {
2748 Self {
2749 model_id: super::models::BARTOWSKI_QWEN3_4B_INSTRUCT_2507_GGUF.into(),
2750 files: vec![super::models::QWEN3_4B_INSTRUCT_2507_GGUF_FILE.into()],
2751 tok_model_id: if cfg!(target_os = "android") {
2752 Some(super::models::QWEN3_4B_INSTRUCT_2507_TOK_MODEL_ID.into())
2753 } else {
2754 None
2755 },
2756 display_name: "Qwen 3 4B Instruct 2507 (Q4_K_M)".into(),
2757 approx_memory: "~2.5 GB".into(),
2758 chat_template: None,
2759 }
2760 }
2761
2762 /// Qwen 3 4B Thinking 2507 (GGUF Q4_K_M) — latest reasoning 4B, ~2.5 GB.
2763 ///
2764 /// Updated thinking-only checkpoint with stronger reasoning and tool-use
2765 /// accuracy. Always emits a `<think>` block — load with `max_tokens ≥ 4096`.
2766 pub fn qwen3_4b_thinking_2507() -> Self {
2767 Self {
2768 model_id: super::models::BARTOWSKI_QWEN3_4B_THINKING_2507_GGUF.into(),
2769 files: vec![super::models::QWEN3_4B_THINKING_2507_GGUF_FILE.into()],
2770 tok_model_id: if cfg!(target_os = "android") {
2771 Some(super::models::QWEN3_4B_THINKING_2507_TOK_MODEL_ID.into())
2772 } else {
2773 None
2774 },
2775 display_name: "Qwen 3 4B Thinking 2507 (Q4_K_M)".into(),
2776 approx_memory: "~2.5 GB".into(),
2777 chat_template: None,
2778 }
2779 }
2780
2781 /// Qwen 3 30B-A3B Instruct 2507 (GGUF Q4_K_M) — flagship MoE, ~18.6 GB.
2782 ///
2783 /// Mixture-of-experts model: 30B total parameters, ~3B active per token, so
2784 /// inference is far cheaper than a 30B dense model while quality rivals it.
2785 /// Loads via the `qwen3moe` GGUF architecture. Requires 32+ GB RAM.
2786 pub fn qwen3_30b_a3b_instruct_2507() -> Self {
2787 Self {
2788 model_id: super::models::BARTOWSKI_QWEN3_30B_A3B_INSTRUCT_2507_GGUF.into(),
2789 files: vec![super::models::QWEN3_30B_A3B_INSTRUCT_2507_GGUF_FILE.into()],
2790 tok_model_id: if cfg!(target_os = "android") {
2791 Some(super::models::QWEN3_30B_A3B_INSTRUCT_2507_TOK_MODEL_ID.into())
2792 } else {
2793 None
2794 },
2795 display_name: "Qwen 3 30B-A3B Instruct 2507 (Q4_K_M)".into(),
2796 approx_memory: "~18.6 GB".into(),
2797 chat_template: None,
2798 }
2799 }
2800
2801 /// DeepSeek Coder v1 6.7B Instruct (GGUF Q4_K_M) — ~3.8 GB.
2802 ///
2803 /// Strong code generation model using the `llama` GGUF architecture.
2804 /// Requires 8+ GB RAM; recommended for macOS desktops and Linux.
2805 pub fn deepseek_coder_6_7b() -> Self {
2806 Self {
2807 model_id: super::models::THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF.into(),
2808 files: vec![super::models::DEEPSEEK_CODER_6_7B_GGUF_FILE.into()],
2809 tok_model_id: if cfg!(target_os = "android") {
2810 Some(super::models::DEEPSEEK_CODER_6_7B_TOK_MODEL_ID.into())
2811 } else {
2812 None
2813 },
2814 display_name: "DeepSeek Coder 6.7B (Q4_K_M)".into(),
2815 approx_memory: "~3.8 GB".into(),
2816 chat_template: Some(DEEPSEEK_CODER_CHAT_TEMPLATE.into()),
2817 }
2818 }
2819
2820 /// Return the platform-appropriate default **coding** model config.
2821 ///
2822 /// - iOS / Android → Qwen 2.5 Coder 1.5B (~941 MB, fits mobile memory budgets)
2823 /// - macOS / Windows / Linux → Qwen 2.5 Coder 3B (~1.93 GB, best desktop quality)
2824 ///
2825 /// Both are dedicated coding models (trained on 5.5T code + math tokens)
2826 /// and use the `qwen2` GGUF architecture supported by mistral.rs.
2827 pub fn platform_default() -> Self {
2828 if cfg!(target_os = "tvos") {
2829 Self::qwen25_0_5b()
2830 } else if cfg!(any(
2831 target_os = "ios",
2832 target_os = "visionos",
2833 target_os = "watchos",
2834 target_os = "android"
2835 )) {
2836 Self::qwen25_coder_1_5b()
2837 } else {
2838 Self::qwen25_coder_3b()
2839 }
2840 }
2841 }
2842
2843 // ═════════════════════════════════════════════════════════════════════════════
2844 // Tests
2845 // ═════════════════════════════════════════════════════════════════════════════
2846
2847 #[cfg(test)]
2848 mod tests {
2849 use super::*;
2850
2851 #[test]
2852 fn truncate_for_log_short() {
2853 assert_eq!(truncate_for_log("hello", 10), "hello");
2854 }
2855
2856 #[test]
2857 fn truncate_for_log_long() {
2858 let long = "a".repeat(200);
2859 let result = truncate_for_log(&long, 50);
2860 assert!(result.ends_with("..."));
2861 assert_eq!(result.len(), 53); // 50 chars + "..."
2862 }
2863
2864 #[test]
2865 fn truncate_for_log_unicode_boundaries() {
2866 for character in ['é', '—', '界', '🦀'] {
2867 let input = format!("{}{}tail", "a".repeat(99), character);
2868 for limit in 99..99 + character.len_utf8() {
2869 assert_eq!(
2870 truncate_for_log(&input, limit),
2871 format!("{}...", "a".repeat(99))
2872 );
2873 }
2874 let boundary = 99 + character.len_utf8();
2875 assert_eq!(
2876 truncate_for_log(&input, boundary),
2877 format!("{}{}...", "a".repeat(99), character)
2878 );
2879 }
2880 }
2881
2882 #[test]
2883 fn truncate_for_log_empty_and_zero_limit() {
2884 assert_eq!(truncate_for_log("", 0), "");
2885 assert_eq!(truncate_for_log("—", 0), "...");
2886 assert_eq!(truncate_for_log("—", 1), "...");
2887 assert_eq!(truncate_for_log("—", 3), "—");
2888 assert_eq!(truncate_for_log("hello", 5), "hello");
2889 }
2890
2891 #[test]
2892 fn gguf_model_config_qwen25_1_5b() {
2893 let cfg = GgufModelConfig::qwen25_1_5b();
2894 assert!(cfg.model_id.contains("1.5B"));
2895 assert_eq!(cfg.files.len(), 1);
2896 }
2897
2898 #[test]
2899 fn gguf_model_config_qwen25_3b() {
2900 let cfg = GgufModelConfig::qwen25_3b();
2901 assert!(cfg.model_id.contains("3B"));
2902 assert_eq!(cfg.files.len(), 1);
2903 }
2904
2905 #[test]
2906 fn gguf_model_config_qwen3_variants() {
2907 // Every Qwen 3 constructor must produce a valid, single-file config
2908 // whose repo ID is registered in the supported-models list.
2909 let configs = [
2910 GgufModelConfig::qwen3_0_6b(),
2911 GgufModelConfig::qwen3_1_7b(),
2912 GgufModelConfig::qwen3_4b(),
2913 GgufModelConfig::qwen3_8b(),
2914 GgufModelConfig::qwen3_14b(),
2915 GgufModelConfig::qwen3_32b(),
2916 GgufModelConfig::qwen3_4b_instruct_2507(),
2917 GgufModelConfig::qwen3_4b_thinking_2507(),
2918 GgufModelConfig::qwen3_30b_a3b_instruct_2507(),
2919 ];
2920 for cfg in &configs {
2921 assert!(
2922 cfg.model_id.contains("Qwen3"),
2923 "unexpected id: {}",
2924 cfg.model_id
2925 );
2926 assert_eq!(cfg.files.len(), 1);
2927 assert!(cfg.files[0].ends_with("-Q4_K_M.gguf"));
2928 assert!(
2929 super::super::models::SUPPORTED_MODELS.contains(&cfg.model_id.as_str()),
2930 "{} missing from SUPPORTED_MODELS",
2931 cfg.model_id
2932 );
2933 }
2934 }
2935
2936 #[test]
2937 fn every_supported_model_resolves_to_download_config() {
2938 for model_id in super::super::models::SUPPORTED_MODELS {
2939 let config = GgufModelConfig::from_supported_model_id(model_id)
2940 .unwrap_or_else(|| panic!("missing config for {model_id}"));
2941 assert_eq!(config.model_id, *model_id);
2942 assert_eq!(config.files.len(), 1);
2943 }
2944 }
2945
2946 #[test]
2947 fn qwen3_thinking_models_request_extended_sampling() {
2948 assert!(GgufModelConfig::qwen3_4b_thinking_2507().uses_extended_thinking());
2949 assert!(!GgufModelConfig::qwen3_4b_instruct_2507().uses_extended_thinking());
2950 }
2951
2952 #[test]
2953 fn gguf_model_config_platform_default() {
2954 let cfg = GgufModelConfig::platform_default();
2955 // On the test host (macOS or Linux), this should be the 3B model.
2956 // On iOS/Android CI it would be 1.5B. We just check it's valid.
2957 assert!(!cfg.model_id.is_empty());
2958 assert!(!cfg.files.is_empty());
2959 }
2960
2961 #[tokio::test]
2962 async fn engine_new_is_unloaded() {
2963 let engine = ChatEngine::new();
2964 assert!(!engine.is_loaded().await);
2965 let info = engine.info().await;
2966 assert_eq!(info.status, EngineStatus::Unloaded);
2967 assert_eq!(info.history_length, 0);
2968 }
2969
2970 #[tokio::test]
2971 async fn engine_send_without_model_errors() {
2972 let engine = ChatEngine::new();
2973 let result = engine.send_message("hello").await;
2974 assert!(result.is_err());
2975 match result.unwrap_err() {
2976 InferenceError::NoModelLoaded => {} // expected
2977 other => panic!("Expected NoModelLoaded, got: {:?}", other),
2978 }
2979 }
2980
2981 #[tokio::test]
2982 async fn engine_load_uqff_without_files_errors() {
2983 let engine = ChatEngine::new();
2984 let result = engine
2985 .load_uqff_model(
2986 UqffModelConfig {
2987 model_id: "mistralrs-community/gemma-4-E4B-it-UQFF".into(),
2988 files: Vec::new(),
2989 display_name: "Gemma 4 E4B (UQFF Q4K)".into(),
2990 approx_memory: "~2.5 GB (UQFF Q4K)".into(),
2991 chat_template: None,
2992 },
2993 None,
2994 None,
2995 )
2996 .await;
2997
2998 match result.unwrap_err() {
2999 InferenceError::ModelBuild { reason } => {
3000 assert!(reason.contains("at least one UQFF file"));
3001 }
3002 other => panic!("Expected ModelBuild, got: {:?}", other),
3003 }
3004 }
3005
3006 #[tokio::test]
3007 async fn engine_history_empty_when_no_model() {
3008 let engine = ChatEngine::new();
3009 assert!(engine.history().await.is_empty());
3010 assert_eq!(engine.clear_history().await, 0);
3011 }
3012
3013 #[tokio::test]
3014 async fn engine_unload_when_none() {
3015 let engine = ChatEngine::new();
3016 assert!(engine.unload_model().await.is_none());
3017 }
3018
3019 #[tokio::test]
3020 async fn complete_without_a_model() {
3021 let engine = ChatEngine::new();
3022 let err = engine
3023 .complete(vec![ChatTurn::User("hi".into())], &[], None)
3024 .await
3025 .unwrap_err();
3026 assert!(matches!(err, InferenceError::NoModelLoaded), "{err:?}");
3027 }
3028
3029 #[test]
3030 fn split_think_separates_reasoning() {
3031 assert_eq!(
3032 split_think("<think>\nC major has no sharps.\n</think>\n\nIt has none."),
3033 (
3034 "It has none.".to_string(),
3035 Some("C major has no sharps.".to_string())
3036 )
3037 );
3038 assert_eq!(
3039 split_think(" Plain reply. "),
3040 ("Plain reply.".to_string(), None)
3041 );
3042 assert_eq!(
3043 split_think("<think>\n\n</think>Reply"),
3044 ("Reply".to_string(), None)
3045 );
3046 // Out of tokens mid-thought: no reply yet.
3047 assert_eq!(
3048 split_think("<think>still going"),
3049 (String::new(), Some("still going".to_string()))
3050 );
3051 // Only a leading block counts.
3052 assert_eq!(
3053 split_think("Use <think> tags."),
3054 ("Use <think> tags.".to_string(), None)
3055 );
3056 }
3057
3058 #[test]
3059 fn chat_turns_become_history_entries() {
3060 let call = ToolCallRequest {
3061 id: "call_1".into(),
3062 function_name: "theory_chord".into(),
3063 arguments: r#"{"symbol":"Cmaj7"}"#.into(),
3064 };
3065 let entries: Vec<HistoryEntry> = vec![
3066 ChatTurn::System("You are helpful.".into()),
3067 ChatTurn::User("Spell Cmaj7".into()),
3068 ChatTurn::Assistant {
3069 content: String::new(),
3070 tool_calls: vec![call.clone()],
3071 },
3072 ChatTurn::Tool {
3073 tool_call_id: "call_1".into(),
3074 content: "C E G B".into(),
3075 },
3076 ChatTurn::Assistant {
3077 content: "C, E, G, B.".into(),
3078 tool_calls: vec![],
3079 },
3080 ]
3081 .into_iter()
3082 .map(HistoryEntry::from)
3083 .collect();
3084
3085 assert!(
3086 matches!(&entries[0], HistoryEntry::Text(m) if m.role == ChatRole::System && m.content == "You are helpful.")
3087 );
3088 assert!(matches!(&entries[1], HistoryEntry::Text(m) if m.role == ChatRole::User));
3089 assert!(
3090 matches!(&entries[2], HistoryEntry::AssistantToolCall { tool_calls, .. } if tool_calls == &vec![call])
3091 );
3092 assert!(
3093 matches!(&entries[3], HistoryEntry::ToolResult { tool_call_id, content } if tool_call_id == "call_1" && content == "C E G B")
3094 );
3095 // A plain assistant reply stays a text message, as the engine records it.
3096 assert!(
3097 matches!(&entries[4], HistoryEntry::Text(m) if m.role == ChatRole::Assistant && m.content == "C, E, G, B.")
3098 );
3099 }
3100 }