@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
copilot/assess-inference-engine-complexity
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
rs 2,151 lines 78.4 KB
Raw
1 //! On-device LLM chat inference engine powered by [mistral.rs](https://github.com/EricLBuehler/mistral.rs).
2 //!
3 //! `ChatEngine` provides a high-level, framework-agnostic API for:
4 //!
5 //! - Loading GGUF-quantized models (the primary format for mobile/desktop)
6 //! - Multi-turn chat with conversation history management
7 //! - Both blocking (non-streaming) and streaming inference
8 //! - Model lifecycle management (load / unload / status)
9 //!
10 //! # Architecture
11 //!
12 //! ```text
13 //! ┌──────────────────────────────────────────┐
14 //! │ App / UniFFI binding / test │
15 //! └──────────────┬───────────────────────────┘
16 //! │ (framework-agnostic API)
17 //! ▼
18 //! ┌──────────────────────────────────────────┐
19 //! │ ChatEngine │
20 //! │ ┌────────────────────────────────────┐ │
21 //! │ │ Mutex<Option<LoadedModel>> │ │
22 //! │ │ · Arc<mistralrs::Model> │ │
23 //! │ │ · config (GgufModelConfig) │ │
24 //! │ │ · history (Vec<ChatMessage>) │ │
25 //! │ │ · sampling (SamplingConfig) │ │
26 //! │ └────────────────────────────────────┘ │
27 //! └──────────────┬───────────────────────────┘
28 //! │ (delegates to)
29 //! ▼
30 //! ┌──────────────────────────────────────────┐
31 //! │ mistralrs::Model │
32 //! │ (wraps Arc<MistralRs> engine thread) │
33 //! └──────────────────────────────────────────┘
34 //! ```
35 //!
36 //! # Example
37 //!
38 //! ```rust,ignore
39 //! use onde::inference::engine::ChatEngine;
40 //! use onde::inference::types::*;
41 //!
42 //! let engine = ChatEngine::new();
43 //!
44 //! let config = GgufModelConfig {
45 //! model_id: "bartowski/Qwen2.5-1.5B-Instruct-GGUF".into(),
46 //! files: vec!["Qwen2.5-1.5B-Instruct-Q4_K_M.gguf".into()],
47 //! tok_model_id: None,
48 //! display_name: "Qwen 2.5 1.5B".into(),
49 //! approx_memory: "~941 MB".into(),
50 //! chat_template: None,
51 //! };
52 //!
53 //! engine.load_gguf_model(config, None).await?;
54 //! engine.set_system_prompt("You are a helpful assistant.");
55 //!
56 //! let result = engine.send_message("Hello!").await?;
57 //! println!("{}", result.text);
58 //! ```
59
60 #[cfg(any(
61 target_os = "macos",
62 target_os = "ios",
63 target_os = "tvos",
64 target_os = "visionos",
65 target_os = "watchos",
66 target_os = "windows",
67 target_os = "linux",
68 target_os = "android"
69 ))]
70 use std::sync::Arc;
71
72 #[cfg(any(
73 target_os = "macos",
74 target_os = "ios",
75 target_os = "tvos",
76 target_os = "visionos",
77 target_os = "watchos",
78 target_os = "windows",
79 target_os = "linux",
80 target_os = "android"
81 ))]
82 use tokio::sync::Mutex;
83
84 use super::types::*;
85
86 // ── Platform-gated mistralrs imports ─────────────────────────────────────────
87
88 #[cfg(any(
89 target_os = "macos",
90 target_os = "ios",
91 target_os = "tvos",
92 target_os = "visionos",
93 target_os = "watchos",
94 target_os = "windows",
95 target_os = "linux",
96 target_os = "android"
97 ))]
98 use mistralrs::{
99 CalledFunction, Function, GgufModelBuilder, Model, RequestBuilder, TextMessageRole,
100 Tool as MistralTool, ToolCallResponse, ToolCallType, ToolChoice, ToolType,
101 };
102
103 #[cfg(any(
104 target_os = "macos",
105 target_os = "ios",
106 target_os = "tvos",
107 target_os = "visionos",
108 target_os = "watchos",
109 target_os = "windows",
110 target_os = "linux",
111 target_os = "android"
112 ))]
113 use std::collections::HashMap;
114
115 // ISQ types are only used by load_isq_model, which is macOS-only.
116 #[cfg(target_os = "macos")]
117 use mistralrs::{IsqBits, TextModelBuilder};
118
119 // ── Internals ────────────────────────────────────────────────────────────────
120
121 /// How a model was loaded — carries the data needed for status reporting.
122 #[cfg(any(
123 target_os = "macos",
124 target_os = "ios",
125 target_os = "tvos",
126 target_os = "visionos",
127 target_os = "watchos",
128 target_os = "windows",
129 target_os = "linux",
130 target_os = "android"
131 ))]
132 enum LoadedModelConfig {
133 Gguf(GgufModelConfig),
134 #[cfg(target_os = "macos")]
135 Isq(IsqModelConfig),
136 }
137
138 #[cfg(any(
139 target_os = "macos",
140 target_os = "ios",
141 target_os = "tvos",
142 target_os = "visionos",
143 target_os = "watchos",
144 target_os = "windows",
145 target_os = "linux",
146 target_os = "android"
147 ))]
148 impl LoadedModelConfig {
149 fn display_name(&self) -> &str {
150 match self {
151 LoadedModelConfig::Gguf(c) => &c.display_name,
152 #[cfg(target_os = "macos")]
153 LoadedModelConfig::Isq(c) => &c.display_name,
154 }
155 }
156
157 fn approx_memory(&self) -> &str {
158 match self {
159 LoadedModelConfig::Gguf(c) => &c.approx_memory,
160 #[cfg(target_os = "macos")]
161 LoadedModelConfig::Isq(c) => &c.approx_memory,
162 }
163 }
164 }
165
166 /// State of a loaded model held inside the engine's mutex.
167 #[cfg(any(
168 target_os = "macos",
169 target_os = "ios",
170 target_os = "tvos",
171 target_os = "visionos",
172 target_os = "watchos",
173 target_os = "windows",
174 target_os = "linux",
175 target_os = "android"
176 ))]
177 struct LoadedModel {
178 /// The mistral.rs model handle. `Model` wraps `Arc<MistralRs>` so
179 /// cloning is a cheap pointer copy — safe to snapshot before inference
180 /// while releasing the mutex.
181 model: Arc<Model>,
182 /// Configuration used to load this model (kept for status reporting).
183 config: LoadedModelConfig,
184 /// Conversation history (system prompt is stored separately).
185 history: Vec<HistoryEntry>,
186 /// System prompt prepended to every request.
187 system_prompt: Option<String>,
188 /// Sampling parameters for generation.
189 sampling: SamplingConfig,
190 }
191
192 /// An entry in the conversation history (internal only).
193 /// Supports tool-related messages alongside regular text.
194 #[cfg(any(
195 target_os = "macos",
196 target_os = "ios",
197 target_os = "tvos",
198 target_os = "visionos",
199 target_os = "watchos",
200 target_os = "windows",
201 target_os = "linux",
202 target_os = "android"
203 ))]
204 #[derive(Debug, Clone)]
205 enum HistoryEntry {
206 /// A regular text message (user, assistant, system).
207 Text(ChatMessage),
208 /// An assistant response that includes tool calls.
209 AssistantToolCall {
210 content: String,
211 tool_calls: Vec<ToolCallRequest>,
212 },
213 /// A tool execution result.
214 ToolResult {
215 tool_call_id: String,
216 content: String,
217 },
218 }
219
220 // ═════════════════════════════════════════════════════════════════════════════
221 // ChatEngine — the public API
222 // ═════════════════════════════════════════════════════════════════════════════
223
224 /// A reusable on-device LLM chat inference engine.
225 ///
226 /// Thread-safe (`Send + Sync`) — safe to store in a `once_cell::sync::Lazy`,
227 /// `tokio::sync::OnceCell`, or shared application state.
228 ///
229 /// All mutating operations acquire an internal `tokio::sync::Mutex`. The
230 /// mutex is released *before* the actual (potentially slow) model inference
231 /// runs, so other tasks can still query status or history while generation
232 /// is in progress.
233 #[cfg(any(
234 target_os = "macos",
235 target_os = "ios",
236 target_os = "tvos",
237 target_os = "visionos",
238 target_os = "watchos",
239 target_os = "windows",
240 target_os = "linux",
241 target_os = "android"
242 ))]
243 pub struct ChatEngine {
244 inner: Mutex<Option<LoadedModel>>,
245 pulse: Option<crate::pulse::PulseClient>,
246 }
247
248 #[cfg(any(
249 target_os = "macos",
250 target_os = "ios",
251 target_os = "tvos",
252 target_os = "visionos",
253 target_os = "watchos",
254 target_os = "windows",
255 target_os = "linux",
256 target_os = "android"
257 ))]
258 impl ChatEngine {
259 // ── Construction ─────────────────────────────────────────────────────
260
261 /// Create a new engine with no model loaded.
262 pub fn new() -> Self {
263 let environment = Self::pulse_environment();
264 let edge_id = std::env::var("ONDE_EDGE_ID").unwrap_or_else(|_| "onde-unknown".to_string());
265 let pulse = crate::pulse::PulseClient::new(environment, edge_id);
266
267 match &pulse {
268 Some(_) => {
269 log::info!("ChatEngine: pulse telemetry enabled (environment={environment})")
270 }
271 None => log::info!(
272 "ChatEngine: pulse telemetry disabled \
273 (GRESIQ_API_KEY / GRESIQ_API_SECRET not embedded at SDK build time)"
274 ),
275 }
276
277 Self {
278 inner: Mutex::new(None),
279 pulse,
280 }
281 }
282
283 /// Resolve the pulse environment from the `GRESIQ_ENVIRONMENT` env var.
284 /// Defaults to `Production` when the var is absent or unrecognised.
285 fn pulse_environment() -> smbcloud_gresiq_sdk::Environment {
286 match std::env::var("GRESIQ_ENVIRONMENT").as_deref() {
287 Ok("dev") => smbcloud_gresiq_sdk::Environment::Dev,
288 _ => smbcloud_gresiq_sdk::Environment::Production,
289 }
290 }
291
292 // ── Model lifecycle ──────────────────────────────────────────────────
293
294 /// Load a GGUF model into the engine.
295 ///
296 /// If a model is already loaded it will be unloaded first (the previous
297 /// `Model` is dropped, which terminates its engine thread).
298 ///
299 /// # Arguments
300 ///
301 /// * `config` — Which model to load (repo ID, filename, etc.).
302 /// * `system_prompt` — Optional system prompt to prepend to every request.
303 /// * `sampling` — Sampling parameters; pass `None` for platform-aware
304 /// defaults (mobile gets [`SamplingConfig::mobile`],
305 /// desktop gets [`SamplingConfig::default`]).
306 ///
307 /// # Errors
308 ///
309 /// Returns [`InferenceError::ModelBuild`] if the model fails to download
310 /// or load.
311 pub async fn load_gguf_model(
312 &self,
313 config: GgufModelConfig,
314 system_prompt: Option<String>,
315 sampling: Option<SamplingConfig>,
316 ) -> Result<std::time::Duration, InferenceError> {
317 use log::info;
318 use std::time::Instant;
319
320 info!(
321 "ChatEngine: loading GGUF model {} (files: {:?})",
322 config.model_id, config.files
323 );
324
325 // ── Sandboxed platforms: seed GLOBAL_HF_CACHE ────────────────────
326 //
327 // On sandboxed platforms the default `~/.cache/huggingface/hub` path
328 // is either inaccessible or non-existent:
329 // - Android: `dirs::home_dir()` returns `None` → `Cache::default()` panics.
330 // - iOS/tvOS: `~/.cache` is outside the app container → os error 1.
331 //
332 // The `get_paths_gguf!` macro in mistralrs falls back to
333 // `Cache::default()` when `GLOBAL_HF_CACHE` (a OnceLock) is empty.
334 //
335 // `HF_HOME` must be set by the host app (via `configure_cache_dir`
336 // or `download_model(app_data_dir:)`) before any model load.
337 // `get_or_init` is a no-op if already seeded — safe to call repeatedly.
338 #[cfg(any(
339 target_os = "android",
340 target_os = "ios",
341 target_os = "tvos",
342 target_os = "visionos",
343 target_os = "watchos"
344 ))]
345 {
346 let hf_home = std::env::var("HF_HOME")
347 .map(std::path::PathBuf::from)
348 .map_err(|_| InferenceError::ModelBuild {
349 reason: "HF_HOME is not set — cannot initialise HF cache. \
350 On iOS/tvOS/Android, call configure_cache_dir() or \
351 download_model(app_data_dir:) before load_gguf_model()."
352 .to_string(),
353 })?;
354 let hf_hub_cache = hf_home.join("hub");
355 if let Err(e) = std::fs::create_dir_all(&hf_hub_cache) {
356 log::warn!(
357 "ChatEngine: could not create HF hub cache dir {}: {}",
358 hf_hub_cache.display(),
359 e
360 );
361 }
362 mistralrs_core::GLOBAL_HF_CACHE
363 .get_or_init(|| hf_hub::Cache::new(hf_hub_cache.clone()));
364 std::env::set_var("HF_HUB_CACHE", &hf_hub_cache);
365 log::debug!(
366 "ChatEngine: GLOBAL_HF_CACHE seeded at {}",
367 hf_hub_cache.display()
368 );
369 }
370
371 // Clean up stale HF cache artefacts before loading.
372 crate::hf_cache::clean_stale_lock_files(&config.model_id);
373 crate::hf_cache::repair_hf_cache_symlinks(&config.model_id);
374
375 let start = Instant::now();
376
377 let mut builder = GgufModelBuilder::new(&config.model_id, config.files.clone())
378 .with_token_source(super::token::hf_token_source())
379 .with_logging();
380
381 // On Android the GGUF embedded tokenizer is not supported by the
382 // candle backend — an explicit tok_model_id is required.
383 if let Some(ref tok_id) = config.tok_model_id {
384 builder = builder.with_tok_model_id(tok_id);
385 }
386
387 // Some older GGUF files (e.g. TheBloke) do not embed a chat template.
388 // When the config provides one, write it to a temporary .jinja file
389 // and pass the path — mistral.rs only accepts file paths ending in
390 // .json or .jinja, not literal template strings.
391 let _chat_template_tempfile = if let Some(ref template) = config.chat_template {
392 let tmp_dir = std::env::temp_dir().join("onde-chat-templates");
393 std::fs::create_dir_all(&tmp_dir).ok();
394 let tmp_path = tmp_dir.join("chat_template.jinja");
395 std::fs::write(&tmp_path, template).map_err(|e| InferenceError::ModelBuild {
396 reason: format!(
397 "Failed to write chat template to {}: {}",
398 tmp_path.display(),
399 e
400 ),
401 })?;
402 builder = builder.with_chat_template(tmp_path.to_string_lossy().to_string());
403 Some(tmp_path)
404 } else {
405 None
406 };
407
408 let model = builder
409 .build()
410 .await
411 .map_err(|e| InferenceError::ModelBuild {
412 reason: format!("Failed to build {} model: {}", config.display_name, e),
413 })?;
414
415 let elapsed = start.elapsed();
416
417 let sampling = sampling.unwrap_or_else(|| {
418 if cfg!(any(
419 target_os = "ios",
420 target_os = "tvos",
421 target_os = "visionos",
422 target_os = "watchos",
423 target_os = "android"
424 )) {
425 SamplingConfig::mobile()
426 } else {
427 SamplingConfig::default()
428 }
429 });
430
431 info!(
432 "ChatEngine: model {} loaded in {} (sampling: temp={:?}, max_tokens={:?})",
433 config.display_name,
434 format_duration(elapsed),
435 sampling.temperature,
436 sampling.max_tokens,
437 );
438
439 // Capture before config is moved into LoadedModel.
440 let pulse_model_id = config.model_id.clone();
441 let pulse_model_name = config.display_name.clone();
442
443 // Swap the new model in and take the old one out — all under the lock
444 // so the engine is never seen as None by a concurrent prompt. Then
445 // drop the old weights *outside* the lock so we don't block inference
446 // while the allocator frees potentially several GB of tensors.
447 let old_model = {
448 let mut guard = self.inner.lock().await;
449 guard.replace(LoadedModel {
450 model: Arc::new(model),
451 config: LoadedModelConfig::Gguf(config),
452 history: Vec::new(),
453 system_prompt,
454 sampling,
455 })
456 };
457 drop(old_model); // free old weights outside the lock
458
459 if let Some(ref pulse) = self.pulse {
460 pulse.record_model_loaded(pulse_model_id, pulse_model_name, elapsed.as_millis() as u64);
461 }
462
463 Ok(elapsed)
464 }
465
466 /// Fetch the model assigned to this Onde app from the SDK model-config
467 /// endpoint and load it.
468 ///
469 /// Authenticates using the app's own `onde_app_id` + `onde_app_secret`
470 /// (the SDK credentials shown in the ondeinference.com dashboard).
471 /// No end-user JWT is required — model assignment is an operator-level
472 /// configuration that is independent of which user is currently signed in.
473 ///
474 /// **Fallback behaviour:**
475 /// - HTTP 404 (no model assigned) → loads [`GgufModelConfig::platform_default()`].
476 /// - Missing `hf_repo_id` or `gguf_file` in response → loads platform default.
477 /// - Any HTTP or network error → returns [`InferenceError::ModelBuild`] so
478 /// the caller can apply its own fallback.
479 pub async fn load_assigned_model(
480 &self,
481 environment: smbcloud_gresiq_sdk::Environment,
482 onde_app_id: &str,
483 onde_app_secret: &str,
484 system_prompt: Option<String>,
485 sampling: Option<SamplingConfig>,
486 ) -> Result<std::time::Duration, InferenceError> {
487 use log::{info, warn};
488
489 #[derive(serde::Deserialize)]
490 struct ModelConfigResponse {
491 hf_repo_id: Option<String>,
492 gguf_file: Option<String>,
493 name: Option<String>,
494 approx_size_bytes: Option<i64>,
495 }
496
497 let url = format!(
498 "{}://{}/v1/client/onde_sdk/model_config?app_id={}&app_secret={}",
499 environment.api_protocol(),
500 environment.api_host(),
501 onde_app_id,
502 onde_app_secret,
503 );
504
505 let response = reqwest::Client::new()
506 .get(&url)
507 .header("Content-Type", "application/json")
508 .send()
509 .await
510 .map_err(|e| InferenceError::ModelBuild {
511 reason: format!("SDK model_config request failed: {e}"),
512 })?;
513
514 if response.status().as_u16() == 404 {
515 warn!(
516 "ChatEngine: no model assigned to Onde app {onde_app_id}; loading platform default."
517 );
518 return self
519 .load_gguf_model(GgufModelConfig::platform_default(), system_prompt, sampling)
520 .await;
521 }
522
523 if !response.status().is_success() {
524 return Err(InferenceError::ModelBuild {
525 reason: format!(
526 "SDK model_config returned HTTP {}",
527 response.status().as_u16()
528 ),
529 });
530 }
531
532 let resp: ModelConfigResponse =
533 response
534 .json()
535 .await
536 .map_err(|e| InferenceError::ModelBuild {
537 reason: format!("Failed to parse model_config response: {e}"),
538 })?;
539
540 let hf_repo_id = resp.hf_repo_id.as_deref().unwrap_or_default();
541 let gguf_file = resp.gguf_file.as_deref().unwrap_or_default();
542
543 if hf_repo_id.is_empty() || gguf_file.is_empty() {
544 warn!(
545 "ChatEngine: model_config response missing hf_repo_id or gguf_file; loading platform default."
546 );
547 return self
548 .load_gguf_model(GgufModelConfig::platform_default(), system_prompt, sampling)
549 .await;
550 }
551
552 #[cfg(target_os = "android")]
553 let tok_model_id = super::models::tok_model_id_for_repo(hf_repo_id).map(|s| s.to_string());
554 #[cfg(not(target_os = "android"))]
555 let tok_model_id: Option<String> = None;
556
557 let approx_memory = resp
558 .approx_size_bytes
559 .map(|b| {
560 let gb = b as f64 / 1_073_741_824.0;
561 if gb >= 1.0 {
562 format!("~{:.2} GB", gb)
563 } else {
564 format!("~{} MB", b / 1_048_576)
565 }
566 })
567 .unwrap_or_else(|| "—".to_string());
568
569 info!(
570 "ChatEngine: resolved SDK model assignment → {} / {} ({})",
571 hf_repo_id, gguf_file, approx_memory
572 );
573
574 let config = GgufModelConfig {
575 model_id: hf_repo_id.to_string(),
576 files: vec![gguf_file.to_string()],
577 tok_model_id,
578 display_name: resp.name.unwrap_or_else(|| hf_repo_id.to_string()),
579 approx_memory,
580 chat_template: None,
581 };
582
583 self.load_gguf_model(config, system_prompt, sampling).await
584 }
585
586 /// Load an ISQ (in-situ quantised) model into the engine.
587 ///
588 /// Unlike [`load_gguf_model`], this downloads the full-precision safetensors
589 /// from HuggingFace and quantises the weights in-situ on Metal (macOS only).
590 /// The quantisation happens once at load time; subsequent inference uses the
591 /// compressed weights.
592 ///
593 /// # macOS only
594 ///
595 /// ISQ with Metal requires the `metal` feature on mistral.rs and is therefore
596 /// restricted to macOS at the Rust level. Other platforms should continue to
597 /// use [`load_gguf_model`] with pre-quantised GGUF files.
598 ///
599 /// # Arguments
600 ///
601 /// * `config` — Which model to load and with how many ISQ bits.
602 /// * `system_prompt` — Optional system prompt prepended to every request.
603 /// * `sampling` — Sampling parameters; `None` uses [`SamplingConfig::default`].
604 ///
605 /// # Errors
606 ///
607 /// Returns [`InferenceError::ModelBuild`] if the model fails to download
608 /// or quantise.
609 #[cfg(target_os = "macos")]
610 pub async fn load_isq_model(
611 &self,
612 config: IsqModelConfig,
613 system_prompt: Option<String>,
614 sampling: Option<SamplingConfig>,
615 ) -> Result<std::time::Duration, InferenceError> {
616 use log::info;
617 use std::time::Instant;
618
619 info!(
620 "ChatEngine: loading ISQ model {} (bits={})",
621 config.model_id, config.isq_bits
622 );
623
624 // Clean up stale HF cache artefacts before loading.
625 crate::hf_cache::clean_stale_lock_files(&config.model_id);
626 crate::hf_cache::repair_hf_cache_symlinks(&config.model_id);
627
628 let start = Instant::now();
629
630 // Choose ISQ bit width.
631 let isq_bits = match config.isq_bits {
632 8 => IsqBits::Eight,
633 _ => IsqBits::Four, // default to 4-bit
634 };
635
636 let model = TextModelBuilder::new(&config.model_id)
637 .with_token_source(super::token::hf_token_source())
638 .with_auto_isq(isq_bits)
639 .with_logging()
640 .build()
641 .await
642 .map_err(|e| InferenceError::ModelBuild {
643 reason: format!("Failed to build ISQ model {}: {}", config.display_name, e),
644 })?;
645
646 let elapsed = start.elapsed();
647
648 let sampling = sampling.unwrap_or_default();
649
650 info!(
651 "ChatEngine: ISQ model {} loaded in {} (sampling: temp={:?}, max_tokens={:?})",
652 config.display_name,
653 format_duration(elapsed),
654 sampling.temperature,
655 sampling.max_tokens,
656 );
657
658 let mut guard = self.inner.lock().await;
659 *guard = Some(LoadedModel {
660 model: Arc::new(model),
661 config: LoadedModelConfig::Isq(config),
662 history: Vec::new(),
663 system_prompt,
664 sampling,
665 });
666
667 Ok(elapsed)
668 }
669
670 /// Unload the current model, freeing all memory.
671 ///
672 /// Dropping the `Model` sends a `Terminate` message to the mistral.rs
673 /// engine thread, which tears down the KV cache, activations, and model
674 /// weights.
675 ///
676 /// Returns the display name of the model that was unloaded, or `None`
677 /// if no model was loaded.
678 pub async fn unload_model(&self) -> Option<String> {
679 let mut guard = self.inner.lock().await;
680 if let Some(loaded) = guard.take() {
681 let name = loaded.config.display_name().to_string();
682 log::info!("ChatEngine: unloading model {}", name);
683 // `loaded` is dropped here → Model → Arc<MistralRs> (last ref)
684 // → MistralRs::drop() → engine thread termination.
685 Some(name)
686 } else {
687 log::debug!("ChatEngine: unload_model called but no model was loaded.");
688 None
689 }
690 }
691
692 /// Check whether a model is currently loaded.
693 pub async fn is_loaded(&self) -> bool {
694 self.inner.lock().await.is_some()
695 }
696
697 /// Get a snapshot of the engine's current state.
698 pub async fn info(&self) -> EngineInfo {
699 let guard = self.inner.lock().await;
700 match guard.as_ref() {
701 Some(loaded) => EngineInfo {
702 status: EngineStatus::Ready,
703 model_name: Some(loaded.config.display_name().to_string()),
704 approx_memory: Some(loaded.config.approx_memory().to_string()),
705 history_length: loaded.history.len() as u64,
706 },
707 None => EngineInfo {
708 status: EngineStatus::Unloaded,
709 model_name: None,
710 approx_memory: None,
711 history_length: 0u64,
712 },
713 }
714 }
715
716 // ── System prompt ────────────────────────────────────────────────────
717
718 /// Set or replace the system prompt.
719 ///
720 /// The system prompt is prepended to every inference request (it is NOT
721 /// stored in the conversation history).
722 pub async fn set_system_prompt(&self, prompt: impl Into<String>) {
723 if let Some(loaded) = self.inner.lock().await.as_mut() {
724 loaded.system_prompt = Some(prompt.into());
725 }
726 }
727
728 /// Clear the system prompt.
729 pub async fn clear_system_prompt(&self) {
730 if let Some(loaded) = self.inner.lock().await.as_mut() {
731 loaded.system_prompt = None;
732 }
733 }
734
735 // ── Sampling configuration ───────────────────────────────────────────
736
737 /// Replace the sampling configuration.
738 pub async fn set_sampling(&self, sampling: SamplingConfig) {
739 if let Some(loaded) = self.inner.lock().await.as_mut() {
740 loaded.sampling = sampling;
741 }
742 }
743
744 // ── Conversation history ─────────────────────────────────────────────
745
746 /// Get a clone of the full conversation history.
747 ///
748 /// Tool-related history entries are converted back to plain
749 /// [`ChatMessage`] values (assistant tool-call entries become assistant
750 /// messages with just their text content; tool results are omitted).
751 pub async fn history(&self) -> Vec<ChatMessage> {
752 let guard = self.inner.lock().await;
753 match guard.as_ref() {
754 Some(loaded) => loaded
755 .history
756 .iter()
757 .filter_map(|entry| match entry {
758 HistoryEntry::Text(msg) => Some(msg.clone()),
759 HistoryEntry::AssistantToolCall { content, .. } => {
760 Some(ChatMessage::assistant(content))
761 }
762 HistoryEntry::ToolResult { .. } => None,
763 })
764 .collect(),
765 None => Vec::new(),
766 }
767 }
768
769 /// Clear the conversation history but keep the model loaded.
770 ///
771 /// Returns the number of turns that were removed.
772 pub async fn clear_history(&self) -> usize {
773 let mut guard = self.inner.lock().await;
774 match guard.as_mut() {
775 Some(loaded) => {
776 let count = loaded.history.len();
777 loaded.history.clear();
778 log::info!("ChatEngine: cleared {} history turns.", count);
779 count
780 }
781 None => 0,
782 }
783 }
784
785 /// Append a message to history without running inference.
786 ///
787 /// Useful for restoring a saved conversation or injecting context.
788 pub async fn push_history(&self, message: ChatMessage) {
789 if let Some(loaded) = self.inner.lock().await.as_mut() {
790 loaded.history.push(HistoryEntry::Text(message));
791 }
792 }
793
794 // ── Non-streaming inference ──────────────────────────────────────────
795
796 /// Send a user message and receive a complete assistant reply.
797 ///
798 /// The user message and assistant reply are automatically appended to
799 /// the conversation history on success.
800 ///
801 /// # Errors
802 ///
803 /// - [`InferenceError::NoModelLoaded`] if no model is loaded.
804 /// - [`InferenceError::Inference`] if the model fails to generate.
805 pub async fn send_message(
806 &self,
807 user_message: impl Into<String>,
808 ) -> Result<InferenceResult, InferenceError> {
809 let user_message = user_message.into();
810
811 // ── 1. Snapshot model handle + build request, then release lock ──
812 let (model, request, pulse_model_id) = {
813 let guard = self.inner.lock().await;
814 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
815 let request = self::build_request(loaded, &user_message);
816 let pulse_model_id = match &loaded.config {
817 LoadedModelConfig::Gguf(c) => c.model_id.clone(),
818 #[cfg(target_os = "macos")]
819 LoadedModelConfig::Isq(c) => c.model_id.clone(),
820 };
821 (loaded.model.clone(), request, pulse_model_id)
822 }; // ← mutex released before inference
823
824 log::info!(
825 "ChatEngine: inference START — message: \"{}\"",
826 truncate_for_log(&user_message, 100)
827 );
828
829 // ── 2. Run inference (potentially slow — mutex is NOT held) ──────
830 let start = std::time::Instant::now();
831 let response =
832 model
833 .send_chat_request(request)
834 .await
835 .map_err(|e| InferenceError::Inference {
836 reason: e.to_string(),
837 })?;
838 let elapsed = start.elapsed();
839
840 let reply = response.choices[0]
841 .message
842 .content
843 .as_ref()
844 .map(|c| c.trim().to_string())
845 .unwrap_or_else(|| "(empty response)".to_string());
846
847 let finish_reason = response.choices[0].finish_reason.clone();
848
849 log::info!(
850 "ChatEngine: inference END — {} — reply: \"{}\"",
851 format_duration(elapsed),
852 truncate_for_log(&reply, 100)
853 );
854
855 // ── 3. Persist turns to history (brief lock re-acquisition) ──────
856 {
857 let mut guard = self.inner.lock().await;
858 if let Some(loaded) = guard.as_mut() {
859 loaded
860 .history
861 .push(HistoryEntry::Text(ChatMessage::user(user_message)));
862 loaded
863 .history
864 .push(HistoryEntry::Text(ChatMessage::assistant(reply.clone())));
865 }
866 }
867
868 if let Some(ref pulse) = self.pulse {
869 pulse.record_inference(
870 pulse_model_id,
871 crate::pulse::next_request_id(),
872 elapsed.as_millis() as u64,
873 "success".to_string(),
874 );
875 }
876
877 Ok(InferenceResult {
878 text: reply,
879 duration_secs: elapsed.as_secs_f64(),
880 duration_display: format_duration(elapsed),
881 finish_reason,
882 tool_calls: vec![],
883 })
884 }
885
886 /// Run inference on an explicit list of messages WITHOUT modifying the
887 /// engine's internal history.
888 ///
889 /// This is useful for one-shot prompts (e.g. prompt enhancement) where
890 /// you don't want side-effects on the main conversation.
891 ///
892 /// The system prompt set on the engine is still prepended.
893 pub async fn generate(
894 &self,
895 messages: Vec<ChatMessage>,
896 sampling: Option<SamplingConfig>,
897 ) -> Result<InferenceResult, InferenceError> {
898 let (model, request) = {
899 let guard = self.inner.lock().await;
900 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
901
902 let sampling = sampling.as_ref().unwrap_or(&loaded.sampling);
903 let mut req = RequestBuilder::new();
904
905 // Apply sampling parameters.
906 req = apply_sampling(req, sampling);
907
908 // System prompt.
909 if let Some(ref sp) = loaded.system_prompt {
910 req = req.add_message(TextMessageRole::System, sp);
911 }
912
913 // Provided messages.
914 for msg in &messages {
915 req = req.add_message(chat_role_to_mistral(&msg.role), &msg.content);
916 }
917
918 (loaded.model.clone(), req)
919 };
920
921 let start = std::time::Instant::now();
922 let response =
923 model
924 .send_chat_request(request)
925 .await
926 .map_err(|e| InferenceError::Inference {
927 reason: e.to_string(),
928 })?;
929 let elapsed = start.elapsed();
930
931 let reply = response.choices[0]
932 .message
933 .content
934 .as_ref()
935 .map(|c| c.trim().to_string())
936 .unwrap_or_else(|| "(empty response)".to_string());
937
938 let finish_reason = response.choices[0].finish_reason.clone();
939
940 Ok(InferenceResult {
941 text: reply,
942 duration_secs: elapsed.as_secs_f64(),
943 duration_display: format_duration(elapsed),
944 finish_reason,
945 tool_calls: vec![],
946 })
947 }
948
949 // ── Streaming inference ──────────────────────────────────────────────
950
951 /// Send a user message and receive an async stream of token chunks.
952 ///
953 /// The user message and full assembled reply are automatically appended
954 /// to the conversation history once the stream finishes.
955 ///
956 /// # Returns
957 ///
958 /// A `tokio::sync::mpsc::Receiver<StreamChunk>`. The caller should
959 /// receive from it until `chunk.done == true` or the channel closes.
960 ///
961 /// # Errors
962 ///
963 /// Returns immediately with [`InferenceError::NoModelLoaded`] if no
964 /// model is loaded. Inference errors are delivered as the final chunk
965 /// with `done = true` and an empty delta.
966 pub async fn stream_message(
967 &self,
968 user_message: impl Into<String>,
969 ) -> Result<tokio::sync::mpsc::Receiver<StreamChunk>, InferenceError> {
970 let user_message = user_message.into();
971
972 let (model, request) = {
973 let guard = self.inner.lock().await;
974 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
975 let request = self::build_request(loaded, &user_message);
976 (loaded.model.clone(), request)
977 };
978
979 let (tx, rx) = tokio::sync::mpsc::channel::<StreamChunk>(64);
980
981 // We need a reference to `self` inside the spawned task so we can
982 // update history when the stream completes. Since ChatEngine is
983 // behind a shared reference (callers hold &self or Arc<ChatEngine>),
984 // we borrow the inner Mutex via a raw pointer. This is safe because
985 // ChatEngine's lifetime exceeds the spawned task (it's typically in
986 // a Lazy static or shared application state).
987 //
988 // Alternative: require ChatEngine to always be wrapped in Arc and
989 // accept `self: Arc<Self>`. We avoid that to keep the API simple.
990 let inner_ptr = &self.inner as *const Mutex<Option<LoadedModel>>;
991 // SAFETY: ChatEngine is stored in a Lazy static or equivalent with
992 // 'static lifetime. The spawned task cannot outlive the engine.
993 let inner_ref: &'static Mutex<Option<LoadedModel>> = unsafe { &*inner_ptr };
994
995 let user_msg_clone = user_message.clone();
996
997 tokio::task::spawn(async move {
998 // `model` is an `Arc<Model>` moved into this task, so it's owned.
999 // `stream_chat_request` borrows `&Model` — the borrow is scoped
1000 // to this async block and does NOT need to be `'static`.
1001 let stream_result = model.stream_chat_request(request).await;
1002
1003 match stream_result {
1004 Ok(mut stream) => {
1005 let mut assembled = String::new();
1006 let mut last_finish_reason = None;
1007
1008 // `Stream::next()` is an inherent async method on
1009 // mistralrs::Stream that returns `Option<Response>`.
1010 // No `futures::StreamExt` import needed.
1011 while let Some(response) = stream.next().await {
1012 match response {
1013 mistralrs::Response::Chunk(chunk) => {
1014 if let Some(choice) = chunk.choices.first() {
1015 if let Some(ref text) = choice.delta.content {
1016 assembled.push_str(text);
1017 let _ = tx
1018 .send(StreamChunk {
1019 delta: text.clone(),
1020 done: false,
1021 finish_reason: None,
1022 })
1023 .await;
1024 }
1025 if let Some(ref reason) = choice.finish_reason {
1026 last_finish_reason = Some(reason.clone());
1027 }
1028 }
1029 }
1030 mistralrs::Response::Done(_) => {
1031 // Non-streaming response arrived on a streaming
1032 // channel — should not happen, but handle gracefully.
1033 break;
1034 }
1035 mistralrs::Response::InternalError(e) => {
1036 log::error!("ChatEngine stream internal error: {}", e);
1037 break;
1038 }
1039 mistralrs::Response::ValidationError(e) => {
1040 log::error!("ChatEngine stream validation error: {}", e);
1041 break;
1042 }
1043 mistralrs::Response::ModelError(msg, _) => {
1044 log::error!("ChatEngine stream model error: {}", msg);
1045 break;
1046 }
1047 _ => {
1048 // Completion / Image / Speech variants — not expected
1049 // for chat streaming.
1050 break;
1051 }
1052 }
1053 }
1054
1055 // Persist turns to history.
1056 {
1057 let mut guard = inner_ref.lock().await;
1058 if let Some(loaded) = guard.as_mut() {
1059 loaded
1060 .history
1061 .push(HistoryEntry::Text(ChatMessage::user(user_msg_clone)));
1062 loaded
1063 .history
1064 .push(HistoryEntry::Text(ChatMessage::assistant(assembled.trim())));
1065 }
1066 }
1067
1068 // Send the final "done" chunk.
1069 let _ = tx
1070 .send(StreamChunk {
1071 delta: String::new(),
1072 done: true,
1073 finish_reason: last_finish_reason,
1074 })
1075 .await;
1076 }
1077 Err(e) => {
1078 log::error!("ChatEngine: stream_chat_request failed: {}", e);
1079 let _ = tx
1080 .send(StreamChunk {
1081 delta: String::new(),
1082 done: true,
1083 finish_reason: Some(format!("error: {}", e)),
1084 })
1085 .await;
1086 }
1087 }
1088 });
1089
1090 Ok(rx)
1091 }
1092
1093 // ── Tool-aware inference (Rust-only) ─────────────────────────────────
1094
1095 /// Send a user message with tool definitions available. Non-streaming.
1096 ///
1097 /// If the model decides to call tools, [`ToolAwareResult::tool_calls`]
1098 /// will be non-empty and `finish_reason` will typically be `"tool_calls"`.
1099 ///
1100 /// The user message and response (including any tool calls) are
1101 /// automatically added to conversation history.
1102 pub async fn send_message_with_tools(
1103 &self,
1104 user_message: impl Into<String>,
1105 tools: &[ToolDefinition],
1106 ) -> Result<ToolAwareResult, InferenceError> {
1107 let user_message = user_message.into();
1108
1109 let (model, request) = {
1110 let guard = self.inner.lock().await;
1111 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1112 let request = build_request_with_tools(loaded, &user_message, tools);
1113 (loaded.model.clone(), request)
1114 };
1115
1116 log::info!(
1117 "ChatEngine: tool inference START — message: \"{}\"",
1118 truncate_for_log(&user_message, 100)
1119 );
1120
1121 let start = std::time::Instant::now();
1122 let response =
1123 model
1124 .send_chat_request(request)
1125 .await
1126 .map_err(|e| InferenceError::Inference {
1127 reason: e.to_string(),
1128 })?;
1129 let elapsed = start.elapsed();
1130
1131 let choice = &response.choices[0];
1132 let reply = choice
1133 .message
1134 .content
1135 .as_ref()
1136 .map(|c| c.trim().to_string())
1137 .unwrap_or_default();
1138 let finish_reason = choice.finish_reason.clone();
1139 let tool_calls = parse_tool_calls(choice);
1140
1141 log::info!(
1142 "ChatEngine: tool inference END — {} — tool_calls: {} — reply: \"{}\"",
1143 format_duration(elapsed),
1144 tool_calls.len(),
1145 truncate_for_log(&reply, 100)
1146 );
1147
1148 // Persist to history.
1149 {
1150 let mut guard = self.inner.lock().await;
1151 if let Some(loaded) = guard.as_mut() {
1152 loaded
1153 .history
1154 .push(HistoryEntry::Text(ChatMessage::user(&user_message)));
1155 if tool_calls.is_empty() {
1156 loaded
1157 .history
1158 .push(HistoryEntry::Text(ChatMessage::assistant(&reply)));
1159 } else {
1160 loaded.history.push(HistoryEntry::AssistantToolCall {
1161 content: reply.clone(),
1162 tool_calls: tool_calls.clone(),
1163 });
1164 }
1165 }
1166 }
1167
1168 Ok(ToolAwareResult {
1169 text: reply,
1170 tool_calls,
1171 duration_secs: elapsed.as_secs_f64(),
1172 duration_display: format_duration(elapsed),
1173 finish_reason,
1174 })
1175 }
1176
1177 /// Send tool execution results back to the model. Non-streaming.
1178 ///
1179 /// Call this after executing the tools from a previous
1180 /// [`send_message_with_tools`](Self::send_message_with_tools) response.
1181 /// Pass the same (or updated) tool definitions if the model should be
1182 /// allowed to make further tool calls.
1183 pub async fn send_tool_results(
1184 &self,
1185 results: Vec<ToolResult>,
1186 tools: Option<&[ToolDefinition]>,
1187 ) -> Result<ToolAwareResult, InferenceError> {
1188 let (model, request) = {
1189 let guard = self.inner.lock().await;
1190 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1191 let request = build_tool_results_request(loaded, &results, tools);
1192 (loaded.model.clone(), request)
1193 };
1194
1195 log::info!(
1196 "ChatEngine: tool results inference START — {} results",
1197 results.len()
1198 );
1199
1200 let start = std::time::Instant::now();
1201 let response =
1202 model
1203 .send_chat_request(request)
1204 .await
1205 .map_err(|e| InferenceError::Inference {
1206 reason: e.to_string(),
1207 })?;
1208 let elapsed = start.elapsed();
1209
1210 let choice = &response.choices[0];
1211 let reply = choice
1212 .message
1213 .content
1214 .as_ref()
1215 .map(|c| c.trim().to_string())
1216 .unwrap_or_default();
1217 let finish_reason = choice.finish_reason.clone();
1218 let tool_calls = parse_tool_calls(choice);
1219
1220 log::info!(
1221 "ChatEngine: tool results inference END — {} — tool_calls: {} — reply: \"{}\"",
1222 format_duration(elapsed),
1223 tool_calls.len(),
1224 truncate_for_log(&reply, 100)
1225 );
1226
1227 // Persist to history.
1228 {
1229 let mut guard = self.inner.lock().await;
1230 if let Some(loaded) = guard.as_mut() {
1231 for result in &results {
1232 loaded.history.push(HistoryEntry::ToolResult {
1233 tool_call_id: result.tool_call_id.clone(),
1234 content: result.content.clone(),
1235 });
1236 }
1237 if tool_calls.is_empty() {
1238 loaded
1239 .history
1240 .push(HistoryEntry::Text(ChatMessage::assistant(&reply)));
1241 } else {
1242 loaded.history.push(HistoryEntry::AssistantToolCall {
1243 content: reply.clone(),
1244 tool_calls: tool_calls.clone(),
1245 });
1246 }
1247 }
1248 }
1249
1250 Ok(ToolAwareResult {
1251 text: reply,
1252 tool_calls,
1253 duration_secs: elapsed.as_secs_f64(),
1254 duration_display: format_duration(elapsed),
1255 finish_reason,
1256 })
1257 }
1258
1259 /// Stream tool execution results back to the model.
1260 ///
1261 /// Similar to [`send_tool_results`](Self::send_tool_results) but returns
1262 /// a streaming receiver. Tool calls in the streaming response are NOT
1263 /// parsed — this is intended for the final text response after all tool
1264 /// rounds complete.
1265 pub async fn stream_tool_results(
1266 &self,
1267 results: Vec<ToolResult>,
1268 tools: Option<Vec<ToolDefinition>>,
1269 ) -> Result<tokio::sync::mpsc::Receiver<StreamChunk>, InferenceError> {
1270 let (model, request) = {
1271 let guard = self.inner.lock().await;
1272 let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1273 let request = build_tool_results_request(loaded, &results, tools.as_deref());
1274 (loaded.model.clone(), request)
1275 };
1276
1277 let (tx, rx) = tokio::sync::mpsc::channel::<StreamChunk>(64);
1278
1279 let inner_ptr = &self.inner as *const Mutex<Option<LoadedModel>>;
1280 // SAFETY: ChatEngine is stored in a Lazy static or equivalent with
1281 // 'static lifetime. The spawned task cannot outlive the engine.
1282 let inner_ref: &'static Mutex<Option<LoadedModel>> = unsafe { &*inner_ptr };
1283
1284 let results_clone = results;
1285
1286 tokio::task::spawn(async move {
1287 let stream_result = model.stream_chat_request(request).await;
1288
1289 match stream_result {
1290 Ok(mut stream) => {
1291 let mut assembled = String::new();
1292 let mut last_finish_reason = None;
1293
1294 while let Some(response) = stream.next().await {
1295 match response {
1296 mistralrs::Response::Chunk(chunk) => {
1297 if let Some(choice) = chunk.choices.first() {
1298 if let Some(ref text) = choice.delta.content {
1299 assembled.push_str(text);
1300 let _ = tx
1301 .send(StreamChunk {
1302 delta: text.clone(),
1303 done: false,
1304 finish_reason: None,
1305 })
1306 .await;
1307 }
1308 if let Some(ref reason) = choice.finish_reason {
1309 last_finish_reason = Some(reason.clone());
1310 }
1311 }
1312 }
1313 mistralrs::Response::Done(_) => break,
1314 mistralrs::Response::InternalError(e) => {
1315 log::error!("stream_tool_results internal error: {}", e);
1316 break;
1317 }
1318 mistralrs::Response::ValidationError(e) => {
1319 log::error!("stream_tool_results validation error: {}", e);
1320 break;
1321 }
1322 mistralrs::Response::ModelError(msg, _) => {
1323 log::error!("stream_tool_results model error: {}", msg);
1324 break;
1325 }
1326 _ => break,
1327 }
1328 }
1329
1330 // Persist to history.
1331 {
1332 let mut guard = inner_ref.lock().await;
1333 if let Some(loaded) = guard.as_mut() {
1334 for result in &results_clone {
1335 loaded.history.push(HistoryEntry::ToolResult {
1336 tool_call_id: result.tool_call_id.clone(),
1337 content: result.content.clone(),
1338 });
1339 }
1340 loaded
1341 .history
1342 .push(HistoryEntry::Text(ChatMessage::assistant(assembled.trim())));
1343 }
1344 }
1345
1346 let _ = tx
1347 .send(StreamChunk {
1348 delta: String::new(),
1349 done: true,
1350 finish_reason: last_finish_reason,
1351 })
1352 .await;
1353 }
1354 Err(e) => {
1355 log::error!("stream_tool_results failed: {}", e);
1356 let _ = tx
1357 .send(StreamChunk {
1358 delta: String::new(),
1359 done: true,
1360 finish_reason: Some(format!("error: {}", e)),
1361 })
1362 .await;
1363 }
1364 }
1365 });
1366
1367 Ok(rx)
1368 }
1369 }
1370
1371 // ═════════════════════════════════════════════════════════════════════════════
1372 // Default impl
1373 // ═════════════════════════════════════════════════════════════════════════════
1374
1375 #[cfg(any(
1376 target_os = "macos",
1377 target_os = "ios",
1378 target_os = "tvos",
1379 target_os = "visionos",
1380 target_os = "watchos",
1381 target_os = "windows",
1382 target_os = "linux",
1383 target_os = "android"
1384 ))]
1385 impl Default for ChatEngine {
1386 fn default() -> Self {
1387 Self::new()
1388 }
1389 }
1390
1391 // ═════════════════════════════════════════════════════════════════════════════
1392 // Internal helpers (platform-gated)
1393 // ═════════════════════════════════════════════════════════════════════════════
1394
1395 /// Build a `RequestBuilder` from the current engine state and a new user message.
1396 #[cfg(any(
1397 target_os = "macos",
1398 target_os = "ios",
1399 target_os = "tvos",
1400 target_os = "visionos",
1401 target_os = "watchos",
1402 target_os = "windows",
1403 target_os = "linux",
1404 target_os = "android"
1405 ))]
1406 fn build_request(loaded: &LoadedModel, user_message: &str) -> RequestBuilder {
1407 let mut req = RequestBuilder::new();
1408
1409 // Apply sampling.
1410 req = apply_sampling(req, &loaded.sampling);
1411
1412 // System prompt.
1413 if let Some(ref sp) = loaded.system_prompt {
1414 req = req.add_message(TextMessageRole::System, sp);
1415 }
1416
1417 // Replay conversation history for multi-turn context.
1418 for entry in &loaded.history {
1419 match entry {
1420 HistoryEntry::Text(msg) => {
1421 req = req.add_message(chat_role_to_mistral(&msg.role), &msg.content);
1422 }
1423 HistoryEntry::AssistantToolCall { content, .. } => {
1424 // When replaying without tools, just use the text content.
1425 req = req.add_message(TextMessageRole::Assistant, content);
1426 }
1427 HistoryEntry::ToolResult { .. } => {
1428 // Skip tool results when not in tool mode.
1429 }
1430 }
1431 }
1432
1433 // Append the current user message.
1434 req = req.add_message(TextMessageRole::User, user_message);
1435
1436 req
1437 }
1438
1439 /// Build a request with tool support, properly replaying tool-related history.
1440 #[cfg(any(
1441 target_os = "macos",
1442 target_os = "ios",
1443 target_os = "tvos",
1444 target_os = "visionos",
1445 target_os = "watchos",
1446 target_os = "windows",
1447 target_os = "linux",
1448 target_os = "android"
1449 ))]
1450 fn build_request_with_tools(
1451 loaded: &LoadedModel,
1452 user_message: &str,
1453 tools: &[ToolDefinition],
1454 ) -> RequestBuilder {
1455 let mut req = RequestBuilder::new();
1456 req = apply_sampling(req, &loaded.sampling);
1457
1458 if let Some(ref sp) = loaded.system_prompt {
1459 req = req.add_message(TextMessageRole::System, sp);
1460 }
1461
1462 req = replay_history_with_tools(req, &loaded.history);
1463
1464 req = req.add_message(TextMessageRole::User, user_message);
1465
1466 // Add tool definitions.
1467 req = attach_tools(req, tools);
1468
1469 req
1470 }
1471
1472 /// Build a request for sending tool results back to the model.
1473 #[cfg(any(
1474 target_os = "macos",
1475 target_os = "ios",
1476 target_os = "tvos",
1477 target_os = "visionos",
1478 target_os = "watchos",
1479 target_os = "windows",
1480 target_os = "linux",
1481 target_os = "android"
1482 ))]
1483 fn build_tool_results_request(
1484 loaded: &LoadedModel,
1485 results: &[ToolResult],
1486 tools: Option<&[ToolDefinition]>,
1487 ) -> RequestBuilder {
1488 let mut req = RequestBuilder::new();
1489 req = apply_sampling(req, &loaded.sampling);
1490
1491 if let Some(ref sp) = loaded.system_prompt {
1492 req = req.add_message(TextMessageRole::System, sp);
1493 }
1494
1495 // Replay full history (including tool call entries).
1496 req = replay_history_with_tools(req, &loaded.history);
1497
1498 // Add new tool results.
1499 for result in results {
1500 req = req.add_tool_message(&result.content, &result.tool_call_id);
1501 }
1502
1503 // Optionally add tools for further rounds.
1504 if let Some(tool_defs) = tools {
1505 req = attach_tools(req, tool_defs);
1506 }
1507
1508 req
1509 }
1510
1511 /// Replay conversation history entries onto a [`RequestBuilder`], preserving
1512 /// tool call and tool result messages.
1513 #[cfg(any(
1514 target_os = "macos",
1515 target_os = "ios",
1516 target_os = "tvos",
1517 target_os = "visionos",
1518 target_os = "watchos",
1519 target_os = "windows",
1520 target_os = "linux",
1521 target_os = "android"
1522 ))]
1523 fn replay_history_with_tools(mut req: RequestBuilder, history: &[HistoryEntry]) -> RequestBuilder {
1524 for entry in history {
1525 match entry {
1526 HistoryEntry::Text(msg) => {
1527 req = req.add_message(chat_role_to_mistral(&msg.role), &msg.content);
1528 }
1529 HistoryEntry::AssistantToolCall {
1530 content,
1531 tool_calls,
1532 } => {
1533 let mistral_tcs: Vec<ToolCallResponse> = tool_calls
1534 .iter()
1535 .enumerate()
1536 .map(|(i, tc)| ToolCallResponse {
1537 index: i,
1538 id: tc.id.clone(),
1539 tp: ToolCallType::Function,
1540 function: CalledFunction {
1541 name: tc.function_name.clone(),
1542 arguments: tc.arguments.clone(),
1543 },
1544 })
1545 .collect();
1546 req = req.add_message_with_tool_call(
1547 TextMessageRole::Assistant,
1548 content.clone(),
1549 mistral_tcs,
1550 );
1551 }
1552 HistoryEntry::ToolResult {
1553 tool_call_id,
1554 content,
1555 } => {
1556 req = req.add_tool_message(content, tool_call_id);
1557 }
1558 }
1559 }
1560 req
1561 }
1562
1563 /// Attach [`ToolDefinition`] slices as mistralrs [`Tool`] values and set
1564 /// [`ToolChoice::Auto`].
1565 #[cfg(any(
1566 target_os = "macos",
1567 target_os = "ios",
1568 target_os = "tvos",
1569 target_os = "visionos",
1570 target_os = "watchos",
1571 target_os = "windows",
1572 target_os = "linux",
1573 target_os = "android"
1574 ))]
1575 fn attach_tools(mut req: RequestBuilder, tools: &[ToolDefinition]) -> RequestBuilder {
1576 if !tools.is_empty() {
1577 let mistral_tools: Vec<MistralTool> = tools
1578 .iter()
1579 .map(|td| {
1580 let params: HashMap<String, serde_json::Value> =
1581 match serde_json::from_str(&td.parameters_schema) {
1582 Ok(p) => p,
1583 Err(err) => {
1584 log::warn!(
1585 "tool '{}': malformed parameters_schema JSON ({}), using empty params",
1586 td.name,
1587 err
1588 );
1589 HashMap::new()
1590 }
1591 };
1592 MistralTool {
1593 tp: ToolType::Function,
1594 function: Function {
1595 description: Some(td.description.clone()),
1596 name: td.name.clone(),
1597 parameters: Some(params),
1598 strict: Some(true),
1599 },
1600 }
1601 })
1602 .collect();
1603 req = req.set_tools(mistral_tools);
1604 req = req.set_tool_choice(ToolChoice::Auto);
1605 }
1606 req
1607 }
1608
1609 /// Parse tool calls from a mistralrs response choice.
1610 #[cfg(any(
1611 target_os = "macos",
1612 target_os = "ios",
1613 target_os = "tvos",
1614 target_os = "visionos",
1615 target_os = "watchos",
1616 target_os = "windows",
1617 target_os = "linux",
1618 target_os = "android"
1619 ))]
1620 fn parse_tool_calls(choice: &mistralrs::Choice) -> Vec<ToolCallRequest> {
1621 choice
1622 .message
1623 .tool_calls
1624 .as_ref()
1625 .map(|tcs| {
1626 tcs.iter()
1627 .enumerate()
1628 .map(|(i, tc)| {
1629 let id = if tc.id.is_empty() {
1630 log::warn!("tool call at index {i} has empty id — generating fallback");
1631 format!("call_{i}")
1632 } else {
1633 tc.id.clone()
1634 };
1635 if serde_json::from_str::<serde_json::Value>(&tc.function.arguments).is_err() {
1636 log::warn!(
1637 "tool call '{}' has malformed arguments JSON: {}",
1638 tc.function.name,
1639 tc.function.arguments.chars().take(200).collect::<String>()
1640 );
1641 }
1642 ToolCallRequest {
1643 id,
1644 function_name: tc.function.name.clone(),
1645 arguments: tc.function.arguments.clone(),
1646 }
1647 })
1648 .collect()
1649 })
1650 .unwrap_or_default()
1651 }
1652
1653 /// Apply a [`SamplingConfig`] to a [`RequestBuilder`].
1654 #[cfg(any(
1655 target_os = "macos",
1656 target_os = "ios",
1657 target_os = "tvos",
1658 target_os = "visionos",
1659 target_os = "watchos",
1660 target_os = "windows",
1661 target_os = "linux",
1662 target_os = "android"
1663 ))]
1664 fn apply_sampling(mut req: RequestBuilder, sampling: &SamplingConfig) -> RequestBuilder {
1665 if let Some(temp) = sampling.temperature {
1666 req = req.set_sampler_temperature(temp);
1667 }
1668 if let Some(top_p) = sampling.top_p {
1669 req = req.set_sampler_topp(top_p);
1670 }
1671 if let Some(top_k) = sampling.top_k {
1672 req = req.set_sampler_topk(top_k as usize);
1673 }
1674 if let Some(min_p) = sampling.min_p {
1675 req = req.set_sampler_minp(min_p);
1676 }
1677 if let Some(max_tokens) = sampling.max_tokens {
1678 req = req.set_sampler_max_len(max_tokens as usize);
1679 }
1680 if let Some(freq) = sampling.frequency_penalty {
1681 req = req.set_sampler_frequency_penalty(freq);
1682 }
1683 if let Some(pres) = sampling.presence_penalty {
1684 req = req.set_sampler_presence_penalty(pres);
1685 }
1686 req
1687 }
1688
1689 /// Convert a [`ChatRole`] to a [`TextMessageRole`].
1690 #[cfg(any(
1691 target_os = "macos",
1692 target_os = "ios",
1693 target_os = "tvos",
1694 target_os = "visionos",
1695 target_os = "watchos",
1696 target_os = "windows",
1697 target_os = "linux",
1698 target_os = "android"
1699 ))]
1700 fn chat_role_to_mistral(role: &ChatRole) -> TextMessageRole {
1701 match role {
1702 ChatRole::System => TextMessageRole::System,
1703 ChatRole::User => TextMessageRole::User,
1704 ChatRole::Assistant => TextMessageRole::Assistant,
1705 }
1706 }
1707
1708 /// Truncate a string for log output, appending `"..."` if truncated.
1709 fn truncate_for_log(s: &str, max_len: usize) -> String {
1710 if s.len() > max_len {
1711 format!("{}...", &s[..max_len])
1712 } else {
1713 s.to_string()
1714 }
1715 }
1716
1717 // ═════════════════════════════════════════════════════════════════════════════
1718 // Unsupported-platform stub
1719 // ═════════════════════════════════════════════════════════════════════════════
1720
1721 /// Stub `ChatEngine` for platforms where mistral.rs is not available.
1722 ///
1723 /// All methods return errors or empty values so that code using the engine
1724 /// can compile on any target without `#[cfg]` at every call site.
1725 #[cfg(not(any(
1726 target_os = "macos",
1727 target_os = "ios",
1728 target_os = "tvos",
1729 target_os = "visionos",
1730 target_os = "watchos",
1731 target_os = "windows",
1732 target_os = "linux",
1733 target_os = "android"
1734 )))]
1735 pub struct ChatEngine;
1736
1737 #[cfg(not(any(
1738 target_os = "macos",
1739 target_os = "ios",
1740 target_os = "tvos",
1741 target_os = "visionos",
1742 target_os = "watchos",
1743 target_os = "windows",
1744 target_os = "linux",
1745 target_os = "android"
1746 )))]
1747 impl ChatEngine {
1748 pub fn new() -> Self {
1749 Self
1750 }
1751
1752 pub async fn load_gguf_model(
1753 &self,
1754 _config: GgufModelConfig,
1755 _system_prompt: Option<String>,
1756 _sampling: Option<SamplingConfig>,
1757 ) -> Result<std::time::Duration, InferenceError> {
1758 Err(InferenceError::Other {
1759 reason: "LLM inference is not supported on this platform.".into(),
1760 })
1761 }
1762
1763 pub async fn unload_model(&self) -> Option<String> {
1764 None
1765 }
1766
1767 pub async fn is_loaded(&self) -> bool {
1768 false
1769 }
1770
1771 pub async fn info(&self) -> EngineInfo {
1772 EngineInfo {
1773 status: EngineStatus::Unloaded,
1774 model_name: None,
1775 approx_memory: None,
1776 history_length: 0u64,
1777 }
1778 }
1779
1780 pub async fn set_system_prompt(&self, _prompt: impl Into<String>) {}
1781
1782 pub async fn clear_system_prompt(&self) {}
1783
1784 pub async fn set_sampling(&self, _sampling: SamplingConfig) {}
1785
1786 pub async fn history(&self) -> Vec<ChatMessage> {
1787 Vec::new()
1788 }
1789
1790 pub async fn clear_history(&self) -> usize {
1791 0
1792 }
1793
1794 pub async fn push_history(&self, _message: ChatMessage) {}
1795
1796 pub async fn send_message(
1797 &self,
1798 _user_message: impl Into<String>,
1799 ) -> Result<InferenceResult, InferenceError> {
1800 Err(InferenceError::Other {
1801 reason: "LLM inference is not supported on this platform.".into(),
1802 })
1803 }
1804
1805 pub async fn generate(
1806 &self,
1807 _messages: Vec<ChatMessage>,
1808 _sampling: Option<SamplingConfig>,
1809 ) -> Result<InferenceResult, InferenceError> {
1810 Err(InferenceError::Other {
1811 reason: "LLM inference is not supported on this platform.".into(),
1812 })
1813 }
1814
1815 pub async fn stream_message(
1816 &self,
1817 _user_message: impl Into<String>,
1818 ) -> Result<tokio::sync::mpsc::Receiver<StreamChunk>, InferenceError> {
1819 Err(InferenceError::Other {
1820 reason: "LLM inference is not supported on this platform.".into(),
1821 })
1822 }
1823
1824 pub async fn send_message_with_tools(
1825 &self,
1826 _user_message: &str,
1827 _tools: &[ToolDefinition],
1828 ) -> Result<InferenceResult, InferenceError> {
1829 Err(InferenceError::Other {
1830 reason: "LLM inference is not supported on this platform.".into(),
1831 })
1832 }
1833
1834 pub async fn send_tool_results(
1835 &self,
1836 _results: Vec<ToolResult>,
1837 _next_tools: Option<&[ToolDefinition]>,
1838 ) -> Result<InferenceResult, InferenceError> {
1839 Err(InferenceError::Other {
1840 reason: "LLM inference is not supported on this platform.".into(),
1841 })
1842 }
1843 }
1844
1845 #[cfg(not(any(
1846 target_os = "macos",
1847 target_os = "ios",
1848 target_os = "tvos",
1849 target_os = "visionos",
1850 target_os = "watchos",
1851 target_os = "windows",
1852 target_os = "linux",
1853 target_os = "android"
1854 )))]
1855 impl Default for ChatEngine {
1856 fn default() -> Self {
1857 Self::new()
1858 }
1859 }
1860
1861 // ═════════════════════════════════════════════════════════════════════════════
1862 // Prebuilt model configs (convenience constructors)
1863 // ═════════════════════════════════════════════════════════════════════════════
1864
1865 /// DeepSeek Coder v1 Jinja chat template.
1866 ///
1867 /// TheBloke's GGUF files do not embed a chat template — without an explicit
1868 /// template, mistral.rs rejects chat requests with "model does not have a
1869 /// chat template". This template reproduces the official DeepSeek Coder v1
1870 /// `### Instruction: / ### Response:` format.
1871 const DEEPSEEK_CODER_CHAT_TEMPLATE: &str = include_str!("deepseek_coder_chat_template.txt");
1872
1873 /// Convenience constructors for common GGUF model configurations.
1874 ///
1875 /// These mirror the constants in [`super::models`] but return a ready-to-use
1876 /// [`GgufModelConfig`].
1877 impl GgufModelConfig {
1878 /// Qwen 2.5 1.5B Instruct (GGUF Q4_K_M) — ~941 MB.
1879 ///
1880 /// Lightest option, ideal for iOS and memory-constrained Android devices.
1881 pub fn qwen25_1_5b() -> Self {
1882 Self {
1883 model_id: super::models::BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF.into(),
1884 files: vec![super::models::QWEN25_1_5B_GGUF_FILE.into()],
1885 tok_model_id: if cfg!(target_os = "android") {
1886 Some(super::models::QWEN25_1_5B_TOK_MODEL_ID.into())
1887 } else {
1888 None
1889 },
1890 display_name: "Qwen 2.5 1.5B".into(),
1891 approx_memory: "~941 MB (GGUF Q4_K_M)".into(),
1892 chat_template: None,
1893 }
1894 }
1895
1896 /// Qwen 2.5 3B Instruct (GGUF Q4_K_M) — ~1.93 GB.
1897 ///
1898 /// Balanced quality/size for macOS desktops and high-RAM Android devices.
1899 /// Not recommended for iOS (OOM on 8 GB devices).
1900 pub fn qwen25_3b() -> Self {
1901 Self {
1902 model_id: super::models::BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF.into(),
1903 files: vec![super::models::QWEN25_3B_GGUF_FILE.into()],
1904 tok_model_id: if cfg!(target_os = "android") {
1905 Some(super::models::QWEN25_3B_TOK_MODEL_ID.into())
1906 } else {
1907 None
1908 },
1909 display_name: "Qwen 2.5 3B".into(),
1910 approx_memory: "~1.93 GB (GGUF Q4_K_M)".into(),
1911 chat_template: None,
1912 }
1913 }
1914
1915 /// Qwen 2.5 Coder 1.5B Instruct (GGUF Q4_K_M) — ~941 MB.
1916 ///
1917 /// Dedicated coding model fine-tuned on 5.5T tokens of code and math data.
1918 /// Same `qwen2` GGUF architecture as the general 1.5B — loads via the
1919 /// existing `quantized_qwen.rs` path without any mistral.rs changes.
1920 /// Same memory footprint as the general 1.5B but dramatically better at
1921 /// code generation, explanation, and debugging.
1922 ///
1923 /// Preferred default for iOS, Android, and any coding-focused deployment.
1924 pub fn qwen25_coder_1_5b() -> Self {
1925 Self {
1926 model_id: super::models::BARTOWSKI_QWEN25_CODER_1_5B_INSTRUCT_GGUF.into(),
1927 files: vec![super::models::QWEN25_CODER_1_5B_GGUF_FILE.into()],
1928 tok_model_id: if cfg!(target_os = "android") {
1929 Some(super::models::QWEN25_CODER_1_5B_TOK_MODEL_ID.into())
1930 } else {
1931 None
1932 },
1933 display_name: "Qwen 2.5 Coder 1.5B".into(),
1934 approx_memory: "~941 MB (GGUF Q4_K_M)".into(),
1935 chat_template: None,
1936 }
1937 }
1938
1939 /// Qwen 2.5 Coder 3B Instruct (GGUF Q4_K_M) — ~1.93 GB.
1940 ///
1941 /// Best on-device coding quality for macOS desktop. Fine-tuned on 5.5T
1942 /// tokens of code and math. Same `qwen2` architecture as the general 3B.
1943 /// Not recommended for iOS due to memory constraints.
1944 pub fn qwen25_coder_3b() -> Self {
1945 Self {
1946 model_id: super::models::BARTOWSKI_QWEN25_CODER_3B_INSTRUCT_GGUF.into(),
1947 files: vec![super::models::QWEN25_CODER_3B_GGUF_FILE.into()],
1948 tok_model_id: if cfg!(target_os = "android") {
1949 Some(super::models::QWEN25_CODER_3B_TOK_MODEL_ID.into())
1950 } else {
1951 None
1952 },
1953 display_name: "Qwen 2.5 Coder 3B".into(),
1954 approx_memory: "~1.93 GB (GGUF Q4_K_M)".into(),
1955 chat_template: None,
1956 }
1957 }
1958
1959 /// Qwen 2.5 Coder 7B Instruct (GGUF Q4_K_M) — ~4.4 GB.
1960 ///
1961 /// Strong coding model with tool/function calling support via the Qwen2.5
1962 /// chat template. Uses the `qwen2` architecture. Requires 8+ GB RAM.
1963 pub fn qwen25_coder_7b() -> Self {
1964 Self {
1965 model_id: super::models::BARTOWSKI_QWEN25_CODER_7B_INSTRUCT_GGUF.into(),
1966 files: vec![super::models::QWEN25_CODER_7B_GGUF_FILE.into()],
1967 tok_model_id: if cfg!(target_os = "android") {
1968 Some(super::models::QWEN25_CODER_7B_TOK_MODEL_ID.into())
1969 } else {
1970 None
1971 },
1972 display_name: "Qwen 2.5 Coder 7B (Q4_K_M)".into(),
1973 approx_memory: "~4.4 GB".into(),
1974 chat_template: None,
1975 }
1976 }
1977
1978 /// Qwen 3 4B Instruct (GGUF Q4_K_M) — ~2.7 GB.
1979 ///
1980 /// Full OpenAI-compatible tool calling with extended thinking mode.
1981 /// Always load with `max_tokens ≥ 4096`; the `<think>…</think>` block can
1982 /// consume 300–400 tokens before the real response begins.
1983 pub fn qwen3_4b() -> Self {
1984 Self {
1985 model_id: super::models::BARTOWSKI_QWEN3_4B_GGUF.into(),
1986 files: vec![super::models::QWEN3_4B_GGUF_FILE.into()],
1987 tok_model_id: None,
1988 display_name: "Qwen 3 4B (Q4_K_M)".into(),
1989 approx_memory: "~2.7 GB".into(),
1990 chat_template: None,
1991 }
1992 }
1993
1994 /// Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling, ~1.3 GB.
1995 pub fn qwen3_1_7b() -> Self {
1996 Self {
1997 model_id: super::models::BARTOWSKI_QWEN3_1_7B_GGUF.into(),
1998 files: vec![super::models::QWEN3_1_7B_GGUF_FILE.into()],
1999 tok_model_id: None,
2000 display_name: "Qwen 3 1.7B (Q4_K_M)".into(),
2001 approx_memory: "~1.3 GB".into(),
2002 chat_template: None,
2003 }
2004 }
2005
2006 /// Qwen 3 14B Instruct (GGUF Q4_K_M) — ~8.4 GB.
2007 ///
2008 /// Strongest reasoning and tool-calling model with extended thinking.
2009 /// Best all-around model for macOS with 16+ GB RAM.
2010 pub fn qwen3_14b() -> Self {
2011 Self {
2012 model_id: super::models::BARTOWSKI_QWEN3_14B_GGUF.into(),
2013 files: vec![super::models::QWEN3_14B_GGUF_FILE.into()],
2014 tok_model_id: None,
2015 display_name: "Qwen 3 14B (Q4_K_M)".into(),
2016 approx_memory: "~8.4 GB".into(),
2017 chat_template: None,
2018 }
2019 }
2020
2021 /// Strong tool-calling model with extended thinking. Best balance of
2022 /// quality and memory for macOS with 24+ GB RAM.
2023 pub fn qwen3_8b() -> Self {
2024 Self {
2025 model_id: super::models::BARTOWSKI_QWEN3_8B_GGUF.into(),
2026 files: vec![super::models::QWEN3_8B_GGUF_FILE.into()],
2027 tok_model_id: None,
2028 display_name: "Qwen 3 8B (Q4_K_M)".into(),
2029 approx_memory: "~5 GB".into(),
2030 chat_template: None,
2031 }
2032 }
2033
2034 /// DeepSeek Coder v1 6.7B Instruct (GGUF Q4_K_M) — ~3.8 GB.
2035 ///
2036 /// Strong code generation model using the `llama` GGUF architecture.
2037 /// Requires 8+ GB RAM; recommended for macOS desktops and Linux.
2038 pub fn deepseek_coder_6_7b() -> Self {
2039 Self {
2040 model_id: super::models::THEBLOKE_DEEPSEEK_CODER_6_7B_INSTRUCT_GGUF.into(),
2041 files: vec![super::models::DEEPSEEK_CODER_6_7B_GGUF_FILE.into()],
2042 tok_model_id: if cfg!(target_os = "android") {
2043 Some(super::models::DEEPSEEK_CODER_6_7B_TOK_MODEL_ID.into())
2044 } else {
2045 None
2046 },
2047 display_name: "DeepSeek Coder 6.7B (Q4_K_M)".into(),
2048 approx_memory: "~3.8 GB".into(),
2049 chat_template: Some(DEEPSEEK_CODER_CHAT_TEMPLATE.into()),
2050 }
2051 }
2052
2053 /// Return the platform-appropriate default **coding** model config.
2054 ///
2055 /// - iOS / Android → Qwen 2.5 Coder 1.5B (~941 MB, fits mobile memory budgets)
2056 /// - macOS / Windows / Linux → Qwen 2.5 Coder 3B (~1.93 GB, best desktop quality)
2057 ///
2058 /// Both are dedicated coding models (trained on 5.5T code + math tokens)
2059 /// and use the `qwen2` GGUF architecture supported by mistral.rs.
2060 pub fn platform_default() -> Self {
2061 if cfg!(any(
2062 target_os = "ios",
2063 target_os = "tvos",
2064 target_os = "visionos",
2065 target_os = "watchos",
2066 target_os = "android"
2067 )) {
2068 Self::qwen25_coder_1_5b()
2069 } else {
2070 Self::qwen25_coder_3b()
2071 }
2072 }
2073 }
2074
2075 // ═════════════════════════════════════════════════════════════════════════════
2076 // Tests
2077 // ═════════════════════════════════════════════════════════════════════════════
2078
2079 #[cfg(test)]
2080 mod tests {
2081 use super::*;
2082
2083 #[test]
2084 fn truncate_for_log_short() {
2085 assert_eq!(truncate_for_log("hello", 10), "hello");
2086 }
2087
2088 #[test]
2089 fn truncate_for_log_long() {
2090 let long = "a".repeat(200);
2091 let result = truncate_for_log(&long, 50);
2092 assert!(result.ends_with("..."));
2093 assert_eq!(result.len(), 53); // 50 chars + "..."
2094 }
2095
2096 #[test]
2097 fn gguf_model_config_qwen25_1_5b() {
2098 let cfg = GgufModelConfig::qwen25_1_5b();
2099 assert!(cfg.model_id.contains("1.5B"));
2100 assert_eq!(cfg.files.len(), 1);
2101 }
2102
2103 #[test]
2104 fn gguf_model_config_qwen25_3b() {
2105 let cfg = GgufModelConfig::qwen25_3b();
2106 assert!(cfg.model_id.contains("3B"));
2107 assert_eq!(cfg.files.len(), 1);
2108 }
2109
2110 #[test]
2111 fn gguf_model_config_platform_default() {
2112 let cfg = GgufModelConfig::platform_default();
2113 // On the test host (macOS or Linux), this should be the 3B model.
2114 // On iOS/Android CI it would be 1.5B. We just check it's valid.
2115 assert!(!cfg.model_id.is_empty());
2116 assert!(!cfg.files.is_empty());
2117 }
2118
2119 #[tokio::test]
2120 async fn engine_new_is_unloaded() {
2121 let engine = ChatEngine::new();
2122 assert!(!engine.is_loaded().await);
2123 let info = engine.info().await;
2124 assert_eq!(info.status, EngineStatus::Unloaded);
2125 assert_eq!(info.history_length, 0);
2126 }
2127
2128 #[tokio::test]
2129 async fn engine_send_without_model_errors() {
2130 let engine = ChatEngine::new();
2131 let result = engine.send_message("hello").await;
2132 assert!(result.is_err());
2133 match result.unwrap_err() {
2134 InferenceError::NoModelLoaded => {} // expected
2135 other => panic!("Expected NoModelLoaded, got: {:?}", other),
2136 }
2137 }
2138
2139 #[tokio::test]
2140 async fn engine_history_empty_when_no_model() {
2141 let engine = ChatEngine::new();
2142 assert!(engine.history().await.is_empty());
2143 assert_eq!(engine.clear_history().await, 0);
2144 }
2145
2146 #[tokio::test]
2147 async fn engine_unload_when_none() {
2148 let engine = ChatEngine::new();
2149 assert!(engine.unload_model().await.is_none());
2150 }
2151 }