@ondeinference / onde
Code Commits Issues Pulls Cloud Agent CI/CD
development
copilot/assess-inference-engine-complexity copilot/fix-onde-npm-sdk-v1-1-2-release development feature/diffusion-vision main
rs 667 lines 25.6 KB
Raw
1 // Copyright 2026 Splitfire AB (Onde Inference). All rights reserved.
2 // SPDX-License-Identifier: MIT OR Apache-2.0
3
4 //! Shared types for the on-device LLM inference engine.
5 //!
6 //! These types are intentionally framework-agnostic — they do **not** depend on
7 //! any UI framework. They carry UniFFI derive annotations so that
8 //! UniFFI bindgen can generate Swift / Kotlin bindings automatically.
9 //!
10 //! ## UniFFI compatibility notes
11 //!
12 //! - `usize` is **not** supported by UniFFI → we use `u64` / `u32` instead.
13 //! - Enums use `#[derive(uniffi::Enum)]`.
14 //! - Plain data structs use `#[derive(uniffi::Record)]`.
15 //! - Error enums use `#[derive(uniffi::Error)]`.
16
17 use serde::{Deserialize, Serialize};
18
19 // ── Message role ─────────────────────────────────────────────────────────────
20
21 /// Role of a message in a chat conversation.
22 ///
23 /// Mirrors the standard OpenAI / mistral.rs role taxonomy but is decoupled
24 /// from `mistralrs::TextMessageRole` so that callers don't need a direct
25 /// dependency on mistral.rs.
26 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, uniffi::Enum)]
27 #[serde(rename_all = "lowercase")]
28 pub enum ChatRole {
29 System,
30 User,
31 Assistant,
32 }
33
34 impl std::fmt::Display for ChatRole {
35 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
36 match self {
37 ChatRole::System => write!(f, "system"),
38 ChatRole::User => write!(f, "user"),
39 ChatRole::Assistant => write!(f, "assistant"),
40 }
41 }
42 }
43
44 // ── Chat message ─────────────────────────────────────────────────────────────
45
46 /// A single message in a conversation.
47 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
48 pub struct ChatMessage {
49 pub role: ChatRole,
50 pub content: String,
51 }
52
53 impl ChatMessage {
54 pub fn system(content: impl Into<String>) -> Self {
55 Self {
56 role: ChatRole::System,
57 content: content.into(),
58 }
59 }
60
61 pub fn user(content: impl Into<String>) -> Self {
62 Self {
63 role: ChatRole::User,
64 content: content.into(),
65 }
66 }
67
68 pub fn assistant(content: impl Into<String>) -> Self {
69 Self {
70 role: ChatRole::Assistant,
71 content: content.into(),
72 }
73 }
74 }
75
76 // ── Sampling configuration ───────────────────────────────────────────────────
77
78 /// Sampling parameters for text generation.
79 ///
80 /// All fields are optional — `None` means "use the engine default".
81 ///
82 /// Note: `usize` is replaced with `u64` for UniFFI compatibility.
83 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
84 pub struct SamplingConfig {
85 /// Sampling temperature (higher = more random). Typical range: 0.0–2.0.
86 pub temperature: Option<f64>,
87 /// Nucleus (top-p) sampling threshold. Typical value: 0.9–0.95.
88 pub top_p: Option<f64>,
89 /// Top-k sampling limit.
90 pub top_k: Option<u64>,
91 /// Min-p sampling threshold.
92 pub min_p: Option<f64>,
93 /// Maximum number of tokens to generate.
94 pub max_tokens: Option<u64>,
95 /// Frequency penalty (penalise tokens proportional to occurrence count).
96 pub frequency_penalty: Option<f32>,
97 /// Presence penalty (penalise tokens that appeared at all).
98 pub presence_penalty: Option<f32>,
99 }
100
101 impl Default for SamplingConfig {
102 /// Sensible creative-chat defaults.
103 fn default() -> Self {
104 Self {
105 temperature: Some(0.7),
106 top_p: Some(0.95),
107 top_k: None,
108 min_p: None,
109 max_tokens: Some(512),
110 frequency_penalty: None,
111 presence_penalty: None,
112 }
113 }
114 }
115
116 impl SamplingConfig {
117 /// Defaults for models that emit an extended reasoning block before the
118 /// user-visible answer.
119 pub fn reasoning() -> Self {
120 Self {
121 max_tokens: Some(4096),
122 ..Self::default()
123 }
124 }
125
126 /// Deterministic sampling (temperature = 0, greedy decoding).
127 pub fn deterministic() -> Self {
128 Self {
129 temperature: Some(0.0),
130 top_p: None,
131 top_k: None,
132 min_p: None,
133 max_tokens: Some(512),
134 frequency_penalty: None,
135 presence_penalty: None,
136 }
137 }
138
139 /// Conservative mobile defaults — lower max_tokens for faster response on
140 /// constrained devices (CPU-only ARM SoCs doing ~1-3 tok/sec).
141 pub fn mobile() -> Self {
142 Self {
143 temperature: Some(0.7),
144 top_p: Some(0.95),
145 top_k: None,
146 min_p: None,
147 max_tokens: Some(128),
148 frequency_penalty: None,
149 presence_penalty: None,
150 }
151 }
152
153 /// Sampling config tuned for coding assistants.
154 ///
155 /// Key differences from the general-purpose default:
156 ///
157 /// - **`frequency_penalty = 1.15`** — the primary fix for the repeated-token
158 /// loop you saw with small GGUF models. Penalises each token proportional
159 /// to how many times it has already appeared in the output, making it
160 /// progressively harder to emit the same word again. 1.15 is aggressive
161 /// enough to kill echo loops without hurting code token repetition
162 /// (identifiers, keywords) at typical response lengths.
163 ///
164 /// - **`presence_penalty = 0.1`** — a light nudge away from any token that
165 /// has appeared at all, encouraging lexical variety across the response.
166 ///
167 /// - **`temperature = 0.2`** — low randomness keeps generated code
168 /// deterministic and syntactically correct. High temperature causes
169 /// hallucinated variable names and broken syntax in code blocks.
170 ///
171 /// - **`top_p = 0.95`** — nucleus sampling retained to avoid degenerate
172 /// greedy outputs while still keeping the distribution tight.
173 ///
174 /// - **`max_tokens = 1024`** — doubled vs. the chat default to allow
175 /// complete function / class explanations without truncation.
176 pub fn coding() -> Self {
177 Self {
178 temperature: Some(0.2),
179 top_p: Some(0.95),
180 top_k: None,
181 min_p: None,
182 max_tokens: Some(1024),
183 frequency_penalty: Some(1.15),
184 presence_penalty: Some(0.1),
185 }
186 }
187
188 /// Like [`coding`](Self::coding) but capped at 256 tokens for mobile
189 /// devices where latency matters more than completeness.
190 pub fn coding_mobile() -> Self {
191 Self {
192 max_tokens: Some(256),
193 ..Self::coding()
194 }
195 }
196 }
197
198 // ── GGUF model configuration ─────────────────────────────────────────────────
199
200 /// Configuration for loading a GGUF model via mistral.rs.
201 ///
202 /// This is the primary model format for on-device mobile/desktop inference
203 /// because GGUF files are pre-quantized and load at their compressed size,
204 /// avoiding the large memory spike of in-situ quantization.
205 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
206 pub struct GgufModelConfig {
207 /// HuggingFace model repository ID, e.g. `"bartowski/Qwen2.5-1.5B-Instruct-GGUF"`.
208 pub model_id: String,
209 /// GGUF filename(s) within the repository, e.g. `["Qwen2.5-1.5B-Instruct-Q4_K_M.gguf"]`.
210 pub files: Vec<String>,
211 /// Optional: explicit tokenizer model ID (required on Android where GGUF
212 /// embedded tokenizers are not supported by the candle backend).
213 pub tok_model_id: Option<String>,
214 /// Human-friendly display name, e.g. `"Qwen 2.5 1.5B"`.
215 pub display_name: String,
216 /// Approximate memory footprint description, e.g. `"~941 MB (GGUF Q4_K_M)"`.
217 pub approx_memory: String,
218 /// Optional Jinja chat template string. When set, this overrides the
219 /// template embedded in the GGUF file (if any). Required for older GGUF
220 /// files that don't embed a chat template (e.g. TheBloke models).
221 pub chat_template: Option<String>,
222 }
223
224 // ── UQFF model configuration ─────────────────────────────────────────────────
225
226 /// Configuration for loading a UQFF model via mistral.rs
227 /// `UqffTextModelBuilder`.
228 ///
229 /// UQFF files store pre-quantised weights and load directly
230 /// without the runtime memory spike of in-situ quantisation. `model_id` points
231 /// at the UQFF *export* — the directory or repository holding the shards
232 /// alongside `residual.safetensors`, `config.json`, and the tokenizer — and
233 /// `files` identifies the first UQFF shard or explicit UQFF shard files.
234 ///
235 /// Note that `model_id` is **not** the original unquantised model repository:
236 /// mistral.rs resolves the shards, the residual weights, and the JSON assets by
237 /// sibling lookup next to `model_id`, so pointing it at the source repo fails
238 /// with a missing-file error. A UQFF export is self-contained by design.
239 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
240 pub struct UqffModelConfig {
241 /// HuggingFace repository ID or local directory of the UQFF export, e.g.
242 /// `"mistralrs-community/gemma-4-E4B-it-UQFF"`.
243 pub model_id: String,
244 /// UQFF filename(s) within the repository. For sharded UQFFs, passing the
245 /// first shard, e.g. `["q4k-0.uqff"]`, is enough for mistral.rs to discover
246 /// siblings. Numeric and ISQ-name shorthands such as `"4"` and `"q4k"` are
247 /// also accepted by the underlying loader.
248 pub files: Vec<String>,
249 /// Human-friendly display name, e.g. `"Gemma 4 E4B (UQFF Q4K)"`.
250 pub display_name: String,
251 /// Approximate memory footprint description, e.g. `"~2.5 GB (UQFF Q4K)"`.
252 pub approx_memory: String,
253 /// Optional Jinja chat template string. When set, this overrides the
254 /// template resolved from the base model.
255 pub chat_template: Option<String>,
256 }
257
258 // ── ISQ model configuration ──────────────────────────────────────────────────
259
260 /// Configuration for loading an ISQ (in-situ quantised) model via mistral.rs
261 /// `TextModelBuilder`.
262 ///
263 /// Unlike GGUF models, ISQ models are loaded directly from a HuggingFace
264 /// safetensors repo and quantised in-situ on the device. They require more
265 /// RAM during the load phase but support a wider range of architectures
266 /// (e.g. DeepSeek-Coder-V2-Lite, Qwen2.5-Coder-7B).
267 ///
268 /// **macOS-only for now** — ISQ with Metal acceleration requires the `metal`
269 /// feature on mistral.rs. The struct is still defined on all platforms so
270 /// that types.rs compiles everywhere without `cfg` gates on the record
271 /// (UniFFI requires all record types to be reachable without feature flags).
272 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
273 pub struct IsqModelConfig {
274 /// HuggingFace model repository ID, e.g. `"Qwen/Qwen2.5-Coder-7B-Instruct"`.
275 pub model_id: String,
276 /// Number of quantisation bits. `4` → Q4K (AFQ4 on Metal), `8` → Q8_0 (AFQ8 on Metal).
277 /// Any value outside `{4, 8}` defaults to 4-bit.
278 pub isq_bits: u8,
279 /// Human-friendly display name, e.g. `"Qwen 2.5 Coder 7B"`.
280 pub display_name: String,
281 /// Approximate memory footprint description, e.g. `"~4.5 GB (Q4K, Metal)"`.
282 pub approx_memory: String,
283 }
284
285 impl IsqModelConfig {
286 /// Qwen 2.5 Coder 7B Instruct — ISQ 4-bit, Metal-accelerated (~4.5 GB).
287 ///
288 /// Best macOS desktop coding quality. Loaded from the official HF repo
289 /// and quantised in-situ; requires ~8 GB RAM during the load phase.
290 pub fn qwen25_coder_7b_isq4() -> Self {
291 Self {
292 model_id: super::models::QWEN25_CODER_7B_INSTRUCT.into(),
293 isq_bits: 4,
294 display_name: "Qwen 2.5 Coder 7B (ISQ 4-bit)".into(),
295 approx_memory: "~4.5 GB (ISQ Q4K, Metal)".into(),
296 }
297 }
298
299 /// Qwen 2.5 Coder 7B Instruct — ISQ 8-bit, Metal-accelerated (~9 GB).
300 ///
301 /// Higher quality than the 4-bit variant; requires ~12 GB RAM on load.
302 /// Suitable for Macs with 16 GB+ unified memory.
303 pub fn qwen25_coder_7b_isq8() -> Self {
304 Self {
305 model_id: super::models::QWEN25_CODER_7B_INSTRUCT.into(),
306 isq_bits: 8,
307 display_name: "Qwen 2.5 Coder 7B (ISQ 8-bit)".into(),
308 approx_memory: "~9 GB (ISQ Q8K, Metal)".into(),
309 }
310 }
311 }
312
313 // ── Inference result ─────────────────────────────────────────────────────────
314
315 /// The result of a chat inference request.
316 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
317 pub struct InferenceResult {
318 /// The generated assistant reply text.
319 pub text: String,
320 /// Wall-clock duration of the inference in seconds.
321 pub duration_secs: f64,
322 /// Human-readable duration string (e.g. `"2m 3.1s"` or `"4.5s"`).
323 pub duration_display: String,
324 /// Finish reason reported by the model (e.g. `"stop"`, `"length"`).
325 pub finish_reason: String,
326 /// Tool calls requested by the model (empty when no tools were invoked).
327 pub tool_calls: Vec<ToolCallInfo>,
328 }
329
330 // ── Tool calling ─────────────────────────────────────────────────────────────
331
332 /// A tool definition passed to the model so it knows which tools are available.
333 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
334 pub struct ToolDefinition {
335 /// Tool name (e.g. `"read_file"`).
336 pub name: String,
337 /// Human-readable description of what the tool does.
338 pub description: String,
339 /// JSON Schema string describing the tool's parameters.
340 pub parameters_schema: String,
341 }
342
343 /// Information about a single tool call requested by the model.
344 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
345 pub struct ToolCallInfo {
346 /// Unique identifier for this tool call (used to correlate results).
347 pub id: String,
348 /// The name of the function the model wants to invoke.
349 pub function_name: String,
350 /// JSON-encoded arguments for the function.
351 pub arguments: String,
352 }
353
354 /// The result of executing a tool, sent back to the model.
355 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
356 pub struct ToolResult {
357 /// The tool call ID this result corresponds to.
358 pub tool_call_id: String,
359 /// The output produced by executing the tool.
360 pub content: String,
361 }
362
363 /// Whether the loaded model can reliably produce structured tool calls.
364 ///
365 /// `Unknown` is used for custom model repositories that Onde does not ship in
366 /// its built-in catalogue. Callers may choose to try those models, but should
367 /// surface that the capability has not been verified.
368 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, uniffi::Enum)]
369 pub enum ToolCallingSupport {
370 Supported,
371 Unsupported,
372 Unknown,
373 }
374
375 // ── Streaming chunk ──────────────────────────────────────────────────────────
376
377 /// A single streaming token chunk from the inference engine.
378 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
379 pub struct StreamChunk {
380 /// The token text delta (may be empty for the final chunk).
381 pub delta: String,
382 /// Whether this is the final chunk in the stream.
383 pub done: bool,
384 /// Finish reason (only set on the final chunk).
385 pub finish_reason: Option<String>,
386 }
387
388 // ── Engine status ────────────────────────────────────────────────────────────
389
390 /// Lifecycle status of the inference engine.
391 #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, uniffi::Enum)]
392 #[serde(rename_all = "snake_case")]
393 pub enum EngineStatus {
394 /// No model loaded.
395 Unloaded,
396 /// Model is being downloaded / loaded into memory.
397 Loading,
398 /// Model is loaded and ready to accept requests.
399 Ready,
400 /// Model is currently running inference.
401 Generating,
402 /// An error occurred (model may or may not still be loaded).
403 Error,
404 }
405
406 impl std::fmt::Display for EngineStatus {
407 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
408 match self {
409 EngineStatus::Unloaded => write!(f, "unloaded"),
410 EngineStatus::Loading => write!(f, "loading"),
411 EngineStatus::Ready => write!(f, "ready"),
412 EngineStatus::Generating => write!(f, "generating"),
413 EngineStatus::Error => write!(f, "error"),
414 }
415 }
416 }
417
418 // ── Engine info snapshot ─────────────────────────────────────────────────────
419
420 /// A point-in-time snapshot of the engine's state, suitable for status UIs.
421 ///
422 /// Note: `history_length` is `u64` (not `usize`) for UniFFI compatibility.
423 #[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
424 pub struct EngineInfo {
425 pub status: EngineStatus,
426 /// Display name of the currently loaded model, if any.
427 pub model_name: Option<String>,
428 /// Approximate memory footprint description, if a model is loaded.
429 pub approx_memory: Option<String>,
430 /// Number of conversation turns in the current history.
431 pub history_length: u64,
432 }
433
434 // ── Error type ───────────────────────────────────────────────────────────────
435
436 /// Errors that can occur during inference engine operations.
437 ///
438 /// Exposed to Swift / Kotlin as a sealed class / enum via `uniffi::Error`.
439 #[derive(Debug, thiserror::Error, uniffi::Error)]
440 pub enum InferenceError {
441 #[error("No model loaded — call `load_model` first")]
442 NoModelLoaded,
443
444 #[error("Model is already loaded: {model_name}")]
445 AlreadyLoaded { model_name: String },
446
447 #[error("Failed to build model: {reason}")]
448 ModelBuild { reason: String },
449
450 #[error("Inference failed: {reason}")]
451 Inference { reason: String },
452
453 #[error("Model loading was cancelled")]
454 Cancelled,
455
456 #[error("{reason}")]
457 Other { reason: String },
458 }
459
460 // ── Duration formatting helper ───────────────────────────────────────────────
461
462 /// Format a `std::time::Duration` as `Xm Ys` or just `Ys` when under a minute.
463 pub fn format_duration(d: std::time::Duration) -> String {
464 let total_secs = d.as_secs_f64();
465 let mins = (total_secs / 60.0).floor() as u64;
466 let secs = total_secs - (mins as f64 * 60.0);
467 if mins > 0 {
468 format!("{}m {:.1}s", mins, secs)
469 } else {
470 format!("{:.1}s", secs)
471 }
472 }
473
474 // ── Tool calling types (Rust-only) ──────────────────────────────────────────
475 //
476 // These types are intentionally NOT annotated with `uniffi::Record` /
477 // `uniffi::Enum` — they are for Rust consumers only. The UniFFI surface
478 // area (Swift / Kotlin) remains unchanged.
479 // (`ToolDefinition`, `ToolCallInfo`, and `ToolResult` are defined above
480 // with UniFFI derives and shared across both layers.)
481
482 /// A tool call requested by the model.
483 #[derive(Debug, Clone, PartialEq, Eq)]
484 pub struct ToolCallRequest {
485 /// Unique identifier for this tool call (used to correlate results).
486 pub id: String,
487 /// Name of the function the model wants to invoke.
488 pub function_name: String,
489 /// JSON string of the function arguments.
490 pub arguments: String,
491 }
492
493 /// Result from a tool-aware inference call (Rust-only, not UniFFI-exported).
494 #[derive(Debug, Clone)]
495 pub struct ToolAwareResult {
496 /// Text content of the assistant reply (may be empty when the model
497 /// decides to call tools instead of responding with text).
498 pub text: String,
499 /// Tool calls requested by the model. Empty when the model responds
500 /// with text only.
501 pub tool_calls: Vec<ToolCallRequest>,
502 /// Wall-clock inference duration in seconds.
503 pub duration_secs: f64,
504 /// Human-readable duration string (e.g. `"1.3s"`).
505 pub duration_display: String,
506 /// Finish reason reported by the model (e.g. `"stop"`, `"tool_calls"`).
507 pub finish_reason: String,
508 }
509
510 /// One message of a conversation passed to
511 /// [`ChatEngine::complete`](super::ChatEngine::complete), which runs on the
512 /// caller's conversation instead of the engine's history (Rust-only).
513 #[derive(Debug, Clone, PartialEq, Eq)]
514 pub enum ChatTurn {
515 System(String),
516 User(String),
517 /// An assistant reply, with the tool calls it made (empty for plain text).
518 Assistant {
519 content: String,
520 tool_calls: Vec<ToolCallRequest>,
521 },
522 /// The result of one tool call, answering the assistant turn before it.
523 Tool {
524 tool_call_id: String,
525 content: String,
526 },
527 }
528
529 /// Result of [`ChatEngine::complete`](super::ChatEngine::complete) (Rust-only).
530 #[derive(Debug, Clone)]
531 pub struct CompletionResult {
532 /// The reply text, with any `<think>` block moved to [`reasoning`](Self::reasoning).
533 /// May be empty when the model only calls tools.
534 pub text: String,
535 /// The model's reasoning, for thinking models; `None` otherwise.
536 pub reasoning: Option<String>,
537 /// Tool calls requested by the model. Empty for a plain text reply.
538 pub tool_calls: Vec<ToolCallRequest>,
539 /// Finish reason reported by the model (e.g. `"stop"`, `"tool_calls"`, `"length"`).
540 pub finish_reason: String,
541 pub prompt_tokens: usize,
542 pub completion_tokens: usize,
543 /// Wall-clock inference duration in seconds.
544 pub duration_secs: f64,
545 }
546
547 #[cfg(test)]
548 mod tests {
549 use super::*;
550
551 #[test]
552 fn chat_message_constructors() {
553 let sys = ChatMessage::system("You are helpful.");
554 assert_eq!(sys.role, ChatRole::System);
555 assert_eq!(sys.content, "You are helpful.");
556
557 let user = ChatMessage::user("Hello");
558 assert_eq!(user.role, ChatRole::User);
559
560 let asst = ChatMessage::assistant("Hi there!");
561 assert_eq!(asst.role, ChatRole::Assistant);
562 }
563
564 #[test]
565 fn chat_role_display() {
566 assert_eq!(ChatRole::System.to_string(), "system");
567 assert_eq!(ChatRole::User.to_string(), "user");
568 assert_eq!(ChatRole::Assistant.to_string(), "assistant");
569 }
570
571 #[test]
572 fn chat_role_serde_rename() {
573 // Verify that #[serde(rename_all = "lowercase")] is applied by
574 // checking the Display impl matches the expected serialised form.
575 // (Full serde_json round-trip is tested in integration tests that
576 // already depend on serde_json.)
577 assert_eq!(format!("{}", ChatRole::System), "system");
578 assert_eq!(format!("{}", ChatRole::User), "user");
579 assert_eq!(format!("{}", ChatRole::Assistant), "assistant");
580 }
581
582 #[test]
583 fn sampling_config_defaults() {
584 let cfg = SamplingConfig::default();
585 assert_eq!(cfg.temperature, Some(0.7));
586 assert_eq!(cfg.top_p, Some(0.95));
587 assert_eq!(cfg.max_tokens, Some(512));
588 }
589
590 #[test]
591 fn sampling_config_deterministic() {
592 let cfg = SamplingConfig::deterministic();
593 assert_eq!(cfg.temperature, Some(0.0));
594 assert!(cfg.top_p.is_none());
595 }
596
597 #[test]
598 fn sampling_config_mobile() {
599 let cfg = SamplingConfig::mobile();
600 assert_eq!(cfg.max_tokens, Some(128));
601 }
602
603 #[test]
604 fn sampling_config_reasoning_has_extended_budget() {
605 let config = SamplingConfig::reasoning();
606 assert_eq!(config.max_tokens, Some(4096));
607 }
608
609 #[test]
610 fn uqff_model_config_accepts_first_shard() {
611 let config = UqffModelConfig {
612 model_id: "mistralrs-community/gemma-4-E4B-it-UQFF".into(),
613 files: vec!["q4k-0.uqff".into()],
614 display_name: "Gemma 4 E4B (UQFF Q4K)".into(),
615 approx_memory: "~2.5 GB (UQFF Q4K)".into(),
616 chat_template: None,
617 };
618
619 assert_eq!(config.model_id, "mistralrs-community/gemma-4-E4B-it-UQFF");
620 assert_eq!(config.files, vec!["q4k-0.uqff"]);
621 }
622
623 #[test]
624 fn engine_status_display() {
625 assert_eq!(EngineStatus::Ready.to_string(), "ready");
626 assert_eq!(EngineStatus::Generating.to_string(), "generating");
627 }
628
629 #[test]
630 fn format_duration_under_minute() {
631 let d = std::time::Duration::from_secs_f64(4.567);
632 assert_eq!(format_duration(d), "4.6s");
633 }
634
635 #[test]
636 fn format_duration_over_minute() {
637 let d = std::time::Duration::from_secs_f64(125.3);
638 assert_eq!(format_duration(d), "2m 5.3s");
639 }
640
641 #[test]
642 fn inference_error_display() {
643 let err = InferenceError::NoModelLoaded;
644 assert_eq!(err.to_string(), "No model loaded — call `load_model` first");
645
646 let err = InferenceError::ModelBuild {
647 reason: "out of memory".into(),
648 };
649 assert_eq!(err.to_string(), "Failed to build model: out of memory");
650
651 let err = InferenceError::AlreadyLoaded {
652 model_name: "Qwen 2.5".into(),
653 };
654 assert_eq!(err.to_string(), "Model is already loaded: Qwen 2.5");
655 }
656
657 #[test]
658 fn engine_info_history_is_u64() {
659 let info = EngineInfo {
660 status: EngineStatus::Ready,
661 model_name: Some("Test".into()),
662 approx_memory: None,
663 history_length: 42,
664 };
665 assert_eq!(info.history_length, 42u64);
666 }
667 }