@ondeinference / onde / commits / 9bc5898

Add ChatEngine::complete for caller-owned conversations

ChatEngine's tool calling was stateful: send_message_with_tools and send_tool_results append to the engine's history, and push_history can only restore text. A host that owns the conversation, such as an ACP agent that persists and reloads sessions, had no way to run a tool turn on device. complete(turns, tools, sampling) takes the whole conversation as ChatTurns (system, user, assistant with its tool calls, tool result), runs the loaded model with the tools offered, and leaves the engine's history and system prompt alone. Every request was already rebuilt from history, so this adds no cost. It returns a CompletionResult with the text, tool calls, finish reason, token usage and reasoning; a leading <think> block (Qwen3 thinking models) moves from the text to reasoning. GgufModelConfig::from_supported_model_id becomes public so a host can load a catalogue model by its ID. Checked end to end with Qwen3 4B Instruct 2507: the model called the offered tool, answered from its result on the next call, and the engine's history stayed empty. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-Authored-By: siGit Code v1.6.6 <noreply@sigit.si> Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>

Seto Elkahfi committed Oct 9, 2026 at 19:52 UTC 9bc5898d401475fe52ecc8a2cd63dbac8b65394b
4 files changed +305 -6
CHANGELOG.md
+9
@@ -1,3 +1,12 @@
1 +## Unreleased
2 +
3 +### Stateless tool calling (Rust-native only)
4 +
5 +* Added `ChatEngine::complete(turns, tools, sampling)`, which runs the loaded model on a conversation the caller supplies instead of the engine's history. `turns` is a list of `ChatTurn`s (system, user, assistant with its tool calls, tool result) and is the whole conversation: the engine's history and system prompt are neither read nor changed. Hosts that keep their own conversation, such as an ACP agent that persists and reloads sessions, can now use on-device tool calling. Previously only the stateful `send_message_with_tools` and `send_tool_results` could, and `push_history` can't restore tool calls or tool results.
6 +* `complete` returns a `CompletionResult` with the reply text, the tool calls, the finish reason, prompt and completion token counts, and the model's reasoning. Reasoning comes from the model's reasoning output or, for Qwen3 thinking models, from a leading `<think>…</think>` block, which is removed from the reply text.
7 +* `GgufModelConfig::from_supported_model_id` is now public, so a host can load a model from `models::TOOL_CALLING_MODELS` or `models::SUPPORTED_MODELS` by its ID.
8 +* `ToolCallRequest` now implements `PartialEq` and `Eq`.
9 +
10 ## 1.3.1
11
12 ### Fixes
src/inference/engine.rs
+255 -4
@@ -1576,6 +1576,82 @@ impl ChatEngine {
1576 })
1577 }
1578
1579 + /// Run the loaded model on the caller's conversation, with `tools` offered.
1580 + /// Non-streaming.
1581 + ///
1582 + /// Unlike [`send_message_with_tools`](Self::send_message_with_tools), this
1583 + /// neither reads nor changes the engine's history, and the engine's system
1584 + /// prompt is not added: `turns` is the whole conversation, system message
1585 + /// included. Hosts that keep their own conversation (an ACP agent persisting
1586 + /// sessions, for example) call this every turn with the full list, assistant
1587 + /// tool calls and tool results included.
1588 + ///
1589 + /// `sampling` overrides the engine's sampling for this call only.
1590 + pub async fn complete(
1591 + &self,
1592 + turns: Vec<ChatTurn>,
1593 + tools: &[ToolDefinition],
1594 + sampling: Option<SamplingConfig>,
1595 + ) -> Result<CompletionResult, InferenceError> {
1596 + let (model, request) = {
1597 + let guard = self.inner.lock().await;
1598 + let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
1599 + let sampling = sampling.as_ref().unwrap_or(&loaded.sampling);
1600 + (
1601 + loaded.model.clone(),
1602 + build_turns_request(&turns, tools, sampling),
1603 + )
1604 + };
1605 +
1606 + log::info!(
1607 + "ChatEngine: complete START — {} turns, {} tools",
1608 + turns.len(),
1609 + tools.len()
1610 + );
1611 + let start = std::time::Instant::now();
1612 + let response =
1613 + model
1614 + .send_chat_request(request)
1615 + .await
1616 + .map_err(|e| InferenceError::Inference {
1617 + reason: e.to_string(),
1618 + })?;
1619 + let elapsed = start.elapsed();
1620 +
1621 + let choice = response
1622 + .choices
1623 + .first()
1624 + .ok_or_else(|| InferenceError::Inference {
1625 + reason: "the model returned no choices".into(),
1626 + })?;
1627 + let content = choice.message.content.clone().unwrap_or_default();
1628 + let (text, inline_reasoning) = split_think(&content);
1629 + let reasoning = choice
1630 + .message
1631 + .reasoning_content
1632 + .clone()
1633 + .filter(|r| !r.trim().is_empty())
1634 + .or(inline_reasoning);
1635 + let tool_calls = parse_tool_calls(choice);
1636 +
1637 + log::info!(
1638 + "ChatEngine: complete END — {} — tool_calls: {} — reply: \"{}\"",
1639 + format_duration(elapsed),
1640 + tool_calls.len(),
1641 + truncate_for_log(&text, 100)
1642 + );
1643 +
1644 + Ok(CompletionResult {
1645 + text,
1646 + reasoning,
1647 + tool_calls,
1648 + finish_reason: choice.finish_reason.clone(),
1649 + prompt_tokens: response.usage.prompt_tokens,
1650 + completion_tokens: response.usage.completion_tokens,
1651 + duration_secs: elapsed.as_secs_f64(),
1652 + })
1653 + }
1654 +
1655 /// Append tool results to history without starting another inference round.
1656 ///
1657 /// Agent hosts use this when a turn is cancelled after the assistant has
@@ -1948,6 +2024,85 @@ fn replay_history_with_tools(mut req: RequestBuilder, history: &[HistoryEntry])
2024 req
2025 }
2026
2027 +#[cfg(any(
2028 + target_os = "macos",
2029 + target_os = "ios",
2030 + target_os = "tvos",
2031 + target_os = "visionos",
2032 + target_os = "watchos",
2033 + target_os = "windows",
2034 + target_os = "linux",
2035 + target_os = "android"
2036 +))]
2037 +/// Build a request from a caller-supplied conversation for [`ChatEngine::complete`].
2038 +fn build_turns_request(
2039 + turns: &[ChatTurn],
2040 + tools: &[ToolDefinition],
2041 + sampling: &SamplingConfig,
2042 +) -> RequestBuilder {
2043 + let req = apply_sampling(RequestBuilder::new(), sampling);
2044 + let history: Vec<HistoryEntry> = turns.iter().cloned().map(HistoryEntry::from).collect();
2045 + let req = replay_history_with_tools(req, &history);
2046 + attach_tools(req, tools)
2047 +}
2048 +
2049 +#[cfg(any(
2050 + target_os = "macos",
2051 + target_os = "ios",
2052 + target_os = "tvos",
2053 + target_os = "visionos",
2054 + target_os = "watchos",
2055 + target_os = "windows",
2056 + target_os = "linux",
2057 + target_os = "android"
2058 +))]
2059 +impl From<ChatTurn> for HistoryEntry {
2060 + fn from(turn: ChatTurn) -> Self {
2061 + match turn {
2062 + ChatTurn::System(content) => HistoryEntry::Text(ChatMessage::system(content)),
2063 + ChatTurn::User(content) => HistoryEntry::Text(ChatMessage::user(content)),
2064 + ChatTurn::Assistant {
2065 + content,
2066 + tool_calls,
2067 + } if tool_calls.is_empty() => HistoryEntry::Text(ChatMessage::assistant(content)),
2068 + ChatTurn::Assistant {
2069 + content,
2070 + tool_calls,
2071 + } => HistoryEntry::AssistantToolCall {
2072 + content,
2073 + tool_calls,
2074 + },
2075 + ChatTurn::Tool {
2076 + tool_call_id,
2077 + content,
2078 + } => HistoryEntry::ToolResult {
2079 + tool_call_id,
2080 + content,
2081 + },
2082 + }
2083 + }
2084 +}
2085 +
2086 +/// Split a leading `<think>…</think>` block (Qwen3 thinking models) off a
2087 +/// reply: `(reply, reasoning)`. A reply without one comes back trimmed, with
2088 +/// no reasoning. An unclosed block is all reasoning: the model ran out of
2089 +/// tokens while thinking.
2090 +fn split_think(content: &str) -> (String, Option<String>) {
2091 + let trimmed = content.trim_start();
2092 + let Some(rest) = trimmed.strip_prefix("<think>") else {
2093 + return (content.trim().to_string(), None);
2094 + };
2095 + let (reasoning, reply) = match rest.split_once("</think>") {
2096 + Some((reasoning, reply)) => (reasoning, reply),
2097 + None => (rest, ""),
2098 + };
2099 + let reasoning = reasoning.trim();
2100 + (
2101 + reply.trim().to_string(),
2102 + (!reasoning.is_empty()).then(|| reasoning.to_string()),
2103 + )
2104 +}
2105 +
2106 /// Attach [`ToolDefinition`] slices as mistralrs [`Tool`] values and set
2107 /// [`ToolChoice::Auto`].
2108 #[cfg(any(
@@ -2260,6 +2415,17 @@ impl ChatEngine {
2415 reason: "LLM inference is not supported on this platform.".into(),
2416 })
2417 }
2418 +
2419 + pub async fn complete(
2420 + &self,
2421 + _turns: Vec<ChatTurn>,
2422 + _tools: &[ToolDefinition],
2423 + _sampling: Option<SamplingConfig>,
2424 + ) -> Result<CompletionResult, InferenceError> {
2425 + Err(InferenceError::Other {
2426 + reason: "LLM inference is not supported on this platform.".into(),
2427 + })
2428 + }
2429 }
2430
2431 #[cfg(not(any(
@@ -2310,10 +2476,13 @@ impl GgufModelConfig {
2476 )
2477 }
2478
2313 - /// Resolve a model-management ID to the same configuration used by the
2314 - /// inference engine. Keeping this mapping in one place prevents download
2315 - /// and load support from drifting apart.
2316 - pub(crate) fn from_supported_model_id(model_id: &str) -> Option<Self> {
2479 + /// Resolve a model-management ID (a Hugging Face repository from
2480 + /// [`SUPPORTED_MODELS`](super::models::SUPPORTED_MODELS), such as
2481 + /// `bartowski/Qwen_Qwen3-4B-Instruct-2507-GGUF`) to the same configuration
2482 + /// used by the inference engine. Keeping this mapping in one place prevents
2483 + /// download and load support from drifting apart. `None` for an ID Onde
2484 + /// doesn't support.
2485 + pub fn from_supported_model_id(model_id: &str) -> Option<Self> {
2486 match model_id {
2487 super::models::BARTOWSKI_QWEN25_0_5B_INSTRUCT_GGUF => Some(Self::qwen25_0_5b()),
2488 super::models::BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF => Some(Self::qwen25_1_5b()),
@@ -2846,4 +3015,86 @@ mod tests {
3015 let engine = ChatEngine::new();
3016 assert!(engine.unload_model().await.is_none());
3017 }
3018 +
3019 + #[tokio::test]
3020 + async fn complete_without_a_model() {
3021 + let engine = ChatEngine::new();
3022 + let err = engine
3023 + .complete(vec![ChatTurn::User("hi".into())], &[], None)
3024 + .await
3025 + .unwrap_err();
3026 + assert!(matches!(err, InferenceError::NoModelLoaded), "{err:?}");
3027 + }
3028 +
3029 + #[test]
3030 + fn split_think_separates_reasoning() {
3031 + assert_eq!(
3032 + split_think("<think>\nC major has no sharps.\n</think>\n\nIt has none."),
3033 + (
3034 + "It has none.".to_string(),
3035 + Some("C major has no sharps.".to_string())
3036 + )
3037 + );
3038 + assert_eq!(
3039 + split_think(" Plain reply. "),
3040 + ("Plain reply.".to_string(), None)
3041 + );
3042 + assert_eq!(
3043 + split_think("<think>\n\n</think>Reply"),
3044 + ("Reply".to_string(), None)
3045 + );
3046 + // Out of tokens mid-thought: no reply yet.
3047 + assert_eq!(
3048 + split_think("<think>still going"),
3049 + (String::new(), Some("still going".to_string()))
3050 + );
3051 + // Only a leading block counts.
3052 + assert_eq!(
3053 + split_think("Use <think> tags."),
3054 + ("Use <think> tags.".to_string(), None)
3055 + );
3056 + }
3057 +
3058 + #[test]
3059 + fn chat_turns_become_history_entries() {
3060 + let call = ToolCallRequest {
3061 + id: "call_1".into(),
3062 + function_name: "theory_chord".into(),
3063 + arguments: r#"{"symbol":"Cmaj7"}"#.into(),
3064 + };
3065 + let entries: Vec<HistoryEntry> = vec![
3066 + ChatTurn::System("You are helpful.".into()),
3067 + ChatTurn::User("Spell Cmaj7".into()),
3068 + ChatTurn::Assistant {
3069 + content: String::new(),
3070 + tool_calls: vec![call.clone()],
3071 + },
3072 + ChatTurn::Tool {
3073 + tool_call_id: "call_1".into(),
3074 + content: "C E G B".into(),
3075 + },
3076 + ChatTurn::Assistant {
3077 + content: "C, E, G, B.".into(),
3078 + tool_calls: vec![],
3079 + },
3080 + ]
3081 + .into_iter()
3082 + .map(HistoryEntry::from)
3083 + .collect();
3084 +
3085 + assert!(
3086 + matches!(&entries[0], HistoryEntry::Text(m) if m.role == ChatRole::System && m.content == "You are helpful.")
3087 + );
3088 + assert!(matches!(&entries[1], HistoryEntry::Text(m) if m.role == ChatRole::User));
3089 + assert!(
3090 + matches!(&entries[2], HistoryEntry::AssistantToolCall { tool_calls, .. } if tool_calls == &vec![call])
3091 + );
3092 + assert!(
3093 + matches!(&entries[3], HistoryEntry::ToolResult { tool_call_id, content } if tool_call_id == "call_1" && content == "C E G B")
3094 + );
3095 + // A plain assistant reply stays a text message, as the engine records it.
3096 + assert!(
3097 + matches!(&entries[4], HistoryEntry::Text(m) if m.role == ChatRole::Assistant && m.content == "C, E, G, B.")
3098 + );
3099 + }
3100 }
src/inference/mod.rs
+3 -1
@@ -69,6 +69,9 @@ pub use ffi::{OndeChatEngine, StreamChunkListener};
69 pub use types::{
70 ChatMessage,
71 ChatRole,
72 + // Tool calling
73 + ChatTurn,
74 + CompletionResult,
75 EngineInfo,
76 EngineStatus,
77 GgufModelConfig,
@@ -77,7 +80,6 @@ pub use types::{
80 IsqModelConfig,
81 SamplingConfig,
82 StreamChunk,
80 - // Tool calling
83 ToolAwareResult,
84 ToolCallInfo,
85 ToolCallRequest,
src/inference/types.rs
+38 -1
@@ -480,7 +480,7 @@ pub fn format_duration(d: std::time::Duration) -> String {
480 // with UniFFI derives and shared across both layers.)
481
482 /// A tool call requested by the model.
483 -#[derive(Debug, Clone)]
483 +#[derive(Debug, Clone, PartialEq, Eq)]
484 pub struct ToolCallRequest {
485 /// Unique identifier for this tool call (used to correlate results).
486 pub id: String,
@@ -507,6 +507,43 @@ pub struct ToolAwareResult {
507 pub finish_reason: String,
508 }
509
510 +/// One message of a conversation passed to
511 +/// [`ChatEngine::complete`](super::ChatEngine::complete), which runs on the
512 +/// caller's conversation instead of the engine's history (Rust-only).
513 +#[derive(Debug, Clone, PartialEq, Eq)]
514 +pub enum ChatTurn {
515 + System(String),
516 + User(String),
517 + /// An assistant reply, with the tool calls it made (empty for plain text).
518 + Assistant {
519 + content: String,
520 + tool_calls: Vec<ToolCallRequest>,
521 + },
522 + /// The result of one tool call, answering the assistant turn before it.
523 + Tool {
524 + tool_call_id: String,
525 + content: String,
526 + },
527 +}
528 +
529 +/// Result of [`ChatEngine::complete`](super::ChatEngine::complete) (Rust-only).
530 +#[derive(Debug, Clone)]
531 +pub struct CompletionResult {
532 + /// The reply text, with any `<think>` block moved to [`reasoning`](Self::reasoning).
533 + /// May be empty when the model only calls tools.
534 + pub text: String,
535 + /// The model's reasoning, for thinking models; `None` otherwise.
536 + pub reasoning: Option<String>,
537 + /// Tool calls requested by the model. Empty for a plain text reply.
538 + pub tool_calls: Vec<ToolCallRequest>,
539 + /// Finish reason reported by the model (e.g. `"stop"`, `"tool_calls"`, `"length"`).
540 + pub finish_reason: String,
541 + pub prompt_tokens: usize,
542 + pub completion_tokens: usize,
543 + /// Wall-clock inference duration in seconds.
544 + pub duration_secs: f64,
545 +}
546 +
547 #[cfg(test)]
548 mod tests {
549 use super::*;