@ondeinference / onde / commits / 4219517

Add tool-calling support to ChatEngine and types

- Implement send_message_with_tools and send_tool_results in ChatEngine - Add ToolDefinition, ToolCallInfo, and ToolResult types - Extend InferenceResult with tool_calls field - Re-export new types from inference module - Bump version to 0.1.8

keypair34 committed Apr 24, 2026 at 08:26 UTC 4219517af10511bb42e2375574f5c4bc35f88bb1
6 files changed +283 -4
Cargo.lock
+1 -1
@@ -3637,7 +3637,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
3637
3638 [[package]]
3639 name = "onde"
3640 -version = "0.1.7"
3640 +version = "0.1.8"
3641 dependencies = [
3642 "anyhow",
3643 "cc",
Cargo.toml
+1 -1
@@ -1,6 +1,6 @@
1 [package]
2 name = "onde"
3 -version = "0.1.7"
3 +version = "0.1.8"
4 edition = "2021"
5 description = "On-device inference engine for Apple silicon."
6 license = "MIT OR Apache-2.0"
src/inference/engine.rs
+243 -1
@@ -94,7 +94,10 @@ use super::types::*;
94 target_os = "linux",
95 target_os = "android"
96 ))]
97 -use mistralrs::{GgufModelBuilder, Model, RequestBuilder, TextMessageRole};
97 +use mistralrs::{
98 + Function, GgufModelBuilder, Model, RequestBuilder, TextMessageRole, Tool as MistralTool,
99 + ToolType,
100 +};
101
102 // ISQ types are only used by load_isq_model, which is macOS-only.
103 #[cfg(target_os = "macos")]
@@ -788,6 +791,7 @@ impl ChatEngine {
791 duration_secs: elapsed.as_secs_f64(),
792 duration_display: format_duration(elapsed),
793 finish_reason,
794 + tool_calls: vec![],
795 })
796 }
797
@@ -850,6 +854,224 @@ impl ChatEngine {
854 duration_secs: elapsed.as_secs_f64(),
855 duration_display: format_duration(elapsed),
856 finish_reason,
857 + tool_calls: vec![],
858 + })
859 + }
860 +
861 + // ── Tool-calling inference ───────────────────────────────────────────
862 +
863 + /// Send a user message with tool definitions and receive a complete
864 + /// response that may contain tool calls.
865 + ///
866 + /// The user message is appended to the conversation history. If the
867 + /// model responds with text (no tool calls), the assistant reply is
868 + /// also appended. If the model requests tool calls, the caller is
869 + /// responsible for executing them and calling [`send_tool_results`].
870 + pub async fn send_message_with_tools(
871 + &self,
872 + user_message: &str,
873 + tools: &[ToolDefinition],
874 + ) -> Result<InferenceResult, InferenceError> {
875 + let (model, request) = {
876 + let guard = self.inner.lock().await;
877 + let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
878 + let mut req = self::build_request(loaded, user_message);
879 +
880 + // Convert ToolDefinitions → mistralrs Tool objects.
881 + let mistral_tools: Vec<MistralTool> = tools
882 + .iter()
883 + .map(|t| {
884 + let params: std::collections::HashMap<String, serde_json::Value> =
885 + serde_json::from_str(&t.parameters_schema).unwrap_or_default();
886 + MistralTool {
887 + tp: ToolType::Function,
888 + function: Function {
889 + description: Some(t.description.clone()),
890 + name: t.name.clone(),
891 + parameters: Some(params),
892 + strict: None,
893 + },
894 + }
895 + })
896 + .collect();
897 +
898 + req = req.set_tools(mistral_tools);
899 + (loaded.model.clone(), req)
900 + };
901 +
902 + log::info!(
903 + "ChatEngine: tool inference START — message: \"{}\"",
904 + truncate_for_log(user_message, 100)
905 + );
906 +
907 + let start = std::time::Instant::now();
908 + let response =
909 + model
910 + .send_chat_request(request)
911 + .await
912 + .map_err(|e| InferenceError::Inference {
913 + reason: e.to_string(),
914 + })?;
915 + let elapsed = start.elapsed();
916 +
917 + let choice = &response.choices[0];
918 + let reply = choice
919 + .message
920 + .content
921 + .as_ref()
922 + .map(|c| c.trim().to_string())
923 + .unwrap_or_default();
924 + let finish_reason = choice.finish_reason.clone();
925 +
926 + // Extract tool calls from the response.
927 + let tool_calls: Vec<ToolCallInfo> = choice
928 + .message
929 + .tool_calls
930 + .as_ref()
931 + .map(|tcs| {
932 + tcs.iter()
933 + .map(|tc| ToolCallInfo {
934 + id: tc.id.clone(),
935 + function_name: tc.function.name.clone(),
936 + arguments: tc.function.arguments.clone(),
937 + })
938 + .collect()
939 + })
940 + .unwrap_or_default();
941 +
942 + log::info!(
943 + "ChatEngine: tool inference END — {} — {} tool call(s), reply: \"{}\"",
944 + format_duration(elapsed),
945 + tool_calls.len(),
946 + truncate_for_log(&reply, 100)
947 + );
948 +
949 + // Persist the user message to history.
950 + {
951 + let mut guard = self.inner.lock().await;
952 + if let Some(loaded) = guard.as_mut() {
953 + loaded.history.push(ChatMessage::user(user_message));
954 + // Only persist the assistant reply if there are no tool calls.
955 + if tool_calls.is_empty() && !reply.is_empty() {
956 + loaded.history.push(ChatMessage::assistant(reply.clone()));
957 + }
958 + }
959 + }
960 +
961 + Ok(InferenceResult {
962 + text: reply,
963 + duration_secs: elapsed.as_secs_f64(),
964 + duration_display: format_duration(elapsed),
965 + finish_reason,
966 + tool_calls,
967 + })
968 + }
969 +
970 + /// Feed tool execution results back to the model and get the next
971 + /// response.
972 + ///
973 + /// `next_tools` controls whether the model may request further tool
974 + /// calls. Pass `Some(tools)` to allow another round, or `None` to
975 + /// force a text-only response.
976 + pub async fn send_tool_results(
977 + &self,
978 + results: Vec<ToolResult>,
979 + next_tools: Option<&[ToolDefinition]>,
980 + ) -> Result<InferenceResult, InferenceError> {
981 + let (model, request) = {
982 + let guard = self.inner.lock().await;
983 + let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;
984 +
985 + let mut req = RequestBuilder::new();
986 + req = apply_sampling(req, &loaded.sampling);
987 +
988 + // System prompt.
989 + if let Some(ref sp) = loaded.system_prompt {
990 + req = req.add_message(TextMessageRole::System, sp);
991 + }
992 +
993 + // Replay history.
994 + for turn in &loaded.history {
995 + req = req.add_message(chat_role_to_mistral(&turn.role), &turn.content);
996 + }
997 +
998 + // Append tool results as tool-role messages.
999 + for tr in &results {
1000 + req = req.add_tool_message(&tr.content, &tr.tool_call_id);
1001 + }
1002 +
1003 + // Optionally allow further tool calls.
1004 + if let Some(tools) = next_tools {
1005 + let mistral_tools: Vec<MistralTool> = tools
1006 + .iter()
1007 + .map(|t| {
1008 + let params: std::collections::HashMap<String, serde_json::Value> =
1009 + serde_json::from_str(&t.parameters_schema).unwrap_or_default();
1010 + MistralTool {
1011 + tp: ToolType::Function,
1012 + function: Function {
1013 + description: Some(t.description.clone()),
1014 + name: t.name.clone(),
1015 + parameters: Some(params),
1016 + strict: None,
1017 + },
1018 + }
1019 + })
1020 + .collect();
1021 + req = req.set_tools(mistral_tools);
1022 + }
1023 +
1024 + (loaded.model.clone(), req)
1025 + };
1026 +
1027 + let start = std::time::Instant::now();
1028 + let response =
1029 + model
1030 + .send_chat_request(request)
1031 + .await
1032 + .map_err(|e| InferenceError::Inference {
1033 + reason: e.to_string(),
1034 + })?;
1035 + let elapsed = start.elapsed();
1036 +
1037 + let choice = &response.choices[0];
1038 + let reply = choice
1039 + .message
1040 + .content
1041 + .as_ref()
1042 + .map(|c| c.trim().to_string())
1043 + .unwrap_or_default();
1044 + let finish_reason = choice.finish_reason.clone();
1045 +
1046 + let tool_calls: Vec<ToolCallInfo> = choice
1047 + .message
1048 + .tool_calls
1049 + .as_ref()
1050 + .map(|tcs| {
1051 + tcs.iter()
1052 + .map(|tc| ToolCallInfo {
1053 + id: tc.id.clone(),
1054 + function_name: tc.function.name.clone(),
1055 + arguments: tc.function.arguments.clone(),
1056 + })
1057 + .collect()
1058 + })
1059 + .unwrap_or_default();
1060 +
1061 + // Persist the assistant reply if no further tool calls.
1062 + if tool_calls.is_empty() && !reply.is_empty() {
1063 + let mut guard = self.inner.lock().await;
1064 + if let Some(loaded) = guard.as_mut() {
1065 + loaded.history.push(ChatMessage::assistant(reply.clone()));
1066 + }
1067 + }
1068 +
1069 + Ok(InferenceResult {
1070 + text: reply,
1071 + duration_secs: elapsed.as_secs_f64(),
1072 + duration_display: format_duration(elapsed),
1073 + finish_reason,
1074 + tool_calls,
1075 })
1076 }
1077
@@ -1223,6 +1445,26 @@ impl ChatEngine {
1445 reason: "LLM inference is not supported on this platform.".into(),
1446 })
1447 }
1448 +
1449 + pub async fn send_message_with_tools(
1450 + &self,
1451 + _user_message: &str,
1452 + _tools: &[ToolDefinition],
1453 + ) -> Result<InferenceResult, InferenceError> {
1454 + Err(InferenceError::Other {
1455 + reason: "LLM inference is not supported on this platform.".into(),
1456 + })
1457 + }
1458 +
1459 + pub async fn send_tool_results(
1460 + &self,
1461 + _results: Vec<ToolResult>,
1462 + _next_tools: Option<&[ToolDefinition]>,
1463 + ) -> Result<InferenceResult, InferenceError> {
1464 + Err(InferenceError::Other {
1465 + reason: "LLM inference is not supported on this platform.".into(),
1466 + })
1467 + }
1468 }
1469
1470 #[cfg(not(any(
src/inference/mod.rs
+2 -1
@@ -68,5 +68,6 @@ pub use engine::ChatEngine;
68 pub use ffi::{OndeChatEngine, StreamChunkListener};
69 pub use types::{
70 ChatMessage, ChatRole, EngineInfo, EngineStatus, GgufModelConfig, InferenceError,
71 - InferenceResult, IsqModelConfig, SamplingConfig, StreamChunk,
71 + InferenceResult, IsqModelConfig, SamplingConfig, StreamChunk, ToolCallInfo, ToolDefinition,
72 + ToolResult,
73 };
src/inference/token.rs
+1
@@ -63,6 +63,7 @@ const BUILD_TIME_HF_TOKEN: Option<&str> = option_env!("HF_TOKEN");
63 target_os = "visionos",
64 target_os = "watchos",
65 target_os = "windows",
66 + target_os = "linux",
67 target_os = "android"
68 ))]
69 pub fn hf_token_source() -> TokenSource {
src/inference/types.rs
+35
@@ -273,6 +273,41 @@ pub struct InferenceResult {
273 pub duration_display: String,
274 /// Finish reason reported by the model (e.g. `"stop"`, `"length"`).
275 pub finish_reason: String,
276 + /// Tool calls requested by the model (empty when no tools were invoked).
277 + pub tool_calls: Vec<ToolCallInfo>,
278 +}
279 +
280 +// ── Tool calling ─────────────────────────────────────────────────────────────
281 +
282 +/// A tool definition passed to the model so it knows which tools are available.
283 +#[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
284 +pub struct ToolDefinition {
285 + /// Tool name (e.g. `"read_file"`).
286 + pub name: String,
287 + /// Human-readable description of what the tool does.
288 + pub description: String,
289 + /// JSON Schema string describing the tool's parameters.
290 + pub parameters_schema: String,
291 +}
292 +
293 +/// Information about a single tool call requested by the model.
294 +#[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
295 +pub struct ToolCallInfo {
296 + /// Unique identifier for this tool call (used to correlate results).
297 + pub id: String,
298 + /// The name of the function the model wants to invoke.
299 + pub function_name: String,
300 + /// JSON-encoded arguments for the function.
301 + pub arguments: String,
302 +}
303 +
304 +/// The result of executing a tool, sent back to the model.
305 +#[derive(Debug, Clone, Serialize, Deserialize, uniffi::Record)]
306 +pub struct ToolResult {
307 + /// The tool call ID this result corresponds to.
308 + pub tool_call_id: String,
309 + /// The output produced by executing the tool.
310 + pub content: String,
311 }
312
313 // ── Streaming chunk ──────────────────────────────────────────────────────────