@ondeinference / onde / commits / 2f66cf9

wip

keypair34 committed Apr 14, 2026 at 18:43 UTC 2f66cf9a0ed53b506ffe57713e2e05d7ebcdba92

6 files changed +679 -13

Cargo.lock

@@ -3628,6 +3628,7 @@ dependencies = [

3628 "mistralrs",

3629 "mistralrs-core",

3630 "serde",

3631 + "serde_json",

3632 "thiserror 2.0.18",

3633 "tokio",

3634 "tsync",

Cargo.toml

@@ -25,6 +25,7 @@ anyhow = "1"

25 home = "0.5.4"

26 log = "^0.4"

27 serde = { version = "1.0", features = ["derive"] }

28 +serde_json = "1.0"

29 tokio = { version = "1", features = ["macros", "rt", "sync"] }

30 tsync = "2"

src/inference/engine.rs

+571 -11

@@ -94,7 +94,22 @@ use super::types::*;

94 target_os = "linux",

95 target_os = "android"

96 ))]

97 -use mistralrs::{GgufModelBuilder, Model, RequestBuilder, TextMessageRole};

97 +use mistralrs::{

98 + CalledFunction, Function, GgufModelBuilder, Model, RequestBuilder, TextMessageRole, Tool,

99 + ToolCallResponse, ToolCallType, ToolChoice, ToolType,

100 +};

101 +

102 +#[cfg(any(

103 + target_os = "macos",

104 + target_os = "ios",

105 + target_os = "tvos",

106 + target_os = "visionos",

107 + target_os = "watchos",

108 + target_os = "windows",

109 + target_os = "linux",

110 + target_os = "android"

111 +))]

112 +use std::collections::HashMap;

113

114 // ISQ types are only used by load_isq_model, which is macOS-only.

115 #[cfg(target_os = "macos")]

@@ -166,13 +181,41 @@ struct LoadedModel {

181 /// Configuration used to load this model (kept for status reporting).

182 config: LoadedModelConfig,

183 /// Conversation history (system prompt is stored separately).

169 - history: Vec<ChatMessage>,

184 + history: Vec<HistoryEntry>,

185 /// System prompt prepended to every request.

186 system_prompt: Option<String>,

187 /// Sampling parameters for generation.

188 sampling: SamplingConfig,

189 }

190

191 +/// An entry in the conversation history (internal only).

192 +/// Supports tool-related messages alongside regular text.

193 +#[cfg(any(

194 + target_os = "macos",

195 + target_os = "ios",

196 + target_os = "tvos",

197 + target_os = "visionos",

198 + target_os = "watchos",

199 + target_os = "windows",

200 + target_os = "linux",

201 + target_os = "android"

202 +))]

203 +#[derive(Debug, Clone)]

204 +enum HistoryEntry {

205 + /// A regular text message (user, assistant, system).

206 + Text(ChatMessage),

207 + /// An assistant response that includes tool calls.

208 + AssistantToolCall {

209 + content: String,

210 + tool_calls: Vec<ToolCallRequest>,

211 + },

212 + /// A tool execution result.

213 + ToolResult {

214 + tool_call_id: String,

215 + content: String,

216 + },

217 +}

218 +

219 // ═════════════════════════════════════════════════════════════════════════════

220 // ChatEngine — the public API

221 // ═════════════════════════════════════════════════════════════════════════════

@@ -519,10 +562,24 @@ impl ChatEngine {

562 // ── Conversation history ─────────────────────────────────────────────

563

564 /// Get a clone of the full conversation history.

565 + ///

566 + /// Tool-related history entries are converted back to plain

567 + /// [`ChatMessage`] values (assistant tool-call entries become assistant

568 + /// messages with just their text content; tool results are omitted).

569 pub async fn history(&self) -> Vec<ChatMessage> {

570 let guard = self.inner.lock().await;

571 match guard.as_ref() {

525 - Some(loaded) => loaded.history.clone(),

572 + Some(loaded) => loaded

573 + .history

574 + .iter()

575 + .filter_map(|entry| match entry {

576 + HistoryEntry::Text(msg) => Some(msg.clone()),

577 + HistoryEntry::AssistantToolCall { content, .. } => {

578 + Some(ChatMessage::assistant(content))

579 + }

580 + HistoryEntry::ToolResult { .. } => None,

581 + })

582 + .collect(),

583 None => Vec::new(),

584 }

585 }

@@ -548,7 +605,7 @@ impl ChatEngine {

605 /// Useful for restoring a saved conversation or injecting context.

606 pub async fn push_history(&self, message: ChatMessage) {

607 if let Some(loaded) = self.inner.lock().await.as_mut() {

551 - loaded.history.push(message);

608 + loaded.history.push(HistoryEntry::Text(message));

609 }

610 }

611

@@ -612,8 +669,12 @@ impl ChatEngine {

669 {

670 let mut guard = self.inner.lock().await;

671 if let Some(loaded) = guard.as_mut() {

615 - loaded.history.push(ChatMessage::user(user_message));

616 - loaded.history.push(ChatMessage::assistant(reply.clone()));

672 + loaded

673 + .history

674 + .push(HistoryEntry::Text(ChatMessage::user(user_message)));

675 + loaded

676 + .history

677 + .push(HistoryEntry::Text(ChatMessage::assistant(reply.clone())));

678 }

679 }

680

@@ -797,10 +858,12 @@ impl ChatEngine {

858 {

859 let mut guard = inner_ref.lock().await;

860 if let Some(loaded) = guard.as_mut() {

800 - loaded.history.push(ChatMessage::user(user_msg_clone));

861 loaded

862 .history

803 - .push(ChatMessage::assistant(assembled.trim()));

863 + .push(HistoryEntry::Text(ChatMessage::user(user_msg_clone)));

864 + loaded

865 + .history

866 + .push(HistoryEntry::Text(ChatMessage::assistant(assembled.trim())));

867 }

868 }

869

@@ -828,6 +891,283 @@ impl ChatEngine {

891

892 Ok(rx)

893 }

894 +

895 + // ── Tool-aware inference (Rust-only) ─────────────────────────────────

896 +

897 + /// Send a user message with tool definitions available. Non-streaming.

898 + ///

899 + /// If the model decides to call tools, [`ToolAwareResult::tool_calls`]

900 + /// will be non-empty and `finish_reason` will typically be `"tool_calls"`.

901 + ///

902 + /// The user message and response (including any tool calls) are

903 + /// automatically added to conversation history.

904 + pub async fn send_message_with_tools(

905 + &self,

906 + user_message: impl Into<String>,

907 + tools: &[ToolDefinition],

908 + ) -> Result<ToolAwareResult, InferenceError> {

909 + let user_message = user_message.into();

910 +

911 + let (model, request) = {

912 + let guard = self.inner.lock().await;

913 + let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;

914 + let request = build_request_with_tools(loaded, &user_message, tools);

915 + (loaded.model.clone(), request)

916 + };

917 +

918 + log::info!(

919 + "ChatEngine: tool inference START — message: \"{}\"",

920 + truncate_for_log(&user_message, 100)

921 + );

922 +

923 + let start = std::time::Instant::now();

924 + let response =

925 + model

926 + .send_chat_request(request)

927 + .await

928 + .map_err(|e| InferenceError::Inference {

929 + reason: e.to_string(),

930 + })?;

931 + let elapsed = start.elapsed();

932 +

933 + let choice = &response.choices[0];

934 + let reply = choice

935 + .message

936 + .content

937 + .as_ref()

938 + .map(|c| c.trim().to_string())

939 + .unwrap_or_default();

940 + let finish_reason = choice.finish_reason.clone();

941 + let tool_calls = parse_tool_calls(choice);

942 +

943 + log::info!(

944 + "ChatEngine: tool inference END — {} — tool_calls: {} — reply: \"{}\"",

945 + format_duration(elapsed),

946 + tool_calls.len(),

947 + truncate_for_log(&reply, 100)

948 + );

949 +

950 + // Persist to history.

951 + {

952 + let mut guard = self.inner.lock().await;

953 + if let Some(loaded) = guard.as_mut() {

954 + loaded

955 + .history

956 + .push(HistoryEntry::Text(ChatMessage::user(&user_message)));

957 + if tool_calls.is_empty() {

958 + loaded

959 + .history

960 + .push(HistoryEntry::Text(ChatMessage::assistant(&reply)));

961 + } else {

962 + loaded.history.push(HistoryEntry::AssistantToolCall {

963 + content: reply.clone(),

964 + tool_calls: tool_calls.clone(),

965 + });

966 + }

967 + }

968 + }

969 +

970 + Ok(ToolAwareResult {

971 + text: reply,

972 + tool_calls,

973 + duration_secs: elapsed.as_secs_f64(),

974 + duration_display: format_duration(elapsed),

975 + finish_reason,

976 + })

977 + }

978 +

979 + /// Send tool execution results back to the model. Non-streaming.

980 + ///

981 + /// Call this after executing the tools from a previous

982 + /// [`send_message_with_tools`](Self::send_message_with_tools) response.

983 + /// Pass the same (or updated) tool definitions if the model should be

984 + /// allowed to make further tool calls.

985 + pub async fn send_tool_results(

986 + &self,

987 + results: Vec<ToolResult>,

988 + tools: Option<&[ToolDefinition]>,

989 + ) -> Result<ToolAwareResult, InferenceError> {

990 + let (model, request) = {

991 + let guard = self.inner.lock().await;

992 + let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;

993 + let request = build_tool_results_request(loaded, &results, tools);

994 + (loaded.model.clone(), request)

995 + };

996 +

997 + log::info!(

998 + "ChatEngine: tool results inference START — {} results",

999 + results.len()

1000 + );

1001 +

1002 + let start = std::time::Instant::now();

1003 + let response =

1004 + model

1005 + .send_chat_request(request)

1006 + .await

1007 + .map_err(|e| InferenceError::Inference {

1008 + reason: e.to_string(),

1009 + })?;

1010 + let elapsed = start.elapsed();

1011 +

1012 + let choice = &response.choices[0];

1013 + let reply = choice

1014 + .message

1015 + .content

1016 + .as_ref()

1017 + .map(|c| c.trim().to_string())

1018 + .unwrap_or_default();

1019 + let finish_reason = choice.finish_reason.clone();

1020 + let tool_calls = parse_tool_calls(choice);

1021 +

1022 + log::info!(

1023 + "ChatEngine: tool results inference END — {} — tool_calls: {} — reply: \"{}\"",

1024 + format_duration(elapsed),

1025 + tool_calls.len(),

1026 + truncate_for_log(&reply, 100)

1027 + );

1028 +

1029 + // Persist to history.

1030 + {

1031 + let mut guard = self.inner.lock().await;

1032 + if let Some(loaded) = guard.as_mut() {

1033 + for result in &results {

1034 + loaded.history.push(HistoryEntry::ToolResult {

1035 + tool_call_id: result.tool_call_id.clone(),

1036 + content: result.content.clone(),

1037 + });

1038 + }

1039 + if tool_calls.is_empty() {

1040 + loaded

1041 + .history

1042 + .push(HistoryEntry::Text(ChatMessage::assistant(&reply)));

1043 + } else {

1044 + loaded.history.push(HistoryEntry::AssistantToolCall {

1045 + content: reply.clone(),

1046 + tool_calls: tool_calls.clone(),

1047 + });

1048 + }

1049 + }

1050 + }

1051 +

1052 + Ok(ToolAwareResult {

1053 + text: reply,

1054 + tool_calls,

1055 + duration_secs: elapsed.as_secs_f64(),

1056 + duration_display: format_duration(elapsed),

1057 + finish_reason,

1058 + })

1059 + }

1060 +

1061 + /// Stream tool execution results back to the model.

1062 + ///

1063 + /// Similar to [`send_tool_results`](Self::send_tool_results) but returns

1064 + /// a streaming receiver. Tool calls in the streaming response are NOT

1065 + /// parsed — this is intended for the final text response after all tool

1066 + /// rounds complete.

1067 + pub async fn stream_tool_results(

1068 + &self,

1069 + results: Vec<ToolResult>,

1070 + tools: Option<Vec<ToolDefinition>>,

1071 + ) -> Result<tokio::sync::mpsc::Receiver<StreamChunk>, InferenceError> {

1072 + let (model, request) = {

1073 + let guard = self.inner.lock().await;

1074 + let loaded = guard.as_ref().ok_or(InferenceError::NoModelLoaded)?;

1075 + let request = build_tool_results_request(loaded, &results, tools.as_deref());

1076 + (loaded.model.clone(), request)

1077 + };

1078 +

1079 + let (tx, rx) = tokio::sync::mpsc::channel::<StreamChunk>(64);

1080 +

1081 + let inner_ptr = &self.inner as *const Mutex<Option<LoadedModel>>;

1082 + // SAFETY: ChatEngine is stored in a Lazy static or equivalent with

1083 + // 'static lifetime. The spawned task cannot outlive the engine.

1084 + let inner_ref: &'static Mutex<Option<LoadedModel>> = unsafe { &*inner_ptr };

1085 +

1086 + let results_clone = results;

1087 +

1088 + tokio::task::spawn(async move {

1089 + let stream_result = model.stream_chat_request(request).await;

1090 +

1091 + match stream_result {

1092 + Ok(mut stream) => {

1093 + let mut assembled = String::new();

1094 + let mut last_finish_reason = None;

1095 +

1096 + while let Some(response) = stream.next().await {

1097 + match response {

1098 + mistralrs::Response::Chunk(chunk) => {

1099 + if let Some(choice) = chunk.choices.first() {

1100 + if let Some(ref text) = choice.delta.content {

1101 + assembled.push_str(text);

1102 + let _ = tx

1103 + .send(StreamChunk {

1104 + delta: text.clone(),

1105 + done: false,

1106 + finish_reason: None,

1107 + })

1108 + .await;

1109 + }

1110 + if let Some(ref reason) = choice.finish_reason {

1111 + last_finish_reason = Some(reason.clone());

1112 + }

1113 + }

1114 + }

1115 + mistralrs::Response::Done(_) => break,

1116 + mistralrs::Response::InternalError(e) => {

1117 + log::error!("stream_tool_results internal error: {}", e);

1118 + break;

1119 + }

1120 + mistralrs::Response::ValidationError(e) => {

1121 + log::error!("stream_tool_results validation error: {}", e);

1122 + break;

1123 + }

1124 + mistralrs::Response::ModelError(msg, _) => {

1125 + log::error!("stream_tool_results model error: {}", msg);

1126 + break;

1127 + }

1128 + _ => break,

1129 + }

1130 + }

1131 +

1132 + // Persist to history.

1133 + {

1134 + let mut guard = inner_ref.lock().await;

1135 + if let Some(loaded) = guard.as_mut() {

1136 + for result in &results_clone {

1137 + loaded.history.push(HistoryEntry::ToolResult {

1138 + tool_call_id: result.tool_call_id.clone(),

1139 + content: result.content.clone(),

1140 + });

1141 + }

1142 + loaded

1143 + .history

1144 + .push(HistoryEntry::Text(ChatMessage::assistant(assembled.trim())));

1145 + }

1146 + }

1147 +

1148 + let _ = tx

1149 + .send(StreamChunk {

1150 + delta: String::new(),

1151 + done: true,

1152 + finish_reason: last_finish_reason,

1153 + })

1154 + .await;

1155 + }

1156 + Err(e) => {

1157 + log::error!("stream_tool_results failed: {}", e);

1158 + let _ = tx

1159 + .send(StreamChunk {

1160 + delta: String::new(),

1161 + done: true,

1162 + finish_reason: Some(format!("error: {}", e)),

1163 + })

1164 + .await;

1165 + }

1166 + }

1167 + });

1168 +

1169 + Ok(rx)

1170 + }

1171 }

1172

1173 // ═════════════════════════════════════════════════════════════════════════════

@@ -877,8 +1217,19 @@ fn build_request(loaded: &LoadedModel, user_message: &str) -> RequestBuilder {

1217 }

1218

1219 // Replay conversation history for multi-turn context.

880 - for turn in &loaded.history {

881 - req = req.add_message(chat_role_to_mistral(&turn.role), &turn.content);

1220 + for entry in &loaded.history {

1221 + match entry {

1222 + HistoryEntry::Text(msg) => {

1223 + req = req.add_message(chat_role_to_mistral(&msg.role), &msg.content);

1224 + }

1225 + HistoryEntry::AssistantToolCall { content, .. } => {

1226 + // When replaying without tools, just use the text content.

1227 + req = req.add_message(TextMessageRole::Assistant, content);

1228 + }

1229 + HistoryEntry::ToolResult { .. } => {

1230 + // Skip tool results when not in tool mode.

1231 + }

1232 + }

1233 }

1234

1235 // Append the current user message.

@@ -887,6 +1238,193 @@ fn build_request(loaded: &LoadedModel, user_message: &str) -> RequestBuilder {

1238 req

1239 }

1240

1241 +/// Build a request with tool support, properly replaying tool-related history.

1242 +#[cfg(any(

1243 + target_os = "macos",

1244 + target_os = "ios",

1245 + target_os = "tvos",

1246 + target_os = "visionos",

1247 + target_os = "watchos",

1248 + target_os = "windows",

1249 + target_os = "linux",

1250 + target_os = "android"

1251 +))]

1252 +fn build_request_with_tools(

1253 + loaded: &LoadedModel,

1254 + user_message: &str,

1255 + tools: &[ToolDefinition],

1256 +) -> RequestBuilder {

1257 + let mut req = RequestBuilder::new();

1258 + req = apply_sampling(req, &loaded.sampling);

1259 +

1260 + if let Some(ref sp) = loaded.system_prompt {

1261 + req = req.add_message(TextMessageRole::System, sp);

1262 + }

1263 +

1264 + req = replay_history_with_tools(req, &loaded.history);

1265 +

1266 + req = req.add_message(TextMessageRole::User, user_message);

1267 +

1268 + // Add tool definitions.

1269 + req = attach_tools(req, tools);

1270 +

1271 + req

1272 +}

1273 +

1274 +/// Build a request for sending tool results back to the model.

1275 +#[cfg(any(

1276 + target_os = "macos",

1277 + target_os = "ios",

1278 + target_os = "tvos",

1279 + target_os = "visionos",

1280 + target_os = "watchos",

1281 + target_os = "windows",

1282 + target_os = "linux",

1283 + target_os = "android"

1284 +))]

1285 +fn build_tool_results_request(

1286 + loaded: &LoadedModel,

1287 + results: &[ToolResult],

1288 + tools: Option<&[ToolDefinition]>,

1289 +) -> RequestBuilder {

1290 + let mut req = RequestBuilder::new();

1291 + req = apply_sampling(req, &loaded.sampling);

1292 +

1293 + if let Some(ref sp) = loaded.system_prompt {

1294 + req = req.add_message(TextMessageRole::System, sp);

1295 + }

1296 +

1297 + // Replay full history (including tool call entries).

1298 + req = replay_history_with_tools(req, &loaded.history);

1299 +

1300 + // Add new tool results.

1301 + for result in results {

1302 + req = req.add_tool_message(&result.content, &result.tool_call_id);

1303 + }

1304 +

1305 + // Optionally add tools for further rounds.

1306 + if let Some(tool_defs) = tools {

1307 + req = attach_tools(req, tool_defs);

1308 + }

1309 +

1310 + req

1311 +}

1312 +

1313 +/// Replay conversation history entries onto a [`RequestBuilder`], preserving

1314 +/// tool call and tool result messages.

1315 +#[cfg(any(

1316 + target_os = "macos",

1317 + target_os = "ios",

1318 + target_os = "tvos",

1319 + target_os = "visionos",

1320 + target_os = "watchos",

1321 + target_os = "windows",

1322 + target_os = "linux",

1323 + target_os = "android"

1324 +))]

1325 +fn replay_history_with_tools(mut req: RequestBuilder, history: &[HistoryEntry]) -> RequestBuilder {

1326 + for entry in history {

1327 + match entry {

1328 + HistoryEntry::Text(msg) => {

1329 + req = req.add_message(chat_role_to_mistral(&msg.role), &msg.content);

1330 + }

1331 + HistoryEntry::AssistantToolCall {

1332 + content,

1333 + tool_calls,

1334 + } => {

1335 + let mistral_tcs: Vec<ToolCallResponse> = tool_calls

1336 + .iter()

1337 + .map(|tc| ToolCallResponse {

1338 + index: 0,

1339 + id: tc.id.clone(),

1340 + tp: ToolCallType::Function,

1341 + function: CalledFunction {

1342 + name: tc.function_name.clone(),

1343 + arguments: tc.arguments.clone(),

1344 + },

1345 + })

1346 + .collect();

1347 + req = req.add_message_with_tool_call(

1348 + TextMessageRole::Assistant,

1349 + content.clone(),

1350 + mistral_tcs,

1351 + );

1352 + }

1353 + HistoryEntry::ToolResult {

1354 + tool_call_id,

1355 + content,

1356 + } => {

1357 + req = req.add_tool_message(content, tool_call_id);

1358 + }

1359 + }

1360 + }

1361 + req

1362 +}

1363 +

1364 +/// Attach [`ToolDefinition`] slices as mistralrs [`Tool`] values and set

1365 +/// [`ToolChoice::Auto`].

1366 +#[cfg(any(

1367 + target_os = "macos",

1368 + target_os = "ios",

1369 + target_os = "tvos",

1370 + target_os = "visionos",

1371 + target_os = "watchos",

1372 + target_os = "windows",

1373 + target_os = "linux",

1374 + target_os = "android"

1375 +))]

1376 +fn attach_tools(mut req: RequestBuilder, tools: &[ToolDefinition]) -> RequestBuilder {

1377 + if !tools.is_empty() {

1378 + let mistral_tools: Vec<Tool> = tools

1379 + .iter()

1380 + .map(|td| {

1381 + let params: HashMap<String, serde_json::Value> =

1382 + serde_json::from_str(&td.parameters_schema).unwrap_or_default();

1383 + Tool {

1384 + tp: ToolType::Function,

1385 + function: Function {

1386 + description: Some(td.description.clone()),

1387 + name: td.name.clone(),

1388 + parameters: Some(params),

1389 + strict: Some(true),

1390 + },

1391 + }

1392 + })

1393 + .collect();

1394 + req = req.set_tools(mistral_tools);

1395 + req = req.set_tool_choice(ToolChoice::Auto);

1396 + }

1397 + req

1398 +}

1399 +

1400 +/// Parse tool calls from a mistralrs response choice.

1401 +#[cfg(any(

1402 + target_os = "macos",

1403 + target_os = "ios",

1404 + target_os = "tvos",

1405 + target_os = "visionos",

1406 + target_os = "watchos",

1407 + target_os = "windows",

1408 + target_os = "linux",

1409 + target_os = "android"

1410 +))]

1411 +fn parse_tool_calls(choice: &mistralrs::Choice) -> Vec<ToolCallRequest> {

1412 + choice

1413 + .message

1414 + .tool_calls

1415 + .as_ref()

1416 + .map(|tcs| {

1417 + tcs.iter()

1418 + .map(|tc| ToolCallRequest {

1419 + id: tc.id.clone(),

1420 + function_name: tc.function.name.clone(),

1421 + arguments: tc.function.arguments.clone(),

1422 + })

1423 + .collect()

1424 + })

1425 + .unwrap_or_default()

1426 +}

1427 +

1428 /// Apply a [`SamplingConfig`] to a [`RequestBuilder`].

1429 #[cfg(any(

1430 target_os = "macos",

@@ -923,7 +1461,7 @@ fn apply_sampling(mut req: RequestBuilder, sampling: &SamplingConfig) -> Request

1461 req

1462 }

1463

926 -/// Convert [`ChatRole`] → [`TextMessageRole`].

1464 +/// Convert a [`ChatRole`] to a [`TextMessageRole`].

1465 #[cfg(any(

1466 target_os = "macos",

1467 target_os = "ios",

@@ -1161,6 +1699,28 @@ impl GgufModelConfig {

1699 }

1700 }

1701

1702 + /// Qwen 3 4B (GGUF Q4_K_M) — tool-calling-capable, ~2.7 GB.

1703 + pub fn qwen3_4b() -> Self {

1704 + Self {

1705 + model_id: super::models::BARTOWSKI_QWEN3_4B_GGUF.into(),

1706 + files: vec![super::models::QWEN3_4B_GGUF_FILE.into()],

1707 + tok_model_id: None,

1708 + display_name: "Qwen 3 4B (Q4_K_M)".into(),

1709 + approx_memory: "~2.7 GB".into(),

1710 + }

1711 + }

1712 +

1713 + /// Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling, ~1.3 GB.

1714 + pub fn qwen3_1_7b() -> Self {

1715 + Self {

1716 + model_id: super::models::BARTOWSKI_QWEN3_1_7B_GGUF.into(),

1717 + files: vec![super::models::QWEN3_1_7B_GGUF_FILE.into()],

1718 + tok_model_id: None,

1719 + display_name: "Qwen 3 1.7B (Q4_K_M)".into(),

1720 + approx_memory: "~1.3 GB".into(),

1721 + }

1722 + }

1723 +

1724 /// Return the platform-appropriate default **coding** model config.

1725 ///

1726 /// - iOS / Android → Qwen 2.5 Coder 1.5B (~941 MB, fits mobile memory budgets)

src/inference/mod.rs

+15 -2

@@ -67,6 +67,19 @@ pub mod types;

67 pub use engine::ChatEngine;

68 pub use ffi::{OndeChatEngine, StreamChunkListener};

69 pub use types::{

70 - ChatMessage, ChatRole, EngineInfo, EngineStatus, GgufModelConfig, InferenceError,

71 - InferenceResult, IsqModelConfig, SamplingConfig, StreamChunk,

70 + ChatMessage,

71 + ChatRole,

72 + EngineInfo,

73 + EngineStatus,

74 + GgufModelConfig,

75 + InferenceError,

76 + InferenceResult,

77 + IsqModelConfig,

78 + SamplingConfig,

79 + StreamChunk,

80 + // Tool calling (Rust-only, not UniFFI-exported)

81 + ToolAwareResult,

82 + ToolCallRequest,

83 + ToolDefinition,

84 + ToolResult,

85 };

src/inference/models.rs

+37

@@ -63,12 +63,33 @@ pub const QWEN25_3B_GGUF_FILE: &str = "Qwen2.5-3B-Instruct-Q4_K_M.gguf";

63 /// Base model repo used for the HF tokenizer (tokenizer.json + tokenizer_config.json).

64 pub const QWEN25_3B_TOK_MODEL_ID: &str = "Qwen/Qwen2.5-3B-Instruct";

66 +/// Pre-quantized Qwen 3 4B (GGUF Q4_K_M) — tool-calling-capable coding model (~2.7 GB).

67 +///

68 +/// Qwen 3 is the first Qwen family supported by mistral.rs for structured tool calling

69 +/// (via the `<tool_call>` / `</tool_call>` XML format). The 4B variant offers a good

70 +/// balance of tool-calling reliability and memory footprint for macOS desktops.

71 +///

72 +/// bartowski's GGUF embeds the full tokenizer and chat template.

73 +pub const BARTOWSKI_QWEN3_4B_GGUF: &str = "bartowski/Qwen_Qwen3-4B-GGUF";

74 +/// The specific GGUF filename to download from the bartowski Qwen3 4B repo.

75 +pub const QWEN3_4B_GGUF_FILE: &str = "Qwen_Qwen3-4B-Q4_K_M.gguf";

76 +

77 +/// Pre-quantized Qwen 3 1.7B (GGUF Q4_K_M) — lightweight tool-calling model (~1.3 GB).

78 +///

79 +/// Smallest Qwen 3 variant with tool calling support. Suitable for mobile devices

80 +/// where the 4B model would be too large.

81 +pub const BARTOWSKI_QWEN3_1_7B_GGUF: &str = "bartowski/Qwen_Qwen3-1.7B-GGUF";

82 +/// The specific GGUF filename for the Qwen3 1.7B repo.

83 +pub const QWEN3_1_7B_GGUF_FILE: &str = "Qwen_Qwen3-1.7B-Q4_K_M.gguf";

84 +

85 /// All model IDs that the Onde inference engine supports.

86 /// Used by `list_local_hf_models` to filter the HuggingFace cache

87 /// to only show models that can actually be used for generation.

88 pub const SUPPORTED_MODELS: &[&str] = &[

89 BARTOWSKI_QWEN25_1_5B_INSTRUCT_GGUF,

90 BARTOWSKI_QWEN25_3B_INSTRUCT_GGUF,

91 + BARTOWSKI_QWEN3_4B_GGUF,

92 + BARTOWSKI_QWEN3_1_7B_GGUF,

93 ];

95 /// Rich metadata for a supported model, used by the frontend to display

@@ -117,4 +138,20 @@ pub const SUPPORTED_MODEL_INFO: &[SupportedModelInfo] = &[

138 // Exact file size from HuggingFace API siblings[].size.

139 expected_size_bytes: 1_929_903_264,

140 },

141 + SupportedModelInfo {

142 + id: BARTOWSKI_QWEN3_4B_GGUF,

143 + name: "Qwen 3 4B (GGUF)",

144 + org: "Qwen / Alibaba",

145 + description: "Tool-calling-capable model for macOS — balanced quality and size (~2.7 GB). \

146 + First Qwen family with structured tool calling support in mistral.rs.",

147 + expected_size_bytes: 2_600_000_000,

148 + },

149 + SupportedModelInfo {

150 + id: BARTOWSKI_QWEN3_1_7B_GGUF,

151 + name: "Qwen 3 1.7B (GGUF)",

152 + org: "Qwen / Alibaba",

153 + description: "Lightweight tool-calling model for mobile (~1.3 GB). \

154 + Smallest Qwen 3 variant with tool calling support.",

155 + expected_size_bytes: 1_282_439_584,

156 + },

157 ];

src/inference/types.rs

+54

@@ -374,6 +374,60 @@ pub fn format_duration(d: std::time::Duration) -> String {

374 }

375 }

376

377 +// ── Tool calling types (Rust-only) ──────────────────────────────────────────

378 +//

379 +// These types are intentionally NOT annotated with `uniffi::Record` /

380 +// `uniffi::Enum` — they are for Rust consumers only. The UniFFI surface

381 +// area (Swift / Kotlin) remains unchanged.

382 +

383 +/// Definition of a tool the model can invoke.

384 +#[derive(Debug, Clone)]

385 +pub struct ToolDefinition {

386 + /// Tool function name (e.g. `"get_weather"`).

387 + pub name: String,

388 + /// Human-readable description of what the tool does.

389 + pub description: String,

390 + /// JSON Schema for function parameters, as a serialised JSON string.

391 + pub parameters_schema: String,

392 +}

393 +

394 +/// A tool call requested by the model.

395 +#[derive(Debug, Clone)]

396 +pub struct ToolCallRequest {

397 + /// Unique identifier for this tool call (used to correlate results).

398 + pub id: String,

399 + /// Name of the function the model wants to invoke.

400 + pub function_name: String,

401 + /// JSON string of the function arguments.

402 + pub arguments: String,

403 +}

404 +

405 +/// Result of executing a tool, to send back to the model.

406 +#[derive(Debug, Clone)]

407 +pub struct ToolResult {

408 + /// The `id` from the corresponding [`ToolCallRequest`].

409 + pub tool_call_id: String,

410 + /// The tool's output (plain text or JSON).

411 + pub content: String,

412 +}

413 +

414 +/// Result from a tool-aware inference call (Rust-only, not UniFFI-exported).

415 +#[derive(Debug, Clone)]

416 +pub struct ToolAwareResult {

417 + /// Text content of the assistant reply (may be empty when the model

418 + /// decides to call tools instead of responding with text).

419 + pub text: String,

420 + /// Tool calls requested by the model. Empty when the model responds

421 + /// with text only.

422 + pub tool_calls: Vec<ToolCallRequest>,

423 + /// Wall-clock inference duration in seconds.

424 + pub duration_secs: f64,

425 + /// Human-readable duration string (e.g. `"1.3s"`).

426 + pub duration_display: String,

427 + /// Finish reason reported by the model (e.g. `"stop"`, `"tool_calls"`).

428 + pub finish_reason: String,

429 +}

430 +

431 #[cfg(test)]

432 mod tests {

433 use super::*;