2 files changed
+31
-5
src/chat.rs
+14
-3
@@ -675,9 +675,20 @@ async fn run_inference_task(
675
}
676
}
677
678
- // Send the final text response.
679
- if !result.text.is_empty() && result.tool_calls.is_empty() {
680
- let _ = tx.send(InferenceUpdate::Response(result.text)).await;
678
+ // Send the final text response, or a fallback if the model returned nothing.
679
+ if result.tool_calls.is_empty() {
680
+ if result.text.is_empty() {
681
+ log::warn!("model returned empty reply — may have exhausted max_tokens on thinking");
682
+ let _ = tx
683
+ .send(InferenceUpdate::Error(
684
+ "(empty response — the model may have used all tokens on internal reasoning. \
685
+ Try a shorter or simpler prompt.)"
686
+ .to_string(),
687
+ ))
688
+ .await;
689
+ } else {
690
+ let _ = tx.send(InferenceUpdate::Response(result.text)).await;
691
+ }
692
}
693
694
log::info!("inference complete — {} tool round(s)", round);
src/main.rs
+17
-2
@@ -30,6 +30,8 @@ use std::fs::File;
30
use std::io::IsTerminal;
31
use std::sync::Arc;
32
33
+use onde::inference::SamplingConfig;
34
+
35
use agent_client_protocol::{
36
Agent, AgentCapabilities, AgentSideConnection, AuthenticateRequest, AuthenticateResponse,
37
CancelNotification, Client, ContentBlock, ContentChunk, Implementation, InitializeRequest,
@@ -145,8 +147,12 @@ impl Agent for SiGitAgent {
147
// Qwen 3 4B is required for tool calling support.
148
log::info!("loading Qwen 3 4B model (this may take a minute on first run)...");
149
let config = GgufModelConfig::qwen3_4b();
150
+ let sampling = SamplingConfig {
151
+ max_tokens: Some(4096),
152
+ ..SamplingConfig::default()
153
+ };
154
self.engine
149
- .load_gguf_model(config, Some(SYSTEM_PROMPT.to_string()), None)
155
+ .load_gguf_model(config, Some(SYSTEM_PROMPT.to_string()), Some(sampling))
156
.await
157
.map_err(|e| {
158
log::error!("model load failed: {e}");
@@ -329,8 +335,17 @@ async fn run_interactive() -> anyhow::Result<()> {
335
336
let engine = Arc::new(ChatEngine::new());
337
let config = GgufModelConfig::qwen3_4b();
338
+
339
+ // Qwen 3 uses a thinking mode (<think>…</think>) that can easily
340
+ // consume 300-400 tokens before the real response. The default 512
341
+ // leaves almost nothing for tool calls or text — bump to 4096.
342
+ let sampling = SamplingConfig {
343
+ max_tokens: Some(4096),
344
+ ..SamplingConfig::default()
345
+ };
346
+
347
engine
333
- .load_gguf_model(config, Some(SYSTEM_PROMPT.to_string()), None)
348
+ .load_gguf_model(config, Some(SYSTEM_PROMPT.to_string()), Some(sampling))
349
.await
350
.map_err(|e| anyhow::anyhow!("model load failed: {e}"))?;
351