96
/// session started (the original request) and where it stands now. Cutting by
97
/// whole lines keeps individual messages intact. A transcript already under
98
/// the cap is returned unchanged.
99
+///
100
+/// Line boundaries are best-effort: when one side of the cut can't land on a
101
+/// newline (a transcript of very long lines, or a single huge message), that
102
+/// side gives up its share rather than the other — the newest exchanges are
103
+/// the ones the summary can least afford to lose, so the head shrinks to make
104
+/// room. Both sides failing degrades to keeping only the newest budget.
105
fn truncate_transcript_middle(transcript: &str, cap: usize) -> String {
106
let budget = cap * 4; // estimate_tokens is chars / 4; invert it.
107
if transcript.len() <= budget {
108
return transcript.to_string();
109
}
104
- // Split the budget between head and tail, then walk each inward to a line
105
- // boundary so no message is severed mid-line.
106
- let half = budget / 2;
107
- let mut head_end = half.min(transcript.len());
108
- while head_end < transcript.len() && !transcript.is_char_boundary(head_end) {
109
- head_end += 1;
110
- }
111
- let head_end = transcript[..head_end]
112
- .rfind('\n')
113
- .map(|index| index + 1)
114
- .unwrap_or(0);
110
116
- let mut tail_start = transcript.len().saturating_sub(half);
117
- while tail_start > 0 && !transcript.is_char_boundary(tail_start) {
118
- tail_start -= 1;
111
+ let total = transcript.len();
112
+ // Walk each half to a line boundary. A side with no boundary in reach
113
+ // contributes nothing; the other side may then use the whole budget.
114
+ let head_end = {
115
+ let mut cut = budget.min(total);
116
+ while cut < total && !transcript.is_char_boundary(cut) {
117
+ cut += 1;
118
+ }
119
+ cut = cut.min(total);
120
+ if transcript[..cut].contains('\n') {
121
+ transcript[..cut].rfind('\n').unwrap() + 1
122
+ } else {
123
+ 0
124
+ }
125
+ };
126
+ let tail_start = {
127
+ let mut cut = total.saturating_sub(budget);
128
+ while cut > 0 && !transcript.is_char_boundary(cut) {
129
+ cut -= 1;
130
+ }
131
+ if transcript[cut..].contains('\n') {
132
+ transcript[cut..]
133
+ .find('\n')
134
+ .map(|offset| cut + offset + 1)
135
+ .unwrap()
136
+ } else {
137
+ total
138
+ }
139
+ };
140
+
141
+ if tail_start == total {
142
+ // The closing exchange has no line boundary in reach (a transcript of
143
+ // one huge message, or a giant newline-less tool result at the end):
144
+ // keep the newest budget. A head-only cut would invert the contract
145
+ // while the placeholder claims older messages were dropped.
146
+ let mut start = total.saturating_sub(budget);
147
+ while start > 0 && !transcript.is_char_boundary(start) {
148
+ start -= 1;
149
+ }
150
+ return format!(
151
+ "{}\n[…older messages omitted…]",
152
+ transcript[start..].trim_start()
153
+ );
154
}
120
- let tail_start = transcript[tail_start..]
121
- .find('\n')
122
- .map(|offset| tail_start + offset + 1)
123
- .unwrap_or(transcript.len());
155
156
if head_end >= tail_start {
126
- // The cut points crossed (a transcript of very long lines); fall back
127
- // to a hard tail cut rather than duplicating content.
128
- let mut end = budget.min(transcript.len());
129
- while end > 0 && !transcript.is_char_boundary(end) {
130
- end -= 1;
157
+ // No line boundary in reach anywhere: keep the newest budget.
158
+ let mut start = total.saturating_sub(budget);
159
+ while start > 0 && !transcript.is_char_boundary(start) {
160
+ start -= 1;
161
}
132
- return format!("{}\n[…older messages omitted…]", &transcript[..end]);
162
+ return format!(
163
+ "{}\n[…older messages omitted…]",
164
+ transcript[start..].trim_start()
165
+ );
166
}
167
168
let omitted_chars = transcript[head_end..tail_start].chars().count();
484
self.apply_pending_history().await?;
485
let snapshot = self.engine.history().await;
486
454
- let non_system: Vec<&ChatMessage> = snapshot
487
+ // Leading system messages carry session context; keep them all.
488
+ let system: Vec<ChatMessage> = snapshot
489
.iter()
490
+ .take_while(|message| message.role == ChatRole::System)
491
+ .cloned()
492
+ .collect();
493
+ let non_system: Vec<ChatMessage> = snapshot
494
+ .into_iter()
495
.filter(|message| message.role != ChatRole::System)
496
.collect();
497
let tail_start = non_system.len().saturating_sub(keep_last);
459
- let tail: Vec<ChatMessage> = non_system[tail_start..]
460
- .iter()
461
- .map(|message| (*message).clone())
462
- .collect();
498
+ let tail: Vec<ChatMessage> = non_system[tail_start..].to_vec();
499
464
- // One plain (tool-free) inference round produces the summary. The
465
- // transcript is capped like the remote path's: an over-budget session
466
- // must not send its whole history back into the window that is already
467
- // too small for it (issue #125). send_message mutates engine history,
468
- // but the clear below wipes whatever it appended, so a failed round
469
- // leaves nothing behind.
500
+ // One plain (tool-free) inference round produces the summary. Note the
501
+ // cap only bounds this request's own payload: the engine still replays
502
+ // its full (over-budget) history underneath it, so an already
503
+ // over-budget local session usually fails here and lands in the
504
+ // fallback below — the cap is about not making the doomed round worse.
505
let transcript = truncate_transcript_middle(
471
- &chat_messages_transcript(&snapshot),
506
+ &chat_messages_transcript(&non_system),
507
SUMMARY_TRANSCRIPT_TOKEN_CAP,
508
);
509
let result = self
516
Ok(result) => {
517
// Local models may reason in <think> blocks; keep only the visible part.
518
let (_think, summary) = crate::chat::strip_think_blocks(&result.text);
484
- let mut rebuilt: Vec<ChatMessage> = snapshot
485
- .iter()
486
- .take_while(|message| message.role == ChatRole::System)
487
- .cloned()
488
- .collect();
519
+ let mut rebuilt = system;
520
rebuilt.push(ChatMessage::user(format!(
521
"[Conversation summary]\n{summary}"
522
)));
527
log::warn!(
528
"local summarization round failed ({error}); falling back to plain truncation"
529
);
499
- let system_json: Option<serde_json::Value> = snapshot
500
- .iter()
501
- .find(|message| message.role == ChatRole::System)
502
- .map(|message| serde_json::json!({
503
- "role": message.role.to_string(),
504
- "content": message.content,
505
- }));
506
- let tail_json: Vec<serde_json::Value> = tail
507
- .iter()
508
- .map(|message| serde_json::json!({
509
- "role": message.role.to_string(),
510
- "content": message.content,
511
- }))
512
- .collect();
513
- let Some(fallback) =
514
- truncate_history_to_budget(system_json.as_ref(), tail_json)
515
- else {
530
+ // Same deterministic shrink as the remote path, on the plain
531
+ // chat shapes the engine holds (its history has no tool-call
532
+ // structure, so there are no orphaned tool results to guard).
533
+ let system_chars: usize = system.iter().map(|message| message.content.len()).sum();
534
+ let mut kept: Vec<ChatMessage> = tail;
535
+ while kept.iter().map(|m| m.content.len()).sum::<usize>() + system_chars
536
+ > DEFAULT_CONTEXT_TOKEN_BUDGET * 4
537
+ && kept.len() > 1
538
+ {
539
+ kept.remove(0);
540
+ }
541
+ if kept.iter().map(|m| m.content.len()).sum::<usize>() + system_chars
542
+ > DEFAULT_CONTEXT_TOKEN_BUDGET * 4
543
+ {
544
+ // Even one message overflows the window: unrecoverable.
545
return Err(error);
517
- };
518
- fallback
519
- .iter()
520
- .filter_map(|entry| {
521
- let role = entry["role"].as_str().unwrap_or("");
522
- let content = entry["content"].as_str().unwrap_or("").to_string();
523
- match role {
524
- "system" => Some(ChatMessage::system(content)),
525
- "user" => Some(ChatMessage::user(content)),
526
- "assistant" => Some(ChatMessage::assistant(content)),
527
- _ => None,
528
- }
529
- })
530
- .collect()
546
+ }
547
+ let mut rebuilt = system;
548
+ rebuilt.push(ChatMessage::user(TRUNCATION_PLACEHOLDER.to_string()));
549
+ rebuilt.extend(kept);
550
+ rebuilt
551
}
552
};
553
785
/// instead of failing the session (issue #125).
786
async fn summarize(
787
&self,
768
- system: Option<&serde_json::Value>,
788
+ system: &[serde_json::Value],
789
transcript: &str,
790
) -> Result<String, BackendError> {
791
let url = format!("{}/chat/completions", self.base_url.trim_end_matches('/'));
772
- let mut messages = Vec::with_capacity(2);
773
- if let Some(system) = system {
774
- messages.push(system.clone());
775
- }
792
+ let mut messages = Vec::with_capacity(system.len() + 1);
793
+ messages.extend(system.iter().cloned());
794
messages.push(serde_json::json!({
795
"role": "user",
796
"content": format!("{transcript}\n\n{SUMMARIZE_PROMPT}"),
1322
async fn compact_history(&self, keep_last: usize) -> Result<(), BackendError> {
1323
let snapshot: Vec<serde_json::Value> = self.history.lock().await.clone();
1324
1307
- let system = snapshot
1308
- .first()
1309
- .filter(|message| message["role"] == "system")
1310
- .cloned();
1325
+ // Leading system messages carry session context (the seeded prompt,
1326
+ // pushed context); keep them all, like the summarization-success path.
1327
+ let system: Vec<serde_json::Value> = snapshot
1328
+ .iter()
1329
+ .take_while(|message| message["role"] == "system")
1330
+ .cloned()
1331
+ .collect();
1332
1333
let non_system: Vec<serde_json::Value> = snapshot
1334
.iter()
1359
&transcript_for_summary(&snapshot),
1360
SUMMARY_TRANSCRIPT_TOKEN_CAP,
1361
);
1341
- let summary = match self.summarize(system.as_ref(), &transcript).await {
1362
+ let summary = match self.summarize(&system, &transcript).await {
1363
Ok(summary) => summary,
1364
Err(error) => {
1365
log::warn!(
1369
// a placeholder and drop the oldest messages until the rebuild
1370
// fits the budget. Needs no endpoint, so it cannot fail the way
1371
// the inference round just did (issue #125).
1351
- let rebuilt = truncate_history_to_budget(system.as_ref(), tail.clone())
1352
- .ok_or(error)?;
1372
+ let rebuilt = truncate_history_to_budget(&system, tail.clone()).ok_or(error)?;
1373
*self.history.lock().await = rebuilt;
1374
return Ok(());
1375
}
1376
};
1377
1358
- let mut rebuilt = Vec::new();
1359
- if let Some(system) = system {
1360
- rebuilt.push(system);
1361
- }
1378
+ let mut rebuilt = Vec::with_capacity(system.len() + tail.len() + 1);
1379
+ rebuilt.extend(system);
1380
rebuilt.push(serde_json::json!({
1381
"role": "user",
1382
"content": format!("[Conversation summary]\n{summary}"),
1401
/// caller then reports the original summarization error and leaves history
1402
/// untouched.
1403
fn truncate_history_to_budget(
1386
- system: Option<&serde_json::Value>,
1404
+ system: &[serde_json::Value],
1405
mut tail: Vec<serde_json::Value>,
1406
) -> Option<Vec<serde_json::Value>> {
1407
/// A `role: "tool"` entry only makes sense behind the assistant message
1408
/// that requested it; dropping that message strands it.
1409
fn drop_orphaned_tool_results(tail: &mut Vec<serde_json::Value>) {
1392
- while tail.first().is_some_and(|message| message["role"] == "tool") {
1410
+ while tail
1411
+ .first()
1412
+ .is_some_and(|message| message["role"] == "tool")
1413
+ {
1414
tail.remove(0);
1415
}
1416
}
1419
// results whose call message already fell outside the kept window.
1420
drop_orphaned_tool_results(&mut tail);
1421
1401
- while !tail.is_empty() {
1402
- let mut candidate = Vec::with_capacity(tail.len() + 2);
1403
- if let Some(system) = system {
1404
- candidate.push(system.clone());
1405
- }
1422
+ loop {
1423
+ let mut candidate = Vec::with_capacity(tail.len() + system.len() + 1);
1424
+ candidate.extend(system.iter().cloned());
1425
candidate.push(serde_json::json!({
1426
"role": "user",
1427
"content": TRUNCATION_PLACEHOLDER,
1431
return Some(candidate);
1432
}
1433
1415
- if tail.len() == 1 {
1416
- // One message and it already overflows: nothing left to drop.
1434
+ // Nothing left to drop: even the minimal rebuild (system + placeholder,
1435
+ // no conversation) overflows — the session is genuinely unrecoverable.
1436
+ if tail.is_empty() {
1437
return None;
1438
}
1439
if tail[0]["role"] == "assistant"
1445
// Dropping an assistant tool-call message would orphan its
1446
// results; drop the whole round (call + results) together.
1447
let mut end = 1;
1428
- while tail.get(end).is_some_and(|message| message["role"] == "tool") {
1448
+ while tail
1449
+ .get(end)
1450
+ .is_some_and(|message| message["role"] == "tool")
1451
+ {
1452
end += 1;
1453
}
1454
tail.drain(..end);
1457
drop_orphaned_tool_results(&mut tail);
1458
}
1459
}
1437
- None
1460
}
1461
1462
// ── OpenAI error shape ────────────────────────────────────────────────────────
1748
1749
#[test]
1750
fn a_body_that_is_not_an_error_envelope_keeps_the_status() {
1729
- let described =
1730
- describe_api_error(reqwest::StatusCode::BAD_GATEWAY, "upstream said no");
1751
+ let described = describe_api_error(reqwest::StatusCode::BAD_GATEWAY, "upstream said no");
1752
1753
assert!(described.contains("502"), "{described}");
1754
assert!(described.contains("upstream said no"), "{described}");
2352
/// the whole window.
2353
#[tokio::test]
2354
async fn compact_history_failure_leaves_history_intact() {
2334
- // No listener at this address: the summarization request fails.
2335
- let backend =
2336
- OpenAiBackend::new("http://127.0.0.1:9", "", "test-model", Some("sys".into()));
2337
- backend.history.lock().await.push(serde_json::json!({
2338
- "role": "user",
2339
- "content": "x".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000),
2340
- }));
2355
+ // No listener at this address, so summarization fails; the system
2356
+ // prompt alone is bigger than the window, so even the minimal
2357
+ // fallback rebuild cannot fit — the genuinely unrecoverable case.
2358
+ let backend = OpenAiBackend::new(
2359
+ "http://127.0.0.1:9",
2360
+ "",
2361
+ "test-model",
2362
+ Some("s".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000)),
2363
+ );
2364
+ backend
2365
+ .history
2366
+ .lock()
2367
+ .await
2368
+ .push(serde_json::json!({ "role": "user", "content": "hello" }));
2369
let before = backend.history_snapshot().await;
2370
2371
assert!(backend.compact_history(2).await.is_err());
2416
/// fallback drops nothing beyond what `keep_last` already folded away.
2417
#[tokio::test]
2418
async fn truncation_fallback_keeps_the_newest_messages_that_fit() {
2391
- let system = serde_json::json!({ "role": "system", "content": "sys" });
2419
+ let system = vec![serde_json::json!({ "role": "system", "content": "sys" })];
2420
let tail: Vec<serde_json::Value> = (0..4)
2421
.map(|i| serde_json::json!({ "role": "user", "content": format!("m{i}") }))
2422
.collect();
2423
2396
- let rebuilt = truncate_history_to_budget(Some(&system), tail).unwrap();
2424
+ let rebuilt = truncate_history_to_budget(&system, tail).unwrap();
2425
2426
assert_eq!(rebuilt[0]["role"], "system");
2427
assert_eq!(rebuilt[1]["content"], TRUNCATION_PLACEHOLDER);
2430
assert_eq!(rebuilt[5]["content"], "m3");
2431
}
2432
2433
+ /// A multi-system-message session keeps every leading system message in
2434
+ /// the fallback, exactly like the summarization-success path.
2435
+ #[test]
2436
+ fn truncation_fallback_keeps_all_leading_system_messages() {
2437
+ let system = vec![
2438
+ serde_json::json!({ "role": "system", "content": "seed prompt" }),
2439
+ serde_json::json!({ "role": "system", "content": "session context" }),
2440
+ ];
2441
+ let tail = vec![serde_json::json!({ "role": "user", "content": "hi" })];
2442
+
2443
+ let rebuilt = truncate_history_to_budget(&system, tail).unwrap();
2444
+
2445
+ assert_eq!(
2446
+ rebuilt[0]["content"], "seed prompt",
2447
+ "first system message kept: {rebuilt:?}"
2448
+ );
2449
+ assert_eq!(
2450
+ rebuilt[1]["content"], "session context",
2451
+ "second system message kept: {rebuilt:?}"
2452
+ );
2453
+ }
2454
+
2455
#[test]
2456
fn truncation_fallback_drops_orphaned_tool_results_with_their_call() {
2457
// A caller that split mid-round hands in a tail starting with tool
2464
serde_json::json!({ "role": "assistant", "content": "done" }),
2465
];
2466
2417
- let rebuilt = truncate_history_to_budget(None, tail).unwrap();
2467
+ let rebuilt = truncate_history_to_budget(&[], tail).unwrap();
2468
2419
- let roles: Vec<&str> = rebuilt
2420
- .iter()
2421
- .filter_map(|m| m["role"].as_str())
2422
- .collect();
2469
+ let roles: Vec<&str> = rebuilt.iter().filter_map(|m| m["role"].as_str()).collect();
2470
assert_eq!(
2471
roles,
2472
["user", "user", "assistant"],
2474
);
2475
}
2476
2477
+ /// Review follow-up: a tail that drains to empty (one oversized tool
2478
+ /// round) must still test the minimal rebuild — system + placeholder fits,
2479
+ /// so the session is recoverable, not "unfixable".
2480
+ #[test]
2481
+ fn truncation_fallback_recovers_when_the_whole_tail_was_one_oversized_round() {
2482
+ let system = vec![serde_json::json!({ "role": "system", "content": "sys" })];
2483
+ let tail = vec![
2484
+ serde_json::json!({
2485
+ "role": "assistant",
2486
+ "content": null,
2487
+ "tool_calls": [{
2488
+ "id": "call_1", "type": "function",
2489
+ "function": { "name": "read_file", "arguments": "{}" },
2490
+ }],
2491
+ }),
2492
+ serde_json::json!({
2493
+ "role": "tool",
2494
+ "tool_call_id": "call_1",
2495
+ "content": "y".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000),
2496
+ }),
2497
+ ];
2498
+
2499
+ let rebuilt = truncate_history_to_budget(&system, tail).unwrap();
2500
+
2501
+ assert_eq!(rebuilt.len(), 2, "{rebuilt:?}");
2502
+ assert_eq!(rebuilt[0]["role"], "system");
2503
+ assert_eq!(rebuilt[1]["content"], TRUNCATION_PLACEHOLDER);
2504
+ }
2505
+
2506
#[test]
2507
fn truncation_fallback_reports_when_nothing_fits() {
2432
- // A single message bigger than the whole window can never fit.
2433
- let tail = vec![serde_json::json!({
2434
- "role": "user",
2435
- "content": "x".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000),
2508
+ // Even the minimal rebuild (system + placeholder, no conversation)
2509
+ // overflows: the system messages alone are bigger than the window.
2510
+ let system = vec![serde_json::json!({
2511
+ "role": "system",
2512
+ "content": "s".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000),
2513
})];
2514
2438
- assert!(truncate_history_to_budget(None, tail).is_none());
2515
+ assert!(truncate_history_to_budget(&system, Vec::new()).is_none());
2516
}
2517
2518
#[test]
2522
assert_eq!(truncate_transcript_middle(transcript, 1_000), transcript);
2523
}
2524
2525
+ /// Review follow-up: a huge final message with no newline in the tail
2526
+ /// half must not be silently dropped — the closing exchange is what the
2527
+ /// summary most needs. The head gives up its share instead.
2528
+ #[test]
2529
+ fn truncate_transcript_middle_keeps_a_newlineless_final_message() {
2530
+ let mut lines: Vec<String> = (0..100).map(|i| format!("user: message {i}")).collect();
2531
+ lines.push(format!(
2532
+ "user: {}current state: pending review",
2533
+ "noise ".repeat(600)
2534
+ ));
2535
+ let transcript = lines.join("\n");
2536
+
2537
+ let trimmed = truncate_transcript_middle(&transcript, 400); // ~1600 chars
2538
+
2539
+ assert!(
2540
+ trimmed.contains("current state: pending review"),
2541
+ "the closing exchange survives: {trimmed}"
2542
+ );
2543
+ assert!(
2544
+ trimmed.contains("older messages omitted"),
2545
+ "the gap is marked: {trimmed}"
2546
+ );
2547
+ assert!(
2548
+ !trimmed.contains("message 0"),
2549
+ "the head gives up its share to the unbreakable tail: {trimmed}"
2550
+ );
2551
+ }
2552
+
2553
+ /// Review follow-up: no newline anywhere in reach — keep the newest
2554
+ /// budget, not the oldest (which would invert the contract while the
2555
+ /// placeholder claims older messages were dropped).
2556
+ #[test]
2557
+ fn truncate_transcript_middle_without_any_newline_keeps_the_newest() {
2558
+ let transcript = format!("old decisions {} current state", "x".repeat(4_000));
2559
+
2560
+ let trimmed = truncate_transcript_middle(&transcript, 200); // ~800 chars
2561
+
2562
+ assert!(
2563
+ trimmed.contains("current state"),
2564
+ "the newest content survives: {trimmed}"
2565
+ );
2566
+ assert!(
2567
+ !trimmed.contains("old decisions"),
2568
+ "the oldest goes when only one side can survive: {trimmed}"
2569
+ );
2570
+ }
2571
+
2572
#[test]
2573
fn truncate_transcript_middle_cuts_the_middle_not_the_ends() {
2574
let mut lines: Vec<String> = Vec::new();