@sigitcodereview / sigit / commits / 095725b

fix(compaction): split the transcript cap between head and tail

The middle cut gave each side the full budget, so a trimmed transcript could reach twice SUMMARY_TRANSCRIPT_TOKEN_CAP. Each side now gets half. A side with no line boundary in reach still hands the whole budget to the newest content. The local truncation fallback also refused to drop the last message, so a final message bigger than the window failed the session where the remote path recovers. It now lives in truncate_chat_messages_to_budget, which lets the tail drain to empty and only gives up when the system messages alone don't fit. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-Authored-By: siGit Code v1.5.11 <noreply@sigit.si>

Seto Elkahfi committed Oct 4, 2026 at 23:34 UTC 095725b2947cbdb40580c88320af29dd70afec6e

1 file changed +109 -22

src/backend.rs

+109 -22

@@ -109,10 +109,12 @@ fn truncate_transcript_middle(transcript: &str, cap: usize) -> String {

109 }

110

111 let total = transcript.len();

112 + // Each side gets half the budget, so the two together stay inside the cap.

113 // Walk each half to a line boundary. A side with no boundary in reach

113 - // contributes nothing; the other side may then use the whole budget.

114 + // contributes nothing; the newest side may then use the whole budget.

115 + let half = budget / 2;

116 let head_end = {

115 - let mut cut = budget.min(total);

117 + let mut cut = half.min(total);

118 while cut < total && !transcript.is_char_boundary(cut) {

119 cut += 1;

120 }

@@ -124,9 +126,9 @@ fn truncate_transcript_middle(transcript: &str, cap: usize) -> String {

126 }

127 };

128 let tail_start = {

127 - let mut cut = total.saturating_sub(budget);

128 - while cut > 0 && !transcript.is_char_boundary(cut) {

129 - cut -= 1;

129 + let mut cut = total.saturating_sub(half);

130 + while cut < total && !transcript.is_char_boundary(cut) {

131 + cut += 1;

132 }

133 if transcript[cut..].contains('\n') {

134 transcript[cut..]

@@ -530,24 +532,11 @@ impl InferenceBackend for LocalBackend {

532 // Same deterministic shrink as the remote path, on the plain

533 // chat shapes the engine holds (its history has no tool-call

534 // structure, so there are no orphaned tool results to guard).

533 - let system_chars: usize = system.iter().map(|message| message.content.len()).sum();

534 - let mut kept: Vec<ChatMessage> = tail;

535 - while kept.iter().map(|m| m.content.len()).sum::<usize>() + system_chars

536 - > DEFAULT_CONTEXT_TOKEN_BUDGET * 4

537 - && kept.len() > 1

538 - {

539 - kept.remove(0);

540 - }

541 - if kept.iter().map(|m| m.content.len()).sum::<usize>() + system_chars

542 - > DEFAULT_CONTEXT_TOKEN_BUDGET * 4

543 - {

544 - // Even one message overflows the window: unrecoverable.

545 - return Err(error);

535 + match truncate_chat_messages_to_budget(system, tail) {

536 + Some(rebuilt) => rebuilt,

537 + // The system messages alone overflow the window: unrecoverable.

538 + None => return Err(error),

539 }

547 - let mut rebuilt = system;

548 - rebuilt.push(ChatMessage::user(TRUNCATION_PLACEHOLDER.to_string()));

549 - rebuilt.extend(kept);

550 - rebuilt

540 }

541 };

542

@@ -559,6 +548,41 @@ impl InferenceBackend for LocalBackend {

548 }

549 }

550

551 +/// The local counterpart of `truncate_history_to_budget`: system messages, the

552 +/// truncation placeholder, and the newest of `tail` that fit the budget.

553 +///

554 +/// The tail may drain to empty. A newest message that overflows the window on

555 +/// its own (a giant tool result, the very thing that triggers compaction) is

556 +/// dropped rather than failing the session, so `None` only means the system

557 +/// messages alone don't fit.

558 +fn truncate_chat_messages_to_budget(

559 + system: Vec<ChatMessage>,

560 + mut tail: Vec<ChatMessage>,

561 +) -> Option<Vec<ChatMessage>> {

562 + let budget = DEFAULT_CONTEXT_TOKEN_BUDGET * 4; // estimate_tokens is chars / 4.

563 + let fixed_chars: usize = system

564 + .iter()

565 + .map(|message| message.content.len())

566 + .sum::<usize>()

567 + + TRUNCATION_PLACEHOLDER.len();

568 + if fixed_chars > budget {

569 + return None;

570 + }

571 +

572 + let mut tail_chars: usize = tail.iter().map(|message| message.content.len()).sum();

573 + let mut drop = 0;

574 + while fixed_chars + tail_chars > budget {

575 + tail_chars -= tail[drop].content.len();

576 + drop += 1;

577 + }

578 + tail.drain(..drop);

579 +

580 + let mut rebuilt = system;

581 + rebuilt.push(ChatMessage::user(TRUNCATION_PLACEHOLDER.to_string()));

582 + rebuilt.extend(tail);

583 + Some(rebuilt)

584 +}

585 +

586 /// Flatten `onde` chat messages into the transcript shape summarization

587 /// expects (the local engine's history has no tool-call structure, so this is

588 /// a plain role/content rendering).

@@ -2515,6 +2539,69 @@ mod tests {

2539 assert!(truncate_history_to_budget(&system, Vec::new()).is_none());

2540 }

2541

2542 + /// Review follow-up: the local fallback recovers from a newest message

2543 + /// that overflows the window on its own, the same way the remote one does.

2544 + #[test]

2545 + fn local_truncation_fallback_drops_an_oversized_final_message() {

2546 + let system = vec![ChatMessage::system("sys")];

2547 + let tail = vec![

2548 + ChatMessage::user("hi"),

2549 + ChatMessage::user("y".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000)),

2550 + ];

2551 +

2552 + let rebuilt = truncate_chat_messages_to_budget(system, tail).unwrap();

2553 +

2554 + assert_eq!(rebuilt.len(), 2, "{rebuilt:?}");

2555 + assert_eq!(rebuilt[0].content, "sys");

2556 + assert_eq!(rebuilt[1].content, TRUNCATION_PLACEHOLDER);

2557 + }

2558 +

2559 + #[test]

2560 + fn local_truncation_fallback_keeps_the_newest_messages_that_fit() {

2561 + let big = DEFAULT_CONTEXT_TOKEN_BUDGET * 4 / 2;

2562 + let tail = vec![

2563 + ChatMessage::user("a".repeat(big)),

2564 + ChatMessage::user("b".repeat(big)),

2565 + ChatMessage::user("newest"),

2566 + ];

2567 +

2568 + let rebuilt = truncate_chat_messages_to_budget(Vec::new(), tail).unwrap();

2569 +

2570 + assert_eq!(rebuilt.len(), 3, "oldest dropped, rest kept");

2571 + assert_eq!(rebuilt[0].content, TRUNCATION_PLACEHOLDER);

2572 + assert!(rebuilt[1].content.starts_with('b'));

2573 + assert_eq!(rebuilt[2].content, "newest");

2574 + }

2575 +

2576 + #[test]

2577 + fn local_truncation_fallback_reports_when_the_system_messages_overflow() {

2578 + let system = vec![ChatMessage::system(

2579 + "s".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000),

2580 + )];

2581 +

2582 + assert!(truncate_chat_messages_to_budget(system, Vec::new()).is_none());

2583 + }

2584 +

2585 + /// Review follow-up: head and tail share the budget, so the trimmed

2586 + /// transcript stays near the cap instead of reaching twice it.

2587 + #[test]

2588 + fn truncate_transcript_middle_stays_within_the_cap() {

2589 + let lines: Vec<String> = (0..2_000)

2590 + .map(|i| format!("user: message {i} with some padding text"))

2591 + .collect();

2592 + let transcript = lines.join("\n");

2593 +

2594 + let trimmed = truncate_transcript_middle(&transcript, 200); // ~800 chars

2595 +

2596 + assert!(trimmed.contains("message 0 "), "opening kept: {trimmed}");

2597 + assert!(trimmed.contains("message 1999"), "closing kept: {trimmed}");

2598 + assert!(

2599 + trimmed.len() <= 800 + 80,

2600 + "cap plus the omission marker, got {} chars",

2601 + trimmed.len()

2602 + );

2603 + }

2604 +

2605 #[test]

2606 fn truncate_transcript_middle_keeps_short_transcripts_verbatim() {

2607 let transcript = "user: hello\n\nassistant: hi";