2ee9e1b · sigitcode-old/sigit · siGit Code & Deploy

@@ -786,8 +786,6 @@ impl InferenceBackend for LocalBackend {

786 .into_iter()

787 .filter(|message| message.role != ChatRole::System)

788 .collect();

789 - let tail_start = non_system.len().saturating_sub(keep_last);

790 - let tail: Vec<ChatMessage> = non_system[tail_start..].to_vec();

789

790 // One plain (tool-free) inference round produces the summary. Note the

791 // cap only bounds this request's own payload: the engine still replays

@@ -798,6 +796,14 @@ impl InferenceBackend for LocalBackend {

796 &chat_messages_transcript(&non_system),

797 SUMMARY_TRANSCRIPT_TOKEN_CAP,

798 );

799 +

800 + // Keep the opening request verbatim; the tail is taken from what is

801 + // left so the pin is neither duplicated nor counted against `keep_last`.

802 + let mut non_system = non_system;

803 + let pin = take_pinned_first_chat_message(&mut non_system);

804 + let tail_start = non_system.len().saturating_sub(keep_last);

805 + let tail: Vec<ChatMessage> = non_system[tail_start..].to_vec();

806 +

807 let result = self

808 .engine

809 .send_message(format!("{transcript}\n\n{SUMMARIZE_PROMPT}"))

@@ -809,6 +815,7 @@ impl InferenceBackend for LocalBackend {

815 // Local models may reason in <think> blocks; keep only the visible part.

816 let (_think, summary) = crate::chat::strip_think_blocks(&result.text);

817 let mut rebuilt = system;

818 + rebuilt.extend(pin);

819 rebuilt.push(ChatMessage::user(format!(

820 "[Conversation summary]\n{summary}"

821 )));

@@ -822,7 +829,7 @@ impl InferenceBackend for LocalBackend {

829 // Same deterministic shrink as the remote path, on the plain

830 // chat shapes the engine holds (its history has no tool-call

831 // structure, so there are no orphaned tool results to guard).

825 - match truncate_chat_messages_to_budget(system, tail) {

832 + match truncate_chat_messages_keeping_pin(system, pin, tail) {

833 Some(rebuilt) => rebuilt,

834 // The system messages alone overflow the window: unrecoverable.

835 None => return Err(error),

@@ -838,6 +845,29 @@ impl InferenceBackend for LocalBackend {

845 }

846 }

847

848 +/// The local counterpart of `take_pinned_first_message`.

849 +fn take_pinned_first_chat_message(non_system: &mut Vec<ChatMessage>) -> Option<ChatMessage> {

850 + let first = non_system.first()?;

851 + (first.role == ChatRole::User && is_pinnable_user_content(&first.content))

852 + .then(|| non_system.remove(0))

853 +}

854 +

855 +/// The local counterpart of `truncate_history_keeping_pin`.

856 +fn truncate_chat_messages_keeping_pin(

857 + system: Vec<ChatMessage>,

858 + pin: Option<ChatMessage>,

859 + tail: Vec<ChatMessage>,

860 +) -> Option<Vec<ChatMessage>> {

861 + if let Some(pin) = pin {

862 + let mut prefix = system.clone();

863 + prefix.push(pin);

864 + if let Some(rebuilt) = truncate_chat_messages_to_budget(prefix, tail.clone()) {

865 + return Some(rebuilt);

866 + }

867 + }

868 + truncate_chat_messages_to_budget(system, tail)

869 +}

870 +

871 /// The local counterpart of `truncate_history_to_budget`: system messages, the

872 /// truncation placeholder, and the newest of `tail` that fit the budget.

873 ///

@@ -1883,11 +1913,14 @@ impl InferenceBackend for OpenAiBackend {

1913 .cloned()

1914 .collect();

1915

1886 - let non_system: Vec<serde_json::Value> = snapshot

1916 + let mut non_system: Vec<serde_json::Value> = snapshot

1917 .iter()

1918 .filter(|message| message["role"] != "system")

1919 .cloned()

1920 .collect();

1921 + // The opening request stays verbatim; the tail is taken from what is

1922 + // left so the pin is neither duplicated nor counted against `keep_last`.

1923 + let pin = take_pinned_first_message(&mut non_system);

1924 let tail_start = non_system.len().saturating_sub(keep_last);

1925 let mut tail = non_system[tail_start..].to_vec();

1926 // Drop leading tool results whose assistant tool-call message was

@@ -1922,14 +1955,16 @@ impl InferenceBackend for OpenAiBackend {

1955 // a placeholder and drop the oldest messages until the rebuild

1956 // fits the budget. Needs no endpoint, so it cannot fail the way

1957 // the inference round just did (issue #125).

1925 - let rebuilt = truncate_history_to_budget(&system, tail.clone()).ok_or(error)?;

1958 + let rebuilt = truncate_history_keeping_pin(&system, pin.as_ref(), tail.clone())

1959 + .ok_or(error)?;

1960 *self.history.lock().await = rebuilt;

1961 return Ok(());

1962 }

1963 };

1964

1931 - let mut rebuilt = Vec::with_capacity(system.len() + tail.len() + 1);

1965 + let mut rebuilt = Vec::with_capacity(system.len() + tail.len() + 2);

1966 rebuilt.extend(system);

1967 + rebuilt.extend(pin);

1968 rebuilt.push(serde_json::json!({

1969 "role": "user",

1970 "content": format!("[Conversation summary]\n{summary}"),

@@ -1940,6 +1975,54 @@ impl InferenceBackend for OpenAiBackend {

1975 }

1976 }

1977

1978 +/// The largest first user message that compaction keeps verbatim, in

1979 +/// `estimate_tokens` units. A third of the context budget: big enough for a

1980 +/// long task description, small enough that pinning it cannot keep the history

1981 +/// over budget and trigger compaction on every round.

1982 +const PINNED_MESSAGE_TOKEN_CAP: usize = 8_000;

1983 +

1984 +/// Whether `content`, sent as a user message, should survive compaction

1985 +/// verbatim as the opening request of the session.

1986 +///

1987 +/// A restored summary or truncation placeholder is derived text, not a

1988 +/// request, and a message over [`PINNED_MESSAGE_TOKEN_CAP`] is summarized like

1989 +/// any other.

1990 +fn is_pinnable_user_content(content: &str) -> bool {

1991 + !content.starts_with("[Conversation summary]")

1992 + && !content.starts_with(TRUNCATION_PLACEHOLDER)

1993 + && estimate_tokens(&[serde_json::json!({ "role": "user", "content": content })])

1994 + <= PINNED_MESSAGE_TOKEN_CAP

1995 +}

1996 +

1997 +/// Remove and return the first non-system message when it qualifies as the

1998 +/// pinned opening request (see [`is_pinnable_user_content`]).

1999 +fn take_pinned_first_message(non_system: &mut Vec<serde_json::Value>) -> Option<serde_json::Value> {

2000 + let first = non_system.first()?;

2001 + let pinnable = first["role"] == "user"

2002 + && first["content"]

2003 + .as_str()

2004 + .is_some_and(is_pinnable_user_content);

2005 + pinnable.then(|| non_system.remove(0))

2006 +}

2007 +

2008 +/// [`truncate_history_to_budget`] with the pin placed after the system

2009 +/// messages. If that rebuild cannot fit, retry without the pin, so the pin

2010 +/// cannot turn a recoverable session into a failed one.

2011 +fn truncate_history_keeping_pin(

2012 + system: &[serde_json::Value],

2013 + pin: Option<&serde_json::Value>,

2014 + tail: Vec<serde_json::Value>,

2015 +) -> Option<Vec<serde_json::Value>> {

2016 + if let Some(pin) = pin {

2017 + let mut prefix = system.to_vec();

2018 + prefix.push(pin.clone());

2019 + if let Some(rebuilt) = truncate_history_to_budget(&prefix, tail.clone()) {

2020 + return Some(rebuilt);

2021 + }

2022 + }

2023 + truncate_history_to_budget(system, tail)

2024 +}

2025 +

2026 /// The placeholder message replacing dropped history when compaction falls

2027 /// back to plain truncation (the model reads it, so it explains the gap).

2028 const TRUNCATION_PLACEHOLDER: &str = "[Older conversation omitted: the session outgrew \

@@ -3393,23 +3476,257 @@ mod tests {

3476 backend.compact_history(2).await.unwrap();

3477

3478 let history = backend.history_snapshot().await;

3396 - assert_eq!(history.len(), 4, "system + summary + last 2: {history:?}");

3479 + assert_eq!(

3480 + history.len(),

3481 + 5,

3482 + "system + pinned first message + summary + last 2: {history:?}"

3483 + );

3484 assert_eq!(history[0]["role"], "system");

3485 assert_eq!(history[0]["content"], "be helpful");

3399 - assert_eq!(history[1]["role"], "user");

3400 - let summary_text = history[1]["content"].as_str().unwrap();

3486 + assert_eq!(

3487 + history[1],

3488 + serde_json::json!({ "role": "user", "content": "message 0" })

3489 + );

3490 + assert_eq!(history[2]["role"], "user");

3491 + let summary_text = history[2]["content"].as_str().unwrap();

3492 assert!(summary_text.starts_with("[Conversation summary]\n"));

3493 assert!(summary_text.contains("We refactored backend.rs; tests pass."));

3494 assert_eq!(

3404 - history[2],

3495 + history[3],

3496 serde_json::json!({ "role": "assistant", "content": "message 3" })

3497 );

3498 assert_eq!(

3408 - history[3],

3499 + history[4],

3500 serde_json::json!({ "role": "user", "content": "message 4" })

3501 );

3502 }

3503

3504 + /// Push `count` alternating user/assistant messages after the first one.

3505 + async fn push_exchange(backend: &OpenAiBackend, first: &str, count: usize) {

3506 + let mut history = backend.history.lock().await;

3507 + history.push(serde_json::json!({ "role": "user", "content": first }));

3508 + for i in 1..count {

3509 + let role = if i % 2 == 0 { "user" } else { "assistant" };

3510 + history.push(serde_json::json!({ "role": role, "content": format!("message {i}") }));

3511 + }

3512 + }

3513 +

3514 + fn count_content(history: &[serde_json::Value], content: &str) -> usize {

3515 + history

3516 + .iter()

3517 + .filter(|message| message["content"] == content)

3518 + .count()

3519 + }

3520 +

3521 + #[tokio::test]

3522 + async fn compaction_keeps_the_first_user_message_through_repeat_compactions() {

3523 + let first = "Open the app, sign in with the credentials, then report.";

3524 + let (addr, _requests) = spawn_completion_stub("first summary");

3525 + let backend = OpenAiBackend::new(

3526 + format!("http://{addr}/v1"),

3527 + "test-key",

3528 + "test-model",

3529 + Some("sys".into()),

3530 + );

3531 + push_exchange(&backend, first, 8).await;

3532 + backend.compact_history(2).await.unwrap();

3533 + let once = backend.history_snapshot().await;

3534 + assert_eq!(once[1]["content"], first, "{once:?}");

3535 + assert_eq!(count_content(&once, first), 1, "{once:?}");

3536 +

3537 + // Second compaction on a fresh endpoint (the stub serves one request),

3538 + // after the session has grown again.

3539 + let (addr, _requests) = spawn_completion_stub("second summary");

3540 + let second = OpenAiBackend::new(

3541 + format!("http://{addr}/v1"),

3542 + "test-key",

3543 + "test-model",

3544 + Some("sys".into()),

3545 + );

3546 + second.restore_history(once).await;

3547 + for i in 0..6 {

3548 + second

3549 + .history

3550 + .lock()

3551 + .await

3552 + .push(serde_json::json!({ "role": "user", "content": format!("later {i}") }));

3553 + }

3554 + second.compact_history(2).await.unwrap();

3555 +

3556 + let twice = second.history_snapshot().await;

3557 + assert_eq!(twice[0]["role"], "system");

3558 + assert_eq!(twice[1]["content"], first, "{twice:?}");

3559 + assert_eq!(count_content(&twice, first), 1, "{twice:?}");

3560 + assert!(

3561 + twice[2]["content"]

3562 + .as_str()

3563 + .unwrap()

3564 + .starts_with("[Conversation summary]\n"),

3565 + "{twice:?}"

3566 + );

3567 + assert!(

3568 + twice[2]["content"]

3569 + .as_str()

3570 + .unwrap()

3571 + .contains("second summary")

3572 + );

3573 + }

3574 +

3575 + #[tokio::test]

3576 + async fn compaction_does_not_pin_a_first_message_over_the_cap() {

3577 + let (addr, _requests) = spawn_completion_stub("a summary");

3578 + let backend = OpenAiBackend::new(

3579 + format!("http://{addr}/v1"),

3580 + "test-key",

3581 + "test-model",

3582 + Some("sys".into()),

3583 + );

3584 + let huge = "x".repeat(PINNED_MESSAGE_TOKEN_CAP * 4 + 400);

3585 + push_exchange(&backend, &huge, 6).await;

3586 +

3587 + backend.compact_history(2).await.unwrap();

3588 +

3589 + let history = backend.history_snapshot().await;

3590 + assert_eq!(

3591 + count_content(&history, &huge),

3592 + 0,

3593 + "{} messages",

3594 + history.len()

3595 + );

3596 + assert_eq!(history.len(), 4, "system + summary + tail");

3597 + assert!(

3598 + history[1]["content"]

3599 + .as_str()

3600 + .unwrap()

3601 + .starts_with("[Conversation summary]")

3602 + );

3603 + }

3604 +

3605 + #[tokio::test]

3606 + async fn compaction_does_not_pin_a_restored_summary() {

3607 + let (addr, _requests) = spawn_completion_stub("a summary");

3608 + let backend = OpenAiBackend::new(

3609 + format!("http://{addr}/v1"),

3610 + "test-key",

3611 + "test-model",

3612 + Some("sys".into()),

3613 + );

3614 + let old_summary = "[Conversation summary]\nearlier work";

3615 + push_exchange(&backend, old_summary, 6).await;

3616 +

3617 + backend.compact_history(2).await.unwrap();

3618 +

3619 + let history = backend.history_snapshot().await;

3620 + assert_eq!(count_content(&history, old_summary), 0, "{history:?}");

3621 + assert_eq!(history.len(), 4, "{history:?}");

3622 + }

3623 +

3624 + #[tokio::test]

3625 + async fn compaction_does_not_duplicate_a_first_message_inside_the_tail() {

3626 + let (addr, _requests) = spawn_completion_stub("a summary");

3627 + let backend = OpenAiBackend::new(

3628 + format!("http://{addr}/v1"),

3629 + "test-key",

3630 + "test-model",

3631 + Some("sys".into()),

3632 + );

3633 + push_exchange(&backend, "the only request", 3).await;

3634 +

3635 + backend.compact_history(6).await.unwrap();

3636 +

3637 + let history = backend.history_snapshot().await;

3638 + assert_eq!(

3639 + count_content(&history, "the only request"),

3640 + 1,

3641 + "{history:?}"

3642 + );

3643 + assert_eq!(history[1]["content"], "the only request");

3644 + assert_eq!(history.len(), 5, "system + pin + summary + 2: {history:?}");

3645 + }

3646 +

3647 + #[tokio::test]

3648 + async fn truncation_fallback_keeps_the_pinned_first_message() {

3649 + // Nothing listens here, so summarization fails.

3650 + let backend =

3651 + OpenAiBackend::new("http://127.0.0.1:9", "", "test-model", Some("sys".into()));

3652 + {

3653 + let mut history = backend.history.lock().await;

3654 + history.push(serde_json::json!({ "role": "user", "content": "the request" }));

3655 + for i in 0..20 {

3656 + history.push(serde_json::json!({

3657 + "role": "user",

3658 + "content": format!("message {i}: {}", "padding ".repeat(200)),

3659 + }));

3660 + }

3661 + }

3662 +

3663 + backend.compact_history(4).await.unwrap();

3664 +

3665 + let history = backend.history_snapshot().await;

3666 + assert_eq!(history[0]["role"], "system");

3667 + assert_eq!(history[1]["content"], "the request", "{history:?}");

3668 + assert_eq!(history[2]["content"], TRUNCATION_PLACEHOLDER);

3669 + assert!(estimate_tokens(&history) <= DEFAULT_CONTEXT_TOKEN_BUDGET);

3670 + }

3671 +

3672 + #[tokio::test]

3673 + async fn truncation_fallback_retries_without_the_pin_when_it_does_not_fit() {

3674 + // The system prompt fits the window on its own, but not beside the pin.

3675 + let backend = OpenAiBackend::new(

3676 + "http://127.0.0.1:9",

3677 + "",

3678 + "test-model",

3679 + Some("s".repeat((DEFAULT_CONTEXT_TOKEN_BUDGET - 7_000) * 4)),

3680 + );

3681 + let pin = "p".repeat((PINNED_MESSAGE_TOKEN_CAP - 500) * 4);

3682 + push_exchange(&backend, &pin, 4).await;

3683 +

3684 + backend.compact_history(2).await.unwrap();

3685 +

3686 + let history = backend.history_snapshot().await;

3687 + assert_eq!(count_content(&history, &pin), 0, "pin dropped to fit");

3688 + assert_eq!(history[1]["content"], TRUNCATION_PLACEHOLDER);

3689 + assert!(estimate_tokens(&history) <= DEFAULT_CONTEXT_TOKEN_BUDGET);

3690 + }

3691 +

3692 + #[test]

3693 + fn local_pin_helpers_follow_the_same_rules() {

3694 + let mut messages = vec![ChatMessage::user("request"), ChatMessage::user("later")];

3695 + let pin = take_pinned_first_chat_message(&mut messages);

3696 + assert_eq!(pin.map(|m| m.content), Some("request".to_string()));

3697 + assert_eq!(messages.len(), 1);

3698 +

3699 + let mut summary = vec![ChatMessage::user("[Conversation summary]\nold")];

3700 + assert!(take_pinned_first_chat_message(&mut summary).is_none());

3701 + assert_eq!(summary.len(), 1);

3702 +

3703 + let mut huge = vec![ChatMessage::user(

3704 + "x".repeat(PINNED_MESSAGE_TOKEN_CAP * 4 + 400),

3705 + )];

3706 + assert!(take_pinned_first_chat_message(&mut huge).is_none());

3707 +

3708 + // The pin leads the rebuild; when it cannot fit beside the system

3709 + // messages the rebuild drops it instead of failing.

3710 + let rebuilt = truncate_chat_messages_keeping_pin(

3711 + vec![ChatMessage::system("sys")],

3712 + Some(ChatMessage::user("request")),

3713 + vec![ChatMessage::user("newest")],

3714 + )

3715 + .unwrap();

3716 + let texts: Vec<&str> = rebuilt.iter().map(|m| m.content.as_str()).collect();

3717 + assert_eq!(texts, ["sys", "request", TRUNCATION_PLACEHOLDER, "newest"]);

3718 +

3719 + let system_len = (DEFAULT_CONTEXT_TOKEN_BUDGET - 100) * 4;

3720 + let rebuilt = truncate_chat_messages_keeping_pin(

3721 + vec![ChatMessage::system("s".repeat(system_len))],

3722 + Some(ChatMessage::user("p".repeat(1_000))),

3723 + Vec::new(),

3724 + )

3725 + .unwrap();

3726 + assert_eq!(rebuilt.len(), 2, "pin dropped: {}", rebuilt.len());

3727 + assert_eq!(rebuilt[1].content, TRUNCATION_PLACEHOLDER);

3728 + }

3729 +

3730 /// Compacting a tool-heavy session must not put tool shapes on the wire.

3731 /// The summarization round offers no `tools`, and endpoints reject tool

3732 /// calls and tool results that arrive without a schema — which used to make

@@ -3516,7 +3833,15 @@ mod tests {

3833

3834 let history = backend.history_snapshot().await;

3835 assert_eq!(history[0]["role"], "system", "{history:?}");

3519 - assert_eq!(history[1]["content"], TRUNCATION_PLACEHOLDER, "{history:?}");

3836 + // The opening message is pinned right after the system prompt.

3837 + assert!(

3838 + history[1]["content"]

3839 + .as_str()

3840 + .unwrap()

3841 + .starts_with("message 0:"),

3842 + "{history:?}"

3843 + );

3844 + assert_eq!(history[2]["content"], TRUNCATION_PLACEHOLDER, "{history:?}");

3845 // The newest messages survive; the oldest were dropped to fit.

3846 assert_eq!(

3847 history.last().unwrap()["content"]

Keep the first user message through compaction