@sigitcodereview / sigit / commits / 38d585f

Carry the conversation across a model switch

Switching models mid-thread in ACP restarted the conversation. Two things dropped it: loading GGUF weights resets the engine's history, and picking a cloud tier built a fresh OpenAiBackend holding nothing but a system prompt. Both switch paths now snapshot the live history first and replay it under the new model's prompt. carryover_history drops the old system messages and repairs half-finished tool plumbing: a switch can land after the assistant asked for a tool but before the results came back, and strict OpenAI-compatible endpoints reject a session carrying an unanswered tool call or an orphaned tool result. The two /models handlers no longer clear_history after a local switch — that was wiping what the replay had just restored. Fixes #56 Co-Authored-By: siGit Code <noreply@sigit.si>

Seto Elkahfi committed Sep 2, 2026 at 22:17 UTC 38d585fcb458eadb1922bdb093c66c87e4173117

2 files changed +238 -4

src/backend.rs

+202

@@ -880,6 +880,89 @@ struct StreamFunctionDelta {

880 arguments: Option<String>,

881 }

882

883 +// ── Carrying a conversation across a model switch ────────────────────────────

884 +

885 +/// The part of a history snapshot that survives a model switch: every

886 +/// non-system message, with half-finished tool plumbing repaired.

887 +///

888 +/// System messages are dropped because the backend being switched *to* seeds

889 +/// its own — a different model's prompt, or a freshly pushed session-context

890 +/// message. A switch can land mid-turn, after the assistant asked for a tool

891 +/// but before the results came back; a tool call with no matching result (or a

892 +/// result with no matching call) makes strict OpenAI-compatible endpoints

893 +/// reject every later request in the session, so those halves are stripped

894 +/// instead of carried.

895 +pub fn carryover_history(snapshot: Vec<serde_json::Value>) -> Vec<serde_json::Value> {

896 + let answered: std::collections::HashSet<String> = snapshot

897 + .iter()

898 + .filter(|message| message["role"] == "tool")

899 + .filter_map(|message| message["tool_call_id"].as_str().map(str::to_string))

900 + .collect();

901 +

902 + let mut carried: Vec<serde_json::Value> = Vec::with_capacity(snapshot.len());

903 + let mut called: std::collections::HashSet<String> = std::collections::HashSet::new();

904 +

905 + for mut message in snapshot {

906 + match message["role"].as_str().unwrap_or_default() {

907 + "system" => continue,

908 + "assistant" => {

909 + if let Some(calls) = message["tool_calls"].as_array() {

910 + let kept: Vec<serde_json::Value> = calls

911 + .iter()

912 + .filter(|call| call["id"].as_str().is_some_and(|id| answered.contains(id)))

913 + .cloned()

914 + .collect();

915 + if kept.is_empty() {

916 + // Only unanswered requests: keep whatever text came

917 + // with them, drop the message if there was none.

918 + if let Some(object) = message.as_object_mut() {

919 + object.remove("tool_calls");

920 + }

921 + if message["content"].as_str().unwrap_or_default().is_empty() {

922 + continue;

923 + }

924 + } else {

925 + for call in &kept {

926 + called.insert(call["id"].as_str().unwrap_or_default().to_string());

927 + }

928 + message["tool_calls"] = serde_json::Value::Array(kept);

929 + }

930 + }

931 + carried.push(message);

932 + }

933 + "tool" => {

934 + let answers_a_kept_call = message["tool_call_id"]

935 + .as_str()

936 + .is_some_and(|id| called.contains(id));

937 + if answers_a_kept_call {

938 + carried.push(message);

939 + }

940 + }

941 + _ => carried.push(message),

942 + }

943 + }

944 +

945 + carried

946 +}

947 +

948 +/// Replay `carried` (from [`carryover_history`]) into `backend`, on top of the

949 +/// system messages `backend` seeded for itself. Used when a model switch

950 +/// installs a new backend — or reloads the on-device engine, which wipes its

951 +/// history — so the thread continues under the new model instead of restarting.

952 +pub async fn adopt_carryover(backend: &dyn InferenceBackend, carried: Vec<serde_json::Value>) {

953 + if carried.is_empty() {

954 + return;

955 + }

956 + let mut rebuilt: Vec<serde_json::Value> = backend

957 + .history_snapshot()

958 + .await

959 + .into_iter()

960 + .take_while(|message| message["role"] == "system")

961 + .collect();

962 + rebuilt.extend(carried);

963 + backend.restore_history(rebuilt).await;

964 +}

965 +

966 #[cfg(test)]

967 mod tests {

968 use super::*;

@@ -968,6 +1051,125 @@ mod tests {

1051 assert_eq!(last["content"], "cancelled by the user");

1052 }

1053

1054 + #[test]

1055 + fn carryover_drops_system_messages_and_keeps_the_turns() {

1056 + let snapshot = vec![

1057 + serde_json::json!({ "role": "system", "content": "old prompt" }),

1058 + serde_json::json!({ "role": "user", "content": "hello" }),

1059 + serde_json::json!({ "role": "assistant", "content": "hi" }),

1060 + ];

1061 +

1062 + let carried = carryover_history(snapshot);

1063 +

1064 + assert_eq!(carried.len(), 2);

1065 + assert_eq!(carried[0]["role"], "user");

1066 + assert_eq!(carried[1]["content"], "hi");

1067 + }

1068 +

1069 + #[test]

1070 + fn carryover_strips_a_tool_call_that_never_got_a_result() {

1071 + // Switching mid-turn: the assistant asked for a tool, nothing answered.

1072 + let snapshot = vec![

1073 + serde_json::json!({ "role": "user", "content": "read a.rs" }),

1074 + streamed_assistant_history(

1075 + "on it",

1076 + &[ToolCall {

1077 + id: "call_1".to_string(),

1078 + name: "read_file".to_string(),

1079 + arguments: "{}".to_string(),

1080 + }],

1081 + ),

1082 + ];

1083 +

1084 + let carried = carryover_history(snapshot);

1085 +

1086 + assert_eq!(carried.len(), 2);

1087 + assert_eq!(carried[1]["content"], "on it");

1088 + assert!(

1089 + carried[1].get("tool_calls").is_none(),

1090 + "an unanswered tool call must not survive the switch"

1091 + );

1092 + }

1093 +

1094 + #[test]

1095 + fn carryover_drops_a_textless_unanswered_tool_call_and_its_late_result() {

1096 + let snapshot = vec![

1097 + streamed_assistant_history(

1098 + "",

1099 + &[ToolCall {

1100 + id: "call_1".to_string(),

1101 + name: "read_file".to_string(),

1102 + arguments: "{}".to_string(),

1103 + }],

1104 + ),

1105 + // An orphan: its assistant message is gone with the line above.

1106 + serde_json::json!({

1107 + "role": "tool",

1108 + "tool_call_id": "call_other",

1109 + "content": "file contents",

1110 + }),

1111 + ];

1112 +

1113 + assert!(carryover_history(snapshot).is_empty());

1114 + }

1115 +

1116 + #[test]

1117 + fn carryover_keeps_an_answered_tool_call_paired_with_its_result() {

1118 + let snapshot = vec![

1119 + streamed_assistant_history(

1120 + "",

1121 + &[ToolCall {

1122 + id: "call_1".to_string(),

1123 + name: "read_file".to_string(),

1124 + arguments: "{}".to_string(),

1125 + }],

1126 + ),

1127 + serde_json::json!({

1128 + "role": "tool",

1129 + "tool_call_id": "call_1",

1130 + "content": "file contents",

1131 + }),

1132 + ];

1133 +

1134 + let carried = carryover_history(snapshot);

1135 +

1136 + assert_eq!(carried.len(), 2);

1137 + assert_eq!(carried[0]["tool_calls"][0]["id"], "call_1");

1138 + assert_eq!(carried[1]["tool_call_id"], "call_1");

1139 + }

1140 +

1141 + #[tokio::test]

1142 + async fn adopt_carryover_replays_the_thread_under_the_new_system_prompt() {

1143 + let new_backend =

1144 + OpenAiBackend::new("http://localhost", "", "m", Some("new prompt".into()));

1145 +

1146 + let carried = carryover_history(vec![

1147 + serde_json::json!({ "role": "system", "content": "old prompt" }),

1148 + serde_json::json!({ "role": "user", "content": "hello" }),

1149 + serde_json::json!({ "role": "assistant", "content": "hi" }),

1150 + ]);

1151 + adopt_carryover(&new_backend, carried).await;

1152 +

1153 + let history = new_backend.history_snapshot().await;

1154 + assert_eq!(history.len(), 3);

1155 + assert_eq!(history[0]["role"], "system");

1156 + assert_eq!(history[0]["content"], "new prompt");

1157 + assert_eq!(history[1]["content"], "hello");

1158 + assert_eq!(history[2]["content"], "hi");

1159 + }

1160 +

1161 + #[tokio::test]

1162 + async fn adopt_carryover_leaves_a_fresh_backend_alone() {

1163 + let new_backend =

1164 + OpenAiBackend::new("http://localhost", "", "m", Some("new prompt".into()));

1165 +

1166 + adopt_carryover(&new_backend, Vec::new()).await;

1167 +

1168 + let history = new_backend.history_snapshot().await;

1169 + assert_eq!(history.len(), 1);

1170 + assert_eq!(history[0]["content"], "new prompt");

1171 + }

1172 +

1173 #[test]

1174 fn estimate_tokens_scales_with_serialized_size() {

1175 assert_eq!(estimate_tokens(&[]), 0);

src/main.rs

+36 -4

@@ -901,6 +901,17 @@ impl SiGitAgent {

901 new_config.display_name

902 );

903

904 + // Loading weights resets the engine's history, which would restart the

905 + // conversation under the new model. Keep the turns so far and replay

906 + // them below, once the new prompt and session context are seeded.

907 + let carried = self

908 + .engine

909 + .history()

910 + .await

911 + .into_iter()

912 + .filter(|message| message.role != onde::inference::ChatRole::System)

913 + .collect::<Vec<_>>();

914 +

915 let sampling = SamplingConfig {

916 max_tokens: Some(max_tokens),

917 ..SamplingConfig::default()

@@ -969,6 +980,10 @@ impl SiGitAgent {

980 .await;

981 }

982

983 + for message in carried {

984 + self.engine.push_history(message).await;

985 + }

986 +

987 Ok(new_config)

988 }

989 }

@@ -1820,6 +1835,10 @@ impl SiGitAgent {

1835 cfg.model,

1836 Some(system_prompt),

1837 ));

1838 + // Continue the thread on the new backend instead of starting over.

1839 + let carried =

1840 + backend::carryover_history(self.backend.lock().await.history_snapshot().await);

1841 + backend::adopt_carryover(cloud_backend.as_ref(), carried).await;

1842 *self.backend.lock().await = cloud_backend;

1843

1844 let cloud_config = GgufModelConfig {

@@ -1862,11 +1881,26 @@ impl SiGitAgent {

1881 }

1882

1883 /// Route inference back on-device. Used after leaving a cloud tier for a

1865 - /// local model. The `LocalBackend` reads the live `engine`, so this just

1866 - /// repoints the active backend.

1884 + /// local model. The `LocalBackend` reads the live `engine`, so this repoints

1885 + /// the active backend and moves the conversation onto the engine with it.

1886 async fn reset_to_local_backend(&self) {

1887 + // Coming back from a cloud tier, the turns since the switch live in the

1888 + // cloud backend's history, not the engine's — carry them over so the

1889 + // on-device model picks the thread up where the cloud one left it. A

1890 + // local-to-local switch needs nothing: `switch_model_by_id` has already

1891 + // replayed the engine's own history past the reload.

1892 + let carried = {

1893 + let current = self.backend.lock().await;

1894 + if current.is_remote() {

1895 + backend::carryover_history(current.history_snapshot().await)

1896 + } else {

1897 + Vec::new()

1898 + }

1899 + };

1900 +

1901 let local_backend: Arc<dyn InferenceBackend> =

1902 Arc::new(LocalBackend::new(Arc::clone(&self.engine)));

1903 + backend::adopt_carryover(local_backend.as_ref(), carried).await;

1904 *self.backend.lock().await = local_backend;

1905 }

1906

@@ -2764,7 +2798,6 @@ async fn exec_slash_acp(

2798 Ok(new_config) => {

2799 agent.reset_to_local_backend().await;

2800 let _ = settings::set_local_inference(true);

2767 - agent.engine.clear_history().await;

2801 agent

2802 .send_assistant_message(

2803 cx,

@@ -2798,7 +2831,6 @@ async fn exec_slash_acp(

2831 let switched = agent.switch_model_by_id(&model.config.model_id).await?;

2832 agent.reset_to_local_backend().await;

2833 let _ = settings::set_local_inference(true);

2801 - agent.engine.clear_history().await;

2834

2835 agent

2836 .send_assistant_message(