@sigitcodereview / sigit / commits / 868119b

refactor(backend): read history messages that use content parts

A user message with an image has to use OpenAI's content-part array for `content`. Several places read `content` with `as_str()` and would treat such a message as empty. - `message_text` and `message_image_count` read either shape. - The summary transcript keeps the text and notes that an image was attached, without the base64. - `carryover_history` and the on-device history restore take the text through `message_text`, so they no longer see such a message as empty. - `estimate_tokens` counts an image as a flat 1,000 tokens. Counting its base64 as text would put a session with one screenshot over the compaction budget on every round. Nothing sends an image yet, so behaviour is unchanged for now. Refs #134 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Seto Elkahfi committed Oct 5, 2026 at 09:32 UTC 868119b169055600a8f547c12ae5a24fe8b2224f

1 file changed +118 -11

src/backend.rs

+118 -11

@@ -199,10 +199,15 @@ fn transcript_for_summary(history: &[serde_json::Value]) -> String {

199 }

200

201 let mut parts: Vec<String> = Vec::new();

202 - if let Some(text) = message["content"].as_str()

203 - && !text.trim().is_empty()

204 - {

205 - parts.push(text.to_string());

202 + let text = message_text(message);

203 + if !text.trim().is_empty() {

204 + parts.push(text);

205 + }

206 + // A summary cannot carry a picture, but it should say one was there.

207 + match message_image_count(message) {

208 + 0 => {}

209 + 1 => parts.push("[attached an image]".to_string()),

210 + count => parts.push(format!("[attached {count} images]")),

211 }

212 for call in message["tool_calls"].as_array().into_iter().flatten() {

213 parts.push(format!(

@@ -225,11 +230,54 @@ fn transcript_for_summary(history: &[serde_json::Value]) -> String {

230 /// Deliberately model-agnostic — it only needs to be in the right ballpark to

231 /// decide when compaction is worth an extra inference round.

232 pub fn estimate_tokens(history: &[serde_json::Value]) -> usize {

228 - let chars: usize = history

229 - .iter()

230 - .map(|message| message.to_string().chars().count())

231 - .sum();

232 - chars / 4

233 + let mut chars = 0;

234 + let mut images = 0;

235 + for message in history {

236 + let image_count = message_image_count(message);

237 + if image_count == 0 {

238 + chars += message.to_string().chars().count();

239 + } else {

240 + // An image travels as base64, megabytes of it, but a model bills

241 + // it by resolution. Counting the payload as text would put every

242 + // session with one screenshot permanently over budget.

243 + chars += message_text(message).chars().count();

244 + images += image_count;

245 + }

246 + }

247 + chars / 4 + images * IMAGE_TOKEN_ESTIMATE

248 +}

249 +

250 +/// What one attached image is assumed to cost. Providers land between a few

251 +/// hundred and a couple of thousand tokens depending on resolution; like the

252 +/// rest of [`estimate_tokens`] this only has to be in the right ballpark.

253 +const IMAGE_TOKEN_ESTIMATE: usize = 1_000;

254 +

255 +/// The text of a history message. `content` is a plain string for almost every

256 +/// message; a user message that carries an image uses OpenAI's content-part

257 +/// array instead, and its text parts are joined here.

258 +pub fn message_text(message: &serde_json::Value) -> String {

259 + match &message["content"] {

260 + serde_json::Value::String(text) => text.clone(),

261 + serde_json::Value::Array(parts) => parts

262 + .iter()

263 + .filter_map(|part| part["text"].as_str())

264 + .collect::<Vec<_>>()

265 + .join("\n"),

266 + _ => String::new(),

267 + }

268 +}

269 +

270 +/// How many images a history message carries as `image_url` content parts.

271 +pub fn message_image_count(message: &serde_json::Value) -> usize {

272 + message["content"]

273 + .as_array()

274 + .map(|parts| {

275 + parts

276 + .iter()

277 + .filter(|part| part["type"] == "image_url")

278 + .count()

279 + })

280 + .unwrap_or(0)

281 }

282

283 /// A sink for streaming assistant text deltas to the UI as they are produced.

@@ -341,7 +389,9 @@ impl LocalBackend {

389 self.engine.clear_history().await;

390 for entry in history {

391 let role = entry["role"].as_str().unwrap_or("");

344 - let content = entry["content"].as_str().unwrap_or("").to_string();

392 + // The on-device engine keeps text only, so an image attached

393 + // while a cloud model was active stays behind here.

394 + let content = message_text(&entry);

395 // Tool-call-only assistant entries and empty tool results carry no

396 // text a plain chat history can replay; drop them.

397 if content.is_empty() && role != "user" && role != "system" {

@@ -1695,7 +1745,7 @@ pub fn carryover_history(snapshot: Vec<serde_json::Value>) -> Vec<serde_json::Va

1745 if let Some(object) = message.as_object_mut() {

1746 object.remove("tool_calls");

1747 }

1698 - if message["content"].as_str().unwrap_or_default().is_empty() {

1748 + if message_text(&message).is_empty() {

1749 continue;

1750 }

1751 } else {

@@ -2052,6 +2102,63 @@ mod tests {

2102 assert_eq!(history[0]["content"], "new prompt");

2103 }

2104

2105 + fn user_with_image(text: &str, payload_chars: usize) -> serde_json::Value {

2106 + serde_json::json!({

2107 + "role": "user",

2108 + "content": [

2109 + { "type": "text", "text": text },

2110 + { "type": "image_url", "image_url": {

2111 + "url": format!("data:image/png;base64,{}", "A".repeat(payload_chars)),

2112 + }},

2113 + ],

2114 + })

2115 + }

2116 +

2117 + #[test]

2118 + fn message_text_reads_both_content_shapes() {

2119 + assert_eq!(

2120 + message_text(&serde_json::json!({ "role": "user", "content": "hello" })),

2121 + "hello"

2122 + );

2123 + let with_image = user_with_image("what is this?", 16);

2124 + assert_eq!(message_text(&with_image), "what is this?");

2125 + assert_eq!(message_image_count(&with_image), 1);

2126 + // A tool-call-only assistant message has no content at all.

2127 + let no_content = serde_json::json!({ "role": "assistant", "content": null });

2128 + assert_eq!(message_text(&no_content), "");

2129 + assert_eq!(message_image_count(&no_content), 0);

2130 + }

2131 +

2132 + #[test]

2133 + fn estimate_tokens_does_not_count_an_image_payload_as_text() {

2134 + // Two megabytes of base64 would read as half a million tokens.

2135 + let history = vec![user_with_image("what is this?", 2_000_000)];

2136 + let estimate = estimate_tokens(&history);

2137 + assert!(

2138 + (IMAGE_TOKEN_ESTIMATE..IMAGE_TOKEN_ESTIMATE + 100).contains(&estimate),

2139 + "{estimate}"

2140 + );

2141 + }

2142 +

2143 + #[test]

2144 + fn summary_transcript_keeps_the_text_of_a_message_with_an_image() {

2145 + let transcript = transcript_for_summary(&[user_with_image("what is this?", 64)]);

2146 + assert!(transcript.contains("what is this?"), "{transcript}");

2147 + assert!(transcript.contains("[attached an image]"), "{transcript}");

2148 + assert!(!transcript.contains("base64"), "{transcript}");

2149 + }

2150 +

2151 + #[test]

2152 + fn carryover_keeps_a_user_message_that_carries_an_image() {

2153 + let carried = carryover_history(vec![

2154 + serde_json::json!({ "role": "system", "content": "prompt" }),

2155 + user_with_image("what is this?", 16),

2156 + serde_json::json!({ "role": "assistant", "content": "a cat" }),

2157 + ]);

2158 + assert_eq!(carried.len(), 2);

2159 + assert_eq!(message_image_count(&carried[0]), 1);

2160 + }

2161 +

2162 #[test]

2163 fn estimate_tokens_scales_with_serialized_size() {

2164 assert_eq!(estimate_tokens(&[]), 0);