backend.rs · sigitcodereview/sigit

Code Commits Issues Pulls Cloud Agent CI/CD

main

claude/github-issue-40-0au5ua claude/linked-issue-acp-tui-u4vvpa claude/minor-release-cut-vkugow claude/sigit-acp-local-chat-cx6380 claude/sigit-cloud-agent-expansion-reox0a claude/tool-permission-system claude/zen-feynman-0u78dk development feature/acp-cancel-request feature/acp-session-list-cursor feature/acp-v1-conformance feature/agent-tools-multiedit-glob-todos-remember feature/cargo-workspace feature/hooks feature/load-local-model-explicitly feature/sigit-agent-profile feature/sigit-code-cloud feature/xcode-mcp-server fix/acp-replay-loaded-session-history fix/acp-replay-reasoning-and-attachments fix/dsml-doubled-bar-markers main release/v1.5.10 release/v1.5.6 release/v1.6.1

rs 3,536 lines 138 KB

Raw

1	//! Inference backend abstraction.
2	//!
3	//! The agent loop only needs to send a turn (optionally with tools) and return
4	//! tool results. This module defines that seam as the `InferenceBackend` trait
5	//! plus a few neutral types, with two implementations:
6	//!
7	//! - `LocalBackend` runs on-device through the `onde` crate (`ChatEngine`).
8	//! - `OpenAiBackend` talks to any OpenAI-compatible HTTP endpoint, configured by
9	//! `base_url`, `api_key`, and `model`.
10	//!
11	//! The trait exposes neither `onde` nor OpenAI types, so the loop does not depend
12	//! on a specific backend.
13	//!
14	//! The seam is consumed by both surfaces: the interactive client (`#[cfg(unix)]`,
15	//! see `run_interactive` in `main.rs` and `mod tui` in `chat.rs`) and the ACP
16	//! server's prompt loop. Some items are still reached only through the
17	//! Unix-only interactive paths, so the dead-code lint stays suppressed on
18	//! non-Unix targets only — Unix builds keep full coverage.
19	#![cfg_attr(not(unix), allow(dead_code))]
20
21	use std::sync::Arc;
22
23	use async_trait::async_trait;
24	use onde::inference::{ChatEngine, ChatMessage, ChatRole, EngineStatus, ToolDefinition};
25	use serde::Deserialize;
26	use tokio::sync::Mutex;
27
28	// ── Neutral types ───────────────────────────────────────────────────────────────
29
30	/// A tool the model may call, in a provider-neutral form. `parameters_schema` is
31	/// a JSON Schema encoded as a string (matching how siGit Code already declares
32	/// tools).
33	#[derive(Debug, Clone)]
34	pub struct ToolSpec {
35	pub name: String,
36	pub description: String,
37	pub parameters_schema: String,
38	}
39
40	/// A tool call requested by the model.
41	#[derive(Debug, Clone)]
42	pub struct ToolCall {
43	pub id: String,
44	pub name: String,
45	/// Arguments as a JSON-encoded string.
46	pub arguments: String,
47	}
48
49	/// The output of executing one tool call, fed back to the model.
50	#[derive(Debug, Clone)]
51	pub struct ToolResult {
52	pub tool_call_id: String,
53	pub content: String,
54	}
55
56	/// The result of one assistant turn: free text and/or tool calls.
57	#[derive(Debug, Clone, Default)]
58	pub struct TurnResult {
59	pub text: String,
60	pub tool_calls: Vec<ToolCall>,
61	/// Why the model stopped. Only the last round of a turn decides how the
62	/// turn is reported; a round that goes on to run tools is not the end.
63	pub finish: FinishReason,
64	}
65
66	/// Why the model stopped generating, reduced to what the agent loop acts on.
67	/// Endpoints report it as `finish_reason`; without it a reply cut off at the
68	/// token limit is indistinguishable from one the model finished.
69	#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
70	pub enum FinishReason {
71	/// The model finished on its own: `stop`, `tool_calls`, or no reason given.
72	#[default]
73	Complete,
74	/// The output ran into the token limit, so the reply is cut short.
75	Length,
76	/// The endpoint withheld the reply instead of answering.
77	ContentFilter,
78	/// The harness cut the reply off because it had degenerated into
79	/// repetition (see `repetition`).
80	Repetition,
81	}
82
83	impl FinishReason {
84	/// Map a wire `finish_reason`. OpenAI's spellings are `length` and
85	/// `content_filter`; gateways in front of other providers sometimes pass
86	/// the upstream's own word through, so those are accepted too.
87	fn from_wire(reason: Option<&str>) -> Self {
88	match reason {
89	Some("length" \| "max_tokens") => Self::Length,
90	Some("content_filter" \| "refusal") => Self::ContentFilter,
91	_ => Self::Complete,
92	}
93	}
94	}
95
96	/// Backend errors are plain strings. Callers map them to ACP errors.
97	pub type BackendError = String;
98
99	/// Rough context budget for a conversation, in estimated tokens (see
100	/// [`estimate_tokens`]). When a snapshot exceeds this, the agent loops compact
101	/// history before the next tool round.
102	pub const DEFAULT_CONTEXT_TOKEN_BUDGET: usize = 24_000;
103
104	/// How many trailing messages survive a compaction verbatim (the rest are
105	/// folded into the summary).
106	pub const COMPACT_KEEP_LAST: usize = 6;
107
108	/// The summarization request sent to the model when compacting history.
109	const SUMMARIZE_PROMPT: &str = "Summarize this coding session so far: decisions made, \
110	files touched, current state, open items. Be concise and factual.";
111
112	/// Largest transcript (in estimated tokens, see [`estimate_tokens`]) sent to
113	/// the model for summarization in one go.
114	///
115	/// Compaction runs because the conversation has outgrown the context budget,
116	/// so shipping the whole transcript verbatim asks the endpoint to process
117	/// exactly the payload that is already too big — on siGit Code Cloud that
118	/// request sits in inference until the gateway gives up (504), which is the
119	/// failure behind issue #125. Capping the transcript keeps the summarization
120	/// round inside the window the model actually has; the oldest and newest
121	/// exchanges carry the intent and the current state, while the truncated
122	/// middle is the part a summary can afford to lose.
123	const SUMMARY_TRANSCRIPT_TOKEN_CAP: usize = 12_000;
124
125	/// Trim `transcript` to about `cap` estimated tokens by cutting whole lines
126	/// from the middle, keeping the opening and closing exchanges.
127	///
128	/// A middle cut preserves the two ends that matter most to a summary: how the
129	/// session started (the original request) and where it stands now. Cutting by
130	/// whole lines keeps individual messages intact. A transcript already under
131	/// the cap is returned unchanged.
132	///
133	/// Line boundaries are best-effort. A head with no newline in its half is
134	/// dropped and the tail keeps its own half. A tail with no newline in its half
135	/// (a single huge message, or a giant final tool result) can't be cut on a
136	/// line, so the newest `cap` worth of text is kept instead and the head goes:
137	/// the newest exchanges are the ones the summary can least afford to lose.
138	fn truncate_transcript_middle(transcript: &str, cap: usize) -> String {
139	let budget = cap * 4; // estimate_tokens is chars / 4; invert it.
140	if transcript.len() <= budget {
141	return transcript.to_string();
142	}
143
144	let total = transcript.len();
145	// Each side gets half the budget, so the two together stay inside the cap.
146	// Walk each half to a line boundary. A head with no boundary in reach
147	// contributes nothing; a tail with none takes the hard cut below.
148	let half = budget / 2;
149	let head_end = {
150	let mut cut = half.min(total);
151	while cut < total && !transcript.is_char_boundary(cut) {
152	cut += 1;
153	}
154	cut = cut.min(total);
155	if transcript[..cut].contains('\n') {
156	transcript[..cut].rfind('\n').unwrap() + 1
157	} else {
158	0
159	}
160	};
161	let tail_start = {
162	let mut cut = total.saturating_sub(half);
163	while cut < total && !transcript.is_char_boundary(cut) {
164	cut += 1;
165	}
166	if transcript[cut..].contains('\n') {
167	transcript[cut..]
168	.find('\n')
169	.map(\|offset\| cut + offset + 1)
170	.unwrap()
171	} else {
172	total
173	}
174	};
175
176	if tail_start == total {
177	// The closing exchange has no line boundary in reach (a transcript of
178	// one huge message, or a giant newline-less tool result at the end):
179	// keep the newest budget. A head-only cut would invert the contract
180	// while the placeholder claims older messages were dropped.
181	let mut start = total.saturating_sub(budget);
182	while start > 0 && !transcript.is_char_boundary(start) {
183	start -= 1;
184	}
185	return format!(
186	"{}\n[…older messages omitted…]",
187	transcript[start..].trim_start()
188	);
189	}
190
191	if head_end >= tail_start {
192	// No line boundary in reach anywhere: keep the newest budget.
193	let mut start = total.saturating_sub(budget);
194	while start > 0 && !transcript.is_char_boundary(start) {
195	start -= 1;
196	}
197	return format!(
198	"{}\n[…older messages omitted…]",
199	transcript[start..].trim_start()
200	);
201	}
202
203	let omitted_chars = transcript[head_end..tail_start].chars().count();
204	format!(
205	"{}\n[…{} characters of older messages omitted…]\n{}",
206	transcript[..head_end].trim_end(),
207	omitted_chars,
208	transcript[tail_start..].trim_start(),
209	)
210	}
211
212	/// Render a history snapshot as a plain-text transcript, with tool calls and
213	/// tool results spelled out as prose rather than left in their wire shapes.
214	///
215	/// Compaction summarizes a conversation that is, by definition, thick with
216	/// `tool_calls` and `role: "tool"` messages — but the summarization round asks
217	/// for a plain answer and so sends no `tools` array. Forwarding the raw shapes
218	/// in that request produces tool blocks with no schema to validate against,
219	/// which strict endpoints reject outright: Anthropic answers 400, so every
220	/// compaction of a session that had ever run a tool failed, permanently, no
221	/// matter how small the history was. Flattening to text keeps everything the
222	/// summary actually needs and drops the shapes that only make sense alongside
223	/// a tool schema. It also sidesteps orphaned `tool_call_id`s and role-
224	/// alternation rules, neither of which a transcript can violate.
225	fn transcript_for_summary(history: &[serde_json::Value]) -> String {
226	let mut lines: Vec<String> = Vec::new();
227	for message in history {
228	let role = message["role"].as_str().unwrap_or("user");
229	// The system prompt is carried over verbatim, so it needn't be summarized.
230	if role == "system" {
231	continue;
232	}
233
234	let mut parts: Vec<String> = Vec::new();
235	let text = message_text(message);
236	if !text.trim().is_empty() {
237	parts.push(text);
238	}
239	// A summary cannot carry a picture, but it should say one was there.
240	match message_image_count(message) {
241	0 => {}
242	1 => parts.push("[attached an image]".to_string()),
243	count => parts.push(format!("[attached {count} images]")),
244	}
245	for call in message["tool_calls"].as_array().into_iter().flatten() {
246	parts.push(format!(
247	"called {}({})",
248	call["function"]["name"].as_str().unwrap_or("tool"),
249	call["function"]["arguments"].as_str().unwrap_or_default(),
250	));
251	}
252	if parts.is_empty() {
253	continue;
254	}
255
256	let label = if role == "tool" { "tool result" } else { role };
257	lines.push(format!("{label}: {}", parts.join("\n")));
258	}
259	lines.join("\n\n")
260	}
261
262	/// Crude token estimate for a history snapshot: serialized characters / 4.
263	/// Deliberately model-agnostic — it only needs to be in the right ballpark to
264	/// decide when compaction is worth an extra inference round.
265	pub fn estimate_tokens(history: &[serde_json::Value]) -> usize {
266	let mut chars = 0;
267	let mut images = 0;
268	for message in history {
269	let image_count = message_image_count(message);
270	if image_count == 0 {
271	chars += message.to_string().chars().count();
272	} else {
273	// An image travels as base64, megabytes of it, but a model bills
274	// it by resolution. Counting the payload as text would put every
275	// session with one screenshot permanently over budget.
276	chars += message_text(message).chars().count();
277	images += image_count;
278	}
279	}
280	chars / 4 + images * IMAGE_TOKEN_ESTIMATE
281	}
282
283	/// What one attached image is assumed to cost. Providers land between a few
284	/// hundred and a couple of thousand tokens depending on resolution; like the
285	/// rest of [`estimate_tokens`] this only has to be in the right ballpark.
286	const IMAGE_TOKEN_ESTIMATE: usize = 1_000;
287
288	/// The text of a history message. `content` is a plain string for almost every
289	/// message; a user message that carries an image uses OpenAI's content-part
290	/// array instead, and its text parts are joined here.
291	pub fn message_text(message: &serde_json::Value) -> String {
292	match &message["content"] {
293	serde_json::Value::String(text) => text.clone(),
294	serde_json::Value::Array(parts) => parts
295	.iter()
296	.filter_map(\|part\| part["text"].as_str())
297	.collect::<Vec<_>>()
298	.join("\n"),
299	_ => String::new(),
300	}
301	}
302
303	/// How many content parts of `part_type` a history message carries.
304	fn message_part_count(message: &serde_json::Value, part_type: &str) -> usize {
305	message["content"]
306	.as_array()
307	.map(\|parts\| {
308	parts
309	.iter()
310	.filter(\|part\| part["type"] == part_type)
311	.count()
312	})
313	.unwrap_or(0)
314	}
315
316	/// How many images a history message carries as `image_url` content parts.
317	pub fn message_image_count(message: &serde_json::Value) -> usize {
318	message_part_count(message, "image_url")
319	}
320
321	/// How many audio clips a history message carries as `input_audio` content
322	/// parts.
323	pub fn message_audio_count(message: &serde_json::Value) -> usize {
324	message_part_count(message, "input_audio")
325	}
326
327	/// One streamed fragment, tagged by whether it is visible prose or the model's
328	/// reasoning.
329	///
330	/// Reasoning reaches a backend two ways: inline `<think>…</think>` tags inside
331	/// the ordinary content stream (local models, and some OpenAI-compatible
332	/// endpoints), or a separate `reasoning_content` delta field (DeepSeek-style
333	/// endpoints). Inline tags can't be classified at the source — a tag may span
334	/// chunk boundaries — so they travel as [`TokenChunk::Visible`] and are split
335	/// out downstream by `StreamedReply`. The `reasoning_content` field is
336	/// unambiguous at the source, so the backend tags it [`TokenChunk::Reasoning`]
337	/// directly.
338	#[derive(Debug, Clone, PartialEq, Eq)]
339	pub enum TokenChunk {
340	/// Content-stream text. May still contain inline `<think>` tags that a
341	/// downstream consumer splits into reasoning and prose.
342	Visible(String),
343	/// Text from a dedicated reasoning channel, already known to be reasoning.
344	Reasoning(String),
345	}
346
347	/// A sink for streaming assistant fragments to the UI as they are produced.
348	///
349	/// When a caller passes `Some(sink)`, a streaming-capable backend forwards each
350	/// fragment through it as the model emits it, tagged [`TokenChunk::Visible`] or
351	/// [`TokenChunk::Reasoning`]; the returned [`TurnResult`] still carries the
352	/// fully assembled (visible) text and any tool calls. When the sink is `None`,
353	/// the backend runs in non-streaming mode. Unbounded so the inference task
354	/// never blocks on a slow consumer.
355	pub type TokenSink = tokio::sync::mpsc::UnboundedSender<TokenChunk>;
356
357	/// An image attached to a user message: base64 data and its media type, as an
358	/// ACP client sends it.
359	#[derive(Debug, Clone, PartialEq, Eq)]
360	pub struct ImageInput {
361	pub mime_type: String,
362	pub data: String,
363	}
364
365	/// An audio clip attached to a user message: base64 data and its media type,
366	/// as an ACP client sends it.
367	#[derive(Debug, Clone, PartialEq, Eq)]
368	pub struct AudioInput {
369	pub mime_type: String,
370	pub data: String,
371	}
372
373	impl AudioInput {
374	/// The `format` OpenAI's `input_audio` part expects (`wav`, `mp3`, ...),
375	/// from the MIME type an ACP client sends (`audio/wav`, `audio/mpeg`, ...).
376	fn format(&self) -> String {
377	let subtype = self
378	.mime_type
379	.split(';')
380	.next()
381	.unwrap_or_default()
382	.trim()
383	.to_lowercase();
384	let subtype = subtype.strip_prefix("audio/").unwrap_or(&subtype);
385	match subtype.strip_prefix("x-").unwrap_or(subtype) {
386	"mpeg" \| "mp3" => "mp3".to_string(),
387	"wave" \| "wav" \| "vnd.wave" => "wav".to_string(),
388	other => other.to_string(),
389	}
390	}
391	}
392
393	/// What a model that cannot read images is shown in place of one.
394	const IMAGE_OMITTED_NOTE: &str = "[image omitted: this model cannot read images]";
395
396	/// What a model that cannot take audio is shown in place of a clip.
397	const AUDIO_OMITTED_NOTE: &str = "[audio omitted: this model cannot take audio]";
398
399	/// A user message in history form. Plain text keeps the string `content` every
400	/// other message uses; with attachments it becomes OpenAI's content-part
401	/// array, text first, each image as a base64 `data:` URL and each audio clip
402	/// as an `input_audio` part.
403	fn user_message(text: &str, images: &[ImageInput], audio: &[AudioInput]) -> serde_json::Value {
404	if images.is_empty() && audio.is_empty() {
405	return serde_json::json!({ "role": "user", "content": text });
406	}
407	let mut parts = Vec::with_capacity(images.len() + audio.len() + 1);
408	if !text.is_empty() {
409	parts.push(serde_json::json!({ "type": "text", "text": text }));
410	}
411	for image in images {
412	parts.push(serde_json::json!({
413	"type": "image_url",
414	"image_url": {
415	"url": format!("data:{};base64,{}", image.mime_type, image.data),
416	},
417	}));
418	}
419	for clip in audio {
420	parts.push(serde_json::json!({
421	"type": "input_audio",
422	"input_audio": { "data": clip.data, "format": clip.format() },
423	}));
424	}
425	serde_json::json!({ "role": "user", "content": parts })
426	}
427
428	/// `history` with every content part of `part_type` replaced by `note`, for a
429	/// model that cannot take that kind of attachment. A message left with only
430	/// text is flattened back to a plain string.
431	///
432	/// History itself keeps the attachments. A thread can move to a model that
433	/// reads them (or back to one), so what was attached is not thrown away just
434	/// because the model active right now cannot use it.
435	fn without_parts(
436	history: &[serde_json::Value],
437	part_type: &str,
438	note: &str,
439	) -> Vec<serde_json::Value> {
440	history
441	.iter()
442	.map(\|message\| {
443	let Some(parts) = message["content"].as_array() else {
444	return message.clone();
445	};
446	if !parts.iter().any(\|part\| part["type"] == part_type) {
447	return message.clone();
448	}
449	let mut kept: Vec<serde_json::Value> = Vec::with_capacity(parts.len());
450	let mut notes = Vec::new();
451	for part in parts {
452	if part["type"] == part_type {
453	notes.push(serde_json::json!({ "type": "text", "text": note }));
454	} else {
455	kept.push(part.clone());
456	}
457	}
458	kept.extend(notes);
459	let mut stripped = message.clone();
460	stripped["content"] = if kept.iter().all(\|part\| part["type"] == "text") {
461	let text = kept
462	.iter()
463	.filter_map(\|part\| part["text"].as_str())
464	.filter(\|text\| !text.is_empty())
465	.collect::<Vec<_>>()
466	.join("\n");
467	serde_json::Value::String(text)
468	} else {
469	serde_json::Value::Array(kept)
470	};
471	stripped
472	})
473	.collect()
474	}
475
476	/// `history` as a model that cannot read images has to receive it, with a
477	/// note where each image was.
478	fn without_images(history: &[serde_json::Value]) -> Vec<serde_json::Value> {
479	without_parts(history, "image_url", IMAGE_OMITTED_NOTE)
480	}
481
482	/// `history` as a model that cannot take audio has to receive it, with a note
483	/// where each clip was.
484	fn without_audio(history: &[serde_json::Value]) -> Vec<serde_json::Value> {
485	without_parts(history, "input_audio", AUDIO_OMITTED_NOTE)
486	}
487
488	// ── The trait ───────────────────────────────────────────────────────────────────
489
490	/// A swappable inference backend driving siGit Code's agent loop.
491	#[async_trait]
492	pub trait InferenceBackend: Send + Sync {
493	/// Start an assistant turn from a new user message, offering `tools`.
494	///
495	/// If `sink` is `Some`, text is streamed through it as it is generated. A
496	/// backend may decline to stream a given round (for example, on-device
497	/// inference cannot stream while it is still deciding whether to call a
498	/// tool); in that case the text is delivered only via the returned result.
499	async fn send_message_with_tools(
500	&self,
501	text: &str,
502	tools: &[ToolSpec],
503	sink: Option<&TokenSink>,
504	) -> Result<TurnResult, BackendError>;
505
506	/// Continue the turn by returning tool results. `allow_tool_calls` controls
507	/// whether `tools` are offered again; the complete catalog remains available
508	/// so a disabled round can recognize and suppress tool-shaped model output.
509	/// `sink` streams assistant text when set.
510	async fn send_tool_results(
511	&self,
512	results: Vec<ToolResult>,
513	tools: &[ToolSpec],
514	allow_tool_calls: bool,
515	sink: Option<&TokenSink>,
516	) -> Result<TurnResult, BackendError>;
517
518	/// Record tool results in the conversation history without asking the
519	/// model to continue the turn. Used when a turn is abandoned mid-round
520	/// (the user cancelled at the permission gate): by then the assistant
521	/// message carrying the tool calls is already in history, and leaving them
522	/// unanswered makes strict OpenAI-compatible endpoints reject every later
523	/// request in the session.
524	async fn record_cancelled_tool_results(&self, results: Vec<ToolResult>);
525
526	/// Start a turn from a user message that carries images or audio.
527	///
528	/// The default drops the attachments and sends the text, which is right
529	/// for a backend that cannot read them. Callers check
530	/// [`Self::accepts_images`] and [`Self::accepts_audio`] first so the user
531	/// can be told, instead of the attachment vanishing.
532	async fn send_message_with_attachments(
533	&self,
534	text: &str,
535	images: &[ImageInput],
536	audio: &[AudioInput],
537	tools: &[ToolSpec],
538	sink: Option<&TokenSink>,
539	) -> Result<TurnResult, BackendError> {
540	let _ = (images, audio);
541	self.send_message_with_tools(text, tools, sink).await
542	}
543
544	/// Whether the model behind this backend reads images. On-device models do
545	/// not; a remote one answers from its model id.
546	fn accepts_images(&self) -> bool {
547	false
548	}
549
550	/// Whether the model behind this backend takes audio. On-device models do
551	/// not; a remote one answers from its model id.
552	fn accepts_audio(&self) -> bool {
553	false
554	}
555
556	/// Whether inference runs over the network (a configured provider) rather
557	/// than on-device. Drives UI labelling so the displayed model can't claim a
558	/// local model while requests actually go to the cloud.
559	fn is_remote(&self) -> bool;
560
561	/// A backend for the same endpoint and model with an empty conversation,
562	/// or `None` when the conversation cannot be separated from the backend
563	/// (on-device, the engine holds it). An editor keeps several threads open
564	/// in one process; this is what lets each of them own its history instead
565	/// of taking turns on a shared one.
566	fn fresh(&self) -> Option<Arc<dyn InferenceBackend>> {
567	None
568	}
569
570	/// A serializable snapshot of the conversation history, one JSON object per
571	/// message (`{"role": ..., "content": ...}` at minimum). The snapshot is
572	/// what the session store persists; it includes any seeded system message
573	/// so [`InferenceBackend::restore_history`] can replace state wholesale.
574	async fn history_snapshot(&self) -> Vec<serde_json::Value>;
575
576	/// Replace the conversation history with a previously saved snapshot.
577	/// Backends that cannot represent every entry (e.g. on-device history has
578	/// no tool-call structure) flatten what they can and drop the rest.
579	async fn restore_history(&self, history: Vec<serde_json::Value>);
580
581	/// Shrink the conversation history: summarize everything so far with one
582	/// extra (non-streaming) inference round, then rebuild history as
583	/// `[system message, summary, last keep_last non-system messages]`. When
584	/// the summarization round itself fails, a backend should fall back to a
585	/// deterministic shrink (dropping the oldest messages) rather than leave
586	/// an over-budget session stuck; an `Err` here means even that fallback
587	/// could not fit the history in the budget.
588	async fn compact_history(&self, keep_last: usize) -> Result<(), BackendError>;
589	}
590
591	// ── Local backend (onde ChatEngine) ──────────────────────────────────────────────
592
593	/// On-device inference. A thin adapter over `onde::ChatEngine`.
594	pub struct LocalBackend {
595	engine: Arc<ChatEngine>,
596	/// History restored before an on-device model is loaded. Onde ignores
597	/// `push_history` while unloaded, so keep the snapshot here until the
598	/// engine can accept it instead of silently dropping a loaded session.
599	pending_history: Mutex<Option<Vec<serde_json::Value>>>,
600	}
601
602	impl LocalBackend {
603	pub fn new(engine: Arc<ChatEngine>) -> Self {
604	Self {
605	engine,
606	pending_history: Mutex::new(None),
607	}
608	}
609
610	async fn apply_pending_history(&self) -> Result<(), BackendError> {
611	if self.pending_history.lock().await.is_none() {
612	return Ok(());
613	}
614	let status = self.engine.info().await.status;
615	if status != EngineStatus::Ready {
616	return Err(format!(
617	"on-device model is not ready to restore session history (status: {status})"
618	));
619	}
620	let Some(history) = self.pending_history.lock().await.take() else {
621	return Ok(());
622	};
623
624	self.engine.clear_history().await;
625	for entry in history {
626	let role = entry["role"].as_str().unwrap_or("");
627	// The on-device engine keeps text only, so an image attached
628	// while a cloud model was active stays behind here.
629	let content = message_text(&entry);
630	// Tool-call-only assistant entries and empty tool results carry no
631	// text a plain chat history can replay; drop them.
632	if content.is_empty() && role != "user" && role != "system" {
633	continue;
634	}
635	let message = match role {
636	"system" => ChatMessage::system(content),
637	"user" => ChatMessage::user(content),
638	"assistant" => ChatMessage::assistant(content),
639	// Tool results flatten to plain text (MVP; acceptable loss).
640	"tool" => ChatMessage::user(format!("[tool result]\n{content}")),
641	_ => continue,
642	};
643	self.engine.push_history(message).await;
644	}
645	Ok(())
646	}
647	}
648
649	fn to_onde_tools(tools: &[ToolSpec]) -> Vec<ToolDefinition> {
650	tools
651	.iter()
652	.map(\|tool\| ToolDefinition {
653	name: tool.name.clone(),
654	description: tool.description.clone(),
655	parameters_schema: tool.parameters_schema.clone(),
656	})
657	.collect()
658	}
659
660	#[async_trait]
661	impl InferenceBackend for LocalBackend {
662	async fn send_message_with_tools(
663	&self,
664	text: &str,
665	tools: &[ToolSpec],
666	sink: Option<&TokenSink>,
667	) -> Result<TurnResult, BackendError> {
668	self.apply_pending_history().await?;
669	// onde's tool-aware path is non-streaming: it has to buffer the whole
670	// reply to detect tool calls. We can only stream when no tools are on
671	// offer (a plain answer), which is exactly the tools-disabled case.
672	if let Some(sink) = sink
673	&& tools.is_empty()
674	{
675	let rx = self
676	.engine
677	.stream_message(text)
678	.await
679	.map_err(\|error\| error.to_string())?;
680	return drain_onde_stream(rx, sink).await;
681	}
682
683	let onde_tools = to_onde_tools(tools);
684	let result = self
685	.engine
686	.send_message_with_tools(text, &onde_tools)
687	.await
688	.map_err(\|error\| error.to_string())?;
689	Ok(onde_result_to_turn(result))
690	}
691
692	async fn send_tool_results(
693	&self,
694	results: Vec<ToolResult>,
695	tools: &[ToolSpec],
696	allow_tool_calls: bool,
697	sink: Option<&TokenSink>,
698	) -> Result<TurnResult, BackendError> {
699	self.apply_pending_history().await?;
700	let onde_results: Vec<onde::inference::ToolResult> = results
701	.into_iter()
702	.map(\|result\| onde::inference::ToolResult {
703	tool_call_id: result.tool_call_id,
704	content: result.content,
705	})
706	.collect();
707
708	// A forced-text round is the only round onde can stream, since no
709	// further tool calls are parsed.
710	if let Some(sink) = sink
711	&& !allow_tool_calls
712	{
713	let rx = self
714	.engine
715	.stream_tool_results(onde_results, None)
716	.await
717	.map_err(\|error\| error.to_string())?;
718	return drain_onde_stream(rx, sink).await;
719	}
720
721	let onde_tools = allow_tool_calls.then(\|\| to_onde_tools(tools));
722	let result = self
723	.engine
724	.send_tool_results(onde_results, onde_tools.as_deref())
725	.await
726	.map_err(\|error\| error.to_string())?;
727	Ok(onde_result_to_turn(result))
728	}
729
730	async fn record_cancelled_tool_results(&self, _results: Vec<ToolResult>) {
731	// onde's public API cannot append tool-result history entries without
732	// running another inference round, so the dangling tool call stays in
733	// its history. The chat template replays it as-is, which local models
734	// tolerate — worst case the model re-issues the call next turn.
735	}
736
737	fn is_remote(&self) -> bool {
738	false
739	}
740
741	async fn history_snapshot(&self) -> Vec<serde_json::Value> {
742	if let Some(history) = self.pending_history.lock().await.as_ref() {
743	return history.clone();
744	}
745	// onde's `history()` already flattens tool entries: assistant tool
746	// calls become plain assistant text and tool results are omitted, so
747	// the snapshot is lossy for tool-heavy turns (acceptable in this MVP).
748	self.engine
749	.history()
750	.await
751	.iter()
752	.map(\|message\| {
753	serde_json::json!({
754	"role": message.role.to_string(),
755	"content": message.content,
756	})
757	})
758	.collect()
759	}
760
761	async fn restore_history(&self, history: Vec<serde_json::Value>) {
762	*self.pending_history.lock().await = Some(history);
763	if self.engine.info().await.status == EngineStatus::Ready {
764	// Ready was just observed, so failure here can only mean a status
765	// transition; keep the pending copy for the next inference call.
766	let _ = self.apply_pending_history().await;
767	}
768	}
769
770	async fn compact_history(&self, keep_last: usize) -> Result<(), BackendError> {
771	self.apply_pending_history().await?;
772	let snapshot = self.engine.history().await;
773
774	// Leading system messages carry session context; keep them all.
775	let system: Vec<ChatMessage> = snapshot
776	.iter()
777	.take_while(\|message\| message.role == ChatRole::System)
778	.cloned()
779	.collect();
780	let non_system: Vec<ChatMessage> = snapshot
781	.into_iter()
782	.filter(\|message\| message.role != ChatRole::System)
783	.collect();
784	let tail_start = non_system.len().saturating_sub(keep_last);
785	let tail: Vec<ChatMessage> = non_system[tail_start..].to_vec();
786
787	// One plain (tool-free) inference round produces the summary. Note the
788	// cap only bounds this request's own payload: the engine still replays
789	// its full (over-budget) history underneath it, so an already
790	// over-budget local session usually fails here and lands in the
791	// fallback below — the cap is about not making the doomed round worse.
792	let transcript = truncate_transcript_middle(
793	&chat_messages_transcript(&non_system),
794	SUMMARY_TRANSCRIPT_TOKEN_CAP,
795	);
796	let result = self
797	.engine
798	.send_message(format!("{transcript}\n\n{SUMMARIZE_PROMPT}"))
799	.await
800	.map_err(\|error\| error.to_string());
801
802	let rebuild: Vec<ChatMessage> = match result {
803	Ok(result) => {
804	// Local models may reason in <think> blocks; keep only the visible part.
805	let (_think, summary) = crate::chat::strip_think_blocks(&result.text);
806	let mut rebuilt = system;
807	rebuilt.push(ChatMessage::user(format!(
808	"[Conversation summary]\n{summary}"
809	)));
810	rebuilt.extend(tail);
811	rebuilt
812	}
813	Err(error) => {
814	log::warn!(
815	"local summarization round failed ({error}); falling back to plain truncation"
816	);
817	// Same deterministic shrink as the remote path, on the plain
818	// chat shapes the engine holds (its history has no tool-call
819	// structure, so there are no orphaned tool results to guard).
820	match truncate_chat_messages_to_budget(system, tail) {
821	Some(rebuilt) => rebuilt,
822	// The system messages alone overflow the window: unrecoverable.
823	None => return Err(error),
824	}
825	}
826	};
827
828	self.engine.clear_history().await;
829	for message in rebuild {
830	self.engine.push_history(message).await;
831	}
832	Ok(())
833	}
834	}
835
836	/// The local counterpart of `truncate_history_to_budget`: system messages, the
837	/// truncation placeholder, and the newest of `tail` that fit the budget.
838	///
839	/// The tail may drain to empty. A newest message that overflows the window on
840	/// its own (a giant tool result, the very thing that triggers compaction) is
841	/// dropped rather than failing the session, so `None` only means the system
842	/// messages alone don't fit.
843	fn truncate_chat_messages_to_budget(
844	system: Vec<ChatMessage>,
845	mut tail: Vec<ChatMessage>,
846	) -> Option<Vec<ChatMessage>> {
847	let budget = DEFAULT_CONTEXT_TOKEN_BUDGET * 4; // estimate_tokens is chars / 4.
848	let fixed_chars: usize = system
849	.iter()
850	.map(\|message\| message.content.len())
851	.sum::<usize>()
852	+ TRUNCATION_PLACEHOLDER.len();
853	if fixed_chars > budget {
854	return None;
855	}
856
857	let mut tail_chars: usize = tail.iter().map(\|message\| message.content.len()).sum();
858	let mut drop = 0;
859	while fixed_chars + tail_chars > budget {
860	tail_chars -= tail[drop].content.len();
861	drop += 1;
862	}
863	tail.drain(..drop);
864
865	let mut rebuilt = system;
866	rebuilt.push(ChatMessage::user(TRUNCATION_PLACEHOLDER.to_string()));
867	rebuilt.extend(tail);
868	Some(rebuilt)
869	}
870
871	/// Flatten `onde` chat messages into the transcript shape summarization
872	/// expects (the local engine's history has no tool-call structure, so this is
873	/// a plain role/content rendering).
874	fn chat_messages_transcript(history: &[ChatMessage]) -> String {
875	history
876	.iter()
877	.filter(\|message\| message.role != ChatRole::System && !message.content.trim().is_empty())
878	.map(\|message\| format!("{}: {}", message.role, message.content))
879	.collect::<Vec<_>>()
880	.join("\n\n")
881	}
882
883	/// Drain an onde streaming receiver, forwarding each token to `sink` and
884	/// assembling the full text. onde reports stream failures as a final chunk whose
885	/// `finish_reason` is `"error: …"`; surface those as a backend error.
886	async fn drain_onde_stream(
887	mut rx: tokio::sync::mpsc::Receiver<onde::inference::StreamChunk>,
888	sink: &TokenSink,
889	) -> Result<TurnResult, BackendError> {
890	let mut text = String::new();
891	let mut finish = FinishReason::default();
892	while let Some(chunk) = rx.recv().await {
893	if !chunk.delta.is_empty() {
894	text.push_str(&chunk.delta);
895	// The receiver is the UI; if it's gone the turn is being cancelled,
896	// so stop assembling rather than spinning the model to completion.
897	// onde reports reasoning as inline `<think>` tags in `delta`, so it
898	// is visible content that a downstream consumer splits out.
899	if sink.send(TokenChunk::Visible(chunk.delta)).is_err() {
900	break;
901	}
902	}
903	if chunk.done {
904	if let Some(reason) = chunk.finish_reason.as_deref()
905	&& let Some(message) = reason.strip_prefix("error: ")
906	{
907	return Err(message.to_string());
908	}
909	finish = FinishReason::from_wire(chunk.finish_reason.as_deref());
910	break;
911	}
912	}
913	Ok(TurnResult {
914	text,
915	tool_calls: Vec::new(),
916	finish,
917	})
918	}
919
920	/// Convert an `onde` tool-aware result into the neutral [`TurnResult`].
921	fn onde_result_to_turn(result: onde::inference::ToolAwareResult) -> TurnResult {
922	TurnResult {
923	finish: FinishReason::from_wire(Some(&result.finish_reason)),
924	text: result.text,
925	tool_calls: result
926	.tool_calls
927	.into_iter()
928	.map(\|call\| ToolCall {
929	id: call.id,
930	name: call.function_name,
931	arguments: call.arguments,
932	})
933	.collect(),
934	}
935	}
936
937	// ── OpenAI-compatible backend ─────────────────────────────────────────────────────
938
939	/// Inference against any OpenAI-compatible Chat Completions endpoint.
940	///
941	/// Conversation state is held client-side and replayed on every request, so the
942	/// endpoint can be stateless. Standard OpenAI function-calling is used end to
943	/// end (`tools`, `choices[].message.tool_calls`, `role: "tool"` follow-ups).
944	pub struct OpenAiBackend {
945	base_url: String,
946	api_key: String,
947	model: String,
948	http: reqwest::Client,
949	/// The full message list sent on each request (system + turns + tool results).
950	history: Mutex<Vec<serde_json::Value>>,
951	/// Whether `model` reads images (see `provider::model_accepts_images`).
952	/// When it does not, images already in `history` are left out of each
953	/// request rather than sent to an endpoint that would refuse them.
954	accepts_images: bool,
955	/// Whether `model` takes audio (see `provider::model_accepts_audio`), with
956	/// the same effect on audio already in `history`.
957	accepts_audio: bool,
958	}
959
960	impl OpenAiBackend {
961	/// Build a backend for `{base_url, api_key, model}`, seeding the optional
962	/// system prompt. `base_url` should include the API root (e.g. ending in
963	/// `/v1`); the chat path is appended.
964	pub fn new(
965	base_url: impl Into<String>,
966	api_key: impl Into<String>,
967	model: impl Into<String>,
968	system_prompt: Option<String>,
969	) -> Self {
970	let mut history = Vec::new();
971	if let Some(prompt) = system_prompt {
972	history.push(serde_json::json!({ "role": "system", "content": prompt }));
973	}
974	let model = model.into();
975	Self {
976	base_url: base_url.into(),
977	api_key: api_key.into(),
978	accepts_images: crate::provider::model_accepts_images(&model),
979	accepts_audio: crate::provider::model_accepts_audio(&model),
980	model,
981	http: reqwest::Client::new(),
982	history: Mutex::new(history),
983	}
984	}
985
986	fn tools_json(tools: &[ToolSpec]) -> Vec<serde_json::Value> {
987	tools
988	.iter()
989	.map(\|tool\| {
990	// parameters_schema is a JSON string; parse it, defaulting to an
991	// empty object schema if malformed.
992	let parameters: serde_json::Value = serde_json::from_str(&tool.parameters_schema)
993	.unwrap_or_else(\|_\| serde_json::json!({ "type": "object", "properties": {} }));
994	serde_json::json!({
995	"type": "function",
996	"function": {
997	"name": tool.name,
998	"description": tool.description,
999	"parameters": parameters,
1000	}
1001	})
1002	})
1003	.collect()
1004	}
1005
1006	/// POST the current history and apply the assistant reply to history.
1007	/// `tools` is always the known catalog, while `allow_tool_calls` determines
1008	/// whether it is advertised to the model and whether returned calls may run.
1009	/// Streams via SSE when `sink` is set; otherwise reads a single JSON response.
1010	///
1011	/// A reply whose only attempt at a tool call was a block that couldn't be
1012	/// parsed gets one retry: the model is told the call didn't run and asked
1013	/// to make it again through the structured interface. Without that the
1014	/// turn ends on whatever prose preceded the broken block, and the call is
1015	/// silently lost.
1016	async fn complete(
1017	&self,
1018	tools: &[ToolSpec],
1019	allow_tool_calls: bool,
1020	sink: Option<&TokenSink>,
1021	) -> Result<TurnResult, BackendError> {
1022	let (result, malformed) = self.request(tools, allow_tool_calls, sink).await?;
1023	let retry_note = if !allow_tool_calls \|\| !result.tool_calls.is_empty() {
1024	None
1025	} else if malformed > 0 {
1026	log::warn!(
1027	"dropped {malformed} unparseable inline tool call(s); asking the model to retry"
1028	);
1029	Some(MALFORMED_TOOL_CALL_RETRY)
1030	} else if result.finish == FinishReason::Repetition {
1031	log::warn!("reply was cut off as a repetition loop; asking the model to retry");
1032	Some(REPETITION_RETRY)
1033	} else {
1034	None
1035	};
1036	let Some(retry_note) = retry_note else {
1037	return Ok(result);
1038	};
1039
1040	self.history.lock().await.push(serde_json::json!({
1041	"role": "user",
1042	"content": retry_note,
1043	}));
1044	if let Some(sink) = sink
1045	&& !result.text.trim().is_empty()
1046	{
1047	// The retry's text follows what already streamed; keep it from
1048	// running on into the previous sentence.
1049	let _ = sink.send(TokenChunk::Visible("\n\n".to_string()));
1050	}
1051	let (retry, retry_malformed) = self.request(tools, allow_tool_calls, sink).await?;
1052	if retry_malformed > 0 && retry.tool_calls.is_empty() {
1053	log::warn!(
1054	"the retry also wrote {retry_malformed} unparseable inline tool call(s); \
1055	ending the turn without them"
1056	);
1057	}
1058	Ok(TurnResult {
1059	text: join_reply_text(&result.text, &retry.text),
1060	tool_calls: retry.tool_calls,
1061	finish: retry.finish,
1062	})
1063	}
1064
1065	/// One chat-completion round trip. Returns the turn plus how many inline
1066	/// tool-call blocks were dropped because they couldn't be parsed.
1067	async fn request(
1068	&self,
1069	tools: &[ToolSpec],
1070	allow_tool_calls: bool,
1071	sink: Option<&TokenSink>,
1072	) -> Result<(TurnResult, usize), BackendError> {
1073	let url = format!("{}/chat/completions", self.base_url.trim_end_matches('/'));
1074	let streaming = sink.is_some();
1075
1076	let messages = {
1077	let history = self.history.lock().await;
1078	let messages = if self.accepts_images {
1079	history.clone()
1080	} else {
1081	without_images(&history)
1082	};
1083	if self.accepts_audio {
1084	messages
1085	} else {
1086	without_audio(&messages)
1087	}
1088	};
1089	let mut body = serde_json::json!({
1090	"model": self.model,
1091	"messages": messages,
1092	"stream": streaming,
1093	});
1094	if allow_tool_calls && !tools.is_empty() {
1095	body["tools"] = serde_json::Value::Array(Self::tools_json(tools));
1096	// OpenAI specifies `auto` as the default when tools are present,
1097	// but not every OpenAI-compatible gateway implements that default.
1098	// Sending it explicitly is important for the cloud path: otherwise
1099	// a model can describe the action it intends to take as text and
1100	// end the turn without returning a tool call at all.
1101	body["tool_choice"] = serde_json::Value::String("auto".to_string());
1102	}
1103
1104	let response = self
1105	.http
1106	.post(&url)
1107	.bearer_auth(&self.api_key)
1108	.json(&body)
1109	.send()
1110	.await
1111	.map_err(\|error\| format!("request to {url} failed: {error}"))?;
1112
1113	if !response.status().is_success() {
1114	let status = response.status();
1115	let body = response.text().await.unwrap_or_default();
1116	return Err(describe_api_error(status, &body));
1117	}
1118
1119	if let Some(sink) = sink {
1120	self.consume_stream(response, sink, tools, allow_tool_calls)
1121	.await
1122	} else {
1123	self.consume_json(response, tools, allow_tool_calls).await
1124	}
1125	}
1126
1127	/// One self-contained summarization round: summarize `transcript` without
1128	/// touching the live history.
1129	///
1130	/// Unlike [`complete`], which replays and mutates `self.history`, this
1131	/// sends its own one-off message list. Compaction only rewrites history
1132	/// once it has a summary in hand, so a failed round leaves the session
1133	/// exactly as it was — and the caller can fall back to plain truncation
1134	/// instead of failing the session (issue #125).
1135	async fn summarize(
1136	&self,
1137	system: &[serde_json::Value],
1138	transcript: &str,
1139	) -> Result<String, BackendError> {
1140	let url = format!("{}/chat/completions", self.base_url.trim_end_matches('/'));
1141	let mut messages = Vec::with_capacity(system.len() + 1);
1142	messages.extend(system.iter().cloned());
1143	messages.push(serde_json::json!({
1144	"role": "user",
1145	"content": format!("{transcript}\n\n{SUMMARIZE_PROMPT}"),
1146	}));
1147	let body = serde_json::json!({
1148	"model": self.model,
1149	"messages": messages,
1150	"stream": false,
1151	});
1152
1153	let response = self
1154	.http
1155	.post(&url)
1156	.bearer_auth(&self.api_key)
1157	.json(&body)
1158	.send()
1159	.await
1160	.map_err(\|error\| format!("request to {url} failed: {error}"))?;
1161
1162	if !response.status().is_success() {
1163	let status = response.status();
1164	let body = response.text().await.unwrap_or_default();
1165	return Err(describe_api_error(status, &body));
1166	}
1167
1168	let parsed: ChatCompletion = response
1169	.json()
1170	.await
1171	.map_err(\|error\| format!("response parse error: {error}"))?;
1172	parsed
1173	.choices
1174	.into_iter()
1175	.next()
1176	.and_then(\|choice\| choice.message.content)
1177	.filter(\|text\| !text.trim().is_empty())
1178	.ok_or_else(\|\| "endpoint returned no summary".to_string())
1179	}
1180
1181	/// Parse a single non-streaming chat-completion response.
1182	async fn consume_json(
1183	&self,
1184	response: reqwest::Response,
1185	tools: &[ToolSpec],
1186	allow_tool_calls: bool,
1187	) -> Result<(TurnResult, usize), BackendError> {
1188	let parsed: ChatCompletion = response
1189	.json()
1190	.await
1191	.map_err(\|error\| format!("response parse error: {error}"))?;
1192
1193	let choice = parsed
1194	.choices
1195	.into_iter()
1196	.next()
1197	.ok_or_else(\|\| "endpoint returned no choices".to_string())?;
1198	let finish = FinishReason::from_wire(choice.finish_reason.as_deref());
1199	let mut message = choice.message;
1200
1201	let text = message.content.clone().unwrap_or_default();
1202	let mut tool_calls: Vec<ToolCall> = message
1203	.tool_calls
1204	.iter()
1205	.flatten()
1206	.map(\|call\| ToolCall {
1207	id: call.id.clone(),
1208	name: call.function.name.clone(),
1209	arguments: call.function.arguments.clone(),
1210	})
1211	.collect();
1212
1213	let mut extracted = crate::inline_tool_calls::extract(&text, tools);
1214	// Markup the model made up in the harness's shape goes too, from the
1215	// reply and the history alike. See `harness_markup`.
1216	extracted.text = crate::harness_markup::strip(&extracted.text);
1217	let malformed = extracted.malformed;
1218	if malformed > 0 {
1219	log::warn!("dropped {malformed} unparseable inline tool call(s) from the reply");
1220	}
1221
1222	if !allow_tool_calls {
1223	let recovered = extracted.calls;
1224	let text = extracted.text;
1225	let suppressed = tool_calls.len() + recovered.len();
1226	if suppressed > 0 {
1227	let names = tool_calls
1228	.iter()
1229	.map(\|call\| call.name.as_str())
1230	.chain(recovered.iter().map(\|call\| call.name.as_str()))
1231	.collect::<Vec<_>>()
1232	.join(", ");
1233	log::warn!(
1234	"suppressed {suppressed} tool call(s) from a forced-text response: {names}"
1235	);
1236	}
1237	self.history
1238	.lock()
1239	.await
1240	.push(streamed_assistant_history(&text, &[]));
1241	return Ok((
1242	TurnResult {
1243	text,
1244	tool_calls: Vec::new(),
1245	finish,
1246	},
1247	malformed,
1248	));
1249	}
1250
1251	// Some models write a tool call out as literal `<tool_call>` text
1252	// instead of using the structured field (see `inline_tool_calls`).
1253	// Recover it, or the turn ends with the tag rendered as prose and
1254	// whatever the model meant to do is dropped.
1255	if tool_calls.is_empty() && (malformed > 0 \|\| !extracted.calls.is_empty()) {
1256	let cleaned = extracted.text;
1257	let recovered = extracted.calls;
1258	if !recovered.is_empty() {
1259	log::warn!(
1260	"recovered {} tool call(s) the model emitted as text instead of a structured call",
1261	recovered.len()
1262	);
1263	let tool_calls: Vec<ToolCall> = recovered
1264	.into_iter()
1265	.enumerate()
1266	.map(\|(index, call)\| ToolCall {
1267	id: format!("call_recovered_{index}"),
1268	name: call.name,
1269	arguments: call.arguments,
1270	})
1271	.collect();
1272	// Record the recovered shape, not the raw tag: the tool
1273	// results that follow have to answer an assistant message
1274	// that actually carries these calls, or the next request is
1275	// rejected for orphaned tool results.
1276	self.history
1277	.lock()
1278	.await
1279	.push(streamed_assistant_history(&cleaned, &tool_calls));
1280	return Ok((
1281	TurnResult {
1282	text: cleaned,
1283	tool_calls,
1284	finish,
1285	},
1286	malformed,
1287	));
1288	}
1289	// Nothing recovered, so what was found is a broken block. Keep it
1290	// out of history as well as the reply; see `inline_tool_calls` for
1291	// why it poisons later turns.
1292	self.push_reply_without_calls(&cleaned).await;
1293	return Ok((
1294	TurnResult {
1295	text: cleaned,
1296	tool_calls,
1297	finish,
1298	},
1299	malformed,
1300	));
1301	} else if malformed > 0 \|\| !extracted.calls.is_empty() {
1302	// Structured calls arrived with inline blocks beside them. As on
1303	// the streaming path, inline calls that parse run alongside the
1304	// structured ones, broken blocks are dropped, and neither kind of
1305	// block is left in the reply or the history.
1306	let offset = tool_calls.len();
1307	tool_calls.extend(
1308	extracted
1309	.calls
1310	.into_iter()
1311	.enumerate()
1312	.map(\|(index, call)\| ToolCall {
1313	id: format!("call_recovered_{}", offset + index),
1314	name: call.name,
1315	arguments: call.arguments,
1316	}),
1317	);
1318	self.history
1319	.lock()
1320	.await
1321	.push(streamed_assistant_history(&extracted.text, &tool_calls));
1322	return Ok((
1323	TurnResult {
1324	text: extracted.text,
1325	tool_calls,
1326	finish,
1327	},
1328	malformed,
1329	));
1330	}
1331
1332	// Record the assistant turn so later tool results have context. No
1333	// inline block was found, so the only change `extracted.text` can
1334	// carry is the stripped markup.
1335	let text = extracted.text;
1336	if message.content.is_some() {
1337	message.content = Some(text.clone());
1338	}
1339	self.history.lock().await.push(message.into_history_value());
1340
1341	Ok((
1342	TurnResult {
1343	text,
1344	tool_calls,
1345	finish,
1346	},
1347	0,
1348	))
1349	}
1350
1351	/// Record a text-only assistant reply. An empty one is skipped: strict
1352	/// endpoints reject an assistant message with neither content nor tool
1353	/// calls, and there is nothing in it worth replaying.
1354	async fn push_reply_without_calls(&self, text: &str) {
1355	if !text.is_empty() {
1356	self.history
1357	.lock()
1358	.await
1359	.push(streamed_assistant_history(text, &[]));
1360	}
1361	}
1362
1363	/// Consume an OpenAI Server-Sent Events stream, forwarding content deltas to
1364	/// `sink` and reassembling any tool calls (which arrive fragmented across
1365	/// chunks, keyed by `index`).
1366	async fn consume_stream(
1367	&self,
1368	response: reqwest::Response,
1369	sink: &TokenSink,
1370	tools: &[ToolSpec],
1371	allow_tool_calls: bool,
1372	) -> Result<(TurnResult, usize), BackendError> {
1373	use futures::StreamExt;
1374
1375	let mut stream = response.bytes_stream();
1376	// Newlines are ASCII, so splitting raw bytes on `\n` never bisects a
1377	// multibyte UTF-8 sequence; we only lossily decode whole lines.
1378	let mut buffer: Vec<u8> = Vec::new();
1379	let mut text = String::new();
1380	let mut tool_accum: Vec<StreamingToolCall> = Vec::new();
1381	let mut done = false;
1382	// Arrives on a late chunk, usually one with an empty delta.
1383	let mut finish = FinishReason::default();
1384	// Recovers a tool call the model wrote as literal `<tool_call>` text
1385	// instead of a structured delta. Scanning here (rather than after the
1386	// stream) keeps the tag off the UI: content goes straight to `sink` as
1387	// it arrives, so by the time a whole turn is assembled the tag has
1388	// already been rendered. See `inline_tool_calls`.
1389	let mut scanner = crate::inline_tool_calls::StreamScanner::new(tools);
1390	// What it lets through then loses any markup the model made up in the
1391	// harness's shape, like a `<system_warning>` block. See
1392	// `harness_markup`.
1393	let mut markup = crate::harness_markup::MarkupFilter::new();
1394	let mut recovered: Vec<ToolCall> = Vec::new();
1395	let mut malformed = 0usize;
1396	// Stops a reply that falls into a repetition loop. See `repetition`.
1397	let mut guard = crate::repetition::RepetitionGuard::new();
1398	let mut degenerate = false;
1399
1400	while let Some(item) = stream.next().await {
1401	let bytes = item.map_err(\|error\| format!("stream read error: {error}"))?;
1402	buffer.extend_from_slice(&bytes);
1403
1404	while let Some(pos) = buffer.iter().position(\|&b\| b == b'\n') {
1405	let line: Vec<u8> = buffer.drain(..=pos).collect();
1406	let line = String::from_utf8_lossy(&line);
1407	let line = line.trim();
1408
1409	let Some(data) = line.strip_prefix("data:") else {
1410	continue;
1411	};
1412	let data = data.trim();
1413	if data == "[DONE]" {
1414	done = true;
1415	break;
1416	}
1417	if data.is_empty() {
1418	continue;
1419	}
1420
1421	// An endpoint that fails after the status line is already sent
1422	// reports it in the stream instead, as an ordinary `data:`
1423	// frame holding an error envelope. siGit Code Cloud does this
1424	// when the upstream fails mid-turn. It has no `choices`, so
1425	// without this it parses as an empty chunk and is skipped, and
1426	// the turn ends looking like the model simply said nothing.
1427	if let Some(message) = api_error_message(data) {
1428	return Err(message);
1429	}
1430
1431	let chunk: StreamCompletion = match serde_json::from_str(data) {
1432	Ok(chunk) => chunk,
1433	// Skip keep-alive comments and anything we can't parse rather
1434	// than aborting a turn over one malformed frame.
1435	Err(_) => continue,
1436	};
1437
1438	let Some(choice) = chunk.choices.into_iter().next() else {
1439	continue;
1440	};
1441	if let Some(reason) = choice.finish_reason.as_deref() {
1442	finish = FinishReason::from_wire(Some(reason));
1443	}
1444	// Reasoning streamed in its own field is tagged at the source;
1445	// it bypasses the inline-tag scanner and is never kept in the
1446	// assistant message text.
1447	if let Some(reasoning) = choice.delta.reasoning_content
1448	&& !reasoning.is_empty()
1449	&& sink.send(TokenChunk::Reasoning(reasoning)).is_err()
1450	{
1451	// Consumer dropped (turn cancelled).
1452	done = true;
1453	break;
1454	}
1455	if let Some(content) = choice.delta.content
1456	&& !content.is_empty()
1457	{
1458	let mut cancelled = false;
1459	for event in scanner.push(&content) {
1460	match event {
1461	crate::inline_tool_calls::ScanEvent::Text(chunk) => {
1462	let chunk = markup.push(&chunk);
1463	if chunk.is_empty() {
1464	continue;
1465	}
1466	text.push_str(&chunk);
1467	let looping = guard.push(&chunk);
1468	if sink.send(TokenChunk::Visible(chunk)).is_err() {
1469	// Consumer dropped (turn cancelled).
1470	cancelled = true;
1471	break;
1472	}
1473	if looping {
1474	degenerate = true;
1475	break;
1476	}
1477	}
1478	crate::inline_tool_calls::ScanEvent::ToolCall(call) => {
1479	if allow_tool_calls {
1480	log::warn!(
1481	"recovered tool call '{}' the model emitted as text instead of a structured call",
1482	call.name
1483	);
1484	}
1485	recovered.push(ToolCall {
1486	id: format!("call_recovered_{}", recovered.len()),
1487	name: call.name,
1488	arguments: call.arguments,
1489	});
1490	}
1491	crate::inline_tool_calls::ScanEvent::Malformed(block) => {
1492	log_malformed_block(&block);
1493	malformed += 1;
1494	}
1495	}
1496	}
1497	if cancelled \|\| degenerate {
1498	done = true;
1499	break;
1500	}
1501	}
1502	for delta in choice.delta.tool_calls.into_iter().flatten() {
1503	let index = delta.index.unwrap_or(0) as usize;
1504	if tool_accum.len() <= index {
1505	tool_accum.resize_with(index + 1, StreamingToolCall::default);
1506	}
1507	let slot = &mut tool_accum[index];
1508	if let Some(id) = delta.id {
1509	slot.id = id;
1510	}
1511	if let Some(function) = delta.function {
1512	if let Some(name) = function.name {
1513	slot.name = name;
1514	}
1515	if let Some(arguments) = function.arguments {
1516	slot.arguments.push_str(&arguments);
1517	}
1518	}
1519	}
1520	}
1521
1522	if done {
1523	break;
1524	}
1525	}
1526
1527	// Flush what was held back. A partial marker is just text; a
1528	// tool-call block that never closed is dropped like any other
1529	// unparseable one.
1530	let mut leftover = String::new();
1531	if degenerate {
1532	// Dropping the stream below closes the connection, so the endpoint
1533	// stops generating. Nothing held back is worth flushing.
1534	let keep = guard.loop_start().unwrap_or(text.len()).min(text.len());
1535	log::warn!(
1536	"reply degenerated into repetition; cut {} of {} bytes",
1537	text.len() - keep,
1538	text.len()
1539	);
1540	text.truncate(keep);
1541	text.truncate(text.trim_end().len());
1542	finish = FinishReason::Repetition;
1543	let _ = sink.send(TokenChunk::Visible(REPETITION_NOTICE.to_string()));
1544	} else {
1545	match scanner.finish() {
1546	Some(crate::inline_tool_calls::ScanEvent::Text(rest)) => {
1547	leftover = markup.push(&rest);
1548	}
1549	Some(crate::inline_tool_calls::ScanEvent::Malformed(block)) => {
1550	log_malformed_block(&block);
1551	malformed += 1;
1552	}
1553	Some(crate::inline_tool_calls::ScanEvent::ToolCall(_)) \| None => {}
1554	}
1555	leftover.push_str(&markup.finish());
1556	if !leftover.is_empty() {
1557	text.push_str(&leftover);
1558	let _ = sink.send(TokenChunk::Visible(leftover));
1559	}
1560	}
1561
1562	let mut tool_calls: Vec<ToolCall> = tool_accum
1563	.iter()
1564	.filter(\|call\| !call.name.is_empty())
1565	.enumerate()
1566	.map(\|(index, call)\| ToolCall {
1567	id: if call.id.is_empty() {
1568	format!("call_{index}")
1569	} else {
1570	call.id.clone()
1571	},
1572	name: call.name.clone(),
1573	arguments: call.arguments.clone(),
1574	})
1575	.collect();
1576	tool_calls.extend(recovered);
1577
1578	if !allow_tool_calls && !tool_calls.is_empty() {
1579	let names = tool_calls
1580	.iter()
1581	.map(\|call\| call.name.as_str())
1582	.collect::<Vec<_>>()
1583	.join(", ");
1584	log::warn!(
1585	"suppressed {} tool call(s) from a forced-text response: {names}",
1586	tool_calls.len()
1587	);
1588	tool_calls.clear();
1589	}
1590
1591	// Record the assistant turn so later tool results have context.
1592	if (malformed > 0 \|\| degenerate) && tool_calls.is_empty() {
1593	self.push_reply_without_calls(&text).await;
1594	} else {
1595	self.history
1596	.lock()
1597	.await
1598	.push(streamed_assistant_history(&text, &tool_calls));
1599	}
1600
1601	Ok((
1602	TurnResult {
1603	text,
1604	tool_calls,
1605	finish,
1606	},
1607	malformed,
1608	))
1609	}
1610	}
1611
1612	/// Sent as a user turn after a reply whose tool call couldn't be parsed.
1613	const MALFORMED_TOOL_CALL_RETRY: &str = "[siGit Code] Your last reply tried to call a tool by \
1614	writing the call out as text, and it could not be parsed, so nothing ran and the user did \
1615	not see it. Do not write tool calls or tool results as text. Make the call again using the \
1616	tool-calling interface, or answer in plain prose if no tool is needed.";
1617
1618	/// Sent as a user turn after a reply was cut off as a repetition loop.
1619	const REPETITION_RETRY: &str = "[siGit Code] Your last reply started repeating the same few \
1620	words and was cut off. Do not continue it. Make the next tool call you need, or answer \
1621	briefly in plain prose.";
1622
1623	/// Shown to the user, who has already watched the loop stream. Not kept in
1624	/// history.
1625	const REPETITION_NOTICE: &str =
1626	"\n\n[siGit Code stopped this reply because the model started repeating itself.]";
1627
1628	fn log_malformed_block(block: &str) {
1629	log::warn!(
1630	"dropped an unparseable inline tool call ({} chars): {}",
1631	block.len(),
1632	block.chars().take(200).collect::<String>()
1633	);
1634	}
1635
1636	/// Join the visible text of a reply and its retry the way the stream showed
1637	/// them: as separate paragraphs.
1638	fn join_reply_text(first: &str, retry: &str) -> String {
1639	match (first.trim().is_empty(), retry.trim().is_empty()) {
1640	(true, _) => retry.to_string(),
1641	(false, true) => first.to_string(),
1642	(false, false) => format!("{first}\n\n{retry}"),
1643	}
1644	}
1645
1646	/// One tool call being reassembled from streamed deltas.
1647	#[derive(Default)]
1648	struct StreamingToolCall {
1649	id: String,
1650	name: String,
1651	arguments: String,
1652	}
1653
1654	/// Rebuild the assistant message for replay in history after a streamed turn,
1655	/// preserving any tool calls so the follow-up request is well-formed. Mirrors
1656	/// [`ResponseMessage::into_history_value`] for the non-streaming path.
1657	fn streamed_assistant_history(text: &str, tool_calls: &[ToolCall]) -> serde_json::Value {
1658	let mut message = serde_json::json!({ "role": "assistant" });
1659	message["content"] = if text.is_empty() {
1660	serde_json::Value::Null
1661	} else {
1662	serde_json::Value::String(text.to_string())
1663	};
1664	if !tool_calls.is_empty() {
1665	message["tool_calls"] = serde_json::json!(
1666	tool_calls
1667	.iter()
1668	.map(\|call\| serde_json::json!({
1669	"id": call.id,
1670	"type": "function",
1671	"function": {
1672	"name": call.name,
1673	"arguments": call.arguments,
1674	}
1675	}))
1676	.collect::<Vec<_>>()
1677	);
1678	}
1679	message
1680	}
1681
1682	#[async_trait]
1683	impl InferenceBackend for OpenAiBackend {
1684	async fn send_message_with_tools(
1685	&self,
1686	text: &str,
1687	tools: &[ToolSpec],
1688	sink: Option<&TokenSink>,
1689	) -> Result<TurnResult, BackendError> {
1690	self.history
1691	.lock()
1692	.await
1693	.push(serde_json::json!({ "role": "user", "content": text }));
1694	self.complete(tools, true, sink).await
1695	}
1696
1697	async fn send_tool_results(
1698	&self,
1699	results: Vec<ToolResult>,
1700	tools: &[ToolSpec],
1701	allow_tool_calls: bool,
1702	sink: Option<&TokenSink>,
1703	) -> Result<TurnResult, BackendError> {
1704	{
1705	let mut history = self.history.lock().await;
1706	for result in results {
1707	history.push(serde_json::json!({
1708	"role": "tool",
1709	"tool_call_id": result.tool_call_id,
1710	"content": result.content,
1711	}));
1712	}
1713	}
1714	self.complete(tools, allow_tool_calls, sink).await
1715	}
1716
1717	async fn record_cancelled_tool_results(&self, results: Vec<ToolResult>) {
1718	let mut history = self.history.lock().await;
1719	for result in results {
1720	// Cancellation races the in-flight continuation request. That
1721	// request appends its tool results before awaiting HTTP, so cleanup
1722	// must be safe whether cancellation won just before or just after
1723	// the append.
1724	let matching_call = history.iter().rposition(\|message\| {
1725	message["role"] == "assistant"
1726	&& message["tool_calls"].as_array().is_some_and(\|calls\| {
1727	calls.iter().any(\|call\| call["id"] == result.tool_call_id)
1728	})
1729	});
1730	let already_recorded = matching_call.is_some_and(\|call_index\| {
1731	history[call_index + 1..].iter().any(\|message\| {
1732	message["role"] == "tool" && message["tool_call_id"] == result.tool_call_id
1733	})
1734	});
1735	if already_recorded {
1736	continue;
1737	}
1738	history.push(serde_json::json!({
1739	"role": "tool",
1740	"tool_call_id": result.tool_call_id,
1741	"content": result.content,
1742	}));
1743	}
1744	}
1745
1746	async fn send_message_with_attachments(
1747	&self,
1748	text: &str,
1749	images: &[ImageInput],
1750	audio: &[AudioInput],
1751	tools: &[ToolSpec],
1752	sink: Option<&TokenSink>,
1753	) -> Result<TurnResult, BackendError> {
1754	self.history
1755	.lock()
1756	.await
1757	.push(user_message(text, images, audio));
1758	self.complete(tools, true, sink).await
1759	}
1760
1761	fn accepts_images(&self) -> bool {
1762	self.accepts_images
1763	}
1764
1765	fn accepts_audio(&self) -> bool {
1766	self.accepts_audio
1767	}
1768
1769	fn is_remote(&self) -> bool {
1770	true
1771	}
1772
1773	fn fresh(&self) -> Option<Arc<dyn InferenceBackend>> {
1774	Some(Arc::new(Self {
1775	base_url: self.base_url.clone(),
1776	api_key: self.api_key.clone(),
1777	model: self.model.clone(),
1778	// The client is a handle on one connection pool; sharing it keeps
1779	// connections to the endpoint warm across sessions.
1780	http: self.http.clone(),
1781	history: Mutex::new(Vec::new()),
1782	accepts_images: self.accepts_images,
1783	accepts_audio: self.accepts_audio,
1784	}))
1785	}
1786
1787	async fn history_snapshot(&self) -> Vec<serde_json::Value> {
1788	self.history.lock().await.clone()
1789	}
1790
1791	async fn restore_history(&self, history: Vec<serde_json::Value>) {
1792	// The snapshot includes the seeded system message, so a wholesale
1793	// replacement restores exactly what was saved.
1794	*self.history.lock().await = history;
1795	}
1796
1797	async fn compact_history(&self, keep_last: usize) -> Result<(), BackendError> {
1798	let snapshot: Vec<serde_json::Value> = self.history.lock().await.clone();
1799
1800	// Leading system messages carry session context (the seeded prompt,
1801	// pushed context); keep them all, like the summarization-success path.
1802	let system: Vec<serde_json::Value> = snapshot
1803	.iter()
1804	.take_while(\|message\| message["role"] == "system")
1805	.cloned()
1806	.collect();
1807
1808	let non_system: Vec<serde_json::Value> = snapshot
1809	.iter()
1810	.filter(\|message\| message["role"] != "system")
1811	.cloned()
1812	.collect();
1813	let tail_start = non_system.len().saturating_sub(keep_last);
1814	let mut tail = non_system[tail_start..].to_vec();
1815	// Drop leading tool results whose assistant tool-call message was
1816	// summarized away — strict endpoints reject orphaned `role: "tool"`
1817	// entries on the very next request.
1818	while tail
1819	.first()
1820	.is_some_and(\|message\| message["role"] == "tool")
1821	{
1822	tail.remove(0);
1823	}
1824
1825	// Ask the endpoint for a summary of the conversation so far, through a
1826	// direct completion call. The request carries the conversation as a
1827	// flattened transcript in a single user message rather than the live
1828	// history: this round offers no tools, and a tool-shaped history sent
1829	// without a tool schema is rejected upstream (see
1830	// `transcript_for_summary`). The transcript is capped so the request
1831	// cannot be as large as the over-budget conversation that triggered
1832	// compaction (see `SUMMARY_TRANSCRIPT_TOKEN_CAP`).
1833	let transcript = truncate_transcript_middle(
1834	&transcript_for_summary(&snapshot),
1835	SUMMARY_TRANSCRIPT_TOKEN_CAP,
1836	);
1837	let summary = match self.summarize(&system, &transcript).await {
1838	Ok(summary) => summary,
1839	Err(error) => {
1840	log::warn!(
1841	"summarization round failed ({error}); falling back to plain truncation"
1842	);
1843	// Compaction must not fail the session: fold the overflow into
1844	// a placeholder and drop the oldest messages until the rebuild
1845	// fits the budget. Needs no endpoint, so it cannot fail the way
1846	// the inference round just did (issue #125).
1847	let rebuilt = truncate_history_to_budget(&system, tail.clone()).ok_or(error)?;
1848	*self.history.lock().await = rebuilt;
1849	return Ok(());
1850	}
1851	};
1852
1853	let mut rebuilt = Vec::with_capacity(system.len() + tail.len() + 1);
1854	rebuilt.extend(system);
1855	rebuilt.push(serde_json::json!({
1856	"role": "user",
1857	"content": format!("[Conversation summary]\n{summary}"),
1858	}));
1859	rebuilt.extend(tail);
1860	*self.history.lock().await = rebuilt;
1861	Ok(())
1862	}
1863	}
1864
1865	/// The placeholder message replacing dropped history when compaction falls
1866	/// back to plain truncation (the model reads it, so it explains the gap).
1867	const TRUNCATION_PLACEHOLDER: &str = "[Older conversation omitted: the session outgrew \
1868	the context window and the summarization service was unavailable. Earlier \
1869	messages were dropped, so answers may lack that context.]";
1870
1871	/// Last-resort compaction when the summarization round cannot run: rebuild the
1872	/// history as `[system, placeholder, newest messages that fit the budget]`.
1873	///
1874	/// Returns `None` when even the smallest rebuild exceeds the budget (a
1875	/// pathological case — e.g. one message larger than the whole window); the
1876	/// caller then reports the original summarization error and leaves history
1877	/// untouched.
1878	fn truncate_history_to_budget(
1879	system: &[serde_json::Value],
1880	mut tail: Vec<serde_json::Value>,
1881	) -> Option<Vec<serde_json::Value>> {
1882	/// A `role: "tool"` entry only makes sense behind the assistant message
1883	/// that requested it; dropping that message strands it.
1884	fn drop_orphaned_tool_results(tail: &mut Vec<serde_json::Value>) {
1885	while tail
1886	.first()
1887	.is_some_and(\|message\| message["role"] == "tool")
1888	{
1889	tail.remove(0);
1890	}
1891	}
1892
1893	// A caller splitting mid-round can hand in a tail that starts with tool
1894	// results whose call message already fell outside the kept window.
1895	drop_orphaned_tool_results(&mut tail);
1896
1897	loop {
1898	let mut candidate = Vec::with_capacity(tail.len() + system.len() + 1);
1899	candidate.extend(system.iter().cloned());
1900	candidate.push(serde_json::json!({
1901	"role": "user",
1902	"content": TRUNCATION_PLACEHOLDER,
1903	}));
1904	candidate.extend(tail.iter().cloned());
1905	if estimate_tokens(&candidate) <= DEFAULT_CONTEXT_TOKEN_BUDGET {
1906	return Some(candidate);
1907	}
1908
1909	// Nothing left to drop: even the minimal rebuild (system + placeholder,
1910	// no conversation) overflows — the session is genuinely unrecoverable.
1911	if tail.is_empty() {
1912	return None;
1913	}
1914	if tail[0]["role"] == "assistant"
1915	&& tail[0]["tool_calls"]
1916	.as_array()
1917	.is_some_and(\|calls\| !calls.is_empty())
1918	&& tail.get(1).is_some_and(\|message\| message["role"] == "tool")
1919	{
1920	// Dropping an assistant tool-call message would orphan its
1921	// results; drop the whole round (call + results) together.
1922	let mut end = 1;
1923	while tail
1924	.get(end)
1925	.is_some_and(\|message\| message["role"] == "tool")
1926	{
1927	end += 1;
1928	}
1929	tail.drain(..end);
1930	} else {
1931	tail.remove(0);
1932	drop_orphaned_tool_results(&mut tail);
1933	}
1934	}
1935	}
1936
1937	// ── OpenAI error shape ────────────────────────────────────────────────────────
1938
1939	/// The OpenAI error envelope: `{"error": {"message": ..., "type": ...}}`.
1940	///
1941	/// Every endpoint siGit talks to speaks it, including siGit Code Cloud, whose
1942	/// messages are written for the person reading them ("Monthly siGit Code Cloud
1943	/// allowance reached..."). Worth unwrapping rather than pasting the raw body
1944	/// into the editor's error banner.
1945	#[derive(Debug, Deserialize)]
1946	struct ApiErrorEnvelope {
1947	error: ApiErrorBody,
1948	}
1949
1950	#[derive(Debug, Deserialize)]
1951	struct ApiErrorBody {
1952	#[serde(default)]
1953	message: Option<String>,
1954	}
1955
1956	/// How much of an unparseable error body to keep. An endpoint behind a proxy
1957	/// can answer with a full HTML page, and the whole thing ends up in the
1958	/// editor's error banner.
1959	const ERROR_BODY_LIMIT: usize = 500;
1960
1961	/// Turn an error response into something worth showing a person.
1962	///
1963	/// The endpoint's own message wins when there is one: it is written for the
1964	/// user, and the status code repeats what it already says. Anything else falls
1965	/// back to the status plus whatever the body held, which is all there is to go
1966	/// on — except an HTML page, which is a proxy or Rails error page and tells
1967	/// the person nothing the status doesn't (pasting it into chat is the dump
1968	/// seen in issue #125's screenshots).
1969	fn describe_api_error(status: reqwest::StatusCode, body: &str) -> String {
1970	if let Some(message) = api_error_message(body) {
1971	return message;
1972	}
1973
1974	let body = body.trim();
1975	if body.is_empty() \|\| is_probable_html(body) {
1976	return format!("endpoint returned {status}");
1977	}
1978
1979	let mut detail = body;
1980	if detail.len() > ERROR_BODY_LIMIT {
1981	let mut cut = ERROR_BODY_LIMIT;
1982	while !detail.is_char_boundary(cut) {
1983	cut -= 1;
1984	}
1985	detail = &detail[..cut];
1986	return format!("endpoint returned {status}: {detail}…");
1987	}
1988	format!("endpoint returned {status}: {detail}")
1989	}
1990
1991	/// The human-readable message out of an OpenAI error envelope, if the body is
1992	/// one and carries a non-empty message.
1993	fn api_error_message(body: &str) -> Option<String> {
1994	let envelope: ApiErrorEnvelope = serde_json::from_str(body).ok()?;
1995	let message = envelope.error.message?;
1996	let message = message.trim();
1997	(!message.is_empty()).then(\|\| message.to_string())
1998	}
1999
2000	/// Whether `body` looks like an HTML page rather than prose: a doctype or an
2001	/// opening tag in the first chunk. Gateways and Rails apps answer failures
2002	/// with full pages; none of that markup is worth showing.
2003	fn is_probable_html(body: &str) -> bool {
2004	let head: String = body.chars().take(512).collect();
2005	let head = head.trim_start().to_ascii_lowercase();
2006	head.starts_with("<!doctype") \|\| head.starts_with("<html") \|\| head.starts_with("<head")
2007	}
2008
2009	// ── OpenAI response shapes ────────────────────────────────────────────────────────
2010
2011	#[derive(Debug, Deserialize)]
2012	struct ChatCompletion {
2013	#[serde(default)]
2014	choices: Vec<CompletionChoice>,
2015	}
2016
2017	#[derive(Debug, Deserialize)]
2018	struct CompletionChoice {
2019	message: ResponseMessage,
2020	#[serde(default)]
2021	finish_reason: Option<String>,
2022	}
2023
2024	#[derive(Debug, Deserialize)]
2025	struct ResponseMessage {
2026	#[serde(default)]
2027	content: Option<String>,
2028	#[serde(default)]
2029	tool_calls: Option<Vec<ResponseToolCall>>,
2030	}
2031
2032	impl ResponseMessage {
2033	/// Reconstruct the assistant message for replay in history, preserving any
2034	/// tool calls so the follow-up request is well-formed.
2035	fn into_history_value(self) -> serde_json::Value {
2036	let mut message = serde_json::json!({ "role": "assistant" });
2037	message["content"] = match self.content {
2038	Some(text) => serde_json::Value::String(text),
2039	None => serde_json::Value::Null,
2040	};
2041	if let Some(tool_calls) = self.tool_calls {
2042	message["tool_calls"] = serde_json::json!(
2043	tool_calls
2044	.into_iter()
2045	.map(\|call\| serde_json::json!({
2046	"id": call.id,
2047	"type": "function",
2048	"function": {
2049	"name": call.function.name,
2050	"arguments": call.function.arguments,
2051	}
2052	}))
2053	.collect::<Vec<_>>()
2054	);
2055	}
2056	message
2057	}
2058	}
2059
2060	#[derive(Debug, Deserialize)]
2061	struct ResponseToolCall {
2062	id: String,
2063	function: ResponseFunction,
2064	}
2065
2066	#[derive(Debug, Deserialize)]
2067	struct ResponseFunction {
2068	name: String,
2069	#[serde(default)]
2070	arguments: String,
2071	}
2072
2073	// ── OpenAI streaming (SSE) chunk shapes ─────────────────────────────────────────
2074
2075	#[derive(Debug, Deserialize)]
2076	struct StreamCompletion {
2077	#[serde(default)]
2078	choices: Vec<StreamChoice>,
2079	}
2080
2081	#[derive(Debug, Deserialize)]
2082	struct StreamChoice {
2083	#[serde(default)]
2084	delta: StreamDelta,
2085	#[serde(default)]
2086	finish_reason: Option<String>,
2087	}
2088
2089	#[derive(Debug, Default, Deserialize)]
2090	struct StreamDelta {
2091	#[serde(default)]
2092	content: Option<String>,
2093	// Some endpoints stream model reasoning in a dedicated field rather than
2094	// inline `<think>` tags. It is unambiguously reasoning at the source, so it
2095	// travels as `TokenChunk::Reasoning` and never enters the content scanner.
2096	#[serde(default, alias = "reasoning")]
2097	reasoning_content: Option<String>,
2098	#[serde(default)]
2099	tool_calls: Option<Vec<StreamToolCallDelta>>,
2100	}
2101
2102	#[derive(Debug, Deserialize)]
2103	struct StreamToolCallDelta {
2104	#[serde(default)]
2105	index: Option<u32>,
2106	#[serde(default)]
2107	id: Option<String>,
2108	#[serde(default)]
2109	function: Option<StreamFunctionDelta>,
2110	}
2111
2112	#[derive(Debug, Deserialize)]
2113	struct StreamFunctionDelta {
2114	#[serde(default)]
2115	name: Option<String>,
2116	#[serde(default)]
2117	arguments: Option<String>,
2118	}
2119
2120	// ── Carrying a conversation across a model switch ────────────────────────────
2121
2122	/// The part of a history snapshot that survives a model switch: every
2123	/// non-system message, with half-finished tool plumbing repaired.
2124	///
2125	/// System messages are dropped because the backend being switched to seeds
2126	/// its own — a different model's prompt, or a freshly pushed session-context
2127	/// message. A switch can land mid-turn, after the assistant asked for a tool
2128	/// but before the results came back; a tool call with no matching result (or a
2129	/// result with no matching call) makes strict OpenAI-compatible endpoints
2130	/// reject every later request in the session, so those halves are stripped
2131	/// instead of carried.
2132	pub fn carryover_history(snapshot: Vec<serde_json::Value>) -> Vec<serde_json::Value> {
2133	let answered: std::collections::HashSet<String> = snapshot
2134	.iter()
2135	.filter(\|message\| message["role"] == "tool")
2136	.filter_map(\|message\| message["tool_call_id"].as_str().map(str::to_string))
2137	.collect();
2138
2139	let mut carried: Vec<serde_json::Value> = Vec::with_capacity(snapshot.len());
2140	let mut called: std::collections::HashSet<String> = std::collections::HashSet::new();
2141
2142	for mut message in snapshot {
2143	match message["role"].as_str().unwrap_or_default() {
2144	"system" => continue,
2145	"assistant" => {
2146	if let Some(calls) = message["tool_calls"].as_array() {
2147	let kept: Vec<serde_json::Value> = calls
2148	.iter()
2149	.filter(\|call\| call["id"].as_str().is_some_and(\|id\| answered.contains(id)))
2150	.cloned()
2151	.collect();
2152	if kept.is_empty() {
2153	// Only unanswered requests: keep whatever text came
2154	// with them, drop the message if there was none.
2155	if let Some(object) = message.as_object_mut() {
2156	object.remove("tool_calls");
2157	}
2158	if message_text(&message).is_empty() {
2159	continue;
2160	}
2161	} else {
2162	for call in &kept {
2163	called.insert(call["id"].as_str().unwrap_or_default().to_string());
2164	}
2165	message["tool_calls"] = serde_json::Value::Array(kept);
2166	}
2167	}
2168	carried.push(message);
2169	}
2170	"tool" => {
2171	let answers_a_kept_call = message["tool_call_id"]
2172	.as_str()
2173	.is_some_and(\|id\| called.contains(id));
2174	if answers_a_kept_call {
2175	carried.push(message);
2176	}
2177	}
2178	_ => carried.push(message),
2179	}
2180	}
2181
2182	carried
2183	}
2184
2185	/// Take a cancelled prompt's user message back out of the live history.
2186	///
2187	/// The client drops a cancelled prompt from its thread, so the model must not
2188	/// keep answering it. Only the first inference round can cancel with the user
2189	/// message as the last entry; later rounds end on tool or assistant output.
2190	/// A prompt that fails with an error is not taken back: the client keeps it,
2191	/// so the history keeps it too and a model switch carries it over.
2192	pub async fn forget_trailing_user_message(backend: &dyn InferenceBackend) {
2193	let mut history = backend.history_snapshot().await;
2194	if history
2195	.last()
2196	.is_some_and(\|message\| message["role"] == "user")
2197	{
2198	history.pop();
2199	backend.restore_history(history).await;
2200	}
2201	}
2202
2203	/// Replay `carried` (from [`carryover_history`]) into `backend`, on top of the
2204	/// system messages `backend` seeded for itself. Used when a model switch
2205	/// installs a new backend — or reloads the on-device engine, which wipes its
2206	/// history — so the thread continues under the new model instead of restarting.
2207	pub async fn adopt_carryover(backend: &dyn InferenceBackend, carried: Vec<serde_json::Value>) {
2208	if carried.is_empty() {
2209	return;
2210	}
2211	let mut rebuilt: Vec<serde_json::Value> = backend
2212	.history_snapshot()
2213	.await
2214	.into_iter()
2215	.take_while(\|message\| message["role"] == "system")
2216	.collect();
2217	rebuilt.extend(carried);
2218	backend.restore_history(rebuilt).await;
2219	}
2220
2221	#[cfg(test)]
2222	mod tests {
2223	use super::*;
2224
2225	/// The case from the issue: the endpoint says something a person can act
2226	/// on, and the status code and JSON around it are noise.
2227	#[test]
2228	fn an_endpoints_own_message_is_what_the_user_sees() {
2229	let body = r#"{"error":{"message":"Monthly siGit Code Cloud allowance reached. It resets at the start of your next billing period.","type":"server_error"}}"#;
2230
2231	let described = describe_api_error(reqwest::StatusCode::TOO_MANY_REQUESTS, body);
2232
2233	assert_eq!(
2234	described,
2235	"Monthly siGit Code Cloud allowance reached. \
2236	It resets at the start of your next billing period."
2237	);
2238	}
2239
2240	#[test]
2241	fn a_body_that_is_not_an_error_envelope_keeps_the_status() {
2242	let described = describe_api_error(reqwest::StatusCode::BAD_GATEWAY, "upstream said no");
2243
2244	assert!(described.contains("502"), "{described}");
2245	assert!(described.contains("upstream said no"), "{described}");
2246	}
2247
2248	/// The issue-#125 screenshots: a gateway's HTML error page pasted into
2249	/// chat tells the person nothing the status doesn't already say.
2250	#[test]
2251	fn an_html_error_page_is_not_pasted_into_the_message() {
2252	let page = "<!doctype html>\n<html lang=\"en\">\n<head>\n\
2253	<title>We're sorry, but something went wrong (500 Internal Server Error)</title>\n\
2254	</head>\n<body>…</body>\n</html>";
2255
2256	let described = describe_api_error(reqwest::StatusCode::GATEWAY_TIMEOUT, page);
2257
2258	assert_eq!(described, "endpoint returned 504 Gateway Timeout");
2259	}
2260
2261	#[test]
2262	fn an_empty_body_still_says_something() {
2263	let described = describe_api_error(reqwest::StatusCode::INTERNAL_SERVER_ERROR, " ");
2264
2265	assert_eq!(described, "endpoint returned 500 Internal Server Error");
2266	}
2267
2268	#[test]
2269	fn an_oversized_body_is_cut_down() {
2270	let body = "x".repeat(ERROR_BODY_LIMIT * 3);
2271
2272	let described = describe_api_error(reqwest::StatusCode::BAD_GATEWAY, &body);
2273
2274	assert!(described.len() < ERROR_BODY_LIMIT + 100, "{described}");
2275	assert!(described.ends_with('…'), "{described}");
2276	}
2277
2278	/// An envelope with nothing useful in it must not shadow the status, which
2279	/// would leave the user with an empty error.
2280	#[test]
2281	fn an_envelope_with_a_blank_message_falls_back() {
2282	let described = describe_api_error(
2283	reqwest::StatusCode::FORBIDDEN,
2284	r#"{"error":{"message":" "}}"#,
2285	);
2286
2287	assert!(described.contains("403"), "{described}");
2288	}
2289
2290	#[test]
2291	fn tools_json_wraps_function_schema() {
2292	let tools = vec![ToolSpec {
2293	name: "read_file".to_string(),
2294	description: "Read a file".to_string(),
2295	parameters_schema: r#"{"type":"object","properties":{"path":{"type":"string"}}}"#
2296	.to_string(),
2297	}];
2298	let json = OpenAiBackend::tools_json(&tools);
2299	assert_eq!(json[0]["type"], "function");
2300	assert_eq!(json[0]["function"]["name"], "read_file");
2301	assert_eq!(
2302	json[0]["function"]["parameters"]["properties"]["path"]["type"],
2303	"string"
2304	);
2305	}
2306
2307	#[test]
2308	fn malformed_schema_falls_back_to_empty_object() {
2309	let tools = vec![ToolSpec {
2310	name: "x".to_string(),
2311	description: String::new(),
2312	parameters_schema: "not json".to_string(),
2313	}];
2314	let json = OpenAiBackend::tools_json(&tools);
2315	assert_eq!(json[0]["function"]["parameters"]["type"], "object");
2316	}
2317
2318	#[test]
2319	fn streamed_assistant_history_omits_empty_tool_calls() {
2320	let value = streamed_assistant_history("hello", &[]);
2321	assert_eq!(value["role"], "assistant");
2322	assert_eq!(value["content"], "hello");
2323	assert!(value.get("tool_calls").is_none());
2324	}
2325
2326	#[test]
2327	fn streamed_assistant_history_preserves_tool_calls() {
2328	let calls = vec![ToolCall {
2329	id: "call_0".to_string(),
2330	name: "read_file".to_string(),
2331	arguments: r#"{"path":"a.rs"}"#.to_string(),
2332	}];
2333	let value = streamed_assistant_history("", &calls);
2334	assert!(value["content"].is_null());
2335	assert_eq!(value["tool_calls"][0]["id"], "call_0");
2336	assert_eq!(value["tool_calls"][0]["type"], "function");
2337	assert_eq!(value["tool_calls"][0]["function"]["name"], "read_file");
2338	assert_eq!(
2339	value["tool_calls"][0]["function"]["arguments"],
2340	r#"{"path":"a.rs"}"#
2341	);
2342	}
2343
2344	#[tokio::test]
2345	async fn cancelled_tool_results_close_out_history() {
2346	let backend = OpenAiBackend::new("http://localhost", "", "test-model", None);
2347	backend
2348	.history
2349	.lock()
2350	.await
2351	.push(streamed_assistant_history(
2352	"",
2353	&[ToolCall {
2354	id: "call_9".to_string(),
2355	name: "run_command".to_string(),
2356	arguments: r#"{"command":"ls"}"#.to_string(),
2357	}],
2358	));
2359
2360	backend
2361	.record_cancelled_tool_results(vec![ToolResult {
2362	tool_call_id: "call_9".to_string(),
2363	content: "cancelled by the user".to_string(),
2364	}])
2365	.await;
2366
2367	let history = backend.history.lock().await;
2368	let last = history.last().unwrap();
2369	assert_eq!(last["role"], "tool");
2370	assert_eq!(last["tool_call_id"], "call_9");
2371	assert_eq!(last["content"], "cancelled by the user");
2372
2373	drop(history);
2374	backend
2375	.record_cancelled_tool_results(vec![ToolResult {
2376	tool_call_id: "call_9".to_string(),
2377	content: "duplicate cleanup".to_string(),
2378	}])
2379	.await;
2380	let history = backend.history.lock().await;
2381	assert_eq!(
2382	history
2383	.iter()
2384	.filter(\|message\| message["tool_call_id"] == "call_9")
2385	.count(),
2386	1
2387	);
2388	}
2389
2390	#[test]
2391	fn carryover_drops_system_messages_and_keeps_the_turns() {
2392	let snapshot = vec![
2393	serde_json::json!({ "role": "system", "content": "old prompt" }),
2394	serde_json::json!({ "role": "user", "content": "hello" }),
2395	serde_json::json!({ "role": "assistant", "content": "hi" }),
2396	];
2397
2398	let carried = carryover_history(snapshot);
2399
2400	assert_eq!(carried.len(), 2);
2401	assert_eq!(carried[0]["role"], "user");
2402	assert_eq!(carried[1]["content"], "hi");
2403	}
2404
2405	#[test]
2406	fn carryover_keeps_a_trailing_user_message_from_a_failed_turn() {
2407	// The client keeps a prompt whose inference errored, so a switch must
2408	// not drop it (issue #124).
2409	let carried = carryover_history(vec![
2410	serde_json::json!({ "role": "system", "content": "prompt" }),
2411	serde_json::json!({ "role": "user", "content": "completed question" }),
2412	serde_json::json!({ "role": "assistant", "content": "completed answer" }),
2413	serde_json::json!({ "role": "user", "content": "failed question" }),
2414	]);
2415
2416	assert_eq!(carried.len(), 3, "{carried:#?}");
2417	assert_eq!(carried.last().unwrap()["content"], "failed question");
2418	}
2419
2420	#[tokio::test]
2421	async fn forget_trailing_user_message_drops_only_a_cancelled_prompt() {
2422	let backend = OpenAiBackend::new("http://localhost", "", "m", None);
2423	backend
2424	.restore_history(vec![
2425	serde_json::json!({ "role": "user", "content": "completed question" }),
2426	serde_json::json!({ "role": "assistant", "content": "completed answer" }),
2427	serde_json::json!({ "role": "user", "content": "cancelled question" }),
2428	])
2429	.await;
2430
2431	forget_trailing_user_message(&backend).await;
2432
2433	let history = backend.history_snapshot().await;
2434	assert_eq!(history.len(), 2, "{history:#?}");
2435	assert_eq!(history[1]["content"], "completed answer");
2436
2437	forget_trailing_user_message(&backend).await;
2438	assert_eq!(backend.history_snapshot().await.len(), 2);
2439	}
2440
2441	#[test]
2442	fn carryover_strips_a_tool_call_that_never_got_a_result() {
2443	// Switching mid-turn: the assistant asked for a tool, nothing answered.
2444	let snapshot = vec![
2445	serde_json::json!({ "role": "user", "content": "read a.rs" }),
2446	streamed_assistant_history(
2447	"on it",
2448	&[ToolCall {
2449	id: "call_1".to_string(),
2450	name: "read_file".to_string(),
2451	arguments: "{}".to_string(),
2452	}],
2453	),
2454	];
2455
2456	let carried = carryover_history(snapshot);
2457
2458	assert_eq!(carried.len(), 2);
2459	assert_eq!(carried[1]["content"], "on it");
2460	assert!(
2461	carried[1].get("tool_calls").is_none(),
2462	"an unanswered tool call must not survive the switch"
2463	);
2464	}
2465
2466	#[test]
2467	fn carryover_drops_a_textless_unanswered_tool_call_and_its_late_result() {
2468	let snapshot = vec![
2469	streamed_assistant_history(
2470	"",
2471	&[ToolCall {
2472	id: "call_1".to_string(),
2473	name: "read_file".to_string(),
2474	arguments: "{}".to_string(),
2475	}],
2476	),
2477	// An orphan: its assistant message is gone with the line above.
2478	serde_json::json!({
2479	"role": "tool",
2480	"tool_call_id": "call_other",
2481	"content": "file contents",
2482	}),
2483	];
2484
2485	assert!(carryover_history(snapshot).is_empty());
2486	}
2487
2488	#[test]
2489	fn carryover_keeps_an_answered_tool_call_paired_with_its_result() {
2490	let snapshot = vec![
2491	streamed_assistant_history(
2492	"",
2493	&[ToolCall {
2494	id: "call_1".to_string(),
2495	name: "read_file".to_string(),
2496	arguments: "{}".to_string(),
2497	}],
2498	),
2499	serde_json::json!({
2500	"role": "tool",
2501	"tool_call_id": "call_1",
2502	"content": "file contents",
2503	}),
2504	];
2505
2506	let carried = carryover_history(snapshot);
2507
2508	assert_eq!(carried.len(), 2);
2509	assert_eq!(carried[0]["tool_calls"][0]["id"], "call_1");
2510	assert_eq!(carried[1]["tool_call_id"], "call_1");
2511	}
2512
2513	#[tokio::test]
2514	async fn adopt_carryover_replays_the_thread_under_the_new_system_prompt() {
2515	let new_backend =
2516	OpenAiBackend::new("http://localhost", "", "m", Some("new prompt".into()));
2517
2518	let carried = carryover_history(vec![
2519	serde_json::json!({ "role": "system", "content": "old prompt" }),
2520	serde_json::json!({ "role": "user", "content": "hello" }),
2521	serde_json::json!({ "role": "assistant", "content": "hi" }),
2522	]);
2523	adopt_carryover(&new_backend, carried).await;
2524
2525	let history = new_backend.history_snapshot().await;
2526	assert_eq!(history.len(), 3);
2527	assert_eq!(history[0]["role"], "system");
2528	assert_eq!(history[0]["content"], "new prompt");
2529	assert_eq!(history[1]["content"], "hello");
2530	assert_eq!(history[2]["content"], "hi");
2531	}
2532
2533	#[tokio::test]
2534	async fn adopt_carryover_leaves_a_fresh_backend_alone() {
2535	let new_backend =
2536	OpenAiBackend::new("http://localhost", "", "m", Some("new prompt".into()));
2537
2538	adopt_carryover(&new_backend, Vec::new()).await;
2539
2540	let history = new_backend.history_snapshot().await;
2541	assert_eq!(history.len(), 1);
2542	assert_eq!(history[0]["content"], "new prompt");
2543	}
2544
2545	fn user_with_image(text: &str, payload_chars: usize) -> serde_json::Value {
2546	serde_json::json!({
2547	"role": "user",
2548	"content": [
2549	{ "type": "text", "text": text },
2550	{ "type": "image_url", "image_url": {
2551	"url": format!("data:image/png;base64,{}", "A".repeat(payload_chars)),
2552	}},
2553	],
2554	})
2555	}
2556
2557	#[test]
2558	fn user_message_uses_content_parts_only_when_there_is_an_image() {
2559	assert_eq!(
2560	user_message("hello", &[], &[]),
2561	serde_json::json!({ "role": "user", "content": "hello" })
2562	);
2563
2564	let image = ImageInput {
2565	mime_type: "image/png".to_string(),
2566	data: "AAAA".to_string(),
2567	};
2568	assert_eq!(
2569	user_message("what is this?", std::slice::from_ref(&image), &[]),
2570	serde_json::json!({
2571	"role": "user",
2572	"content": [
2573	{ "type": "text", "text": "what is this?" },
2574	{ "type": "image_url", "image_url": { "url": "data:image/png;base64,AAAA" } },
2575	],
2576	})
2577	);
2578	// An image with no text sends no empty text part.
2579	let only_image = user_message("", &[image], &[]);
2580	assert_eq!(only_image["content"].as_array().unwrap().len(), 1);
2581	}
2582
2583	#[test]
2584	fn audio_is_sent_as_an_input_audio_part() {
2585	let clip = AudioInput {
2586	mime_type: "audio/mpeg".to_string(),
2587	data: "BBBB".to_string(),
2588	};
2589	assert_eq!(
2590	user_message("transcribe this", &[], &[clip]),
2591	serde_json::json!({
2592	"role": "user",
2593	"content": [
2594	{ "type": "text", "text": "transcribe this" },
2595	{ "type": "input_audio", "input_audio": { "data": "BBBB", "format": "mp3" } },
2596	],
2597	})
2598	);
2599	}
2600
2601	#[test]
2602	fn audio_format_follows_the_mime_type() {
2603	let format = \|mime: &str\| {
2604	AudioInput {
2605	mime_type: mime.to_string(),
2606	data: String::new(),
2607	}
2608	.format()
2609	};
2610	assert_eq!(format("audio/wav"), "wav");
2611	assert_eq!(format("audio/x-wav"), "wav");
2612	assert_eq!(format("audio/wave"), "wav");
2613	assert_eq!(format("audio/mpeg"), "mp3");
2614	assert_eq!(format("Audio/MP3; codecs=mp3"), "mp3");
2615	assert_eq!(format("audio/flac"), "flac");
2616	}
2617
2618	#[test]
2619	fn a_model_without_audio_gets_a_note_and_keeps_the_image() {
2620	let history = vec![serde_json::json!({
2621	"role": "user",
2622	"content": [
2623	{ "type": "text", "text": "what is this?" },
2624	{ "type": "image_url", "image_url": { "url": "data:image/png;base64,AAAA" } },
2625	{ "type": "input_audio", "input_audio": { "data": "BBBB", "format": "wav" } },
2626	],
2627	})];
2628
2629	let sent = without_audio(&history);
2630	assert_eq!(
2631	sent[0]["content"],
2632	serde_json::json!([
2633	{ "type": "text", "text": "what is this?" },
2634	{ "type": "image_url", "image_url": { "url": "data:image/png;base64,AAAA" } },
2635	{ "type": "text", "text": AUDIO_OMITTED_NOTE },
2636	])
2637	);
2638	// With neither, the message is plain text again.
2639	assert_eq!(
2640	without_images(&sent)[0]["content"],
2641	format!("what is this?\n{AUDIO_OMITTED_NOTE}\n{IMAGE_OMITTED_NOTE}")
2642	);
2643	// History keeps the clip for a model that can take it later.
2644	assert_eq!(message_audio_count(&history[0]), 1);
2645	}
2646
2647	#[test]
2648	fn a_text_only_model_is_sent_a_note_in_place_of_each_image() {
2649	let history = vec![
2650	serde_json::json!({ "role": "system", "content": "prompt" }),
2651	user_with_image("what is this?", 16),
2652	serde_json::json!({ "role": "assistant", "content": "a cat" }),
2653	];
2654	let sent = without_images(&history);
2655
2656	assert_eq!(sent[0], history[0]);
2657	assert_eq!(sent[2], history[2]);
2658	assert_eq!(
2659	sent[1]["content"],
2660	format!("what is this?\n{IMAGE_OMITTED_NOTE}")
2661	);
2662	// History keeps the image for a model that can read it later.
2663	assert_eq!(message_image_count(&history[1]), 1);
2664	}
2665
2666	#[test]
2667	fn a_remote_backend_reads_images_by_model_id() {
2668	let own_endpoint = OpenAiBackend::new("http://localhost", "", "gpt-4o-mini", None);
2669	assert!(own_endpoint.accepts_images());
2670	let text_tier = OpenAiBackend::new("http://localhost", "", "onde-nova", None);
2671	assert!(!text_tier.accepts_images());
2672	let image_tier = OpenAiBackend::new("http://localhost", "", "onde-large", None);
2673	assert!(image_tier.accepts_images());
2674	}
2675
2676	#[test]
2677	fn a_remote_backend_takes_audio_only_on_the_users_own_endpoint() {
2678	let own_endpoint = OpenAiBackend::new("http://localhost", "", "gpt-4o-audio", None);
2679	assert!(own_endpoint.accepts_audio());
2680	let cloud_tier = OpenAiBackend::new("http://localhost", "", "onde-large", None);
2681	assert!(!cloud_tier.accepts_audio());
2682	}
2683
2684	#[test]
2685	fn message_text_reads_both_content_shapes() {
2686	assert_eq!(
2687	message_text(&serde_json::json!({ "role": "user", "content": "hello" })),
2688	"hello"
2689	);
2690	let with_image = user_with_image("what is this?", 16);
2691	assert_eq!(message_text(&with_image), "what is this?");
2692	assert_eq!(message_image_count(&with_image), 1);
2693	// A tool-call-only assistant message has no content at all.
2694	let no_content = serde_json::json!({ "role": "assistant", "content": null });
2695	assert_eq!(message_text(&no_content), "");
2696	assert_eq!(message_image_count(&no_content), 0);
2697	}
2698
2699	#[test]
2700	fn estimate_tokens_does_not_count_an_image_payload_as_text() {
2701	// Two megabytes of base64 would read as half a million tokens.
2702	let history = vec![user_with_image("what is this?", 2_000_000)];
2703	let estimate = estimate_tokens(&history);
2704	assert!(
2705	(IMAGE_TOKEN_ESTIMATE..IMAGE_TOKEN_ESTIMATE + 100).contains(&estimate),
2706	"{estimate}"
2707	);
2708	}
2709
2710	#[test]
2711	fn summary_transcript_keeps_the_text_of_a_message_with_an_image() {
2712	let transcript = transcript_for_summary(&[user_with_image("what is this?", 64)]);
2713	assert!(transcript.contains("what is this?"), "{transcript}");
2714	assert!(transcript.contains("[attached an image]"), "{transcript}");
2715	assert!(!transcript.contains("base64"), "{transcript}");
2716	}
2717
2718	#[test]
2719	fn carryover_keeps_a_user_message_that_carries_an_image() {
2720	let carried = carryover_history(vec![
2721	serde_json::json!({ "role": "system", "content": "prompt" }),
2722	user_with_image("what is this?", 16),
2723	serde_json::json!({ "role": "assistant", "content": "a cat" }),
2724	]);
2725	assert_eq!(carried.len(), 2);
2726	assert_eq!(message_image_count(&carried[0]), 1);
2727	}
2728
2729	#[test]
2730	fn estimate_tokens_scales_with_serialized_size() {
2731	assert_eq!(estimate_tokens(&[]), 0);
2732
2733	let short = vec![serde_json::json!({ "role": "user", "content": "hi" })];
2734	let long = vec![serde_json::json!({ "role": "user", "content": "x".repeat(4_000) })];
2735	let short_estimate = estimate_tokens(&short);
2736	let long_estimate = estimate_tokens(&long);
2737
2738	assert!(short_estimate > 0, "non-empty history estimates > 0 tokens");
2739	assert!(long_estimate > short_estimate, "longer history costs more");
2740	// 4,000 content chars / 4 ≈ 1,000 tokens, plus a little JSON framing.
2741	assert!((1_000..1_100).contains(&long_estimate), "{long_estimate}");
2742	}
2743
2744	#[tokio::test]
2745	async fn openai_snapshot_restore_round_trips_exactly() {
2746	let backend = OpenAiBackend::new("http://localhost", "", "m", Some("be helpful".into()));
2747	{
2748	let mut history = backend.history.lock().await;
2749	history.push(serde_json::json!({ "role": "user", "content": "hello" }));
2750	history.push(streamed_assistant_history(
2751	"",
2752	&[ToolCall {
2753	id: "call_1".to_string(),
2754	name: "read_file".to_string(),
2755	arguments: r#"{"path":"a.rs"}"#.to_string(),
2756	}],
2757	));
2758	history.push(serde_json::json!({
2759	"role": "tool", "tool_call_id": "call_1", "content": "fn main() {}",
2760	}));
2761	history.push(serde_json::json!({ "role": "assistant", "content": "done" }));
2762	}
2763	let snapshot = backend.history_snapshot().await;
2764	assert_eq!(
2765	snapshot[0]["role"], "system",
2766	"snapshot keeps the system message"
2767	);
2768
2769	// Restoring into a backend seeded with a different system prompt must
2770	// replace everything, including that seed.
2771	let restored = OpenAiBackend::new("http://localhost", "", "m", Some("other seed".into()));
2772	restored.restore_history(snapshot.clone()).await;
2773	assert_eq!(restored.history_snapshot().await, snapshot);
2774	}
2775
2776	#[tokio::test]
2777	async fn local_restore_replaces_engine_history_and_system_context() {
2778	let engine = Arc::new(ChatEngine::new());
2779	let backend = LocalBackend::new(Arc::clone(&engine));
2780	backend
2781	.restore_history(vec![
2782	serde_json::json!({ "role": "system", "content": "thread A context" }),
2783	serde_json::json!({ "role": "user", "content": "thread A question" }),
2784	])
2785	.await;
2786
2787	backend
2788	.restore_history(vec![
2789	serde_json::json!({ "role": "system", "content": "thread B context" }),
2790	serde_json::json!({ "role": "user", "content": "thread B question" }),
2791	])
2792	.await;
2793
2794	let restored = backend.history_snapshot().await;
2795	assert_eq!(restored.len(), 2, "{restored:#?}");
2796	assert_eq!(restored[0]["content"], "thread B context");
2797	assert_eq!(restored[1]["content"], "thread B question");
2798	assert!(
2799	restored.iter().all(\|message\| !message["content"]
2800	.as_str()
2801	.unwrap_or_default()
2802	.contains("thread A")),
2803	"restore_history must replace, not append to, the local engine: {restored:#?}"
2804	);
2805
2806	let error = backend
2807	.send_message_with_tools("must not run", &[], None)
2808	.await
2809	.unwrap_err();
2810	assert!(error.contains("not ready"), "{error}");
2811	assert_eq!(backend.history_snapshot().await, restored);
2812	}
2813
2814	/// Minimal scripted OpenAI-compatible endpoint: accepts one HTTP request on
2815	/// a std listener and answers with a fixed non-streaming completion. The
2816	/// receiver yields the request body the backend actually put on the wire.
2817	fn spawn_completion_stub(
2818	summary: &str,
2819	) -> (std::net::SocketAddr, std::sync::mpsc::Receiver<String>) {
2820	spawn_message_stub(serde_json::json!({ "role": "assistant", "content": summary }))
2821	}
2822
2823	/// Like [`spawn_completion_stub`], answering with an arbitrary message.
2824	fn spawn_message_stub(
2825	message: serde_json::Value,
2826	) -> (std::net::SocketAddr, std::sync::mpsc::Receiver<String>) {
2827	spawn_choice_stub(serde_json::json!({ "message": message }))
2828	}
2829
2830	/// Like [`spawn_message_stub`], answering with a whole choice, so a test
2831	/// can set what sits beside the message.
2832	fn spawn_choice_stub(
2833	choice: serde_json::Value,
2834	) -> (std::net::SocketAddr, std::sync::mpsc::Receiver<String>) {
2835	use std::io::{Read, Write};
2836
2837	let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2838	let addr = listener.local_addr().unwrap();
2839	let (sender, receiver) = std::sync::mpsc::channel();
2840	let body = serde_json::json!({ "choices": [choice] }).to_string();
2841	std::thread::spawn(move \|\| {
2842	let (mut stream, _) = listener.accept().unwrap();
2843	// Read until the full request (headers + content-length body) is in.
2844	let mut request = Vec::new();
2845	let mut chunk = [0u8; 4096];
2846	loop {
2847	let n = stream.read(&mut chunk).unwrap_or(0);
2848	if n == 0 {
2849	break;
2850	}
2851	request.extend_from_slice(&chunk[..n]);
2852	if let Some(headers_end) =
2853	request.windows(4).position(\|window\| window == b"\r\n\r\n")
2854	{
2855	let headers = String::from_utf8_lossy(&request[..headers_end]);
2856	let content_length = headers
2857	.lines()
2858	.find_map(\|line\| {
2859	line.to_ascii_lowercase()
2860	.strip_prefix("content-length:")
2861	.map(\|value\| value.trim().parse::<usize>().unwrap_or(0))
2862	})
2863	.unwrap_or(0);
2864	if request.len() >= headers_end + 4 + content_length {
2865	let _ = sender.send(
2866	String::from_utf8_lossy(&request[headers_end + 4..]).into_owned(),
2867	);
2868	break;
2869	}
2870	}
2871	}
2872	let response = format!(
2873	"HTTP/1.1 200 OK\r\ncontent-type: application/json\r\n\
2874	content-length: {}\r\nconnection: close\r\n\r\n{}",
2875	body.len(),
2876	body
2877	);
2878	let _ = stream.write_all(response.as_bytes());
2879	});
2880	(addr, receiver)
2881	}
2882
2883	/// Scripted SSE endpoint: one response per connection, in order. The
2884	/// receiver yields each request body.
2885	fn spawn_sse_sequence_stub(
2886	replies: Vec<Vec<String>>,
2887	) -> (std::net::SocketAddr, std::sync::mpsc::Receiver<String>) {
2888	use std::io::{Read, Write};
2889
2890	let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
2891	let addr = listener.local_addr().unwrap();
2892	let (sender, receiver) = std::sync::mpsc::channel();
2893	std::thread::spawn(move \|\| {
2894	for deltas in replies {
2895	let (mut stream, _) = listener.accept().unwrap();
2896	let mut request = Vec::new();
2897	let mut buf = [0u8; 4096];
2898	loop {
2899	let n = stream.read(&mut buf).unwrap_or(0);
2900	if n == 0 {
2901	break;
2902	}
2903	request.extend_from_slice(&buf[..n]);
2904	if let Some(end) = request.windows(4).position(\|w\| w == b"\r\n\r\n") {
2905	let headers = String::from_utf8_lossy(&request[..end]).to_ascii_lowercase();
2906	let length = headers
2907	.lines()
2908	.find_map(\|l\| l.strip_prefix("content-length:")?.trim().parse().ok())
2909	.unwrap_or(0usize);
2910	if request.len() >= end + 4 + length {
2911	let _ = sender
2912	.send(String::from_utf8_lossy(&request[end + 4..]).into_owned());
2913	break;
2914	}
2915	}
2916	}
2917	let mut body = String::new();
2918	for delta in deltas {
2919	let frame = serde_json::json!({
2920	"choices": [{ "delta": { "content": delta } }]
2921	});
2922	body.push_str(&format!("data: {frame}\n\n"));
2923	}
2924	body.push_str("data: [DONE]\n\n");
2925	let response = format!(
2926	"HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n\
2927	content-length: {}\r\nconnection: close\r\n\r\n{body}",
2928	body.len()
2929	);
2930	let _ = stream.write_all(response.as_bytes());
2931	}
2932	});
2933	(addr, receiver)
2934	}
2935
2936	/// Issue #123: a streamed reply that degenerates into a repetition loop is
2937	/// cut off, history keeps only what came before the loop, and the model
2938	/// gets one retry.
2939	#[tokio::test]
2940	async fn a_repetition_loop_is_cut_and_retried_once() {
2941	let intro = "Let me look at the commits. ";
2942	let looped: Vec<String> = std::iter::once(intro.to_string())
2943	.chain(
2944	crate::repetition::issue_loop_text()
2945	.as_bytes()
2946	.chunks(40)
2947	.map(\|c\| String::from_utf8_lossy(c).into_owned()),
2948	)
2949	.collect();
2950	let (addr, requests) =
2951	spawn_sse_sequence_stub(vec![looped, vec!["Here is the answer.".to_string()]]);
2952	let backend =
2953	OpenAiBackend::new(format!("http://{addr}/v1"), "test-key", "test-model", None);
2954	let (sink, mut tokens) = tokio::sync::mpsc::unbounded_channel();
2955	let tools = vec![ToolSpec {
2956	name: "run_command".to_string(),
2957	description: "Run a command".to_string(),
2958	parameters_schema: r#"{"type":"object","properties":{}}"#.to_string(),
2959	}];
2960
2961	let result = backend
2962	.send_message_with_tools("what changed", &tools, Some(&sink))
2963	.await
2964	.unwrap();
2965
2966	assert_eq!(result.finish, FinishReason::Complete);
2967	assert!(result.text.starts_with(intro.trim_end()), "{}", result.text);
2968	assert!(result.text.ends_with("Here is the answer."));
2969	assert!(
2970	result.text.len() < 600,
2971	"junk kept: {} bytes",
2972	result.text.len()
2973	);
2974
2975	let first = requests.recv().unwrap();
2976	assert!(!first.contains(REPETITION_RETRY));
2977	let second = requests.recv().unwrap();
2978	assert!(second.contains(REPETITION_RETRY));
2979
2980	let history = backend.history_snapshot().await;
2981	let kept = history
2982	.iter()
2983	.find(\|m\| m["role"] == "assistant")
2984	.and_then(\|m\| m["content"].as_str())
2985	.unwrap();
2986	assert!(
2987	kept.len() < 600,
2988	"history kept the loop: {} bytes",
2989	kept.len()
2990	);
2991
2992	drop(sink);
2993	let mut shown = String::new();
2994	while let Ok(chunk) = tokens.try_recv() {
2995	if let TokenChunk::Visible(text) = chunk {
2996	shown.push_str(&text);
2997	}
2998	}
2999	assert!(shown.contains("stopped this reply"), "{shown}");
3000	}
3001
3002	#[test]
3003	fn finish_reasons_map_from_the_wire() {
3004	for reason in [None, Some("stop"), Some("tool_calls"), Some("unheard-of")] {
3005	assert_eq!(FinishReason::from_wire(reason), FinishReason::Complete);
3006	}
3007	assert_eq!(
3008	FinishReason::from_wire(Some("length")),
3009	FinishReason::Length
3010	);
3011	assert_eq!(
3012	FinishReason::from_wire(Some("content_filter")),
3013	FinishReason::ContentFilter
3014	);
3015	}
3016
3017	/// The non-streaming path (headless runs, subagents) reads the reason off
3018	/// the choice; a reply cut off at the token limit must not look finished.
3019	#[tokio::test]
3020	async fn a_truncated_json_reply_reports_length() {
3021	let (addr, _requests) = spawn_choice_stub(serde_json::json!({
3022	"message": { "role": "assistant", "content": "The answer is" },
3023	"finish_reason": "length",
3024	}));
3025	let backend =
3026	OpenAiBackend::new(format!("http://{addr}/v1"), "test-key", "test-model", None);
3027
3028	let result = backend
3029	.send_message_with_tools("explain", &[], None)
3030	.await
3031	.unwrap();
3032
3033	assert_eq!(result.text, "The answer is");
3034	assert_eq!(result.finish, FinishReason::Length);
3035	}
3036
3037	/// A structured call can come back with a broken inline block in the same
3038	/// message. The call runs; the block reaches neither the reply nor the
3039	/// history, where the model would read back a call that never ran.
3040	#[tokio::test]
3041	async fn a_broken_inline_block_beside_a_structured_call_is_dropped() {
3042	let (addr, _requests) = spawn_message_stub(serde_json::json!({
3043	"role": "assistant",
3044	"content": "Checking.<tool_call>run_command CheckStatus=nope</tool_call>",
3045	"tool_calls": [{
3046	"id": "call_1",
3047	"type": "function",
3048	"function": { "name": "run_command", "arguments": "{\"command\":\"pwd\"}" },
3049	}],
3050	}));
3051	let backend =
3052	OpenAiBackend::new(format!("http://{addr}/v1"), "test-key", "test-model", None);
3053	let tools = vec![ToolSpec {
3054	name: "run_command".to_string(),
3055	description: "Run a command".to_string(),
3056	parameters_schema: r#"{"type":"object","properties":{"command":{"type":"string"}}}"#
3057	.to_string(),
3058	}];
3059
3060	let result = backend
3061	.send_message_with_tools("where am I", &tools, None)
3062	.await
3063	.unwrap();
3064
3065	assert_eq!(result.text, "Checking.");
3066	assert_eq!(result.tool_calls.len(), 1);
3067	assert_eq!(result.tool_calls[0].id, "call_1");
3068	let history = backend.history_snapshot().await;
3069	let reply = history.last().unwrap();
3070	assert_eq!(reply["content"], "Checking.");
3071	assert_eq!(reply["tool_calls"][0]["id"], "call_1");
3072	}
3073
3074	/// A structured call plus a well-formed inline block: both run, and the
3075	/// inline markup leaves the reply and history, as on the streaming path.
3076	#[tokio::test]
3077	async fn a_well_formed_inline_block_beside_a_structured_call_also_runs() {
3078	let (addr, _requests) = spawn_message_stub(serde_json::json!({
3079	"role": "assistant",
3080	"content": "Checking.<tool_call>run_command<arg_key>command</arg_key><arg_value>ls</arg_value></tool_call>",
3081	"tool_calls": [{
3082	"id": "call_1",
3083	"type": "function",
3084	"function": { "name": "run_command", "arguments": "{\"command\":\"pwd\"}" },
3085	}],
3086	}));
3087	let backend =
3088	OpenAiBackend::new(format!("http://{addr}/v1"), "test-key", "test-model", None);
3089	let tools = vec![ToolSpec {
3090	name: "run_command".to_string(),
3091	description: "Run a command".to_string(),
3092	parameters_schema: r#"{"type":"object","properties":{"command":{"type":"string"}}}"#
3093	.to_string(),
3094	}];
3095
3096	let result = backend
3097	.send_message_with_tools("where am I", &tools, None)
3098	.await
3099	.unwrap();
3100
3101	assert_eq!(result.text, "Checking.");
3102	let ids: Vec<&str> = result.tool_calls.iter().map(\|c\| c.id.as_str()).collect();
3103	assert_eq!(ids, ["call_1", "call_recovered_1"]);
3104	assert_eq!(result.tool_calls[1].arguments, r#"{"command":"ls"}"#);
3105	let history = backend.history_snapshot().await;
3106	let reply = history.last().unwrap();
3107	assert_eq!(reply["content"], "Checking.");
3108	assert_eq!(reply["tool_calls"].as_array().unwrap().len(), 2);
3109	}
3110
3111	#[tokio::test]
3112	async fn compact_history_rebuilds_system_summary_and_tail() {
3113	let (addr, _requests) = spawn_completion_stub("We refactored backend.rs; tests pass.");
3114	let backend = OpenAiBackend::new(
3115	format!("http://{addr}/v1"),
3116	"test-key",
3117	"test-model",
3118	Some("be helpful".into()),
3119	);
3120	{
3121	let mut history = backend.history.lock().await;
3122	for i in 0..5 {
3123	let role = if i % 2 == 0 { "user" } else { "assistant" };
3124	history.push(serde_json::json!({
3125	"role": role, "content": format!("message {i}"),
3126	}));
3127	}
3128	}
3129
3130	backend.compact_history(2).await.unwrap();
3131
3132	let history = backend.history_snapshot().await;
3133	assert_eq!(history.len(), 4, "system + summary + last 2: {history:?}");
3134	assert_eq!(history[0]["role"], "system");
3135	assert_eq!(history[0]["content"], "be helpful");
3136	assert_eq!(history[1]["role"], "user");
3137	let summary_text = history[1]["content"].as_str().unwrap();
3138	assert!(summary_text.starts_with("[Conversation summary]\n"));
3139	assert!(summary_text.contains("We refactored backend.rs; tests pass."));
3140	assert_eq!(
3141	history[2],
3142	serde_json::json!({ "role": "assistant", "content": "message 3" })
3143	);
3144	assert_eq!(
3145	history[3],
3146	serde_json::json!({ "role": "user", "content": "message 4" })
3147	);
3148	}
3149
3150	/// Compacting a tool-heavy session must not put tool shapes on the wire.
3151	/// The summarization round offers no `tools`, and endpoints reject tool
3152	/// calls and tool results that arrive without a schema — which used to make
3153	/// compaction fail forever in any session that had run a single tool.
3154	#[tokio::test]
3155	async fn compact_history_sends_no_tool_artifacts() {
3156	let (addr, requests) = spawn_completion_stub("Ran git status on main.");
3157	let backend = OpenAiBackend::new(
3158	format!("http://{addr}/v1"),
3159	"test-key",
3160	"test-model",
3161	Some("be helpful".into()),
3162	);
3163	{
3164	let mut history = backend.history.lock().await;
3165	history.push(serde_json::json!({ "role": "user", "content": "check the repo" }));
3166	history.push(serde_json::json!({
3167	"role": "assistant",
3168	"content": null,
3169	"tool_calls": [{
3170	"id": "call_1",
3171	"type": "function",
3172	"function": { "name": "run_command", "arguments": "{\"command\":\"git status\"}" },
3173	}],
3174	}));
3175	history.push(serde_json::json!({
3176	"role": "tool",
3177	"tool_call_id": "call_1",
3178	"content": "on branch main",
3179	}));
3180	}
3181
3182	backend.compact_history(2).await.unwrap();
3183
3184	let body: serde_json::Value =
3185	serde_json::from_str(&requests.recv().unwrap()).expect("request body is JSON");
3186	assert!(
3187	body.get("tools").is_none(),
3188	"summarization offers no tools: {body}"
3189	);
3190	for message in body["messages"].as_array().unwrap() {
3191	assert!(
3192	message.get("tool_calls").is_none(),
3193	"no tool_calls may be sent without a schema: {message}"
3194	);
3195	assert_ne!(
3196	message["role"], "tool",
3197	"no tool results may be sent without a schema: {message}"
3198	);
3199	}
3200
3201	// The tool round still has to survive into the summary request as prose,
3202	// or the summary loses the work the session actually did.
3203	let transcript = body["messages"].as_array().unwrap().last().unwrap()["content"]
3204	.as_str()
3205	.unwrap();
3206	assert!(transcript.contains("called run_command({\"command\":\"git status\"})"));
3207	assert!(transcript.contains("tool result: on branch main"));
3208	}
3209
3210	/// Compaction only fails when even the truncation fallback cannot fit the
3211	/// history in the budget — here the newest message alone is bigger than
3212	/// the whole window.
3213	#[tokio::test]
3214	async fn compact_history_failure_leaves_history_intact() {
3215	// No listener at this address, so summarization fails; the system
3216	// prompt alone is bigger than the window, so even the minimal
3217	// fallback rebuild cannot fit — the genuinely unrecoverable case.
3218	let backend = OpenAiBackend::new(
3219	"http://127.0.0.1:9",
3220	"",
3221	"test-model",
3222	Some("s".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000)),
3223	);
3224	backend
3225	.history
3226	.lock()
3227	.await
3228	.push(serde_json::json!({ "role": "user", "content": "hello" }));
3229	let before = backend.history_snapshot().await;
3230
3231	assert!(backend.compact_history(2).await.is_err());
3232	assert_eq!(backend.history_snapshot().await, before);
3233	}
3234
3235	/// The issue-#125 path: the summarization endpoint is down, and the
3236	/// session must still shrink rather than die with "start a new thread".
3237	#[tokio::test]
3238	async fn failed_summarization_falls_back_to_truncation() {
3239	// Nothing listens here, so the summarization round always fails.
3240	let backend =
3241	OpenAiBackend::new("http://127.0.0.1:9", "", "test-model", Some("sys".into()));
3242	{
3243	let mut history = backend.history.lock().await;
3244	for i in 0..20 {
3245	history.push(serde_json::json!({
3246	"role": "user",
3247	"content": format!("message {i}: {}", "padding ".repeat(200)),
3248	}));
3249	}
3250	}
3251
3252	backend.compact_history(4).await.unwrap();
3253
3254	let history = backend.history_snapshot().await;
3255	assert_eq!(history[0]["role"], "system", "{history:?}");
3256	assert_eq!(history[1]["content"], TRUNCATION_PLACEHOLDER, "{history:?}");
3257	// The newest messages survive; the oldest were dropped to fit.
3258	assert_eq!(
3259	history.last().unwrap()["content"]
3260	.as_str()
3261	.unwrap()
3262	.split(':')
3263	.next()
3264	.unwrap(),
3265	"message 19",
3266	"{history:?}"
3267	);
3268	assert!(
3269	estimate_tokens(&history) <= DEFAULT_CONTEXT_TOKEN_BUDGET,
3270	"fallback result must fit the budget: ~{} tokens",
3271	estimate_tokens(&history)
3272	);
3273	}
3274
3275	/// A history small enough to begin with keeps its whole tail: the
3276	/// fallback drops nothing beyond what `keep_last` already folded away.
3277	#[tokio::test]
3278	async fn truncation_fallback_keeps_the_newest_messages_that_fit() {
3279	let system = vec![serde_json::json!({ "role": "system", "content": "sys" })];
3280	let tail: Vec<serde_json::Value> = (0..4)
3281	.map(\|i\| serde_json::json!({ "role": "user", "content": format!("m{i}") }))
3282	.collect();
3283
3284	let rebuilt = truncate_history_to_budget(&system, tail).unwrap();
3285
3286	assert_eq!(rebuilt[0]["role"], "system");
3287	assert_eq!(rebuilt[1]["content"], TRUNCATION_PLACEHOLDER);
3288	assert_eq!(rebuilt.len(), 6, "{rebuilt:?}");
3289	assert_eq!(rebuilt[2]["content"], "m0");
3290	assert_eq!(rebuilt[5]["content"], "m3");
3291	}
3292
3293	/// A multi-system-message session keeps every leading system message in
3294	/// the fallback, exactly like the summarization-success path.
3295	#[test]
3296	fn truncation_fallback_keeps_all_leading_system_messages() {
3297	let system = vec![
3298	serde_json::json!({ "role": "system", "content": "seed prompt" }),
3299	serde_json::json!({ "role": "system", "content": "session context" }),
3300	];
3301	let tail = vec![serde_json::json!({ "role": "user", "content": "hi" })];
3302
3303	let rebuilt = truncate_history_to_budget(&system, tail).unwrap();
3304
3305	assert_eq!(
3306	rebuilt[0]["content"], "seed prompt",
3307	"first system message kept: {rebuilt:?}"
3308	);
3309	assert_eq!(
3310	rebuilt[1]["content"], "session context",
3311	"second system message kept: {rebuilt:?}"
3312	);
3313	}
3314
3315	#[test]
3316	fn truncation_fallback_drops_orphaned_tool_results_with_their_call() {
3317	// A caller that split mid-round hands in a tail starting with tool
3318	// results whose assistant call message already fell outside the kept
3319	// window; those orphans must go before anything else, or strict
3320	// endpoints reject the next request.
3321	let tail = vec![
3322	serde_json::json!({ "role": "tool", "tool_call_id": "call_1", "content": "ok" }),
3323	serde_json::json!({ "role": "user", "content": "thanks" }),
3324	serde_json::json!({ "role": "assistant", "content": "done" }),
3325	];
3326
3327	let rebuilt = truncate_history_to_budget(&[], tail).unwrap();
3328
3329	let roles: Vec<&str> = rebuilt.iter().filter_map(\|m\| m["role"].as_str()).collect();
3330	assert_eq!(
3331	roles,
3332	["user", "user", "assistant"],
3333	"placeholder + the surviving exchange, orphan gone: {rebuilt:?}"
3334	);
3335	}
3336
3337	/// Review follow-up: a tail that drains to empty (one oversized tool
3338	/// round) must still test the minimal rebuild — system + placeholder fits,
3339	/// so the session is recoverable, not "unfixable".
3340	#[test]
3341	fn truncation_fallback_recovers_when_the_whole_tail_was_one_oversized_round() {
3342	let system = vec![serde_json::json!({ "role": "system", "content": "sys" })];
3343	let tail = vec![
3344	serde_json::json!({
3345	"role": "assistant",
3346	"content": null,
3347	"tool_calls": [{
3348	"id": "call_1", "type": "function",
3349	"function": { "name": "read_file", "arguments": "{}" },
3350	}],
3351	}),
3352	serde_json::json!({
3353	"role": "tool",
3354	"tool_call_id": "call_1",
3355	"content": "y".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000),
3356	}),
3357	];
3358
3359	let rebuilt = truncate_history_to_budget(&system, tail).unwrap();
3360
3361	assert_eq!(rebuilt.len(), 2, "{rebuilt:?}");
3362	assert_eq!(rebuilt[0]["role"], "system");
3363	assert_eq!(rebuilt[1]["content"], TRUNCATION_PLACEHOLDER);
3364	}
3365
3366	#[test]
3367	fn truncation_fallback_reports_when_nothing_fits() {
3368	// Even the minimal rebuild (system + placeholder, no conversation)
3369	// overflows: the system messages alone are bigger than the window.
3370	let system = vec![serde_json::json!({
3371	"role": "system",
3372	"content": "s".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000),
3373	})];
3374
3375	assert!(truncate_history_to_budget(&system, Vec::new()).is_none());
3376	}
3377
3378	/// Review follow-up: the local fallback recovers from a newest message
3379	/// that overflows the window on its own, the same way the remote one does.
3380	#[test]
3381	fn local_truncation_fallback_drops_an_oversized_final_message() {
3382	let system = vec![ChatMessage::system("sys")];
3383	let tail = vec![
3384	ChatMessage::user("hi"),
3385	ChatMessage::user("y".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000)),
3386	];
3387
3388	let rebuilt = truncate_chat_messages_to_budget(system, tail).unwrap();
3389
3390	assert_eq!(rebuilt.len(), 2, "{rebuilt:?}");
3391	assert_eq!(rebuilt[0].content, "sys");
3392	assert_eq!(rebuilt[1].content, TRUNCATION_PLACEHOLDER);
3393	}
3394
3395	#[test]
3396	fn local_truncation_fallback_keeps_the_newest_messages_that_fit() {
3397	let big = DEFAULT_CONTEXT_TOKEN_BUDGET * 4 / 2;
3398	let tail = vec![
3399	ChatMessage::user("a".repeat(big)),
3400	ChatMessage::user("b".repeat(big)),
3401	ChatMessage::user("newest"),
3402	];
3403
3404	let rebuilt = truncate_chat_messages_to_budget(Vec::new(), tail).unwrap();
3405
3406	assert_eq!(rebuilt.len(), 3, "oldest dropped, rest kept");
3407	assert_eq!(rebuilt[0].content, TRUNCATION_PLACEHOLDER);
3408	assert!(rebuilt[1].content.starts_with('b'));
3409	assert_eq!(rebuilt[2].content, "newest");
3410	}
3411
3412	#[test]
3413	fn local_truncation_fallback_reports_when_the_system_messages_overflow() {
3414	let system = vec![ChatMessage::system(
3415	"s".repeat(DEFAULT_CONTEXT_TOKEN_BUDGET * 4 + 1000),
3416	)];
3417
3418	assert!(truncate_chat_messages_to_budget(system, Vec::new()).is_none());
3419	}
3420
3421	/// Review follow-up: head and tail share the budget, so the trimmed
3422	/// transcript stays near the cap instead of reaching twice it.
3423	#[test]
3424	fn truncate_transcript_middle_stays_within_the_cap() {
3425	let lines: Vec<String> = (0..2_000)
3426	.map(\|i\| format!("user: message {i} with some padding text"))
3427	.collect();
3428	let transcript = lines.join("\n");
3429
3430	let trimmed = truncate_transcript_middle(&transcript, 200); // ~800 chars
3431
3432	assert!(trimmed.contains("message 0 "), "opening kept: {trimmed}");
3433	assert!(trimmed.contains("message 1999"), "closing kept: {trimmed}");
3434	assert!(
3435	trimmed.len() <= 800 + 80,
3436	"cap plus the omission marker, got {} chars",
3437	trimmed.len()
3438	);
3439	}
3440
3441	#[test]
3442	fn truncate_transcript_middle_keeps_short_transcripts_verbatim() {
3443	let transcript = "user: hello\n\nassistant: hi";
3444
3445	assert_eq!(truncate_transcript_middle(transcript, 1_000), transcript);
3446	}
3447
3448	/// Review follow-up: a huge final message with no newline in the tail
3449	/// half must not be silently dropped — the closing exchange is what the
3450	/// summary most needs. The head gives up its share instead.
3451	#[test]
3452	fn truncate_transcript_middle_keeps_a_newlineless_final_message() {
3453	let mut lines: Vec<String> = (0..100).map(\|i\| format!("user: message {i}")).collect();
3454	lines.push(format!(
3455	"user: {}current state: pending review",
3456	"noise ".repeat(600)
3457	));
3458	let transcript = lines.join("\n");
3459
3460	let trimmed = truncate_transcript_middle(&transcript, 400); // ~1600 chars
3461
3462	assert!(
3463	trimmed.contains("current state: pending review"),
3464	"the closing exchange survives: {trimmed}"
3465	);
3466	assert!(
3467	trimmed.contains("older messages omitted"),
3468	"the gap is marked: {trimmed}"
3469	);
3470	assert!(
3471	!trimmed.contains("message 0"),
3472	"the head gives up its share to the unbreakable tail: {trimmed}"
3473	);
3474	}
3475
3476	/// Review follow-up: no newline anywhere in reach — keep the newest
3477	/// budget, not the oldest (which would invert the contract while the
3478	/// placeholder claims older messages were dropped).
3479	#[test]
3480	fn truncate_transcript_middle_without_any_newline_keeps_the_newest() {
3481	let transcript = format!("old decisions {} current state", "x".repeat(4_000));
3482
3483	let trimmed = truncate_transcript_middle(&transcript, 200); // ~800 chars
3484
3485	assert!(
3486	trimmed.contains("current state"),
3487	"the newest content survives: {trimmed}"
3488	);
3489	assert!(
3490	!trimmed.contains("old decisions"),
3491	"the oldest goes when only one side can survive: {trimmed}"
3492	);
3493	}
3494
3495	#[test]
3496	fn truncate_transcript_middle_cuts_the_middle_not_the_ends() {
3497	let mut lines: Vec<String> = Vec::new();
3498	for i in 0..200 {
3499	lines.push(format!("user: message {i} with some padding text"));
3500	}
3501	let transcript = lines.join("\n");
3502
3503	let trimmed = truncate_transcript_middle(&transcript, 200); // ~800 chars
3504
3505	assert!(trimmed.contains("message 0"), "opening kept: {trimmed}");
3506	assert!(trimmed.contains("message 199"), "closing kept: {trimmed}");
3507	assert!(
3508	trimmed.contains("older messages omitted"),
3509	"gap is marked: {trimmed}"
3510	);
3511	assert!(
3512	trimmed.len() <= transcript.len(),
3513	"never grows the transcript"
3514	);
3515	}
3516
3517	#[test]
3518	fn assistant_message_with_tool_calls_round_trips() {
3519	let message = ResponseMessage {
3520	content: None,
3521	tool_calls: Some(vec![ResponseToolCall {
3522	id: "call_1".to_string(),
3523	function: ResponseFunction {
3524	name: "read_file".to_string(),
3525	arguments: r#"{"path":"a.rs"}"#.to_string(),
3526	},
3527	}]),
3528	};
3529	let value = message.into_history_value();
3530	assert_eq!(value["role"], "assistant");
3531	assert!(value["content"].is_null());
3532	assert_eq!(value["tool_calls"][0]["id"], "call_1");
3533	assert_eq!(value["tool_calls"][0]["type"], "function");
3534	assert_eq!(value["tool_calls"][0]["function"]["name"], "read_file");
3535	}
3536	}