Replay reasoning and attachments on session/load
session/load redraws a thread from the updates the agent sends, and the replay carried only the text of each message. A reopened thread lost the model's reasoning and any image or audio the user had attached. Reasoning now comes back as an agent_thought_chunk under the reply's messageId, the way it goes out during a turn. A user message's image_url and input_audio parts are turned back into image and audio blocks under the id of its text. An image stored as a remote URL has no data to send and is left out. The replay test asserted that reasoning stays hidden "exactly as it did live". That was written before live turns sent thought chunks, so it now expects the thought. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-Authored-By: siGit Code v1.6.7-acp <noreply@sigit.si>
Seto Elkahfi committed
Oct 12, 2026 at 09:48 UTC
6b2222c957c80ab4b79993537051e28b90440f95
1 file changed
+147
-29
src/main.rs
+147
-29
@@ -74,28 +74,29 @@ use onde::inference::SamplingConfig;
74
// schema types moved under `schema::v1` in agent-client-protocol 1.0.
75
use agent_client_protocol::schema::ProtocolVersion;
76
use agent_client_protocol::schema::v1::{
77
- AgentAuthCapabilities, AgentCapabilities, AuthMethod, AuthMethodAgent, AuthenticateRequest,
78
- AuthenticateResponse, AvailableCommand, AvailableCommandInput, AvailableCommandsUpdate,
79
- CancelNotification, CloseSessionRequest, CloseSessionResponse, CompleteElicitationNotification,
80
- ConfigOptionUpdate, ContentBlock, ContentChunk, CreateElicitationRequest,
81
- CreateTerminalRequest, CurrentModeUpdate, DeleteSessionRequest, DeleteSessionResponse, Diff,
82
- ElicitationAction, ElicitationRequestScope, ElicitationUrlMode, EmbeddedResourceResource,
83
- ErrorCode, ForkSessionRequest, ForkSessionResponse, Implementation, InitializeRequest,
84
- InitializeResponse, KillTerminalRequest, ListSessionsRequest, ListSessionsResponse,
85
- LoadSessionRequest, LoadSessionResponse, LogoutCapabilities, LogoutRequest, LogoutResponse,
86
- McpCapabilities, McpServer, MessageId, Meta, NewSessionRequest, NewSessionResponse,
87
- PermissionOption, PermissionOptionKind, Plan, PlanEntry, PlanEntryPriority, PlanEntryStatus,
88
- PromptCapabilities, PromptRequest, PromptResponse, ReadTextFileRequest, ReleaseTerminalRequest,
89
- RequestId, RequestPermissionOutcome, RequestPermissionRequest, ResumeSessionRequest,
90
- ResumeSessionResponse, SessionAdditionalDirectoriesCapabilities, SessionCapabilities,
91
- SessionCloseCapabilities, SessionConfigOption, SessionConfigOptionCategory,
92
- SessionConfigSelectOption, SessionConfigValueId, SessionDeleteCapabilities,
93
- SessionForkCapabilities, SessionId, SessionInfo, SessionInfoUpdate, SessionListCapabilities,
94
- SessionMode, SessionModeState, SessionNotification, SessionResumeCapabilities, SessionUpdate,
95
- SetSessionConfigOptionRequest, SetSessionConfigOptionResponse, SetSessionModeRequest,
96
- SetSessionModeResponse, StopReason, Terminal, TerminalOutputRequest, ToolCall, ToolCallContent,
97
- ToolCallLocation, ToolCallStatus, ToolCallUpdate, ToolCallUpdateFields, ToolKind,
98
- UnstructuredCommandInput, UsageUpdate, WaitForTerminalExitRequest, WriteTextFileRequest,
77
+ AgentAuthCapabilities, AgentCapabilities, AudioContent, AuthMethod, AuthMethodAgent,
78
+ AuthenticateRequest, AuthenticateResponse, AvailableCommand, AvailableCommandInput,
79
+ AvailableCommandsUpdate, CancelNotification, CloseSessionRequest, CloseSessionResponse,
80
+ CompleteElicitationNotification, ConfigOptionUpdate, ContentBlock, ContentChunk,
81
+ CreateElicitationRequest, CreateTerminalRequest, CurrentModeUpdate, DeleteSessionRequest,
82
+ DeleteSessionResponse, Diff, ElicitationAction, ElicitationRequestScope, ElicitationUrlMode,
83
+ EmbeddedResourceResource, ErrorCode, ForkSessionRequest, ForkSessionResponse, ImageContent,
84
+ Implementation, InitializeRequest, InitializeResponse, KillTerminalRequest,
85
+ ListSessionsRequest, ListSessionsResponse, LoadSessionRequest, LoadSessionResponse,
86
+ LogoutCapabilities, LogoutRequest, LogoutResponse, McpCapabilities, McpServer, MessageId, Meta,
87
+ NewSessionRequest, NewSessionResponse, PermissionOption, PermissionOptionKind, Plan, PlanEntry,
88
+ PlanEntryPriority, PlanEntryStatus, PromptCapabilities, PromptRequest, PromptResponse,
89
+ ReadTextFileRequest, ReleaseTerminalRequest, RequestId, RequestPermissionOutcome,
90
+ RequestPermissionRequest, ResumeSessionRequest, ResumeSessionResponse,
91
+ SessionAdditionalDirectoriesCapabilities, SessionCapabilities, SessionCloseCapabilities,
92
+ SessionConfigOption, SessionConfigOptionCategory, SessionConfigSelectOption,
93
+ SessionConfigValueId, SessionDeleteCapabilities, SessionForkCapabilities, SessionId,
94
+ SessionInfo, SessionInfoUpdate, SessionListCapabilities, SessionMode, SessionModeState,
95
+ SessionNotification, SessionResumeCapabilities, SessionUpdate, SetSessionConfigOptionRequest,
96
+ SetSessionConfigOptionResponse, SetSessionModeRequest, SetSessionModeResponse, StopReason,
97
+ Terminal, TerminalOutputRequest, ToolCall, ToolCallContent, ToolCallLocation, ToolCallStatus,
98
+ ToolCallUpdate, ToolCallUpdateFields, ToolKind, UnstructuredCommandInput, UsageUpdate,
99
+ WaitForTerminalExitRequest, WriteTextFileRequest,
100
};
101
use agent_client_protocol::{Agent, ByteStreams, Client, ConnectionTo, Responder};
102
use onde::inference::{ChatEngine, GgufModelConfig};
@@ -649,6 +650,36 @@ fn history_message_text(message: &serde_json::Value) -> String {
650
}
651
}
652
653
+/// The images and audio clips a saved user message carries, as the ACP
654
+/// content blocks the client sent them in. History keeps them as OpenAI
655
+/// content parts (see `backend::user_message`): an image as a base64 `data:`
656
+/// URL and a clip as `input_audio` with a bare format name. Anything that does
657
+/// not turn back into a block (an image by remote URL, say) is left out.
658
+fn history_message_attachments(message: &serde_json::Value) -> Vec<ContentBlock> {
659
+ let Some(parts) = message["content"].as_array() else {
660
+ return Vec::new();
661
+ };
662
+ parts
663
+ .iter()
664
+ .filter_map(|part| match part["type"].as_str()? {
665
+ "image_url" => {
666
+ let url = part["image_url"]["url"].as_str()?;
667
+ let (mime_type, data) = url.strip_prefix("data:")?.split_once(";base64,")?;
668
+ Some(ContentBlock::Image(ImageContent::new(data, mime_type)))
669
+ }
670
+ "input_audio" => {
671
+ let data = part["input_audio"]["data"].as_str()?;
672
+ let mime_type = match part["input_audio"]["format"].as_str()? {
673
+ "mp3" => "audio/mpeg".to_string(),
674
+ format => format!("audio/{format}"),
675
+ };
676
+ Some(ContentBlock::Audio(AudioContent::new(data, mime_type)))
677
+ }
678
+ _ => None,
679
+ })
680
+ .collect()
681
+}
682
+
683
/// How many sessions one `session/list` reply carries. A client asks for the
684
/// rest with the `nextCursor` of the page before.
685
const SESSION_LIST_PAGE_SIZE: usize = 100;
@@ -766,18 +797,34 @@ fn history_replay_updates(history: &[serde_json::Value]) -> Vec<SessionUpdate> {
797
// Seeded context and project instructions were never on screen.
798
"system" => {}
799
"user" => {
800
+ // The text and the attachments were one message when they
801
+ // were sent, so they replay under one id.
802
+ let message_id = new_message_id();
803
let text = history_message_text(message);
804
if !text.trim().is_empty() {
805
updates.push(SessionUpdate::UserMessageChunk(
772
- ContentChunk::new(ContentBlock::from(text)).message_id(new_message_id()),
806
+ ContentChunk::new(ContentBlock::from(text)).message_id(message_id.clone()),
807
+ ));
808
+ }
809
+ for attachment in history_message_attachments(message) {
810
+ updates.push(SessionUpdate::UserMessageChunk(
811
+ ContentChunk::new(attachment).message_id(message_id.clone()),
812
));
813
}
814
}
815
"assistant" => {
777
- let (_think, visible) = chat::strip_think_blocks(&history_message_text(message));
816
+ // Reasoning went out as thought chunks under the reply's id
817
+ // while the turn ran; it comes back the same way.
818
+ let message_id = new_message_id();
819
+ let (think, visible) = chat::strip_think_blocks(&history_message_text(message));
820
+ if !think.trim().is_empty() {
821
+ updates.push(SessionUpdate::AgentThoughtChunk(
822
+ ContentChunk::new(ContentBlock::from(think)).message_id(message_id.clone()),
823
+ ));
824
+ }
825
if !visible.trim().is_empty() {
826
updates.push(SessionUpdate::AgentMessageChunk(
780
- ContentChunk::new(ContentBlock::from(visible)).message_id(new_message_id()),
827
+ ContentChunk::new(ContentBlock::from(visible)).message_id(message_id),
828
));
829
}
830
for call in message["tool_calls"].as_array().into_iter().flatten() {
@@ -7729,7 +7776,7 @@ mod tests {
7776
let updates = history_replay_updates(&history);
7777
7778
// The system message seeded the model; it was never on screen.
7732
- assert_eq!(updates.len(), 4, "{updates:#?}");
7779
+ assert_eq!(updates.len(), 5, "{updates:#?}");
7780
7781
match &updates[0] {
7782
SessionUpdate::UserMessageChunk(chunk) => {
@@ -7737,16 +7784,23 @@ mod tests {
7784
}
7785
other => panic!("expected the user message first, got {other:?}"),
7786
}
7787
+ // Reasoning comes back as a thought, apart from the reply, exactly
7788
+ // as it went out live.
7789
match &updates[1] {
7790
+ SessionUpdate::AgentThoughtChunk(chunk) => {
7791
+ assert!(format!("{:?}", chunk.content).contains("which file?"));
7792
+ }
7793
+ other => panic!("expected the reasoning, got {other:?}"),
7794
+ }
7795
+ match &updates[2] {
7796
SessionUpdate::AgentMessageChunk(chunk) => {
7797
let rendered = format!("{:?}", chunk.content);
7798
assert!(rendered.contains("Reading it now."));
7744
- // Reasoning stays hidden on replay, exactly as it did live.
7799
assert!(!rendered.contains("which file?"));
7800
}
7801
other => panic!("expected the assistant reply, got {other:?}"),
7802
}
7749
- match &updates[2] {
7803
+ match &updates[3] {
7804
SessionUpdate::ToolCall(call) => {
7805
assert_eq!(call.title, format!("read_file · {changelog_path_text}"));
7806
assert_eq!(call.kind, ToolKind::Read);
@@ -7769,7 +7823,7 @@ mod tests {
7823
}
7824
other => panic!("expected the tool call, got {other:?}"),
7825
}
7772
- assert!(matches!(updates[3], SessionUpdate::AgentMessageChunk(_)));
7826
+ assert!(matches!(updates[4], SessionUpdate::AgentMessageChunk(_)));
7827
7828
// Each replayed message has an id of its own.
7829
let ids: Vec<_> = updates
@@ -7875,6 +7929,70 @@ mod tests {
7929
assert!(history_replay_updates(&history).is_empty());
7930
}
7931
7932
+ #[test]
7933
+ fn history_replay_sends_reasoning_as_a_thought_under_the_reply_id() {
7934
+ let history = vec![serde_json::json!({
7935
+ "role": "assistant",
7936
+ "content": "<think>check the lockfile first</think>The lockfile is stale.",
7937
+ })];
7938
+ match history_replay_updates(&history).as_slice() {
7939
+ [
7940
+ SessionUpdate::AgentThoughtChunk(thought),
7941
+ SessionUpdate::AgentMessageChunk(reply),
7942
+ ] => {
7943
+ assert!(
7944
+ matches!(&thought.content, ContentBlock::Text(t) if t.text.contains("check the lockfile first"))
7945
+ );
7946
+ assert!(
7947
+ matches!(&reply.content, ContentBlock::Text(t) if t.text == "The lockfile is stale.")
7948
+ );
7949
+ assert!(thought.message_id.is_some());
7950
+ assert_eq!(thought.message_id, reply.message_id);
7951
+ }
7952
+ other => panic!("expected a thought and a reply, got {other:?}"),
7953
+ }
7954
+ }
7955
+
7956
+ #[test]
7957
+ fn history_replay_gives_back_the_attachments_of_a_user_message() {
7958
+ let history = vec![serde_json::json!({
7959
+ "role": "user",
7960
+ "content": [
7961
+ { "type": "text", "text": "what is in these?" },
7962
+ { "type": "image_url", "image_url": { "url": "data:image/png;base64,AAAA" } },
7963
+ { "type": "image_url", "image_url": { "url": "https://example.com/cat.png" } },
7964
+ { "type": "input_audio", "input_audio": { "data": "BBBB", "format": "mp3" } },
7965
+ { "type": "input_audio", "input_audio": { "data": "CCCC", "format": "wav" } },
7966
+ ],
7967
+ })];
7968
+ match history_replay_updates(&history).as_slice() {
7969
+ [
7970
+ SessionUpdate::UserMessageChunk(text),
7971
+ SessionUpdate::UserMessageChunk(image),
7972
+ SessionUpdate::UserMessageChunk(mp3),
7973
+ SessionUpdate::UserMessageChunk(wav),
7974
+ ] => {
7975
+ assert!(
7976
+ matches!(&text.content, ContentBlock::Text(t) if t.text == "what is in these?")
7977
+ );
7978
+ assert!(
7979
+ matches!(&image.content, ContentBlock::Image(i) if i.data == "AAAA" && i.mime_type == "image/png")
7980
+ );
7981
+ assert!(
7982
+ matches!(&mp3.content, ContentBlock::Audio(a) if a.data == "BBBB" && a.mime_type == "audio/mpeg")
7983
+ );
7984
+ assert!(
7985
+ matches!(&wav.content, ContentBlock::Audio(a) if a.data == "CCCC" && a.mime_type == "audio/wav")
7986
+ );
7987
+ assert!(text.message_id.is_some());
7988
+ for chunk in [image, mp3, wav] {
7989
+ assert_eq!(chunk.message_id, text.message_id);
7990
+ }
7991
+ }
7992
+ other => panic!("expected the text and three attachments, got {other:?}"),
7993
+ }
7994
+ }
7995
+
7996
fn listed(id: &str, modified_secs: u64) -> session_store::SessionEntry {
7997
session_store::SessionEntry {
7998
id: id.to_string(),