@sigitcodereview / sigit / commits / 574e283

feat(acp): accept audio in prompts

Advertise promptCapabilities.audio and send an attached clip to the model as an OpenAI input_audio part. It follows the image path: the capability is fixed for the connection, and each prompt decides whether the active model gets the clip. No siGit Code Cloud tier takes audio, since onde-cloud drops input_audio parts when it parses a request, and on-device models are text only. So model_accepts_audio says yes only for a user's own endpoint, the same pass-through rule images use there. Everywhere else the clip is replaced by a note to the model and the editor is told it was left out. Audio already in history is stripped per request after a switch to a model without audio, and the user hears about it once. images_for_turn is now a thin wrapper over attachments_for_turn, which audio_for_turn shares. without_images likewise sits on a without_parts helper that drops one part type and keeps the others, so a message with both an image and a clip keeps the image for a model that reads images. Fixes #143 Co-Authored-By: siGit Code v1.6.4 <noreply@sigit.si> Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

Seto Elkahfi committed Oct 8, 2026 at 07:15 UTC 574e28356a005ff1a7a5e2fc7f3f634edb53fb39

6 files changed +746 -78

.agents/AGENTS.md

@@ -163,6 +163,11 @@ feeds results back. Neither the loop nor ACP/TUI surfaces depend on a concrete b

163 when its model cannot read them, which is what makes a mid-thread switch to a text-only tier

164 safe. Read history text through `backend::message_text`, never `content.as_str()`.

165 `IMAGE_TIERS` mirrors onde-cloud's `IMAGE_CANDIDATES`; update both together.

166 + Audio attachments take the same path (`promptCapabilities.audio`, `accepts_audio`,

167 + `audio_for_turn`, `without_audio`), sent as OpenAI `input_audio` parts. No cloud tier

168 + takes audio, because onde-cloud drops `input_audio` parts when it parses a request, so

169 + `provider::model_accepts_audio` says no for every `onde-*` id and yes for a user's own

170 + endpoint. If onde-cloud ever routes audio, give it a tier table like `IMAGE_TIERS`.

171 - **`src/provider.rs`** — decides *which* backend serves inference. Resolution order, first match

172 wins: (1) override via `OPENAI_BASE_URL`+`OPENAI_API_KEY` or active profile in

173 `~/.config/sigit/providers.toml`; (2) siGit Code Cloud when logged in; (3) on-device.

CHANGELOG.md

@@ -9,6 +9,13 @@

9 contents in the prompt instead of sending a link to it. Before, a client

10 that follows the spec only sent `resource_link`, and siGit Code read the

11 file from disk, missing whatever the user had not saved yet (#141).

12 +- **A prompt can carry audio.** siGit Code now advertises

13 + `promptCapabilities.audio` and sends an attached clip to the model as an

14 + OpenAI `input_audio` part. It works the way image attachments do: a model on

15 + an endpoint you configure yourself gets the clip and decides for itself,

16 + while siGit Code Cloud tiers and on-device models, none of which take audio

17 + today, get a note in its place and the editor says the clip was left out

18 + (#143).

20 ### Changed

src/backend.rs

+215 -36

@@ -297,19 +297,30 @@ pub fn message_text(message: &serde_json::Value) -> String {

297 }

298 }

299

300 -/// How many images a history message carries as `image_url` content parts.

301 -pub fn message_image_count(message: &serde_json::Value) -> usize {

300 +/// How many content parts of `part_type` a history message carries.

301 +fn message_part_count(message: &serde_json::Value, part_type: &str) -> usize {

302 message["content"]

303 .as_array()

304 .map(|parts| {

305 parts

306 .iter()

307 - .filter(|part| part["type"] == "image_url")

307 + .filter(|part| part["type"] == part_type)

308 .count()

309 })

310 .unwrap_or(0)

311 }

312

313 +/// How many images a history message carries as `image_url` content parts.

314 +pub fn message_image_count(message: &serde_json::Value) -> usize {

315 + message_part_count(message, "image_url")

316 +}

317 +

318 +/// How many audio clips a history message carries as `input_audio` content

319 +/// parts.

320 +pub fn message_audio_count(message: &serde_json::Value) -> usize {

321 + message_part_count(message, "input_audio")

322 +}

323 +

324 /// A sink for streaming assistant text deltas to the UI as they are produced.

325 ///

326 /// When a caller passes `Some(sink)`, a streaming-capable backend forwards each

@@ -327,17 +338,49 @@ pub struct ImageInput {

338 pub data: String,

339 }

340

341 +/// An audio clip attached to a user message: base64 data and its media type,

342 +/// as an ACP client sends it.

343 +#[derive(Debug, Clone, PartialEq, Eq)]

344 +pub struct AudioInput {

345 + pub mime_type: String,

346 + pub data: String,

347 +}

348 +

349 +impl AudioInput {

350 + /// The `format` OpenAI's `input_audio` part expects (`wav`, `mp3`, ...),

351 + /// from the MIME type an ACP client sends (`audio/wav`, `audio/mpeg`, ...).

352 + fn format(&self) -> String {

353 + let subtype = self

354 + .mime_type

355 + .split(';')

356 + .next()

357 + .unwrap_or_default()

358 + .trim()

359 + .to_lowercase();

360 + let subtype = subtype.strip_prefix("audio/").unwrap_or(&subtype);

361 + match subtype.strip_prefix("x-").unwrap_or(subtype) {

362 + "mpeg" | "mp3" => "mp3".to_string(),

363 + "wave" | "wav" | "vnd.wave" => "wav".to_string(),

364 + other => other.to_string(),

365 + }

366 + }

367 +}

368 +

369 /// What a model that cannot read images is shown in place of one.

370 const IMAGE_OMITTED_NOTE: &str = "[image omitted: this model cannot read images]";

371

372 +/// What a model that cannot take audio is shown in place of a clip.

373 +const AUDIO_OMITTED_NOTE: &str = "[audio omitted: this model cannot take audio]";

374 +

375 /// A user message in history form. Plain text keeps the string `content` every

334 -/// other message uses; with images it becomes OpenAI's content-part array,

335 -/// text first, each image as a base64 `data:` URL.

336 -fn user_message(text: &str, images: &[ImageInput]) -> serde_json::Value {

337 - if images.is_empty() {

376 +/// other message uses; with attachments it becomes OpenAI's content-part

377 +/// array, text first, each image as a base64 `data:` URL and each audio clip

378 +/// as an `input_audio` part.

379 +fn user_message(text: &str, images: &[ImageInput], audio: &[AudioInput]) -> serde_json::Value {

380 + if images.is_empty() && audio.is_empty() {

381 return serde_json::json!({ "role": "user", "content": text });

382 }

340 - let mut parts = Vec::with_capacity(images.len() + 1);

383 + let mut parts = Vec::with_capacity(images.len() + audio.len() + 1);

384 if !text.is_empty() {

385 parts.push(serde_json::json!({ "type": "text", "text": text }));

386 }

@@ -349,37 +392,75 @@ fn user_message(text: &str, images: &[ImageInput]) -> serde_json::Value {

392 },

393 }));

394 }

395 + for clip in audio {

396 + parts.push(serde_json::json!({

397 + "type": "input_audio",

398 + "input_audio": { "data": clip.data, "format": clip.format() },

399 + }));

400 + }

401 serde_json::json!({ "role": "user", "content": parts })

402 }

403

355 -/// `history` as a text-only model has to receive it: a message that carries

356 -/// images is flattened back to a string, with a note where each image was.

404 +/// `history` with every content part of `part_type` replaced by `note`, for a

405 +/// model that cannot take that kind of attachment. A message left with only

406 +/// text is flattened back to a plain string.

407 ///

358 -/// History itself keeps the images. A thread can move to a model that reads

359 -/// them (or back to one), so what was attached is not thrown away just because

360 -/// the model active right now cannot use it.

361 -fn without_images(history: &[serde_json::Value]) -> Vec<serde_json::Value> {

408 +/// History itself keeps the attachments. A thread can move to a model that

409 +/// reads them (or back to one), so what was attached is not thrown away just

410 +/// because the model active right now cannot use it.

411 +fn without_parts(

412 + history: &[serde_json::Value],

413 + part_type: &str,

414 + note: &str,

415 +) -> Vec<serde_json::Value> {

416 history

417 .iter()

418 .map(|message| {

365 - let images = message_image_count(message);

366 - if images == 0 {

419 + let Some(parts) = message["content"].as_array() else {

420 + return message.clone();

421 + };

422 + if !parts.iter().any(|part| part["type"] == part_type) {

423 return message.clone();

424 }

369 - let mut text = message_text(message);

370 - for _ in 0..images {

371 - if !text.is_empty() {

372 - text.push('\n');

425 + let mut kept: Vec<serde_json::Value> = Vec::with_capacity(parts.len());

426 + let mut notes = Vec::new();

427 + for part in parts {

428 + if part["type"] == part_type {

429 + notes.push(serde_json::json!({ "type": "text", "text": note }));

430 + } else {

431 + kept.push(part.clone());

432 }

374 - text.push_str(IMAGE_OMITTED_NOTE);

433 }

376 - let mut flattened = message.clone();

377 - flattened["content"] = serde_json::Value::String(text);

378 - flattened

434 + kept.extend(notes);

435 + let mut stripped = message.clone();

436 + stripped["content"] = if kept.iter().all(|part| part["type"] == "text") {

437 + let text = kept

438 + .iter()

439 + .filter_map(|part| part["text"].as_str())

440 + .filter(|text| !text.is_empty())

441 + .collect::<Vec<_>>()

442 + .join("\n");

443 + serde_json::Value::String(text)

444 + } else {

445 + serde_json::Value::Array(kept)

446 + };

447 + stripped

448 })

449 .collect()

450 }

451

452 +/// `history` as a model that cannot read images has to receive it, with a

453 +/// note where each image was.

454 +fn without_images(history: &[serde_json::Value]) -> Vec<serde_json::Value> {

455 + without_parts(history, "image_url", IMAGE_OMITTED_NOTE)

456 +}

457 +

458 +/// `history` as a model that cannot take audio has to receive it, with a note

459 +/// where each clip was.

460 +fn without_audio(history: &[serde_json::Value]) -> Vec<serde_json::Value> {

461 + without_parts(history, "input_audio", AUDIO_OMITTED_NOTE)

462 +}

463 +

464 // ── The trait ───────────────────────────────────────────────────────────────────

465

466 /// A swappable inference backend driving siGit Code's agent loop.

@@ -418,19 +499,21 @@ pub trait InferenceBackend: Send + Sync {

499 /// request in the session.

500 async fn record_cancelled_tool_results(&self, results: Vec<ToolResult>);

501

421 - /// Start a turn from a user message that carries images.

502 + /// Start a turn from a user message that carries images or audio.

503 ///

423 - /// The default drops the images and sends the text, which is right for a

424 - /// backend that cannot read them. Callers check [`Self::accepts_images`]

425 - /// first so the user can be told, instead of the image vanishing.

426 - async fn send_message_with_images(

504 + /// The default drops the attachments and sends the text, which is right

505 + /// for a backend that cannot read them. Callers check

506 + /// [`Self::accepts_images`] and [`Self::accepts_audio`] first so the user

507 + /// can be told, instead of the attachment vanishing.

508 + async fn send_message_with_attachments(

509 &self,

510 text: &str,

511 images: &[ImageInput],

512 + audio: &[AudioInput],

513 tools: &[ToolSpec],

514 sink: Option<&TokenSink>,

515 ) -> Result<TurnResult, BackendError> {

433 - let _ = images;

516 + let _ = (images, audio);

517 self.send_message_with_tools(text, tools, sink).await

518 }

519

@@ -440,6 +523,12 @@ pub trait InferenceBackend: Send + Sync {

523 false

524 }

525

526 + /// Whether the model behind this backend takes audio. On-device models do

527 + /// not; a remote one answers from its model id.

528 + fn accepts_audio(&self) -> bool {

529 + false

530 + }

531 +

532 /// Whether inference runs over the network (a configured provider) rather

533 /// than on-device. Drives UI labelling so the displayed model can't claim a

534 /// local model while requests actually go to the cloud.

@@ -837,6 +926,9 @@ pub struct OpenAiBackend {

926 /// When it does not, images already in `history` are left out of each

927 /// request rather than sent to an endpoint that would refuse them.

928 accepts_images: bool,

929 + /// Whether `model` takes audio (see `provider::model_accepts_audio`), with

930 + /// the same effect on audio already in `history`.

931 + accepts_audio: bool,

932 }

933

934 impl OpenAiBackend {

@@ -858,6 +950,7 @@ impl OpenAiBackend {

950 base_url: base_url.into(),

951 api_key: api_key.into(),

952 accepts_images: crate::provider::model_accepts_images(&model),

953 + accepts_audio: crate::provider::model_accepts_audio(&model),

954 model,

955 http: reqwest::Client::new(),

956 history: Mutex::new(history),

@@ -946,10 +1039,15 @@ impl OpenAiBackend {

1039

1040 let messages = {

1041 let history = self.history.lock().await;

949 - if self.accepts_images {

1042 + let messages = if self.accepts_images {

1043 history.clone()

1044 } else {

1045 without_images(&history)

1046 + };

1047 + if self.accepts_audio {

1048 + messages

1049 + } else {

1050 + without_audio(&messages)

1051 }

1052 };

1053 let mut body = serde_json::json!({

@@ -1544,14 +1642,18 @@ impl InferenceBackend for OpenAiBackend {

1642 }

1643 }

1644

1547 - async fn send_message_with_images(

1645 + async fn send_message_with_attachments(

1646 &self,

1647 text: &str,

1648 images: &[ImageInput],

1649 + audio: &[AudioInput],

1650 tools: &[ToolSpec],

1651 sink: Option<&TokenSink>,

1652 ) -> Result<TurnResult, BackendError> {

1554 - self.history.lock().await.push(user_message(text, images));

1653 + self.history

1654 + .lock()

1655 + .await

1656 + .push(user_message(text, images, audio));

1657 self.complete(tools, true, sink).await

1658 }

1659

@@ -1559,6 +1661,10 @@ impl InferenceBackend for OpenAiBackend {

1661 self.accepts_images

1662 }

1663

1664 + fn accepts_audio(&self) -> bool {

1665 + self.accepts_audio

1666 + }

1667 +

1668 fn is_remote(&self) -> bool {

1669 true

1670 }

@@ -1573,6 +1679,7 @@ impl InferenceBackend for OpenAiBackend {

1679 http: self.http.clone(),

1680 history: Mutex::new(Vec::new()),

1681 accepts_images: self.accepts_images,

1682 + accepts_audio: self.accepts_audio,

1683 }))

1684 }

1685

@@ -2344,7 +2451,7 @@ mod tests {

2451 #[test]

2452 fn user_message_uses_content_parts_only_when_there_is_an_image() {

2453 assert_eq!(

2347 - user_message("hello", &[]),

2454 + user_message("hello", &[], &[]),

2455 serde_json::json!({ "role": "user", "content": "hello" })

2456 );

2457

@@ -2353,7 +2460,7 @@ mod tests {

2460 data: "AAAA".to_string(),

2461 };

2462 assert_eq!(

2356 - user_message("what is this?", std::slice::from_ref(&image)),

2463 + user_message("what is this?", std::slice::from_ref(&image), &[]),

2464 serde_json::json!({

2465 "role": "user",

2466 "content": [

@@ -2363,10 +2470,74 @@ mod tests {

2470 })

2471 );

2472 // An image with no text sends no empty text part.

2366 - let only_image = user_message("", &[image]);

2473 + let only_image = user_message("", &[image], &[]);

2474 assert_eq!(only_image["content"].as_array().unwrap().len(), 1);

2475 }

2476

2477 + #[test]

2478 + fn audio_is_sent_as_an_input_audio_part() {

2479 + let clip = AudioInput {

2480 + mime_type: "audio/mpeg".to_string(),

2481 + data: "BBBB".to_string(),

2482 + };

2483 + assert_eq!(

2484 + user_message("transcribe this", &[], &[clip]),

2485 + serde_json::json!({

2486 + "role": "user",

2487 + "content": [

2488 + { "type": "text", "text": "transcribe this" },

2489 + { "type": "input_audio", "input_audio": { "data": "BBBB", "format": "mp3" } },

2490 + ],

2491 + })

2492 + );

2493 + }

2494 +

2495 + #[test]

2496 + fn audio_format_follows_the_mime_type() {

2497 + let format = |mime: &str| {

2498 + AudioInput {

2499 + mime_type: mime.to_string(),

2500 + data: String::new(),

2501 + }

2502 + .format()

2503 + };

2504 + assert_eq!(format("audio/wav"), "wav");

2505 + assert_eq!(format("audio/x-wav"), "wav");

2506 + assert_eq!(format("audio/wave"), "wav");

2507 + assert_eq!(format("audio/mpeg"), "mp3");

2508 + assert_eq!(format("Audio/MP3; codecs=mp3"), "mp3");

2509 + assert_eq!(format("audio/flac"), "flac");

2510 + }

2511 +

2512 + #[test]

2513 + fn a_model_without_audio_gets_a_note_and_keeps_the_image() {

2514 + let history = vec![serde_json::json!({

2515 + "role": "user",

2516 + "content": [

2517 + { "type": "text", "text": "what is this?" },

2518 + { "type": "image_url", "image_url": { "url": "data:image/png;base64,AAAA" } },

2519 + { "type": "input_audio", "input_audio": { "data": "BBBB", "format": "wav" } },

2520 + ],

2521 + })];

2522 +

2523 + let sent = without_audio(&history);

2524 + assert_eq!(

2525 + sent[0]["content"],

2526 + serde_json::json!([

2527 + { "type": "text", "text": "what is this?" },

2528 + { "type": "image_url", "image_url": { "url": "data:image/png;base64,AAAA" } },

2529 + { "type": "text", "text": AUDIO_OMITTED_NOTE },

2530 + ])

2531 + );

2532 + // With neither, the message is plain text again.

2533 + assert_eq!(

2534 + without_images(&sent)[0]["content"],

2535 + format!("what is this?\n{AUDIO_OMITTED_NOTE}\n{IMAGE_OMITTED_NOTE}")

2536 + );

2537 + // History keeps the clip for a model that can take it later.

2538 + assert_eq!(message_audio_count(&history[0]), 1);

2539 + }

2540 +

2541 #[test]

2542 fn a_text_only_model_is_sent_a_note_in_place_of_each_image() {

2543 let history = vec![

@@ -2396,6 +2567,14 @@ mod tests {

2567 assert!(image_tier.accepts_images());

2568 }

2569

2570 + #[test]

2571 + fn a_remote_backend_takes_audio_only_on_the_users_own_endpoint() {

2572 + let own_endpoint = OpenAiBackend::new("http://localhost", "", "gpt-4o-audio", None);

2573 + assert!(own_endpoint.accepts_audio());

2574 + let cloud_tier = OpenAiBackend::new("http://localhost", "", "onde-large", None);

2575 + assert!(!cloud_tier.accepts_audio());

2576 + }

2577 +

2578 #[test]

2579 fn message_text_reads_both_content_shapes() {

2580 assert_eq!(

src/main.rs

+191 -42

@@ -94,8 +94,8 @@ use agent_client_protocol::{Agent, ByteStreams, Client, ConnectionTo, Responder}

94 use onde::inference::{ChatEngine, GgufModelConfig};

96 use crate::backend::{

97 - ImageInput, InferenceBackend, LocalBackend, OpenAiBackend, ToolResult as BackendToolResult,

98 - ToolSpec, TurnResult,

97 + AudioInput, ImageInput, InferenceBackend, LocalBackend, OpenAiBackend,

98 + ToolResult as BackendToolResult, ToolSpec, TurnResult,

99 };

100 use std::path::PathBuf;

101 use std::sync::atomic::{AtomicBool, Ordering};

@@ -1706,70 +1706,154 @@ impl SiGitAgent {

1706 /// the note about a model switch is shown once and not on every prompt.

1707 static IMAGE_GAP_NOTED: std::sync::Mutex<Vec<(String, String)>> = std::sync::Mutex::new(Vec::new());

1708

1709 -/// Decide what happens to a prompt's images for the model about to answer.

1709 +/// The same for earlier audio clips.

1710 +static AUDIO_GAP_NOTED: std::sync::Mutex<Vec<(String, String)>> = std::sync::Mutex::new(Vec::new());

1711 +

1712 +/// The wording [`attachments_for_turn`] uses for one kind of attachment.

1713 +struct AttachmentKind {

1714 + /// "image"

1715 + noun: &'static str,

1716 + /// "images"

1717 + nouns: &'static str,

1718 + /// "Image", for the notice headings.

1719 + heading: &'static str,

1720 + /// "Images", for the notice headings.

1721 + headings: &'static str,

1722 + /// "read images"

1723 + ability: &'static str,

1724 + /// Where the user can find a model that can.

1725 + hint: &'static str,

1726 + /// Session and model pairs already told about earlier attachments.

1727 + noted: &'static std::sync::Mutex<Vec<(String, String)>>,

1728 +}

1729 +

1730 +const IMAGES: AttachmentKind = AttachmentKind {

1731 + noun: "image",

1732 + nouns: "images",

1733 + heading: "Image",

1734 + headings: "images",

1735 + ability: "read images",

1736 + hint: "Models marked \"reads images\" in the model picker can.",

1737 + noted: &IMAGE_GAP_NOTED,

1738 +};

1739 +

1740 +const AUDIO: AttachmentKind = AttachmentKind {

1741 + noun: "audio clip",

1742 + nouns: "audio clips",

1743 + heading: "Audio",

1744 + headings: "audio clips",

1745 + ability: "take audio",

1746 + hint: "No siGit Code Cloud or on-device model can; a model on an endpoint you \

1747 + configure yourself can, if it accepts audio input.",

1748 + noted: &AUDIO_GAP_NOTED,

1749 +};

1750 +

1751 +/// Decide what happens to a prompt's attachments of one `kind` for the model

1752 +/// about to answer.

1753 ///

1711 -/// The editor can attach an image whatever model is selected, because ACP

1712 -/// fixes `promptCapabilities.image` for the whole connection. Returns the text

1713 -/// and images to send, plus a note for the user when something is being left

1714 -/// out:

1754 +/// The editor can attach images and audio whatever model is selected, because

1755 +/// ACP fixes `promptCapabilities` for the whole connection. Returns the text

1756 +/// and attachments to send, plus a note for the user when something is being

1757 +/// left out:

1758 ///

1716 -/// - the model cannot read images and the prompt has some: they are dropped,

1759 +/// - the model cannot take them and the prompt has some: they are dropped,

1760 /// the user is told, and the model is told too, so it does not answer as if

1761 /// it had seen them;

1719 -/// - the model cannot read images and the thread has some from before a model

1720 -/// switch (`earlier_images`): the user is told once per session and model.

1721 -fn images_for_turn(

1762 +/// - the model cannot take them and the thread has some from before a model

1763 +/// switch (`earlier`): the user is told once per session and model.

1764 +fn attachments_for_turn<T>(

1765 mut user_text: String,

1723 - images: Vec<ImageInput>,

1724 - accepts_images: bool,

1766 + attachments: Vec<T>,

1767 + accepts: bool,

1768 model_name: &str,

1726 - earlier_images: usize,

1769 + earlier: usize,

1770 session_id: &str,

1728 -) -> (String, Vec<ImageInput>, Option<String>) {

1729 - if accepts_images {

1730 - return (user_text, images, None);

1771 + kind: &AttachmentKind,

1772 +) -> (String, Vec<T>, Option<String>) {

1773 + if accepts {

1774 + return (user_text, attachments, None);

1775 }

1776

1733 - const HINT: &str = "Models marked \"reads images\" in the model picker can.";

1734 - let plural = |count: usize| if count == 1 { "image" } else { "images" };

1777 + let nouns = |count: usize| if count == 1 { kind.noun } else { kind.nouns };

1778 + let ability = kind.ability;

1779 + let hint = kind.hint;

1780

1736 - if !images.is_empty() {

1737 - let count = images.len();

1781 + if !attachments.is_empty() {

1782 + let count = attachments.len();

1783 if !user_text.is_empty() {

1784 user_text.push_str("\n\n");

1785 }

1786 user_text.push_str(&format!(

1742 - "[The user attached {count} {}. This model cannot read images, so it was not \

1787 + "[The user attached {count} {}. This model cannot {ability}, so it was not \

1788 included. Say so if the request depends on it.]",

1744 - plural(count)

1789 + nouns(count)

1790 ));

1791 let notice = format!(

1747 - "> **Image not sent.** {model_name} cannot read images, so the {count} attached {} \

1748 - left out of this message. {HINT}",

1749 - if count == 1 {

1750 - "image was"

1751 - } else {

1752 - "images were"

1753 - }

1792 + "> **{} not sent.** {model_name} cannot {ability}, so the {count} attached {} {} \

1793 + left out of this message. {hint}",

1794 + kind.heading,

1795 + nouns(count),

1796 + if count == 1 { "was" } else { "were" }

1797 );

1798 return (user_text, Vec::new(), Some(notice));

1799 }

1800

1758 - if earlier_images > 0 {

1801 + if earlier > 0 {

1802 let key = (session_id.to_string(), model_name.to_string());

1760 - let mut noted = IMAGE_GAP_NOTED.lock().unwrap_or_else(|e| e.into_inner());

1803 + let mut noted = kind.noted.lock().unwrap_or_else(|e| e.into_inner());

1804 if !noted.contains(&key) {

1805 noted.push(key);

1806 let notice = format!(

1764 - "> **Earlier images are hidden.** {model_name} cannot read images, so the \

1765 - {earlier_images} {} attached earlier in this thread {} not sent to it. {HINT}",

1766 - plural(earlier_images),

1767 - if earlier_images == 1 { "is" } else { "are" }

1807 + "> **Earlier {} are hidden.** {model_name} cannot {ability}, so the \

1808 + {earlier} {} attached earlier in this thread {} not sent to it. {hint}",

1809 + kind.headings,

1810 + nouns(earlier),

1811 + if earlier == 1 { "is" } else { "are" }

1812 );

1769 - return (user_text, images, Some(notice));

1813 + return (user_text, attachments, Some(notice));

1814 }

1815 }

1772 - (user_text, images, None)

1816 + (user_text, attachments, None)

1817 +}

1818 +

1819 +/// [`attachments_for_turn`] for a prompt's images.

1820 +fn images_for_turn(

1821 + user_text: String,

1822 + images: Vec<ImageInput>,

1823 + accepts_images: bool,

1824 + model_name: &str,

1825 + earlier_images: usize,

1826 + session_id: &str,

1827 +) -> (String, Vec<ImageInput>, Option<String>) {

1828 + attachments_for_turn(

1829 + user_text,

1830 + images,

1831 + accepts_images,

1832 + model_name,

1833 + earlier_images,

1834 + session_id,

1835 + &IMAGES,

1836 + )

1837 +}

1838 +

1839 +/// [`attachments_for_turn`] for a prompt's audio clips.

1840 +fn audio_for_turn(

1841 + user_text: String,

1842 + audio: Vec<AudioInput>,

1843 + accepts_audio: bool,

1844 + model_name: &str,

1845 + earlier_audio: usize,

1846 + session_id: &str,

1847 +) -> (String, Vec<AudioInput>, Option<String>) {

1848 + attachments_for_turn(

1849 + user_text,

1850 + audio,

1851 + accepts_audio,

1852 + model_name,

1853 + earlier_audio,

1854 + session_id,

1855 + &AUDIO,

1856 + )

1857 }

1858

1859 /// The servers out of a session request's `mcpServers` that siGit Code can

@@ -1905,8 +1989,15 @@ impl SiGitAgent {

1989 // contents, e.g. an `@file` mention with unsaved edits.

1990 // Without it a client sends only a `resource_link`, and

1991 // sigit reads the file from disk.

1992 + //

1993 + // Audio works the way images do. No cloud tier or on-device

1994 + // model takes it today, so in practice only a model on the

1995 + // user's own endpoint receives it (see `audio_for_turn`).

1996 .prompt_capabilities(

1909 - PromptCapabilities::new().image(true).embedded_context(true),

1997 + PromptCapabilities::new()

1998 + .image(true)

1999 + .audio(true)

2000 + .embedded_context(true),

2001 )

2002 // Clients only pass HTTP MCP servers in `mcpServers` to an

2003 // agent that says it can reach them. SSE stays off: the

@@ -2812,6 +2903,7 @@ impl SiGitAgent {

2903

2904 let mut parts: Vec<String> = Vec::new();

2905 let mut images: Vec<ImageInput> = Vec::new();

2906 + let mut audio: Vec<AudioInput> = Vec::new();

2907

2908 for block in &args.prompt {

2909 match block {

@@ -2824,6 +2916,12 @@ impl SiGitAgent {

2916 data: image.data.clone(),

2917 });

2918 }

2919 + ContentBlock::Audio(clip) => {

2920 + audio.push(AudioInput {

2921 + mime_type: clip.mime_type.clone(),

2922 + data: clip.data.clone(),

2923 + });

2924 + }

2925 ContentBlock::Resource(embedded) => {

2926 // editor inlined the file content already

2927 match &embedded.resource {

@@ -2991,7 +3089,7 @@ impl SiGitAgent {

3089 .map(backend::message_image_count)

3090 .sum()

3091 };

2994 - let (user_text, images, notice) = images_for_turn(

3092 + let (user_text, images, image_notice) = images_for_turn(

3093 user_text,

3094 images,

3095 backend.accepts_images(),

@@ -2999,7 +3097,25 @@ impl SiGitAgent {

3097 earlier_images,

3098 &session_id.to_string(),

3099 );

3002 - if let Some(notice) = notice {

3100 + let earlier_audio = if backend.accepts_audio() {

3101 + 0

3102 + } else {

3103 + backend

3104 + .history_snapshot()

3105 + .await

3106 + .iter()

3107 + .map(backend::message_audio_count)

3108 + .sum()

3109 + };

3110 + let (user_text, audio, audio_notice) = audio_for_turn(

3111 + user_text,

3112 + audio,

3113 + backend.accepts_audio(),

3114 + &model_name,

3115 + earlier_audio,

3116 + &session_id.to_string(),

3117 + );

3118 + for notice in [image_notice, audio_notice].into_iter().flatten() {

3119 self.send_assistant_message(

3120 cx,

3121 session_id.clone(),

@@ -3033,7 +3149,13 @@ impl SiGitAgent {

3149 .drain_turn(

3150 cx,

3151 &session_id,

3036 - backend.send_message_with_images(&user_text, &images, &tools, Some(&sink)),

3152 + backend.send_message_with_attachments(

3153 + &user_text,

3154 + &images,

3155 + &audio,

3156 + &tools,

3157 + Some(&sink),

3158 + ),

3159 &mut sink_rx,

3160 &mut reply,

3161 &cancellation,

@@ -7091,6 +7213,33 @@ mod tests {

7213 );

7214 }

7215

7216 + #[test]

7217 + fn a_model_without_audio_gets_a_note_and_the_user_is_told() {

7218 + let clip = AudioInput {

7219 + mime_type: "audio/wav".to_string(),

7220 + data: "AAAA".to_string(),

7221 + };

7222 + let (text, audio, notice) = audio_for_turn(

7223 + "listen".to_string(),

7224 + vec![clip],

7225 + false,

7226 + "Nova",

7227 + 0,

7228 + "s-audio",

7229 + );

7230 + assert!(audio.is_empty());

7231 + assert!(

7232 + text.starts_with("listen\n\n[The user attached 1 audio clip."),

7233 + "{text}"

7234 + );

7235 + let notice = notice.expect("the user must be told the audio was left out");

7236 + assert!(

7237 + notice.starts_with("> **Audio not sent.** Nova cannot take audio"),

7238 + "{notice}"

7239 + );

7240 + assert!(notice.contains("1 attached audio clip was"), "{notice}");

7241 + }

7242 +

7243 #[test]

7244 fn session_roots_must_be_absolute() {

7245 let absolute = std::env::temp_dir();

src/provider.rs

+18

@@ -79,6 +79,16 @@ pub fn model_accepts_images(model: &str) -> bool {

79 .any(|tier| tier_to_model(tier) == model && tier_accepts_images(tier))

80 }

82 +/// Whether the model id sent on the wire takes audio.

83 +///

84 +/// No cloud tier does: onde-cloud drops `input_audio` parts when it parses a

85 +/// request, so a clip sent to an `onde-*` id would vanish without a word. Any

86 +/// other id is the user's own endpoint and, as with images, gets the audio and

87 +/// decides for itself.

88 +pub fn model_accepts_audio(model: &str) -> bool {

89 + !model.starts_with("onde-")

90 +}

91 +

92 /// Base URL of the siGit Code Cloud inference endpoint. Override with

93 /// `SIGIT_CLOUD_URL` (dev: `http://localhost:8090/v1`).

94 pub fn cloud_base_url() -> String {

@@ -338,4 +348,12 @@ mod tests {

348 assert!(CLOUD_TIERS.contains(tier), "{tier} is not a cloud tier");

349 }

350 }

351 +

352 + #[test]

353 + fn audio_goes_only_to_the_users_own_endpoint() {

354 + for tier in CLOUD_TIERS {

355 + assert!(!model_accepts_audio(&tier_to_model(tier)), "{tier}");

356 + }

357 + assert!(model_accepts_audio("gpt-4o-audio-preview"));

358 + }

359 }

tests/acp_audio.rs new

+310

@@ -0,0 +1,310 @@

1 +//! End-to-end check of audio attachments over ACP (issue #143).

2 +//!

3 +//! Audio works the way images do (see `acp_images.rs`): the capability is

4 +//! advertised for the whole connection and the decision is made per prompt.

5 +//! No cloud tier takes audio (onde-cloud drops `input_audio` parts), so:

6 +//!

7 +//! - a model on the user's own endpoint receives the clip as an OpenAI

8 +//! `input_audio` content part;

9 +//! - an `onde-*` tier is sent text only, with a note in place of the clip,

10 +//! and the user is told the clip was left out.

11 +

12 +use std::collections::VecDeque;

13 +use std::io::{BufRead, BufReader, Read, Write};

14 +use std::net::TcpListener;

15 +use std::process::{Child, ChildStdin, Command, Stdio};

16 +use std::sync::mpsc::{Receiver, channel};

17 +use std::sync::{Arc, Mutex};

18 +use std::time::{Duration, Instant};

19 +

20 +use serde_json::{Value, json};

21 +

22 +const TIMEOUT: Duration = Duration::from_secs(60);

23 +

24 +// ── Scripted OpenAI-compatible endpoint ─────────────────────────────────────

25 +

26 +fn sse_body(events: &[Value]) -> String {

27 + let mut body = String::new();

28 + for event in events {

29 + body.push_str("data: ");

30 + body.push_str(&event.to_string());

31 + body.push_str("\n\n");

32 + }

33 + body.push_str("data: [DONE]\n\n");

34 + body

35 +}

36 +

37 +fn sse_text(text: &str) -> String {

38 + sse_body(&[json!({"choices": [{"delta": {"content": text}}]})])

39 +}

40 +

41 +/// Serves one scripted SSE response per request and records each request body.

42 +struct FakeEndpoint {

43 + port: u16,

44 + requests: Arc<Mutex<Vec<Value>>>,

45 +}

46 +

47 +fn start_fake_endpoint(responses: Vec<String>) -> FakeEndpoint {

48 + let listener = TcpListener::bind("127.0.0.1:0").expect("bind fake endpoint");

49 + let port = listener.local_addr().unwrap().port();

50 + let requests: Arc<Mutex<Vec<Value>>> = Arc::default();

51 + let recorded = Arc::clone(&requests);

52 + let queue = Mutex::new(VecDeque::from(responses));

53 +

54 + std::thread::spawn(move || {

55 + for stream in listener.incoming() {

56 + let Ok(mut stream) = stream else { continue };

57 + let mut reader = BufReader::new(match stream.try_clone() {

58 + Ok(clone) => clone,

59 + Err(_) => continue,

60 + });

61 + let mut content_length = 0usize;

62 + loop {

63 + let mut line = String::new();

64 + if reader.read_line(&mut line).unwrap_or(0) == 0 {

65 + break;

66 + }

67 + let line = line.trim();

68 + if line.is_empty() {

69 + break;

70 + }

71 + if let Some(length) = line.to_ascii_lowercase().strip_prefix("content-length:") {

72 + content_length = length.trim().parse().unwrap_or(0);

73 + }

74 + }

75 + let mut body = vec![0u8; content_length];

76 + if reader.read_exact(&mut body).is_err() {

77 + continue;

78 + }

79 + if let Ok(request) = serde_json::from_slice::<Value>(&body) {

80 + recorded.lock().unwrap().push(request);

81 + }

82 + let payload = queue

83 + .lock()

84 + .unwrap()

85 + .pop_front()

86 + .unwrap_or_else(|| sse_text("out of scripted responses"));

87 + let response = format!(

88 + "HTTP/1.1 200 OK\r\ncontent-type: text/event-stream\r\n\

89 + content-length: {}\r\nconnection: close\r\n\r\n{}",

90 + payload.len(),

91 + payload

92 + );

93 + let _ = stream.write_all(response.as_bytes());

94 + }

95 + });

96 +

97 + FakeEndpoint { port, requests }

98 +}

99 +

100 +// ── ACP client over the binary's stdio ──────────────────────────────────────

101 +

102 +struct AgentUnderTest {

103 + child: Child,

104 + stdin: ChildStdin,

105 + incoming: Receiver<Value>,

106 + next_id: u64,

107 +}

108 +

109 +fn spawn_agent(port: u16, config_dir: &std::path::Path, model: &str) -> AgentUnderTest {

110 + let mut child = Command::new(env!("CARGO_BIN_EXE_sigit"))

111 + .env("OPENAI_BASE_URL", format!("http://127.0.0.1:{port}"))

112 + .env("OPENAI_API_KEY", "test-key")

113 + .env("SIGIT_MODEL", model)

114 + .env("SIGIT_CONFIG_DIR", config_dir)

115 + .env("SIGIT_MCP", "off")

116 + .env("SIGIT_PERMISSIONS", "allow")

117 + .env_remove("SIGIT_LOCAL_INFERENCE")

118 + .stdin(Stdio::piped())

119 + .stdout(Stdio::piped())

120 + .stderr(Stdio::null())

121 + .spawn()

122 + .expect("spawn sigit in ACP mode");

123 +

124 + let stdout = child.stdout.take().unwrap();

125 + let (message_tx, incoming) = channel();

126 + std::thread::spawn(move || {

127 + for line in BufReader::new(stdout).lines() {

128 + let Ok(line) = line else { break };

129 + if let Ok(message) = serde_json::from_str::<Value>(&line)

130 + && message_tx.send(message).is_err()

131 + {

132 + break;

133 + }

134 + }

135 + });

136 +

137 + let stdin = child.stdin.take().unwrap();

138 + AgentUnderTest {

139 + child,

140 + stdin,

141 + incoming,

142 + next_id: 0,

143 + }

144 +}

145 +

146 +impl AgentUnderTest {

147 + fn send(&mut self, message: Value) {

148 + let mut line = message.to_string();

149 + line.push('\n');

150 + self.stdin

151 + .write_all(line.as_bytes())

152 + .expect("write to agent stdin");

153 + self.stdin.flush().expect("flush agent stdin");

154 + }

155 +

156 + fn request(&mut self, method: &str, params: Value) -> u64 {

157 + self.next_id += 1;

158 + let id = self.next_id;

159 + self.send(json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params}));

160 + id

161 + }

162 +

163 + /// Wait for the response to our request `id`, collecting the raw JSON of

164 + /// every `session/update` notification that arrives before it.

165 + fn wait_for_response_collecting_updates(&mut self, id: u64) -> (Value, String) {

166 + let deadline = Instant::now() + TIMEOUT;

167 + let mut updates = String::new();

168 + loop {

169 + let remaining = deadline.saturating_duration_since(Instant::now());

170 + match self.incoming.recv_timeout(remaining) {

171 + Ok(message) if message["id"] == id && message.get("method").is_none() => {

172 + assert!(

173 + message.get("error").is_none(),

174 + "request {id} failed: {message}"

175 + );

176 + return (message, updates);

177 + }

178 + Ok(message) => {

179 + if message["method"] == "session/update" {

180 + updates.push_str(&message["params"].to_string());

181 + updates.push('\n');

182 + }

183 + }

184 + Err(_) => panic!("timed out waiting for response to request {id}"),

185 + }

186 + }

187 + }

188 +

189 + fn wait_for_response(&mut self, id: u64) -> Value {

190 + self.wait_for_response_collecting_updates(id).0

191 + }

192 +}

193 +

194 +impl Drop for AgentUnderTest {

195 + fn drop(&mut self) {

196 + let _ = self.child.kill();

197 + let _ = self.child.wait();

198 + }

199 +}

200 +

201 +/// A few bytes of base64; the endpoint is scripted, so nothing decodes them.

202 +const CLIP: &str = "UklGRiQAAABXQVZFZm10IA==";

203 +

204 +/// Run one prompt that carries text and an audio clip against `model`.

205 +/// Returns the `initialize` response, the raw `session/update` stream of the

206 +/// prompt, and the first completion request the endpoint received.

207 +fn prompt_with_audio(model: &str, tag: &str) -> (Value, String, Value) {

208 + let endpoint = start_fake_endpoint(vec![sse_text("done")]);

209 + let scratch =

210 + std::env::temp_dir().join(format!("sigit_acp_audio_{tag}_{}", std::process::id()));

211 + let config_dir = scratch.join("config");

212 + let project = scratch.join("project");

213 + std::fs::create_dir_all(&config_dir).unwrap();

214 + std::fs::create_dir_all(&project).unwrap();

215 +

216 + let mut agent = spawn_agent(endpoint.port, &config_dir, model);

217 +

218 + let id = agent.request(

219 + "initialize",

220 + json!({"protocolVersion": 1, "clientCapabilities": {}}),

221 + );

222 + let initialize = agent.wait_for_response(id);

223 +

224 + let id = agent.request("session/new", json!({"cwd": project, "mcpServers": []}));

225 + let session_id = agent.wait_for_response(id)["result"]["sessionId"]

226 + .as_str()

227 + .expect("session id")

228 + .to_string();

229 +

230 + let id = agent.request(

231 + "session/prompt",

232 + json!({

233 + "sessionId": session_id,

234 + "prompt": [

235 + {"type": "text", "text": "what is said here?"},

236 + {"type": "audio", "mimeType": "audio/wav", "data": CLIP},

237 + ],

238 + }),

239 + );

240 + let (response, updates) = agent.wait_for_response_collecting_updates(id);

241 + assert_eq!(response["result"]["stopReason"], "end_turn");

242 +

243 + let request = endpoint

244 + .requests

245 + .lock()

246 + .unwrap()

247 + .first()

248 + .cloned()

249 + .expect("the endpoint received a completion request");

250 +

251 + drop(agent);

252 + let _ = std::fs::remove_dir_all(&scratch);

253 + (initialize, updates, request)

254 +}

255 +

256 +fn last_user_message(request: &Value) -> &Value {

257 + request["messages"]

258 + .as_array()

259 + .expect("messages")

260 + .iter()

261 + .rev()

262 + .find(|message| message["role"] == "user")

263 + .expect("a user message")

264 +}

265 +

266 +#[test]

267 +fn audio_reaches_a_model_on_the_users_own_endpoint() {

268 + let (initialize, updates, request) = prompt_with_audio("scripted-audio-model", "own");

269 +

270 + assert_eq!(

271 + initialize["result"]["agentCapabilities"]["promptCapabilities"]["audio"], true,

272 + "a client only attaches audio for an agent that advertises it: {initialize}"

273 + );

274 +

275 + let message = last_user_message(&request);

276 + assert_eq!(

277 + message["content"],

278 + json!([

279 + {"type": "text", "text": "what is said here?"},

280 + {"type": "input_audio", "input_audio": {"data": CLIP, "format": "wav"}},

281 + ]),

282 + "the clip must travel as an input_audio content part"

283 + );

284 + assert!(

285 + !updates.contains("Audio not sent"),

286 + "nothing was left out, so nothing should be flagged: {updates}"

287 + );

288 +}

289 +

290 +#[test]

291 +fn a_cloud_tier_gets_the_text_and_the_user_is_told() {

292 + let (_, updates, request) = prompt_with_audio("onde-large", "cloud");

293 +

294 + let message = last_user_message(&request);

295 + let content = message["content"]

296 + .as_str()

297 + .expect("with the clip dropped, the message is plain text");

298 + assert!(

299 + content.starts_with("what is said here?\n\n[The user attached 1 audio clip."),

300 + "the model must be told a clip was left out: {content}"

301 + );

302 + assert!(

303 + !request.to_string().contains(CLIP),

304 + "the clip must not reach a tier that drops it"

305 + );

306 + assert!(

307 + updates.contains("Audio not sent"),

308 + "the user must be told the clip was left out: {updates}"

309 + );

310 +}