| 1 | //! SplitFire's ACP tests over raw JSON-RPC against the real binary. The protocol itself (version |
| 2 | //! negotiation, auth, sessions, streaming, approval, MCP) is covered by `ed-acp-testkit`'s |
| 3 | //! conformance suite, run here through [`ed_acp_testkit::conformance_suite!`]. The rest are |
| 4 | //! SplitFire's own: slash commands, audio, the music tools and the built-in stem separator. |
| 5 | |
| 6 | mod common; |
| 7 | |
| 8 | use std::path::{Path, PathBuf}; |
| 9 | |
| 10 | use common::{BASE_TOOL_COUNT, target}; |
| 11 | use ed_acp_testkit::{Raw, Step, mock, read_pid, stub_server, temp_dir, tool_text, wait_dead}; |
| 12 | use serde_json::json; |
| 13 | |
| 14 | ed_acp_testkit::conformance_suite!(target); |
| 15 | |
| 16 | fn wav_bytes() -> Vec<u8> { |
| 17 | let mut cursor = std::io::Cursor::new(Vec::new()); |
| 18 | let spec = hound::WavSpec { |
| 19 | channels: 1, |
| 20 | sample_rate: 22_050, |
| 21 | bits_per_sample: 16, |
| 22 | sample_format: hound::SampleFormat::Int, |
| 23 | }; |
| 24 | let mut w = hound::WavWriter::new(&mut cursor, spec).unwrap(); |
| 25 | for i in 0..22_050 { |
| 26 | let s = (i as f32 * 2.0 * std::f32::consts::PI * 440.0 / 22_050.0).sin(); |
| 27 | w.write_sample((s * 0.5 * i16::MAX as f32) as i16).unwrap(); |
| 28 | } |
| 29 | w.finalize().unwrap(); |
| 30 | cursor.into_inner() |
| 31 | } |
| 32 | |
| 33 | /// Audio links never reach the model as text to read; SplitFire points it at `analyze_audio`. |
| 34 | #[tokio::test] |
| 35 | async fn audio_links_point_the_model_at_the_analyzer() { |
| 36 | let data = temp_dir("sf-data"); |
| 37 | let work = temp_dir("sf-work"); |
| 38 | std::fs::write(work.join("take.wav"), wav_bytes()).unwrap(); |
| 39 | let (url, stats) = mock(vec![Step::Final("ok")]).await; |
| 40 | let mut agent = Raw::spawn(&target().env(&data, &url)); |
| 41 | agent.initialize().await; |
| 42 | let sid = agent.new_session(&work, json!([])).await; |
| 43 | let take = format!("file://{}", work.join("take.wav").display()); |
| 44 | agent |
| 45 | .prompt( |
| 46 | &sid, |
| 47 | json!([ |
| 48 | {"type": "text", "text": "What key is this take in?"}, |
| 49 | {"type": "resource_link", "name": "take.wav", "uri": take}, |
| 50 | ]), |
| 51 | ) |
| 52 | .await; |
| 53 | let payload = stats.payloads.lock().unwrap()[0].clone(); |
| 54 | let user = payload["messages"].as_array().unwrap().last().unwrap()["content"] |
| 55 | .as_str() |
| 56 | .unwrap() |
| 57 | .to_string(); |
| 58 | assert!( |
| 59 | user.contains("Referenced audio file") && user.contains("analyze_audio"), |
| 60 | "{user}" |
| 61 | ); |
| 62 | agent.finish().await; |
| 63 | } |
| 64 | |
| 65 | #[tokio::test] |
| 66 | async fn slash_commands_are_advertised_and_expanded() { |
| 67 | let data = temp_dir("sf-data"); |
| 68 | let work = temp_dir("sf-work"); |
| 69 | let (url, stats) = mock(vec![Step::Final("ok")]).await; |
| 70 | let mut agent = Raw::spawn(&target().env(&data, &url)); |
| 71 | agent.initialize().await; |
| 72 | let sid = agent.new_session(&work, json!([])).await; |
| 73 | let cmds = agent.updates("available_commands_update"); |
| 74 | let names: Vec<&str> = cmds[0]["availableCommands"] |
| 75 | .as_array() |
| 76 | .unwrap() |
| 77 | .iter() |
| 78 | .map(|c| c["name"].as_str().unwrap()) |
| 79 | .collect(); |
| 80 | assert_eq!( |
| 81 | names, |
| 82 | ["analyze", "chords", "library", "practice"], |
| 83 | "no /stems without a separator" |
| 84 | ); |
| 85 | |
| 86 | agent.text_prompt(&sid, "/chords Dm7 G7 Cmaj7").await; |
| 87 | let payload = stats.payloads.lock().unwrap()[0].clone(); |
| 88 | let msgs = payload["messages"].as_array().unwrap(); |
| 89 | let system = msgs[0]["content"].as_str().unwrap(); |
| 90 | // Disconnected: the separation tool is not offered, but the prompt says how to enable it. |
| 91 | assert!( |
| 92 | !system.contains("mcp__demucs__separate_stems"), |
| 93 | "separation tool offered while disconnected" |
| 94 | ); |
| 95 | assert!( |
| 96 | system.contains("SPLITFIRE_DEMUCS_BIN"), |
| 97 | "no hint on enabling stems" |
| 98 | ); |
| 99 | assert!( |
| 100 | system.contains("Co-Authored-By: SplitFire v") |
| 101 | && system.contains("-acp <noreply@splitfire.ai>") |
| 102 | ); |
| 103 | let user = msgs.last().unwrap()["content"].as_str().unwrap(); |
| 104 | assert!( |
| 105 | user.contains("theory_progression") && user.contains("Dm7 G7 Cmaj7"), |
| 106 | "{user}" |
| 107 | ); |
| 108 | assert_eq!(payload["tools"].as_array().unwrap().len(), BASE_TOOL_COUNT); |
| 109 | agent.finish().await; |
| 110 | } |
| 111 | |
| 112 | // --------------------------------------------------------------------------- |
| 113 | // audio |
| 114 | // --------------------------------------------------------------------------- |
| 115 | |
| 116 | #[tokio::test] |
| 117 | async fn audio_blocks_are_saved_to_scratch_and_cleaned_on_close() { |
| 118 | use base64::Engine; |
| 119 | let data = temp_dir("sf-data"); |
| 120 | let work = temp_dir("sf-work"); |
| 121 | let (url, stats) = mock(vec![Step::Final("heard it")]).await; |
| 122 | let mut agent = Raw::spawn(&target().env(&data, &url)); |
| 123 | agent.initialize().await; |
| 124 | let sid = agent.new_session(&work, json!([])).await; |
| 125 | let b64 = base64::engine::general_purpose::STANDARD.encode(wav_bytes()); |
| 126 | let r = agent |
| 127 | .prompt( |
| 128 | &sid, |
| 129 | json!([ |
| 130 | {"type": "text", "text": "what is this?"}, |
| 131 | {"type": "audio", "data": b64, "mimeType": "audio/wav"} |
| 132 | ]), |
| 133 | ) |
| 134 | .await; |
| 135 | assert_eq!(r["stopReason"], "end_turn"); |
| 136 | let payload = stats.payloads.lock().unwrap()[0].clone(); |
| 137 | let user = payload["messages"].as_array().unwrap().last().unwrap()["content"] |
| 138 | .as_str() |
| 139 | .unwrap() |
| 140 | .to_string(); |
| 141 | let scratch = data.join("scratch").join(&sid); |
| 142 | let saved: Vec<PathBuf> = std::fs::read_dir(&scratch) |
| 143 | .unwrap() |
| 144 | .map(|e| e.unwrap().path()) |
| 145 | .collect(); |
| 146 | assert_eq!( |
| 147 | saved.len(), |
| 148 | 1, |
| 149 | "one scratch file expected in {}", |
| 150 | scratch.display() |
| 151 | ); |
| 152 | assert!( |
| 153 | user.contains(&saved[0].display().to_string()), |
| 154 | "model must see the path: {user}" |
| 155 | ); |
| 156 | assert_eq!(std::fs::read(&saved[0]).unwrap(), wav_bytes()); |
| 157 | agent.ok("session/close", json!({"sessionId": sid})).await; |
| 158 | assert!(!scratch.exists(), "scratch dir must be removed on close"); |
| 159 | agent.finish().await; |
| 160 | } |
| 161 | |
| 162 | #[tokio::test] |
| 163 | async fn analyze_audio_runs_through_the_agent() { |
| 164 | let data = temp_dir("sf-data"); |
| 165 | let work = temp_dir("sf-work"); |
| 166 | std::fs::write(work.join("tone.wav"), wav_bytes()).unwrap(); |
| 167 | let (url, stats) = mock(vec![ |
| 168 | Step::ToolCall { |
| 169 | name: "analyze_audio", |
| 170 | arguments: json!({"path": "tone.wav"}), |
| 171 | }, |
| 172 | Step::Final("done"), |
| 173 | ]) |
| 174 | .await; |
| 175 | let mut agent = Raw::spawn(&target().env(&data, &url)); |
| 176 | agent.initialize().await; |
| 177 | let sid = agent.new_session(&work, json!([])).await; |
| 178 | agent.text_prompt(&sid, "analyze tone.wav").await; |
| 179 | assert!( |
| 180 | agent.permissions.is_empty(), |
| 181 | "read-only analysis must not prompt" |
| 182 | ); |
| 183 | let calls = agent.updates("tool_call"); |
| 184 | assert_eq!(calls[0]["kind"], "read"); |
| 185 | assert!( |
| 186 | calls[0]["locations"][0]["path"] |
| 187 | .as_str() |
| 188 | .unwrap() |
| 189 | .ends_with("tone.wav") |
| 190 | ); |
| 191 | let payloads = stats.payloads.lock().unwrap().clone(); |
| 192 | let tool = payloads[1]["messages"] |
| 193 | .as_array() |
| 194 | .unwrap() |
| 195 | .iter() |
| 196 | .find(|m| m["role"] == "tool") |
| 197 | .unwrap() |
| 198 | .clone(); |
| 199 | let text = tool["content"].as_str().unwrap(); |
| 200 | assert!( |
| 201 | text.contains("22050 Hz") && text.contains("Peak:") && text.contains("estimate"), |
| 202 | "{text}" |
| 203 | ); |
| 204 | agent.finish().await; |
| 205 | } |
| 206 | |
| 207 | #[tokio::test] |
| 208 | async fn theory_tools_are_ungated_and_exact() { |
| 209 | let data = temp_dir("sf-data"); |
| 210 | let work = temp_dir("sf-work"); |
| 211 | let (url, stats) = mock(vec![ |
| 212 | Step::ToolCall { |
| 213 | name: "theory_scale", |
| 214 | arguments: json!({"root": "C#", "scale": "major"}), |
| 215 | }, |
| 216 | Step::Final("done"), |
| 217 | ]) |
| 218 | .await; |
| 219 | let mut agent = Raw::spawn(&target().env(&data, &url)); |
| 220 | agent.initialize().await; |
| 221 | let sid = agent.new_session(&work, json!([])).await; |
| 222 | agent.text_prompt(&sid, "C# major scale?").await; |
| 223 | assert!(agent.permissions.is_empty()); |
| 224 | let calls = agent.updates("tool_call"); |
| 225 | assert_eq!(calls[0]["kind"], "think"); |
| 226 | let payloads = stats.payloads.lock().unwrap().clone(); |
| 227 | let tool = payloads[1]["messages"] |
| 228 | .as_array() |
| 229 | .unwrap() |
| 230 | .iter() |
| 231 | .find(|m| m["role"] == "tool") |
| 232 | .unwrap() |
| 233 | .clone(); |
| 234 | assert!( |
| 235 | tool["content"] |
| 236 | .as_str() |
| 237 | .unwrap() |
| 238 | .contains("C# D# E# F# G# A# B#") |
| 239 | ); |
| 240 | agent.finish().await; |
| 241 | } |
| 242 | |
| 243 | // --------------------------------------------------------------------------- |
| 244 | // built-in demucs |
| 245 | // --------------------------------------------------------------------------- |
| 246 | |
| 247 | #[tokio::test] |
| 248 | async fn builtin_demucs_appears_only_when_configured() { |
| 249 | let data = temp_dir("sf-data"); |
| 250 | let work = temp_dir("sf-work"); |
| 251 | let pid_file = data.join("stub.pid"); |
| 252 | let input = work.join("song.wav"); |
| 253 | std::fs::write(&input, wav_bytes()).unwrap(); |
| 254 | let (url, stats) = mock(vec![ |
| 255 | Step::ToolCall { |
| 256 | name: "mcp__demucs__separate_stems", |
| 257 | arguments: json!({"input": input}), |
| 258 | }, |
| 259 | Step::Final("separated"), |
| 260 | ]) |
| 261 | .await; |
| 262 | |
| 263 | // Configured: SPLITFIRE_DEMUCS_BIN points at a server speaking `demucs --mcp`. |
| 264 | let mut env = target().env(&data, &url); |
| 265 | env.set("SPLITFIRE_MCP_DEMUCS", "on"); |
| 266 | env.set( |
| 267 | "SPLITFIRE_DEMUCS_BIN", |
| 268 | target().mcp_stub.display().to_string(), |
| 269 | ); |
| 270 | env.set("STUB_PID_FILE", pid_file.display().to_string()); |
| 271 | let mut agent = Raw::spawn(&env); |
| 272 | agent.initialize().await; |
| 273 | let sid = agent.new_session(&work, json!([])).await; |
| 274 | let cmds = agent.updates("available_commands_update"); |
| 275 | let names: Vec<&str> = cmds[0]["availableCommands"] |
| 276 | .as_array() |
| 277 | .unwrap() |
| 278 | .iter() |
| 279 | .map(|c| c["name"].as_str().unwrap()) |
| 280 | .collect(); |
| 281 | assert!(names.contains(&"stems"), "{names:?}"); |
| 282 | let r = agent.text_prompt(&sid, "split song.wav").await; |
| 283 | assert_eq!(r["stopReason"], "end_turn"); |
| 284 | let payload = stats.payloads.lock().unwrap()[0].clone(); |
| 285 | let system = payload["messages"][0]["content"].as_str().unwrap(); |
| 286 | assert!(system.contains("mcp__demucs__separate_stems")); |
| 287 | let tool_names: Vec<&str> = payload["tools"] |
| 288 | .as_array() |
| 289 | .unwrap() |
| 290 | .iter() |
| 291 | .map(|t| t["function"]["name"].as_str().unwrap()) |
| 292 | .collect(); |
| 293 | assert!( |
| 294 | tool_names.contains(&"mcp__demucs__separate_stems") |
| 295 | && tool_names.contains(&"mcp__demucs__list_models") |
| 296 | ); |
| 297 | |
| 298 | // The permission prompt states the download size from list_models. |
| 299 | assert_eq!(agent.permissions.len(), 1); |
| 300 | let prompt_text = agent.permissions[0]["toolCall"]["content"][0]["content"]["text"] |
| 301 | .as_str() |
| 302 | .unwrap() |
| 303 | .to_string(); |
| 304 | assert!(prompt_text.contains("80 MB"), "{prompt_text}"); |
| 305 | assert!(prompt_text.contains("song.wav"), "{prompt_text}"); |
| 306 | |
| 307 | // Stem paths become tool-call locations. |
| 308 | let update = agent |
| 309 | .updates("tool_call_update") |
| 310 | .into_iter() |
| 311 | .rfind(|u| u["status"] == "completed") |
| 312 | .unwrap(); |
| 313 | let locs: Vec<&str> = update["locations"] |
| 314 | .as_array() |
| 315 | .unwrap() |
| 316 | .iter() |
| 317 | .map(|l| l["path"].as_str().unwrap()) |
| 318 | .collect(); |
| 319 | assert_eq!(locs.len(), 4, "{update}"); |
| 320 | assert!( |
| 321 | locs.iter() |
| 322 | .all(|p| Path::new(p).is_absolute() && p.ends_with(".wav")) |
| 323 | ); |
| 324 | |
| 325 | // One demucs per process: a second session shares it. |
| 326 | let pid = read_pid(&pid_file); |
| 327 | agent.new_session(&work, json!([])).await; |
| 328 | assert_eq!(read_pid(&pid_file), pid); |
| 329 | agent.finish().await; |
| 330 | assert!(wait_dead(pid).await, "demucs outlived the agent"); |
| 331 | |
| 332 | // Unset (no binary, nothing on PATH): no tools, no /stems, and the prompt offers no |
| 333 | // separation tool, only how to enable it. |
| 334 | for mode in ["off", "on"] { |
| 335 | let mut env = target().env(&data, &url); |
| 336 | env.set("SPLITFIRE_MCP_DEMUCS", mode); |
| 337 | let (url2, stats2) = mock(vec![Step::Final("x")]).await; |
| 338 | env.set("SPLITFIRE_BASE_URL", url2); |
| 339 | let mut agent = Raw::spawn(&env); |
| 340 | agent.initialize().await; |
| 341 | let sid = agent.new_session(&work, json!([])).await; |
| 342 | let cmds = agent.updates("available_commands_update"); |
| 343 | assert!(!cmds[0]["availableCommands"].to_string().contains("stems")); |
| 344 | agent.text_prompt(&sid, "hi").await; |
| 345 | let payload = stats2.payloads.lock().unwrap()[0].clone(); |
| 346 | assert_eq!( |
| 347 | payload["tools"].as_array().unwrap().len(), |
| 348 | BASE_TOOL_COUNT, |
| 349 | "mode {mode}" |
| 350 | ); |
| 351 | let system = payload["messages"][0]["content"].as_str().unwrap(); |
| 352 | assert!(!system.contains("mcp__demucs__"), "mode {mode}"); |
| 353 | assert!(system.contains("SPLITFIRE_DEMUCS_BIN"), "mode {mode}"); |
| 354 | agent.finish().await; |
| 355 | } |
| 356 | } |
| 357 | |
| 358 | #[tokio::test] |
| 359 | async fn client_supplied_demucs_wins_over_the_builtin() { |
| 360 | let data = temp_dir("sf-data"); |
| 361 | let work = temp_dir("sf-work"); |
| 362 | let builtin_pid = data.join("builtin.pid"); |
| 363 | let client_pid = data.join("client.pid"); |
| 364 | let (url, _) = mock(vec![Step::Final("x")]).await; |
| 365 | let mut env = target().env(&data, &url); |
| 366 | env.set("SPLITFIRE_MCP_DEMUCS", "on"); |
| 367 | env.set( |
| 368 | "SPLITFIRE_DEMUCS_BIN", |
| 369 | target().mcp_stub.display().to_string(), |
| 370 | ); |
| 371 | env.set("STUB_PID_FILE", builtin_pid.display().to_string()); |
| 372 | let mut agent = Raw::spawn(&env); |
| 373 | agent.initialize().await; |
| 374 | agent |
| 375 | .new_session( |
| 376 | &work, |
| 377 | json!([stub_server(&target().mcp_stub, "demucs", Some(&client_pid))]), |
| 378 | ) |
| 379 | .await; |
| 380 | assert!(client_pid.exists(), "the client's server should run"); |
| 381 | assert!( |
| 382 | !builtin_pid.exists(), |
| 383 | "the built-in must not start when the client supplies demucs" |
| 384 | ); |
| 385 | agent.finish().await; |
| 386 | } |
| 387 | |
| 388 | // --------------------------------------------------------------------------- |
| 389 | // Real demucs (opt-in) |
| 390 | // --------------------------------------------------------------------------- |
| 391 | |
| 392 | /// End to end with a real `demucs --mcp`. Skipped unless `SPLITFIRE_E2E_DEMUCS_BIN` points at a |
| 393 | /// demucs binary (model weights are downloaded on first use): |
| 394 | /// `SPLITFIRE_E2E_DEMUCS_BIN=$(which demucs) cargo test --test acp_compliance real_demucs -- --nocapture` |
| 395 | #[tokio::test] |
| 396 | async fn real_demucs_end_to_end() { |
| 397 | let Some(bin) = std::env::var("SPLITFIRE_E2E_DEMUCS_BIN") |
| 398 | .ok() |
| 399 | .filter(|b| !b.is_empty()) |
| 400 | else { |
| 401 | eprintln!("skipping: SPLITFIRE_E2E_DEMUCS_BIN not set"); |
| 402 | return; |
| 403 | }; |
| 404 | let data = temp_dir("sf-data"); |
| 405 | let work = temp_dir("sf-work"); |
| 406 | // Four seconds of stereo 44.1 kHz: a low note plus a decaying high one. |
| 407 | let input = work.join("short.wav"); |
| 408 | let spec = hound::WavSpec { |
| 409 | channels: 2, |
| 410 | sample_rate: 44_100, |
| 411 | bits_per_sample: 16, |
| 412 | sample_format: hound::SampleFormat::Int, |
| 413 | }; |
| 414 | let mut w = hound::WavWriter::create(&input, spec).unwrap(); |
| 415 | for i in 0..44_100 * 4 { |
| 416 | let t = i as f32 / 44_100.0; |
| 417 | let s = 0.3 * (2.0 * std::f32::consts::PI * 110.0 * t).sin() |
| 418 | + 0.2 * (2.0 * std::f32::consts::PI * 1_320.0 * t).sin() * (-(t % 0.5) * 8.0).exp(); |
| 419 | let v = (s * i16::MAX as f32) as i16; |
| 420 | w.write_sample(v).unwrap(); |
| 421 | w.write_sample(v).unwrap(); |
| 422 | } |
| 423 | w.finalize().unwrap(); |
| 424 | |
| 425 | let (url, stats) = mock(vec![ |
| 426 | Step::ToolCall { |
| 427 | name: "mcp__demucs__separate_stems", |
| 428 | arguments: json!({"input": input}), |
| 429 | }, |
| 430 | Step::Final("separated"), |
| 431 | ]) |
| 432 | .await; |
| 433 | let mut env = target().env(&data, &url); |
| 434 | env.set("SPLITFIRE_MCP_DEMUCS", "on"); |
| 435 | env.set("SPLITFIRE_DEMUCS_BIN", bin); |
| 436 | let mut agent = Raw::spawn(&env); |
| 437 | agent.initialize().await; |
| 438 | let sid = agent.new_session(&work, json!([])).await; |
| 439 | let r = agent.text_prompt(&sid, "/stems short.wav").await; |
| 440 | assert_eq!(r["stopReason"], "end_turn"); |
| 441 | let prompt_text = agent.permissions[0]["toolCall"]["content"][0]["content"]["text"] |
| 442 | .as_str() |
| 443 | .unwrap() |
| 444 | .to_string(); |
| 445 | eprintln!("permission prompt: {prompt_text}"); |
| 446 | let updates = agent.updates("tool_call_update"); |
| 447 | let progress: Vec<String> = updates |
| 448 | .iter() |
| 449 | .map(tool_text) |
| 450 | .filter(|t| t.contains('%')) |
| 451 | .collect(); |
| 452 | eprintln!("progress: {progress:?}"); |
| 453 | assert!(!progress.is_empty(), "no progress reached the tool call"); |
| 454 | let done = updates |
| 455 | .iter() |
| 456 | .rev() |
| 457 | .find(|u| u["status"] == "completed") |
| 458 | .expect("completed update"); |
| 459 | let paths: Vec<&str> = done["locations"] |
| 460 | .as_array() |
| 461 | .unwrap() |
| 462 | .iter() |
| 463 | .map(|l| l["path"].as_str().unwrap()) |
| 464 | .collect(); |
| 465 | assert_eq!(paths.len(), 4, "{done}"); |
| 466 | for p in &paths { |
| 467 | assert!(Path::new(p).is_file(), "missing stem {p}"); |
| 468 | } |
| 469 | let payloads = stats.payloads.lock().unwrap().clone(); |
| 470 | let tool = payloads[1]["messages"] |
| 471 | .as_array() |
| 472 | .unwrap() |
| 473 | .iter() |
| 474 | .find(|m| m["role"] == "tool") |
| 475 | .unwrap() |
| 476 | .clone(); |
| 477 | eprintln!("model saw: {}", tool["content"]); |
| 478 | agent.finish().await; |
| 479 | } |