@splitfireai / agent
Code Commits Issues Pulls Cloud Agent CI/CD
main
development main
rs 141 lines 5.31 KB
Raw
1 //! Live smoke tests against the real Onde Cloud endpoint (cloud.ondeinference.com).
2 //!
3 //! Skipped unless `ONDE_API_KEY` (format `app-id:app-secret`) is set: they hit the production
4 //! service and cost real tokens, so they are for manual or scheduled verification.
5 //!
6 //! Run with: ONDE_API_KEY=app-id:app-secret cargo test --test live -- --nocapture
7
8 use std::sync::{Arc, Mutex};
9
10 const BIN: &str = env!("CARGO_BIN_EXE_splitfire-agent");
11
12 fn live_key() -> Option<String> {
13 std::env::var("ONDE_API_KEY").ok().filter(|k| !k.is_empty())
14 }
15
16 /// A clean command: no stored config, no overrides leaking in from the developer's shell.
17 fn clean_cmd(data_dir: &std::path::Path) -> std::process::Command {
18 let mut cmd = std::process::Command::new(BIN);
19 cmd.env("SPLITFIRE_DATA_DIR", data_dir)
20 .env("SPLITFIRE_MCP_DEMUCS", "off")
21 .env_remove("SPLITFIRE_BASE_URL")
22 .env_remove("SPLITFIRE_MODEL")
23 .env_remove("SPLITFIRE_MODELS");
24 cmd
25 }
26
27 #[test]
28 fn onde_lists_models() {
29 if live_key().is_none() {
30 eprintln!("skipping: ONDE_API_KEY not set");
31 return;
32 }
33 let dir = std::env::temp_dir().join(format!("sf-live-{}", uuid::Uuid::new_v4()));
34 let out = clean_cmd(&dir)
35 .arg("--list-models")
36 .output()
37 .expect("failed to run splitfire-agent --list-models");
38 let stdout = String::from_utf8_lossy(&out.stdout);
39 let stderr = String::from_utf8_lossy(&out.stderr);
40 assert!(
41 out.status.success(),
42 "--list-models failed\nstdout: {stdout}\nstderr: {stderr}"
43 );
44 let ids: Vec<&str> = stdout
45 .lines()
46 .map(|l| l.split('\t').next().unwrap())
47 .collect();
48 assert!(!ids.is_empty(), "endpoint returned no models");
49 assert!(
50 ids.iter().any(|id| id.starts_with("onde")),
51 "expected at least one onde-* model, got: {ids:?}"
52 );
53 }
54
55 /// One completion through the ACP path that must use a theory tool: ask for the C# major
56 /// scale and expect the exact spelling (E# and B#) in the answer.
57 #[tokio::test(flavor = "multi_thread")]
58 async fn onde_answers_with_theory_tool() {
59 use agent_client_protocol::schema::ProtocolVersion;
60 use agent_client_protocol::schema::v1::{
61 ContentBlock, InitializeRequest, NewSessionRequest, PromptRequest,
62 RequestPermissionRequest, SessionNotification, SessionUpdate, StopReason, TextContent,
63 };
64 use agent_client_protocol::{AcpAgent, AcpAgentConfig, Agent, Client, ConnectionTo};
65
66 if live_key().is_none() {
67 eprintln!("skipping: ONDE_API_KEY not set");
68 return;
69 }
70 let dir = std::env::temp_dir().join(format!("sf-live-{}", uuid::Uuid::new_v4()));
71 let text: Arc<Mutex<String>> = Arc::default();
72 let text_notify = text.clone();
73 let tools: Arc<Mutex<Vec<String>>> = Arc::default();
74 let tools_notify = tools.clone();
75
76 let stop_reason = Client
77 .builder()
78 .on_receive_notification(
79 async move |n: SessionNotification, _cx| {
80 match n.update {
81 SessionUpdate::AgentMessageChunk(chunk) => {
82 if let ContentBlock::Text(t) = chunk.content {
83 text_notify.lock().unwrap().push_str(&t.text);
84 }
85 }
86 SessionUpdate::ToolCall(tc) => tools_notify.lock().unwrap().push(tc.title),
87 _ => {}
88 }
89 Ok(())
90 },
91 agent_client_protocol::on_receive_notification!(),
92 )
93 .on_receive_request(
94 // Theory tools are read-only; a permission request means something else ran.
95 async move |_req: RequestPermissionRequest, responder, _cx| {
96 responder.respond_with_error(agent_client_protocol::Error::internal_error())
97 },
98 agent_client_protocol::on_receive_request!(),
99 )
100 .connect_with(
101 AcpAgent::new(
102 AcpAgentConfig::new(BIN)
103 .env("SPLITFIRE_DATA_DIR", dir.display().to_string())
104 .env("SPLITFIRE_MCP_DEMUCS", "off"),
105 ),
106 |connection: ConnectionTo<Agent>| async move {
107 connection
108 .send_request(InitializeRequest::new(ProtocolVersion::V1))
109 .block_task()
110 .await?;
111 let session = connection
112 .send_request(NewSessionRequest::new(std::env::current_dir().unwrap()))
113 .block_task()
114 .await?;
115 let resp = connection
116 .send_request(PromptRequest::new(
117 session.session_id,
118 vec![ContentBlock::Text(TextContent::new(
119 "Spell the C# major scale. Use your theory tool and list the seven notes.",
120 ))],
121 ))
122 .block_task()
123 .await?;
124 Ok(resp.stop_reason)
125 },
126 )
127 .await
128 .expect("ACP session failed");
129
130 assert_eq!(stop_reason, StopReason::EndTurn);
131 let text = text.lock().unwrap();
132 eprintln!(
133 "tool calls: {:?}\nmodel replied: {}",
134 tools.lock().unwrap(),
135 text.trim()
136 );
137 assert!(
138 text.contains("E#") && text.contains("B#"),
139 "expected exact spelling: {text}"
140 );
141 }