@ondeinference / ed
Code Commits Issues Pulls Cloud Agent CI/CD
release/1.3.1
main release/1.3.1
rs 155 lines 5.98 KB
Raw
1 //! Test support for Onde ACP agents: a mock Onde Cloud endpoint ([`mock`]), a raw JSON-RPC
2 //! client that drives the real agent binary over stdio ([`Raw`]), the ACP registry's validator
3 //! request ([`REGISTRY_VALIDATOR_INITIALIZE`]),
4 //! and the ACP v1 conformance suite ([`conformance`]).
5 //!
6 //! The test MCP server lives in `ed-mcp` (feature `test-server`) because it must be a binary,
7 //! and binaries cannot use dev-dependencies. An agent adopts the whole suite with one macro call
8 //! in an integration test:
9 //!
10 //! ```ignore
11 //! fn target() -> ed_acp_testkit::Target {
12 //! ed_acp_testkit::Target {
13 //! bin: env!("CARGO_BIN_EXE_my-agent").into(),
14 //! name: "my-agent",
15 //! env_prefix: "MY_AGENT",
16 //! mcp_stub: env!("CARGO_BIN_EXE_mcp_stdio_stub").into(),
17 //! base_tool_count: 5,
18 //! env: vec![],
19 //! }
20 //! }
21 //! ed_acp_testkit::conformance_suite!(target);
22 //! ```
23
24 pub mod conformance;
25 pub mod mock;
26 mod raw;
27
28 use std::path::{Path, PathBuf};
29
30 use serde_json::{Value, json};
31
32 pub use mock::{MODEL, MockStats, Script, Step, TEST_KEY, mock, start_mock_llm, temp_dir};
33 pub use raw::{
34 Env, Perm, Raw, TIMEOUT, alive, error_code, read_pid, stub_server, tool_text, wait_dead,
35 };
36
37 /// The agent under test.
38 #[derive(Debug, Clone)]
39 pub struct Target {
40 /// The agent binary, usually `env!("CARGO_BIN_EXE_<name>")`.
41 pub bin: PathBuf,
42 /// Expected `agentInfo.name`.
43 pub name: &'static str,
44 /// The agent's environment prefix: the suite sets `<PREFIX>_DATA_DIR`, `<PREFIX>_BASE_URL`
45 /// and `<PREFIX>_MODEL`.
46 pub env_prefix: &'static str,
47 /// A binary whose `main` runs `ed_mcp::test_server::run`.
48 pub mcp_stub: PathBuf,
49 /// How many tools the agent offers with no MCP server connected.
50 pub base_tool_count: usize,
51 /// Extra environment for every run, e.g. switching built-in MCP servers off.
52 pub env: Vec<(String, String)>,
53 }
54
55 impl Target {
56 /// The environment for one run: an isolated data dir, the mock endpoint at `base_url`, the
57 /// test key, and `PATH` set to the data dir so no stray binaries are found.
58 pub fn env(&self, data_dir: &Path, base_url: &str) -> Env {
59 let mut e = Env {
60 bin: self.bin.clone(),
61 prefix: self.env_prefix,
62 vars: Vec::new(),
63 };
64 let (data, url, model) = (e.var("DATA_DIR"), e.var("BASE_URL"), e.var("MODEL"));
65 e.set(&data, data_dir.display().to_string());
66 e.set(&url, base_url);
67 e.set(&model, MODEL);
68 e.set("ONDE_API_KEY", TEST_KEY);
69 e.set("PATH", data_dir.display().to_string());
70 for (k, v) in &self.env {
71 e.set(k, v.clone());
72 }
73 e
74 }
75 }
76
77 /// The `initialize` params the ACP registry's validator sends, copied from
78 /// `agentclientprotocol/registry` `.github/workflows/client.py`. It signals terminal-auth
79 /// support only through `_meta`, never through `auth.terminal`.
80 pub fn registry_validator_initialize() -> Value {
81 json!({
82 "protocolVersion": 1,
83 "clientInfo": {"name": "ACP Registry Validator", "version": "1.0.0"},
84 "clientCapabilities": {
85 "terminal": true,
86 "fs": {"readTextFile": true, "writeTextFile": true},
87 "_meta": {"terminal_output": true, "terminal-auth": true}
88 }
89 })
90 }
91
92 /// See [`registry_validator_initialize`].
93 pub const REGISTRY_VALIDATOR_INITIALIZE: &str = r#"{"protocolVersion":1,"clientInfo":{"name":"ACP Registry Validator","version":"1.0.0"},"clientCapabilities":{"terminal":true,"fs":{"readTextFile":true,"writeTextFile":true},"_meta":{"terminal_output":true,"terminal-auth":true}}}"#;
94
95 /// Generate one `#[tokio::test]` per conformance check, in a module named `acp_conformance`.
96 /// `$target` names a function, in scope where the macro is called, that returns the [`Target`].
97 #[macro_export]
98 macro_rules! conformance_suite {
99 ($target:ident) => {
100 mod acp_conformance {
101 $crate::conformance_suite!(@tests $target;
102 initialize_negotiates_v1_and_advertises_capabilities,
103 registry_validator_probe_sees_terminal_auth,
104 auth_required_then_authenticate_reloads_config,
105 sessions_survive_a_killed_process_and_replay_tool_output,
106 resume_close_delete_and_unknown_sessions,
107 streaming_reports_message_ids_usage_and_inlines_file_links,
108 permissions_reject_always_raw_output_and_auto_approve,
109 auto_approve_is_hidden_from_clients_without_boolean_options,
110 content_filter_is_a_refusal_and_private_flags_stay_off_the_wire,
111 client_supplied_mcp_server_is_offered_gated_and_streams_progress,
112 mcp_children_die_when_the_agent_exits,
113 broken_mcp_server_is_not_fatal,
114 cancelling_a_turn_cancels_the_mcp_call,
115 );
116 }
117 };
118 (@tests $target:ident; $($name:ident),* $(,)?) => {
119 $(
120 #[tokio::test]
121 async fn $name() {
122 $crate::conformance::$name(&super::$target()).await;
123 }
124 )*
125 };
126 }
127
128 #[cfg(test)]
129 mod tests {
130 use super::*;
131
132 #[test]
133 fn the_constant_and_the_value_agree() {
134 let parsed: Value = serde_json::from_str(REGISTRY_VALIDATOR_INITIALIZE).unwrap();
135 assert_eq!(parsed, registry_validator_initialize());
136 }
137
138 #[test]
139 fn env_uses_the_prefix_and_target_overrides() {
140 let t = Target {
141 bin: "/bin/agent".into(),
142 name: "agent",
143 env_prefix: "MY",
144 mcp_stub: "/bin/stub".into(),
145 base_tool_count: 0,
146 env: vec![("MY_MODEL".into(), "custom".into())],
147 };
148 let e = t.env(Path::new("/data"), "http://x/v1");
149 let get = |k: &str| e.vars.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str());
150 assert_eq!(get("MY_DATA_DIR"), Some("/data"));
151 assert_eq!(get("MY_BASE_URL"), Some("http://x/v1"));
152 assert_eq!(get("MY_MODEL"), Some("custom"));
153 assert_eq!(get("ONDE_API_KEY"), Some(TEST_KEY));
154 }
155 }