@ondeinference / ed / commits / b08dd59

Add ed-acp-testkit: mock endpoint, raw client, ACP conformance suite

A dev-dependency for ACP agents: a mock Onde Cloud endpoint, a raw JSON-RPC client for the agent binary, the registry probe, and the ACP v1 conformance suite behind conformance_suite!. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-Authored-By: siGit Code v1.6.3 <noreply@sigit.si>

Seto Elkahfi committed Oct 9, 2026 at 00:12 UTC b08dd59ccac2cf3d9d40f8316a84e03546a0a960
8 files changed +1491
Cargo.lock
+9
@@ -1964,6 +1964,15 @@ dependencies = [
1964 "uuid 1.27.0",
1965 ]
1966
1967 +[[package]]
1968 +name = "ed-acp-testkit"
1969 +version = "1.0.1"
1970 +dependencies = [
1971 + "serde_json",
1972 + "tokio",
1973 + "uuid 1.27.0",
1974 +]
1975 +
1976 [[package]]
1977 name = "ed-agent"
1978 version = "1.0.1"
Cargo.toml
+2
@@ -6,6 +6,7 @@ members = [
6 "crates/ed-agent-tauri",
7 "crates/ed-mcp",
8 "crates/ed-acp",
9 + "crates/ed-acp-testkit",
10 "uniffi-bindgen",
11 ]
12
@@ -58,6 +59,7 @@ base64 = "0.23"
59 ed-agent = { version = "1.0.1", path = "crates/ed-agent" }
60 ed-mcp = { version = "1.0.1", path = "crates/ed-mcp" }
61 ed-acp = { version = "1.0.1", path = "crates/ed-acp" }
62 +ed-acp-testkit = { version = "1.0.1", path = "crates/ed-acp-testkit" }
63
64 # gemm-common (via candle, via onde) depends on sysctl for every
65 # `target_vendor = "apple"`, but sysctl 0.6.0 only has a `sys` module behind
crates/ed-acp-testkit/Cargo.toml new
+17
@@ -0,0 +1,17 @@
1 +[package]
2 +name = "ed-acp-testkit"
3 +description = "Test kit for Onde ACP agents: a mock Onde Cloud endpoint, a raw JSON-RPC client for the agent binary, a test MCP server, the ACP registry probe and the ACP v1 conformance suite."
4 +keywords = ["acp", "agent", "testing", "conformance", "onde"]
5 +categories = ["development-tools::testing"]
6 +version.workspace = true
7 +edition = "2024"
8 +rust-version.workspace = true
9 +license.workspace = true
10 +authors.workspace = true
11 +repository.workspace = true
12 +homepage.workspace = true
13 +
14 +[dependencies]
15 +tokio = { workspace = true, features = ["full"] }
16 +serde_json.workspace = true
17 +uuid.workspace = true
crates/ed-acp-testkit/README.md new
+17
@@ -0,0 +1,17 @@
1 +# ed-acp-testkit
2 +
3 +Test support for Onde ACP agents, used as a dev-dependency.
4 +
5 +- A mock Onde Cloud endpoint (streaming, tool calls, usage, `content_filter`).
6 +- A raw JSON-RPC client that drives the real agent binary over stdio.
7 +- The ACP registry validator's `initialize` probe.
8 +- The ACP v1 conformance suite, adopted with one macro call in an integration test:
9 + `ed_acp_testkit::conformance_suite!(target);`
10 +
11 +The scriptable MCP server for tests is in [`ed-mcp`](https://crates.io/crates/ed-mcp) behind its
12 +`test-server` feature.
13 +
14 +## Copyright
15 +
16 +© 2026 [Splitfire AB](https://5mb.app) ([Onde Inference](https://ondeinference.com)).
17 +Licensed under MIT or Apache-2.0.
crates/ed-acp-testkit/src/conformance.rs new
+795
@@ -0,0 +1,795 @@
1 +//! The ACP v1 conformance suite (Onde Agent Platform §5.5). Each function drives the real
2 +//! agent binary over stdio against the mock Onde endpoint. Run them all with
3 +//! [`conformance_suite!`](crate::conformance_suite).
4 +//!
5 +//! The suite assumes the agent offers [`WorkspaceTools`] (`read_file`, `write_file`) and
6 +//! terminal auth, as every Onde ACP agent does today.
7 +//!
8 +//! [`WorkspaceTools`]: https://docs.rs/ed-acp
9 +
10 +use std::time::Duration;
11 +
12 +use serde_json::{Value, json};
13 +
14 +use crate::Target;
15 +use crate::mock::{MODEL, Step, TEST_KEY, mock, temp_dir};
16 +use crate::raw::{Perm, Raw, alive, error_code, read_pid, stub_server, tool_text, wait_dead};
17 +
18 +pub async fn initialize_negotiates_v1_and_advertises_capabilities(t: &Target) {
19 + let data = temp_dir("sf-data");
20 + let (url, _) = mock(vec![Step::Final("x")]).await;
21 + let mut agent = Raw::spawn(&t.env(&data, &url));
22 + // A client from the future gets v1 back, not an echo of its own version.
23 + let r = agent
24 + .ok(
25 + "initialize",
26 + json!({"protocolVersion": 99, "clientCapabilities": {"auth": {"terminal": true}}}),
27 + )
28 + .await;
29 + assert_eq!(r["protocolVersion"], 1);
30 + let caps = &r["agentCapabilities"];
31 + assert_eq!(caps["loadSession"], true);
32 + assert_eq!(caps["promptCapabilities"]["image"], true);
33 + assert_eq!(caps["promptCapabilities"]["embeddedContext"], true);
34 + let sc = &caps["sessionCapabilities"];
35 + for k in ["list", "resume", "close", "delete", "additionalDirectories"] {
36 + assert!(sc.get(k).is_some(), "sessionCapabilities.{k} missing: {sc}");
37 + }
38 + assert_eq!(r["agentInfo"]["name"], t.name);
39 + assert!(
40 + caps["auth"].get("logout").is_some(),
41 + "auth.logout missing: {caps}"
42 + );
43 + // Terminal auth is offered to clients that declare they can run it...
44 + assert_eq!(r["authMethods"][0]["id"], "terminal-setup");
45 + assert_eq!(r["authMethods"][0]["args"][0], "--setup");
46 + agent.finish().await;
47 +
48 + let mut agent = Raw::spawn(&t.env(&data, &url));
49 + let r = agent
50 + .ok(
51 + "initialize",
52 + json!({"protocolVersion": 1, "clientCapabilities": {}}),
53 + )
54 + .await;
55 + assert_eq!(r["protocolVersion"], 1);
56 + // ...and not to clients that can't.
57 + assert!(
58 + r.get("authMethods")
59 + .and_then(Value::as_array)
60 + .is_none_or(|m| m.is_empty()),
61 + "{r}"
62 + );
63 + agent.finish().await;
64 +}
65 +
66 +/// The ACP registry probes every listed agent with this exact `initialize` (copied from
67 +/// `.github/workflows/client.py` in agentclientprotocol/registry), from an empty HOME with no
68 +/// credentials, and rejects the agent if `authMethods` is empty. It signals terminal-auth
69 +/// support only through `_meta`, never through `auth.terminal`.
70 +pub async fn registry_validator_probe_sees_terminal_auth(t: &Target) {
71 + let data = temp_dir("sf-data");
72 + let (url, _) = mock(vec![Step::Final("x")]).await;
73 + let mut env = t.env(&data, &url);
74 + env.unset("ONDE_API_KEY");
75 + let mut agent = Raw::spawn(&env);
76 + let r = agent
77 + .ok(
78 + "initialize",
79 + json!({
80 + "protocolVersion": 1,
81 + "clientInfo": {"name": "ACP Registry Validator", "version": "1.0.0"},
82 + "clientCapabilities": {
83 + "terminal": true,
84 + "fs": {"readTextFile": true, "writeTextFile": true},
85 + "_meta": {"terminal_output": true, "terminal-auth": true}
86 + }
87 + }),
88 + )
89 + .await;
90 + let methods = r["authMethods"].as_array().expect("authMethods");
91 + assert!(!methods.is_empty(), "registry validation would fail: {r}");
92 + assert_eq!(methods[0]["type"], "terminal", "{methods:?}");
93 + assert_eq!(methods[0]["args"][0], "--setup");
94 + agent.finish().await;
95 +}
96 +
97 +pub async fn auth_required_then_authenticate_reloads_config(t: &Target) {
98 + let data = temp_dir("sf-data");
99 + let (url, _) = mock(vec![Step::Final("hello")]).await;
100 + let mut env = t.env(&data, &url);
101 + env.unset("ONDE_API_KEY");
102 + let mut agent = Raw::spawn(&env);
103 + agent.initialize().await;
104 + let cwd = temp_dir("sf-work");
105 +
106 + // No key anywhere: session/new and session/prompt are refused with auth_required (-32000).
107 + let e = agent
108 + .call("session/new", json!({"cwd": cwd, "mcpServers": []}))
109 + .await
110 + .unwrap_err();
111 + assert_eq!(error_code(&e), -32000, "{e}");
112 + let e = agent
113 + .call("session/prompt", json!({"sessionId": "x", "prompt": []}))
114 + .await
115 + .unwrap_err();
116 + assert_eq!(error_code(&e), -32000, "{e}");
117 + // authenticate fails while there is still no key, and rejects unknown methods.
118 + let e = agent
119 + .call("authenticate", json!({"methodId": "terminal-setup"}))
120 + .await
121 + .unwrap_err();
122 + assert_eq!(error_code(&e), -32000, "{e}");
123 + let e = agent
124 + .call("authenticate", json!({"methodId": "nope"}))
125 + .await
126 + .unwrap_err();
127 + assert_eq!(error_code(&e), -32602, "{e}");
128 +
129 + // load/resume also answer auth_required, so an editor reopening a thread shows sign-in.
130 + let e = agent
131 + .call(
132 + "session/load",
133 + json!({"sessionId": "x", "cwd": cwd, "mcpServers": []}),
134 + )
135 + .await
136 + .unwrap_err();
137 + assert_eq!(error_code(&e), -32000, "{e}");
138 +
139 + // `--setup` stored a key in the config file; authenticate picks it up without a restart.
140 + std::fs::write(data.join("env"), format!("ONDE_API_KEY={TEST_KEY}\n")).unwrap();
141 + agent
142 + .ok("authenticate", json!({"methodId": "terminal-setup"}))
143 + .await;
144 + let sid = agent.new_session(&cwd, json!([])).await;
145 + let r = agent.text_prompt(&sid, "hi").await;
146 + assert_eq!(r["stopReason"], "end_turn");
147 +
148 + // logout removes the stored key; the agent is signed out without a restart.
149 + agent.ok("logout", json!({})).await;
150 + assert!(!data.join("env").exists());
151 + let e = agent
152 + .call("session/new", json!({"cwd": cwd, "mcpServers": []}))
153 + .await
154 + .unwrap_err();
155 + assert_eq!(error_code(&e), -32000, "{e}");
156 +
157 + // Zed's flow: terminal auth runs `--setup` in another process, then retries session/new
158 + // on this one without calling authenticate. The stored key must be picked up anyway.
159 + std::fs::write(data.join("env"), format!("ONDE_API_KEY={TEST_KEY}\n")).unwrap();
160 + agent.new_session(&cwd, json!([])).await;
161 + agent.finish().await;
162 +}
163 +
164 +pub async fn sessions_survive_a_killed_process_and_replay_tool_output(t: &Target) {
165 + let data = temp_dir("sf-data");
166 + let work = temp_dir("sf-work");
167 + std::fs::write(work.join("notes.txt"), "chorus in Dm").unwrap();
168 + let (url, _) = mock(vec![
169 + Step::ToolCall {
170 + name: "read_file",
171 + arguments: json!({"path": "notes.txt"}),
172 + },
173 + Step::ToolCall {
174 + name: "read_file",
175 + arguments: json!({"path": "missing.txt"}),
176 + },
177 + Step::Final("The chorus is in Dm."),
178 + ])
179 + .await;
180 + let env = t.env(&data, &url);
181 +
182 + let mut agent = Raw::spawn(&env);
183 + agent.initialize().await;
184 + let sid = agent.new_session(&work, json!([])).await;
185 + let r = agent.text_prompt(&sid, "What key is the chorus in?").await;
186 + assert_eq!(r["stopReason"], "end_turn");
187 + // Title is pushed as a session_info_update.
188 + let infos = agent.updates("session_info_update");
189 + assert!(
190 + infos
191 + .iter()
192 + .any(|u| u["title"] == "What key is the chorus in?"),
193 + "no title update: {infos:?}"
194 + );
195 + // The process dies without a graceful close.
196 + agent.child.kill().await.unwrap();
197 + drop(agent);
198 +
199 + let mut agent = Raw::spawn(&env);
200 + agent.initialize().await;
201 + let list = agent.ok("session/list", json!({})).await;
202 + let sessions = list["sessions"].as_array().unwrap();
203 + assert_eq!(sessions.len(), 1, "{list}");
204 + assert_eq!(sessions[0]["sessionId"], sid.as_str());
205 + assert_eq!(sessions[0]["title"], "What key is the chorus in?");
206 + assert!(sessions[0]["updatedAt"].as_str().unwrap().ends_with('Z'));
207 + // cwd filter.
208 + let other = agent.ok("session/list", json!({"cwd": "/nowhere"})).await;
209 + assert!(other["sessions"].as_array().unwrap().is_empty());
210 +
211 + agent.notes.clear();
212 + agent
213 + .ok(
214 + "session/load",
215 + json!({"sessionId": sid, "cwd": work, "mcpServers": []}),
216 + )
217 + .await;
218 + agent.settle().await;
219 + let user = agent.updates("user_message_chunk");
220 + assert_eq!(user[0]["content"]["text"], "What key is the chorus in?");
221 + let said: String = agent
222 + .updates("agent_message_chunk")
223 + .iter()
224 + .filter_map(|u| u["content"]["text"].as_str())
225 + .collect();
226 + assert!(said.contains("The chorus is in Dm."), "{said}");
227 + let calls = agent.updates("tool_call");
228 + assert_eq!(calls.len(), 2, "{calls:?}");
229 + assert_eq!(calls[0]["status"], "completed");
230 + assert!(
231 + tool_text(&calls[0]).contains("chorus in Dm"),
232 + "{}",
233 + calls[0]
234 + );
235 + assert_eq!(
236 + calls[1]["status"], "failed",
237 + "failed call must replay as failed: {}",
238 + calls[1]
239 + );
240 + assert!(
241 + calls[0]["locations"][0]["path"]
242 + .as_str()
243 + .unwrap()
244 + .ends_with("notes.txt")
245 + );
246 + // Commands are advertised after the load response.
247 + let cmds = agent.updates("available_commands_update");
248 + assert!(!cmds.is_empty());
249 + agent.finish().await;
250 +}
251 +
252 +pub async fn resume_close_delete_and_unknown_sessions(t: &Target) {
253 + let data = temp_dir("sf-data");
254 + let work = temp_dir("sf-work");
255 + let work2 = temp_dir("sf-work");
256 + let (url, stats) = mock(vec![Step::Final("ok")]).await;
257 + let env = t.env(&data, &url);
258 + let mut agent = Raw::spawn(&env);
259 + agent.initialize().await;
260 +
261 + let sid = agent.new_session(&work, json!([])).await;
262 + agent.text_prompt(&sid, "first").await;
263 +
264 + // Unknown sessions are invalid_params where nothing can be reopened. load/resume of a
265 + // well-formed id reopen it instead (covered below); a malformed id can't name a session.
266 + for (method, params) in [
267 + (
268 + "session/load",
269 + json!({"sessionId": "../nope", "cwd": work, "mcpServers": []}),
270 + ),
271 + (
272 + "session/resume",
273 + json!({"sessionId": "../nope", "cwd": work, "mcpServers": []}),
274 + ),
275 + (
276 + "session/load",
277 + json!({"sessionId": "relative-cwd", "cwd": "work", "mcpServers": []}),
278 + ),
279 + ("session/new", json!({"cwd": "work", "mcpServers": []})),
280 + ("session/close", json!({"sessionId": "nope"})),
281 + ("session/list", json!({"cursor": "stale"})),
282 + (
283 + "session/prompt",
284 + json!({"sessionId": "nope", "prompt": [{"type": "text", "text": "x"}]}),
285 + ),
286 + (
287 + "session/set_config_option",
288 + json!({"sessionId": "nope", "configId": "model", "value": "other-model"}),
289 + ),
290 + ] {
291 + let e = agent.call(method, params).await.unwrap_err();
292 + assert_eq!(error_code(&e), -32602, "{method}: {e}");
293 + }
294 +
295 + // Config options: model picker lists the endpoint's models; bad ids are rejected.
296 + let created = agent
297 + .ok("session/new", json!({"cwd": work, "mcpServers": []}))
298 + .await;
299 + let opts = created["configOptions"].as_array().unwrap();
300 + assert_eq!(opts[0]["id"], "model");
301 + assert_eq!(opts[0]["category"], "model");
302 + assert_eq!(opts[0]["currentValue"], MODEL);
303 + let values: Vec<&str> = opts[0]["options"]
304 + .as_array()
305 + .unwrap()
306 + .iter()
307 + .map(|o| o["value"].as_str().unwrap())
308 + .collect();
309 + assert!(
310 + values.contains(&MODEL) && values.contains(&"other-model"),
311 + "{values:?}"
312 + );
313 + let sid2 = created["sessionId"].as_str().unwrap().to_string();
314 + let e = agent
315 + .call(
316 + "session/set_config_option",
317 + json!({"sessionId": sid2, "configId": "model", "value": "bogus"}),
318 + )
319 + .await
320 + .unwrap_err();
321 + assert_eq!(error_code(&e), -32602, "{e}");
322 + let e = agent
323 + .call(
324 + "session/set_config_option",
325 + json!({"sessionId": sid2, "configId": "temperature", "value": "x"}),
326 + )
327 + .await
328 + .unwrap_err();
329 + assert_eq!(error_code(&e), -32602, "{e}");
330 + let r = agent
331 + .ok(
332 + "session/set_config_option",
333 + json!({"sessionId": sid2, "configId": "model", "value": "other-model"}),
334 + )
335 + .await;
336 + assert_eq!(r["configOptions"][0]["currentValue"], "other-model");
337 + agent.text_prompt(&sid2, "second").await;
338 + let last = stats.payloads.lock().unwrap().last().cloned().unwrap();
339 + assert_eq!(last["model"], "other-model");
340 +
341 + // Resume rebinds the cwd without replaying anything.
342 + agent.notes.clear();
343 + agent
344 + .ok(
345 + "session/resume",
346 + json!({"sessionId": sid, "cwd": work2, "mcpServers": []}),
347 + )
348 + .await;
349 + assert!(
350 + agent.updates("user_message_chunk").is_empty(),
351 + "resume must not replay"
352 + );
353 + let listed = agent.ok("session/list", json!({"cwd": work2})).await;
354 + assert_eq!(listed["sessions"].as_array().unwrap().len(), 1);
355 +
356 + // Close keeps the stored conversation; delete removes it.
357 + agent.ok("session/close", json!({"sessionId": sid})).await;
358 + let e = agent.call("session/close", json!({"sessionId": sid})).await;
359 + assert!(
360 + e.is_ok(),
361 + "close of a stored-only session is a no-op, not an error"
362 + );
363 + let all = agent.ok("session/list", json!({})).await;
364 + assert_eq!(all["sessions"].as_array().unwrap().len(), 2);
365 + agent.ok("session/delete", json!({"sessionId": sid})).await;
366 + let all = agent.ok("session/list", json!({})).await;
367 + assert_eq!(all["sessions"].as_array().unwrap().len(), 1);
368 + assert!(!data.join("sessions").join(format!("{sid}.json")).exists());
369 + // Delete is idempotent: a second delete, or one for a session that never existed, succeeds.
370 + agent.ok("session/delete", json!({"sessionId": sid})).await;
371 + agent
372 + .ok("session/delete", json!({"sessionId": "nope"}))
373 + .await;
374 +
375 + // A well-formed id with no saved history (a thread from a process that never saved it)
376 + // reopens empty under the same id, with a notice, instead of failing to launch.
377 + agent.notes.clear();
378 + let r = agent
379 + .ok(
380 + "session/load",
381 + json!({"sessionId": "lost-thread", "cwd": work, "mcpServers": []}),
382 + )
383 + .await;
384 + assert_eq!(r["configOptions"][0]["id"], "model");
385 + let notice = agent.updates("agent_message_chunk");
386 + assert_eq!(notice.len(), 1, "{notice:?}");
387 + assert!(
388 + notice[0]["content"]["text"]
389 + .as_str()
390 + .unwrap()
391 + .contains("starts fresh"),
392 + "{notice:?}"
393 + );
394 + assert!(notice[0]["messageId"].is_string());
395 + agent.text_prompt("lost-thread", "hello again").await;
396 + agent.finish().await;
397 +}
398 +
399 +pub async fn streaming_reports_message_ids_usage_and_inlines_file_links(t: &Target) {
400 + let data = temp_dir("sf-data");
401 + let work = temp_dir("sf-work");
402 + std::fs::write(
403 + work.join("chart.md"),
404 + "Verse: Am F C G\nChorus: F G Em Am\nBridge: Dm E\n",
405 + )
406 + .unwrap();
407 + let (url, stats) = mock(vec![Step::Final("Looks like A minor.")]).await;
408 + let mut env = t.env(&data, &url);
409 + let window = env.var("CONTEXT_WINDOW");
410 + env.set(&window, "1000");
411 + let mut agent = Raw::spawn(&env);
412 + agent.initialize().await;
413 + let sid = agent.new_session(&work, json!([])).await;
414 + let chart = format!("file://{}?column=1#L2:2", work.join("chart.md").display());
415 + agent
416 + .prompt(
417 + &sid,
418 + json!([
419 + {"type": "text", "text": "What key is the chorus in?"},
420 + {"type": "resource_link", "name": "chart.md", "uri": chart},
421 + ]),
422 + )
423 + .await;
424 +
425 + // The selection link is inlined as just that line.
426 + let payload = stats.payloads.lock().unwrap()[0].clone();
427 + assert!(payload["stream_options"]["include_usage"] == true);
428 + let user = payload["messages"].as_array().unwrap().last().unwrap()["content"]
429 + .as_str()
430 + .unwrap()
431 + .to_string();
432 + assert!(user.contains("Chorus: F G Em Am"), "{user}");
433 + assert!(
434 + !user.contains("Verse:") && !user.contains("Bridge:"),
435 + "{user}"
436 + );
437 +
438 + // Every chunk of one assistant message carries the same message id.
439 + let chunks = agent.updates("agent_message_chunk");
440 + assert!(!chunks.is_empty());
441 + assert!(
442 + chunks.iter().all(|c| c["messageId"].is_string()),
443 + "{chunks:?}"
444 + );
445 + // Usage from the endpoint, sized by <PREFIX>_CONTEXT_WINDOW.
446 + let usage = agent.updates("usage_update");
447 + assert_eq!(usage.len(), 1, "{usage:?}");
448 + assert_eq!(
449 + (usage[0]["used"].as_u64(), usage[0]["size"].as_u64()),
450 + (Some(150), Some(1000))
451 + );
452 + agent.finish().await;
453 +}
454 +
455 +pub async fn permissions_reject_always_raw_output_and_auto_approve(t: &Target) {
456 + let data = temp_dir("sf-data");
457 + let work = temp_dir("sf-work");
458 + let write = |path: &'static str| Step::ToolCall {
459 + name: "write_file",
460 + arguments: json!({"path": path, "content": "x"}),
461 + };
462 + let (url, _) = mock(vec![write("a.txt"), write("b.txt"), Step::Final("done")]).await;
463 + let mut agent = Raw::spawn(&t.env(&data, &url));
464 + // A client that supports boolean config options gets the auto-approve toggle.
465 + let init = agent
466 + .ok(
467 + "initialize",
468 + json!({"protocolVersion": 1, "clientCapabilities": {"session": {"configOptions": {"boolean": {}}}}}),
469 + )
470 + .await;
471 + assert_eq!(init["protocolVersion"], 1);
472 + let created = agent
473 + .ok("session/new", json!({"cwd": work, "mcpServers": []}))
474 + .await;
475 + agent.settle().await;
476 + let sid = created["sessionId"].as_str().unwrap().to_string();
477 + let opts = created["configOptions"].as_array().unwrap();
478 + assert_eq!(opts[1]["id"], "auto_approve");
479 + assert_eq!(opts[1]["type"], "boolean");
480 + assert_eq!(opts[1]["currentValue"], false);
481 +
482 + // "Always reject" is offered, and once chosen the tool is refused without asking again.
483 + agent.perm = Perm::RejectAlways;
484 + agent.text_prompt(&sid, "write two files").await;
485 + assert_eq!(
486 + agent.permissions.len(),
487 + 1,
488 + "asked again after reject_always"
489 + );
490 + let kinds: Vec<&str> = agent.permissions[0]["options"]
491 + .as_array()
492 + .unwrap()
493 + .iter()
494 + .map(|o| o["kind"].as_str().unwrap())
495 + .collect();
496 + assert!(kinds.contains(&"reject_always"), "{kinds:?}");
497 + assert!(!work.join("a.txt").exists() && !work.join("b.txt").exists());
498 + // Tool calls carry the tool name; updates carry the raw output the model saw.
499 + let calls = agent.updates("tool_call");
500 + assert!(calls.iter().all(|c| c["name"] == "write_file"), "{calls:?}");
501 + let done = agent.updates("tool_call_update");
502 + let failed: Vec<&Value> = done.iter().filter(|u| u["status"] == "failed").collect();
503 + assert_eq!(failed.len(), 2, "{done:?}");
504 + assert_eq!(failed[0]["rawOutput"]["failed"], true);
505 + assert!(
506 + failed[0]["rawOutput"]["output"]
507 + .as_str()
508 + .unwrap()
509 + .contains("rejected")
510 + );
511 +
512 + // Turning auto-approve on is broadcast as a config_option_update and skips the prompts.
513 + agent.notes.clear();
514 + let r = agent
515 + .ok(
516 + "session/set_config_option",
517 + json!({"sessionId": sid, "configId": "auto_approve", "type": "boolean", "value": true}),
518 + )
519 + .await;
520 + assert_eq!(r["configOptions"][1]["currentValue"], true);
521 + let broadcast = agent.updates("config_option_update");
522 + assert_eq!(broadcast.len(), 1, "{broadcast:?}");
523 + let e = agent
524 + .call(
525 + "session/set_config_option",
526 + json!({"sessionId": sid, "configId": "auto_approve", "value": "yes"}),
527 + )
528 + .await
529 + .unwrap_err();
530 + assert_eq!(error_code(&e), -32602, "{e}");
531 +
532 + // A fresh session (no reject_always grant) now writes without asking.
533 + let sid2 = agent.new_session(&work, json!([])).await;
534 + agent
535 + .ok(
536 + "session/set_config_option",
537 + json!({"sessionId": sid2, "configId": "auto_approve", "type": "boolean", "value": true}),
538 + )
539 + .await;
540 + agent.permissions.clear();
541 + agent.text_prompt(&sid2, "write two files").await;
542 + assert!(agent.permissions.is_empty(), "{:?}", agent.permissions);
543 + assert!(work.join("a.txt").exists() && work.join("b.txt").exists());
544 + agent.finish().await;
545 +}
546 +
547 +pub async fn auto_approve_is_hidden_from_clients_without_boolean_options(t: &Target) {
548 + let data = temp_dir("sf-data");
549 + let work = temp_dir("sf-work");
550 + let (url, _) = mock(vec![Step::Final("x")]).await;
551 + let mut agent = Raw::spawn(&t.env(&data, &url));
552 + agent.initialize().await;
553 + let created = agent
554 + .ok("session/new", json!({"cwd": work, "mcpServers": []}))
555 + .await;
556 + agent.settle().await;
557 + let opts = created["configOptions"].as_array().unwrap();
558 + assert_eq!(opts.len(), 1, "{opts:?}");
559 + let e = agent
560 + .call(
561 + "session/set_config_option",
562 + json!({"sessionId": created["sessionId"], "configId": "auto_approve", "type": "boolean", "value": true}),
563 + )
564 + .await
565 + .unwrap_err();
566 + assert_eq!(error_code(&e), -32602, "{e}");
567 + agent.finish().await;
568 +}
569 +
570 +pub async fn content_filter_is_a_refusal_and_private_flags_stay_off_the_wire(t: &Target) {
571 + let data = temp_dir("sf-data");
572 + let work = temp_dir("sf-work");
573 + let (url, stats) = mock(vec![
574 + Step::ToolCall {
575 + name: "read_file",
576 + arguments: json!({"path": "missing.txt"}),
577 + },
578 + Step::ContentFilter,
579 + ])
580 + .await;
581 + let mut agent = Raw::spawn(&t.env(&data, &url));
582 + agent.initialize().await;
583 + let sid = agent.new_session(&work, json!([])).await;
584 + let r = agent.text_prompt(&sid, "read it").await;
585 + assert_eq!(r["stopReason"], "refusal");
586 + let payloads = stats.payloads.lock().unwrap().clone();
587 + let second = payloads.last().unwrap()["messages"]
588 + .as_array()
589 + .unwrap()
590 + .clone();
591 + let tool = second.iter().find(|m| m["role"] == "tool").unwrap();
592 + assert!(
593 + tool["content"].as_str().unwrap().starts_with("Error:"),
594 + "{tool}"
595 + );
596 + assert!(
597 + tool.get("x_failed").is_none(),
598 + "private field leaked to the model: {tool}"
599 + );
600 + // The system prompt is rebuilt each turn and never stored with the history.
601 + assert_eq!(second[0]["role"], "system");
602 + let stored: Value = serde_json::from_slice(
603 + &std::fs::read(data.join("sessions").join(format!("{sid}.json"))).unwrap(),
604 + )
605 + .unwrap();
606 + assert!(
607 + stored["messages"]
608 + .as_array()
609 + .unwrap()
610 + .iter()
611 + .all(|m| m["role"] != "system")
612 + );
613 + agent.finish().await;
614 +}
615 +
616 +pub async fn client_supplied_mcp_server_is_offered_gated_and_streams_progress(t: &Target) {
617 + let data = temp_dir("sf-data");
618 + let work = temp_dir("sf-work");
619 + let pid_file = data.join("stub.pid");
620 + let (url, stats) = mock(vec![
621 + Step::ToolCall {
622 + name: "mcp__stub__echo",
623 + arguments: json!({"text": "hi"}),
624 + },
625 + Step::ToolCall {
626 + name: "mcp__stub__slow_progress",
627 + arguments: json!({}),
628 + },
629 + Step::Final("all done"),
630 + ])
631 + .await;
632 + let mut agent = Raw::spawn(&t.env(&data, &url));
633 + agent.initialize().await;
634 + let sid = agent
635 + .new_session(
636 + &work,
637 + json!([stub_server(&t.mcp_stub, "stub", Some(&pid_file))]),
638 + )
639 + .await;
640 + let r = agent.text_prompt(&sid, "use the stub").await;
641 + assert_eq!(r["stopReason"], "end_turn");
642 +
643 + // Offered: the namespaced tools ride along with the built-ins.
644 + let payload = stats.payloads.lock().unwrap()[0].clone();
645 + let names: Vec<String> = payload["tools"]
646 + .as_array()
647 + .unwrap()
648 + .iter()
649 + .map(|t| t["function"]["name"].as_str().unwrap().to_string())
650 + .collect();
651 + assert!(names.contains(&"mcp__stub__echo".to_string()), "{names:?}");
652 + assert_eq!(names.len(), t.base_tool_count + 5, "{names:?}");
653 +
654 + // Gated: only the tool without readOnlyHint asked for permission.
655 + assert_eq!(agent.permissions.len(), 1, "{:?}", agent.permissions);
656 + let perm = &agent.permissions[0];
657 + assert_eq!(perm["toolCall"]["toolCallId"], "call_2");
658 +
659 + // Callable, with the result fed back to the model.
660 + let payloads = stats.payloads.lock().unwrap().clone();
661 + let tools: Vec<&Value> = payloads[1]["messages"]
662 + .as_array()
663 + .unwrap()
664 + .iter()
665 + .filter(|m| m["role"] == "tool")
666 + .collect();
667 + assert_eq!(tools[0]["content"], "echo: hi");
668 +
669 + // Progress: MCP notifications/progress became tool_call_update content.
670 + let progress: Vec<String> = agent
671 + .updates("tool_call_update")
672 + .iter()
673 + .filter(|u| u["toolCallId"] == "call_2")
674 + .map(tool_text)
675 + .collect();
676 + assert!(
677 + progress.iter().any(|t| t == "halfway (50%)"),
678 + "{progress:?}"
679 + );
680 + let done = agent
681 + .updates("tool_call_update")
682 + .into_iter()
683 + .rfind(|u| u["toolCallId"] == "call_2")
684 + .unwrap();
685 + assert_eq!(done["status"], "completed");
686 +
687 + // Children are killed when the session closes.
688 + let pid = read_pid(&pid_file);
689 + assert!(alive(pid));
690 + agent.ok("session/close", json!({"sessionId": sid})).await;
691 + assert!(
692 + wait_dead(pid).await,
693 + "stub server still running after session/close"
694 + );
695 + agent.finish().await;
696 +}
697 +
698 +pub async fn mcp_children_die_when_the_agent_exits(t: &Target) {
699 + let data = temp_dir("sf-data");
700 + let work = temp_dir("sf-work");
701 + let pid_file = data.join("stub.pid");
702 + let (url, _) = mock(vec![Step::Final("x")]).await;
703 + let mut agent = Raw::spawn(&t.env(&data, &url));
704 + agent.initialize().await;
705 + agent
706 + .new_session(
707 + &work,
708 + json!([stub_server(&t.mcp_stub, "stub", Some(&pid_file))]),
709 + )
710 + .await;
711 + let pid = read_pid(&pid_file);
712 + assert!(alive(pid));
713 + agent.finish().await;
714 + assert!(wait_dead(pid).await, "stub server outlived the agent");
715 +}
716 +
717 +pub async fn broken_mcp_server_is_not_fatal(t: &Target) {
718 + let data = temp_dir("sf-data");
719 + let work = temp_dir("sf-work");
720 + let (url, stats) = mock(vec![Step::Final("fine")]).await;
721 + let mut agent = Raw::spawn(&t.env(&data, &url));
722 + agent.initialize().await;
723 + let sid = agent
724 + .new_session(
725 + &work,
726 + json!([
727 + {"name": "ghost", "command": "/definitely/not/here", "args": [], "env": []},
728 + {"name": "http", "type": "http", "url": "http://127.0.0.1:1/mcp", "headers": []}
729 + ]),
730 + )
731 + .await;
732 + let r = agent.text_prompt(&sid, "hello").await;
733 + assert_eq!(r["stopReason"], "end_turn");
734 + let payload = stats.payloads.lock().unwrap()[0].clone();
735 + assert_eq!(
736 + payload["tools"].as_array().unwrap().len(),
737 + t.base_tool_count
738 + );
739 + agent.finish().await;
740 +}
741 +
742 +pub async fn cancelling_a_turn_cancels_the_mcp_call(t: &Target) {
743 + let data = temp_dir("sf-data");
744 + let work = temp_dir("sf-work");
745 + let (url, _) = mock(vec![
746 + Step::ToolCall {
747 + name: "mcp__stub__wait_for_cancel",
748 + arguments: json!({}),
749 + },
750 + Step::Final("should not get here"),
751 + ])
752 + .await;
753 + let mut agent = Raw::spawn(&t.env(&data, &url));
754 + agent.initialize().await;
755 + let sid = agent
756 + .new_session(&work, json!([stub_server(&t.mcp_stub, "stub", None)]))
757 + .await;
758 + // Fire the prompt without waiting, then cancel once the call is in flight.
759 + let id = agent.next_id;
760 + agent.next_id += 1;
761 + agent
762 + .send(
763 + json!({"jsonrpc": "2.0", "id": id, "method": "session/prompt",
764 + "params": {"sessionId": sid, "prompt": [{"type": "text", "text": "wait"}]}}),
765 + )
766 + .await;
767 + let mut cancelled = false;
768 + let stop = loop {
769 + let msg = agent.next_message().await;
770 + if msg.get("method").is_some() {
771 + if msg.get("id").is_some() {
772 + agent.serve(&msg).await;
773 + // The permission request means the tool call is about to start.
774 + tokio::time::sleep(Duration::from_millis(300)).await;
775 + agent
776 + .notify("session/cancel", json!({"sessionId": sid}))
777 + .await;
778 + cancelled = true;
779 + } else {
780 + agent.notes.push(msg);
781 + }
782 + } else if msg["id"] == json!(id) {
783 + break msg["result"]["stopReason"].clone();
784 + }
785 + };
786 + assert!(cancelled, "the call should have asked for permission first");
787 + assert_eq!(stop, "cancelled");
788 + let last = agent
789 + .updates("tool_call_update")
790 + .into_iter()
791 + .last()
792 + .unwrap();
793 + assert_eq!(last["status"], "failed");
794 + agent.finish().await;
795 +}
crates/ed-acp-testkit/src/lib.rs new
+155
@@ -0,0 +1,155 @@
1 +//! Test support for Onde ACP agents: a mock Onde Cloud endpoint ([`mock`]), a raw JSON-RPC
2 +//! client that drives the real agent binary over stdio ([`Raw`]), the ACP registry's validator
3 +//! request ([`REGISTRY_VALIDATOR_INITIALIZE`]),
4 +//! and the ACP v1 conformance suite ([`conformance`]).
5 +//!
6 +//! The test MCP server lives in `ed-mcp` (feature `test-server`) because it must be a binary,
7 +//! and binaries cannot use dev-dependencies. An agent adopts the whole suite with one macro call
8 +//! in an integration test:
9 +//!
10 +//! ```ignore
11 +//! fn target() -> ed_acp_testkit::Target {
12 +//! ed_acp_testkit::Target {
13 +//! bin: env!("CARGO_BIN_EXE_my-agent").into(),
14 +//! name: "my-agent",
15 +//! env_prefix: "MY_AGENT",
16 +//! mcp_stub: env!("CARGO_BIN_EXE_mcp_stdio_stub").into(),
17 +//! base_tool_count: 5,
18 +//! env: vec![],
19 +//! }
20 +//! }
21 +//! ed_acp_testkit::conformance_suite!(target);
22 +//! ```
23 +
24 +pub mod conformance;
25 +pub mod mock;
26 +mod raw;
27 +
28 +use std::path::{Path, PathBuf};
29 +
30 +use serde_json::{Value, json};
31 +
32 +pub use mock::{MODEL, MockStats, Script, Step, TEST_KEY, mock, start_mock_llm, temp_dir};
33 +pub use raw::{
34 + Env, Perm, Raw, TIMEOUT, alive, error_code, read_pid, stub_server, tool_text, wait_dead,
35 +};
36 +
37 +/// The agent under test.
38 +#[derive(Debug, Clone)]
39 +pub struct Target {
40 + /// The agent binary, usually `env!("CARGO_BIN_EXE_<name>")`.
41 + pub bin: PathBuf,
42 + /// Expected `agentInfo.name`.
43 + pub name: &'static str,
44 + /// The agent's environment prefix: the suite sets `<PREFIX>_DATA_DIR`, `<PREFIX>_BASE_URL`
45 + /// and `<PREFIX>_MODEL`.
46 + pub env_prefix: &'static str,
47 + /// A binary whose `main` runs `ed_mcp::test_server::run`.
48 + pub mcp_stub: PathBuf,
49 + /// How many tools the agent offers with no MCP server connected.
50 + pub base_tool_count: usize,
51 + /// Extra environment for every run, e.g. switching built-in MCP servers off.
52 + pub env: Vec<(String, String)>,
53 +}
54 +
55 +impl Target {
56 + /// The environment for one run: an isolated data dir, the mock endpoint at `base_url`, the
57 + /// test key, and `PATH` set to the data dir so no stray binaries are found.
58 + pub fn env(&self, data_dir: &Path, base_url: &str) -> Env {
59 + let mut e = Env {
60 + bin: self.bin.clone(),
61 + prefix: self.env_prefix,
62 + vars: Vec::new(),
63 + };
64 + let (data, url, model) = (e.var("DATA_DIR"), e.var("BASE_URL"), e.var("MODEL"));
65 + e.set(&data, data_dir.display().to_string());
66 + e.set(&url, base_url);
67 + e.set(&model, MODEL);
68 + e.set("ONDE_API_KEY", TEST_KEY);
69 + e.set("PATH", data_dir.display().to_string());
70 + for (k, v) in &self.env {
71 + e.set(k, v.clone());
72 + }
73 + e
74 + }
75 +}
76 +
77 +/// The `initialize` params the ACP registry's validator sends, copied from
78 +/// `agentclientprotocol/registry` `.github/workflows/client.py`. It signals terminal-auth
79 +/// support only through `_meta`, never through `auth.terminal`.
80 +pub fn registry_validator_initialize() -> Value {
81 + json!({
82 + "protocolVersion": 1,
83 + "clientInfo": {"name": "ACP Registry Validator", "version": "1.0.0"},
84 + "clientCapabilities": {
85 + "terminal": true,
86 + "fs": {"readTextFile": true, "writeTextFile": true},
87 + "_meta": {"terminal_output": true, "terminal-auth": true}
88 + }
89 + })
90 +}
91 +
92 +/// See [`registry_validator_initialize`].
93 +pub const REGISTRY_VALIDATOR_INITIALIZE: &str = r#"{"protocolVersion":1,"clientInfo":{"name":"ACP Registry Validator","version":"1.0.0"},"clientCapabilities":{"terminal":true,"fs":{"readTextFile":true,"writeTextFile":true},"_meta":{"terminal_output":true,"terminal-auth":true}}}"#;
94 +
95 +/// Generate one `#[tokio::test]` per conformance check, in a module named `acp_conformance`.
96 +/// `$target` names a function, in scope where the macro is called, that returns the [`Target`].
97 +#[macro_export]
98 +macro_rules! conformance_suite {
99 + ($target:ident) => {
100 + mod acp_conformance {
101 + $crate::conformance_suite!(@tests $target;
102 + initialize_negotiates_v1_and_advertises_capabilities,
103 + registry_validator_probe_sees_terminal_auth,
104 + auth_required_then_authenticate_reloads_config,
105 + sessions_survive_a_killed_process_and_replay_tool_output,
106 + resume_close_delete_and_unknown_sessions,
107 + streaming_reports_message_ids_usage_and_inlines_file_links,
108 + permissions_reject_always_raw_output_and_auto_approve,
109 + auto_approve_is_hidden_from_clients_without_boolean_options,
110 + content_filter_is_a_refusal_and_private_flags_stay_off_the_wire,
111 + client_supplied_mcp_server_is_offered_gated_and_streams_progress,
112 + mcp_children_die_when_the_agent_exits,
113 + broken_mcp_server_is_not_fatal,
114 + cancelling_a_turn_cancels_the_mcp_call,
115 + );
116 + }
117 + };
118 + (@tests $target:ident; $($name:ident),* $(,)?) => {
119 + $(
120 + #[tokio::test]
121 + async fn $name() {
122 + $crate::conformance::$name(&super::$target()).await;
123 + }
124 + )*
125 + };
126 +}
127 +
128 +#[cfg(test)]
129 +mod tests {
130 + use super::*;
131 +
132 + #[test]
133 + fn the_constant_and_the_value_agree() {
134 + let parsed: Value = serde_json::from_str(REGISTRY_VALIDATOR_INITIALIZE).unwrap();
135 + assert_eq!(parsed, registry_validator_initialize());
136 + }
137 +
138 + #[test]
139 + fn env_uses_the_prefix_and_target_overrides() {
140 + let t = Target {
141 + bin: "/bin/agent".into(),
142 + name: "agent",
143 + env_prefix: "MY",
144 + mcp_stub: "/bin/stub".into(),
145 + base_tool_count: 0,
146 + env: vec![("MY_MODEL".into(), "custom".into())],
147 + };
148 + let e = t.env(Path::new("/data"), "http://x/v1");
149 + let get = |k: &str| e.vars.iter().find(|(n, _)| n == k).map(|(_, v)| v.as_str());
150 + assert_eq!(get("MY_DATA_DIR"), Some("/data"));
151 + assert_eq!(get("MY_BASE_URL"), Some("http://x/v1"));
152 + assert_eq!(get("MY_MODEL"), Some("custom"));
153 + assert_eq!(get("ONDE_API_KEY"), Some(TEST_KEY));
154 + }
155 +}
crates/ed-acp-testkit/src/mock.rs new
+207
@@ -0,0 +1,207 @@
1 +//! A scripted OpenAI-compatible streaming endpoint that stands in for Onde Cloud.
2 +//!
3 +//! It answers `GET /v1/models` with [`MODEL`] and `other-model`, records every
4 +//! `/chat/completions` body, and replies with the next [`Step`] of its script, chosen by how many
5 +//! tool results the request already carries. Final answers report usage when the request asks
6 +//! for it with `stream_options.include_usage`, as Onde Cloud does.
7 +
8 +use std::path::PathBuf;
9 +use std::sync::atomic::{AtomicUsize, Ordering};
10 +use std::sync::{Arc, Mutex};
11 +
12 +use serde_json::{Value, json};
13 +use tokio::io::{AsyncReadExt, AsyncWriteExt};
14 +use tokio::net::TcpListener;
15 +
16 +/// Dummy Onde credentials; the mock asserts they arrive as the bearer token.
17 +pub const TEST_KEY: &str = "test-app:test-secret";
18 +pub const MODEL: &str = "scripted-test-model";
19 +
20 +#[derive(Debug, Default)]
21 +pub struct MockStats {
22 + pub requests: AtomicUsize,
23 + pub saw_bearer: Mutex<bool>,
24 + /// Every `/chat/completions` request body, in arrival order.
25 + pub payloads: Mutex<Vec<Value>>,
26 +}
27 +
28 +/// What the scripted model does on its next completion.
29 +#[derive(Clone)]
30 +pub enum Step {
31 + /// Emit a tool call, then wait for the tool result.
32 + ToolCall {
33 + name: &'static str,
34 + arguments: Value,
35 + },
36 + /// Emit final assistant text and finish.
37 + Final(&'static str),
38 + /// Stop with `finish_reason: content_filter`.
39 + ContentFilter,
40 +}
41 +
42 +#[derive(Clone)]
43 +pub struct Script(pub Vec<Step>);
44 +
45 +impl Script {
46 + /// Count completed tool results to know how far into the script we are.
47 + fn step_for(&self, messages: &[Value]) -> (usize, Step) {
48 + let done = messages.iter().filter(|m| m["role"] == "tool").count();
49 + (
50 + done,
51 + self.0
52 + .get(done)
53 + .unwrap_or_else(|| self.0.last().expect("empty script"))
54 + .clone(),
55 + )
56 + }
57 +}
58 +
59 +fn sse_chunk(
60 + content: Option<&str>,
61 + tool_call: Option<(&str, &str, &str)>,
62 + finish: Option<&str>,
63 +) -> String {
64 + let mut delta = json!({});
65 + if let Some(c) = content {
66 + delta["content"] = json!(c);
67 + }
68 + if let Some((id, name, args)) = tool_call {
69 + delta["tool_calls"] = json!([{
70 + "index": 0,
71 + "id": id,
72 + "function": { "name": name, "arguments": args },
73 + }]);
74 + }
75 + let chunk = json!({ "choices": [{ "delta": delta, "finish_reason": finish }] });
76 + format!("data: {chunk}\n\n")
77 +}
78 +
79 +async fn read_http_request(socket: &mut tokio::net::TcpStream) -> (String, String) {
80 + let mut buf = Vec::new();
81 + let mut tmp = [0u8; 4096];
82 + let header_end;
83 + let mut content_length = 0usize;
84 + loop {
85 + let n = socket.read(&mut tmp).await.unwrap();
86 + if n == 0 {
87 + panic!("connection closed before headers complete");
88 + }
89 + buf.extend_from_slice(&tmp[..n]);
90 + if let Some(pos) = find_subslice(&buf, b"\r\n\r\n") {
91 + header_end = pos + 4;
92 + let headers = String::from_utf8_lossy(&buf[..pos]).to_string();
93 + for line in headers.lines() {
94 + if let Some(v) = line.to_ascii_lowercase().strip_prefix("content-length:") {
95 + content_length = v.trim().parse().unwrap();
96 + }
97 + }
98 + break;
99 + }
100 + }
101 + while buf.len() < header_end + content_length {
102 + let n = socket.read(&mut tmp).await.unwrap();
103 + if n == 0 {
104 + break;
105 + }
106 + buf.extend_from_slice(&tmp[..n]);
107 + }
108 + let head = String::from_utf8_lossy(&buf[..header_end]).to_string();
109 + let body = String::from_utf8_lossy(&buf[header_end..header_end + content_length]).to_string();
110 + (head, body)
111 +}
112 +
113 +fn find_subslice(haystack: &[u8], needle: &[u8]) -> Option<usize> {
114 + haystack.windows(needle.len()).position(|w| w == needle)
115 +}
116 +
117 +async fn respond(socket: &mut tokio::net::TcpStream, content_type: &str, body: &str) {
118 + let resp = format!(
119 + "HTTP/1.1 200 OK\r\ncontent-type: {content_type}\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
120 + body.len()
121 + );
122 + socket.write_all(resp.as_bytes()).await.unwrap();
123 + socket.shutdown().await.unwrap();
124 +}
125 +
126 +/// Spawn the mock server; returns its base URL (e.g. `http://127.0.0.1:PORT/v1`).
127 +pub async fn start_mock_llm(script: Script, stats: Arc<MockStats>) -> String {
128 + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
129 + let port = listener.local_addr().unwrap().port();
130 + tokio::spawn(async move {
131 + loop {
132 + let Ok((mut socket, _)) = listener.accept().await else {
133 + return;
134 + };
135 + let stats = stats.clone();
136 + let script = script.clone();
137 + tokio::spawn(async move {
138 + let (head, body) = read_http_request(&mut socket).await;
139 + let first = head.lines().next().unwrap_or("").to_string();
140 + if first.starts_with("GET /v1/models") {
141 + let list =
142 + json!({"object": "list", "data": [{"id": MODEL}, {"id": "other-model"}]});
143 + return respond(&mut socket, "application/json", &list.to_string()).await;
144 + }
145 + stats.requests.fetch_add(1, Ordering::SeqCst);
146 + if head
147 + .to_ascii_lowercase()
148 + .contains(&format!("authorization: bearer {TEST_KEY}").to_ascii_lowercase())
149 + {
150 + *stats.saw_bearer.lock().unwrap() = true;
151 + }
152 + assert!(
153 + first.starts_with("POST /v1/chat/completions"),
154 + "unexpected request: {first}"
155 + );
156 + let payload: Value = serde_json::from_str(&body).unwrap();
157 + assert_eq!(payload["stream"], true);
158 + stats.payloads.lock().unwrap().push(payload.clone());
159 + let messages = payload["messages"].as_array().unwrap();
160 + let (done, step) = script.step_for(messages);
161 +
162 + let mut data = String::new();
163 + match step {
164 + Step::ToolCall { name, arguments } => {
165 + let id = format!("call_{}", done + 1);
166 + data.push_str(&sse_chunk(
167 + None,
168 + Some((&id, name, &arguments.to_string())),
169 + None,
170 + ));
171 + data.push_str(&sse_chunk(None, None, Some("tool_calls")));
172 + }
173 + Step::Final(text) => {
174 + data.push_str(&sse_chunk(Some(text), None, None));
175 + data.push_str(&sse_chunk(None, None, Some("stop")));
176 + // `stream_options.include_usage`: a final chunk with no choices.
177 + if payload.get("stream_options").is_some() {
178 + let usage = json!({"choices": [], "usage": {
179 + "prompt_tokens": 120, "completion_tokens": 30, "total_tokens": 150
180 + }});
181 + data.push_str(&format!("data: {usage}\n\n"));
182 + }
183 + }
184 + Step::ContentFilter => {
185 + data.push_str(&sse_chunk(None, None, Some("content_filter")));
186 + }
187 + }
188 + data.push_str("data: [DONE]\n\n");
189 + respond(&mut socket, "text/event-stream", &data).await;
190 + });
191 + }
192 + });
193 + format!("http://127.0.0.1:{port}/v1")
194 +}
195 +
196 +pub fn temp_dir(prefix: &str) -> PathBuf {
197 + let dir = std::env::temp_dir().join(format!("{prefix}-{}", uuid::Uuid::new_v4()));
198 + std::fs::create_dir_all(&dir).unwrap();
199 + dir
200 +}
201 +
202 +/// Start a mock with `steps`; returns its base URL and the recorded requests.
203 +pub async fn mock(steps: Vec<Step>) -> (String, Arc<MockStats>) {
204 + let stats = Arc::new(MockStats::default());
205 + let url = start_mock_llm(Script(steps), stats.clone()).await;
206 + (url, stats)
207 +}
crates/ed-acp-testkit/src/raw.rs new
+289
@@ -0,0 +1,289 @@
1 +//! The raw JSON-RPC test client and its environment.
2 +
3 +use std::collections::VecDeque;
4 +use std::path::{Path, PathBuf};
5 +use std::process::Stdio;
6 +use std::time::Duration;
7 +
8 +use serde_json::{Value, json};
9 +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader};
10 +use tokio::process::{Child, ChildStdin, ChildStdout, Command};
11 +
12 +/// How long to wait for any one message from the agent.
13 +pub const TIMEOUT: Duration = Duration::from_secs(30);
14 +
15 +/// What the fake client does when the agent asks for permission.
16 +#[derive(Debug, Clone, Copy)]
17 +pub enum Perm {
18 + Allow,
19 + Reject,
20 + RejectAlways,
21 +}
22 +
23 +/// An ACP client speaking raw JSON-RPC to the agent's stdio, so tests see exactly what is on
24 +/// the wire. It answers `session/request_permission` per [`Raw::perm`] and fails every other
25 +/// agent-to-client request.
26 +pub struct Raw {
27 + pub child: Child,
28 + stdin: Option<ChildStdin>,
29 + lines: tokio::io::Lines<BufReader<ChildStdout>>,
30 + pub next_id: u64,
31 + /// Notifications (`session/update` etc.), in arrival order.
32 + pub notes: Vec<Value>,
33 + /// Permission requests seen, as the full JSON-RPC `params`.
34 + pub permissions: Vec<Value>,
35 + pub perm: Perm,
36 + queued: VecDeque<Value>,
37 +}
38 +
39 +/// The agent's command and environment for one test. The process gets nothing else from the
40 +/// test's environment.
41 +#[derive(Debug, Clone)]
42 +pub struct Env {
43 + pub bin: PathBuf,
44 + pub prefix: &'static str,
45 + pub vars: Vec<(String, String)>,
46 +}
47 +
48 +impl Env {
49 + /// `<PREFIX>_<suffix>`.
50 + pub fn var(&self, suffix: &str) -> String {
51 + format!("{}_{suffix}", self.prefix)
52 + }
53 + pub fn set(&mut self, k: &str, v: impl Into<String>) -> &mut Self {
54 + self.vars.retain(|(n, _)| n != k);
55 + self.vars.push((k.to_string(), v.into()));
56 + self
57 + }
58 + pub fn unset(&mut self, k: &str) -> &mut Self {
59 + self.vars.retain(|(n, _)| n != k);
60 + self
61 + }
62 +}
63 +
64 +impl Raw {
65 + /// Start the agent with `--acp`.
66 + pub fn spawn(env: &Env) -> Raw {
67 + let mut cmd = Command::new(&env.bin);
68 + cmd.arg("--acp")
69 + .env_clear()
70 + .stdin(Stdio::piped())
71 + .stdout(Stdio::piped())
72 + .stderr(Stdio::inherit())
73 + .kill_on_drop(true);
74 + for (k, v) in &env.vars {
75 + cmd.env(k, v);
76 + }
77 + let mut child = cmd.spawn().expect("spawn agent");
78 + let stdin = child.stdin.take();
79 + let stdout = child.stdout.take().unwrap();
80 + Raw {
81 + child,
82 + stdin,
83 + lines: BufReader::new(stdout).lines(),
84 + next_id: 1,
85 + notes: Vec::new(),
86 + permissions: Vec::new(),
87 + perm: Perm::Allow,
88 + queued: VecDeque::new(),
89 + }
90 + }
91 +
92 + pub async fn send(&mut self, msg: Value) {
93 + let stdin = self.stdin.as_mut().expect("stdin closed");
94 + stdin
95 + .write_all(format!("{msg}\n").as_bytes())
96 + .await
97 + .unwrap();
98 + stdin.flush().await.unwrap();
99 + }
100 +
101 + pub async fn notify(&mut self, method: &str, params: Value) {
102 + self.send(json!({"jsonrpc": "2.0", "method": method, "params": params}))
103 + .await;
104 + }
105 +
106 + pub async fn next_message(&mut self) -> Value {
107 + if let Some(m) = self.queued.pop_front() {
108 + return m;
109 + }
110 + let line = tokio::time::timeout(TIMEOUT, self.lines.next_line())
111 + .await
112 + .expect("timed out waiting for the agent")
113 + .unwrap()
114 + .expect("agent closed stdout");
115 + serde_json::from_str(&line).unwrap_or_else(|e| panic!("non-JSON on stdout ({e}): {line}"))
116 + }
117 +
118 + /// Send a request and wait for its response, serving agent→client requests meanwhile.
119 + pub async fn call(&mut self, method: &str, params: Value) -> Result<Value, Value> {
120 + let id = self.next_id;
121 + self.next_id += 1;
122 + self.send(json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params}))
123 + .await;
124 + loop {
125 + let msg = self.next_message().await;
126 + if msg.get("method").is_some() {
127 + if msg.get("id").is_some() {
128 + self.serve(&msg).await;
129 + } else {
130 + self.notes.push(msg);
131 + }
132 + continue;
133 + }
134 + if msg["id"] == json!(id) {
135 + return match msg.get("error") {
136 + Some(e) => Err(e.clone()),
137 + None => Ok(msg["result"].clone()),
138 + };
139 + }
140 + }
141 + }
142 +
143 + pub async fn serve(&mut self, req: &Value) {
144 + let id = req["id"].clone();
145 + let reply = match req["method"].as_str().unwrap() {
146 + "session/request_permission" => {
147 + self.permissions.push(req["params"].clone());
148 + match self.perm {
149 + Perm::Allow => {
150 + json!({"outcome": {"outcome": "selected", "optionId": "allow_once"}})
151 + }
152 + Perm::Reject => {
153 + json!({"outcome": {"outcome": "selected", "optionId": "reject_once"}})
154 + }
155 + Perm::RejectAlways => {
156 + json!({"outcome": {"outcome": "selected", "optionId": "reject_always"}})
157 + }
158 + }
159 + }
160 + other => {
161 + self.send(json!({"jsonrpc": "2.0", "id": id, "error": {"code": -32601, "message": format!("unsupported {other}")}})).await;
162 + return;
163 + }
164 + };
165 + self.send(json!({"jsonrpc": "2.0", "id": id, "result": reply}))
166 + .await;
167 + }
168 +
169 + /// Read whatever the agent sends right after a response (e.g. the commands update that
170 + /// follows `session/new`) until it has been quiet for a moment.
171 + pub async fn settle(&mut self) {
172 + while let Ok(line) =
173 + tokio::time::timeout(Duration::from_millis(300), self.lines.next_line()).await
174 + {
175 + let Ok(Some(line)) = line else { return };
176 + let msg: Value = serde_json::from_str(&line).unwrap();
177 + if msg.get("id").is_some() && msg.get("method").is_some() {
178 + self.serve(&msg).await;
179 + } else {
180 + self.notes.push(msg);
181 + }
182 + }
183 + }
184 +
185 + pub async fn ok(&mut self, method: &str, params: Value) -> Value {
186 + self.call(method, params)
187 + .await
188 + .unwrap_or_else(|e| panic!("{method} failed: {e}"))
189 + }
190 +
191 + pub async fn initialize(&mut self) -> Value {
192 + self.ok(
193 + "initialize",
194 + json!({"protocolVersion": 1, "clientCapabilities": {}}),
195 + )
196 + .await
197 + }
198 +
199 + pub async fn new_session(&mut self, cwd: &Path, mcp: Value) -> String {
200 + let r = self
201 + .ok("session/new", json!({"cwd": cwd, "mcpServers": mcp}))
202 + .await;
203 + self.settle().await;
204 + r["sessionId"].as_str().unwrap().to_string()
205 + }
206 +
207 + pub async fn prompt(&mut self, sid: &str, blocks: Value) -> Value {
208 + self.ok(
209 + "session/prompt",
210 + json!({"sessionId": sid, "prompt": blocks}),
211 + )
212 + .await
213 + }
214 +
215 + pub async fn text_prompt(&mut self, sid: &str, text: &str) -> Value {
216 + self.prompt(sid, json!([{"type": "text", "text": text}]))
217 + .await
218 + }
219 +
220 + /// Notifications for a session update kind, e.g. `tool_call_update`.
221 + pub fn updates(&self, kind: &str) -> Vec<Value> {
222 + self.notes
223 + .iter()
224 + .filter(|n| {
225 + n["method"] == "session/update" && n["params"]["update"]["sessionUpdate"] == kind
226 + })
227 + .map(|n| n["params"]["update"].clone())
228 + .collect()
229 + }
230 +
231 + /// Close stdin and wait for the agent to exit on its own.
232 + pub async fn finish(mut self) {
233 + drop(self.stdin.take());
234 + let _ = tokio::time::timeout(TIMEOUT, self.child.wait())
235 + .await
236 + .expect("agent did not exit");
237 + }
238 +}
239 +
240 +/// The JSON-RPC error code.
241 +pub fn error_code(e: &Value) -> i64 {
242 + e["code"].as_i64().unwrap()
243 +}
244 +
245 +/// The text content of a tool call or update.
246 +pub fn tool_text(update: &Value) -> String {
247 + update["content"]
248 + .as_array()
249 + .into_iter()
250 + .flatten()
251 + .filter_map(|c| c.pointer("/content/text").and_then(Value::as_str))
252 + .collect::<Vec<_>>()
253 + .join("\n")
254 +}
255 +
256 +pub fn alive(pid: u32) -> bool {
257 + std::process::Command::new("kill")
258 + .args(["-0", &pid.to_string()])
259 + .stderr(Stdio::null())
260 + .status()
261 + .is_ok_and(|s| s.success())
262 +}
263 +
264 +/// Wait up to five seconds for `pid` to exit.
265 +pub async fn wait_dead(pid: u32) -> bool {
266 + for _ in 0..100 {
267 + if !alive(pid) {
268 + return true;
269 + }
270 + tokio::time::sleep(Duration::from_millis(50)).await;
271 + }
272 + false
273 +}
274 +
275 +pub fn read_pid(file: &Path) -> u32 {
276 + std::fs::read_to_string(file)
277 + .expect("stub pid file")
278 + .trim()
279 + .parse()
280 + .unwrap()
281 +}
282 +
283 +/// An `mcpServers` entry for the test MCP server under `name`.
284 +pub fn stub_server(stub: &Path, name: &str, pid_file: Option<&Path>) -> Value {
285 + let env: Vec<Value> = pid_file
286 + .map(|p| vec![json!({"name": "STUB_PID_FILE", "value": p})])
287 + .unwrap_or_default();
288 + json!({"name": name, "command": stub, "args": [], "env": env})
289 +}