3 files changed
+147
-19
crates/xcrs/src/mcp.rs
+21
-7
@@ -584,6 +584,8 @@ pub struct UiTargetArgs {
584
/// Local ControlKit JSON-RPC port. Defaults to 12004.
585
#[serde(default)]
586
pub controlkit_port: Option<u16>,
587
+ /// Bundle identifier of the app whose accessibility hierarchy should be read.
588
+ pub bundle_id: String,
589
}
590
591
#[derive(Debug, Deserialize, JsonSchema)]
@@ -957,9 +959,9 @@ macro_rules! xcrs_mcp_tools {
959
960
#[::rmcp::tool(
961
name = $app_install_launch_name,
960
- title = "Install and launch iOS app",
961
- annotations(title = "Install and launch iOS app", read_only_hint = false, destructive_hint = true, idempotent_hint = false),
962
- description = "Purpose: one-shot end-to-end setup for a freshly built iOS app: boot the simulator, install the .app bundle, optionally terminate a previous instance, launch it, and return its app container path. When to use vs siblings: use this once to get a build under test onto a simulator; use app_launch/app_terminate afterwards for an already-installed app, and screen_capture/ui_describe to inspect it. Behavior: boots the target simulator if it is not already booted, installs app_path, optionally force-terminates bundle_id first, launches bundle_id, then reads back the app's container directory. Prerequisites: Xcode command line tools installed; app_path must point to an existing .app bundle built for the simulator (not a device) architecture. Failure modes: errors if neither simulator_name nor simulator_udid is given, if the simulator cannot be found, or if any underlying `simctl` step fails. Limitations: iOS-simulator-only; there is no Android or physical-device equivalent in this tool."
962
+ title = "Install and launch Apple simulator app",
963
+ annotations(title = "Install and launch Apple simulator app", read_only_hint = false, destructive_hint = true, idempotent_hint = false),
964
+ description = "Purpose: one-shot end-to-end setup for a freshly built Apple-platform app: boot an iOS, tvOS, watchOS, or visionOS simulator, install the .app bundle, optionally terminate a previous instance, launch it, and return its app container path. When to use vs siblings: use this once to get a build under test onto a simulator; use app_launch/app_terminate afterwards for an already-installed app, and screen_capture/ui_describe to inspect it. Behavior: boots the target simulator if it is not already booted, installs app_path, optionally force-terminates bundle_id first, launches bundle_id, then reads back the app's container directory. Prerequisites: Xcode command line tools installed; app_path must point to an existing .app bundle built for the selected simulator platform and architecture. Failure modes: errors if neither simulator_name nor simulator_udid is given, if the simulator cannot be found, if the app bundle targets a different platform, or if any underlying `simctl` step fails. Limitations: simulator-only; there is no Android or physical-device equivalent in this tool."
965
)]
966
async fn app_install_launch(
967
&self,
@@ -1317,7 +1319,7 @@ macro_rules! xcrs_mcp_tools {
1319
name = $ui_describe_name,
1320
title = "Describe UI",
1321
annotations(title = "Describe UI", read_only_hint = true, idempotent_hint = true),
1320
- description = "Purpose: return the full accessibility hierarchy of the foreground app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: calls the resolved target's ControlKit `device.dump.ui` method and returns the raw hierarchy alongside the resolved simulator (if any). Prerequisites: a reachable ControlKit endpoint. Failure modes: errors if no target can be resolved, or if the resolved target is Android (Apple-only tool; ControlKit UI introspection has no Android equivalent), or if the ControlKit call fails."
1322
+ description = "Purpose: return the full accessibility hierarchy of an Apple app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method and returns the raw hierarchy alongside the resolved simulator (if any). Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool; ControlKit UI introspection has no Android equivalent), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable."
1323
)]
1324
async fn ui_describe(
1325
&self,
@@ -1339,7 +1341,13 @@ macro_rules! xcrs_mcp_tools {
1341
)?;
1342
let (simulator, controlkit) = Self::controlkit_for_target(&target)?;
1343
let result = controlkit
1342
- .call("device.dump.ui", ::serde_json::json!({ "format": "json" }))
1344
+ .call(
1345
+ "device.dump.ui",
1346
+ ::serde_json::json!({
1347
+ "format": "json",
1348
+ "bundleId": args.bundle_id,
1349
+ }),
1350
+ )
1351
.await
1352
.map_err(|error| {
1353
::rmcp::model::ErrorData::internal_error(error.to_string(), None)
@@ -1356,7 +1364,7 @@ macro_rules! xcrs_mcp_tools {
1364
name = $ui_element_list_name,
1365
title = "List UI elements",
1366
annotations(title = "List UI elements", read_only_hint = true, idempotent_hint = true),
1359
- description = "Purpose: list just the actionable accessibility elements of the foreground app with their labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: calls the resolved target's ControlKit `device.dump.ui` method, then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: a reachable ControlKit endpoint. Failure modes: errors if no target can be resolved, or if the resolved target is Android (Apple-only tool), or if the ControlKit call fails."
1367
+ description = "Purpose: list just the actionable accessibility elements of an Apple app with their labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method, then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable."
1368
)]
1369
async fn ui_element_list(
1370
&self,
@@ -1378,7 +1386,13 @@ macro_rules! xcrs_mcp_tools {
1386
)?;
1387
let (simulator, controlkit) = Self::controlkit_for_target(&target)?;
1388
let ui = controlkit
1381
- .call("device.dump.ui", ::serde_json::json!({ "format": "json" }))
1389
+ .call(
1390
+ "device.dump.ui",
1391
+ ::serde_json::json!({
1392
+ "format": "json",
1393
+ "bundleId": args.bundle_id,
1394
+ }),
1395
+ )
1396
.await
1397
.map_err(|error| {
1398
::rmcp::model::ErrorData::internal_error(error.to_string(), None)
crates/xcrs/src/xcrs.rs
+119
-10
@@ -10,6 +10,7 @@ use std::time::Duration;
10
pub mod mcp;
11
12
const IOS_SIMULATOR_DESTINATION_PREFIX: &str = "platform=iOS Simulator,id=";
13
+const CONTROLKIT_METHOD_NOT_FOUND: i64 = -32601;
14
15
pub fn encode_base64(data: impl AsRef<[u8]>) -> String {
16
use base64::{engine::general_purpose::STANDARD, Engine};
@@ -46,7 +47,7 @@ fn collect_controlkit_elements(element: &serde_json::Value, elements: &mut Vec<s
47
object
48
.get(*key)
49
.and_then(serde_json::Value::as_str)
49
- .is_some()
50
+ .is_some_and(|value| !value.is_empty())
51
});
52
53
if has_visible_rect && has_identity {
@@ -58,6 +59,10 @@ fn collect_controlkit_elements(element: &serde_json::Value, elements: &mut Vec<s
59
"placeholderValue",
60
"rawIdentifier",
61
"rect",
62
+ "depth",
63
+ "enabled",
64
+ "selected",
65
+ "hittable",
66
];
67
let element = keys
68
.iter()
@@ -126,6 +131,33 @@ impl ControlKit {
131
}
132
133
pub async fn call(&self, method: &str, params: serde_json::Value) -> Result<serde_json::Value> {
134
+ let body = self.send(method, params).await?;
135
+
136
+ if let Some(error) = body.get("error") {
137
+ let runner_info = if error.get("code").and_then(serde_json::Value::as_i64)
138
+ == Some(CONTROLKIT_METHOD_NOT_FOUND)
139
+ && method != "device.info"
140
+ {
141
+ self.send("device.info", serde_json::json!({}))
142
+ .await
143
+ .ok()
144
+ .and_then(|body| body.get("result").cloned())
145
+ } else {
146
+ None
147
+ };
148
+ return Err(anyhow!(controlkit_rpc_error(
149
+ method,
150
+ error,
151
+ runner_info.as_ref()
152
+ )));
153
+ }
154
+
155
+ body.get("result")
156
+ .cloned()
157
+ .ok_or_else(|| anyhow!("ControlKit response for '{method}' did not contain a result"))
158
+ }
159
+
160
+ async fn send(&self, method: &str, params: serde_json::Value) -> Result<serde_json::Value> {
161
let response = self
162
.client
163
.post(format!("{}/rpc", self.base_url))
@@ -150,16 +182,43 @@ impl ControlKit {
182
return Err(anyhow!("ControlKit returned HTTP {}: {}", status, body));
183
}
184
153
- if let Some(error) = body.get("error") {
154
- return Err(anyhow!("ControlKit method '{method}' failed: {error}"));
155
- }
156
-
157
- body.get("result")
158
- .cloned()
159
- .ok_or_else(|| anyhow!("ControlKit response for '{method}' did not contain a result"))
185
+ Ok(body)
186
}
187
}
188
189
+fn controlkit_rpc_error(
190
+ method: &str,
191
+ error: &serde_json::Value,
192
+ runner_info: Option<&serde_json::Value>,
193
+) -> String {
194
+ let code = error
195
+ .get("code")
196
+ .and_then(serde_json::Value::as_i64)
197
+ .unwrap_or_default();
198
+ let message = error
199
+ .get("message")
200
+ .and_then(serde_json::Value::as_str)
201
+ .unwrap_or("unknown JSON-RPC error");
202
+
203
+ if code != CONTROLKIT_METHOD_NOT_FOUND {
204
+ return format!("ControlKit method '{method}' failed ({code}): {message}");
205
+ }
206
+
207
+ let runner = runner_info
208
+ .and_then(|info| info.get("runner"))
209
+ .and_then(serde_json::Value::as_str)
210
+ .unwrap_or("unknown ControlKit runner");
211
+ let protocol = runner_info
212
+ .and_then(|info| info.get("protocolVersion"))
213
+ .and_then(serde_json::Value::as_u64)
214
+ .map(|version| format!(" protocol {version}"))
215
+ .unwrap_or_default();
216
+
217
+ format!(
218
+ "{runner}{protocol} does not implement ControlKit method '{method}'. Rebuild or upgrade xcrs-controlkit, restart its runner, and retry."
219
+ )
220
+}
221
+
222
#[derive(Debug, Clone)]
223
pub struct XcodeCommandLineTools {
224
xcrun_path: PathBuf,
@@ -297,14 +356,14 @@ impl Simctl<'_> {
356
self.list_simulators()?
357
.into_iter()
358
.find(|simulator| simulator.name == name)
300
- .ok_or_else(|| anyhow!("iOS simulator named '{name}' was not found"))
359
+ .ok_or_else(|| anyhow!("Apple simulator named '{name}' was not found"))
360
}
361
362
pub fn find_simulator_by_udid(&self, udid: &str) -> Result<Simulator> {
363
self.list_simulators()?
364
.into_iter()
365
.find(|simulator| simulator.udid == udid)
307
- .ok_or_else(|| anyhow!("iOS simulator with UDID '{udid}' was not found"))
366
+ .ok_or_else(|| anyhow!("Apple simulator with UDID '{udid}' was not found"))
367
}
368
369
pub fn boot(&self, udid: &str) -> Result<()> {
@@ -1273,6 +1332,56 @@ mod tests {
1332
assert_eq!(controlkit.base_url, "http://[fdb4:e020:7377::1]:12006");
1333
}
1334
1335
+ #[test]
1336
+ fn reports_actionable_controlkit_method_mismatch() {
1337
+ let error = serde_json::json!({
1338
+ "code": -32601,
1339
+ "message": "Method not found"
1340
+ });
1341
+ let runner_info = serde_json::json!({
1342
+ "runner": "XCRSControlKit",
1343
+ "protocolVersion": 1
1344
+ });
1345
+
1346
+ let message = controlkit_rpc_error("device.dump.ui", &error, Some(&runner_info));
1347
+
1348
+ assert!(message.contains("XCRSControlKit protocol 1"));
1349
+ assert!(message.contains("device.dump.ui"));
1350
+ assert!(message.contains("Rebuild or upgrade"));
1351
+ }
1352
+
1353
+ #[test]
1354
+ fn extracts_identified_visible_controlkit_elements() {
1355
+ let hierarchy = serde_json::json!({
1356
+ "type": "Application",
1357
+ "rect": { "x": 0, "y": 0, "width": 1920, "height": 1080 },
1358
+ "children": [
1359
+ {
1360
+ "type": "Button",
1361
+ "label": "Play",
1362
+ "rawIdentifier": "play-button",
1363
+ "rect": { "x": 100, "y": 200, "width": 80, "height": 40 },
1364
+ "enabled": true,
1365
+ "selected": false,
1366
+ "hittable": true,
1367
+ "children": []
1368
+ },
1369
+ {
1370
+ "type": "Image",
1371
+ "label": "",
1372
+ "rect": { "x": 0, "y": 0, "width": 40, "height": 40 },
1373
+ "children": []
1374
+ }
1375
+ ]
1376
+ });
1377
+
1378
+ let elements = extract_controlkit_elements(&hierarchy);
1379
+
1380
+ assert_eq!(elements.len(), 1);
1381
+ assert_eq!(elements[0]["label"], serde_json::json!("Play"));
1382
+ assert_eq!(elements[0]["hittable"], serde_json::json!(true));
1383
+ }
1384
+
1385
#[test]
1386
fn parses_device_tunnel_address() {
1387
let output = "• Device Name: iPhone\n• Tunnel IP Address: fd55:33ce:ad87::1\n";
docs/controlkit.md
+7
-2
@@ -117,8 +117,13 @@ expose the same unprefixed tool names.
117
| `input_button` | iOS / tvOS, simulator or physical | Press a Home or tvOS remote button. |
118
| `app_launch`/`app_terminate` | simulator or physical | Launch/terminate an app by bundle ID. On a physical device this calls the runner's `device.apps.launch`/`device.apps.terminate` RPC methods instead of `simctl`. |
119
| `screen_capture` | Simulator or physical | Capture a PNG screenshot. Uses `simctl` for a simulator, or `devicectl device capture screenshot` when a `device` identifier is given. |
120
-| `ui_describe` | All UI-test runners | Read the accessibility hierarchy. |
121
-| `ui_element_list` | All UI-test runners | Read only actionable elements with tap coordinates. |
120
+| `ui_describe` | All UI-test runners | Read the accessibility hierarchy for the supplied `bundle_id`. |
121
+| `ui_element_list` | All UI-test runners | Read actionable elements with tap coordinates for the supplied `bundle_id`. |
122
+
123
+UI introspection requires a recent ControlKit runner that implements
124
+`device.dump.ui`. Both UI tools require the target app's `bundle_id`; this
125
+lets XCTest attach to an already-running app instead of accidentally reading
126
+the ControlKit host app.
127
128
For a local macOS runner, omit simulator fields and pass `host` and
129
`controlkit_port` when needed: