7 files changed
+523
-74
crates/cli/src/mcp_xcrs.rs
+4
-2
@@ -39,6 +39,7 @@ xcrs::xcrs_mcp_tools!(
39
"url_open",
40
"ui_describe",
41
"ui_element_list",
42
+ "ui_tap",
43
"input_tap",
44
"input_text",
45
"input_swipe",
@@ -89,7 +90,7 @@ pub async fn serve() -> Result<()> {
90
mod tests {
91
use super::*;
92
92
- const CANONICAL_TOOL_NAMES: [&str; 18] = [
93
+ const CANONICAL_TOOL_NAMES: [&str; 19] = [
94
"device_list",
95
"device_select",
96
"device_capabilities",
@@ -100,6 +101,7 @@ mod tests {
101
"url_open",
102
"ui_describe",
103
"ui_element_list",
104
+ "ui_tap",
105
"input_tap",
106
"input_text",
107
"input_swipe",
@@ -111,7 +113,7 @@ mod tests {
113
];
114
115
#[test]
114
- fn automation_router_exposes_exactly_the_canonical_eighteen_tools() {
116
+ fn automation_router_exposes_exactly_the_canonical_nineteen_tools() {
117
let tools = AutomationMcpServer::xcrs_tool_router().list_all();
118
let mut names: Vec<&str> = tools.iter().map(|tool| tool.name.as_ref()).collect();
119
names.sort_unstable();
crates/xcrs/README.md
+6
@@ -5,6 +5,8 @@ physical Apple devices, Android phones, and Android TV devices. XCRS combines
5
Xcode, ControlKit, CoreDevice, and adb behind one target-aware automation
6
workflow.
7
8
+https://mobilenext.ai/docs/architecture#how-a-single-action-flows
9
+
10
MCP Registry name: `mcp-name: io.github.smbcloudXYZ/xcrs`
11
12
## Run the MCP server
@@ -18,6 +20,10 @@ Android tools use `adb` from `ANDROID_SDK_ROOT`, `ANDROID_HOME`, the standard
20
Android SDK locations, or `PATH`. Connect a device with USB debugging enabled
21
and authorize the host before selecting it with `device_select`.
22
23
+For Android accessibility-tree inspection and semantic `ui_tap`, install the
24
+optional [XCRS AndroidKit](../../docs/androidkit.md) companion runner. Raw adb
25
+actions remain available without it.
26
+
27
The same automation profile is available from the full smbCloud CLI:
28
29
```sh
crates/xcrs/src/main.rs
+49
-1
@@ -1,7 +1,7 @@
1
use anyhow::{anyhow, Result};
2
use clap::{Parser, Subcommand};
3
use std::path::PathBuf;
4
-use xcrs::{IosAppTest, XcodeCommandLineTools};
4
+use xcrs::{AndroidDebugBridge, AndroidKit, IosAppTest, XcodeCommandLineTools};
5
6
#[derive(Debug, Parser)]
7
#[command(name = "xcrs")]
@@ -19,6 +19,8 @@ struct Cli {
19
enum Commands {
20
#[command(subcommand)]
21
Simulator(SimulatorCommand),
22
+ #[command(subcommand)]
23
+ AndroidKit(AndroidKitCommand),
24
IosAppTest {
25
#[arg(long, conflicts_with = "simulator_udid")]
26
simulator_name: Option<String>,
@@ -35,6 +37,30 @@ enum Commands {
37
},
38
}
39
40
+#[derive(Debug, Subcommand)]
41
+enum AndroidKitCommand {
42
+ Install {
43
+ #[arg(long)]
44
+ serial: String,
45
+ #[arg(long)]
46
+ apk: PathBuf,
47
+ },
48
+ Start {
49
+ #[arg(long)]
50
+ serial: String,
51
+ },
52
+ Status {
53
+ #[arg(long)]
54
+ serial: String,
55
+ #[arg(long)]
56
+ json: bool,
57
+ },
58
+ Stop {
59
+ #[arg(long)]
60
+ serial: String,
61
+ },
62
+}
63
+
64
#[derive(Debug, Subcommand)]
65
enum SimulatorCommand {
66
Find {
@@ -63,6 +89,28 @@ async fn main() -> Result<()> {
89
let tools = XcodeCommandLineTools::new();
90
91
match command {
92
+ Commands::AndroidKit(AndroidKitCommand::Install { serial, apk }) => {
93
+ AndroidDebugBridge::new().install_androidkit(&serial, &apk)?;
94
+ println!("Installed XCRS AndroidKit on {serial}");
95
+ }
96
+ Commands::AndroidKit(AndroidKitCommand::Start { serial }) => {
97
+ AndroidDebugBridge::new().start_androidkit(&serial)?;
98
+ println!("Started XCRS AndroidKit on {serial}");
99
+ }
100
+ Commands::AndroidKit(AndroidKitCommand::Status { serial, json }) => {
101
+ let info = AndroidKit::new()
102
+ .call(serial.clone(), "device.info", serde_json::json!({}))
103
+ .await?;
104
+ if json {
105
+ println!("{}", serde_json::to_string_pretty(&info)?);
106
+ } else {
107
+ println!("XCRS AndroidKit is ready on {serial}");
108
+ }
109
+ }
110
+ Commands::AndroidKit(AndroidKitCommand::Stop { serial }) => {
111
+ AndroidDebugBridge::new().stop_androidkit(&serial)?;
112
+ println!("Stopped XCRS AndroidKit on {serial}");
113
+ }
114
Commands::Simulator(SimulatorCommand::Find { name, json }) => {
115
let simulator = tools.simctl().find_simulator_by_name(&name)?;
116
if json {
crates/xcrs/src/mcp.rs
+162
-66
@@ -314,6 +314,7 @@ pub fn android_capabilities() -> serde_json::Value {
314
"app_install_launch": false,
315
"ui_describe": false,
316
"ui_element_list": false,
317
+ "ui_tap": false,
318
"orientation_get": false,
319
"orientation_set": false,
320
"input_click": false,
@@ -602,11 +603,25 @@ pub struct UiTargetArgs {
603
/// Local ControlKit JSON-RPC port. Defaults to 12004.
604
#[serde(default)]
605
pub controlkit_port: Option<u16>,
605
- /// Bundle identifier of the app whose accessibility hierarchy should be read.
606
+ /// Android device serial from `adb devices -l`.
607
+ #[serde(default)]
608
+ pub android_serial: Option<String>,
609
+ /// Target-neutral app identifier: an Apple bundle identifier or Android package name.
610
+ #[serde(default)]
611
+ pub app_id: Option<String>,
612
+ /// Backward-compatible alias for `app_id` on Apple targets.
613
#[serde(default)]
614
pub bundle_id: Option<String>,
615
}
616
617
+#[derive(Debug, Deserialize, JsonSchema)]
618
+pub struct UiTapArgs {
619
+ #[serde(flatten)]
620
+ pub target: UiTargetArgs,
621
+ /// Exact accessible text, description, identifier, value, or hint to activate.
622
+ pub element: String,
623
+}
624
+
625
#[derive(Debug, Deserialize, JsonSchema)]
626
pub struct OrientationSetArgs {
627
/// Exact Apple simulator name. Omit to use the target selected with
@@ -672,6 +687,7 @@ macro_rules! xcrs_mcp_tools {
687
$url_open_name:literal,
688
$ui_describe_name:literal,
689
$ui_element_list_name:literal,
690
+ $ui_tap_name:literal,
691
$input_tap_name:literal,
692
$input_text_name:literal,
693
$input_swipe_name:literal,
@@ -697,23 +713,29 @@ macro_rules! xcrs_mcp_tools {
713
.and_then(|slot| slot.clone())
714
}
715
700
- fn require_bundle_id(
716
+ fn require_app_id(
717
tool_name: &str,
718
+ app_id: Option<String>,
719
bundle_id: Option<String>,
720
) -> ::std::result::Result<String, ::rmcp::model::ErrorData> {
704
- let bundle_id = bundle_id
705
- .as_deref()
706
- .map(str::trim)
707
- .filter(|bundle_id| !bundle_id.is_empty())
708
- .ok_or_else(|| {
721
+ let app_id = app_id.as_deref().map(str::trim).filter(|value| !value.is_empty());
722
+ let bundle_id = bundle_id.as_deref().map(str::trim).filter(|value| !value.is_empty());
723
+ if let (Some(app_id), Some(bundle_id)) = (app_id, bundle_id) {
724
+ if app_id != bundle_id {
725
+ return Err(::rmcp::model::ErrorData::invalid_request(
726
+ format!("{tool_name} received conflicting app_id and bundle_id values."),
727
+ None,
728
+ ));
729
+ }
730
+ }
731
+ app_id.or(bundle_id).map(str::to_string).ok_or_else(|| {
732
::rmcp::model::ErrorData::invalid_request(
733
format!(
711
- "{tool_name} requires bundle_id. Pass the bundle identifier of the foreground Apple app."
734
+ "{tool_name} requires app_id. Pass the Apple bundle identifier or Android package name of the foreground app."
735
),
736
None,
737
)
715
- })?;
716
- Ok(bundle_id.to_string())
738
+ })
739
}
740
741
/// Validate and tag a target from per-call fields, falling back to the
@@ -947,7 +969,7 @@ macro_rules! xcrs_mcp_tools {
969
name = $device_capabilities_name,
970
title = "Get device capabilities",
971
annotations(title = "Get device capabilities", read_only_hint = true, idempotent_hint = true),
950
- description = "Purpose: report which automation actions the resolved target supports before driving it. When to use vs siblings: call this after device_select (or with explicit target fields) and before UI actions, to confirm e.g. that orientation control or UI introspection is available on this target. Behavior: for an Apple target, forwards to the target's ControlKit `device.capabilities` JSON-RPC method and returns its raw result alongside the resolved simulator (if any); for an Android target, returns a static capability description reflecting exactly the adb actions this crate implements (screen capture, app launch/terminate, URL open, tap/text/swipe/button), since adb has no capability-discovery RPC. Prerequisites: for Apple, a reachable ControlKit endpoint (local companion app for a simulator, or the given host) must be running; for Android, adb must be able to reach the resolved device. Failure modes: returns an error if no target can be resolved, or if the ControlKit call fails or times out."
972
+ description = "Purpose: report which automation actions the resolved target supports before driving it. When to use vs siblings: call this after device_select (or with explicit target fields) and before UI actions, to confirm e.g. that orientation control or UI introspection is available on this target. Behavior: for an Apple target, forwards to the target's ControlKit `device.capabilities` JSON-RPC method and returns its raw result alongside the resolved simulator (if any); for an Android target, starts from the adb actions this crate implements, then probes XCRS AndroidKit and enables semantic UI actions when its instrumentation runner is reachable. Prerequisites: for Apple, a reachable ControlKit endpoint (local companion app for a simulator, or the given host) must be running; for Android, adb must be able to reach the resolved device, and XCRS AndroidKit must be installed and started for semantic UI actions. Failure modes: returns an error if no target can be resolved, or if the ControlKit call fails or times out."
973
)]
974
async fn device_capabilities(
975
&self,
@@ -968,11 +990,34 @@ macro_rules! xcrs_mcp_tools {
990
args.android_serial,
991
)?;
992
match &target {
971
- $crate::mcp::SelectedTarget::Android { .. } => {
993
+ $crate::mcp::SelectedTarget::Android { serial } => {
994
+ let device = Self::android_device_for(serial.clone()).await?;
995
+ let mut capabilities = $crate::mcp::android_capabilities();
996
+ let androidkit = $crate::AndroidKit::new()
997
+ .call(device.serial.clone(), "device.info", ::serde_json::json!({}))
998
+ .await;
999
+ if let Some(object) = capabilities.as_object_mut() {
1000
+ match androidkit {
1001
+ Ok(info) => {
1002
+ object.insert("ui_describe".to_string(), ::serde_json::json!(true));
1003
+ object.insert("ui_element_list".to_string(), ::serde_json::json!(true));
1004
+ object.insert("ui_tap".to_string(), ::serde_json::json!(true));
1005
+ object.insert("androidkit".to_string(), ::serde_json::json!({ "available": true, "info": info }));
1006
+ }
1007
+ Err(error) => {
1008
+ object.insert("ui_tap".to_string(), ::serde_json::json!(false));
1009
+ object.insert("androidkit".to_string(), ::serde_json::json!({
1010
+ "available": false,
1011
+ "setup": "Install and start XCRS AndroidKit with xcrs android-kit install/start.",
1012
+ "error": error.to_string(),
1013
+ }));
1014
+ }
1015
+ }
1016
+ }
1017
Ok(::rmcp::model::CallToolResult::success(vec![
1018
::rmcp::model::ContentBlock::json(&::serde_json::json!({
1019
"platform": "android",
975
- "capabilities": $crate::mcp::android_capabilities(),
1020
+ "capabilities": capabilities,
1021
}))?,
1022
]))
1023
}
@@ -1357,7 +1402,7 @@ macro_rules! xcrs_mcp_tools {
1402
name = $ui_describe_name,
1403
title = "Describe UI",
1404
annotations(title = "Describe UI", read_only_hint = true, idempotent_hint = true),
1360
- description = "Purpose: return the full accessibility hierarchy of an Apple app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method and returns the raw hierarchy alongside the resolved simulator (if any). Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool; ControlKit UI introspection has no Android equivalent), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable."
1405
+ description = "Purpose: return the full accessibility hierarchy of the foreground app as JSON. When to use vs siblings: use this to see everything on screen before tapping or typing; prefer ui_element_list when you only need actionable elements and their tap coordinates, since it is smaller and already filtered. Behavior: for Apple targets, attaches to app_id (or bundle_id) through ControlKit `device.dump.ui`; for Android targets, attaches to app_id through XCRS AndroidKit. Prerequisites: Apple targets need a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; Android targets need XCRS AndroidKit installed and started; app_id must identify the foreground app. Failure modes: errors if no target can be resolved, if app_id is invalid, if the platform runner is unavailable or outdated, or if the requested app is not foreground."
1406
)]
1407
async fn ui_describe(
1408
&self,
@@ -1370,30 +1415,35 @@ macro_rules! xcrs_mcp_tools {
1415
::rmcp::model::CallToolResult,
1416
::rmcp::model::ErrorData,
1417
> {
1373
- let bundle_id = Self::require_bundle_id($ui_describe_name, args.bundle_id)?;
1374
- let target = Self::dispatch_apple_target(
1375
- $ui_describe_name,
1418
+ let app_id = Self::require_app_id($ui_describe_name, args.app_id, args.bundle_id)?;
1419
+ let target = Self::dispatch_target(
1420
args.simulator_name,
1421
args.simulator_udid,
1422
args.host,
1423
args.controlkit_port,
1424
+ args.android_serial,
1425
)?;
1381
- let (simulator, controlkit) = Self::controlkit_for_target(&target)?;
1382
- let result = controlkit
1383
- .call(
1384
- "device.dump.ui",
1385
- ::serde_json::json!({
1386
- "format": "json",
1387
- "bundleId": bundle_id,
1388
- }),
1389
- )
1390
- .await
1391
- .map_err(|error| {
1392
- ::rmcp::model::ErrorData::internal_error(error.to_string(), None)
1393
- })?;
1426
+ let (simulator, result) = match &target {
1427
+ $crate::mcp::SelectedTarget::Android { serial } => {
1428
+ let device = Self::android_device_for(serial.clone()).await?;
1429
+ let ui = $crate::AndroidKit::new()
1430
+ .call(device.serial.clone(), "device.dump.ui", ::serde_json::json!({ "appId": app_id }))
1431
+ .await
1432
+ .map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?;
1433
+ (None, ui)
1434
+ }
1435
+ _ => {
1436
+ let (simulator, controlkit) = Self::controlkit_for_target(&target)?;
1437
+ let ui = controlkit.call("device.dump.ui", ::serde_json::json!({
1438
+ "format": "json", "bundleId": app_id,
1439
+ })).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?;
1440
+ (simulator, ui)
1441
+ }
1442
+ };
1443
Ok(::rmcp::model::CallToolResult::success(vec![
1444
::rmcp::model::ContentBlock::json(&::serde_json::json!({
1445
"simulator": simulator,
1446
+ "target": target,
1447
"ui": result,
1448
}))?,
1449
]))
@@ -1403,7 +1453,7 @@ macro_rules! xcrs_mcp_tools {
1453
name = $ui_element_list_name,
1454
title = "List UI elements",
1455
annotations(title = "List UI elements", read_only_hint = true, idempotent_hint = true),
1406
- description = "Purpose: list just the actionable accessibility elements of an Apple app with their labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: attaches to bundle_id through the resolved target's ControlKit `device.dump.ui` method, then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; bundle_id must identify an installed app. Failure modes: errors if no target can be resolved, if the resolved target is Android (Apple-only tool), if bundle_id is invalid, or if the ControlKit runner is outdated or unavailable."
1456
+ description = "Purpose: list just the actionable accessibility elements of the foreground app with labels and tap coordinates. When to use vs siblings: use this to decide where to tap; use ui_describe when you need the full hierarchy instead of a filtered, flatter list. Behavior: for Apple targets, attaches to app_id (or bundle_id) through ControlKit `device.dump.ui`; for Android targets, attaches to app_id through XCRS AndroidKit; then filters to elements that have both a visible rect and an identifying label/name/value/rawIdentifier. Prerequisites: Apple targets need a reachable ControlKit endpoint built from a version that implements `device.dump.ui`; Android targets need XCRS AndroidKit installed and started; app_id must identify the foreground app. Failure modes: errors if no target can be resolved, if app_id is invalid, if the platform runner is unavailable or outdated, or if the requested app is not foreground."
1457
)]
1458
async fn ui_element_list(
1459
&self,
@@ -1416,36 +1466,76 @@ macro_rules! xcrs_mcp_tools {
1466
::rmcp::model::CallToolResult,
1467
::rmcp::model::ErrorData,
1468
> {
1419
- let bundle_id = Self::require_bundle_id($ui_element_list_name, args.bundle_id)?;
1420
- let target = Self::dispatch_apple_target(
1421
- $ui_element_list_name,
1422
- args.simulator_name,
1423
- args.simulator_udid,
1424
- args.host,
1425
- args.controlkit_port,
1469
+ let app_id = Self::require_app_id($ui_element_list_name, args.app_id, args.bundle_id)?;
1470
+ let target = Self::dispatch_target(
1471
+ args.simulator_name, args.simulator_udid, args.host, args.controlkit_port, args.android_serial,
1472
)?;
1427
- let (simulator, controlkit) = Self::controlkit_for_target(&target)?;
1428
- let ui = controlkit
1429
- .call(
1430
- "device.dump.ui",
1431
- ::serde_json::json!({
1432
- "format": "json",
1433
- "bundleId": bundle_id,
1434
- }),
1435
- )
1436
- .await
1437
- .map_err(|error| {
1438
- ::rmcp::model::ErrorData::internal_error(error.to_string(), None)
1439
- })?;
1440
- let elements = $crate::extract_controlkit_elements(&ui);
1473
+ let (simulator, elements) = match &target {
1474
+ $crate::mcp::SelectedTarget::Android { serial } => {
1475
+ let device = Self::android_device_for(serial.clone()).await?;
1476
+ let ui = $crate::AndroidKit::new().call(
1477
+ device.serial.clone(), "device.dump.ui", ::serde_json::json!({ "appId": app_id }),
1478
+ ).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?;
1479
+ (None, $crate::extract_androidkit_elements(&ui))
1480
+ }
1481
+ _ => {
1482
+ let (simulator, controlkit) = Self::controlkit_for_target(&target)?;
1483
+ let ui = controlkit.call("device.dump.ui", ::serde_json::json!({
1484
+ "format": "json", "bundleId": app_id,
1485
+ })).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?;
1486
+ (simulator, $crate::extract_controlkit_elements(&ui))
1487
+ }
1488
+ };
1489
Ok(::rmcp::model::CallToolResult::success(vec![
1490
::rmcp::model::ContentBlock::json(&::serde_json::json!({
1491
"simulator": simulator,
1492
+ "target": target,
1493
"elements": elements,
1494
}))?,
1495
]))
1496
}
1497
1498
+ #[::rmcp::tool(
1499
+ name = $ui_tap_name,
1500
+ title = "Tap UI element",
1501
+ annotations(title = "Tap UI element", read_only_hint = false, destructive_hint = false, idempotent_hint = false),
1502
+ description = "Purpose: activate one uniquely identified accessible element in the foreground app. When to use vs siblings: use this for semantic interaction by label or identifier; use input_tap only when you deliberately need raw screen coordinates. Behavior: accepts app_id (or Apple-compatible bundle_id) and an exact element value. Apple targets use ControlKit; Android targets use XCRS AndroidKit over adb forwarding. Prerequisites: the app is foreground and its platform runner is installed and reachable. Failure modes: returns an error when the app is not foreground, AndroidKit/ControlKit is unavailable, or zero or multiple elements match."
1503
+ )]
1504
+ async fn ui_tap(
1505
+ &self,
1506
+ ::rmcp::handler::server::wrapper::Parameters(args): ::rmcp::handler::server::wrapper::Parameters<$crate::mcp::UiTapArgs>,
1507
+ ) -> ::std::result::Result<::rmcp::model::CallToolResult, ::rmcp::model::ErrorData> {
1508
+ let app_id = Self::require_app_id($ui_tap_name, args.target.app_id, args.target.bundle_id)?;
1509
+ let element = args.element;
1510
+ if element.trim().is_empty() {
1511
+ return Err(::rmcp::model::ErrorData::invalid_request("ui_tap requires a non-empty element.", None));
1512
+ }
1513
+ let target = Self::dispatch_target(
1514
+ args.target.simulator_name,
1515
+ args.target.simulator_udid,
1516
+ args.target.host,
1517
+ args.target.controlkit_port,
1518
+ args.target.android_serial,
1519
+ )?;
1520
+ match &target {
1521
+ $crate::mcp::SelectedTarget::Android { serial } => {
1522
+ let device = Self::android_device_for(serial.clone()).await?;
1523
+ $crate::AndroidKit::new().call(
1524
+ device.serial.clone(), "device.ui.tap", ::serde_json::json!({ "appId": app_id, "element": element }),
1525
+ ).await.map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?;
1526
+ }
1527
+ _ => {
1528
+ let (_, controlkit) = Self::controlkit_for_target(&target)?;
1529
+ controlkit.call("device.ui.tap", ::serde_json::json!({ "bundleId": app_id, "element": element }))
1530
+ .await
1531
+ .map_err(|error| ::rmcp::model::ErrorData::internal_error(error.to_string(), None))?;
1532
+ }
1533
+ }
1534
+ Ok(::rmcp::model::CallToolResult::success(vec![
1535
+ ::rmcp::model::ContentBlock::text(format!("Tapped '{}' on {}.", element, Self::target_label(&target, &None))),
1536
+ ]))
1537
+ }
1538
+
1539
#[::rmcp::tool(
1540
name = $input_tap_name,
1541
title = "Tap (iOS/tvOS/Android)",
@@ -1900,6 +1990,7 @@ xcrs_mcp_tools!(
1990
"url_open",
1991
"ui_describe",
1992
"ui_element_list",
1993
+ "ui_tap",
1994
"input_tap",
1995
"input_text",
1996
"input_swipe",
@@ -1922,7 +2013,7 @@ impl ServerHandler for XcrsMcpServer {
2013
.with_instructions(
2014
"xcrs exposes a canonical cross-platform automation contract (device_list, \
2015
device_select, device_capabilities, app_install_launch, screen_capture, \
1925
- app_launch, app_terminate, url_open, ui_describe, ui_element_list, input_tap, \
2016
+ app_launch, app_terminate, url_open, ui_describe, ui_element_list, ui_tap, input_tap, \
2017
input_text, input_swipe, input_button, input_click, input_spatial_tap, \
2018
orientation_get, orientation_set) over Xcode/simctl/devicectl/ControlKit for \
2019
Apple platforms and adb for Android.",
@@ -1946,9 +2037,9 @@ pub async fn serve() -> Result<()> {
2037
mod tests {
2038
use super::*;
2039
1949
- /// The 18 canonical tool names shared by the standalone and embedded
2040
+ /// The 19 canonical tool names shared by the standalone and embedded
2041
/// automation servers.
1951
- const CANONICAL_TOOL_NAMES: [&str; 18] = [
2042
+ const CANONICAL_TOOL_NAMES: [&str; 19] = [
2043
"device_list",
2044
"device_select",
2045
"device_capabilities",
@@ -1959,6 +2050,7 @@ mod tests {
2050
"url_open",
2051
"ui_describe",
2052
"ui_element_list",
2053
+ "ui_tap",
2054
"input_tap",
2055
"input_text",
2056
"input_swipe",
@@ -1969,7 +2061,7 @@ mod tests {
2061
"orientation_set",
2062
];
2063
1972
- /// Fully-qualified tool names this crate used before the 18-tool
2064
+ /// Fully-qualified tool names this crate used before the canonical-tool
2065
/// consolidation (e.g. `xcrs_list_simulators`, `xcrs_use_target`,
2066
/// `xcrs_gesture`). None of these must reappear as a canonical name; a
2067
/// match here means a rename regressed back to pre-consolidation naming.
@@ -2005,7 +2097,7 @@ mod tests {
2097
/// types it does not own, so they intentionally advertise no
2098
/// `output_schema`. Only `device_select` returns structured content built
2099
/// entirely from types this module owns, so it is the only tool with one.
2008
- const TOOLS_WITHOUT_OUTPUT_SCHEMA: [&str; 17] = [
2100
+ const TOOLS_WITHOUT_OUTPUT_SCHEMA: [&str; 18] = [
2101
"device_list",
2102
"device_capabilities",
2103
"app_install_launch",
@@ -2015,6 +2107,7 @@ mod tests {
2107
"url_open",
2108
"ui_describe",
2109
"ui_element_list",
2110
+ "ui_tap",
2111
"input_tap",
2112
"input_text",
2113
"input_swipe",
@@ -2026,7 +2119,7 @@ mod tests {
2119
];
2120
2121
#[test]
2029
- fn tool_router_exposes_exactly_the_eighteen_canonical_tools() {
2122
+ fn tool_router_exposes_exactly_the_nineteen_canonical_tools() {
2123
let tools = XcrsMcpServer::xcrs_tool_router().list_all();
2124
2125
assert_eq!(
@@ -2192,9 +2285,9 @@ mod tests {
2285
}
2286
2287
#[test]
2195
- fn ui_tools_keep_bundle_id_optional_in_the_input_schema() {
2288
+ fn ui_tools_keep_app_identifiers_optional_in_the_input_schema() {
2289
let tools = XcrsMcpServer::xcrs_tool_router().list_all();
2197
- for tool_name in ["ui_describe", "ui_element_list"] {
2290
+ for tool_name in ["ui_describe", "ui_element_list", "ui_tap"] {
2291
let tool = tools
2292
.iter()
2293
.find(|tool| tool.name.as_ref() == tool_name)
@@ -2207,7 +2300,9 @@ mod tests {
2300
.unwrap_or_default();
2301
2302
assert!(tool.input_schema["properties"].get("bundle_id").is_some());
2303
+ assert!(tool.input_schema["properties"].get("app_id").is_some());
2304
assert!(!required.contains(&serde_json::json!("bundle_id")));
2305
+ assert!(!required.contains(&serde_json::json!("app_id")));
2306
}
2307
}
2308
@@ -2237,20 +2332,21 @@ mod tests {
2332
}
2333
2334
#[test]
2240
- fn ui_target_args_accept_omitted_bundle_id_and_validate_it_explicitly() {
2335
+ fn ui_target_args_accept_omitted_app_id_and_validate_it_explicitly() {
2336
let args: UiTargetArgs =
2337
serde_json::from_value(serde_json::json!({})).expect("arguments should deserialize");
2338
2339
assert!(args.bundle_id.is_none());
2245
- let error = XcrsMcpServer::require_bundle_id("ui_describe", args.bundle_id)
2246
- .expect_err("missing bundle_id should fail validation");
2247
- assert!(error.to_string().contains("ui_describe requires bundle_id"));
2340
+ let error = XcrsMcpServer::require_app_id("ui_describe", args.app_id, args.bundle_id)
2341
+ .expect_err("missing app_id should fail validation");
2342
+ assert!(error.to_string().contains("ui_describe requires app_id"));
2343
assert_eq!(
2249
- XcrsMcpServer::require_bundle_id(
2344
+ XcrsMcpServer::require_app_id(
2345
"ui_describe",
2346
+ Some(" com.example.app ".to_string()),
2347
Some(" com.example.app ".to_string())
2348
)
2253
- .expect("non-empty bundle_id should pass validation"),
2349
+ .expect("matching app identifiers should pass validation"),
2350
"com.example.app"
2351
);
2352
}
crates/xcrs/src/xcrs.rs
+269
@@ -25,6 +25,82 @@ pub fn extract_controlkit_elements(root: &serde_json::Value) -> Vec<serde_json::
25
elements
26
}
27
28
+/// Normalize actionable AndroidKit nodes into the public element-list shape.
29
+pub fn extract_androidkit_elements(root: &serde_json::Value) -> Vec<serde_json::Value> {
30
+ let mut elements = Vec::new();
31
+ if let Some(hierarchy) = root.get("hierarchy").and_then(serde_json::Value::as_array) {
32
+ for element in hierarchy {
33
+ collect_androidkit_elements(element, &mut elements);
34
+ }
35
+ } else {
36
+ collect_androidkit_elements(root, &mut elements);
37
+ }
38
+ elements
39
+}
40
+
41
+fn collect_androidkit_elements(element: &serde_json::Value, elements: &mut Vec<serde_json::Value>) {
42
+ let Some(object) = element.as_object() else {
43
+ return;
44
+ };
45
+
46
+ let rect = object.get("rect").and_then(serde_json::Value::as_object);
47
+ let has_visible_rect = object
48
+ .get("visible")
49
+ .and_then(serde_json::Value::as_bool)
50
+ .unwrap_or(false)
51
+ && rect
52
+ .and_then(|rect| {
53
+ Some((
54
+ rect.get("x")?.as_f64()?,
55
+ rect.get("y")?.as_f64()?,
56
+ rect.get("width")?.as_f64()?,
57
+ rect.get("height")?.as_f64()?,
58
+ ))
59
+ })
60
+ .is_some_and(|(_, _, width, height)| width > 0.0 && height > 0.0);
61
+ let text = object
62
+ .get("text")
63
+ .and_then(serde_json::Value::as_str)
64
+ .unwrap_or_default();
65
+ let description = object
66
+ .get("contentDescription")
67
+ .and_then(serde_json::Value::as_str)
68
+ .unwrap_or_default();
69
+ let resource_id = object
70
+ .get("resourceId")
71
+ .and_then(serde_json::Value::as_str)
72
+ .unwrap_or_default();
73
+ let hint = object
74
+ .get("hint")
75
+ .and_then(serde_json::Value::as_str)
76
+ .unwrap_or_default();
77
+
78
+ if has_visible_rect
79
+ && [text, description, resource_id, hint]
80
+ .iter()
81
+ .any(|value| !value.is_empty())
82
+ {
83
+ elements.push(serde_json::json!({
84
+ "type": object.get("role").cloned().unwrap_or(serde_json::Value::Null),
85
+ "label": if !text.is_empty() { text } else { description },
86
+ "name": resource_id,
87
+ "value": text,
88
+ "placeholderValue": hint,
89
+ "rawIdentifier": resource_id,
90
+ "rect": object.get("rect").cloned().unwrap_or(serde_json::Value::Null),
91
+ "enabled": object.get("enabled").cloned().unwrap_or(serde_json::Value::Bool(false)),
92
+ "selected": object.get("selected").cloned().unwrap_or(serde_json::Value::Bool(false)),
93
+ "hittable": object.get("clickable").cloned().unwrap_or(serde_json::Value::Bool(false)),
94
+ }));
95
+ }
96
+
97
+ if let Some(children) = object.get("children").and_then(serde_json::Value::as_array) {
98
+ for child in children {
99
+ collect_androidkit_elements(child, elements);
100
+ }
101
+ }
102
+}
103
+
104
fn collect_controlkit_elements(element: &serde_json::Value, elements: &mut Vec<serde_json::Value>) {
105
let object = match element.as_object() {
106
Some(object) => object,
@@ -609,6 +685,92 @@ pub struct AndroidDebugBridge {
685
adb_path: PathBuf,
686
}
687
688
+/// Client for an XCRS AndroidKit instrumentation runner forwarded over adb.
689
+#[derive(Debug, Clone)]
690
+pub struct AndroidKit {
691
+ client: reqwest::Client,
692
+}
693
+
694
+impl Default for AndroidKit {
695
+ fn default() -> Self {
696
+ Self {
697
+ client: reqwest::Client::new(),
698
+ }
699
+ }
700
+}
701
+
702
+impl AndroidKit {
703
+ pub const PACKAGE_NAME: &'static str = "xyz.smbcloud.xcrs.androidkit";
704
+
705
+ pub fn new() -> Self {
706
+ Self::default()
707
+ }
708
+
709
+ pub async fn call(
710
+ &self,
711
+ serial: String,
712
+ method: &str,
713
+ params: serde_json::Value,
714
+ ) -> Result<serde_json::Value> {
715
+ let bridge = AndroidDebugBridge::new();
716
+ let local_port = tokio::task::spawn_blocking({
717
+ let serial = serial.clone();
718
+ move || bridge.forward_androidkit(&serial)
719
+ })
720
+ .await
721
+ .context("AndroidKit forwarding task failed")??;
722
+
723
+ let response = self
724
+ .client
725
+ .post(format!("http://127.0.0.1:{local_port}/rpc"))
726
+ .timeout(Duration::from_secs(10))
727
+ .json(&serde_json::json!({
728
+ "jsonrpc": "2.0",
729
+ "id": 1,
730
+ "method": method,
731
+ "params": params,
732
+ }))
733
+ .send()
734
+ .await
735
+ .with_context(|| format!("failed to connect to XCRS AndroidKit for {serial}"));
736
+
737
+ let cleanup_bridge = AndroidDebugBridge::new();
738
+ let cleanup_serial = serial.clone();
739
+ let cleanup = tokio::task::spawn_blocking(move || {
740
+ cleanup_bridge.remove_forward(&cleanup_serial, local_port)
741
+ })
742
+ .await
743
+ .context("AndroidKit forwarding cleanup task failed")?;
744
+ cleanup.context("failed to remove AndroidKit adb forward")?;
745
+
746
+ let response = response?;
747
+ let status = response.status();
748
+ let body: serde_json::Value = response
749
+ .json()
750
+ .await
751
+ .context("failed to decode AndroidKit JSON-RPC response")?;
752
+ if !status.is_success() {
753
+ return Err(anyhow!("AndroidKit returned HTTP {status}: {body}"));
754
+ }
755
+ if let Some(error) = body.get("error") {
756
+ let code = error
757
+ .get("code")
758
+ .and_then(serde_json::Value::as_i64)
759
+ .unwrap_or_default();
760
+ let message = error
761
+ .get("message")
762
+ .and_then(serde_json::Value::as_str)
763
+ .unwrap_or("unknown JSON-RPC error");
764
+ return Err(anyhow!(
765
+ "AndroidKit method '{method}' failed ({code}): {message}"
766
+ ));
767
+ }
768
+ body.get("result")
769
+ .cloned()
770
+ .ok_or_else(|| anyhow!("AndroidKit response for '{method}' did not contain a result"))
771
+ }
772
+}
773
+
774
impl Default for AndroidDebugBridge {
775
fn default() -> Self {
776
Self {
@@ -772,6 +934,86 @@ impl AndroidDebugBridge {
934
Ok(())
935
}
936
937
+ pub fn install_androidkit(&self, serial: &str, apk_path: &Path) -> Result<()> {
938
+ if !apk_path.is_file() {
939
+ return Err(anyhow!(
940
+ "AndroidKit APK does not exist: {}",
941
+ apk_path.display()
942
+ ));
943
+ }
944
+ run_command(
945
+ &self.adb_path,
946
+ [
947
+ "-s",
948
+ serial,
949
+ "install",
950
+ "-r",
951
+ apk_path
952
+ .to_str()
953
+ .ok_or_else(|| anyhow!("AndroidKit APK path is not valid UTF-8"))?,
954
+ ],
955
+ )?;
956
+ Ok(())
957
+ }
958
+
959
+ pub fn start_androidkit(&self, serial: &str) -> Result<()> {
960
+ Command::new(&self.adb_path)
961
+ .args([
962
+ "-s",
963
+ serial,
964
+ "shell",
965
+ "am",
966
+ "instrument",
967
+ "-w",
968
+ "-r",
969
+ "xyz.smbcloud.xcrs.androidkit/.AndroidKitInstrumentation",
970
+ ])
971
+ .stdout(Stdio::null())
972
+ .stderr(Stdio::null())
973
+ .spawn()
974
+ .with_context(|| format!("failed to start AndroidKit on {serial}"))?;
975
+ Ok(())
976
+ }
977
+
978
+ pub fn stop_androidkit(&self, serial: &str) -> Result<()> {
979
+ self.run_shell(
980
+ serial,
981
+ &format!("am force-stop {}", AndroidKit::PACKAGE_NAME),
982
+ )?;
983
+ Ok(())
984
+ }
985
+
986
+ fn forward_androidkit(&self, serial: &str) -> Result<u16> {
987
+ let output = run_command(
988
+ &self.adb_path,
989
+ [
990
+ "-s",
991
+ serial,
992
+ "forward",
993
+ "tcp:0",
994
+ "localabstract:xcrs-androidkit",
995
+ ],
996
+ )?;
997
+ output
998
+ .trim()
999
+ .parse::<u16>()
1000
+ .with_context(|| format!("adb returned an invalid AndroidKit forward port: {output:?}"))
1001
+ }
1002
+
1003
+ fn remove_forward(&self, serial: &str, local_port: u16) -> Result<()> {
1004
+ run_command(
1005
+ &self.adb_path,
1006
+ [
1007
+ "-s",
1008
+ serial,
1009
+ "forward",
1010
+ "--remove",
1011
+ &format!("tcp:{local_port}"),
1012
+ ],
1013
+ )?;
1014
+ Ok(())
1015
+ }
1016
+
1017
fn run_shell(&self, serial: &str, command: &str) -> Result<String> {
1018
run_command(&self.adb_path, ["-s", serial, "shell", command])
1019
}
@@ -1455,6 +1697,33 @@ mod tests {
1697
assert_eq!(elements[0]["hittable"], serde_json::json!(true));
1698
}
1699
1700
+ #[test]
1701
+ fn normalizes_visible_androidkit_elements() {
1702
+ let hierarchy = serde_json::json!({
1703
+ "hierarchy": [{
1704
+ "role": "android.widget.Button",
1705
+ "text": "Continue",
1706
+ "hint": "",
1707
+ "contentDescription": "",
1708
+ "resourceId": "com.example:id/continue",
1709
+ "enabled": true,
1710
+ "clickable": true,
1711
+ "selected": false,
1712
+ "visible": true,
1713
+ "rect": { "x": 10, "y": 20, "width": 100, "height": 40 }
1714
+ }]
1715
+ });
1716
+
1717
+ let elements = extract_androidkit_elements(&hierarchy);
1718
+
1719
+ assert_eq!(elements.len(), 1);
1720
+ assert_eq!(elements[0]["label"], serde_json::json!("Continue"));
1721
+ assert_eq!(
1722
+ elements[0]["rawIdentifier"],
1723
+ serde_json::json!("com.example:id/continue")
1724
+ );
1725
+ }
1726
+
1727
#[test]
1728
fn parses_device_tunnel_address() {
1729
let output = "• Device Name: iPhone\n• Tunnel IP Address: fd55:33ce:ad87::1\n";
docs/androidkit.md
new
+27
@@ -0,0 +1,27 @@
1
+# XCRS AndroidKit
2
+
3
+XCRS AndroidKit is the optional Android companion runner for semantic E2E
4
+automation. It is separate from adb: adb remains sufficient for screenshots,
5
+coordinate input, app lifecycle, and system buttons.
6
+
7
+Build an AndroidKit APK from the public `xcrs-androidkit` repository, then
8
+install and start it on the selected Android device:
9
+
10
+```sh
11
+xcrs android-kit install --serial <serial> --apk <path-to-apk>
12
+xcrs android-kit start --serial <serial>
13
+xcrs android-kit status --serial <serial>
14
+```
15
+
16
+AndroidKit binds only to an Android local socket. xcrs creates and removes its
17
+own `adb forward` connection for each RPC call; it does not expose a network
18
+service on the device.
19
+
20
+When AndroidKit is available, `ui_describe`, `ui_element_list`, and `ui_tap`
21
+accept `app_id` as the Android package name. `ui_tap` uses an exact accessible
22
+text, content description, resource identifier, or hint and fails if it finds
23
+zero or more than one visible enabled target. The app must be foreground.
24
+
25
+`bundle_id` remains accepted as a compatibility alias for Apple callers. New
26
+cross-platform clients should send `app_id`; passing different `app_id` and
27
+`bundle_id` values is invalid.
docs/mcp.md
+6
-5
@@ -276,8 +276,8 @@ them by server.
276
2. Call `device_select` with the Apple simulator/runner details or Android adb
277
serial. Later actions use that target by default.
278
3. Call `device_capabilities` before choosing an input or inspection action.
279
-4. Use `screen_capture` or the Apple-only `ui_describe`/`ui_element_list` to
280
- inspect the app, then drive it with the `input_*` tools.
279
+4. Use `screen_capture` or `ui_describe`/`ui_element_list` to
280
+ inspect the app, then use semantic `ui_tap` or the raw `input_*` tools.
281
282
| Tool | Purpose |
283
| --- | --- |
@@ -288,14 +288,15 @@ them by server.
288
| `app_launch` / `app_terminate` | Start or stop an installed app on the selected target. |
289
| `screen_capture` | Capture the selected target as a PNG image. |
290
| `url_open` | Open an HTTP(S) URL or custom scheme where the target supports it. |
291
-| `ui_describe` / `ui_element_list` | Inspect the Apple ControlKit accessibility hierarchy. |
291
+| `ui_describe` / `ui_element_list` / `ui_tap` | Inspect or activate accessible elements through ControlKit (Apple) or AndroidKit (Android). |
292
| `input_tap` / `input_text` / `input_swipe` / `input_button` | Drive shared touch, text, gesture, and button actions. |
293
| `input_click` | Click a macOS target with pointer coordinates. |
294
| `input_spatial_tap` | Perform a visionOS spatial tap. |
295
| `orientation_get` / `orientation_set` | Read or update orientation where supported. |
296
297
-See [ControlKit runners](./controlkit.md) for Apple runner setup. Android tools
298
-require adb and an authorized device; Android TV app launches resolve the
297
+See [ControlKit runners](./controlkit.md) for Apple runner setup and
298
+[XCRS AndroidKit](./androidkit.md) for Android semantic UI automation. Android
299
+tools require adb and an authorized device; Android TV app launches resolve the
300
device's Leanback launcher activity.
301
302
#### Automation migration in 0.5.0