Add smb_device_screenshot and document physical-device ControlKit support
Simctl has no equivalent for a real device, so smb_simulator_screenshot couldn't capture one. Add a devicectl-backed Devicectl::screenshot next to Simctl::screenshot and expose it as smb_device_screenshot/ xcrs_device_screenshot. Also document the host field's physical-device usage across the ControlKit tool surface, which the previous commit added but left undocumented.
keypair34 committed
Jul 31, 2026 at 00:22 UTC
8be27ebd7e2b7164b1fb33097af9b6e7658307ce
5 files changed
+150
-18
crates/cli/src/mcp_xcrs.rs
+2
-1
@@ -21,5 +21,6 @@ xcrs::xcrs_mcp_tools!(
21
"smb_controlkit_capabilities",
22
"smb_macos_click",
23
"smb_visionos_spatial_tap",
24
- "smb_watchos_tap"
24
+ "smb_watchos_tap",
25
+ "smb_device_screenshot"
26
);
crates/xcrs/src/mcp.rs
+38
-2
@@ -16,6 +16,13 @@ pub struct SimulatorFindArgs {
16
pub name: String,
17
}
18
19
+#[derive(Debug, Deserialize, JsonSchema)]
20
+pub struct DeviceScreenshotArgs {
21
+ /// Physical device identifier known to CoreDevice: UDID, ECID, serial
22
+ /// number, user-provided name, or DNS name.
23
+ pub device: String,
24
+}
25
+
26
#[derive(Debug, Deserialize, JsonSchema)]
27
pub struct IosAppTestArgs {
28
/// Exact simulator name to use.
@@ -261,7 +268,8 @@ macro_rules! xcrs_mcp_tools {
268
$controlkit_capabilities_name:literal,
269
$macos_click_name:literal,
270
$visionos_spatial_tap_name:literal,
264
- $watchos_tap_name:literal
271
+ $watchos_tap_name:literal,
272
+ $device_screenshot_name:literal
273
) => {
274
#[::rmcp::tool_router(router = xcrs_tool_router, vis = "pub(crate)")]
275
impl $server {
@@ -620,6 +628,33 @@ macro_rules! xcrs_mcp_tools {
628
]))
629
}
630
631
+ #[::rmcp::tool(
632
+ name = $device_screenshot_name,
633
+ description = "Capture a PNG screenshot from a physical iOS, tvOS, watchOS, or visionOS device paired with this Mac. Accepts a UDID, ECID, serial number, user-provided name, or DNS name."
634
+ )]
635
+ async fn device_screenshot(
636
+ &self,
637
+ ::rmcp::handler::server::wrapper::Parameters(
638
+ args,
639
+ ): ::rmcp::handler::server::wrapper::Parameters<
640
+ $crate::mcp::DeviceScreenshotArgs,
641
+ >,
642
+ ) -> ::std::result::Result<
643
+ ::rmcp::model::CallToolResult,
644
+ ::rmcp::model::ErrorData,
645
+ > {
646
+ let screenshot = $crate::XcodeCommandLineTools::new()
647
+ .devicectl()
648
+ .screenshot(&args.device)
649
+ .map_err(|error| {
650
+ ::rmcp::model::ErrorData::internal_error(error.to_string(), None)
651
+ })?;
652
+ let encoded = $crate::encode_base64(screenshot);
653
+ Ok(::rmcp::model::CallToolResult::success(vec![
654
+ ::rmcp::model::ContentBlock::image(encoded, "image/png"),
655
+ ]))
656
+ }
657
+
658
#[::rmcp::tool(
659
name = $simulator_launch_app_name,
660
description = "Launch an installed app on an Apple simulator by bundle identifier."
@@ -1143,7 +1178,8 @@ xcrs_mcp_tools!(
1178
"xcrs_controlkit_capabilities",
1179
"xcrs_macos_click",
1180
"xcrs_visionos_spatial_tap",
1146
- "xcrs_watchos_tap"
1181
+ "xcrs_watchos_tap",
1182
+ "xcrs_device_screenshot"
1183
);
1184
1185
#[rmcp::tool_handler(router = Self::xcrs_tool_router())]
crates/xcrs/src/xcrs.rs
+35
@@ -194,6 +194,10 @@ impl XcodeCommandLineTools {
194
Xcodebuild { tools: self }
195
}
196
197
+ pub fn devicectl(&self) -> Devicectl<'_> {
198
+ Devicectl { tools: self }
199
+ }
200
+
201
fn run_xcrun<I, S>(&self, args: I) -> Result<String>
202
where
203
I: IntoIterator<Item = S>,
@@ -384,6 +388,37 @@ impl Simctl<'_> {
388
}
389
}
390
391
+pub struct Devicectl<'a> {
392
+ tools: &'a XcodeCommandLineTools,
393
+}
394
+
395
+impl Devicectl<'_> {
396
+ /// Captures a PNG screenshot from a physical device known to CoreDevice.
397
+ /// `device` accepts anything `devicectl --device` does: UDID, ECID,
398
+ /// serial number, user-provided name, or DNS name.
399
+ pub fn screenshot(&self, device: &str) -> Result<Vec<u8>> {
400
+ let temporary_directory = tempfile::tempdir()?;
401
+ let screenshot_path = temporary_directory.path().join("screenshot.png");
402
+ let screenshot_path_string = screenshot_path
403
+ .to_str()
404
+ .ok_or_else(|| anyhow!("screenshot path is not valid UTF-8"))?;
405
+
406
+ self.tools.run_xcrun([
407
+ "devicectl",
408
+ "device",
409
+ "capture",
410
+ "screenshot",
411
+ "--device",
412
+ device,
413
+ "--destination",
414
+ screenshot_path_string,
415
+ ])?;
416
+
417
+ std::fs::read(&screenshot_path)
418
+ .with_context(|| format!("failed to read {}", screenshot_path.display()))
419
+ }
420
+}
421
+
422
#[derive(Debug, Clone)]
423
pub enum XcodeProject {
424
Project(PathBuf),
docs/controlkit.md
+54
-6
@@ -57,6 +57,38 @@ The shared XCTest runner starts a local ControlKit JSON-RPC server on
57
The macOS UI-test target includes the network-server entitlement required to
58
bind the local port.
59
60
+## Run on a physical device
61
+
62
+The same runner also works installed on a real iOS, tvOS, watchOS, or
63
+visionOS device, not just a simulator. Build and start it against the
64
+device's UDID (find it with `xcrun devicectl list devices`) instead of a
65
+simulator destination:
66
+
67
+```sh
68
+xcodebuild -project XCRSControlKitRunner.xcodeproj \
69
+ -scheme ControlKit-<platform> \
70
+ -destination 'id=<device-udid>' \
71
+ build-for-testing
72
+
73
+TEST_RUNNER_CONTROLKIT_LISTEN_HOST=0.0.0.0 xcodebuild \
74
+ -project XCRSControlKitRunner.xcodeproj \
75
+ -scheme ControlKit-<platform> \
76
+ -destination 'id=<device-udid>' \
77
+ test-without-building
78
+```
79
+
80
+The `CONTROLKIT_LISTEN_HOST` environment variable controls what the
81
+in-process RPC server binds to (`Shared/ControlKitRPCServer.swift`) — it
82
+defaults to `127.0.0.1`, which is loopback-only *on the device itself* and
83
+unreachable from your Mac. `xcodebuild` forwards any `TEST_RUNNER_`-prefixed
84
+environment variable into the test process with the prefix stripped, so
85
+setting `TEST_RUNNER_CONTROLKIT_LISTEN_HOST=0.0.0.0` makes the server bind on
86
+every interface, including the one reachable over your LAN. Find the device's
87
+address with its Bonjour hostname (`<device-name>.local`, discoverable via
88
+`dns-sd -B _airplay._tcp local` for an Apple TV) or its CoreDevice tunnel
89
+address (`xcrun devicectl device info details --device <udid>`, under
90
+`Tunnel IP Address`).
91
+
92
## Connect through `smb --mcp`
93
94
Start the CLI MCP server:
@@ -65,8 +97,10 @@ Start the CLI MCP server:
97
smb --mcp
98
```
99
68
-The runner tools accept either `simulator_name` or `simulator_udid`. The
69
-standalone `xcrs --mcp` server exposes the same tools with the `xcrs_` prefix.
100
+The runner tools accept either `simulator_name` or `simulator_udid` for a
101
+simulator, or `host` (plus `controlkit_port`, defaulting to `12004`) for a
102
+physical device or any other remote runner. The standalone `xcrs --mcp`
103
+server exposes the same tools with the `xcrs_` prefix.
104
105
| Tool | Runner | Purpose |
106
| --- | --- | --- |
@@ -74,10 +108,13 @@ standalone `xcrs --mcp` server exposes the same tools with the `xcrs_` prefix.
108
| `smb_macos_click` | macOS | Click at screen coordinates. |
109
| `smb_visionos_spatial_tap` | visionOS | Perform a spatial tap. |
110
| `smb_watchos_tap` | watchOS | Perform a touch tap. |
77
-| `smb_simulator_tap` | iOS / tvOS | Perform a touch tap. |
78
-| `smb_simulator_type_text` | iOS / tvOS | Type into the focused field. |
79
-| `smb_simulator_swipe` | iOS / tvOS | Swipe between coordinates. |
80
-| `smb_simulator_screenshot` | All simulator runners | Capture a PNG screenshot. |
111
+| `smb_simulator_tap` | iOS / tvOS, simulator or physical | Perform a touch tap. |
112
+| `smb_simulator_type_text` | iOS / tvOS, simulator or physical | Type into the focused field. |
113
+| `smb_simulator_swipe` | iOS / tvOS, simulator or physical | Swipe between coordinates. |
114
+| `smb_simulator_press_button` | iOS / tvOS, simulator or physical | Press a Home or tvOS remote button. |
115
+| `smb_simulator_launch_app`/`terminate_app` | simulator or physical | Launch/terminate an app by bundle ID. On a physical device this calls the runner's `device.apps.launch`/`device.apps.terminate` RPC methods instead of `simctl`. |
116
+| `smb_simulator_screenshot` | Simulator only | Capture a PNG screenshot via `simctl`. |
117
+| `smb_device_screenshot` | Physical device only | Capture a PNG screenshot via `devicectl device capture screenshot` — works independently of the ControlKit runner, so it succeeds even if the RPC server isn't running. |
118
| `smb_simulator_ui_dump` | All UI-test runners | Read the accessibility hierarchy. |
119
120
For a local macOS runner, omit simulator fields and pass `host` and
@@ -92,6 +129,17 @@ For a local macOS runner, omit simulator fields and pass `host` and
129
}
130
```
131
132
+For a physical device, pass its Bonjour hostname or tunnel address as `host`
133
+instead:
134
+
135
+```json
136
+{
137
+ "host": "<device-name>.local",
138
+ "controlkit_port": 12004,
139
+ "button": "select"
140
+}
141
+```
142
+
143
Platform-specific operations intentionally reject unsupported input. For
144
example, visionOS rejects raw touch, watchOS rejects pointer and spatial input,
145
and macOS rejects touch and Home-button requests.
docs/mcp.md
+21
-9
@@ -223,6 +223,17 @@ ControlKit arguments default to `127.0.0.1:12004`. For a local macOS runner,
223
omit the simulator fields and use the optional `host` and `controlkit_port`
224
fields.
225
226
+The same `host` field also targets a **physical** iOS/tvOS/watchOS/visionOS
227
+device running a ControlKit runner reachable over the network (see
228
+[ControlKit runners](./controlkit.md)) — omit `simulator_name`/`simulator_udid`
229
+and pass the device's host and the runner's listen port instead. This applies
230
+to `smb_simulator_tap`, `smb_simulator_type_text`, `smb_simulator_swipe`,
231
+`smb_simulator_press_button`, `smb_simulator_press_home`,
232
+`smb_simulator_orientation_get`/`set`, `smb_simulator_ui_dump`,
233
+`smb_simulator_list_elements`, and `smb_simulator_launch_app`/`terminate_app`
234
+(the latter two route through the runner's `device.apps.launch`/`terminate`
235
+RPC methods instead of `simctl` when `host` is given).
236
+
237
| Tool | Arguments | Returns |
238
| --- | --- | --- |
239
| `smb_simulator_list` | _(none)_ | All Apple simulator devices known to Xcode. |
@@ -231,17 +242,18 @@ fields.
242
| `smb_macos_click` | `x`, `y`, plus optional endpoint fields | A pointer click on a macOS runner. |
243
| `smb_visionos_spatial_tap` | `x`, `y`, plus optional endpoint fields | A spatial tap on a visionOS runner. |
244
| `smb_watchos_tap` | `x`, `y`, plus optional endpoint fields | A touch tap on a watchOS runner. |
234
-| `smb_simulator_tap` | simulator target, `x`, `y` | A touch tap through ControlKit. |
235
-| `smb_simulator_type_text` | simulator target, `text` | Text input through ControlKit. |
236
-| `smb_simulator_swipe` | simulator target, `x1`, `y1`, `x2`, `y2` | A swipe through ControlKit. |
237
-| `smb_simulator_press_button` | simulator target, `button` | A supported remote-button press. |
238
-| `smb_simulator_press_home` | simulator target | An iOS Home-button press. |
239
-| `smb_simulator_orientation_get`/`set` | simulator target; `orientation` for `set` | Current or updated iOS orientation. |
240
-| `smb_simulator_launch_app`/`terminate_app` | simulator target, `bundle_id` | App lifecycle confirmation. |
245
+| `smb_simulator_tap` | simulator target or `host`, `x`, `y` | A touch tap through ControlKit. |
246
+| `smb_simulator_type_text` | simulator target or `host`, `text` | Text input through ControlKit. |
247
+| `smb_simulator_swipe` | simulator target or `host`, `x1`, `y1`, `x2`, `y2` | A swipe through ControlKit. |
248
+| `smb_simulator_press_button` | simulator target or `host`, `button` | A supported remote-button press. |
249
+| `smb_simulator_press_home` | simulator target or `host` | An iOS Home-button press. |
250
+| `smb_simulator_orientation_get`/`set` | simulator target or `host`; `orientation` for `set` | Current or updated iOS orientation. |
251
+| `smb_simulator_launch_app`/`terminate_app` | simulator target or `host`, `bundle_id` | App lifecycle confirmation. |
252
| `smb_simulator_open_url` | simulator target, `url` | URL-open confirmation. |
242
-| `smb_simulator_ui_dump` | simulator target | Accessibility UI hierarchy. |
243
-| `smb_simulator_list_elements` | simulator target | Actionable accessibility elements. |
253
+| `smb_simulator_ui_dump` | simulator target or `host` | Accessibility UI hierarchy. |
254
+| `smb_simulator_list_elements` | simulator target or `host` | Actionable accessibility elements. |
255
| `smb_simulator_screenshot` | simulator target | A PNG image. |
256
+| `smb_device_screenshot` | `device` (UDID, ECID, serial number, name, or DNS name) | A PNG image captured from a physical device via `devicectl`. |
257
| `smb_ios_app_test` | simulator target, `app_path`, `bundle_id` | Installed/launched app details. |
258
259
The standalone `xcrs --mcp` server exposes the same runner surface with the