Add ControlKit UI tap action
Co-Authored-By: siGit Code <noreply@sigit.si>
Seto Elkahfi committed
Sep 9, 2026 at 13:55 UTC
0fee093993f2d8095d0fa2d86504576629fde511
8 files changed
+157
-41
Runners/Shared/ControlKitRPCServer.swift
+104
-34
@@ -13,22 +13,7 @@ private struct JSONRPCRequest {
13
14
@MainActor
15
final class ControlKitRPCServer {
16
- private static let protocolVersion = 1
17
- private static let supportedMethods = [
18
- "device.apps.launch",
19
- "device.apps.terminate",
20
- "device.capabilities",
21
- "device.dump.ui",
22
- "device.info",
23
- "device.io.button",
24
- "device.io.click",
25
- "device.io.spatial.tap",
26
- "device.io.tap",
27
- "device.io.text"
28
- ]
29
-
30
- private let port: UInt16
31
- private let host: String
16
+ // MARK: - Internal
17
18
init() {
19
port = UInt16(ProcessInfo.processInfo.environment["CONTROLKIT_LISTEN_PORT"] ?? "12004") ?? 12004
@@ -60,6 +45,40 @@ final class ControlKitRPCServer {
45
try await channel.closeFuture.get()
46
}
47
48
+ // MARK: - Private
49
+
50
+ private static let protocolVersion = 1
51
+ private static let supportedMethods = [
52
+ "device.apps.launch",
53
+ "device.apps.terminate",
54
+ "device.capabilities",
55
+ "device.dump.ui",
56
+ "device.ui.tap",
57
+ "device.info",
58
+ "device.io.button",
59
+ "device.io.click",
60
+ "device.io.spatial.tap",
61
+ "device.io.tap",
62
+ "device.io.text"
63
+ ]
64
+
65
+ private static var platform: XCRSControlKitPlatform {
66
+ #if os(tvOS)
67
+ return .tvOS
68
+ #elseif os(visionOS)
69
+ return .visionOS
70
+ #elseif os(macOS)
71
+ return .macOS
72
+ #elseif os(watchOS)
73
+ return .watchOS
74
+ #else
75
+ return .iOS
76
+ #endif
77
+ }
78
+
79
+ private let port: UInt16
80
+ private let host: String
81
+
82
private func handle(_ body: Data) async -> Data {
83
do {
84
if body == Data("health".utf8) {
@@ -113,6 +132,15 @@ final class ControlKitRPCServer {
132
bundleIdentifier: bundleIdentifier
133
)
134
return try response(result: dictionary(hierarchy), id: request.id)
135
+ case "device.ui.tap":
136
+ let bundleIdentifier = try stringParameter("bundleId", request.params)
137
+ let element = try stringParameter("element", request.params)
138
+ .trimmingCharacters(in: .whitespacesAndNewlines)
139
+ guard !element.isEmpty else {
140
+ throw RunnerError.invalidParameter("element")
141
+ }
142
+ try tapElement(named: element, in: bundleIdentifier)
143
+ return try response(result: ["success": true], id: request.id)
144
case "device.apps.launch":
145
let bundleIdentifier = try stringParameter("bundleId", request.params)
146
let application = XCUIApplication(bundleIdentifier: bundleIdentifier)
@@ -185,20 +213,6 @@ final class ControlKitRPCServer {
213
}
214
}
215
188
- private static var platform: XCRSControlKitPlatform {
189
- #if os(tvOS)
190
- return .tvOS
191
- #elseif os(visionOS)
192
- return .visionOS
193
- #elseif os(macOS)
194
- return .macOS
195
- #elseif os(watchOS)
196
- return .watchOS
197
- #else
198
- return .iOS
199
- #endif
200
- }
201
-
216
private func foregroundApplication(_ params: [String: Any]) -> XCUIApplication {
217
if let bundleIdentifier = params["bundleId"] as? String {
218
return XCUIApplication(bundleIdentifier: bundleIdentifier)
@@ -222,6 +236,46 @@ final class ControlKitRPCServer {
236
)
237
}
238
239
+ private func tapElement(named element: String, in bundleIdentifier: String) throws {
240
+ let application = XCUIApplication(bundleIdentifier: bundleIdentifier)
241
+ guard application.state == .runningForeground else {
242
+ throw RunnerError.applicationNotInForeground(bundleIdentifier)
243
+ }
244
+ let predicate = NSPredicate(
245
+ format: "label == %@ OR identifier == %@ OR value == %@",
246
+ element,
247
+ element,
248
+ element
249
+ )
250
+ let matches = application.descendants(matching: .any).matching(predicate)
251
+ let matchCount = matches.count
252
+ guard matchCount > 0 else {
253
+ throw RunnerError.elementNotFound(element)
254
+ }
255
+ guard matchCount == 1 else {
256
+ throw RunnerError.ambiguousElement(element, matchCount)
257
+ }
258
+ try activate(matches.element(boundBy: 0), named: element)
259
+ }
260
+
261
+ private func activate(_ element: XCUIElement, named name: String) throws {
262
+ #if os(tvOS)
263
+ guard element.hasFocus else {
264
+ throw RunnerError.elementNotFocused(name)
265
+ }
266
+ XCUIRemote.shared.press(.select)
267
+ #else
268
+ guard element.isHittable else {
269
+ throw RunnerError.elementNotHittable(name)
270
+ }
271
+ #if os(macOS)
272
+ element.click()
273
+ #else
274
+ element.tap()
275
+ #endif
276
+ #endif
277
+ }
278
+
279
private func dictionary(_ node: XCRSControlKitAccessibilityNode) throws -> [String: Any] {
280
let data = try JSONEncoder().encode(node)
281
guard let object = try JSONSerialization.jsonObject(with: data) as? [String: Any] else {
@@ -281,13 +335,11 @@ final class ControlKitRPCServer {
335
}
336
337
private final class RPCHandler: ChannelInboundHandler {
338
+ // MARK: - Internal
339
+
340
typealias InboundIn = HTTPServerRequestPart
341
typealias OutboundOut = HTTPServerResponsePart
342
287
- private let requestHandler: (HTTPMethod, String, Data) async -> Data
288
- private var requestHead: HTTPRequestHead?
289
- private var requestBody = Data()
290
-
343
init(requestHandler: @escaping (HTTPMethod, String, Data) async -> Data) {
344
self.requestHandler = requestHandler
345
}
@@ -333,23 +385,41 @@ private final class RPCHandler: ChannelInboundHandler {
385
}
386
}
387
}
388
+
389
+ // MARK: - Private
390
+
391
+ private let requestHandler: (HTTPMethod, String, Data) async -> Data
392
+ private var requestHead: HTTPRequestHead?
393
+ private var requestBody = Data()
394
}
395
396
private enum RunnerError: LocalizedError {
397
+ case ambiguousElement(String, Int)
398
case applicationNotInForeground(String)
399
case invalidAccessibilityHierarchy
400
case invalidParameter(String)
401
+ case elementNotFound(String)
402
+ case elementNotHittable(String)
403
+ case elementNotFocused(String)
404
case unsupportedButton(String)
405
case unsupportedInteraction(String)
406
407
var errorDescription: String? {
408
switch self {
409
+ case .ambiguousElement(let element, let count):
410
+ return "Multiple accessibility elements matched \(element): \(count)"
411
case .applicationNotInForeground(let bundleIdentifier):
412
return "Application is not running in the foreground: \(bundleIdentifier)"
413
case .invalidAccessibilityHierarchy:
414
return "Could not serialize the accessibility hierarchy"
415
case .invalidParameter(let name):
416
return "Missing or invalid parameter: \(name)"
417
+ case .elementNotFound(let element):
418
+ return "No accessibility element matched: \(element)"
419
+ case .elementNotHittable(let element):
420
+ return "Accessibility element is not hittable: \(element)"
421
+ case .elementNotFocused(let element):
422
+ return "Accessibility element is not focused on tvOS: \(element)"
423
case .unsupportedButton(let button):
424
return "Unsupported button: \(button)"
425
case .unsupportedInteraction(let interaction):
Runners/Shared/ControlKitRunnerUITests.swift
+2
@@ -1,6 +1,8 @@
1
import XCTest
2
3
final class ControlKitRunnerUITests: XCTestCase {
4
+ // MARK: - Internal
5
+
6
@MainActor
7
func testRunAutomation() async throws {
8
continueAfterFailure = true
Runners/iOS/ControlKitRunnerApp.swift
+7
-1
@@ -2,6 +2,8 @@ import SwiftUI
2
3
@main
4
struct ControlKitRunnerApp: App {
5
+ // MARK: - Internal
6
+
7
var body: some Scene {
8
WindowGroup {
9
ControlKitShowcaseView()
@@ -10,7 +12,7 @@ struct ControlKitRunnerApp: App {
12
}
13
14
private struct ControlKitShowcaseView: View {
13
- private let repositoryURL = URL(string: "https://github.com/ondeinference/xcrs-controlkit")!
15
+ // MARK: - Internal
16
17
var body: some View {
18
ZStack {
@@ -59,6 +61,10 @@ private struct ControlKitShowcaseView: View {
61
.statusBarHidden(true)
62
}
63
64
+ // MARK: - Private
65
+
66
+ private let repositoryURL = URL(string: "https://github.com/ondeinference/xcrs-controlkit")!
67
+
68
private var logo: some View {
69
Text("XC\nRS")
70
.font(.system(size: 30, weight: .black, design: .rounded))
Runners/macOS/ControlKitRunnerApp.swift
+7
-1
@@ -2,6 +2,8 @@ import SwiftUI
2
3
@main
4
struct ControlKitRunnerApp: App {
5
+ // MARK: - Internal
6
+
7
var body: some Scene {
8
WindowGroup {
9
ControlKitShowcaseView()
@@ -11,7 +13,7 @@ struct ControlKitRunnerApp: App {
13
}
14
15
private struct ControlKitShowcaseView: View {
14
- private let repositoryURL = URL(string: "https://github.com/ondeinference/xcrs-controlkit")!
16
+ // MARK: - Internal
17
18
var body: some View {
19
ScrollView {
@@ -47,6 +49,10 @@ private struct ControlKitShowcaseView: View {
49
.frame(minWidth: 640, minHeight: 480)
50
}
51
52
+ // MARK: - Private
53
+
54
+ private let repositoryURL = URL(string: "https://github.com/ondeinference/xcrs-controlkit")!
55
+
56
private var logo: some View {
57
Text("XC\nRS")
58
.font(.system(size: 30, weight: .black, design: .rounded))
Runners/tvOS/ControlKitRunnerApp.swift
+6
@@ -2,6 +2,8 @@ import SwiftUI
2
3
@main
4
struct ControlKitRunnerApp: App {
5
+ // MARK: - Internal
6
+
7
var body: some Scene {
8
WindowGroup {
9
ControlKitShowcaseView()
@@ -10,6 +12,8 @@ struct ControlKitRunnerApp: App {
12
}
13
14
private struct ControlKitShowcaseView: View {
15
+ // MARK: - Internal
16
+
17
var body: some View {
18
ZStack {
19
LinearGradient(
@@ -46,6 +50,8 @@ private struct ControlKitShowcaseView: View {
50
}
51
}
52
53
+ // MARK: - Private
54
+
55
private var logo: some View {
56
Text("XC\nRS")
57
.font(.system(size: 38, weight: .black, design: .rounded))
Runners/visionOS/ControlKitRunnerApp.swift
+7
-1
@@ -2,6 +2,8 @@ import SwiftUI
2
3
@main
4
struct ControlKitRunnerApp: App {
5
+ // MARK: - Internal
6
+
7
var body: some Scene {
8
WindowGroup {
9
ControlKitShowcaseView()
@@ -11,7 +13,7 @@ struct ControlKitRunnerApp: App {
13
}
14
15
private struct ControlKitShowcaseView: View {
14
- private let repositoryURL = URL(string: "https://github.com/ondeinference/xcrs-controlkit")!
16
+ // MARK: - Internal
17
18
var body: some View {
19
ScrollView {
@@ -46,6 +48,10 @@ private struct ControlKitShowcaseView: View {
48
}
49
}
50
51
+ // MARK: - Private
52
+
53
+ private let repositoryURL = URL(string: "https://github.com/ondeinference/xcrs-controlkit")!
54
+
55
private var logo: some View {
56
Text("XC\nRS")
57
.font(.system(size: 30, weight: .black, design: .rounded))
Runners/watchOS/ControlKitRunnerApp.swift
+6
@@ -2,6 +2,8 @@ import SwiftUI
2
3
@main
4
struct ControlKitRunnerApp: App {
5
+ // MARK: - Internal
6
+
7
var body: some Scene {
8
WindowGroup {
9
ControlKitShowcaseView()
@@ -10,6 +12,8 @@ struct ControlKitRunnerApp: App {
12
}
13
14
private struct ControlKitShowcaseView: View {
15
+ // MARK: - Internal
16
+
17
var body: some View {
18
ScrollView {
19
VStack(alignment: .leading, spacing: 12) {
@@ -33,6 +37,8 @@ private struct ControlKitShowcaseView: View {
37
.foregroundStyle(Color.controlKitCream)
38
}
39
40
+ // MARK: - Private
41
+
42
private var logo: some View {
43
Text("XC\nRS")
44
.font(.system(size: 14, weight: .black, design: .rounded))
Sources/XCRSControlKit/XCRSControlKit.swift
+18
-4
@@ -28,6 +28,8 @@ public enum XCRSControlKitCapability: String, Codable, CaseIterable, Sendable {
28
29
/// A simulator or device controlled by a runner.
30
public struct XCRSControlKitTarget: Codable, Equatable, Sendable {
31
+ // MARK: - Public
32
+
33
public let platform: XCRSControlKitPlatform
34
public let identifier: String
35
public let name: String?
@@ -45,6 +47,8 @@ public struct XCRSControlKitTarget: Codable, Equatable, Sendable {
47
48
/// The capabilities advertised by a runner.
49
public struct XCRSControlKitCapabilities: Codable, Equatable, Sendable {
50
+ // MARK: - Public
51
+
52
public let platform: XCRSControlKitPlatform
53
public let values: Set<XCRSControlKitCapability>
54
@@ -113,6 +117,8 @@ public struct XCRSControlKitCapabilities: Codable, Equatable, Sendable {
117
118
/// A screen-space rectangle reported by the accessibility hierarchy.
119
public struct XCRSControlKitRect: Codable, Equatable, Sendable {
120
+ // MARK: - Public
121
+
122
public let x: Double
123
public let y: Double
124
public let width: Double
@@ -128,6 +134,8 @@ public struct XCRSControlKitRect: Codable, Equatable, Sendable {
134
135
/// A transport-stable accessibility element returned by `device.dump.ui`.
136
public struct XCRSControlKitAccessibilityNode: Codable, Equatable, Sendable {
137
+ // MARK: - Public
138
+
139
public let type: String
140
public let label: String?
141
public let name: String?
@@ -175,10 +183,7 @@ public struct XCRSControlKitAccessibilityNode: Codable, Equatable, Sendable {
183
184
/// Converts XCTest's single-snapshot hierarchy into ControlKit nodes.
185
public enum XCRSControlKitAccessibilityParser {
178
- private struct ParsedNode {
179
- let node: XCRSControlKitAccessibilityNode
180
- let indentation: Int
181
- }
186
+ // MARK: - Public
187
188
public static func parse(
189
debugDescription: String
@@ -211,6 +216,13 @@ public enum XCRSControlKitAccessibilityParser {
216
return root
217
}
218
219
+ // MARK: - Private
220
+
221
+ private struct ParsedNode {
222
+ let node: XCRSControlKitAccessibilityNode
223
+ let indentation: Int
224
+ }
225
+
226
private static func parseLine(
227
_ line: String
228
) -> ParsedNode? {
@@ -390,6 +402,8 @@ public enum XCRSControlKitAccessibilityParser {
402
403
/// A transport-neutral command sent to a runner.
404
public struct XCRSControlKitRequest: Codable, Equatable, Sendable {
405
+ // MARK: - Public
406
+
407
public let id: String
408
public let method: String
409
public let parameters: [String: String]