hvf.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 2,902 lines 94.3 KB

Raw

1	/*
2	* QEMU Hypervisor.framework support for Apple Silicon
3
4	* Copyright 2020 Alexander Graf <agraf@csgraf.de>
5	* Copyright 2020 Google LLC
6	*
7	* This work is licensed under the terms of the GNU GPL, version 2 or later.
8	* See the COPYING file in the top-level directory.
9	*
10	*/
11
12	#include "qemu/osdep.h"
13	#include "qemu/error-report.h"
14	#include "qemu/log.h"
15
16	#include "system/runstate.h"
17	#include "system/hvf.h"
18	#include "system/hvf_int.h"
19	#include "system/hw_accel.h"
20	#include "hvf_arm.h"
21	#include "cpregs.h"
22	#include "cpu-sysregs.h"
23
24	#include <mach/mach_time.h>
25
26	#include "system/address-spaces.h"
27	#include "system/memory.h"
28	#include "hw/core/boards.h"
29	#include "hw/core/irq.h"
30	#include "hw/arm/virt.h"
31	#include "qemu/main-loop.h"
32	#include "qemu/timer.h"
33	#include "system/cpus.h"
34	#include "arm-powerctl.h"
35	#include "target/arm/cpu.h"
36	#include "target/arm/internals.h"
37	#include "target/arm/multiprocessing.h"
38	#include "target/arm/gtimer.h"
39	#include "target/arm/trace.h"
40	#include "trace.h"
41	#include "migration/vmstate.h"
42
43	#include "gdbstub/enums.h"
44
45	#define MDSCR_EL1_SS_SHIFT 0
46	#define MDSCR_EL1_MDE_SHIFT 15
47
48	static const uint16_t dbgbcr_regs[] = {
49	HV_SYS_REG_DBGBCR0_EL1,
50	HV_SYS_REG_DBGBCR1_EL1,
51	HV_SYS_REG_DBGBCR2_EL1,
52	HV_SYS_REG_DBGBCR3_EL1,
53	HV_SYS_REG_DBGBCR4_EL1,
54	HV_SYS_REG_DBGBCR5_EL1,
55	HV_SYS_REG_DBGBCR6_EL1,
56	HV_SYS_REG_DBGBCR7_EL1,
57	HV_SYS_REG_DBGBCR8_EL1,
58	HV_SYS_REG_DBGBCR9_EL1,
59	HV_SYS_REG_DBGBCR10_EL1,
60	HV_SYS_REG_DBGBCR11_EL1,
61	HV_SYS_REG_DBGBCR12_EL1,
62	HV_SYS_REG_DBGBCR13_EL1,
63	HV_SYS_REG_DBGBCR14_EL1,
64	HV_SYS_REG_DBGBCR15_EL1,
65	};
66
67	static const uint16_t dbgbvr_regs[] = {
68	HV_SYS_REG_DBGBVR0_EL1,
69	HV_SYS_REG_DBGBVR1_EL1,
70	HV_SYS_REG_DBGBVR2_EL1,
71	HV_SYS_REG_DBGBVR3_EL1,
72	HV_SYS_REG_DBGBVR4_EL1,
73	HV_SYS_REG_DBGBVR5_EL1,
74	HV_SYS_REG_DBGBVR6_EL1,
75	HV_SYS_REG_DBGBVR7_EL1,
76	HV_SYS_REG_DBGBVR8_EL1,
77	HV_SYS_REG_DBGBVR9_EL1,
78	HV_SYS_REG_DBGBVR10_EL1,
79	HV_SYS_REG_DBGBVR11_EL1,
80	HV_SYS_REG_DBGBVR12_EL1,
81	HV_SYS_REG_DBGBVR13_EL1,
82	HV_SYS_REG_DBGBVR14_EL1,
83	HV_SYS_REG_DBGBVR15_EL1,
84	};
85
86	static const uint16_t dbgwcr_regs[] = {
87	HV_SYS_REG_DBGWCR0_EL1,
88	HV_SYS_REG_DBGWCR1_EL1,
89	HV_SYS_REG_DBGWCR2_EL1,
90	HV_SYS_REG_DBGWCR3_EL1,
91	HV_SYS_REG_DBGWCR4_EL1,
92	HV_SYS_REG_DBGWCR5_EL1,
93	HV_SYS_REG_DBGWCR6_EL1,
94	HV_SYS_REG_DBGWCR7_EL1,
95	HV_SYS_REG_DBGWCR8_EL1,
96	HV_SYS_REG_DBGWCR9_EL1,
97	HV_SYS_REG_DBGWCR10_EL1,
98	HV_SYS_REG_DBGWCR11_EL1,
99	HV_SYS_REG_DBGWCR12_EL1,
100	HV_SYS_REG_DBGWCR13_EL1,
101	HV_SYS_REG_DBGWCR14_EL1,
102	HV_SYS_REG_DBGWCR15_EL1,
103	};
104
105	static const uint16_t dbgwvr_regs[] = {
106	HV_SYS_REG_DBGWVR0_EL1,
107	HV_SYS_REG_DBGWVR1_EL1,
108	HV_SYS_REG_DBGWVR2_EL1,
109	HV_SYS_REG_DBGWVR3_EL1,
110	HV_SYS_REG_DBGWVR4_EL1,
111	HV_SYS_REG_DBGWVR5_EL1,
112	HV_SYS_REG_DBGWVR6_EL1,
113	HV_SYS_REG_DBGWVR7_EL1,
114	HV_SYS_REG_DBGWVR8_EL1,
115	HV_SYS_REG_DBGWVR9_EL1,
116	HV_SYS_REG_DBGWVR10_EL1,
117	HV_SYS_REG_DBGWVR11_EL1,
118	HV_SYS_REG_DBGWVR12_EL1,
119	HV_SYS_REG_DBGWVR13_EL1,
120	HV_SYS_REG_DBGWVR14_EL1,
121	HV_SYS_REG_DBGWVR15_EL1,
122	};
123
124	static inline int hvf_arm_num_brps(hv_vcpu_config_t config)
125	{
126	uint64_t val;
127	hv_return_t ret;
128	ret = hv_vcpu_config_get_feature_reg(config, HV_FEATURE_REG_ID_AA64DFR0_EL1,
129	&val);
130	assert_hvf_ok(ret);
131	return FIELD_EX64(val, ID_AA64DFR0, BRPS) + 1;
132	}
133
134	static inline int hvf_arm_num_wrps(hv_vcpu_config_t config)
135	{
136	uint64_t val;
137	hv_return_t ret;
138	ret = hv_vcpu_config_get_feature_reg(config, HV_FEATURE_REG_ID_AA64DFR0_EL1,
139	&val);
140	assert_hvf_ok(ret);
141	return FIELD_EX64(val, ID_AA64DFR0, WRPS) + 1;
142	}
143
144	void hvf_arm_init_debug(void)
145	{
146	hv_vcpu_config_t config;
147	config = hv_vcpu_config_create();
148
149	max_hw_bps = hvf_arm_num_brps(config);
150	hw_breakpoints =
151	g_array_sized_new(true, true, sizeof(HWBreakpoint), max_hw_bps);
152
153	max_hw_wps = hvf_arm_num_wrps(config);
154	hw_watchpoints =
155	g_array_sized_new(true, true, sizeof(HWWatchpoint), max_hw_wps);
156
157	os_release(config);
158	}
159
160	#define SYSREG_OP0_SHIFT 20
161	#define SYSREG_OP0_MASK 0x3
162	#define SYSREG_OP0(sysreg) ((sysreg >> SYSREG_OP0_SHIFT) & SYSREG_OP0_MASK)
163	#define SYSREG_OP1_SHIFT 14
164	#define SYSREG_OP1_MASK 0x7
165	#define SYSREG_OP1(sysreg) ((sysreg >> SYSREG_OP1_SHIFT) & SYSREG_OP1_MASK)
166	#define SYSREG_CRN_SHIFT 10
167	#define SYSREG_CRN_MASK 0xf
168	#define SYSREG_CRN(sysreg) ((sysreg >> SYSREG_CRN_SHIFT) & SYSREG_CRN_MASK)
169	#define SYSREG_CRM_SHIFT 1
170	#define SYSREG_CRM_MASK 0xf
171	#define SYSREG_CRM(sysreg) ((sysreg >> SYSREG_CRM_SHIFT) & SYSREG_CRM_MASK)
172	#define SYSREG_OP2_SHIFT 17
173	#define SYSREG_OP2_MASK 0x7
174	#define SYSREG_OP2(sysreg) ((sysreg >> SYSREG_OP2_SHIFT) & SYSREG_OP2_MASK)
175
176	#define SYSREG(op0, op1, crn, crm, op2) \
177	((op0 << SYSREG_OP0_SHIFT) \| \
178	(op1 << SYSREG_OP1_SHIFT) \| \
179	(crn << SYSREG_CRN_SHIFT) \| \
180	(crm << SYSREG_CRM_SHIFT) \| \
181	(op2 << SYSREG_OP2_SHIFT))
182	#define SYSREG_MASK \
183	SYSREG(SYSREG_OP0_MASK, \
184	SYSREG_OP1_MASK, \
185	SYSREG_CRN_MASK, \
186	SYSREG_CRM_MASK, \
187	SYSREG_OP2_MASK)
188	#define SYSREG_OSLAR_EL1 SYSREG(2, 0, 1, 0, 4)
189	#define SYSREG_OSLSR_EL1 SYSREG(2, 0, 1, 1, 4)
190	#define SYSREG_OSDLR_EL1 SYSREG(2, 0, 1, 3, 4)
191	#define SYSREG_LORC_EL1 SYSREG(3, 0, 10, 4, 3)
192	#define SYSREG_CNTPCT_EL0 SYSREG(3, 3, 14, 0, 1)
193	#define SYSREG_CNTP_TVAL_EL0 SYSREG(3, 3, 14, 2, 0)
194	#define SYSREG_CNTP_CTL_EL0 SYSREG(3, 3, 14, 2, 1)
195	#define SYSREG_CNTP_CVAL_EL0 SYSREG(3, 3, 14, 2, 2)
196	#define SYSREG_PMCR_EL0 SYSREG(3, 3, 9, 12, 0)
197	#define SYSREG_PMUSERENR_EL0 SYSREG(3, 3, 9, 14, 0)
198	#define SYSREG_PMCNTENSET_EL0 SYSREG(3, 3, 9, 12, 1)
199	#define SYSREG_PMCNTENCLR_EL0 SYSREG(3, 3, 9, 12, 2)
200	#define SYSREG_PMINTENCLR_EL1 SYSREG(3, 0, 9, 14, 2)
201	#define SYSREG_PMOVSCLR_EL0 SYSREG(3, 3, 9, 12, 3)
202	#define SYSREG_PMSWINC_EL0 SYSREG(3, 3, 9, 12, 4)
203	#define SYSREG_PMSELR_EL0 SYSREG(3, 3, 9, 12, 5)
204	#define SYSREG_PMCEID0_EL0 SYSREG(3, 3, 9, 12, 6)
205	#define SYSREG_PMCEID1_EL0 SYSREG(3, 3, 9, 12, 7)
206	#define SYSREG_PMCCNTR_EL0 SYSREG(3, 3, 9, 13, 0)
207	#define SYSREG_PMCCFILTR_EL0 SYSREG(3, 3, 14, 15, 7)
208
209	#define SYSREG_ICC_AP0R0_EL1 SYSREG(3, 0, 12, 8, 4)
210	#define SYSREG_ICC_AP0R1_EL1 SYSREG(3, 0, 12, 8, 5)
211	#define SYSREG_ICC_AP0R2_EL1 SYSREG(3, 0, 12, 8, 6)
212	#define SYSREG_ICC_AP0R3_EL1 SYSREG(3, 0, 12, 8, 7)
213	#define SYSREG_ICC_AP1R0_EL1 SYSREG(3, 0, 12, 9, 0)
214	#define SYSREG_ICC_AP1R1_EL1 SYSREG(3, 0, 12, 9, 1)
215	#define SYSREG_ICC_AP1R2_EL1 SYSREG(3, 0, 12, 9, 2)
216	#define SYSREG_ICC_AP1R3_EL1 SYSREG(3, 0, 12, 9, 3)
217	#define SYSREG_ICC_ASGI1R_EL1 SYSREG(3, 0, 12, 11, 6)
218	#define SYSREG_ICC_BPR0_EL1 SYSREG(3, 0, 12, 8, 3)
219	#define SYSREG_ICC_BPR1_EL1 SYSREG(3, 0, 12, 12, 3)
220	#define SYSREG_ICC_CTLR_EL1 SYSREG(3, 0, 12, 12, 4)
221	#define SYSREG_ICC_DIR_EL1 SYSREG(3, 0, 12, 11, 1)
222	#define SYSREG_ICC_EOIR0_EL1 SYSREG(3, 0, 12, 8, 1)
223	#define SYSREG_ICC_EOIR1_EL1 SYSREG(3, 0, 12, 12, 1)
224	#define SYSREG_ICC_HPPIR0_EL1 SYSREG(3, 0, 12, 8, 2)
225	#define SYSREG_ICC_HPPIR1_EL1 SYSREG(3, 0, 12, 12, 2)
226	#define SYSREG_ICC_IAR0_EL1 SYSREG(3, 0, 12, 8, 0)
227	#define SYSREG_ICC_IAR1_EL1 SYSREG(3, 0, 12, 12, 0)
228	#define SYSREG_ICC_IGRPEN0_EL1 SYSREG(3, 0, 12, 12, 6)
229	#define SYSREG_ICC_IGRPEN1_EL1 SYSREG(3, 0, 12, 12, 7)
230	#define SYSREG_ICC_PMR_EL1 SYSREG(3, 0, 4, 6, 0)
231	#define SYSREG_ICC_RPR_EL1 SYSREG(3, 0, 12, 11, 3)
232	#define SYSREG_ICC_SGI0R_EL1 SYSREG(3, 0, 12, 11, 7)
233	#define SYSREG_ICC_SGI1R_EL1 SYSREG(3, 0, 12, 11, 5)
234	#define SYSREG_ICC_SRE_EL1 SYSREG(3, 0, 12, 12, 5)
235
236	#define SYSREG_MDSCR_EL1 SYSREG(2, 0, 0, 2, 2)
237	#define SYSREG_DBGBVR0_EL1 SYSREG(2, 0, 0, 0, 4)
238	#define SYSREG_DBGBCR0_EL1 SYSREG(2, 0, 0, 0, 5)
239	#define SYSREG_DBGWVR0_EL1 SYSREG(2, 0, 0, 0, 6)
240	#define SYSREG_DBGWCR0_EL1 SYSREG(2, 0, 0, 0, 7)
241	#define SYSREG_DBGBVR1_EL1 SYSREG(2, 0, 0, 1, 4)
242	#define SYSREG_DBGBCR1_EL1 SYSREG(2, 0, 0, 1, 5)
243	#define SYSREG_DBGWVR1_EL1 SYSREG(2, 0, 0, 1, 6)
244	#define SYSREG_DBGWCR1_EL1 SYSREG(2, 0, 0, 1, 7)
245	#define SYSREG_DBGBVR2_EL1 SYSREG(2, 0, 0, 2, 4)
246	#define SYSREG_DBGBCR2_EL1 SYSREG(2, 0, 0, 2, 5)
247	#define SYSREG_DBGWVR2_EL1 SYSREG(2, 0, 0, 2, 6)
248	#define SYSREG_DBGWCR2_EL1 SYSREG(2, 0, 0, 2, 7)
249	#define SYSREG_DBGBVR3_EL1 SYSREG(2, 0, 0, 3, 4)
250	#define SYSREG_DBGBCR3_EL1 SYSREG(2, 0, 0, 3, 5)
251	#define SYSREG_DBGWVR3_EL1 SYSREG(2, 0, 0, 3, 6)
252	#define SYSREG_DBGWCR3_EL1 SYSREG(2, 0, 0, 3, 7)
253	#define SYSREG_DBGBVR4_EL1 SYSREG(2, 0, 0, 4, 4)
254	#define SYSREG_DBGBCR4_EL1 SYSREG(2, 0, 0, 4, 5)
255	#define SYSREG_DBGWVR4_EL1 SYSREG(2, 0, 0, 4, 6)
256	#define SYSREG_DBGWCR4_EL1 SYSREG(2, 0, 0, 4, 7)
257	#define SYSREG_DBGBVR5_EL1 SYSREG(2, 0, 0, 5, 4)
258	#define SYSREG_DBGBCR5_EL1 SYSREG(2, 0, 0, 5, 5)
259	#define SYSREG_DBGWVR5_EL1 SYSREG(2, 0, 0, 5, 6)
260	#define SYSREG_DBGWCR5_EL1 SYSREG(2, 0, 0, 5, 7)
261	#define SYSREG_DBGBVR6_EL1 SYSREG(2, 0, 0, 6, 4)
262	#define SYSREG_DBGBCR6_EL1 SYSREG(2, 0, 0, 6, 5)
263	#define SYSREG_DBGWVR6_EL1 SYSREG(2, 0, 0, 6, 6)
264	#define SYSREG_DBGWCR6_EL1 SYSREG(2, 0, 0, 6, 7)
265	#define SYSREG_DBGBVR7_EL1 SYSREG(2, 0, 0, 7, 4)
266	#define SYSREG_DBGBCR7_EL1 SYSREG(2, 0, 0, 7, 5)
267	#define SYSREG_DBGWVR7_EL1 SYSREG(2, 0, 0, 7, 6)
268	#define SYSREG_DBGWCR7_EL1 SYSREG(2, 0, 0, 7, 7)
269	#define SYSREG_DBGBVR8_EL1 SYSREG(2, 0, 0, 8, 4)
270	#define SYSREG_DBGBCR8_EL1 SYSREG(2, 0, 0, 8, 5)
271	#define SYSREG_DBGWVR8_EL1 SYSREG(2, 0, 0, 8, 6)
272	#define SYSREG_DBGWCR8_EL1 SYSREG(2, 0, 0, 8, 7)
273	#define SYSREG_DBGBVR9_EL1 SYSREG(2, 0, 0, 9, 4)
274	#define SYSREG_DBGBCR9_EL1 SYSREG(2, 0, 0, 9, 5)
275	#define SYSREG_DBGWVR9_EL1 SYSREG(2, 0, 0, 9, 6)
276	#define SYSREG_DBGWCR9_EL1 SYSREG(2, 0, 0, 9, 7)
277	#define SYSREG_DBGBVR10_EL1 SYSREG(2, 0, 0, 10, 4)
278	#define SYSREG_DBGBCR10_EL1 SYSREG(2, 0, 0, 10, 5)
279	#define SYSREG_DBGWVR10_EL1 SYSREG(2, 0, 0, 10, 6)
280	#define SYSREG_DBGWCR10_EL1 SYSREG(2, 0, 0, 10, 7)
281	#define SYSREG_DBGBVR11_EL1 SYSREG(2, 0, 0, 11, 4)
282	#define SYSREG_DBGBCR11_EL1 SYSREG(2, 0, 0, 11, 5)
283	#define SYSREG_DBGWVR11_EL1 SYSREG(2, 0, 0, 11, 6)
284	#define SYSREG_DBGWCR11_EL1 SYSREG(2, 0, 0, 11, 7)
285	#define SYSREG_DBGBVR12_EL1 SYSREG(2, 0, 0, 12, 4)
286	#define SYSREG_DBGBCR12_EL1 SYSREG(2, 0, 0, 12, 5)
287	#define SYSREG_DBGWVR12_EL1 SYSREG(2, 0, 0, 12, 6)
288	#define SYSREG_DBGWCR12_EL1 SYSREG(2, 0, 0, 12, 7)
289	#define SYSREG_DBGBVR13_EL1 SYSREG(2, 0, 0, 13, 4)
290	#define SYSREG_DBGBCR13_EL1 SYSREG(2, 0, 0, 13, 5)
291	#define SYSREG_DBGWVR13_EL1 SYSREG(2, 0, 0, 13, 6)
292	#define SYSREG_DBGWCR13_EL1 SYSREG(2, 0, 0, 13, 7)
293	#define SYSREG_DBGBVR14_EL1 SYSREG(2, 0, 0, 14, 4)
294	#define SYSREG_DBGBCR14_EL1 SYSREG(2, 0, 0, 14, 5)
295	#define SYSREG_DBGWVR14_EL1 SYSREG(2, 0, 0, 14, 6)
296	#define SYSREG_DBGWCR14_EL1 SYSREG(2, 0, 0, 14, 7)
297	#define SYSREG_DBGBVR15_EL1 SYSREG(2, 0, 0, 15, 4)
298	#define SYSREG_DBGBCR15_EL1 SYSREG(2, 0, 0, 15, 5)
299	#define SYSREG_DBGWVR15_EL1 SYSREG(2, 0, 0, 15, 6)
300	#define SYSREG_DBGWCR15_EL1 SYSREG(2, 0, 0, 15, 7)
301
302	/* EL2 registers */
303	#define SYSREG_CNTHCTL_EL2 SYSREG(3, 4, 14, 1, 0)
304	#define SYSREG_MDCCINT_EL1 SYSREG(2, 0, 0, 2, 0)
305
306	#define WFX_IS_WFE (1 << 0)
307
308	#define TMR_CTL_ENABLE (1 << 0)
309	#define TMR_CTL_IMASK (1 << 1)
310	#define TMR_CTL_ISTATUS (1 << 2)
311
312	static void hvf_wfi_timer_cb(void *opaque);
313
314	static uint32_t chosen_ipa_bit_size;
315
316	typedef struct HVFVTimer {
317	/* Vtimer value during migration and paused state */
318	uint64_t vtimer_val;
319	} HVFVTimer;
320
321	static HVFVTimer vtimer;
322
323	typedef struct ARMHostCPUFeatures {
324	ARMISARegisters isar;
325	uint64_t features;
326	uint64_t midr;
327	uint32_t reset_sctlr;
328	uint32_t sme_vq_supported;
329	const char *dtb_compatible;
330	} ARMHostCPUFeatures;
331
332	static ARMHostCPUFeatures arm_host_cpu_features;
333
334	struct hvf_reg_match {
335	int reg;
336	uint64_t offset;
337	};
338
339	static const struct hvf_reg_match hvf_reg_match[] = {
340	{ HV_REG_X0, offsetof(CPUARMState, xregs[0]) },
341	{ HV_REG_X1, offsetof(CPUARMState, xregs[1]) },
342	{ HV_REG_X2, offsetof(CPUARMState, xregs[2]) },
343	{ HV_REG_X3, offsetof(CPUARMState, xregs[3]) },
344	{ HV_REG_X4, offsetof(CPUARMState, xregs[4]) },
345	{ HV_REG_X5, offsetof(CPUARMState, xregs[5]) },
346	{ HV_REG_X6, offsetof(CPUARMState, xregs[6]) },
347	{ HV_REG_X7, offsetof(CPUARMState, xregs[7]) },
348	{ HV_REG_X8, offsetof(CPUARMState, xregs[8]) },
349	{ HV_REG_X9, offsetof(CPUARMState, xregs[9]) },
350	{ HV_REG_X10, offsetof(CPUARMState, xregs[10]) },
351	{ HV_REG_X11, offsetof(CPUARMState, xregs[11]) },
352	{ HV_REG_X12, offsetof(CPUARMState, xregs[12]) },
353	{ HV_REG_X13, offsetof(CPUARMState, xregs[13]) },
354	{ HV_REG_X14, offsetof(CPUARMState, xregs[14]) },
355	{ HV_REG_X15, offsetof(CPUARMState, xregs[15]) },
356	{ HV_REG_X16, offsetof(CPUARMState, xregs[16]) },
357	{ HV_REG_X17, offsetof(CPUARMState, xregs[17]) },
358	{ HV_REG_X18, offsetof(CPUARMState, xregs[18]) },
359	{ HV_REG_X19, offsetof(CPUARMState, xregs[19]) },
360	{ HV_REG_X20, offsetof(CPUARMState, xregs[20]) },
361	{ HV_REG_X21, offsetof(CPUARMState, xregs[21]) },
362	{ HV_REG_X22, offsetof(CPUARMState, xregs[22]) },
363	{ HV_REG_X23, offsetof(CPUARMState, xregs[23]) },
364	{ HV_REG_X24, offsetof(CPUARMState, xregs[24]) },
365	{ HV_REG_X25, offsetof(CPUARMState, xregs[25]) },
366	{ HV_REG_X26, offsetof(CPUARMState, xregs[26]) },
367	{ HV_REG_X27, offsetof(CPUARMState, xregs[27]) },
368	{ HV_REG_X28, offsetof(CPUARMState, xregs[28]) },
369	{ HV_REG_X29, offsetof(CPUARMState, xregs[29]) },
370	{ HV_REG_X30, offsetof(CPUARMState, xregs[30]) },
371	{ HV_REG_PC, offsetof(CPUARMState, pc) },
372	};
373
374	static const struct hvf_reg_match hvf_fpreg_match[] = {
375	{ HV_SIMD_FP_REG_Q0, offsetof(CPUARMState, vfp.zregs[0]) },
376	{ HV_SIMD_FP_REG_Q1, offsetof(CPUARMState, vfp.zregs[1]) },
377	{ HV_SIMD_FP_REG_Q2, offsetof(CPUARMState, vfp.zregs[2]) },
378	{ HV_SIMD_FP_REG_Q3, offsetof(CPUARMState, vfp.zregs[3]) },
379	{ HV_SIMD_FP_REG_Q4, offsetof(CPUARMState, vfp.zregs[4]) },
380	{ HV_SIMD_FP_REG_Q5, offsetof(CPUARMState, vfp.zregs[5]) },
381	{ HV_SIMD_FP_REG_Q6, offsetof(CPUARMState, vfp.zregs[6]) },
382	{ HV_SIMD_FP_REG_Q7, offsetof(CPUARMState, vfp.zregs[7]) },
383	{ HV_SIMD_FP_REG_Q8, offsetof(CPUARMState, vfp.zregs[8]) },
384	{ HV_SIMD_FP_REG_Q9, offsetof(CPUARMState, vfp.zregs[9]) },
385	{ HV_SIMD_FP_REG_Q10, offsetof(CPUARMState, vfp.zregs[10]) },
386	{ HV_SIMD_FP_REG_Q11, offsetof(CPUARMState, vfp.zregs[11]) },
387	{ HV_SIMD_FP_REG_Q12, offsetof(CPUARMState, vfp.zregs[12]) },
388	{ HV_SIMD_FP_REG_Q13, offsetof(CPUARMState, vfp.zregs[13]) },
389	{ HV_SIMD_FP_REG_Q14, offsetof(CPUARMState, vfp.zregs[14]) },
390	{ HV_SIMD_FP_REG_Q15, offsetof(CPUARMState, vfp.zregs[15]) },
391	{ HV_SIMD_FP_REG_Q16, offsetof(CPUARMState, vfp.zregs[16]) },
392	{ HV_SIMD_FP_REG_Q17, offsetof(CPUARMState, vfp.zregs[17]) },
393	{ HV_SIMD_FP_REG_Q18, offsetof(CPUARMState, vfp.zregs[18]) },
394	{ HV_SIMD_FP_REG_Q19, offsetof(CPUARMState, vfp.zregs[19]) },
395	{ HV_SIMD_FP_REG_Q20, offsetof(CPUARMState, vfp.zregs[20]) },
396	{ HV_SIMD_FP_REG_Q21, offsetof(CPUARMState, vfp.zregs[21]) },
397	{ HV_SIMD_FP_REG_Q22, offsetof(CPUARMState, vfp.zregs[22]) },
398	{ HV_SIMD_FP_REG_Q23, offsetof(CPUARMState, vfp.zregs[23]) },
399	{ HV_SIMD_FP_REG_Q24, offsetof(CPUARMState, vfp.zregs[24]) },
400	{ HV_SIMD_FP_REG_Q25, offsetof(CPUARMState, vfp.zregs[25]) },
401	{ HV_SIMD_FP_REG_Q26, offsetof(CPUARMState, vfp.zregs[26]) },
402	{ HV_SIMD_FP_REG_Q27, offsetof(CPUARMState, vfp.zregs[27]) },
403	{ HV_SIMD_FP_REG_Q28, offsetof(CPUARMState, vfp.zregs[28]) },
404	{ HV_SIMD_FP_REG_Q29, offsetof(CPUARMState, vfp.zregs[29]) },
405	{ HV_SIMD_FP_REG_Q30, offsetof(CPUARMState, vfp.zregs[30]) },
406	{ HV_SIMD_FP_REG_Q31, offsetof(CPUARMState, vfp.zregs[31]) },
407	};
408
409	static const struct hvf_reg_match hvf_sme2_zreg_match[] = {
410	{ HV_SME_Z_REG_0, offsetof(CPUARMState, vfp.zregs[0]) },
411	{ HV_SME_Z_REG_1, offsetof(CPUARMState, vfp.zregs[1]) },
412	{ HV_SME_Z_REG_2, offsetof(CPUARMState, vfp.zregs[2]) },
413	{ HV_SME_Z_REG_3, offsetof(CPUARMState, vfp.zregs[3]) },
414	{ HV_SME_Z_REG_4, offsetof(CPUARMState, vfp.zregs[4]) },
415	{ HV_SME_Z_REG_5, offsetof(CPUARMState, vfp.zregs[5]) },
416	{ HV_SME_Z_REG_6, offsetof(CPUARMState, vfp.zregs[6]) },
417	{ HV_SME_Z_REG_7, offsetof(CPUARMState, vfp.zregs[7]) },
418	{ HV_SME_Z_REG_8, offsetof(CPUARMState, vfp.zregs[8]) },
419	{ HV_SME_Z_REG_9, offsetof(CPUARMState, vfp.zregs[9]) },
420	{ HV_SME_Z_REG_10, offsetof(CPUARMState, vfp.zregs[10]) },
421	{ HV_SME_Z_REG_11, offsetof(CPUARMState, vfp.zregs[11]) },
422	{ HV_SME_Z_REG_12, offsetof(CPUARMState, vfp.zregs[12]) },
423	{ HV_SME_Z_REG_13, offsetof(CPUARMState, vfp.zregs[13]) },
424	{ HV_SME_Z_REG_14, offsetof(CPUARMState, vfp.zregs[14]) },
425	{ HV_SME_Z_REG_15, offsetof(CPUARMState, vfp.zregs[15]) },
426	{ HV_SME_Z_REG_16, offsetof(CPUARMState, vfp.zregs[16]) },
427	{ HV_SME_Z_REG_17, offsetof(CPUARMState, vfp.zregs[17]) },
428	{ HV_SME_Z_REG_18, offsetof(CPUARMState, vfp.zregs[18]) },
429	{ HV_SME_Z_REG_19, offsetof(CPUARMState, vfp.zregs[19]) },
430	{ HV_SME_Z_REG_20, offsetof(CPUARMState, vfp.zregs[20]) },
431	{ HV_SME_Z_REG_21, offsetof(CPUARMState, vfp.zregs[21]) },
432	{ HV_SME_Z_REG_22, offsetof(CPUARMState, vfp.zregs[22]) },
433	{ HV_SME_Z_REG_23, offsetof(CPUARMState, vfp.zregs[23]) },
434	{ HV_SME_Z_REG_24, offsetof(CPUARMState, vfp.zregs[24]) },
435	{ HV_SME_Z_REG_25, offsetof(CPUARMState, vfp.zregs[25]) },
436	{ HV_SME_Z_REG_26, offsetof(CPUARMState, vfp.zregs[26]) },
437	{ HV_SME_Z_REG_27, offsetof(CPUARMState, vfp.zregs[27]) },
438	{ HV_SME_Z_REG_28, offsetof(CPUARMState, vfp.zregs[28]) },
439	{ HV_SME_Z_REG_29, offsetof(CPUARMState, vfp.zregs[29]) },
440	{ HV_SME_Z_REG_30, offsetof(CPUARMState, vfp.zregs[30]) },
441	{ HV_SME_Z_REG_31, offsetof(CPUARMState, vfp.zregs[31]) },
442	};
443
444	static const struct hvf_reg_match hvf_sme2_preg_match[] = {
445	{ HV_SME_P_REG_0, offsetof(CPUARMState, vfp.pregs[0]) },
446	{ HV_SME_P_REG_1, offsetof(CPUARMState, vfp.pregs[1]) },
447	{ HV_SME_P_REG_2, offsetof(CPUARMState, vfp.pregs[2]) },
448	{ HV_SME_P_REG_3, offsetof(CPUARMState, vfp.pregs[3]) },
449	{ HV_SME_P_REG_4, offsetof(CPUARMState, vfp.pregs[4]) },
450	{ HV_SME_P_REG_5, offsetof(CPUARMState, vfp.pregs[5]) },
451	{ HV_SME_P_REG_6, offsetof(CPUARMState, vfp.pregs[6]) },
452	{ HV_SME_P_REG_7, offsetof(CPUARMState, vfp.pregs[7]) },
453	{ HV_SME_P_REG_8, offsetof(CPUARMState, vfp.pregs[8]) },
454	{ HV_SME_P_REG_9, offsetof(CPUARMState, vfp.pregs[9]) },
455	{ HV_SME_P_REG_10, offsetof(CPUARMState, vfp.pregs[10]) },
456	{ HV_SME_P_REG_11, offsetof(CPUARMState, vfp.pregs[11]) },
457	{ HV_SME_P_REG_12, offsetof(CPUARMState, vfp.pregs[12]) },
458	{ HV_SME_P_REG_13, offsetof(CPUARMState, vfp.pregs[13]) },
459	{ HV_SME_P_REG_14, offsetof(CPUARMState, vfp.pregs[14]) },
460	{ HV_SME_P_REG_15, offsetof(CPUARMState, vfp.pregs[15]) },
461	};
462
463	/*
464	* QEMU uses KVM system register ids in the migration format.
465	* Conveniently, HVF uses the same encoding of the op* and cr* parameters
466	* within the low 16 bits of the ids. Thus conversion between the
467	* formats is trivial.
468	*/
469
470	#define KVMID_TO_HVF(KVM) ((KVM) & 0xffff)
471	#define HVF_TO_KVMID(HVF) \
472	(CP_REG_ARM64 \| CP_REG_SIZE_U64 \| CP_REG_ARM64_SYSREG \| (HVF))
473
474	/*
475	* In older SDKs, MDCR_EL2 was defined incorrectly.
476	* As such, override it with a #define if compiling with an older macOS SDK.
477	* https://lore.kernel.org/qemu-devel/BCCED674-EAEF-4755-9BE1-116FB36FB5C9@apple.com/
478	*/
479	#if !defined(MAC_OS_VERSION_26_0)
480	#define HV_SYS_REG_MDCR_EL2 0xe089
481	#endif
482
483	/*
484	* Verify this at compile-time.
485	*
486	* SME2 registers are guarded by a runtime availability attribute instead of a
487	* compile-time def, so verify those at runtime in hvf_arch_init_vcpu() below.
488	*
489	* Nested virt registers are handled via a runtime check, so override the
490	* guarded availability check done by Clang.
491	*/
492
493	#pragma clang diagnostic push
494	#pragma clang diagnostic ignored "-Wunguarded-availability"
495
496	#define DEF_SYSREG(HVF_ID, ...) \
497	QEMU_BUILD_BUG_ON(HVF_ID != KVMID_TO_HVF(KVMID_AA64_SYS_REG64(__VA_ARGS__)));
498	#define DEF_SYSREG_15_02(...)
499
500	#define DEF_SYSREG_EL2(HVF_ID, ...) \
501	QEMU_BUILD_BUG_ON(HVF_ID != KVMID_TO_HVF(KVMID_AA64_SYS_REG64(__VA_ARGS__)));
502
503	#define DEF_SYSREG_VGIC(HVF_ID, ...) \
504	QEMU_BUILD_BUG_ON(HVF_ID != KVMID_TO_HVF(KVMID_AA64_SYS_REG64(__VA_ARGS__)));
505
506	#define DEF_SYSREG_VGIC_EL2(HVF_ID, ...) \
507	QEMU_BUILD_BUG_ON(HVF_ID != KVMID_TO_HVF(KVMID_AA64_SYS_REG64(__VA_ARGS__)));
508
509	#include "sysreg.c.inc"
510
511	#undef DEF_SYSREG
512	#undef DEF_SYSREG_15_02
513	#undef DEF_SYSREG_EL2
514	#undef DEF_SYSREG_VGIC
515	#undef DEF_SYSREG_VGIC_EL2
516
517	#define DEF_SYSREG(HVF_ID, op0, op1, crn, crm, op2) {HVF_ID},
518	#define DEF_SYSREG_15_02(...)
519	#define DEF_SYSREG_EL2(HVF_ID, op0, op1, crn, crm, op2) {HVF_ID, .el2 = true},
520	#define DEF_SYSREG_VGIC(HVF_ID, op0, op1, crn, crm, op2) {HVF_ID, .vgic = true},
521	#define DEF_SYSREG_VGIC_EL2(HVF_ID, op0, op1, crn, crm, op2) {HVF_ID, true, true},
522
523	struct hvf_sreg {
524	hv_sys_reg_t sreg;
525	bool vgic;
526	bool el2;
527	};
528
529	static struct hvf_sreg hvf_sreg_list[] = {
530	#include "sysreg.c.inc"
531	};
532
533	#undef DEF_SYSREG
534	#undef DEF_SYSREG_15_02
535	#undef DEF_SYSREG_EL2
536	#undef DEF_SYSREG_VGIC
537	#undef DEF_SYSREG_VGIC_EL2
538
539	#pragma clang diagnostic pop
540
541	#define DEF_SYSREG(...)
542	#define DEF_SYSREG_15_02(HVF_ID, op0, op1, crn, crm, op2) {HVF_ID},
543	#define DEF_SYSREG_EL2(...)
544	#define DEF_SYSREG_VGIC(...)
545	#define DEF_SYSREG_VGIC_EL2(...)
546
547	API_AVAILABLE(macos(15.2))
548	static struct hvf_sreg hvf_sreg_list_sme2[] = {
549	#include "sysreg.c.inc"
550	};
551
552	#undef DEF_SYSREG
553	#undef DEF_SYSREG_15_02
554	#undef DEF_SYSREG_EL2
555	#undef DEF_SYSREG_VGIC
556	#undef DEF_SYSREG_VGIC_EL2
557
558	/*
559	* For FEAT_SME2 migration, we need to store PSTATE.{SM,ZA} bits which are
560	* accessible with the SVCR pseudo-register. However, in the HVF API this is
561	* not exposed as a system-register (i.e. HVF_SYS_REG_SVCR) but a custom
562	* struct, hv_vcpu_sme_state_t. So we need to define our own KVMID in order to
563	* store it in cpreg_values and make it migrateable.
564	*/
565	#define SVCR KVMID_AA64_SYS_REG64(3, 3, 4, 2, 2)
566
567	API_AVAILABLE(macos(15.2))
568	static void hvf_arch_put_sme(CPUState *cpu)
569	{
570	ARMCPU *arm_cpu = ARM_CPU(cpu);
571	CPUARMState *env = &arm_cpu->env;
572	const size_t svl_bytes = hvf_arm_sme2_get_svl();
573	const size_t z_size = svl_bytes;
574	const size_t preg_size = DIV_ROUND_UP(z_size, 8);
575	const size_t za_size = svl_bytes * svl_bytes;
576	hv_vcpu_sme_state_t sme_state = { 0 };
577	hv_return_t ret;
578	uint64_t svcr;
579	int n;
580
581	/*
582	* Set PSTATE.{SM,ZA} bits
583	*/
584	svcr = arm_cpu->cpreg_values[arm_cpu->cpreg_array_len - 1];
585	env->svcr = svcr;
586
587	/*
588	* Construct SVCR (PSTATE.{SM,ZA}) state to pass to HVF:
589	*/
590	sme_state.streaming_sve_mode_enabled = FIELD_EX64(env->svcr, SVCR, SM) > 0;
591	sme_state.za_storage_enabled = FIELD_EX64(env->svcr, SVCR, ZA) > 0;
592	ret = hv_vcpu_set_sme_state(cpu->accel->fd, &sme_state);
593	assert_hvf_ok(ret);
594
595	/*
596	* We only care about Z/P registers if we're in streaming SVE mode, i.e.
597	* PSTATE.SM is set, because only then can instructions that access them be
598	* used. We don't care about the register values otherwise. This is because
599	* when the processing unit exits/enters this mode, it zeroes out those
600	* registers.
601	*/
602	if (sme_state.streaming_sve_mode_enabled) {
603	for (n = 0; n < ARRAY_SIZE(hvf_sme2_zreg_match); ++n) {
604	ret = hv_vcpu_set_sme_z_reg(cpu->accel->fd,
605	hvf_sme2_zreg_match[n].reg,
606	(uint8_t *)&env->vfp.zregs[n].d[0],
607	z_size);
608	assert_hvf_ok(ret);
609	}
610
611	for (n = 0; n < ARRAY_SIZE(hvf_sme2_preg_match); ++n) {
612	ret = hv_vcpu_set_sme_p_reg(cpu->accel->fd,
613	hvf_sme2_preg_match[n].reg,
614	(uint8_t *)&env->vfp.pregs[n].p[0],
615	preg_size);
616	assert_hvf_ok(ret);
617	}
618	}
619
620	/*
621	* If PSTATE.ZA bit is set then ZA and ZT0 are valid, otherwise they are
622	* zeroed out.
623	*/
624	if (sme_state.za_storage_enabled) {
625	hv_sme_zt0_uchar64_t tmp = { 0 };
626
627	memcpy(&tmp, &env->za_state.zt0, 64);
628	ret = hv_vcpu_set_sme_zt0_reg(cpu->accel->fd, &tmp);
629	assert_hvf_ok(ret);
630
631	ret = hv_vcpu_set_sme_za_reg(cpu->accel->fd,
632	(uint8_t *)&env->za_state.za,
633	za_size);
634	assert_hvf_ok(ret);
635	}
636
637	return;
638	}
639
640	API_AVAILABLE(macos(15.2))
641	static void hvf_arch_get_sme(CPUState *cpu)
642	{
643	ARMCPU *arm_cpu = ARM_CPU(cpu);
644	CPUARMState *env = &arm_cpu->env;
645	const size_t svl_bytes = hvf_arm_sme2_get_svl();
646	const size_t z_size = svl_bytes;
647	const size_t preg_size = DIV_ROUND_UP(z_size, 8);
648	const size_t za_size = svl_bytes * svl_bytes;
649	hv_vcpu_sme_state_t sme_state = { 0 };
650	hv_return_t ret;
651	uint64_t svcr;
652	int n;
653
654	/*
655	* Get SVCR (PSTATE.{SM,ZA}) state from HVF:
656	*/
657	ret = hv_vcpu_get_sme_state(cpu->accel->fd, &sme_state);
658	assert_hvf_ok(ret);
659
660	/*
661	* Set SVCR first because changing it will zero out Z/P regs
662	*/
663	svcr =
664	(sme_state.za_storage_enabled ? R_SVCR_ZA_MASK : 0)
665	\| (sme_state.streaming_sve_mode_enabled ? R_SVCR_SM_MASK : 0);
666
667	aarch64_set_svcr(env, svcr, R_SVCR_ZA_MASK \| R_SVCR_SM_MASK);
668	arm_cpu->cpreg_values[arm_cpu->cpreg_array_len - 1] = svcr;
669
670	/*
671	* We only care about Z/P registers if we're in streaming SVE mode, i.e.
672	* PSTATE.SM is set, because only then can instructions that access them be
673	* used. We don't care about the register values otherwise. This is because
674	* when the processing unit exits/enters this mode, it zeroes out those
675	* registers.
676	*/
677	if (sme_state.streaming_sve_mode_enabled) {
678	for (n = 0; n < ARRAY_SIZE(hvf_sme2_zreg_match); ++n) {
679	ret = hv_vcpu_get_sme_z_reg(cpu->accel->fd,
680	hvf_sme2_zreg_match[n].reg,
681	(uint8_t *)&env->vfp.zregs[n].d[0],
682	z_size);
683	assert_hvf_ok(ret);
684	}
685
686	for (n = 0; n < ARRAY_SIZE(hvf_sme2_preg_match); ++n) {
687	ret = hv_vcpu_get_sme_p_reg(cpu->accel->fd,
688	hvf_sme2_preg_match[n].reg,
689	(uint8_t *)&env->vfp.pregs[n].p[0],
690	preg_size);
691	assert_hvf_ok(ret);
692	}
693	}
694
695	/*
696	* If PSTATE.ZA bit is set then ZA and ZT0 are valid, otherwise they are
697	* zeroed out.
698	*/
699	if (sme_state.za_storage_enabled) {
700	hv_sme_zt0_uchar64_t tmp = { 0 };
701
702	/* Get ZT0 in a tmp vector, and then copy it to env.za_state.zt0 */
703	ret = hv_vcpu_get_sme_zt0_reg(cpu->accel->fd, &tmp);
704	assert_hvf_ok(ret);
705
706	memcpy(&env->za_state.zt0, &tmp, 64);
707	ret = hv_vcpu_get_sme_za_reg(cpu->accel->fd,
708	(uint8_t *)&env->za_state.za,
709	za_size);
710	assert_hvf_ok(ret);
711
712	}
713
714	return;
715	}
716
717	static uint32_t hvf_reg2cp_reg(uint32_t reg)
718	{
719	return ENCODE_AA64_CP_REG((reg >> SYSREG_OP0_SHIFT) & SYSREG_OP0_MASK,
720	(reg >> SYSREG_OP1_SHIFT) & SYSREG_OP1_MASK,
721	(reg >> SYSREG_CRN_SHIFT) & SYSREG_CRN_MASK,
722	(reg >> SYSREG_CRM_SHIFT) & SYSREG_CRM_MASK,
723	(reg >> SYSREG_OP2_SHIFT) & SYSREG_OP2_MASK);
724	}
725
726	static bool hvf_sysreg_read_cp(CPUState cpu, const char cpname,
727	uint32_t reg, uint64_t *val)
728	{
729	ARMCPU *arm_cpu = ARM_CPU(cpu);
730	CPUARMState *env = &arm_cpu->env;
731	const ARMCPRegInfo *ri;
732
733	ri = get_arm_cp_reginfo(arm_cpu->cp_regs, hvf_reg2cp_reg(reg));
734	if (ri) {
735	if (!cp_access_ok(1, ri, true)) {
736	return false;
737	}
738	if (ri->accessfn) {
739	if (ri->accessfn(env, ri, true) != CP_ACCESS_OK) {
740	return false;
741	}
742	}
743	if (ri->type & ARM_CP_CONST) {
744	*val = ri->resetvalue;
745	} else if (ri->readfn) {
746	*val = ri->readfn(env, ri);
747	} else {
748	*val = raw_read(env, ri);
749	}
750	trace_hvf_emu_reginfo_read(cpname, ri->name, *val);
751	return true;
752	}
753
754	return false;
755	}
756
757	static bool hvf_sysreg_write_cp(CPUState cpu, const char cpname,
758	uint32_t reg, uint64_t val)
759	{
760	ARMCPU *arm_cpu = ARM_CPU(cpu);
761	CPUARMState *env = &arm_cpu->env;
762	const ARMCPRegInfo *ri;
763
764	ri = get_arm_cp_reginfo(arm_cpu->cp_regs, hvf_reg2cp_reg(reg));
765
766	if (ri) {
767	if (!cp_access_ok(1, ri, false)) {
768	return false;
769	}
770	if (ri->accessfn) {
771	if (ri->accessfn(env, ri, false) != CP_ACCESS_OK) {
772	return false;
773	}
774	}
775	if (ri->writefn) {
776	ri->writefn(env, ri, val);
777	} else {
778	raw_write(env, ri, val);
779	}
780
781	trace_hvf_emu_reginfo_write(cpname, ri->name, val);
782	return true;
783	}
784
785	return false;
786	}
787
788	int hvf_arch_get_registers(CPUState *cpu)
789	{
790	ARMCPU *arm_cpu = ARM_CPU(cpu);
791	CPUARMState *env = &arm_cpu->env;
792	hv_return_t ret;
793	uint64_t val;
794	hv_simd_fp_uchar16_t fpval;
795	int i, n;
796
797	assert(!cpu->vcpu_dirty);
798
799	for (i = 0; i < ARRAY_SIZE(hvf_reg_match); i++) {
800	ret = hv_vcpu_get_reg(cpu->accel->fd, hvf_reg_match[i].reg, &val);
801	(uint64_t )((void *)env + hvf_reg_match[i].offset) = val;
802	assert_hvf_ok(ret);
803	}
804
805	for (i = 0; i < ARRAY_SIZE(hvf_fpreg_match); i++) {
806	ret = hv_vcpu_get_simd_fp_reg(cpu->accel->fd, hvf_fpreg_match[i].reg,
807	&fpval);
808	memcpy((void *)env + hvf_fpreg_match[i].offset, &fpval, sizeof(fpval));
809	assert_hvf_ok(ret);
810	}
811
812	val = 0;
813	ret = hv_vcpu_get_reg(cpu->accel->fd, HV_REG_FPCR, &val);
814	assert_hvf_ok(ret);
815	vfp_set_fpcr(env, val);
816
817	val = 0;
818	ret = hv_vcpu_get_reg(cpu->accel->fd, HV_REG_FPSR, &val);
819	assert_hvf_ok(ret);
820	vfp_set_fpsr(env, val);
821
822	ret = hv_vcpu_get_reg(cpu->accel->fd, HV_REG_CPSR, &val);
823	assert_hvf_ok(ret);
824	pstate_write(env, val);
825
826	for (i = 0, n = arm_cpu->cpreg_array_len; i < n; i++) {
827	uint64_t kvm_id = arm_cpu->cpreg_indexes[i];
828	int hvf_id = KVMID_TO_HVF(kvm_id);
829
830	if (kvm_id == HVF_TO_KVMID(SVCR)) {
831	continue;
832	}
833
834	if (cpu->accel->guest_debug_enabled) {
835	/* Handle debug registers */
836	switch (hvf_id) {
837	case HV_SYS_REG_DBGBVR0_EL1:
838	case HV_SYS_REG_DBGBCR0_EL1:
839	case HV_SYS_REG_DBGWVR0_EL1:
840	case HV_SYS_REG_DBGWCR0_EL1:
841	case HV_SYS_REG_DBGBVR1_EL1:
842	case HV_SYS_REG_DBGBCR1_EL1:
843	case HV_SYS_REG_DBGWVR1_EL1:
844	case HV_SYS_REG_DBGWCR1_EL1:
845	case HV_SYS_REG_DBGBVR2_EL1:
846	case HV_SYS_REG_DBGBCR2_EL1:
847	case HV_SYS_REG_DBGWVR2_EL1:
848	case HV_SYS_REG_DBGWCR2_EL1:
849	case HV_SYS_REG_DBGBVR3_EL1:
850	case HV_SYS_REG_DBGBCR3_EL1:
851	case HV_SYS_REG_DBGWVR3_EL1:
852	case HV_SYS_REG_DBGWCR3_EL1:
853	case HV_SYS_REG_DBGBVR4_EL1:
854	case HV_SYS_REG_DBGBCR4_EL1:
855	case HV_SYS_REG_DBGWVR4_EL1:
856	case HV_SYS_REG_DBGWCR4_EL1:
857	case HV_SYS_REG_DBGBVR5_EL1:
858	case HV_SYS_REG_DBGBCR5_EL1:
859	case HV_SYS_REG_DBGWVR5_EL1:
860	case HV_SYS_REG_DBGWCR5_EL1:
861	case HV_SYS_REG_DBGBVR6_EL1:
862	case HV_SYS_REG_DBGBCR6_EL1:
863	case HV_SYS_REG_DBGWVR6_EL1:
864	case HV_SYS_REG_DBGWCR6_EL1:
865	case HV_SYS_REG_DBGBVR7_EL1:
866	case HV_SYS_REG_DBGBCR7_EL1:
867	case HV_SYS_REG_DBGWVR7_EL1:
868	case HV_SYS_REG_DBGWCR7_EL1:
869	case HV_SYS_REG_DBGBVR8_EL1:
870	case HV_SYS_REG_DBGBCR8_EL1:
871	case HV_SYS_REG_DBGWVR8_EL1:
872	case HV_SYS_REG_DBGWCR8_EL1:
873	case HV_SYS_REG_DBGBVR9_EL1:
874	case HV_SYS_REG_DBGBCR9_EL1:
875	case HV_SYS_REG_DBGWVR9_EL1:
876	case HV_SYS_REG_DBGWCR9_EL1:
877	case HV_SYS_REG_DBGBVR10_EL1:
878	case HV_SYS_REG_DBGBCR10_EL1:
879	case HV_SYS_REG_DBGWVR10_EL1:
880	case HV_SYS_REG_DBGWCR10_EL1:
881	case HV_SYS_REG_DBGBVR11_EL1:
882	case HV_SYS_REG_DBGBCR11_EL1:
883	case HV_SYS_REG_DBGWVR11_EL1:
884	case HV_SYS_REG_DBGWCR11_EL1:
885	case HV_SYS_REG_DBGBVR12_EL1:
886	case HV_SYS_REG_DBGBCR12_EL1:
887	case HV_SYS_REG_DBGWVR12_EL1:
888	case HV_SYS_REG_DBGWCR12_EL1:
889	case HV_SYS_REG_DBGBVR13_EL1:
890	case HV_SYS_REG_DBGBCR13_EL1:
891	case HV_SYS_REG_DBGWVR13_EL1:
892	case HV_SYS_REG_DBGWCR13_EL1:
893	case HV_SYS_REG_DBGBVR14_EL1:
894	case HV_SYS_REG_DBGBCR14_EL1:
895	case HV_SYS_REG_DBGWVR14_EL1:
896	case HV_SYS_REG_DBGWCR14_EL1:
897	case HV_SYS_REG_DBGBVR15_EL1:
898	case HV_SYS_REG_DBGBCR15_EL1:
899	case HV_SYS_REG_DBGWVR15_EL1:
900	case HV_SYS_REG_DBGWCR15_EL1: {
901	/*
902	* If the guest is being debugged, the vCPU's debug registers
903	* are holding the gdbstub's view of the registers (set in
904	* hvf_arch_update_guest_debug()).
905	* Since the environment is used to store only the guest's view
906	* of the registers, don't update it with the values from the
907	* vCPU but simply keep the values from the previous
908	* environment.
909	*/
910	uint32_t key = kvm_to_cpreg_id(kvm_id);
911	const ARMCPRegInfo *ri =
912	get_arm_cp_reginfo(arm_cpu->cp_regs, key);
913
914	val = read_raw_cp_reg(env, ri);
915
916	arm_cpu->cpreg_values[i] = val;
917	continue;
918	}
919	}
920	}
921
922	ret = hv_vcpu_get_sys_reg(cpu->accel->fd, hvf_id, &val);
923	assert_hvf_ok(ret);
924
925	arm_cpu->cpreg_values[i] = val;
926	}
927	if (cpu_isar_feature(aa64_sme, arm_cpu)) {
928	if (__builtin_available(macOS 15.2, *)) {
929	hvf_arch_get_sme(cpu);
930	} else {
931	g_assert_not_reached();
932	}
933	}
934	assert(write_list_to_cpustate(arm_cpu));
935
936	aarch64_restore_sp(env, arm_current_el(env));
937
938	return 0;
939	}
940
941	int hvf_arch_put_registers(CPUState *cpu)
942	{
943	ARMCPU *arm_cpu = ARM_CPU(cpu);
944	CPUARMState *env = &arm_cpu->env;
945	hv_return_t ret;
946	uint64_t val;
947	hv_simd_fp_uchar16_t fpval;
948	int i, n;
949
950	assert(cpu->vcpu_dirty);
951
952	/*
953	* Set SVCR first because changing it will zero out Z/P (including NEON)
954	* regs
955	*/
956	if (cpu_isar_feature(aa64_sme, arm_cpu)) {
957	if (__builtin_available(macOS 15.2, *)) {
958	hvf_arch_put_sme(cpu);
959	} else {
960	g_assert_not_reached();
961	}
962	}
963
964	for (i = 0; i < ARRAY_SIZE(hvf_reg_match); i++) {
965	val = (uint64_t )((void *)env + hvf_reg_match[i].offset);
966	ret = hv_vcpu_set_reg(cpu->accel->fd, hvf_reg_match[i].reg, val);
967	assert_hvf_ok(ret);
968	}
969
970	for (i = 0; i < ARRAY_SIZE(hvf_fpreg_match); i++) {
971	memcpy(&fpval, (void *)env + hvf_fpreg_match[i].offset, sizeof(fpval));
972	ret = hv_vcpu_set_simd_fp_reg(cpu->accel->fd, hvf_fpreg_match[i].reg,
973	fpval);
974	assert_hvf_ok(ret);
975	}
976
977	ret = hv_vcpu_set_reg(cpu->accel->fd, HV_REG_FPCR, vfp_get_fpcr(env));
978	assert_hvf_ok(ret);
979
980	ret = hv_vcpu_set_reg(cpu->accel->fd, HV_REG_FPSR, vfp_get_fpsr(env));
981	assert_hvf_ok(ret);
982
983	ret = hv_vcpu_set_reg(cpu->accel->fd, HV_REG_CPSR, pstate_read(env));
984	assert_hvf_ok(ret);
985
986	aarch64_save_sp(env, arm_current_el(env));
987
988	assert(write_cpustate_to_list(arm_cpu, false));
989	for (i = 0, n = arm_cpu->cpreg_array_len; i < n; i++) {
990	uint64_t kvm_id = arm_cpu->cpreg_indexes[i];
991	int hvf_id = KVMID_TO_HVF(kvm_id);
992
993	if (kvm_id == HVF_TO_KVMID(SVCR)) {
994	continue;
995	}
996
997	if (cpu->accel->guest_debug_enabled) {
998	/* Handle debug registers */
999	switch (hvf_id) {
1000	case HV_SYS_REG_DBGBVR0_EL1:
1001	case HV_SYS_REG_DBGBCR0_EL1:
1002	case HV_SYS_REG_DBGWVR0_EL1:
1003	case HV_SYS_REG_DBGWCR0_EL1:
1004	case HV_SYS_REG_DBGBVR1_EL1:
1005	case HV_SYS_REG_DBGBCR1_EL1:
1006	case HV_SYS_REG_DBGWVR1_EL1:
1007	case HV_SYS_REG_DBGWCR1_EL1:
1008	case HV_SYS_REG_DBGBVR2_EL1:
1009	case HV_SYS_REG_DBGBCR2_EL1:
1010	case HV_SYS_REG_DBGWVR2_EL1:
1011	case HV_SYS_REG_DBGWCR2_EL1:
1012	case HV_SYS_REG_DBGBVR3_EL1:
1013	case HV_SYS_REG_DBGBCR3_EL1:
1014	case HV_SYS_REG_DBGWVR3_EL1:
1015	case HV_SYS_REG_DBGWCR3_EL1:
1016	case HV_SYS_REG_DBGBVR4_EL1:
1017	case HV_SYS_REG_DBGBCR4_EL1:
1018	case HV_SYS_REG_DBGWVR4_EL1:
1019	case HV_SYS_REG_DBGWCR4_EL1:
1020	case HV_SYS_REG_DBGBVR5_EL1:
1021	case HV_SYS_REG_DBGBCR5_EL1:
1022	case HV_SYS_REG_DBGWVR5_EL1:
1023	case HV_SYS_REG_DBGWCR5_EL1:
1024	case HV_SYS_REG_DBGBVR6_EL1:
1025	case HV_SYS_REG_DBGBCR6_EL1:
1026	case HV_SYS_REG_DBGWVR6_EL1:
1027	case HV_SYS_REG_DBGWCR6_EL1:
1028	case HV_SYS_REG_DBGBVR7_EL1:
1029	case HV_SYS_REG_DBGBCR7_EL1:
1030	case HV_SYS_REG_DBGWVR7_EL1:
1031	case HV_SYS_REG_DBGWCR7_EL1:
1032	case HV_SYS_REG_DBGBVR8_EL1:
1033	case HV_SYS_REG_DBGBCR8_EL1:
1034	case HV_SYS_REG_DBGWVR8_EL1:
1035	case HV_SYS_REG_DBGWCR8_EL1:
1036	case HV_SYS_REG_DBGBVR9_EL1:
1037	case HV_SYS_REG_DBGBCR9_EL1:
1038	case HV_SYS_REG_DBGWVR9_EL1:
1039	case HV_SYS_REG_DBGWCR9_EL1:
1040	case HV_SYS_REG_DBGBVR10_EL1:
1041	case HV_SYS_REG_DBGBCR10_EL1:
1042	case HV_SYS_REG_DBGWVR10_EL1:
1043	case HV_SYS_REG_DBGWCR10_EL1:
1044	case HV_SYS_REG_DBGBVR11_EL1:
1045	case HV_SYS_REG_DBGBCR11_EL1:
1046	case HV_SYS_REG_DBGWVR11_EL1:
1047	case HV_SYS_REG_DBGWCR11_EL1:
1048	case HV_SYS_REG_DBGBVR12_EL1:
1049	case HV_SYS_REG_DBGBCR12_EL1:
1050	case HV_SYS_REG_DBGWVR12_EL1:
1051	case HV_SYS_REG_DBGWCR12_EL1:
1052	case HV_SYS_REG_DBGBVR13_EL1:
1053	case HV_SYS_REG_DBGBCR13_EL1:
1054	case HV_SYS_REG_DBGWVR13_EL1:
1055	case HV_SYS_REG_DBGWCR13_EL1:
1056	case HV_SYS_REG_DBGBVR14_EL1:
1057	case HV_SYS_REG_DBGBCR14_EL1:
1058	case HV_SYS_REG_DBGWVR14_EL1:
1059	case HV_SYS_REG_DBGWCR14_EL1:
1060	case HV_SYS_REG_DBGBVR15_EL1:
1061	case HV_SYS_REG_DBGBCR15_EL1:
1062	case HV_SYS_REG_DBGWVR15_EL1:
1063	case HV_SYS_REG_DBGWCR15_EL1:
1064	/*
1065	* If the guest is being debugged, the vCPU's debug registers
1066	* are already holding the gdbstub's view of the registers (set
1067	* in hvf_arch_update_guest_debug()).
1068	*/
1069	continue;
1070	}
1071	}
1072
1073	val = arm_cpu->cpreg_values[i];
1074	ret = hv_vcpu_set_sys_reg(cpu->accel->fd, hvf_id, val);
1075	assert_hvf_ok(ret);
1076	}
1077
1078	ret = hv_vcpu_set_vtimer_offset(cpu->accel->fd, hvf_state->vtimer_offset);
1079	assert_hvf_ok(ret);
1080
1081	return 0;
1082	}
1083
1084	/* Must be called by the owning thread */
1085	static void flush_cpu_state(CPUState *cpu)
1086	{
1087	if (cpu->vcpu_dirty) {
1088	hvf_arch_put_registers(cpu);
1089	cpu->vcpu_dirty = false;
1090	}
1091	}
1092
1093	/* Must be called by the owning thread */
1094	static void hvf_set_reg(CPUState *cpu, int rt, uint64_t val)
1095	{
1096	hv_return_t r;
1097
1098	flush_cpu_state(cpu);
1099
1100	if (rt < 31) {
1101	r = hv_vcpu_set_reg(cpu->accel->fd, HV_REG_X0 + rt, val);
1102	assert_hvf_ok(r);
1103	}
1104	}
1105
1106	/* Must be called by the owning thread */
1107	static uint64_t hvf_get_reg(CPUState *cpu, int rt)
1108	{
1109	uint64_t val = 0;
1110	hv_return_t r;
1111
1112	flush_cpu_state(cpu);
1113
1114	if (rt < 31) {
1115	r = hv_vcpu_get_reg(cpu->accel->fd, HV_REG_X0 + rt, &val);
1116	assert_hvf_ok(r);
1117	}
1118
1119	return val;
1120	}
1121
1122	static void clamp_id_aa64mmfr0_parange_to_ipa_size(ARMISARegisters *isar)
1123	{
1124	uint32_t ipa_size = chosen_ipa_bit_size ?
1125	chosen_ipa_bit_size : hvf_arch_get_max_ipa_bit_size();
1126	uint64_t id_aa64mmfr0;
1127
1128	/* Clamp down the PARange to the IPA size the kernel supports. */
1129	uint8_t index = round_down_to_parange_index(ipa_size);
1130	id_aa64mmfr0 = GET_IDREG(isar, ID_AA64MMFR0);
1131	id_aa64mmfr0 = (id_aa64mmfr0 & ~R_ID_AA64MMFR0_PARANGE_MASK) \| index;
1132	SET_IDREG(isar, ID_AA64MMFR0, id_aa64mmfr0);
1133	}
1134
1135	static bool hvf_arm_get_host_cpu_features(ARMHostCPUFeatures *ahcf)
1136	{
1137	ARMISARegisters host_isar = {};
1138	static const struct isar_regs {
1139	hv_feature_reg_t reg;
1140	ARMIDRegisterIdx index;
1141	} regs[] = {
1142	{ HV_FEATURE_REG_ID_AA64PFR0_EL1, ID_AA64PFR0_EL1_IDX },
1143	{ HV_FEATURE_REG_ID_AA64PFR1_EL1, ID_AA64PFR1_EL1_IDX },
1144	/* Add ID_AA64PFR2_EL1 here when HVF supports it */
1145	{ HV_FEATURE_REG_ID_AA64DFR0_EL1, ID_AA64DFR0_EL1_IDX },
1146	{ HV_FEATURE_REG_ID_AA64DFR1_EL1, ID_AA64DFR1_EL1_IDX },
1147	{ HV_FEATURE_REG_ID_AA64ISAR0_EL1, ID_AA64ISAR0_EL1_IDX },
1148	{ HV_FEATURE_REG_ID_AA64ISAR1_EL1, ID_AA64ISAR1_EL1_IDX },
1149	/* Add ID_AA64ISAR2_EL1 here when HVF supports it */
1150	{ HV_FEATURE_REG_ID_AA64MMFR0_EL1, ID_AA64MMFR0_EL1_IDX },
1151	{ HV_FEATURE_REG_ID_AA64MMFR1_EL1, ID_AA64MMFR1_EL1_IDX },
1152	{ HV_FEATURE_REG_ID_AA64MMFR2_EL1, ID_AA64MMFR2_EL1_IDX },
1153	/* Add ID_AA64MMFR3_EL1 here when HVF supports it */
1154	};
1155	hv_return_t r = HV_SUCCESS;
1156	hv_vcpu_config_t config = hv_vcpu_config_create();
1157	uint64_t t;
1158	int i;
1159
1160	ahcf->dtb_compatible = "arm,armv8";
1161	ahcf->features = (1ULL << ARM_FEATURE_V8) \|
1162	(1ULL << ARM_FEATURE_NEON) \|
1163	(1ULL << ARM_FEATURE_AARCH64) \|
1164	(1ULL << ARM_FEATURE_PMU) \|
1165	(1ULL << ARM_FEATURE_GENERIC_TIMER);
1166
1167	if (hvf_nested_virt_enabled()) {
1168	ahcf->features \|= 1ULL << ARM_FEATURE_EL2;
1169	}
1170
1171	for (i = 0; i < ARRAY_SIZE(regs); i++) {
1172	r \|= hv_vcpu_config_get_feature_reg(config, regs[i].reg,
1173	&host_isar.idregs[regs[i].index]);
1174	}
1175
1176	if (__builtin_available(macOS 15.2, *)) {
1177	static const struct sme_isar_regs {
1178	hv_feature_reg_t reg;
1179	ARMIDRegisterIdx index;
1180	} sme_regs[] = {
1181	{ HV_FEATURE_REG_ID_AA64SMFR0_EL1, ID_AA64SMFR0_EL1_IDX },
1182	{ HV_FEATURE_REG_ID_AA64ZFR0_EL1, ID_AA64ZFR0_EL1_IDX },
1183	};
1184
1185	if (hvf_arm_sme2_supported()) {
1186	for (i = 0; i < ARRAY_SIZE(sme_regs); i++) {
1187	r \|= hv_vcpu_config_get_feature_reg(config, sme_regs[i].reg,
1188	&host_isar.idregs[sme_regs[i].index]);
1189	}
1190	}
1191	}
1192
1193	os_release(config);
1194
1195	/*
1196	* Hardcode MIDR because Apple deliberately doesn't expose a divergent
1197	* MIDR across systems.
1198	*/
1199	t = FIELD_DP64(0, MIDR_EL1, IMPLEMENTER, 0x61); /* Apple */
1200	t = FIELD_DP64(t, MIDR_EL1, ARCHITECTURE, 0xf); /* v7 or later */
1201	t = FIELD_DP64(t, MIDR_EL1, PARTNUM, 0);
1202	t = FIELD_DP64(t, MIDR_EL1, VARIANT, 0);
1203	t = FIELD_DP64(t, MIDR_EL1, REVISION, 0);
1204	ahcf->midr = t;
1205
1206	clamp_id_aa64mmfr0_parange_to_ipa_size(&host_isar);
1207
1208	/*
1209	* Windows wants at least the PMU's cycles counter to be available.
1210	*
1211	* With kernel-irqchip=off, we "emulate" the cycles counter
1212	* in reference to time in QEMU. Having that, even with
1213	* ID_AA64DFR0_EL1.PMUVer = 0 is enough to make Windows happy.
1214	*
1215	* As it's a very inaccurate implementation with its only purpose
1216	* being making Windows boot, expose ID_AA64DFR0_EL1.PMUVer = 0
1217	* when kernel-irqchip=off.
1218	*
1219	* When kernel-irqchip=on and ID_AA64DFR0_EL1.PMUVer = 1,
1220	* the OS provides its own PMU emulation, which is currently
1221	* a cycles counter only emulation.
1222	*/
1223	if (hvf_irqchip_in_kernel()) {
1224	FIELD_DP64_IDREG(&host_isar, ID_AA64DFR0, PMUVER, 0x1);
1225	}
1226
1227	if (hvf_nested_virt_enabled()) {
1228	/* SME is not implemented with nested virt on the Apple side */
1229	FIELD_DP64_IDREG(&host_isar, ID_AA64PFR1, SME, 0);
1230	}
1231
1232	ahcf->isar = host_isar;
1233
1234	/*
1235	* A scratch vCPU returns SCTLR 0, so let's fill our default with the M1
1236	* boot SCTLR from https://github.com/AsahiLinux/m1n1/issues/97
1237	*/
1238	ahcf->reset_sctlr = 0x30100180;
1239	/*
1240	* SPAN is disabled by default when SCTLR.SPAN=1. To improve compatibility,
1241	* let's disable it on boot and then allow guest software to turn it on by
1242	* setting it to 0.
1243	*/
1244	ahcf->reset_sctlr \|= 0x00800000;
1245
1246	ahcf->sme_vq_supported = hvf_arm_sme2_supported() ? hvf_arm_sme2_get_svl() : 0;
1247
1248	/* Make sure we don't advertise AArch32 support for EL0/EL1 */
1249	if ((GET_IDREG(&host_isar, ID_AA64PFR0) & 0xff) != 0x11) {
1250	return false;
1251	}
1252
1253	return r == HV_SUCCESS;
1254	}
1255
1256	uint32_t hvf_arch_get_default_ipa_bit_size(void)
1257	{
1258	uint32_t default_ipa_size;
1259	hv_return_t ret = hv_vm_config_get_default_ipa_size(&default_ipa_size);
1260	assert_hvf_ok(ret);
1261
1262	return default_ipa_size;
1263	}
1264
1265	uint32_t hvf_arch_get_max_ipa_bit_size(void)
1266	{
1267	uint32_t max_ipa_size;
1268	hv_return_t ret = hv_vm_config_get_max_ipa_size(&max_ipa_size);
1269	assert_hvf_ok(ret);
1270
1271	/*
1272	* We clamp any IPA size we want to back the VM with to a valid PARange
1273	* value so the guest doesn't try and map memory outside of the valid range.
1274	* This logic just clamps the passed in IPA bit size to the first valid
1275	* PARange value <= to it.
1276	*/
1277	return round_down_to_parange_bit_size(max_ipa_size);
1278	}
1279
1280	void hvf_arm_set_cpu_features_from_host(ARMCPU *cpu)
1281	{
1282	if (!arm_host_cpu_features.dtb_compatible) {
1283	if (!hvf_enabled() \|\|
1284	!hvf_arm_get_host_cpu_features(&arm_host_cpu_features)) {
1285	/*
1286	* We can't report this error yet, so flag that we need to
1287	* in arm_cpu_realizefn().
1288	*/
1289	cpu->host_cpu_probe_failed = true;
1290	return;
1291	}
1292	}
1293
1294	cpu->dtb_compatible = arm_host_cpu_features.dtb_compatible;
1295	cpu->isar = arm_host_cpu_features.isar;
1296	cpu->env.features = arm_host_cpu_features.features;
1297	cpu->midr = arm_host_cpu_features.midr;
1298	cpu->reset_sctlr = arm_host_cpu_features.reset_sctlr;
1299	cpu->sme_vq.supported = arm_host_cpu_features.sme_vq_supported;
1300	}
1301
1302	void hvf_arch_vcpu_destroy(CPUState *cpu)
1303	{
1304	if (!hvf_irqchip_in_kernel()) {
1305	timer_free(cpu->accel->wfi_timer);
1306	cpu->accel->wfi_timer = NULL;
1307	}
1308	}
1309
1310	static bool hvf_arm_el2_supported(void)
1311	{
1312	bool is_nested_virt_supported;
1313	if (__builtin_available(macOS 15.0, *)) {
1314	hv_return_t ret = hv_vm_config_get_el2_supported(&is_nested_virt_supported);
1315	assert_hvf_ok(ret);
1316	} else {
1317	return false;
1318	}
1319	return is_nested_virt_supported;
1320	}
1321
1322
1323	hv_return_t hvf_arch_vm_create(MachineState *ms, uint32_t pa_range)
1324	{
1325	hv_return_t ret;
1326	hv_vm_config_t config = hv_vm_config_create();
1327
1328	ret = hv_vm_config_set_ipa_size(config, pa_range);
1329	if (ret != HV_SUCCESS) {
1330	goto cleanup;
1331	}
1332	chosen_ipa_bit_size = pa_range;
1333
1334	if (__builtin_available(macOS 15.0, *)) {
1335	if (hvf_nested_virt_enabled()) {
1336	if (!hvf_arm_el2_supported()) {
1337	error_report("Nested virtualization not supported on this system.");
1338	goto cleanup;
1339	}
1340	ret = hv_vm_config_set_el2_enabled(config, true);
1341	if (ret != HV_SUCCESS) {
1342	error_report("Failed to enable nested virtualization.");
1343	goto cleanup;
1344	}
1345	}
1346	}
1347
1348	ret = hv_vm_create(config);
1349	if (hvf_irqchip_in_kernel()) {
1350	if (__builtin_available(macOS 15.0, *)) {
1351	/*
1352	* Instantiate GIC.
1353	* This must be done prior to the creation of any vCPU
1354	* but past hv_vm_create()
1355	*/
1356	hv_gic_config_t cfg = hv_gic_config_create();
1357	hv_gic_config_set_distributor_base(cfg, 0x08000000);
1358	hv_gic_config_set_redistributor_base(cfg, 0x080A0000);
1359	ret = hv_gic_create(cfg);
1360	if (ret != HV_SUCCESS) {
1361	error_report("error creating platform VGIC");
1362	goto cleanup;
1363	}
1364	os_release(cfg);
1365	} else {
1366	error_report("HVF: Unsupported OS for platform vGIC.");
1367	ret = HV_UNSUPPORTED;
1368	goto cleanup;
1369	}
1370	}
1371
1372	cleanup:
1373	os_release(config);
1374
1375	return ret;
1376	}
1377
1378	static uint64_t get_cntfrq_el0(void)
1379	{
1380	uint64_t freq_hz = 0;
1381	asm volatile("mrs %0, cntfrq_el0" : "=r"(freq_hz));
1382	return freq_hz;
1383	}
1384
1385	int hvf_arch_init_vcpu(CPUState *cpu)
1386	{
1387	ARMCPU *arm_cpu = ARM_CPU(cpu);
1388	CPUARMState *env = &arm_cpu->env;
1389	uint32_t sregs_match_len = ARRAY_SIZE(hvf_sreg_list);
1390	uint32_t sregs_cnt = 0;
1391	uint64_t pfr;
1392	hv_return_t ret;
1393	int i;
1394
1395	if (__builtin_available(macOS 15.2, *)) {
1396	if (hvf_arm_sme2_supported()) {
1397	sregs_match_len += ARRAY_SIZE(hvf_sreg_list_sme2) + 1;
1398	}
1399
1400	#define DEF_SYSREG_15_02(HVF_ID, ...) \
1401	g_assert(HVF_ID == KVMID_TO_HVF(KVMID_AA64_SYS_REG64(__VA_ARGS__)));
1402	#define DEF_SYSREG(...)
1403	#define DEF_SYSREG_EL2(...)
1404	#define DEF_SYSREG_VGIC(...)
1405	#define DEF_SYSREG_VGIC_EL2(...)
1406
1407	#include "sysreg.c.inc"
1408
1409	#undef DEF_SYSREG
1410	#undef DEF_SYSREG_15_02
1411	}
1412	env->aarch64 = true;
1413
1414	/* system count frequency sanity check */
1415	assert(arm_cpu->gt_cntfrq_hz == get_cntfrq_el0());
1416
1417	/* Allocate enough space for our sysreg sync */
1418	arm_cpu->cpreg_indexes = g_renew(uint64_t, arm_cpu->cpreg_indexes,
1419	sregs_match_len);
1420	arm_cpu->cpreg_values = g_renew(uint64_t, arm_cpu->cpreg_values,
1421	sregs_match_len);
1422
1423	memset(arm_cpu->cpreg_values, 0, sregs_match_len * sizeof(uint64_t));
1424
1425	/* Populate cp list for all known sysregs */
1426	for (i = 0; i < ARRAY_SIZE(hvf_sreg_list); i++) {
1427	hv_sys_reg_t hvf_id = hvf_sreg_list[i].sreg;
1428	uint64_t kvm_id = HVF_TO_KVMID(hvf_id);
1429	uint32_t key = kvm_to_cpreg_id(kvm_id);
1430	const ARMCPRegInfo *ri = get_arm_cp_reginfo(arm_cpu->cp_regs, key);
1431
1432	if (hvf_sreg_list[i].vgic && !hvf_irqchip_in_kernel()) {
1433	continue;
1434	}
1435
1436	if (hvf_sreg_list[i].el2 && !hvf_nested_virt_enabled()) {
1437	continue;
1438	}
1439
1440	if (ri) {
1441	assert(!(ri->type & ARM_CP_NO_RAW));
1442	arm_cpu->cpreg_indexes[sregs_cnt++] = kvm_id;
1443	}
1444	}
1445	if (__builtin_available(macOS 15.2, *)) {
1446	if (hvf_arm_sme2_supported()) {
1447	for (i = 0; i < ARRAY_SIZE(hvf_sreg_list_sme2); i++) {
1448	hv_sys_reg_t hvf_id = hvf_sreg_list_sme2[i].sreg;
1449	uint64_t kvm_id = HVF_TO_KVMID(hvf_id);
1450	uint32_t key = kvm_to_cpreg_id(kvm_id);
1451	const ARMCPRegInfo *ri = get_arm_cp_reginfo(arm_cpu->cp_regs, key);
1452
1453	if (ri) {
1454	assert(!(ri->type & ARM_CP_NO_RAW));
1455	arm_cpu->cpreg_indexes[sregs_cnt++] = kvm_id;
1456	}
1457	}
1458	/*
1459	* Add SVCR last. It is elsewhere assumed its index is after
1460	* hvf_sreg_list and hvf_sreg_list_sme2.
1461	*/
1462	arm_cpu->cpreg_indexes[sregs_cnt++] = HVF_TO_KVMID(SVCR);
1463	}
1464	}
1465	arm_cpu->cpreg_array_len = sregs_cnt;
1466
1467	/* cpreg tuples must be in strictly ascending order */
1468	qsort(arm_cpu->cpreg_indexes, sregs_cnt, sizeof(uint64_t), compare_u64);
1469
1470	assert(write_cpustate_to_list(arm_cpu, false));
1471
1472	/* Set CP_NO_RAW system registers on init */
1473	ret = hv_vcpu_set_sys_reg(cpu->accel->fd, HV_SYS_REG_MIDR_EL1,
1474	arm_cpu->midr);
1475	assert_hvf_ok(ret);
1476
1477	ret = hv_vcpu_set_sys_reg(cpu->accel->fd, HV_SYS_REG_MPIDR_EL1,
1478	arm_cpu->mp_affinity);
1479	assert_hvf_ok(ret);
1480
1481	pfr = GET_IDREG(&arm_cpu->isar, ID_AA64PFR0);
1482	pfr \|= env->gicv3state ? (1 << 24) : 0;
1483	ret = hv_vcpu_set_sys_reg(cpu->accel->fd, HV_SYS_REG_ID_AA64PFR0_EL1, pfr);
1484	assert_hvf_ok(ret);
1485
1486	ret = hv_vcpu_set_sys_reg(cpu->accel->fd, HV_SYS_REG_ID_AA64ISAR0_EL1,
1487	GET_IDREG(&arm_cpu->isar, ID_AA64ISAR0));
1488	assert_hvf_ok(ret);
1489
1490	clamp_id_aa64mmfr0_parange_to_ipa_size(&arm_cpu->isar);
1491	ret = hv_vcpu_set_sys_reg(cpu->accel->fd, HV_SYS_REG_ID_AA64MMFR0_EL1,
1492	GET_IDREG(&arm_cpu->isar, ID_AA64MMFR0));
1493	assert_hvf_ok(ret);
1494
1495	if (!hvf_irqchip_in_kernel()) {
1496	cpu->accel->wfi_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL,
1497	hvf_wfi_timer_cb, cpu);
1498	}
1499
1500	aarch64_add_sme_properties(OBJECT(cpu));
1501	return 0;
1502	}
1503
1504	bool hvf_arch_cpu_realize(CPUState cs, Error *errp)
1505	{
1506	ARMCPU *cpu = ARM_CPU(cs);
1507
1508	/*
1509	* We must set the counter frequency HVF will be using
1510	* early, before arm_cpu_realizefn initializes the timers
1511	* with it.
1512	*/
1513	cpu->gt_cntfrq_hz = get_cntfrq_el0();
1514
1515	return true;
1516	}
1517
1518	void hvf_kick_vcpu_thread(CPUState *cpu)
1519	{
1520	hv_return_t ret;
1521	trace_hvf_kick_vcpu_thread(cpu->cpu_index, cpu->stop);
1522	cpus_kick_thread(cpu);
1523	ret = hv_vcpus_exit(&cpu->accel->fd, 1);
1524	assert_hvf_ok(ret);
1525	}
1526
1527	static void hvf_raise_exception(CPUState *cpu, uint32_t excp,
1528	uint32_t syndrome, int target_el)
1529	{
1530	ARMCPU *arm_cpu = ARM_CPU(cpu);
1531	CPUARMState *env = &arm_cpu->env;
1532
1533	cpu->exception_index = excp;
1534	env->exception.target_el = target_el;
1535	env->exception.syndrome = syndrome;
1536
1537	arm_cpu_do_interrupt(cpu);
1538	}
1539
1540	static void hvf_psci_cpu_off(ARMCPU *arm_cpu)
1541	{
1542	int32_t ret = arm_set_cpu_off(arm_cpu_mp_affinity(arm_cpu));
1543	assert(ret == QEMU_ARM_POWERCTL_RET_SUCCESS);
1544	}
1545
1546	static int hvf_psci_get_target_el(void)
1547	{
1548	if (hvf_nested_virt_enabled()) {
1549	return 2;
1550	}
1551	return 1;
1552	}
1553	/*
1554	* Handle a PSCI call.
1555	*
1556	* Returns 0 on success
1557	* -1 when the PSCI call is unknown,
1558	*/
1559	static bool hvf_handle_psci_call(CPUState cpu, int excp_ret)
1560	{
1561	ARMCPU *arm_cpu = ARM_CPU(cpu);
1562	CPUARMState *env = &arm_cpu->env;
1563	uint64_t param[4] = {
1564	env->xregs[0],
1565	env->xregs[1],
1566	env->xregs[2],
1567	env->xregs[3]
1568	};
1569	uint64_t context_id, mpidr;
1570	bool target_aarch64 = true;
1571	CPUState *target_cpu_state;
1572	ARMCPU *target_cpu;
1573	uint64_t entry;
1574	int32_t ret = 0;
1575
1576	trace_arm_psci_call(param[0], param[1], param[2], param[3],
1577	arm_cpu_mp_affinity(arm_cpu));
1578
1579	switch (param[0]) {
1580	case QEMU_PSCI_0_2_FN_PSCI_VERSION:
1581	ret = QEMU_PSCI_VERSION_1_1;
1582	break;
1583	case QEMU_PSCI_0_2_FN_MIGRATE_INFO_TYPE:
1584	ret = QEMU_PSCI_0_2_RET_TOS_MIGRATION_NOT_REQUIRED; /* No trusted OS */
1585	break;
1586	case QEMU_PSCI_0_2_FN_AFFINITY_INFO:
1587	case QEMU_PSCI_0_2_FN64_AFFINITY_INFO:
1588	mpidr = param[1];
1589
1590	switch (param[2]) {
1591	case 0:
1592	target_cpu_state = arm_get_cpu_by_id(mpidr);
1593	if (!target_cpu_state) {
1594	ret = QEMU_PSCI_RET_INVALID_PARAMS;
1595	break;
1596	}
1597	target_cpu = ARM_CPU(target_cpu_state);
1598
1599	ret = target_cpu->power_state;
1600	break;
1601	default:
1602	/* Everything above affinity level 0 is always on. */
1603	ret = 0;
1604	}
1605	break;
1606	case QEMU_PSCI_0_2_FN_SYSTEM_RESET:
1607	qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET);
1608	/*
1609	* QEMU reset and shutdown are async requests, but PSCI
1610	* mandates that we never return from the reset/shutdown
1611	* call, so power the CPU off now so it doesn't execute
1612	* anything further.
1613	*/
1614	hvf_psci_cpu_off(arm_cpu);
1615	break;
1616	case QEMU_PSCI_0_2_FN_SYSTEM_OFF:
1617	qemu_system_shutdown_request(SHUTDOWN_CAUSE_GUEST_SHUTDOWN);
1618	hvf_psci_cpu_off(arm_cpu);
1619	break;
1620	case QEMU_PSCI_0_1_FN_CPU_ON:
1621	case QEMU_PSCI_0_2_FN_CPU_ON:
1622	case QEMU_PSCI_0_2_FN64_CPU_ON:
1623	mpidr = param[1];
1624	entry = param[2];
1625	context_id = param[3];
1626	ret = arm_set_cpu_on(mpidr, entry, context_id,
1627	hvf_psci_get_target_el(), target_aarch64);
1628	break;
1629	case QEMU_PSCI_0_1_FN_CPU_OFF:
1630	case QEMU_PSCI_0_2_FN_CPU_OFF:
1631	hvf_psci_cpu_off(arm_cpu);
1632	break;
1633	case QEMU_PSCI_0_1_FN_CPU_SUSPEND:
1634	case QEMU_PSCI_0_2_FN_CPU_SUSPEND:
1635	case QEMU_PSCI_0_2_FN64_CPU_SUSPEND:
1636	/* Affinity levels are not supported in QEMU */
1637	if (param[1] & 0xfffe0000) {
1638	ret = QEMU_PSCI_RET_INVALID_PARAMS;
1639	break;
1640	}
1641	env->xregs[0] = 0;
1642	*excp_ret = EXCP_HLT;
1643	break;
1644	case QEMU_PSCI_0_1_FN_MIGRATE:
1645	case QEMU_PSCI_0_2_FN_MIGRATE:
1646	ret = QEMU_PSCI_RET_NOT_SUPPORTED;
1647	break;
1648	case QEMU_PSCI_1_0_FN_PSCI_FEATURES:
1649	switch (param[1]) {
1650	case QEMU_PSCI_0_2_FN_PSCI_VERSION:
1651	case QEMU_PSCI_0_2_FN_MIGRATE_INFO_TYPE:
1652	case QEMU_PSCI_0_2_FN_AFFINITY_INFO:
1653	case QEMU_PSCI_0_2_FN64_AFFINITY_INFO:
1654	case QEMU_PSCI_0_2_FN_SYSTEM_RESET:
1655	case QEMU_PSCI_0_2_FN_SYSTEM_OFF:
1656	case QEMU_PSCI_0_1_FN_CPU_ON:
1657	case QEMU_PSCI_0_2_FN_CPU_ON:
1658	case QEMU_PSCI_0_2_FN64_CPU_ON:
1659	case QEMU_PSCI_0_1_FN_CPU_OFF:
1660	case QEMU_PSCI_0_2_FN_CPU_OFF:
1661	case QEMU_PSCI_0_1_FN_CPU_SUSPEND:
1662	case QEMU_PSCI_0_2_FN_CPU_SUSPEND:
1663	case QEMU_PSCI_0_2_FN64_CPU_SUSPEND:
1664	case QEMU_PSCI_1_0_FN_PSCI_FEATURES:
1665	ret = 0;
1666	break;
1667	case QEMU_PSCI_0_1_FN_MIGRATE:
1668	case QEMU_PSCI_0_2_FN_MIGRATE:
1669	default:
1670	ret = QEMU_PSCI_RET_NOT_SUPPORTED;
1671	}
1672	break;
1673	default:
1674	return false;
1675	}
1676
1677	env->xregs[0] = ret;
1678	return true;
1679	}
1680
1681	static bool is_id_sysreg(uint32_t reg)
1682	{
1683	return SYSREG_OP0(reg) == 3 &&
1684	SYSREG_OP1(reg) == 0 &&
1685	SYSREG_CRN(reg) == 0 &&
1686	SYSREG_CRM(reg) >= 1 &&
1687	SYSREG_CRM(reg) < 8;
1688	}
1689
1690	static int hvf_sysreg_read(CPUState cpu, uint32_t reg, uint64_t val)
1691	{
1692	ARMCPU *arm_cpu = ARM_CPU(cpu);
1693	CPUARMState *env = &arm_cpu->env;
1694
1695	if (!hvf_irqchip_in_kernel() && arm_feature(env, ARM_FEATURE_PMU)) {
1696	switch (reg) {
1697	case SYSREG_PMCR_EL0:
1698	*val = env->cp15.c9_pmcr;
1699	return 0;
1700	case SYSREG_PMCCNTR_EL0:
1701	pmu_op_start(env);
1702	*val = env->cp15.c15_ccnt;
1703	pmu_op_finish(env);
1704	return 0;
1705	case SYSREG_PMCNTENCLR_EL0:
1706	*val = env->cp15.c9_pmcnten;
1707	return 0;
1708	case SYSREG_PMOVSCLR_EL0:
1709	*val = env->cp15.c9_pmovsr;
1710	return 0;
1711	case SYSREG_PMSELR_EL0:
1712	*val = env->cp15.c9_pmselr;
1713	return 0;
1714	case SYSREG_PMINTENCLR_EL1:
1715	*val = env->cp15.c9_pminten;
1716	return 0;
1717	case SYSREG_PMCCFILTR_EL0:
1718	*val = env->cp15.pmccfiltr_el0;
1719	return 0;
1720	case SYSREG_PMCNTENSET_EL0:
1721	*val = env->cp15.c9_pmcnten;
1722	return 0;
1723	case SYSREG_PMUSERENR_EL0:
1724	*val = env->cp15.c9_pmuserenr;
1725	return 0;
1726	case SYSREG_PMCEID0_EL0:
1727	case SYSREG_PMCEID1_EL0:
1728	/* We can't really count anything yet, declare all events invalid */
1729	*val = 0;
1730	return 0;
1731	}
1732	}
1733
1734	switch (reg) {
1735	case SYSREG_CNTPCT_EL0:
1736	case SYSREG_CNTP_CTL_EL0:
1737	case SYSREG_CNTP_CVAL_EL0:
1738	case SYSREG_CNTP_TVAL_EL0:
1739	assert(!hvf_irqchip_in_kernel());
1740	/* Call the TCG sysreg handler. */
1741	if (hvf_sysreg_read_cp(cpu, "PTimer", reg, val)) {
1742	return 0;
1743	}
1744	break;
1745	case SYSREG_OSLSR_EL1:
1746	*val = env->cp15.oslsr_el1;
1747	return 0;
1748	case SYSREG_OSDLR_EL1:
1749	/* Dummy register */
1750	return 0;
1751	case SYSREG_CNTHCTL_EL2:
1752	if (__builtin_available(macOS 15.0, *)) {
1753	assert_hvf_ok(hv_vcpu_get_sys_reg(cpu->accel->fd, HV_SYS_REG_CNTHCTL_EL2, val));
1754	}
1755	return 0;
1756	case SYSREG_MDCCINT_EL1:
1757	assert_hvf_ok(hv_vcpu_get_sys_reg(cpu->accel->fd, HV_SYS_REG_MDCCINT_EL1, val));
1758	return 0;
1759	case SYSREG_ICC_AP0R0_EL1:
1760	case SYSREG_ICC_AP0R1_EL1:
1761	case SYSREG_ICC_AP0R2_EL1:
1762	case SYSREG_ICC_AP0R3_EL1:
1763	case SYSREG_ICC_AP1R0_EL1:
1764	case SYSREG_ICC_AP1R1_EL1:
1765	case SYSREG_ICC_AP1R2_EL1:
1766	case SYSREG_ICC_AP1R3_EL1:
1767	case SYSREG_ICC_ASGI1R_EL1:
1768	case SYSREG_ICC_BPR0_EL1:
1769	case SYSREG_ICC_BPR1_EL1:
1770	case SYSREG_ICC_DIR_EL1:
1771	case SYSREG_ICC_EOIR0_EL1:
1772	case SYSREG_ICC_EOIR1_EL1:
1773	case SYSREG_ICC_HPPIR0_EL1:
1774	case SYSREG_ICC_HPPIR1_EL1:
1775	case SYSREG_ICC_IAR0_EL1:
1776	case SYSREG_ICC_IAR1_EL1:
1777	case SYSREG_ICC_IGRPEN0_EL1:
1778	case SYSREG_ICC_IGRPEN1_EL1:
1779	case SYSREG_ICC_PMR_EL1:
1780	case SYSREG_ICC_RPR_EL1:
1781	case SYSREG_ICC_SGI0R_EL1:
1782	case SYSREG_ICC_SGI1R_EL1:
1783	case SYSREG_ICC_SRE_EL1:
1784	case SYSREG_ICC_CTLR_EL1:
1785	assert(!hvf_irqchip_in_kernel());
1786	/* Call the TCG sysreg handler. This is only safe for GICv3 regs. */
1787	if (hvf_sysreg_read_cp(cpu, "GICv3", reg, val)) {
1788	return 0;
1789	}
1790	break;
1791	case SYSREG_DBGBVR0_EL1:
1792	case SYSREG_DBGBVR1_EL1:
1793	case SYSREG_DBGBVR2_EL1:
1794	case SYSREG_DBGBVR3_EL1:
1795	case SYSREG_DBGBVR4_EL1:
1796	case SYSREG_DBGBVR5_EL1:
1797	case SYSREG_DBGBVR6_EL1:
1798	case SYSREG_DBGBVR7_EL1:
1799	case SYSREG_DBGBVR8_EL1:
1800	case SYSREG_DBGBVR9_EL1:
1801	case SYSREG_DBGBVR10_EL1:
1802	case SYSREG_DBGBVR11_EL1:
1803	case SYSREG_DBGBVR12_EL1:
1804	case SYSREG_DBGBVR13_EL1:
1805	case SYSREG_DBGBVR14_EL1:
1806	case SYSREG_DBGBVR15_EL1:
1807	*val = env->cp15.dbgbvr[SYSREG_CRM(reg)];
1808	return 0;
1809	case SYSREG_DBGBCR0_EL1:
1810	case SYSREG_DBGBCR1_EL1:
1811	case SYSREG_DBGBCR2_EL1:
1812	case SYSREG_DBGBCR3_EL1:
1813	case SYSREG_DBGBCR4_EL1:
1814	case SYSREG_DBGBCR5_EL1:
1815	case SYSREG_DBGBCR6_EL1:
1816	case SYSREG_DBGBCR7_EL1:
1817	case SYSREG_DBGBCR8_EL1:
1818	case SYSREG_DBGBCR9_EL1:
1819	case SYSREG_DBGBCR10_EL1:
1820	case SYSREG_DBGBCR11_EL1:
1821	case SYSREG_DBGBCR12_EL1:
1822	case SYSREG_DBGBCR13_EL1:
1823	case SYSREG_DBGBCR14_EL1:
1824	case SYSREG_DBGBCR15_EL1:
1825	*val = env->cp15.dbgbcr[SYSREG_CRM(reg)];
1826	return 0;
1827	case SYSREG_DBGWVR0_EL1:
1828	case SYSREG_DBGWVR1_EL1:
1829	case SYSREG_DBGWVR2_EL1:
1830	case SYSREG_DBGWVR3_EL1:
1831	case SYSREG_DBGWVR4_EL1:
1832	case SYSREG_DBGWVR5_EL1:
1833	case SYSREG_DBGWVR6_EL1:
1834	case SYSREG_DBGWVR7_EL1:
1835	case SYSREG_DBGWVR8_EL1:
1836	case SYSREG_DBGWVR9_EL1:
1837	case SYSREG_DBGWVR10_EL1:
1838	case SYSREG_DBGWVR11_EL1:
1839	case SYSREG_DBGWVR12_EL1:
1840	case SYSREG_DBGWVR13_EL1:
1841	case SYSREG_DBGWVR14_EL1:
1842	case SYSREG_DBGWVR15_EL1:
1843	*val = env->cp15.dbgwvr[SYSREG_CRM(reg)];
1844	return 0;
1845	case SYSREG_DBGWCR0_EL1:
1846	case SYSREG_DBGWCR1_EL1:
1847	case SYSREG_DBGWCR2_EL1:
1848	case SYSREG_DBGWCR3_EL1:
1849	case SYSREG_DBGWCR4_EL1:
1850	case SYSREG_DBGWCR5_EL1:
1851	case SYSREG_DBGWCR6_EL1:
1852	case SYSREG_DBGWCR7_EL1:
1853	case SYSREG_DBGWCR8_EL1:
1854	case SYSREG_DBGWCR9_EL1:
1855	case SYSREG_DBGWCR10_EL1:
1856	case SYSREG_DBGWCR11_EL1:
1857	case SYSREG_DBGWCR12_EL1:
1858	case SYSREG_DBGWCR13_EL1:
1859	case SYSREG_DBGWCR14_EL1:
1860	case SYSREG_DBGWCR15_EL1:
1861	*val = env->cp15.dbgwcr[SYSREG_CRM(reg)];
1862	return 0;
1863	default:
1864	if (is_id_sysreg(reg)) {
1865	/* ID system registers read as RES0 */
1866	*val = 0;
1867	return 0;
1868	}
1869	}
1870
1871	cpu_synchronize_state(cpu);
1872	trace_hvf_unhandled_sysreg_read(env->pc, reg,
1873	SYSREG_OP0(reg),
1874	SYSREG_OP1(reg),
1875	SYSREG_CRN(reg),
1876	SYSREG_CRM(reg),
1877	SYSREG_OP2(reg));
1878	hvf_raise_exception(cpu, EXCP_UDEF, syn_uncategorized(), 1);
1879	return 1;
1880	}
1881
1882	static void pmu_update_irq(CPUARMState *env)
1883	{
1884	ARMCPU *cpu = env_archcpu(env);
1885	qemu_set_irq(cpu->pmu_interrupt, (env->cp15.c9_pmcr & PMCRE) &&
1886	(env->cp15.c9_pminten & env->cp15.c9_pmovsr));
1887	}
1888
1889	static bool pmu_event_supported(uint16_t number)
1890	{
1891	return false;
1892	}
1893
1894	/* Returns true if the counter (pass 31 for PMCCNTR) should count events using
1895	* the current EL, security state, and register configuration.
1896	*/
1897	static bool pmu_counter_enabled(CPUARMState *env, uint8_t counter)
1898	{
1899	uint64_t filter;
1900	bool enabled, filtered = true;
1901	int el = arm_current_el(env);
1902
1903	enabled = (env->cp15.c9_pmcr & PMCRE) &&
1904	(env->cp15.c9_pmcnten & (1 << counter));
1905
1906	if (counter == 31) {
1907	filter = env->cp15.pmccfiltr_el0;
1908	} else {
1909	filter = env->cp15.c14_pmevtyper[counter];
1910	}
1911
1912	if (el == 0) {
1913	filtered = filter & PMXEVTYPER_U;
1914	} else if (el == 1) {
1915	filtered = filter & PMXEVTYPER_P;
1916	}
1917
1918	if (counter != 31) {
1919	/*
1920	* If not checking PMCCNTR, ensure the counter is setup to an event we
1921	* support
1922	*/
1923	uint16_t event = filter & PMXEVTYPER_EVTCOUNT;
1924	if (!pmu_event_supported(event)) {
1925	return false;
1926	}
1927	}
1928
1929	return enabled && !filtered;
1930	}
1931
1932	static void pmswinc_write(CPUARMState *env, uint64_t value)
1933	{
1934	unsigned int i;
1935	for (i = 0; i < pmu_num_counters(env); i++) {
1936	/* Increment a counter's count iff: */
1937	if ((value & (1 << i)) && /* counter's bit is set */
1938	/* counter is enabled and not filtered */
1939	pmu_counter_enabled(env, i) &&
1940	/* counter is SW_INCR */
1941	(env->cp15.c14_pmevtyper[i] & PMXEVTYPER_EVTCOUNT) == 0x0) {
1942	/*
1943	* Detect if this write causes an overflow since we can't predict
1944	* PMSWINC overflows like we can for other events
1945	*/
1946	uint32_t new_pmswinc = env->cp15.c14_pmevcntr[i] + 1;
1947
1948	if (env->cp15.c14_pmevcntr[i] & ~new_pmswinc & INT32_MIN) {
1949	env->cp15.c9_pmovsr \|= (1 << i);
1950	pmu_update_irq(env);
1951	}
1952
1953	env->cp15.c14_pmevcntr[i] = new_pmswinc;
1954	}
1955	}
1956	}
1957
1958	static int hvf_sysreg_write(CPUState *cpu, uint32_t reg, uint64_t val)
1959	{
1960	ARMCPU *arm_cpu = ARM_CPU(cpu);
1961	CPUARMState *env = &arm_cpu->env;
1962
1963	trace_hvf_sysreg_write(reg,
1964	SYSREG_OP0(reg),
1965	SYSREG_OP1(reg),
1966	SYSREG_CRN(reg),
1967	SYSREG_CRM(reg),
1968	SYSREG_OP2(reg),
1969	val);
1970
1971	if (!hvf_irqchip_in_kernel() && arm_feature(env, ARM_FEATURE_PMU)) {
1972	switch (reg) {
1973	case SYSREG_PMCCNTR_EL0:
1974	pmu_op_start(env);
1975	env->cp15.c15_ccnt = val;
1976	pmu_op_finish(env);
1977	return 0;
1978	case SYSREG_PMCR_EL0:
1979	pmu_op_start(env);
1980
1981	if (val & PMCRC) {
1982	/* The counter has been reset */
1983	env->cp15.c15_ccnt = 0;
1984	}
1985
1986	if (val & PMCRP) {
1987	unsigned int i;
1988	for (i = 0; i < pmu_num_counters(env); i++) {
1989	env->cp15.c14_pmevcntr[i] = 0;
1990	}
1991	}
1992
1993	env->cp15.c9_pmcr &= ~PMCR_WRITABLE_MASK;
1994	env->cp15.c9_pmcr \|= (val & PMCR_WRITABLE_MASK);
1995
1996	pmu_op_finish(env);
1997	return 0;
1998	case SYSREG_PMUSERENR_EL0:
1999	env->cp15.c9_pmuserenr = val & 0xf;
2000	return 0;
2001	case SYSREG_PMCNTENSET_EL0:
2002	env->cp15.c9_pmcnten \|= (val & pmu_counter_mask(env));
2003	return 0;
2004	case SYSREG_PMCNTENCLR_EL0:
2005	env->cp15.c9_pmcnten &= ~(val & pmu_counter_mask(env));
2006	return 0;
2007	case SYSREG_PMINTENCLR_EL1:
2008	pmu_op_start(env);
2009	env->cp15.c9_pminten \|= val;
2010	pmu_op_finish(env);
2011	return 0;
2012	case SYSREG_PMOVSCLR_EL0:
2013	pmu_op_start(env);
2014	env->cp15.c9_pmovsr &= ~val;
2015	pmu_op_finish(env);
2016	return 0;
2017	case SYSREG_PMSWINC_EL0:
2018	pmu_op_start(env);
2019	pmswinc_write(env, val);
2020	pmu_op_finish(env);
2021	return 0;
2022	case SYSREG_PMSELR_EL0:
2023	env->cp15.c9_pmselr = val & 0x1f;
2024	return 0;
2025	case SYSREG_PMCCFILTR_EL0:
2026	pmu_op_start(env);
2027	env->cp15.pmccfiltr_el0 = val & PMCCFILTR_EL0;
2028	pmu_op_finish(env);
2029	return 0;
2030	}
2031	}
2032
2033	switch (reg) {
2034	case SYSREG_OSLAR_EL1:
2035	env->cp15.oslsr_el1 = val & 1;
2036	return 0;
2037	case SYSREG_CNTP_CTL_EL0:
2038	case SYSREG_CNTP_CVAL_EL0:
2039	case SYSREG_CNTP_TVAL_EL0:
2040	assert(!hvf_irqchip_in_kernel());
2041	/* Call the TCG sysreg handler. */
2042	if (hvf_sysreg_write_cp(cpu, "PTimer", reg, val)) {
2043	return 0;
2044	}
2045	break;
2046	case SYSREG_OSDLR_EL1:
2047	/* Dummy register */
2048	return 0;
2049	case SYSREG_CNTHCTL_EL2:
2050	if (__builtin_available(macOS 15.0, *)) {
2051	assert_hvf_ok(hv_vcpu_set_sys_reg(cpu->accel->fd, HV_SYS_REG_CNTHCTL_EL2, val));
2052	}
2053	return 0;
2054	case SYSREG_MDCCINT_EL1:
2055	assert_hvf_ok(hv_vcpu_set_sys_reg(cpu->accel->fd, HV_SYS_REG_MDCCINT_EL1, val));
2056	return 0;
2057	case SYSREG_LORC_EL1:
2058	/* Dummy register */
2059	return 0;
2060	case SYSREG_ICC_AP0R0_EL1:
2061	case SYSREG_ICC_AP0R1_EL1:
2062	case SYSREG_ICC_AP0R2_EL1:
2063	case SYSREG_ICC_AP0R3_EL1:
2064	case SYSREG_ICC_AP1R0_EL1:
2065	case SYSREG_ICC_AP1R1_EL1:
2066	case SYSREG_ICC_AP1R2_EL1:
2067	case SYSREG_ICC_AP1R3_EL1:
2068	case SYSREG_ICC_ASGI1R_EL1:
2069	case SYSREG_ICC_BPR0_EL1:
2070	case SYSREG_ICC_BPR1_EL1:
2071	case SYSREG_ICC_CTLR_EL1:
2072	case SYSREG_ICC_DIR_EL1:
2073	case SYSREG_ICC_EOIR0_EL1:
2074	case SYSREG_ICC_EOIR1_EL1:
2075	case SYSREG_ICC_HPPIR0_EL1:
2076	case SYSREG_ICC_HPPIR1_EL1:
2077	case SYSREG_ICC_IAR0_EL1:
2078	case SYSREG_ICC_IAR1_EL1:
2079	case SYSREG_ICC_IGRPEN0_EL1:
2080	case SYSREG_ICC_IGRPEN1_EL1:
2081	case SYSREG_ICC_PMR_EL1:
2082	case SYSREG_ICC_RPR_EL1:
2083	case SYSREG_ICC_SGI0R_EL1:
2084	case SYSREG_ICC_SGI1R_EL1:
2085	case SYSREG_ICC_SRE_EL1:
2086	assert(!hvf_irqchip_in_kernel());
2087	/* Call the TCG sysreg handler. This is only safe for GICv3 regs. */
2088	if (hvf_sysreg_write_cp(cpu, "GICv3", reg, val)) {
2089	return 0;
2090	}
2091	break;
2092	case SYSREG_MDSCR_EL1:
2093	env->cp15.mdscr_el1 = val;
2094	return 0;
2095	case SYSREG_DBGBVR0_EL1:
2096	case SYSREG_DBGBVR1_EL1:
2097	case SYSREG_DBGBVR2_EL1:
2098	case SYSREG_DBGBVR3_EL1:
2099	case SYSREG_DBGBVR4_EL1:
2100	case SYSREG_DBGBVR5_EL1:
2101	case SYSREG_DBGBVR6_EL1:
2102	case SYSREG_DBGBVR7_EL1:
2103	case SYSREG_DBGBVR8_EL1:
2104	case SYSREG_DBGBVR9_EL1:
2105	case SYSREG_DBGBVR10_EL1:
2106	case SYSREG_DBGBVR11_EL1:
2107	case SYSREG_DBGBVR12_EL1:
2108	case SYSREG_DBGBVR13_EL1:
2109	case SYSREG_DBGBVR14_EL1:
2110	case SYSREG_DBGBVR15_EL1:
2111	env->cp15.dbgbvr[SYSREG_CRM(reg)] = val;
2112	return 0;
2113	case SYSREG_DBGBCR0_EL1:
2114	case SYSREG_DBGBCR1_EL1:
2115	case SYSREG_DBGBCR2_EL1:
2116	case SYSREG_DBGBCR3_EL1:
2117	case SYSREG_DBGBCR4_EL1:
2118	case SYSREG_DBGBCR5_EL1:
2119	case SYSREG_DBGBCR6_EL1:
2120	case SYSREG_DBGBCR7_EL1:
2121	case SYSREG_DBGBCR8_EL1:
2122	case SYSREG_DBGBCR9_EL1:
2123	case SYSREG_DBGBCR10_EL1:
2124	case SYSREG_DBGBCR11_EL1:
2125	case SYSREG_DBGBCR12_EL1:
2126	case SYSREG_DBGBCR13_EL1:
2127	case SYSREG_DBGBCR14_EL1:
2128	case SYSREG_DBGBCR15_EL1:
2129	env->cp15.dbgbcr[SYSREG_CRM(reg)] = val;
2130	return 0;
2131	case SYSREG_DBGWVR0_EL1:
2132	case SYSREG_DBGWVR1_EL1:
2133	case SYSREG_DBGWVR2_EL1:
2134	case SYSREG_DBGWVR3_EL1:
2135	case SYSREG_DBGWVR4_EL1:
2136	case SYSREG_DBGWVR5_EL1:
2137	case SYSREG_DBGWVR6_EL1:
2138	case SYSREG_DBGWVR7_EL1:
2139	case SYSREG_DBGWVR8_EL1:
2140	case SYSREG_DBGWVR9_EL1:
2141	case SYSREG_DBGWVR10_EL1:
2142	case SYSREG_DBGWVR11_EL1:
2143	case SYSREG_DBGWVR12_EL1:
2144	case SYSREG_DBGWVR13_EL1:
2145	case SYSREG_DBGWVR14_EL1:
2146	case SYSREG_DBGWVR15_EL1:
2147	env->cp15.dbgwvr[SYSREG_CRM(reg)] = val;
2148	return 0;
2149	case SYSREG_DBGWCR0_EL1:
2150	case SYSREG_DBGWCR1_EL1:
2151	case SYSREG_DBGWCR2_EL1:
2152	case SYSREG_DBGWCR3_EL1:
2153	case SYSREG_DBGWCR4_EL1:
2154	case SYSREG_DBGWCR5_EL1:
2155	case SYSREG_DBGWCR6_EL1:
2156	case SYSREG_DBGWCR7_EL1:
2157	case SYSREG_DBGWCR8_EL1:
2158	case SYSREG_DBGWCR9_EL1:
2159	case SYSREG_DBGWCR10_EL1:
2160	case SYSREG_DBGWCR11_EL1:
2161	case SYSREG_DBGWCR12_EL1:
2162	case SYSREG_DBGWCR13_EL1:
2163	case SYSREG_DBGWCR14_EL1:
2164	case SYSREG_DBGWCR15_EL1:
2165	env->cp15.dbgwcr[SYSREG_CRM(reg)] = val;
2166	return 0;
2167	}
2168
2169	cpu_synchronize_state(cpu);
2170	trace_hvf_unhandled_sysreg_write(env->pc, reg,
2171	SYSREG_OP0(reg),
2172	SYSREG_OP1(reg),
2173	SYSREG_CRN(reg),
2174	SYSREG_CRM(reg),
2175	SYSREG_OP2(reg));
2176	hvf_raise_exception(cpu, EXCP_UDEF, syn_uncategorized(), 1);
2177	return 1;
2178	}
2179
2180	/* Must be called by the owning thread */
2181	static int hvf_inject_interrupts(CPUState *cpu)
2182	{
2183	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_FIQ)) {
2184	trace_hvf_inject_fiq();
2185	hv_vcpu_set_pending_interrupt(cpu->accel->fd, HV_INTERRUPT_TYPE_FIQ,
2186	true);
2187	}
2188
2189	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) {
2190	trace_hvf_inject_irq();
2191	hv_vcpu_set_pending_interrupt(cpu->accel->fd, HV_INTERRUPT_TYPE_IRQ,
2192	true);
2193	}
2194
2195	return 0;
2196	}
2197
2198	static uint64_t hvf_vtimer_val_raw(void)
2199	{
2200	/*
2201	* mach_absolute_time() returns the vtimer value without the VM
2202	* offset that we define. Add our own offset on top.
2203	*/
2204	return mach_absolute_time() - hvf_state->vtimer_offset;
2205	}
2206
2207	static void hvf_wfi_timer_cb(void *opaque)
2208	{
2209	CPUState *cpu = opaque;
2210	ARMCPU *arm_cpu = ARM_CPU(cpu);
2211
2212	/*
2213	* vtimer expired while the CPU was halted for WFI.
2214	* Mirror HV_EXIT_REASON_VTIMER_ACTIVATED: raise the vtimer
2215	* interrupt and mark as masked so hvf_sync_vtimer() will
2216	* check and unmask when the guest handles it.
2217	*
2218	* The interrupt delivery chain (GIC -> cpu_interrupt ->
2219	* qemu_cpu_kick) wakes the vCPU thread from halt_cond.
2220	*/
2221	qemu_set_irq(arm_cpu->gt_timer_outputs[GTIMER_VIRT], 1);
2222	cpu->accel->vtimer_masked = true;
2223	}
2224
2225	/*
2226	* Arm a host-side QEMU_CLOCK_VIRTUAL timer to fire when the guest's
2227	* vtimer (CNTV_CVAL_EL0) is scheduled to expire. HVF only delivers
2228	* HV_EXIT_REASON_VTIMER_ACTIVATED during hv_vcpu_run(), which we won't
2229	* call while the vCPU is halted, so we need this to wake the vCPU.
2230	*
2231	* QEMU_CLOCK_VIRTUAL pauses while the VM is stopped, which keeps the
2232	* timer in lockstep with the guest's view of vtime across pause/resume.
2233	*
2234	* Caller must supply the current CNTV_CTL_EL0 and CNTV_CVAL_EL0 values,
2235	* since the appropriate source (HVF vs. env) depends on context.
2236	*
2237	* Returns 0 if the timer was armed (or if the vtimer is disabled/masked
2238	* and the vCPU should still halt waiting on another event), or -1 if
2239	* the vtimer has already expired.
2240	*/
2241	static int hvf_arm_wfi_timer(CPUState *cpu, uint64_t ctl, uint64_t cval)
2242	{
2243	ARMCPU *arm_cpu = ARM_CPU(cpu);
2244	uint64_t now;
2245	int64_t delta_ns;
2246
2247	if (!(ctl & TMR_CTL_ENABLE) \|\| (ctl & TMR_CTL_IMASK)) {
2248	return 0;
2249	}
2250
2251	now = hvf_vtimer_val_raw();
2252	if (cval <= now) {
2253	return -1;
2254	}
2255
2256	delta_ns = muldiv64(cval - now, NANOSECONDS_PER_SECOND,
2257	arm_cpu->gt_cntfrq_hz);
2258	timer_mod(cpu->accel->wfi_timer,
2259	qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + delta_ns);
2260	return 0;
2261	}
2262
2263	static int hvf_wfi(CPUState *cpu)
2264	{
2265	if (cpu_has_work(cpu)) {
2266	/*
2267	* Don't bother to go into our "low power state" if
2268	* we would just wake up immediately.
2269	*/
2270	return 0;
2271	}
2272
2273	if (!hvf_irqchip_in_kernel()) {
2274	uint64_t ctl, cval;
2275	hv_return_t r;
2276
2277	/*
2278	* Read the vtimer state directly from HVF. We're on the vCPU
2279	* thread, just exited from hv_vcpu_run(), so HVF holds the
2280	* authoritative values and env may be stale.
2281	*/
2282	r = hv_vcpu_get_sys_reg(cpu->accel->fd, HV_SYS_REG_CNTV_CTL_EL0,
2283	&ctl);
2284	assert_hvf_ok(r);
2285	r = hv_vcpu_get_sys_reg(cpu->accel->fd, HV_SYS_REG_CNTV_CVAL_EL0,
2286	&cval);
2287	assert_hvf_ok(r);
2288
2289	if (hvf_arm_wfi_timer(cpu, ctl, cval) < 0) {
2290	/* vtimer already expired, don't halt */
2291	return 0;
2292	}
2293	}
2294
2295	cpu->halted = 1;
2296	return EXCP_HLT;
2297	}
2298
2299	/* Must be called by the owning thread */
2300	static void hvf_sync_vtimer(CPUState *cpu)
2301	{
2302	ARMCPU *arm_cpu = ARM_CPU(cpu);
2303	hv_return_t r;
2304	uint64_t ctl;
2305	bool irq_state;
2306
2307	if (!cpu->accel->vtimer_masked) {
2308	/* We will get notified on vtimer changes by hvf, nothing to do */
2309	return;
2310	}
2311
2312	r = hv_vcpu_get_sys_reg(cpu->accel->fd, HV_SYS_REG_CNTV_CTL_EL0, &ctl);
2313	assert_hvf_ok(r);
2314
2315	irq_state = (ctl & (TMR_CTL_ENABLE \| TMR_CTL_IMASK \| TMR_CTL_ISTATUS)) ==
2316	(TMR_CTL_ENABLE \| TMR_CTL_ISTATUS);
2317	qemu_set_irq(arm_cpu->gt_timer_outputs[GTIMER_VIRT], irq_state);
2318
2319	if (!irq_state) {
2320	/* Timer no longer asserting, we can unmask it */
2321	r = hv_vcpu_set_vtimer_mask(cpu->accel->fd, false);
2322	assert_hvf_ok(r);
2323	cpu->accel->vtimer_masked = false;
2324	}
2325	}
2326
2327	static int hvf_handle_exception(CPUState cpu, hv_vcpu_exit_exception_t excp)
2328	{
2329	CPUARMState *env = cpu_env(cpu);
2330	ARMCPU *arm_cpu = env_archcpu(env);
2331	uint64_t syndrome = excp->syndrome;
2332	uint32_t ec = syn_get_ec(syndrome);
2333	bool advance_pc = false;
2334	hv_return_t r;
2335	int ret = 0;
2336
2337	switch (ec) {
2338	case EC_SOFTWARESTEP: {
2339	ret = EXCP_DEBUG;
2340
2341	if (!cpu_single_stepping(cpu)) {
2342	error_report("EC_SOFTWARESTEP but single-stepping not enabled");
2343	}
2344	break;
2345	}
2346	case EC_AA64_BKPT: {
2347	ret = EXCP_DEBUG;
2348
2349	cpu_synchronize_state(cpu);
2350
2351	if (!hvf_find_sw_breakpoint(cpu, env->pc)) {
2352	/* Re-inject into the guest */
2353	ret = 0;
2354	hvf_raise_exception(cpu, EXCP_BKPT, syn_aa64_bkpt(0), 1);
2355	}
2356	break;
2357	}
2358	case EC_BREAKPOINT: {
2359	ret = EXCP_DEBUG;
2360
2361	cpu_synchronize_state(cpu);
2362
2363	if (!find_hw_breakpoint(cpu, env->pc)) {
2364	error_report("EC_BREAKPOINT but unknown hw breakpoint");
2365	}
2366	break;
2367	}
2368	case EC_WATCHPOINT: {
2369	ret = EXCP_DEBUG;
2370
2371	cpu_synchronize_state(cpu);
2372
2373	CPUWatchpoint *wp =
2374	find_hw_watchpoint(cpu, excp->virtual_address);
2375	if (!wp) {
2376	error_report("EXCP_DEBUG but unknown hw watchpoint");
2377	}
2378	cpu->watchpoint_hit = wp;
2379	break;
2380	}
2381	case EC_DATAABORT: {
2382	bool isv = FIELD_EX32(syndrome, DABORT_ISS, ISV);
2383	bool iswrite = FIELD_EX32(syndrome, DABORT_ISS, WNR);
2384	bool s1ptw = FIELD_EX32(syndrome, DABORT_ISS, S1PTW);
2385	bool sse = FIELD_EX32(syndrome, DABORT_ISS, SSE);
2386	uint32_t sas = FIELD_EX32(syndrome, DABORT_ISS, SAS);
2387	uint32_t len = 1 << sas;
2388	uint32_t srt = FIELD_EX32(syndrome, DABORT_ISS, SRT);
2389	uint32_t cm = FIELD_EX32(syndrome, DABORT_ISS, CM);
2390	uint64_t val = 0;
2391	uint64_t ipa = excp->physical_address;
2392	AddressSpace *as = cpu_get_address_space(cpu, ARMASIdx_NS);
2393
2394	trace_hvf_data_abort(excp->virtual_address, ipa, isv,
2395	iswrite, s1ptw, len, srt);
2396
2397	if (cm) {
2398	/* We don't cache MMIO regions */
2399	advance_pc = true;
2400	break;
2401	}
2402
2403	/* Handle dirty page logging for ram. */
2404	if (iswrite) {
2405	hwaddr xlat;
2406	MemoryRegion *mr = address_space_translate(as, ipa, &xlat,
2407	NULL, true,
2408	MEMTXATTRS_UNSPECIFIED);
2409	if (memory_region_is_ram(mr)) {
2410	uintptr_t page_size = qemu_real_host_page_size();
2411	intptr_t page_mask = -(intptr_t)page_size;
2412	uint64_t ipa_page = ipa & page_mask;
2413
2414	/* TODO: Inject exception to the guest. */
2415	assert(!mr->readonly);
2416
2417	if (memory_region_get_dirty_log_mask(mr)) {
2418	memory_region_set_dirty(mr, xlat, page_size);
2419	hvf_unprotect_dirty_range(ipa_page, page_size);
2420	}
2421
2422	/* Retry with page writes enabled. */
2423	break;
2424	}
2425	}
2426
2427	/*
2428	* TODO: If s1ptw, this is an error in the guest os page tables.
2429	* Inject the exception into the guest.
2430	*/
2431	assert(!s1ptw);
2432
2433	/*
2434	* TODO: ISV will be 0 for SIMD or SVE accesses.
2435	* Inject the exception into the guest.
2436	*/
2437	assert(isv);
2438
2439	/*
2440	* Emulate MMIO.
2441	* TODO: Inject faults for errors.
2442	*/
2443	if (iswrite) {
2444	val = hvf_get_reg(cpu, srt);
2445	address_space_write(as, ipa, MEMTXATTRS_UNSPECIFIED, &val, len);
2446	} else {
2447	address_space_read(as, ipa, MEMTXATTRS_UNSPECIFIED, &val, len);
2448	if (sse) {
2449	val = sextract64(val, 0, len * 8);
2450	}
2451	hvf_set_reg(cpu, srt, val);
2452	}
2453	advance_pc = true;
2454	break;
2455	}
2456	case EC_SYSTEMREGISTERTRAP: {
2457	bool isread = (syndrome >> 0) & 1;
2458	uint32_t rt = (syndrome >> 5) & 0x1f;
2459	uint32_t reg = syndrome & SYSREG_MASK;
2460	uint64_t val;
2461	int sysreg_ret = 0;
2462
2463	if (isread) {
2464	sysreg_ret = hvf_sysreg_read(cpu, reg, &val);
2465	if (!sysreg_ret) {
2466	trace_hvf_sysreg_read(reg,
2467	SYSREG_OP0(reg),
2468	SYSREG_OP1(reg),
2469	SYSREG_CRN(reg),
2470	SYSREG_CRM(reg),
2471	SYSREG_OP2(reg),
2472	val);
2473	hvf_set_reg(cpu, rt, val);
2474	}
2475	} else {
2476	val = hvf_get_reg(cpu, rt);
2477	sysreg_ret = hvf_sysreg_write(cpu, reg, val);
2478	}
2479
2480	advance_pc = !sysreg_ret;
2481	break;
2482	}
2483	case EC_WFX_TRAP:
2484	advance_pc = true;
2485	if (!(syndrome & WFX_IS_WFE)) {
2486	ret = hvf_wfi(cpu);
2487	}
2488	break;
2489	case EC_AA64_HVC:
2490	cpu_synchronize_state(cpu);
2491	if (arm_cpu->psci_conduit == QEMU_PSCI_CONDUIT_HVC) {
2492	/* Do NOT advance $pc for HVC */
2493	if (!hvf_handle_psci_call(cpu, &ret)) {
2494	trace_hvf_unknown_hvc(env->pc, env->xregs[0]);
2495	/* SMCCC 1.3 section 5.2 says every unknown SMCCC call returns -1 */
2496	env->xregs[0] = -1;
2497	}
2498	} else {
2499	trace_hvf_unknown_hvc(env->pc, env->xregs[0]);
2500	hvf_raise_exception(cpu, EXCP_UDEF, syn_uncategorized(), 1);
2501	}
2502	break;
2503	case EC_AA64_SMC:
2504	cpu_synchronize_state(cpu);
2505	if (arm_cpu->psci_conduit == QEMU_PSCI_CONDUIT_SMC) {
2506	/* Secure Monitor Call exception, we need to advance $pc */
2507	advance_pc = true;
2508
2509	if (!hvf_handle_psci_call(cpu, &ret)) {
2510	trace_hvf_unknown_smc(env->xregs[0]);
2511	/* SMCCC 1.3 section 5.2 says every unknown SMCCC call returns -1 */
2512	env->xregs[0] = -1;
2513	}
2514	} else {
2515	trace_hvf_unknown_smc(env->xregs[0]);
2516	hvf_raise_exception(cpu, EXCP_UDEF, syn_uncategorized(), 1);
2517	}
2518	break;
2519	case EC_INSNABORT: {
2520	uint32_t set = (syndrome >> 12) & 3;
2521	bool fnv = (syndrome >> 10) & 1;
2522	bool ea = (syndrome >> 9) & 1;
2523	bool s1ptw = (syndrome >> 7) & 1;
2524	uint32_t ifsc = (syndrome >> 0) & 0x3f;
2525
2526	trace_hvf_insn_abort(env->pc, set, fnv, ea, s1ptw, ifsc);
2527
2528	/* fall through */
2529	}
2530	default:
2531	cpu_synchronize_state(cpu);
2532	trace_hvf_exit(syndrome, ec, env->pc);
2533	error_report("0x%llx: unhandled exception ec=0x%x", env->pc, ec);
2534	}
2535
2536	/* flush any changed cpu state back to HVF */
2537	flush_cpu_state(cpu);
2538
2539	if (advance_pc) {
2540	uint64_t pc;
2541
2542
2543	r = hv_vcpu_get_reg(cpu->accel->fd, HV_REG_PC, &pc);
2544	assert_hvf_ok(r);
2545	pc += 4;
2546	r = hv_vcpu_set_reg(cpu->accel->fd, HV_REG_PC, pc);
2547	assert_hvf_ok(r);
2548
2549	/* Handle single-stepping over instructions which trigger a VM exit */
2550	if (cpu_single_stepping(cpu)) {
2551	ret = EXCP_DEBUG;
2552	}
2553	}
2554
2555	return ret;
2556	}
2557
2558	static int hvf_handle_vmexit(CPUState cpu, hv_vcpu_exit_t exit)
2559	{
2560	ARMCPU *arm_cpu = env_archcpu(cpu_env(cpu));
2561	int ret = 0;
2562
2563	switch (exit->reason) {
2564	case HV_EXIT_REASON_EXCEPTION:
2565	if (!hvf_irqchip_in_kernel()) {
2566	hvf_sync_vtimer(cpu);
2567	}
2568	ret = hvf_handle_exception(cpu, &exit->exception);
2569	break;
2570	case HV_EXIT_REASON_VTIMER_ACTIVATED:
2571	assert(!hvf_irqchip_in_kernel());
2572	qemu_set_irq(arm_cpu->gt_timer_outputs[GTIMER_VIRT], 1);
2573	cpu->accel->vtimer_masked = true;
2574	break;
2575	case HV_EXIT_REASON_CANCELED:
2576	/* we got kicked, no exit to process */
2577	ret = -1;
2578	break;
2579	default:
2580	g_assert_not_reached();
2581	}
2582
2583	return ret;
2584	}
2585
2586	int hvf_arch_vcpu_exec(CPUState *cpu)
2587	{
2588	int ret;
2589	hv_return_t r;
2590
2591	if (cpu->halted) {
2592	if (!cpu_has_work(cpu)) {
2593	return EXCP_HLT;
2594	}
2595	cpu->halted = 0;
2596	if (!hvf_irqchip_in_kernel()) {
2597	timer_del(cpu->accel->wfi_timer);
2598	}
2599	}
2600
2601	flush_cpu_state(cpu);
2602
2603	do {
2604	if (!(cpu->singlestep_flags & SSTEP_NOIRQ) &&
2605	hvf_inject_interrupts(cpu)) {
2606	return EXCP_INTERRUPT;
2607	}
2608
2609	bql_unlock();
2610	cpu_exec_start(cpu);
2611	r = hv_vcpu_run(cpu->accel->fd);
2612	cpu_exec_end(cpu);
2613	bql_lock();
2614	switch (r) {
2615	case HV_SUCCESS:
2616	ret = hvf_handle_vmexit(cpu, cpu->accel->exit);
2617	break;
2618	case HV_ILLEGAL_GUEST_STATE:
2619	trace_hvf_illegal_guest_state();
2620	/* fall through */
2621	default:
2622	g_assert_not_reached();
2623	}
2624	} while (ret == 0);
2625
2626	return ret;
2627	}
2628
2629	static const VMStateDescription vmstate_hvf_vtimer = {
2630	.name = "hvf-vtimer",
2631	.version_id = 1,
2632	.minimum_version_id = 1,
2633	.fields = (const VMStateField[]) {
2634	VMSTATE_UINT64(vtimer_val, HVFVTimer),
2635	VMSTATE_END_OF_LIST()
2636	},
2637	};
2638
2639	static void hvf_vm_state_change(void *opaque, bool running, RunState state)
2640	{
2641	HVFVTimer *s = opaque;
2642
2643	if (running) {
2644	/* Update vtimer offset on all CPUs */
2645	hvf_state->vtimer_offset = mach_absolute_time() - s->vtimer_val;
2646	cpu_synchronize_all_states();
2647
2648	/*
2649	* After migration restore (or any resume), the wfi_timer is not
2650	* scheduled on this QEMU instance, so re-arm it for any halted
2651	* vCPU with a pending vtimer. For a non-migration resume the
2652	* QEMU_CLOCK_VIRTUAL timer was already scheduled; recomputing the
2653	* deadline produces the same value and is a harmless no-op.
2654	*
2655	* cpu_synchronize_all_states() above ensures env mirrors the
2656	* authoritative vtimer state (whether that came from HVF or from
2657	* the migration stream), so we can safely read it here from the
2658	* iothread.
2659	*
2660	* Only applies when we own the wfi_timer; with an in-kernel vGIC
2661	* the timer is never allocated and HVF handles vtimer wake-ups.
2662	*/
2663	if (!hvf_irqchip_in_kernel()) {
2664	CPUState *cpu;
2665
2666	CPU_FOREACH(cpu) {
2667	ARMCPU *arm_cpu;
2668	uint64_t ctl, cval;
2669
2670	if (!cpu->accel \|\| !cpu->halted) {
2671	continue;
2672	}
2673
2674	arm_cpu = ARM_CPU(cpu);
2675	ctl = arm_cpu->env.cp15.c14_timer[GTIMER_VIRT].ctl;
2676	cval = arm_cpu->env.cp15.c14_timer[GTIMER_VIRT].cval;
2677
2678	if (hvf_arm_wfi_timer(cpu, ctl, cval) < 0) {
2679	/*
2680	* vtimer already expired while we were paused; raise
2681	* the IRQ now so the halted vCPU wakes up.
2682	*/
2683	hvf_wfi_timer_cb(cpu);
2684	}
2685	}
2686	}
2687	} else {
2688	/* Remember vtimer value on every pause */
2689	s->vtimer_val = hvf_vtimer_val_raw();
2690	}
2691	}
2692
2693	int hvf_arch_init(void)
2694	{
2695	hvf_state->vtimer_offset = mach_absolute_time();
2696	vmstate_register(NULL, 0, &vmstate_hvf_vtimer, &vtimer);
2697	qemu_add_vm_change_state_handler(hvf_vm_state_change, &vtimer);
2698
2699	hvf_arm_init_debug();
2700
2701	return 0;
2702	}
2703
2704	static const uint32_t brk_insn = 0xd4200000;
2705
2706	int hvf_arch_insert_sw_breakpoint(CPUState cpu, struct hvf_sw_breakpoint bp)
2707	{
2708	if (cpu_memory_rw_debug(cpu, bp->pc, (uint8_t *)&bp->saved_insn, 4, 0) \|\|
2709	cpu_memory_rw_debug(cpu, bp->pc, (uint8_t *)&brk_insn, 4, 1)) {
2710	return -EINVAL;
2711	}
2712	return 0;
2713	}
2714
2715	int hvf_arch_remove_sw_breakpoint(CPUState cpu, struct hvf_sw_breakpoint bp)
2716	{
2717	static uint32_t brk;
2718
2719	if (cpu_memory_rw_debug(cpu, bp->pc, (uint8_t *)&brk, 4, 0) \|\|
2720	brk != brk_insn \|\|
2721	cpu_memory_rw_debug(cpu, bp->pc, (uint8_t *)&bp->saved_insn, 4, 1)) {
2722	return -EINVAL;
2723	}
2724	return 0;
2725	}
2726
2727	int hvf_arch_insert_gdbstub_hw_breakpoint(vaddr addr, vaddr len,
2728	GdbBreakpointType type)
2729	{
2730	switch (type) {
2731	case GDB_BREAKPOINT_HW:
2732	return insert_hw_breakpoint(addr);
2733	case GDB_WATCHPOINT_READ:
2734	case GDB_WATCHPOINT_WRITE:
2735	case GDB_WATCHPOINT_ACCESS:
2736	return insert_gdbstub_hw_watchpoint(addr, len, type);
2737	default:
2738	return -ENOSYS;
2739	}
2740	}
2741
2742	int hvf_arch_remove_gdbstub_hw_breakpoint(vaddr addr, vaddr len,
2743	GdbBreakpointType type)
2744	{
2745	switch (type) {
2746	case GDB_BREAKPOINT_HW:
2747	return delete_hw_breakpoint(addr);
2748	case GDB_WATCHPOINT_READ:
2749	case GDB_WATCHPOINT_WRITE:
2750	case GDB_WATCHPOINT_ACCESS:
2751	return delete_gdbstub_hw_watchpoint(addr, len, type);
2752	default:
2753	return -ENOSYS;
2754	}
2755	}
2756
2757	void hvf_arch_remove_all_gdbstub_hw_breakpoints(void)
2758	{
2759	if (cur_hw_wps > 0) {
2760	g_array_remove_range(hw_watchpoints, 0, cur_hw_wps);
2761	}
2762	if (cur_hw_bps > 0) {
2763	g_array_remove_range(hw_breakpoints, 0, cur_hw_bps);
2764	}
2765	}
2766
2767	/*
2768	* Update the vCPU with the gdbstub's view of debug registers. This view
2769	* consists of all hardware breakpoints and watchpoints inserted so far while
2770	* debugging the guest.
2771	* Must be called by the owning thread.
2772	*/
2773	static void hvf_put_gdbstub_debug_registers(CPUState *cpu)
2774	{
2775	hv_return_t r = HV_SUCCESS;
2776	int i;
2777
2778	for (i = 0; i < cur_hw_bps; i++) {
2779	HWBreakpoint *bp = get_hw_bp(i);
2780	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgbcr_regs[i], bp->bcr);
2781	assert_hvf_ok(r);
2782	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgbvr_regs[i], bp->bvr);
2783	assert_hvf_ok(r);
2784	}
2785	for (i = cur_hw_bps; i < max_hw_bps; i++) {
2786	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgbcr_regs[i], 0);
2787	assert_hvf_ok(r);
2788	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgbvr_regs[i], 0);
2789	assert_hvf_ok(r);
2790	}
2791
2792	for (i = 0; i < cur_hw_wps; i++) {
2793	HWWatchpoint *wp = get_hw_wp(i);
2794	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgwcr_regs[i], wp->wcr);
2795	assert_hvf_ok(r);
2796	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgwvr_regs[i], wp->wvr);
2797	assert_hvf_ok(r);
2798	}
2799	for (i = cur_hw_wps; i < max_hw_wps; i++) {
2800	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgwcr_regs[i], 0);
2801	assert_hvf_ok(r);
2802	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgwvr_regs[i], 0);
2803	assert_hvf_ok(r);
2804	}
2805	}
2806
2807	/*
2808	* Update the vCPU with the guest's view of debug registers. This view is kept
2809	* in the environment at all times.
2810	* Must be called by the owning thread.
2811	*/
2812	static void hvf_put_guest_debug_registers(CPUState *cpu)
2813	{
2814	ARMCPU *arm_cpu = ARM_CPU(cpu);
2815	CPUARMState *env = &arm_cpu->env;
2816	hv_return_t r = HV_SUCCESS;
2817	int i;
2818
2819	for (i = 0; i < max_hw_bps; i++) {
2820	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgbcr_regs[i],
2821	env->cp15.dbgbcr[i]);
2822	assert_hvf_ok(r);
2823	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgbvr_regs[i],
2824	env->cp15.dbgbvr[i]);
2825	assert_hvf_ok(r);
2826	}
2827
2828	for (i = 0; i < max_hw_wps; i++) {
2829	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgwcr_regs[i],
2830	env->cp15.dbgwcr[i]);
2831	assert_hvf_ok(r);
2832	r = hv_vcpu_set_sys_reg(cpu->accel->fd, dbgwvr_regs[i],
2833	env->cp15.dbgwvr[i]);
2834	assert_hvf_ok(r);
2835	}
2836	}
2837
2838	static inline bool hvf_arm_hw_debug_active(CPUState *cpu)
2839	{
2840	return ((cur_hw_wps > 0) \|\| (cur_hw_bps > 0));
2841	}
2842
2843	/* Must be called by the owning thread */
2844	static void hvf_arch_set_traps(CPUState *cpu)
2845	{
2846	bool should_enable_traps = false;
2847	hv_return_t r = HV_SUCCESS;
2848
2849	/* Check whether guest debugging is enabled for at least one vCPU; if it
2850	* is, enable exiting the guest on all vCPUs */
2851	should_enable_traps \|= cpu->accel->guest_debug_enabled;
2852	/* Set whether debug exceptions exit the guest */
2853	r = hv_vcpu_set_trap_debug_exceptions(cpu->accel->fd,
2854	should_enable_traps);
2855	assert_hvf_ok(r);
2856
2857	/* Set whether accesses to debug registers exit the guest */
2858	r = hv_vcpu_set_trap_debug_reg_accesses(cpu->accel->fd,
2859	should_enable_traps);
2860	assert_hvf_ok(r);
2861	}
2862
2863	void hvf_arch_update_guest_debug(CPUState *cpu)
2864	{
2865	ARMCPU *arm_cpu = ARM_CPU(cpu);
2866	CPUARMState *env = &arm_cpu->env;
2867
2868	/* Check whether guest debugging is enabled */
2869	cpu->accel->guest_debug_enabled = cpu_single_stepping(cpu) \|\|
2870	hvf_sw_breakpoints_active(cpu) \|\|
2871	hvf_arm_hw_debug_active(cpu);
2872
2873	/* Update debug registers */
2874	if (cpu->accel->guest_debug_enabled) {
2875	hvf_put_gdbstub_debug_registers(cpu);
2876	} else {
2877	hvf_put_guest_debug_registers(cpu);
2878	}
2879
2880	cpu_synchronize_state(cpu);
2881
2882	/* Enable/disable single-stepping */
2883	if (cpu_single_stepping(cpu)) {
2884	env->cp15.mdscr_el1 =
2885	deposit64(env->cp15.mdscr_el1, MDSCR_EL1_SS_SHIFT, 1, 1);
2886	pstate_write(env, pstate_read(env) \| PSTATE_SS);
2887	} else {
2888	env->cp15.mdscr_el1 =
2889	deposit64(env->cp15.mdscr_el1, MDSCR_EL1_SS_SHIFT, 1, 0);
2890	}
2891
2892	/* Enable/disable Breakpoint exceptions */
2893	if (hvf_arm_hw_debug_active(cpu)) {
2894	env->cp15.mdscr_el1 =
2895	deposit64(env->cp15.mdscr_el1, MDSCR_EL1_MDE_SHIFT, 1, 1);
2896	} else {
2897	env->cp15.mdscr_el1 =
2898	deposit64(env->cp15.mdscr_el1, MDSCR_EL1_MDE_SHIFT, 1, 0);
2899	}
2900
2901	hvf_arch_set_traps(cpu);
2902	}