whpx-all.c · samitouri/QOSamiQemu · siGit Code & Deploy

Code Commits Issues Pulls Sessions CI/CD

master

c 3,372 lines 105 KB

Raw

1	/*
2	* QEMU Windows Hypervisor Platform accelerator (WHPX)
3	*
4	* Copyright Microsoft Corp. 2017
5	*
6	* This work is licensed under the terms of the GNU GPL, version 2 or later.
7	* See the COPYING file in the top-level directory.
8	*
9	*/
10
11	#include "qemu/osdep.h"
12	#include "cpu.h"
13	#include "qemu/typedefs.h"
14	#include "system/address-spaces.h"
15	#include "system/ioport.h"
16	#include "gdbstub/helpers.h"
17	#include "qemu/accel.h"
18	#include "accel/accel-ops.h"
19	#include "system/memory.h"
20	#include "system/whpx.h"
21	#include "system/cpus.h"
22	#include "system/runstate.h"
23	#include "qemu/main-loop.h"
24	#include "qemu/memalign.h"
25	#include "hw/core/boards.h"
26	#include "hw/intc/ioapic.h"
27	#include "hw/intc/i8259.h"
28	#include "hw/i386/x86.h"
29	#include "hw/i386/apic_internal.h"
30	#include "qemu/error-report.h"
31	#include "qapi/error.h"
32	#include "qapi/qapi-types-common.h"
33	#include "qapi/qapi-visit-common.h"
34	#include "migration/blocker.h"
35	#include "host-cpu.h"
36	#include "accel/accel-cpu-target.h"
37	#include <winerror.h>
38
39	#include "system/whpx-internal.h"
40	#include "system/whpx-accel-ops.h"
41	#include "system/whpx-all.h"
42	#include "system/whpx-common.h"
43	#include "whpx-i386.h"
44
45	#include "emulate/x86_decode.h"
46	#include "emulate/x86_emu.h"
47	#include "emulate/x86_flags.h"
48	#include "emulate/x86_mmu.h"
49	#include "trace.h"
50
51	#include <winhvplatform.h>
52
53	#define HYPERV_APIC_BUS_FREQUENCY (200000000ULL)
54	/* for kernel-irqchip=off */
55	#define HV_X64_MSR_APIC_FREQUENCY 0x40000023
56	#define HV_X64_MSR_VP_ASSIST_PAGE 0x40000073
57	#define HV_X64_MSR_GUEST_IDLE 0x400000f0
58
59	static bool is_modern_os = true;
60
61	static const WHV_REGISTER_NAME whpx_register_names[] = {
62
63	/* X64 General purpose registers */
64	WHvX64RegisterRax,
65	WHvX64RegisterRcx,
66	WHvX64RegisterRdx,
67	WHvX64RegisterRbx,
68	WHvX64RegisterRsp,
69	WHvX64RegisterRbp,
70	WHvX64RegisterRsi,
71	WHvX64RegisterRdi,
72	WHvX64RegisterR8,
73	WHvX64RegisterR9,
74	WHvX64RegisterR10,
75	WHvX64RegisterR11,
76	WHvX64RegisterR12,
77	WHvX64RegisterR13,
78	WHvX64RegisterR14,
79	WHvX64RegisterR15,
80	WHvX64RegisterRip,
81	WHvX64RegisterRflags,
82
83	/* X64 Segment registers */
84	WHvX64RegisterEs,
85	WHvX64RegisterCs,
86	WHvX64RegisterSs,
87	WHvX64RegisterDs,
88	WHvX64RegisterFs,
89	WHvX64RegisterGs,
90	WHvX64RegisterLdtr,
91	WHvX64RegisterTr,
92
93	/* X64 Table registers */
94	WHvX64RegisterIdtr,
95	WHvX64RegisterGdtr,
96
97	/* X64 Control Registers */
98	WHvX64RegisterCr0,
99	WHvX64RegisterCr2,
100	WHvX64RegisterCr3,
101	WHvX64RegisterCr4,
102
103	/* X64 Debug Registers */
104	/*
105	* WHvX64RegisterDr0,
106	* WHvX64RegisterDr1,
107	* WHvX64RegisterDr2,
108	* WHvX64RegisterDr3,
109	* WHvX64RegisterDr6,
110	* WHvX64RegisterDr7,
111	*/
112
113	/* X64 MSRs */
114	WHvX64RegisterEfer,
115	#ifdef TARGET_X86_64
116	WHvX64RegisterKernelGsBase,
117	#endif
118	WHvX64RegisterPat,
119	WHvX64RegisterSysenterCs,
120	WHvX64RegisterSysenterEip,
121	WHvX64RegisterSysenterEsp,
122	WHvX64RegisterStar,
123	#ifdef TARGET_X86_64
124	WHvX64RegisterLstar,
125	WHvX64RegisterCstar,
126	WHvX64RegisterSfmask,
127	#endif
128
129	/* Interrupt / Event Registers */
130	/*
131	* WHvRegisterPendingInterruption,
132	* WHvRegisterInterruptState,
133	* WHvRegisterPendingEvent0,
134	* WHvRegisterPendingEvent1
135	* WHvX64RegisterDeliverabilityNotifications,
136	*/
137	};
138
139	static const WHV_REGISTER_NAME whpx_register_names_for_vmexit[] = {
140	/* X64 General purpose registers */
141	WHvX64RegisterRax,
142	WHvX64RegisterRcx,
143	WHvX64RegisterRdx,
144	WHvX64RegisterRbx,
145	WHvX64RegisterRsp,
146	WHvX64RegisterRbp,
147	WHvX64RegisterRsi,
148	WHvX64RegisterRdi,
149	WHvX64RegisterR8,
150	WHvX64RegisterR9,
151	WHvX64RegisterR10,
152	WHvX64RegisterR11,
153	WHvX64RegisterR12,
154	WHvX64RegisterR13,
155	WHvX64RegisterR14,
156	WHvX64RegisterR15,
157	};
158
159	static const WHV_REGISTER_NAME whpx_register_names_legacy_fp[] = {
160	/* X64 Floating Point and Vector Registers (non-xsave) */
161	WHvX64RegisterXmm0,
162	WHvX64RegisterXmm1,
163	WHvX64RegisterXmm2,
164	WHvX64RegisterXmm3,
165	WHvX64RegisterXmm4,
166	WHvX64RegisterXmm5,
167	WHvX64RegisterXmm6,
168	WHvX64RegisterXmm7,
169	WHvX64RegisterXmm8,
170	WHvX64RegisterXmm9,
171	WHvX64RegisterXmm10,
172	WHvX64RegisterXmm11,
173	WHvX64RegisterXmm12,
174	WHvX64RegisterXmm13,
175	WHvX64RegisterXmm14,
176	WHvX64RegisterXmm15,
177	WHvX64RegisterFpMmx0,
178	WHvX64RegisterFpMmx1,
179	WHvX64RegisterFpMmx2,
180	WHvX64RegisterFpMmx3,
181	WHvX64RegisterFpMmx4,
182	WHvX64RegisterFpMmx5,
183	WHvX64RegisterFpMmx6,
184	WHvX64RegisterFpMmx7,
185	WHvX64RegisterFpControlStatus,
186	WHvX64RegisterXmmControlStatus,
187	};
188
189	struct whpx_register_set {
190	WHV_REGISTER_VALUE values[RTL_NUMBER_OF(whpx_register_names)];
191	};
192
193	/*
194	* The current implementation of instruction stepping sets the TF flag
195	* in RFLAGS, causing the CPU to raise an INT1 after each instruction.
196	* This corresponds to the WHvX64ExceptionTypeDebugTrapOrFault exception.
197	*
198	* This approach has a few limitations:
199	* 1. Stepping over a PUSHF/SAHF instruction will save the TF flag
200	* along with the other flags, possibly restoring it later. It would
201	* result in another INT1 when the flags are restored, triggering
202	* a stop in gdb that could be cleared by doing another step.
203	*
204	* Stepping over a POPF/LAHF instruction will let it overwrite the
205	* TF flags, ending the stepping mode.
206	*
207	* 2. Stepping over an instruction raising an exception (e.g. INT, DIV,
208	* or anything that could result in a page fault) will save the flags
209	* to the stack, clear the TF flag, and let the guest execute the
210	* handler. Normally, the guest will restore the original flags,
211	* that will continue single-stepping.
212	*
213	* 3. Debuggers running on the guest may wish to set TF to do instruction
214	* stepping. INT1 events generated by it would be intercepted by us,
215	* as long as the gdb is connected to QEMU.
216	*
217	* In practice this means that:
218	* 1. Stepping through flags-modifying instructions may cause gdb to
219	* continue or stop in unexpected places. This will be fully recoverable
220	* and will not crash the target.
221	*
222	* 2. Stepping over an instruction that triggers an exception will step
223	* over the exception handler, not into it.
224	*
225	* 3. Debugging the guest via gdb, while running debugger on the guest
226	* at the same time may lead to unexpected effects. Removing all
227	* breakpoints set via QEMU will prevent any further interference
228	* with the guest-level debuggers.
229	*
230	* The limitations can be addressed as shown below:
231	* 1. PUSHF/SAHF/POPF/LAHF/IRET instructions can be emulated instead of
232	* stepping through them. The exact semantics of the instructions is
233	* defined in the "Combined Volume Set of Intel 64 and IA-32
234	* Architectures Software Developer's Manuals", however it involves a
235	* fair amount of corner cases due to compatibility with real mode,
236	* virtual 8086 mode, and differences between 64-bit and 32-bit modes.
237	*
238	* 2. We could step into the guest's exception handlers using the following
239	* sequence:
240	* a. Temporarily enable catching of all exception types via
241	* whpx_set_exception_exit_bitmap().
242	* b. Once an exception is intercepted, read the IDT/GDT and locate
243	* the original handler.
244	* c. Patch the original handler, injecting an INT3 at the beginning.
245	* d. Update the exception exit bitmap to only catch the
246	* WHvX64ExceptionTypeBreakpointTrap exception.
247	* e. Let the affected CPU run in the exclusive mode.
248	* f. Restore the original handler and the exception exit bitmap.
249	* Note that handling all corner cases related to IDT/GDT is harder
250	* than it may seem. See x86_cpu_translate_for_debug() for a
251	* rough idea.
252	*
253	* 3. In order to properly support guest-level debugging in parallel with
254	* the QEMU-level debugging, we would need to be able to pass some INT1
255	* events to the guest. This could be done via the following methods:
256	* a. Using the WHvRegisterPendingEvent register. As of Windows 21H1,
257	* it seems to only work for interrupts and not software
258	* exceptions.
259	* b. Locating and patching the original handler by parsing IDT/GDT.
260	* This involves relatively complex logic outlined in the previous
261	* paragraph.
262	* c. Emulating the exception invocation (i.e. manually updating RIP,
263	* RFLAGS, and pushing the old values to stack). This is even more
264	* complicated than the previous option, since it involves checking
265	* CPL, gate attributes, and doing various adjustments depending
266	* on the current CPU mode, whether the CPL is changing, etc.
267	*/
268	typedef enum WhpxStepMode {
269	WHPX_STEP_NONE = 0,
270	/* Halt other VCPUs */
271	WHPX_STEP_EXCLUSIVE,
272	} WhpxStepMode;
273
274	static uint32_t max_vcpu_index;
275	static WHV_PROCESSOR_XSAVE_FEATURES whpx_xsave_cap;
276
277	bool whpx_has_xsave(void)
278	{
279	return whpx_xsave_cap.XsaveSupport;
280	}
281
282	bool whpx_has_xsaves(void)
283	{
284	return whpx_xsave_cap.XsaveSupervisorSupport;
285	}
286
287	static bool whpx_rdtsc_cap;
288
289	bool whpx_has_rdtscp(void)
290	{
291	return whpx_rdtsc_cap;
292	}
293
294	static bool whpx_invpcid_cap;
295
296	bool whpx_has_invpcid(void)
297	{
298	return whpx_invpcid_cap;
299	}
300
301	static WHV_X64_SEGMENT_REGISTER whpx_seg_q2h(const SegmentCache *qs, int v86,
302	int r86)
303	{
304	WHV_X64_SEGMENT_REGISTER hs;
305	unsigned flags = qs->flags;
306
307	hs.Base = qs->base;
308	hs.Limit = qs->limit;
309	hs.Selector = qs->selector;
310
311	if (v86) {
312	hs.Attributes = 0;
313	hs.SegmentType = 3;
314	hs.Present = 1;
315	hs.DescriptorPrivilegeLevel = 3;
316	hs.NonSystemSegment = 1;
317
318	} else {
319	hs.Attributes = (flags >> DESC_TYPE_SHIFT);
320
321	if (r86) {
322	/* hs.Base &= 0xfffff; */
323	}
324	}
325
326	return hs;
327	}
328
329	static SegmentCache whpx_seg_h2q(const WHV_X64_SEGMENT_REGISTER *hs)
330	{
331	SegmentCache qs;
332
333	qs.base = hs->Base;
334	qs.limit = hs->Limit;
335	qs.selector = hs->Selector;
336
337	qs.flags = ((uint32_t)hs->Attributes) << DESC_TYPE_SHIFT;
338
339	return qs;
340	}
341
342	/* X64 Extended Control Registers */
343	static void whpx_set_xcrs(CPUState *cpu)
344	{
345	HRESULT hr;
346	struct whpx_state *whpx = &whpx_global;
347	WHV_REGISTER_VALUE xcr0;
348	WHV_REGISTER_NAME xcr0_name = WHvX64RegisterXCr0;
349
350	if (!whpx_has_xsave()) {
351	return;
352	}
353
354	/* Only xcr0 is supported by the hypervisor currently */
355	xcr0.Reg64 = cpu_env(cpu)->xcr0;
356	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
357	whpx->partition, cpu->cpu_index, &xcr0_name, 1, &xcr0);
358	if (FAILED(hr)) {
359	error_report("WHPX: Failed to set register xcr0, hr=%08lx", hr);
360	}
361	}
362
363	static int whpx_set_tsc(CPUState *cpu)
364	{
365	WHV_REGISTER_NAME tsc_reg = WHvX64RegisterTsc;
366	WHV_REGISTER_VALUE tsc_val;
367	HRESULT hr;
368	struct whpx_state *whpx = &whpx_global;
369
370	/*
371	* Suspend the partition prior to setting the TSC to reduce the variance
372	* in TSC across vCPUs. When the first vCPU runs post suspend, the
373	* partition is automatically resumed.
374	*/
375	if (whp_dispatch.WHvSuspendPartitionTime) {
376
377	/*
378	* Unable to suspend partition while setting TSC is not a fatal
379	* error. It just increases the likelihood of TSC variance between
380	* vCPUs and some guest OS are able to handle that just fine.
381	*/
382	hr = whp_dispatch.WHvSuspendPartitionTime(whpx->partition);
383	if (FAILED(hr)) {
384	warn_report("WHPX: Failed to suspend partition, hr=%08lx", hr);
385	}
386	}
387
388	tsc_val.Reg64 = cpu_env(cpu)->tsc;
389	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
390	whpx->partition, cpu->cpu_index, &tsc_reg, 1, &tsc_val);
391	if (FAILED(hr)) {
392	error_report("WHPX: Failed to set TSC, hr=%08lx", hr);
393	return -1;
394	}
395
396	return 0;
397	}
398
399	static bool whpx_is_xsave_enabled(CPUState *cpu)
400	{
401	CPUX86State *env = &X86_CPU(cpu)->env;
402	return env->cr[4] & CR4_OSXSAVE_MASK;
403	}
404
405	static size_t whpx_get_xsave_max_len(void)
406	{
407	return whpx_get_supported_cpuid(0xd, 0, R_ECX);
408	}
409
410	static int whpx_set_xsave_state(const CPUState *cpu)
411	{
412	struct whpx_state *whpx = &whpx_global;
413	X86CPU *x86cpu = X86_CPU(cpu);
414	CPUX86State *env = &x86cpu->env;
415	HRESULT hr;
416	void *xsavec_buf;
417	size_t page = qemu_real_host_page_size();
418	size_t xsavec_buf_len;
419
420	/* allocate and populate compacted buffer */
421	xsavec_buf_len = whpx_get_xsave_max_len();
422	xsavec_buf = qemu_memalign(page, xsavec_buf_len);
423
424	/* save registers to standard format buffer */
425	x86_cpu_xsave_all_areas(x86cpu, env->xsave_buf, env->xsave_buf_len);
426
427	/* store compacted version of xsave area in xsavec_buf */
428	compact_xsave_area(env, xsavec_buf, xsavec_buf_len);
429
430	if (!whpx_is_legacy_os()) {
431	hr = whp_dispatch.WHvSetVirtualProcessorState(
432	whpx->partition, cpu->cpu_index,
433	WHvVirtualProcessorStateTypeXsaveState,
434	xsavec_buf,
435	xsavec_buf_len);
436	} else {
437	hr = whp_dispatch.WHvSetVirtualProcessorXsaveState(
438	whpx->partition, cpu->cpu_index,
439	xsavec_buf,
440	xsavec_buf_len);
441	}
442
443	qemu_vfree(xsavec_buf);
444	if (FAILED(hr)) {
445	error_report("WHPX: Failed to get virtual processor context, hr=%08lx",
446	hr);
447	}
448
449	return 0;
450	}
451
452	static void whpx_set_legacy_fp_registers(CPUState *cpu, WHPXStateLevel level)
453	{
454	struct whpx_state *whpx = &whpx_global;
455	X86CPU *x86_cpu = X86_CPU(cpu);
456	CPUX86State *env = &x86_cpu->env;
457	struct whpx_register_set vcxt;
458	HRESULT hr;
459	int idx = 0;
460	int i;
461	int idx_next;
462
463	assert(cpu_is_stopped(cpu) \|\| qemu_cpu_is_self(cpu));
464
465	/* 16 XMM registers */
466	assert(whpx_register_names_legacy_fp[idx] == WHvX64RegisterXmm0);
467	idx_next = idx + 16;
468	for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
469	vcxt.values[idx].Reg128.Low64 = env->xmm_regs[i].ZMM_Q(0);
470	vcxt.values[idx].Reg128.High64 = env->xmm_regs[i].ZMM_Q(1);
471	}
472	idx = idx_next;
473
474	/* 8 FP registers */
475	assert(whpx_register_names_legacy_fp[idx] == WHvX64RegisterFpMmx0);
476	for (i = 0; i < 8; i += 1, idx += 1) {
477	vcxt.values[idx].Fp.AsUINT128.Low64 = env->fpregs[i].mmx.MMX_Q(0);
478	/* vcxt.values[idx].Fp.AsUINT128.High64 =
479	env->fpregs[i].mmx.MMX_Q(1);
480	*/
481	}
482
483	/* FP control status register */
484	assert(whpx_register_names_legacy_fp[idx] == WHvX64RegisterFpControlStatus);
485	vcxt.values[idx].FpControlStatus.FpControl = env->fpuc;
486	vcxt.values[idx].FpControlStatus.FpStatus =
487	(env->fpus & ~0x3800) \| (env->fpstt & 0x7) << 11;
488	vcxt.values[idx].FpControlStatus.FpTag = 0;
489	for (i = 0; i < 8; ++i) {
490	vcxt.values[idx].FpControlStatus.FpTag \|= (!env->fptags[i]) << i;
491	}
492	vcxt.values[idx].FpControlStatus.Reserved = 0;
493	vcxt.values[idx].FpControlStatus.LastFpOp = env->fpop;
494	vcxt.values[idx].FpControlStatus.LastFpRip = env->fpip;
495	idx += 1;
496
497	/* XMM control status register */
498	assert(whpx_register_names_legacy_fp[idx] == WHvX64RegisterXmmControlStatus);
499	vcxt.values[idx].XmmControlStatus.LastFpRdp = 0;
500	vcxt.values[idx].XmmControlStatus.XmmStatusControl = env->mxcsr;
501	vcxt.values[idx].XmmControlStatus.XmmStatusControlMask = 0x0000ffff;
502	idx += 1;
503
504	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
505	whpx->partition, cpu->cpu_index,
506	whpx_register_names_legacy_fp,
507	idx,
508	&vcxt.values[0]);
509
510	if (FAILED(hr)) {
511	error_report("WHPX: Failed to set virtual processor context, hr=%08lx",
512	hr);
513	}
514	}
515
516	void whpx_set_registers(CPUState *cpu, WHPXStateLevel level)
517	{
518	struct whpx_state *whpx = &whpx_global;
519	AccelCPUState *vcpu = cpu->accel;
520	X86CPU *x86_cpu = X86_CPU(cpu);
521	CPUX86State *env = &x86_cpu->env;
522	struct whpx_register_set vcxt;
523	HRESULT hr;
524	int idx;
525	int idx_next;
526	int i;
527	int v86, r86;
528
529	assert(cpu_is_stopped(cpu) \|\| qemu_cpu_is_self(cpu));
530
531	/*
532	* Following MSRs have side effects on the guest or are too heavy for
533	* runtime. Limit them to full state update.
534	*/
535	if (level >= WHPX_LEVEL_RESET_STATE) {
536	whpx_set_tsc(cpu);
537	}
538
539	memset(&vcxt, 0, sizeof(struct whpx_register_set));
540
541	v86 = (env->eflags & VM_MASK);
542	r86 = !(env->cr[0] & CR0_PE_MASK);
543
544	vcpu->tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
545
546	idx = 0;
547
548	/* Indexes for first 16 registers match between HV and QEMU definitions */
549	idx_next = 16;
550	for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
551	vcxt.values[idx].Reg64 = (uint64_t)env->regs[idx];
552	}
553	idx = idx_next;
554
555	/* Same goes for RIP and RFLAGS */
556	assert(whpx_register_names[idx] == WHvX64RegisterRip);
557	vcxt.values[idx++].Reg64 = env->eip;
558
559	assert(whpx_register_names[idx] == WHvX64RegisterRflags);
560	lflags_to_rflags(env);
561	vcxt.values[idx++].Reg64 = env->eflags;
562	assert(idx == WHvX64RegisterEs);
563
564	if (level > WHPX_LEVEL_FAST_RUNTIME_STATE) {
565
566	/* Translate 6+4 segment registers. HV and QEMU order matches */
567	for (i = 0; i < 6; i += 1, idx += 1) {
568	vcxt.values[idx].Segment = whpx_seg_q2h(&env->segs[i], v86, r86);
569	}
570
571	assert(idx == WHvX64RegisterLdtr);
572	/*
573	* Skip those registers for synchronisation after MMIO accesses
574	* as they're not going to be modified in that case.
575	*/
576
577	vcxt.values[idx++].Segment = whpx_seg_q2h(&env->ldt, 0, 0);
578
579	assert(idx == WHvX64RegisterTr);
580	vcxt.values[idx++].Segment = whpx_seg_q2h(&env->tr, 0, 0);
581
582	assert(idx == WHvX64RegisterIdtr);
583	vcxt.values[idx].Table.Base = env->idt.base;
584	vcxt.values[idx].Table.Limit = env->idt.limit;
585	idx += 1;
586
587	assert(idx == WHvX64RegisterGdtr);
588	vcxt.values[idx].Table.Base = env->gdt.base;
589	vcxt.values[idx].Table.Limit = env->gdt.limit;
590	idx += 1;
591
592	/* CR0, 2, 3, 4, 8 */
593	assert(whpx_register_names[idx] == WHvX64RegisterCr0);
594	vcxt.values[idx++].Reg64 = env->cr[0];
595	assert(whpx_register_names[idx] == WHvX64RegisterCr2);
596	vcxt.values[idx++].Reg64 = env->cr[2];
597	assert(whpx_register_names[idx] == WHvX64RegisterCr3);
598	vcxt.values[idx++].Reg64 = env->cr[3];
599	assert(whpx_register_names[idx] == WHvX64RegisterCr4);
600	vcxt.values[idx++].Reg64 = env->cr[4];
601	/* For kernel-irqchip=on, TPR is managed as part of APIC state */
602	if (!whpx_irqchip_in_kernel()) {
603	WHV_REGISTER_VALUE cr8 = {.Reg64 = vcpu->tpr};
604	whpx_set_reg(cpu, WHvX64RegisterCr8, cr8);
605	}
606
607	/* 8 Debug Registers - Skipped */
608
609	/*
610	* Extended control registers needs to be handled separately depending
611	* on whether xsave is supported/enabled or not.
612	*/
613	whpx_set_xcrs(cpu);
614
615	if (whpx_is_xsave_enabled(cpu)) {
616	whpx_set_xsave_state(cpu);
617	}
618	whpx_set_legacy_fp_registers(cpu, level);
619
620	/* MSRs */
621	assert(whpx_register_names[idx] == WHvX64RegisterEfer);
622	vcxt.values[idx++].Reg64 = env->efer;
623	#ifdef TARGET_X86_64
624	assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
625	vcxt.values[idx++].Reg64 = env->kernelgsbase;
626	#endif
627	assert(whpx_register_names[idx] == WHvX64RegisterPat);
628	vcxt.values[idx++].Reg64 = env->pat;
629	assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
630	vcxt.values[idx++].Reg64 = env->sysenter_cs;
631	assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
632	vcxt.values[idx++].Reg64 = env->sysenter_eip;
633	assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
634	vcxt.values[idx++].Reg64 = env->sysenter_esp;
635	assert(whpx_register_names[idx] == WHvX64RegisterStar);
636	vcxt.values[idx++].Reg64 = env->star;
637	#ifdef TARGET_X86_64
638	assert(whpx_register_names[idx] == WHvX64RegisterLstar);
639	vcxt.values[idx++].Reg64 = env->lstar;
640	assert(whpx_register_names[idx] == WHvX64RegisterCstar);
641	vcxt.values[idx++].Reg64 = env->cstar;
642	assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
643	vcxt.values[idx++].Reg64 = env->fmask;
644	#endif
645
646	/* Interrupt / Event Registers - Skipped */
647
648	assert(idx == RTL_NUMBER_OF(whpx_register_names));
649	}
650
651	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
652	whpx->partition, cpu->cpu_index,
653	whpx_register_names,
654	idx,
655	&vcxt.values[0]);
656
657	if (FAILED(hr)) {
658	error_report("WHPX: Failed to set virtual processor context, hr=%08lx",
659	hr);
660	}
661
662	if (level >= WHPX_LEVEL_FULL_STATE) {
663	WHV_REGISTER_VALUE apic_base = {};
664	apic_base.Reg64 = cpu_get_apic_base(X86_CPU(cpu)->apic_state);
665	whpx_set_reg(cpu, WHvX64RegisterApicBase, apic_base);
666	}
667	}
668
669	static int whpx_get_tsc(CPUState *cpu)
670	{
671	WHV_REGISTER_NAME tsc_reg = WHvX64RegisterTsc;
672	WHV_REGISTER_VALUE tsc_val;
673	HRESULT hr;
674	struct whpx_state *whpx = &whpx_global;
675
676	hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
677	whpx->partition, cpu->cpu_index, &tsc_reg, 1, &tsc_val);
678	if (FAILED(hr)) {
679	error_report("WHPX: Failed to get TSC, hr=%08lx", hr);
680	return -1;
681	}
682
683	cpu_env(cpu)->tsc = tsc_val.Reg64;
684	return 0;
685	}
686
687	/* X64 Extended Control Registers */
688	static void whpx_get_xcrs(CPUState *cpu)
689	{
690	HRESULT hr;
691	struct whpx_state *whpx = &whpx_global;
692	WHV_REGISTER_VALUE xcr0;
693	WHV_REGISTER_NAME xcr0_name = WHvX64RegisterXCr0;
694
695	if (!whpx_has_xsave()) {
696	return;
697	}
698
699	/* Only xcr0 is supported by the hypervisor currently */
700	hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
701	whpx->partition, cpu->cpu_index, &xcr0_name, 1, &xcr0);
702	if (FAILED(hr)) {
703	error_report("WHPX: Failed to get register xcr0, hr=%08lx", hr);
704	return;
705	}
706
707	cpu_env(cpu)->xcr0 = xcr0.Reg64;
708	}
709
710	static void whpx_get_registers_for_vmexit(CPUState *cpu, WHPXStateLevel level)
711	{
712	struct whpx_state *whpx = &whpx_global;
713	AccelCPUState *vcpu = cpu->accel;
714	X86CPU *x86_cpu = X86_CPU(cpu);
715	CPUX86State *env = &x86_cpu->env;
716	struct whpx_register_set vcxt;
717	HRESULT hr;
718	int idx;
719	int idx_next;
720
721	assert(cpu_is_stopped(cpu) \|\| qemu_cpu_is_self(cpu));
722
723	hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
724	whpx->partition, cpu->cpu_index,
725	whpx_register_names_for_vmexit,
726	RTL_NUMBER_OF(whpx_register_names_for_vmexit),
727	&vcxt.values[0]);
728	if (FAILED(hr)) {
729	error_report("WHPX: Failed to get virtual processor context, hr=%08lx",
730	hr);
731	}
732
733	idx = 0;
734
735	/* Indexes for first 16 registers match between HV and QEMU definitions */
736	idx_next = 16;
737	for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
738	env->regs[idx] = vcxt.values[idx].Reg64;
739	}
740	idx = idx_next;
741
742	env->eip = vcpu->exit_ctx.VpContext.Rip;
743	env->eflags = vcpu->exit_ctx.VpContext.Rflags;
744	rflags_to_lflags(env);
745
746	assert(idx == RTL_NUMBER_OF(whpx_register_names_for_vmexit));
747
748	x86_update_hflags(env);
749	}
750
751	static void whpx_get_legacy_fp_registers(CPUState *cpu, WHPXStateLevel level)
752	{
753	struct whpx_state *whpx = &whpx_global;
754	X86CPU *x86_cpu = X86_CPU(cpu);
755	CPUX86State *env = &x86_cpu->env;
756	struct whpx_register_set vcxt;
757	HRESULT hr;
758	int i;
759	int idx;
760	int idx_next;
761
762	assert(cpu_is_stopped(cpu) \|\| qemu_cpu_is_self(cpu));
763
764	hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
765	whpx->partition, cpu->cpu_index,
766	whpx_register_names_legacy_fp,
767	RTL_NUMBER_OF(whpx_register_names_legacy_fp),
768	&vcxt.values[0]);
769
770	if (FAILED(hr)) {
771	error_report("WHPX: Failed to get virtual processor context, hr=%08lx",
772	hr);
773	}
774
775	idx = 0;
776	/* 16 XMM registers */
777	assert(whpx_register_names_legacy_fp[idx] == WHvX64RegisterXmm0);
778	idx_next = idx + 16;
779	for (i = 0; i < sizeof(env->xmm_regs) / sizeof(ZMMReg); i += 1, idx += 1) {
780	env->xmm_regs[i].ZMM_Q(0) = vcxt.values[idx].Reg128.Low64;
781	env->xmm_regs[i].ZMM_Q(1) = vcxt.values[idx].Reg128.High64;
782	}
783	idx = idx_next;
784
785	/* 8 FP registers */
786	assert(whpx_register_names_legacy_fp[idx] == WHvX64RegisterFpMmx0);
787	for (i = 0; i < 8; i += 1, idx += 1) {
788	env->fpregs[i].mmx.MMX_Q(0) = vcxt.values[idx].Fp.AsUINT128.Low64;
789	/* env->fpregs[i].mmx.MMX_Q(1) =
790	vcxt.values[idx].Fp.AsUINT128.High64;
791	*/
792	}
793
794	/* FP control status register */
795	assert(whpx_register_names_legacy_fp[idx] == WHvX64RegisterFpControlStatus);
796	env->fpuc = vcxt.values[idx].FpControlStatus.FpControl;
797	env->fpstt = (vcxt.values[idx].FpControlStatus.FpStatus >> 11) & 0x7;
798	env->fpus = vcxt.values[idx].FpControlStatus.FpStatus & ~0x3800;
799	for (i = 0; i < 8; ++i) {
800	env->fptags[i] = !((vcxt.values[idx].FpControlStatus.FpTag >> i) & 1);
801	}
802	env->fpop = vcxt.values[idx].FpControlStatus.LastFpOp;
803	env->fpip = vcxt.values[idx].FpControlStatus.LastFpRip;
804	idx += 1;
805
806	/* XMM control status register */
807	assert(whpx_register_names_legacy_fp[idx] == WHvX64RegisterXmmControlStatus);
808	env->mxcsr = vcxt.values[idx].XmmControlStatus.XmmStatusControl;
809	idx += 1;
810	}
811
812	static int whpx_get_xsave_state(CPUState *cpu)
813	{
814	struct whpx_state *whpx = &whpx_global;
815	X86CPU *x86cpu = X86_CPU(cpu);
816	CPUX86State *env = &x86cpu->env;
817	int ret;
818	HRESULT hr;
819	void *xsavec_buf;
820	const size_t page = qemu_real_host_page_size();
821	size_t xsavec_buf_len = whpx_get_xsave_max_len();
822	UINT32 bytes_written;
823
824	xsavec_buf = qemu_memalign(page, xsavec_buf_len);
825	memset(xsavec_buf, 0, xsavec_buf_len);
826
827	if (!whpx_is_legacy_os()) {
828	hr = whp_dispatch.WHvGetVirtualProcessorState(
829	whpx->partition, cpu->cpu_index,
830	WHvVirtualProcessorStateTypeXsaveState,
831	xsavec_buf,
832	xsavec_buf_len, &bytes_written);
833	} else {
834	hr = whp_dispatch.WHvGetVirtualProcessorXsaveState(
835	whpx->partition, cpu->cpu_index,
836	xsavec_buf,
837	xsavec_buf_len, &bytes_written);
838	}
839	if (FAILED(hr) \|\| bytes_written == 0) {
840	error_report("failed to get xsave state: %s", strerror(errno));
841	return -errno;
842	}
843
844	ret = decompact_xsave_area(xsavec_buf, xsavec_buf_len, env);
845	qemu_vfree(xsavec_buf);
846	if (ret < 0) {
847	error_report("failed to decompact xsave area");
848	return ret;
849	}
850	x86_cpu_xrstor_all_areas(x86cpu, env->xsave_buf, env->xsave_buf_len);
851
852	return 0;
853	}
854
855	void whpx_get_registers(CPUState *cpu, WHPXStateLevel level)
856	{
857	struct whpx_state *whpx = &whpx_global;
858	AccelCPUState *vcpu = cpu->accel;
859	X86CPU *x86_cpu = X86_CPU(cpu);
860	CPUX86State *env = &x86_cpu->env;
861	struct whpx_register_set vcxt;
862	uint64_t tpr;
863	HRESULT hr;
864	int idx;
865	int idx_next;
866	int i;
867
868	assert(cpu_is_stopped(cpu) \|\| qemu_cpu_is_self(cpu));
869
870	if (level == WHPX_LEVEL_FAST_RUNTIME_STATE) {
871	return whpx_get_registers_for_vmexit(cpu, level);
872	}
873
874	if (!env->tsc_valid) {
875	whpx_get_tsc(cpu);
876	env->tsc_valid = !runstate_is_running();
877	}
878
879	hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
880	whpx->partition, cpu->cpu_index,
881	whpx_register_names,
882	RTL_NUMBER_OF(whpx_register_names),
883	&vcxt.values[0]);
884	if (FAILED(hr)) {
885	error_report("WHPX: Failed to get virtual processor context, hr=%08lx",
886	hr);
887	}
888
889	idx = 0;
890
891	/* Indexes for first 16 registers match between HV and QEMU definitions */
892	idx_next = 16;
893	for (idx = 0; idx < CPU_NB_REGS; idx += 1) {
894	env->regs[idx] = vcxt.values[idx].Reg64;
895	}
896	idx = idx_next;
897
898	/* Same goes for RIP and RFLAGS */
899	assert(whpx_register_names[idx] == WHvX64RegisterRip);
900	env->eip = vcxt.values[idx++].Reg64;
901	assert(whpx_register_names[idx] == WHvX64RegisterRflags);
902	env->eflags = vcxt.values[idx++].Reg64;
903	rflags_to_lflags(env);
904
905	/* Translate 6+4 segment registers. HV and QEMU order matches */
906	assert(idx == WHvX64RegisterEs);
907	for (i = 0; i < 6; i += 1, idx += 1) {
908	env->segs[i] = whpx_seg_h2q(&vcxt.values[idx].Segment);
909	}
910
911	assert(idx == WHvX64RegisterLdtr);
912	env->ldt = whpx_seg_h2q(&vcxt.values[idx++].Segment);
913	assert(idx == WHvX64RegisterTr);
914	env->tr = whpx_seg_h2q(&vcxt.values[idx++].Segment);
915	assert(idx == WHvX64RegisterIdtr);
916	env->idt.base = vcxt.values[idx].Table.Base;
917	env->idt.limit = vcxt.values[idx].Table.Limit;
918	idx += 1;
919	assert(idx == WHvX64RegisterGdtr);
920	env->gdt.base = vcxt.values[idx].Table.Base;
921	env->gdt.limit = vcxt.values[idx].Table.Limit;
922	idx += 1;
923
924	/* CR0, 2, 3, 4, 8 */
925	assert(whpx_register_names[idx] == WHvX64RegisterCr0);
926	env->cr[0] = vcxt.values[idx++].Reg64;
927	assert(whpx_register_names[idx] == WHvX64RegisterCr2);
928	env->cr[2] = vcxt.values[idx++].Reg64;
929	assert(whpx_register_names[idx] == WHvX64RegisterCr3);
930	env->cr[3] = vcxt.values[idx++].Reg64;
931	assert(whpx_register_names[idx] == WHvX64RegisterCr4);
932	env->cr[4] = vcxt.values[idx++].Reg64;
933
934	/* For kernel-irqchip=on, TPR is managed as part of APIC state */
935	if (!whpx_irqchip_in_kernel()) {
936	tpr = vcpu->exit_ctx.VpContext.Cr8;
937	if (tpr != vcpu->tpr) {
938	vcpu->tpr = tpr;
939	cpu_set_apic_tpr(x86_cpu->apic_state, tpr);
940	}
941	}
942
943	/* 8 Debug Registers - Skipped */
944
945	/*
946	* Extended control registers needs to be handled separately depending
947	* on whether xsave is supported/enabled or not.
948	*/
949	whpx_get_xcrs(cpu);
950
951	if (whpx_is_xsave_enabled(cpu)) {
952	whpx_get_xsave_state(cpu);
953	}
954	whpx_get_legacy_fp_registers(cpu, level);
955
956	/* MSRs */
957	assert(whpx_register_names[idx] == WHvX64RegisterEfer);
958	env->efer = vcxt.values[idx++].Reg64;
959	#ifdef TARGET_X86_64
960	assert(whpx_register_names[idx] == WHvX64RegisterKernelGsBase);
961	env->kernelgsbase = vcxt.values[idx++].Reg64;
962	#endif
963	assert(whpx_register_names[idx] == WHvX64RegisterPat);
964	env->pat = vcxt.values[idx++].Reg64;
965	assert(whpx_register_names[idx] == WHvX64RegisterSysenterCs);
966	env->sysenter_cs = vcxt.values[idx++].Reg64;
967	assert(whpx_register_names[idx] == WHvX64RegisterSysenterEip);
968	env->sysenter_eip = vcxt.values[idx++].Reg64;
969	assert(whpx_register_names[idx] == WHvX64RegisterSysenterEsp);
970	env->sysenter_esp = vcxt.values[idx++].Reg64;
971	assert(whpx_register_names[idx] == WHvX64RegisterStar);
972	env->star = vcxt.values[idx++].Reg64;
973	#ifdef TARGET_X86_64
974	assert(whpx_register_names[idx] == WHvX64RegisterLstar);
975	env->lstar = vcxt.values[idx++].Reg64;
976	assert(whpx_register_names[idx] == WHvX64RegisterCstar);
977	env->cstar = vcxt.values[idx++].Reg64;
978	assert(whpx_register_names[idx] == WHvX64RegisterSfmask);
979	env->fmask = vcxt.values[idx++].Reg64;
980	#endif
981
982	/* Interrupt / Event Registers - Skipped */
983
984	assert(idx == RTL_NUMBER_OF(whpx_register_names));
985
986	if (whpx_irqchip_in_kernel()) {
987	whpx_apic_get(x86_cpu->apic_state);
988	}
989
990	x86_update_hflags(env);
991	}
992
993	static int emulate_instruction(CPUState cpu, const uint8_t insn_bytes, size_t insn_len)
994	{
995	X86CPU *x86_cpu = X86_CPU(cpu);
996	CPUX86State *env = &x86_cpu->env;
997	struct x86_decode decode = { 0 };
998	x86_insn_stream stream = { .bytes = insn_bytes, .len = insn_len };
999
1000	whpx_get_registers(cpu, WHPX_LEVEL_FAST_RUNTIME_STATE);
1001	decode_instruction_stream(env, &decode, &stream);
1002	exec_instruction(env, &decode);
1003	whpx_set_registers(cpu, WHPX_LEVEL_FAST_RUNTIME_STATE);
1004
1005	return 0;
1006	}
1007
1008	static int emulate_msr_instruction(CPUState *cpu,
1009	const uint8_t *insn_bytes, size_t insn_len)
1010	{
1011	X86CPU *x86_cpu = X86_CPU(cpu);
1012	CPUX86State *env = &x86_cpu->env;
1013	struct x86_decode decode = { 0 };
1014	x86_insn_stream stream = { .bytes = insn_bytes, .len = insn_len };
1015
1016	whpx_get_registers(cpu, WHPX_LEVEL_FAST_RUNTIME_STATE);
1017	decode_instruction_stream(env, &decode, &stream);
1018
1019	if (decode.cmd != X86_DECODE_CMD_RDMSR
1020	&& decode.cmd != X86_DECODE_CMD_WRMSR) {
1021	return 1;
1022	}
1023
1024	exec_instruction(env, &decode);
1025	whpx_set_registers(cpu, WHPX_LEVEL_FAST_RUNTIME_STATE);
1026	return 0;
1027	}
1028
1029	static int whpx_handle_mmio(CPUState cpu, WHV_RUN_VP_EXIT_CONTEXT exit_ctx)
1030	{
1031	WHV_MEMORY_ACCESS_CONTEXT *ctx = &exit_ctx->MemoryAccess;
1032	int ret;
1033
1034	ret = emulate_instruction(cpu, ctx->InstructionBytes, ctx->InstructionByteCount);
1035	if (ret < 0) {
1036	error_report("failed to emulate mmio");
1037	return -1;
1038	}
1039
1040	return 0;
1041	}
1042
1043	static int whpx_handle_msr_from_gpf(CPUState *cpu)
1044	{
1045	WHV_VP_EXCEPTION_CONTEXT *ctx = &cpu->accel->exit_ctx.VpException;
1046	int ret;
1047
1048	ret = emulate_msr_instruction(cpu, ctx->InstructionBytes, ctx->InstructionByteCount);
1049	if (ret == 1) {
1050	/* Not an MSR instruction */
1051	return 1;
1052	}
1053
1054	return 0;
1055	}
1056
1057	static void whpx_inject_back_gpf(CPUState *cpu)
1058	{
1059	WHV_VP_EXCEPTION_CONTEXT *ctx = &cpu->accel->exit_ctx.VpException;
1060	WHV_REGISTER_VALUE reg = {};
1061
1062	if (ctx->ExceptionInfo.SoftwareException) {
1063	/* TODO */
1064	warn_report("Was asked to inject software exception.");
1065	return;
1066	}
1067
1068	if (ctx->ExceptionType != EXCP0D_GPF) {
1069	warn_report("Was asked to inject exception other than GPF.");
1070	return;
1071	}
1072
1073	reg.ExceptionEvent.EventPending = 1;
1074	reg.ExceptionEvent.EventType = WHvX64PendingEventException;
1075	reg.ExceptionEvent.DeliverErrorCode = ctx->ExceptionInfo.ErrorCodeValid;
1076	reg.ExceptionEvent.Vector = ctx->ExceptionType;
1077	reg.ExceptionEvent.ErrorCode = ctx->ErrorCode;
1078	reg.ExceptionEvent.ExceptionParameter = ctx->ExceptionParameter;
1079	whpx_set_reg(cpu, WHvRegisterPendingEvent, reg);
1080	}
1081
1082	static void whpx_inject_back_db(CPUState *cpu)
1083	{
1084	WHV_VP_EXCEPTION_CONTEXT *ctx = &cpu->accel->exit_ctx.VpException;
1085	WHV_REGISTER_VALUE reg = {};
1086
1087	if (ctx->ExceptionInfo.SoftwareException) {
1088	/* TODO */
1089	warn_report("Was asked to inject software exception.");
1090	return;
1091	}
1092
1093	if (ctx->ExceptionType != EXCP01_DB) {
1094	warn_report("Was asked to inject exception other than debug.");
1095	return;
1096	}
1097
1098	reg.ExceptionEvent.EventPending = 1;
1099	reg.ExceptionEvent.EventType = WHvX64PendingEventException;
1100	reg.ExceptionEvent.DeliverErrorCode = ctx->ExceptionInfo.ErrorCodeValid;
1101	reg.ExceptionEvent.Vector = ctx->ExceptionType;
1102	reg.ExceptionEvent.ErrorCode = ctx->ErrorCode;
1103	reg.ExceptionEvent.ExceptionParameter = ctx->ExceptionParameter;
1104	whpx_set_reg(cpu, WHvRegisterPendingEvent, reg);
1105	}
1106
1107	static void handle_io(CPUState env, uint16_t port, void buffer,
1108	int direction, int size, int count)
1109	{
1110	int i;
1111	uint8_t *ptr = buffer;
1112
1113	for (i = 0; i < count; i++) {
1114	address_space_rw(&address_space_io, port, MEMTXATTRS_UNSPECIFIED,
1115	ptr, size,
1116	direction);
1117	ptr += size;
1118	}
1119	}
1120
1121	static void whpx_bump_rip(CPUState cpu, WHV_RUN_VP_EXIT_CONTEXT exit_ctx)
1122	{
1123	WHV_REGISTER_VALUE reg;
1124	reg.Reg64 = exit_ctx->VpContext.Rip + exit_ctx->VpContext.InstructionLength;
1125	whpx_set_reg(cpu, WHvX64RegisterRip, reg);
1126	}
1127
1128	static int whpx_handle_portio(CPUState *cpu,
1129	WHV_RUN_VP_EXIT_CONTEXT *exit_ctx)
1130	{
1131	WHV_X64_IO_PORT_ACCESS_CONTEXT *ctx = &exit_ctx->IoPortAccess;
1132	X86CPU *x86_cpu = X86_CPU(cpu);
1133	CPUX86State *env = &x86_cpu->env;
1134	int ret;
1135
1136	if (!ctx->AccessInfo.StringOp && !ctx->AccessInfo.IsWrite) {
1137	uint64_t val = 0;
1138	WHV_REGISTER_VALUE reg;
1139
1140	whpx_get_reg(cpu, WHvX64RegisterRax, &reg);
1141	handle_io(cpu, ctx->PortNumber, &val, 0, ctx->AccessInfo.AccessSize, 1);
1142	if (ctx->AccessInfo.AccessSize == 1) {
1143	reg.Reg8 = val;
1144	} else if (ctx->AccessInfo.AccessSize == 2) {
1145	reg.Reg16 = val;
1146	} else if (ctx->AccessInfo.AccessSize == 4) {
1147	reg.Reg64 = (uint32_t)val;
1148	} else {
1149	reg.Reg64 = (uint64_t)val;
1150	}
1151	/* vmport calls cpu_synchronize_state on an I/O port read */
1152	if (!cpu->vcpu_dirty) {
1153	whpx_bump_rip(cpu, exit_ctx);
1154	whpx_set_reg(cpu, WHvX64RegisterRax, reg);
1155	} else {
1156	env->eip = exit_ctx->VpContext.Rip + exit_ctx->VpContext.InstructionLength;
1157	env->regs[R_EAX] = reg.Reg64;
1158	}
1159	return 0;
1160	} else if (!ctx->AccessInfo.StringOp && ctx->AccessInfo.IsWrite) {
1161	RAX(env) = ctx->Rax;
1162	handle_io(cpu, ctx->PortNumber, &RAX(env), 1, ctx->AccessInfo.AccessSize, 1);
1163	if (!cpu->vcpu_dirty) {
1164	whpx_bump_rip(cpu, exit_ctx);
1165	} else {
1166	env->eip = exit_ctx->VpContext.Rip + exit_ctx->VpContext.InstructionLength;
1167	}
1168	return 0;
1169	}
1170
1171	ret = emulate_instruction(cpu, ctx->InstructionBytes, exit_ctx->VpContext.InstructionLength);
1172	if (ret < 0) {
1173	error_report("failed to emulate I/O port access");
1174	return -1;
1175	}
1176
1177	return 0;
1178	}
1179
1180	static void whpx_segment_to_x86_descriptor(CPUState cpu, WHV_X64_SEGMENT_REGISTER reg,
1181	struct x86_segment_descriptor *desc)
1182	{
1183	uint32_t limit;
1184	desc->g = reg->Granularity;
1185
1186	/*
1187	* Hyper-V can return reg->Granularity == 0
1188	* with a higher limit than 0xfffff.
1189	*
1190	* Detect that case and set desc->g
1191	* with shifting the limit properly.
1192	*/
1193	if (!desc->g && reg->Limit <= 0xfffff) {
1194	limit = reg->Limit;
1195	} else {
1196	limit = (reg->Limit >> 12);
1197	desc->g = 1;
1198	}
1199
1200	x86_set_segment_limit(desc, limit);
1201	x86_set_segment_base(desc, reg->Base);
1202
1203	desc->type = reg->SegmentType;
1204	desc->s = reg->NonSystemSegment;
1205	desc->dpl = reg->DescriptorPrivilegeLevel;
1206	desc->p = reg->Present;
1207	desc->avl = reg->Available;
1208	desc->l = reg->Long;
1209	desc->db = reg->Default;
1210	}
1211
1212	static void whpx_read_segment_descriptor(CPUState cpu, WHV_X64_SEGMENT_REGISTER reg,
1213	X86Seg seg)
1214	{
1215	AccelCPUState *vcpu = cpu->accel;
1216	WHV_REGISTER_NAME reg_name = WHvX64RegisterEs + seg;
1217	WHV_REGISTER_VALUE val;
1218
1219	if (seg == R_CS) {
1220	*reg = vcpu->exit_ctx.VpContext.Cs;
1221	return;
1222	}
1223	if (vcpu->exit_ctx.ExitReason == WHvRunVpExitReasonX64IoPortAccess) {
1224	if (seg == R_DS) {
1225	*reg = vcpu->exit_ctx.IoPortAccess.Ds;
1226	return;
1227	} else if (seg == R_ES) {
1228	*reg = vcpu->exit_ctx.IoPortAccess.Es;
1229	return;
1230	}
1231	}
1232
1233	whpx_get_reg(cpu, reg_name, &val);
1234	*reg = val.Segment;
1235	}
1236
1237	static void read_segment_descriptor(CPUState *cpu,
1238	struct x86_segment_descriptor *desc,
1239	enum X86Seg seg_idx)
1240	{
1241	WHV_X64_SEGMENT_REGISTER reg;
1242	whpx_read_segment_descriptor(cpu, &reg, seg_idx);
1243	whpx_segment_to_x86_descriptor(cpu, &reg, desc);
1244	}
1245
1246	static bool is_protected_mode(CPUState *cpu)
1247	{
1248	AccelCPUState *vcpu = cpu->accel;
1249
1250	return vcpu->exit_ctx.VpContext.ExecutionState.Cr0Pe == 1;
1251	}
1252
1253	static bool is_long_mode(CPUState *cpu)
1254	{
1255	AccelCPUState *vcpu = cpu->accel;
1256
1257	return vcpu->exit_ctx.VpContext.ExecutionState.EferLma == 1;
1258	}
1259
1260	static bool is_user_mode(CPUState *cpu)
1261	{
1262	AccelCPUState *vcpu = cpu->accel;
1263	return vcpu->exit_ctx.VpContext.ExecutionState.Cpl == 3;
1264	}
1265
1266	static target_ulong read_cr(CPUState *cpu, int cr)
1267	{
1268	WHV_REGISTER_NAME whv_cr;
1269	WHV_REGISTER_VALUE val;
1270
1271	switch (cr) {
1272	case 0:
1273	whv_cr = WHvX64RegisterCr0;
1274	break;
1275	case 2:
1276	whv_cr = WHvX64RegisterCr2;
1277	break;
1278	case 3:
1279	whv_cr = WHvX64RegisterCr3;
1280	break;
1281	case 4:
1282	whv_cr = WHvX64RegisterCr4;
1283	break;
1284	case 8:
1285	whv_cr = WHvX64RegisterCr8;
1286	break;
1287	default:
1288	abort();
1289	}
1290	whpx_get_reg(cpu, whv_cr, &val);
1291
1292	return val.Reg64;
1293	}
1294
1295	static bool whpx_simulate_rdmsr(CPUState *cs)
1296	{
1297	X86CPU *cpu = X86_CPU(cs);
1298	CPUX86State *env = &cpu->env;
1299	uint32_t msr = ECX(env);
1300	uint64_t val = 0;
1301
1302	switch (msr) {
1303	default:
1304	error_report("WHPX: unknown msr 0x%x", msr);
1305	x86_emul_raise_exception(&X86_CPU(cpu)->env, EXCP0D_GPF, 0);
1306	return 1;
1307	break;
1308	}
1309
1310	RAX(env) = (uint32_t)val;
1311	RDX(env) = (uint32_t)(val >> 32);
1312
1313	return 0;
1314	}
1315
1316	static bool whpx_simulate_wrmsr(CPUState *cs)
1317	{
1318	X86CPU *cpu = X86_CPU(cs);
1319	CPUX86State *env = &cpu->env;
1320	uint32_t msr = ECX(env);
1321	uint64_t data = ((uint64_t)EDX(env) << 32) \| EAX(env);
1322
1323	switch (msr) {
1324	default:
1325	error_report("WHPX: unknown msr 0x%x val %llx", msr, data);
1326	x86_emul_raise_exception(&X86_CPU(cpu)->env, EXCP0D_GPF, 0);
1327	return 1;
1328	break;
1329	}
1330
1331	return 0;
1332	}
1333
1334	static const struct x86_emul_ops whpx_x86_emul_ops = {
1335	.read_segment_descriptor = read_segment_descriptor,
1336	.handle_io = handle_io,
1337	.is_protected_mode = is_protected_mode,
1338	.is_long_mode = is_long_mode,
1339	.is_user_mode = is_user_mode,
1340	.read_cr = read_cr,
1341	.simulate_rdmsr = whpx_simulate_rdmsr,
1342	.simulate_wrmsr = whpx_simulate_wrmsr
1343	};
1344
1345	static void whpx_init_emu(void)
1346	{
1347	init_decoder();
1348	init_emu(&whpx_x86_emul_ops);
1349	}
1350
1351	bool whpx_is_legacy_os(void)
1352	{
1353	return !is_modern_os;
1354	}
1355
1356	uint32_t whpx_get_supported_cpuid(uint32_t func, uint32_t idx, int reg)
1357	{
1358	WHV_CPUID_OUTPUT output = {};
1359	uint32_t eax, ebx, ecx, edx;
1360	uint32_t cpu_index = 0;
1361	bool temp_cpu = true;
1362	HRESULT hr;
1363
1364	/* Legacy OSes don't have WHvGetVirtualProcessorCpuidOutput */
1365	if (whpx_is_legacy_os()) {
1366	return whpx_get_supported_cpuid_legacy(func, idx, reg);
1367	}
1368
1369	hr = whp_dispatch.WHvCreateVirtualProcessor(
1370	whpx_global.partition, cpu_index, 0);
1371
1372	/* This means that the CPU already exists... */
1373	if (FAILED(hr)) {
1374	temp_cpu = false;
1375	}
1376
1377	hr = whp_dispatch.WHvGetVirtualProcessorCpuidOutput(whpx_global.partition,
1378	cpu_index, func, idx, &output);
1379
1380	if (FAILED(hr)) {
1381	abort();
1382	}
1383
1384	if (temp_cpu) {
1385	hr = whp_dispatch.WHvDeleteVirtualProcessor(whpx_global.partition, cpu_index);
1386	if (FAILED(hr)) {
1387	abort();
1388	}
1389	}
1390
1391	eax = output.Eax;
1392	ebx = output.Ebx;
1393	ecx = output.Ecx;
1394	edx = output.Edx;
1395
1396	/*
1397	* We can emulate X2APIC even for the kernel-irqchip=off case.
1398	* CPUID_EXT_HYPERVISOR and CPUID_HT should be considered present
1399	* always, so report them as unconditionally supported here.
1400	*/
1401	if (func == 1) {
1402	ecx \|= CPUID_EXT_X2APIC;
1403	ecx \|= CPUID_EXT_HYPERVISOR;
1404	edx \|= CPUID_HT;
1405	}
1406
1407	switch (reg) {
1408	case R_EAX:
1409	return eax;
1410	case R_EBX:
1411	return ebx;
1412	case R_ECX:
1413	return ecx;
1414	case R_EDX:
1415	return edx;
1416	default:
1417	return 0;
1418	}
1419	}
1420
1421	uint64_t whpx_get_supported_msr_feature(uint32_t index)
1422	{
1423	WHV_CAPABILITY_CODE cap;
1424	uint64_t val = 0;
1425
1426	switch (index) {
1427	case MSR_IA32_VMX_BASIC:
1428	cap = WHvCapabilityCodeVmxBasic;
1429	break;
1430	case MSR_IA32_VMX_MISC:
1431	cap = WHvCapabilityCodeVmxMisc;
1432	break;
1433	case MSR_IA32_VMX_CR0_FIXED0:
1434	cap = WHvCapabilityCodeVmxCr0Fixed0;
1435	break;
1436	case MSR_IA32_VMX_CR0_FIXED1:
1437	cap = WHvCapabilityCodeVmxCr0Fixed1;
1438	break;
1439	case MSR_IA32_VMX_CR4_FIXED0:
1440	cap = WHvCapabilityCodeVmxCr4Fixed0;
1441	break;
1442	case MSR_IA32_VMX_CR4_FIXED1:
1443	cap = WHvCapabilityCodeVmxCr4Fixed1;
1444	break;
1445	case MSR_IA32_VMX_VMCS_ENUM:
1446	cap = WHvCapabilityCodeVmxVmcsEnum;
1447	break;
1448	case MSR_IA32_VMX_PROCBASED_CTLS2:
1449	cap = WHvCapabilityCodeVmxProcbasedCtls2;
1450	break;
1451	case MSR_IA32_VMX_EPT_VPID_CAP:
1452	cap = WHvCapabilityCodeVmxEptVpidCap;
1453	break;
1454	case MSR_IA32_VMX_TRUE_PINBASED_CTLS:
1455	cap = WHvCapabilityCodeVmxPinbasedCtls;
1456	break;
1457	case MSR_IA32_VMX_TRUE_PROCBASED_CTLS:
1458	cap = WHvCapabilityCodeVmxProcbasedCtls;
1459	break;
1460	case MSR_IA32_VMX_TRUE_ENTRY_CTLS:
1461	cap = WHvCapabilityCodeVmxTrueEntryCtls;
1462	break;
1463	case MSR_IA32_VMX_TRUE_EXIT_CTLS:
1464	cap = WHvCapabilityCodeVmxTrueExitCtls;
1465	break;
1466	default:
1467	cap = 0;
1468	}
1469
1470	if (cap != 0) {
1471	HRESULT hr = whp_dispatch.WHvGetCapability(
1472	cap, &val, sizeof(val),
1473	NULL);
1474	if (FAILED(hr)) {
1475	return 0;
1476	}
1477	return val;
1478	}
1479	return 0;
1480	}
1481
1482	static UINT64 whpx_get_default_exceptions(void)
1483	{
1484	struct whpx_state *whpx = &whpx_global;
1485	UINT64 intercepts = 0;
1486
1487	if (whpx->intercept_msr_gp) {
1488	intercepts \|= 1UL << WHvX64ExceptionTypeGeneralProtectionFault;
1489	}
1490
1491	return intercepts;
1492	}
1493
1494	/*
1495	* Controls whether we should intercept various exceptions on the guest,
1496	* namely breakpoint/single-step events.
1497	*
1498	* The 'exceptions' argument accepts a bitmask, e.g:
1499	* (1 << WHvX64ExceptionTypeDebugTrapOrFault) \| (...)
1500	*/
1501	HRESULT whpx_set_exception_exit_bitmap(UINT64 exceptions)
1502	{
1503	struct whpx_state *whpx = &whpx_global;
1504	WHV_PARTITION_PROPERTY prop;
1505	HRESULT hr;
1506
1507	if (exceptions == whpx->exception_exit_bitmap) {
1508	return S_OK;
1509	}
1510
1511	/* Register for MSR and CPUID exits */
1512	memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
1513	prop.ExtendedVmExits.X64MsrExit = 1;
1514	prop.ExtendedVmExits.X64CpuidExit = 1;
1515
1516	if (exceptions != 0 \|\| whpx_get_default_exceptions() != 0) {
1517	prop.ExtendedVmExits.ExceptionExit = 1;
1518	}
1519
1520	hr = whp_dispatch.WHvSetPartitionProperty(
1521	whpx->partition,
1522	WHvPartitionPropertyCodeExtendedVmExits,
1523	&prop,
1524	sizeof(WHV_PARTITION_PROPERTY));
1525	if (FAILED(hr)) {
1526	error_report("WHPX: Failed to enable extended VM exits, hr=%08lx", hr);
1527	return hr;
1528	}
1529
1530	memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
1531	prop.ExceptionExitBitmap = exceptions \| whpx_get_default_exceptions();
1532
1533	hr = whp_dispatch.WHvSetPartitionProperty(
1534	whpx->partition,
1535	WHvPartitionPropertyCodeExceptionExitBitmap,
1536	&prop,
1537	sizeof(WHV_PARTITION_PROPERTY));
1538
1539	if (SUCCEEDED(hr)) {
1540	whpx->exception_exit_bitmap = exceptions;
1541	} else {
1542	error_report("WHPX: Failed to set exception exit bitmap, hr=%08lx", hr);
1543	}
1544
1545	return hr;
1546	}
1547
1548
1549	/*
1550	* This function is called before/after stepping over a single instruction.
1551	* It will update the CPU registers to arm/disarm the instruction stepping
1552	* accordingly.
1553	*/
1554	static HRESULT whpx_vcpu_configure_single_stepping(CPUState *cpu,
1555	bool set,
1556	uint64_t *exit_context_rflags)
1557	{
1558	WHV_REGISTER_NAME reg_name;
1559	WHV_REGISTER_VALUE reg_value;
1560	HRESULT hr;
1561	struct whpx_state *whpx = &whpx_global;
1562
1563	/*
1564	* If we are trying to step over a single instruction, we need to set the
1565	* TF bit in rflags. Otherwise, clear it.
1566	*/
1567	reg_name = WHvX64RegisterRflags;
1568	hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
1569	whpx->partition,
1570	cpu->cpu_index,
1571	&reg_name,
1572	1,
1573	&reg_value);
1574
1575	if (FAILED(hr)) {
1576	error_report("WHPX: Failed to get rflags, hr=%08lx", hr);
1577	return hr;
1578	}
1579
1580	if (exit_context_rflags) {
1581	assert(*exit_context_rflags == reg_value.Reg64);
1582	}
1583
1584	if (set) {
1585	/* Raise WHvX64ExceptionTypeDebugTrapOrFault after each instruction */
1586	reg_value.Reg64 \|= TF_MASK;
1587	} else {
1588	reg_value.Reg64 &= ~TF_MASK;
1589	}
1590
1591	if (exit_context_rflags) {
1592	*exit_context_rflags = reg_value.Reg64;
1593	}
1594
1595	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1596	whpx->partition,
1597	cpu->cpu_index,
1598	&reg_name,
1599	1,
1600	&reg_value);
1601
1602	if (FAILED(hr)) {
1603	error_report("WHPX: Failed to set rflags,"
1604	" hr=%08lx",
1605	hr);
1606	return hr;
1607	}
1608
1609	reg_name = WHvRegisterInterruptState;
1610	reg_value.Reg64 = 0;
1611
1612	/* Suspend delivery of hardware interrupts during single-stepping. */
1613	reg_value.InterruptState.InterruptShadow = set != 0;
1614
1615	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1616	whpx->partition,
1617	cpu->cpu_index,
1618	&reg_name,
1619	1,
1620	&reg_value);
1621
1622	if (FAILED(hr)) {
1623	error_report("WHPX: Failed to set InterruptState,"
1624	" hr=%08lx",
1625	hr);
1626	return hr;
1627	}
1628
1629	if (!set) {
1630	/*
1631	* We have just finished stepping over a single instruction,
1632	* and intercepted the INT1 generated by it.
1633	* We need to now hide the INT1 from the guest,
1634	* as it would not be expecting it.
1635	*/
1636
1637	reg_name = WHvX64RegisterPendingDebugException;
1638	hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
1639	whpx->partition,
1640	cpu->cpu_index,
1641	&reg_name,
1642	1,
1643	&reg_value);
1644
1645	if (FAILED(hr)) {
1646	error_report("WHPX: Failed to get pending debug exceptions,"
1647	"hr=%08lx", hr);
1648	return hr;
1649	}
1650
1651	if (reg_value.PendingDebugException.SingleStep) {
1652	reg_value.PendingDebugException.SingleStep = 0;
1653
1654	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
1655	whpx->partition,
1656	cpu->cpu_index,
1657	&reg_name,
1658	1,
1659	&reg_value);
1660
1661	if (FAILED(hr)) {
1662	error_report("WHPX: Failed to clear pending debug exceptions,"
1663	"hr=%08lx", hr);
1664	return hr;
1665	}
1666	}
1667
1668	}
1669
1670	return S_OK;
1671	}
1672
1673	/*
1674	* Linux uses int3 (0xCC) during startup (see int3_selftest()) and for
1675	* debugging user-mode applications. Since the WHPX API does not offer
1676	* an easy way to pass the intercepted exception back to the guest, we
1677	* resort to using INT1 instead, and let the guest always handle INT3.
1678	*/
1679	static const uint8_t whpx_breakpoint_instruction = 0xF1;
1680
1681	/*
1682	* The WHPX QEMU backend implements breakpoints by writing the INT1
1683	* instruction into memory (ignoring the DRx registers). This raises a few
1684	* issues that need to be carefully handled:
1685	*
1686	* 1. Although unlikely, other parts of QEMU may set multiple breakpoints
1687	* at the same location, and later remove them in arbitrary order.
1688	* This should not cause memory corruption, and should only remove the
1689	* physical breakpoint instruction when the last QEMU breakpoint is gone.
1690	*
1691	* 2. Writing arbitrary virtual memory may fail if it's not mapped to a valid
1692	* physical location. Hence, physically adding/removing a breakpoint can
1693	* theoretically fail at any time. We need to keep track of it.
1694	*
1695	* The function below rebuilds a list of low-level breakpoints (one per
1696	* address, tracking the original instruction and any errors) from the list of
1697	* high-level breakpoints (set via cpu_breakpoint_insert()).
1698	*
1699	* In order to optimize performance, this function stores the list of
1700	* high-level breakpoints (a.k.a. CPU breakpoints) used to compute the
1701	* low-level ones, so that it won't be re-invoked until these breakpoints
1702	* change.
1703	*
1704	* Note that this function decides which breakpoints should be inserted into,
1705	* memory, but doesn't actually do it. The memory accessing is done in
1706	* whpx_apply_breakpoints().
1707	*/
1708	void whpx_translate_cpu_breakpoints(
1709	struct whpx_breakpoints *breakpoints,
1710	CPUState *cpu,
1711	int cpu_breakpoint_count)
1712	{
1713	CPUBreakpoint *bp;
1714	int cpu_bp_index = 0;
1715
1716	breakpoints->original_addresses =
1717	g_renew(vaddr, breakpoints->original_addresses, cpu_breakpoint_count);
1718
1719	breakpoints->original_address_count = cpu_breakpoint_count;
1720
1721	int max_breakpoints = cpu_breakpoint_count +
1722	(breakpoints->breakpoints ? breakpoints->breakpoints->used : 0);
1723
1724	struct whpx_breakpoint_collection *new_breakpoints =
1725	g_malloc0(sizeof(struct whpx_breakpoint_collection)
1726	+ max_breakpoints * sizeof(struct whpx_breakpoint));
1727
1728	new_breakpoints->allocated = max_breakpoints;
1729	new_breakpoints->used = 0;
1730
1731	/*
1732	* 1. Preserve all old breakpoints that could not be automatically
1733	* cleared when the CPU got stopped.
1734	*/
1735	if (breakpoints->breakpoints) {
1736	int i;
1737	for (i = 0; i < breakpoints->breakpoints->used; i++) {
1738	if (breakpoints->breakpoints->data[i].state != WHPX_BP_CLEARED) {
1739	new_breakpoints->data[new_breakpoints->used++] =
1740	breakpoints->breakpoints->data[i];
1741	}
1742	}
1743	}
1744
1745	/* 2. Map all CPU breakpoints to WHPX breakpoints */
1746	QTAILQ_FOREACH(bp, &cpu->breakpoints, entry) {
1747	int i;
1748	bool found = false;
1749
1750	/* This will be used to detect changed CPU breakpoints later. */
1751	breakpoints->original_addresses[cpu_bp_index++] = bp->pc;
1752
1753	for (i = 0; i < new_breakpoints->used; i++) {
1754	/*
1755	* WARNING: This loop has O(N^2) complexity, where N is the
1756	* number of breakpoints. It should not be a bottleneck in
1757	* real-world scenarios, since it only needs to run once after
1758	* the breakpoints have been modified.
1759	* If this ever becomes a concern, it can be optimized by storing
1760	* high-level breakpoint objects in a tree or hash map.
1761	*/
1762
1763	if (new_breakpoints->data[i].address == bp->pc) {
1764	/* There was already a breakpoint at this address. */
1765	if (new_breakpoints->data[i].state == WHPX_BP_CLEAR_PENDING) {
1766	new_breakpoints->data[i].state = WHPX_BP_SET;
1767	} else if (new_breakpoints->data[i].state == WHPX_BP_SET) {
1768	new_breakpoints->data[i].state = WHPX_BP_SET_PENDING;
1769	}
1770
1771	found = true;
1772	break;
1773	}
1774	}
1775
1776	if (!found && new_breakpoints->used < new_breakpoints->allocated) {
1777	/* No WHPX breakpoint at this address. Create one. */
1778	new_breakpoints->data[new_breakpoints->used].address = bp->pc;
1779	new_breakpoints->data[new_breakpoints->used].state =
1780	WHPX_BP_SET_PENDING;
1781	new_breakpoints->used++;
1782	}
1783	}
1784
1785	/*
1786	* Free the previous breakpoint list. This can be optimized by keeping
1787	* it as shadow buffer for the next computation instead of freeing
1788	* it immediately.
1789	*/
1790	g_free(breakpoints->breakpoints);
1791
1792	breakpoints->breakpoints = new_breakpoints;
1793	}
1794
1795	/*
1796	* Physically inserts/removes the breakpoints by reading and writing the
1797	* physical memory, keeping a track of the failed attempts.
1798	*
1799	* Passing resuming=true will try to set all previously unset breakpoints.
1800	* Passing resuming=false will remove all inserted ones.
1801	*/
1802	void whpx_apply_breakpoints(
1803	struct whpx_breakpoint_collection *breakpoints,
1804	CPUState *cpu,
1805	bool resuming)
1806	{
1807	int i, rc;
1808	if (!breakpoints) {
1809	return;
1810	}
1811
1812	for (i = 0; i < breakpoints->used; i++) {
1813	/* Decide what to do right now based on the last known state. */
1814	WhpxBreakpointState state = breakpoints->data[i].state;
1815	switch (state) {
1816	case WHPX_BP_CLEARED:
1817	if (resuming) {
1818	state = WHPX_BP_SET_PENDING;
1819	}
1820	break;
1821	case WHPX_BP_SET_PENDING:
1822	if (!resuming) {
1823	state = WHPX_BP_CLEARED;
1824	}
1825	break;
1826	case WHPX_BP_SET:
1827	if (!resuming) {
1828	state = WHPX_BP_CLEAR_PENDING;
1829	}
1830	break;
1831	case WHPX_BP_CLEAR_PENDING:
1832	if (resuming) {
1833	state = WHPX_BP_SET;
1834	}
1835	break;
1836	}
1837
1838	if (state == WHPX_BP_SET_PENDING) {
1839	/* Remember the original instruction. */
1840	rc = cpu_memory_rw_debug(cpu,
1841	breakpoints->data[i].address,
1842	&breakpoints->data[i].original_instruction,
1843	1,
1844	false);
1845
1846	if (!rc) {
1847	/* Write the breakpoint instruction. */
1848	rc = cpu_memory_rw_debug(cpu,
1849	breakpoints->data[i].address,
1850	(void *)&whpx_breakpoint_instruction,
1851	1,
1852	true);
1853	}
1854
1855	if (!rc) {
1856	state = WHPX_BP_SET;
1857	}
1858
1859	}
1860
1861	if (state == WHPX_BP_CLEAR_PENDING) {
1862	/* Restore the original instruction. */
1863	rc = cpu_memory_rw_debug(cpu,
1864	breakpoints->data[i].address,
1865	&breakpoints->data[i].original_instruction,
1866	1,
1867	true);
1868
1869	if (!rc) {
1870	state = WHPX_BP_CLEARED;
1871	}
1872	}
1873
1874	breakpoints->data[i].state = state;
1875	}
1876	}
1877
1878	void whpx_arch_destroy_vcpu(CPUState *cpu)
1879	{
1880	X86CPU *x86cpu = X86_CPU(cpu);
1881	CPUX86State *env = &x86cpu->env;
1882	g_free(env->emu_mmio_buf);
1883	qemu_vfree(env->xsave_buf);
1884	env->xsave_buf = NULL;
1885	env->xsave_buf_len = 0;
1886	}
1887
1888	/* Returns the address of the next instruction that is about to be executed. */
1889	static vaddr whpx_vcpu_get_pc(CPUState *cpu, bool exit_context_valid)
1890	{
1891	if (cpu->vcpu_dirty) {
1892	/* The CPU registers have been modified by other parts of QEMU. */
1893	return cpu_env(cpu)->eip;
1894	} else if (exit_context_valid) {
1895	/*
1896	* The CPU registers have not been modified by neither other parts
1897	* of QEMU, nor this port by calling WHvSetVirtualProcessorRegisters().
1898	* This is the most common case.
1899	*/
1900	AccelCPUState *vcpu = cpu->accel;
1901	return vcpu->exit_ctx.VpContext.Rip;
1902	} else {
1903	/*
1904	* The CPU registers have been modified by a call to
1905	* WHvSetVirtualProcessorRegisters() and must be re-queried from
1906	* the target.
1907	*/
1908	WHV_REGISTER_VALUE reg_value;
1909	WHV_REGISTER_NAME reg_name = WHvX64RegisterRip;
1910	HRESULT hr;
1911	struct whpx_state *whpx = &whpx_global;
1912
1913	hr = whp_dispatch.WHvGetVirtualProcessorRegisters(
1914	whpx->partition,
1915	cpu->cpu_index,
1916	&reg_name,
1917	1,
1918	&reg_value);
1919
1920	if (FAILED(hr)) {
1921	error_report("WHPX: Failed to get PC, hr=%08lx", hr);
1922	return 0;
1923	}
1924
1925	return reg_value.Reg64;
1926	}
1927	}
1928
1929	static int whpx_handle_halt(CPUState *cpu)
1930	{
1931	X86CPU *x86_cpu = X86_CPU(cpu);
1932	CPUX86State *env = &x86_cpu->env;
1933
1934	int ret = 0;
1935
1936	bql_lock();
1937	if (!(cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD) &&
1938	x86_cpu_interrupts_enabled(env)) &&
1939	!cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) {
1940	cpu->exception_index = EXCP_HLT;
1941	cpu->halted = true;
1942	ret = 1;
1943	}
1944	bql_unlock();
1945
1946	return ret;
1947	}
1948
1949	static int whpx_handle_hyperv_guestidle(CPUState *cpu)
1950	{
1951	X86CPU *x86_cpu = X86_CPU(cpu);
1952	CPUX86State *env = &x86_cpu->env;
1953
1954	env->hflags2 \|= HF2_HYPERV_HLT_MASK;
1955	return whpx_handle_halt(cpu);
1956	}
1957
1958	static void whpx_vcpu_kick_out_of_hlt(CPUState *cpu)
1959	{
1960	WHV_REGISTER_VALUE reg;
1961	whpx_get_reg(cpu, WHvRegisterInternalActivityState, &reg);
1962	if (reg.InternalActivity.HaltSuspend) {
1963	reg.InternalActivity.HaltSuspend = 0;
1964	whpx_set_reg(cpu, WHvRegisterInternalActivityState, reg);
1965	}
1966	}
1967
1968	static void whpx_vcpu_pre_run(CPUState *cpu)
1969	{
1970	HRESULT hr;
1971	struct whpx_state *whpx = &whpx_global;
1972	AccelCPUState *vcpu = cpu->accel;
1973	X86CPU *x86_cpu = X86_CPU(cpu);
1974	CPUX86State *env = &x86_cpu->env;
1975	int irq;
1976	uint8_t tpr;
1977	WHV_X64_PENDING_INTERRUPTION_REGISTER new_int;
1978	UINT32 reg_count = 0;
1979	WHV_REGISTER_VALUE reg_values[3];
1980	WHV_REGISTER_NAME reg_names[3];
1981	int irr = apic_get_highest_priority_irr(x86_cpu->apic_state);
1982
1983	memset(&new_int, 0, sizeof(new_int));
1984	memset(reg_values, 0, sizeof(reg_values));
1985
1986	bql_lock();
1987
1988	/* Inject NMI */
1989	if (!vcpu->interruption_pending &&
1990	cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI \| CPU_INTERRUPT_SMI)) {
1991	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) {
1992	cpu_reset_interrupt(cpu, CPU_INTERRUPT_NMI);
1993	vcpu->interruptable = false;
1994	new_int.InterruptionType = WHvX64PendingNmi;
1995	new_int.InterruptionPending = 1;
1996	new_int.InterruptionVector = 2;
1997	}
1998	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_SMI)) {
1999	cpu_reset_interrupt(cpu, CPU_INTERRUPT_SMI);
2000	}
2001	}
2002
2003	/*
2004	* Force the VCPU out of its inner loop to process any INIT requests or
2005	* commit pending TPR access.
2006	*/
2007	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT \| CPU_INTERRUPT_TPR)) {
2008	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT) &&
2009	!(env->hflags & HF_SMM_MASK)) {
2010	qatomic_set(&cpu->exit_request, true);
2011	}
2012	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_TPR)) {
2013	qatomic_set(&cpu->exit_request, true);
2014	}
2015	}
2016
2017	if (irr == -1) {
2018	if (isa_pic != NULL && pic_get_output(isa_pic)) {
2019	/* In case it's a PIC interrupt */
2020	irr = 0;
2021	} else if (cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) {
2022	abort();
2023	}
2024	}
2025
2026	/* Get pending hard interruption or replay one that was overwritten */
2027	if (!whpx_irqchip_in_kernel()) {
2028	if (!vcpu->interruption_pending &&
2029	vcpu->interruptable && (env->eflags & IF_MASK)
2030	&& (vcpu->tpr < irr \|\| irr == 0)) {
2031	assert(!new_int.InterruptionPending);
2032	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) {
2033	cpu_reset_interrupt(cpu, CPU_INTERRUPT_HARD);
2034	irq = cpu_get_pic_interrupt(env);
2035	if (irq >= 0) {
2036	new_int.InterruptionType = WHvX64PendingInterrupt;
2037	new_int.InterruptionPending = 1;
2038	new_int.InterruptionVector = irq;
2039	}
2040	}
2041	}
2042
2043	/* Setup interrupt state if new one was prepared */
2044	if (new_int.InterruptionPending) {
2045	reg_values[reg_count].PendingInterruption = new_int;
2046	reg_names[reg_count] = WHvRegisterPendingInterruption;
2047	reg_count += 1;
2048	}
2049	} else if (vcpu->ready_for_pic_interrupt &&
2050	cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) {
2051	cpu_reset_interrupt(cpu, CPU_INTERRUPT_HARD);
2052	irq = cpu_get_pic_interrupt(env);
2053	if (irq >= 0) {
2054	reg_names[reg_count] = WHvRegisterPendingEvent;
2055	reg_values[reg_count].ExtIntEvent = (WHV_X64_PENDING_EXT_INT_EVENT)
2056	{
2057	.EventPending = 1,
2058	.EventType = WHvX64PendingEventExtInt,
2059	.Vector = irq,
2060	};
2061	reg_count += 1;
2062	/*
2063	* When the Hyper-V APIC is enabled, to get out of HLT we
2064	* either have to request an interrupt or manually get it away
2065	* from HLT.
2066	*
2067	* We also manually do inject some interrupts via WHvRegisterPendingEvent
2068	* instead of WHVRequestInterrupt, which does not reset the HLT state.
2069	*/
2070	if (whpx_irqchip_in_kernel()) {
2071	whpx_vcpu_kick_out_of_hlt(cpu);
2072	}
2073	}
2074	}
2075
2076	/* Sync the TPR to the CR8 if was modified during the intercept */
2077	tpr = cpu_get_apic_tpr(x86_cpu->apic_state);
2078	if (!whpx_irqchip_in_kernel() && tpr != vcpu->tpr) {
2079	vcpu->tpr = tpr;
2080	reg_values[reg_count].Reg64 = tpr;
2081	qatomic_set(&cpu->exit_request, true);
2082	reg_names[reg_count] = WHvX64RegisterCr8;
2083	reg_count += 1;
2084	}
2085
2086	/* Update the state of the interrupt delivery notification */
2087	if ((!vcpu->window_registered \|\|
2088	(vcpu->window_priority < irr && vcpu->window_priority != 0) \|\|
2089	(irr == 0 && vcpu->window_priority != 0)) &&
2090	cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD)) {
2091	reg_values[reg_count].DeliverabilityNotifications =
2092	(WHV_X64_DELIVERABILITY_NOTIFICATIONS_REGISTER) {
2093	.InterruptNotification = 1,
2094	.InterruptPriority = irr >> 4
2095	};
2096	vcpu->window_registered = 1;
2097	vcpu->window_priority = irr;
2098	reg_names[reg_count] = WHvX64RegisterDeliverabilityNotifications;
2099	reg_count += 1;
2100	}
2101
2102	bql_unlock();
2103	vcpu->ready_for_pic_interrupt = false;
2104
2105	if (reg_count) {
2106	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
2107	whpx->partition, cpu->cpu_index,
2108	reg_names, reg_count, reg_values);
2109	if (FAILED(hr)) {
2110	error_report("WHPX: Failed to set interrupt state registers,"
2111	" hr=%08lx, InterruptPriority=%i", hr, irr >> 4);
2112	}
2113	}
2114	}
2115
2116	static void whpx_vcpu_post_run(CPUState *cpu)
2117	{
2118	AccelCPUState *vcpu = cpu->accel;
2119	X86CPU *x86_cpu = X86_CPU(cpu);
2120	CPUX86State *env = &x86_cpu->env;
2121
2122	env->eflags = vcpu->exit_ctx.VpContext.Rflags;
2123
2124	if (!whpx_irqchip_in_kernel()) {
2125	uint64_t tpr = vcpu->exit_ctx.VpContext.Cr8;
2126	if (vcpu->tpr != tpr) {
2127	vcpu->tpr = tpr;
2128	bql_lock();
2129	cpu_set_apic_tpr(x86_cpu->apic_state, vcpu->tpr);
2130	bql_unlock();
2131	}
2132	}
2133
2134	vcpu->interruption_pending =
2135	vcpu->exit_ctx.VpContext.ExecutionState.InterruptionPending;
2136
2137	vcpu->interruptable =
2138	!vcpu->exit_ctx.VpContext.ExecutionState.InterruptShadow;
2139	}
2140
2141
2142	static void whpx_vcpu_process_async_events(CPUState *cpu)
2143	{
2144	X86CPU *x86_cpu = X86_CPU(cpu);
2145	CPUX86State *env = &x86_cpu->env;
2146	AccelCPUState *vcpu = cpu->accel;
2147
2148	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_INIT) &&
2149	!(env->hflags & HF_SMM_MASK)) {
2150	whpx_cpu_synchronize_state(cpu);
2151	do_cpu_init(x86_cpu);
2152	vcpu->interruptable = true;
2153	}
2154
2155	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_POLL)) {
2156	cpu_reset_interrupt(cpu, CPU_INTERRUPT_POLL);
2157	apic_poll_irq(x86_cpu->apic_state);
2158	}
2159
2160	if ((cpu_test_interrupt(cpu, CPU_INTERRUPT_HARD) &&
2161	((env->eflags & IF_MASK) \|\| (env->hflags2 & HF2_HYPERV_HLT_MASK))) \|\|
2162	cpu_test_interrupt(cpu, CPU_INTERRUPT_NMI)) {
2163	cpu->halted = false;
2164	env->hflags2 &= ~HF2_HYPERV_HLT_MASK;
2165	}
2166
2167	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_SIPI)) {
2168	cpu_reset_interrupt(cpu, CPU_INTERRUPT_SIPI);
2169	whpx_cpu_synchronize_state(cpu);
2170	do_cpu_sipi(x86_cpu);
2171	}
2172
2173	if (cpu_test_interrupt(cpu, CPU_INTERRUPT_TPR)) {
2174	cpu_reset_interrupt(cpu, CPU_INTERRUPT_TPR);
2175	whpx_cpu_synchronize_state(cpu);
2176	apic_handle_tpr_access_report(x86_cpu->apic_state, env->eip,
2177	env->tpr_access_type);
2178	}
2179	}
2180
2181	static void whpx_inject_exceptions(CPUState* cpu)
2182	{
2183	X86CPU *x86_cpu = X86_CPU(cpu);
2184	CPUX86State *env = &x86_cpu->env;
2185
2186	if (env->exception_injected) {
2187	env->exception_injected = 0;
2188	WHV_REGISTER_VALUE reg = {};
2189	reg.ExceptionEvent.EventPending = 1;
2190	reg.ExceptionEvent.EventType = WHvX64PendingEventException;
2191	reg.ExceptionEvent.DeliverErrorCode = env->has_error_code;
2192	reg.ExceptionEvent.Vector = env->exception_nr;
2193	reg.ExceptionEvent.ErrorCode = env->error_code;
2194	if (env->exception_has_payload) {
2195	reg.ExceptionEvent.ExceptionParameter = env->exception_payload;
2196	}
2197	whpx_set_reg(cpu, WHvRegisterPendingEvent, reg);
2198	}
2199	}
2200
2201	int whpx_vcpu_run(CPUState *cpu)
2202	{
2203	HRESULT hr;
2204	struct whpx_state *whpx = &whpx_global;
2205	AccelCPUState *vcpu = cpu->accel;
2206	struct whpx_breakpoint *stepped_over_bp = NULL;
2207	WhpxStepMode exclusive_step_mode = WHPX_STEP_NONE;
2208	int ret;
2209
2210	g_assert(bql_locked());
2211
2212	if (whpx->running_cpus++ == 0) {
2213	/* Insert breakpoints into memory, update exception exit bitmap. */
2214	ret = whpx_first_vcpu_starting(cpu);
2215	if (ret != 0) {
2216	return ret;
2217	}
2218	}
2219
2220	if (whpx->breakpoints.breakpoints &&
2221	whpx->breakpoints.breakpoints->used > 0)
2222	{
2223	uint64_t pc = whpx_vcpu_get_pc(cpu, true);
2224	stepped_over_bp = whpx_lookup_breakpoint_by_addr(pc);
2225	if (stepped_over_bp && stepped_over_bp->state != WHPX_BP_SET) {
2226	stepped_over_bp = NULL;
2227	}
2228
2229	if (stepped_over_bp) {
2230	/*
2231	* We are trying to run the instruction overwritten by an active
2232	* breakpoint. We will temporarily disable the breakpoint, suspend
2233	* other CPUs, and step over the instruction.
2234	*/
2235	exclusive_step_mode = WHPX_STEP_EXCLUSIVE;
2236	}
2237	}
2238
2239	if (exclusive_step_mode == WHPX_STEP_NONE) {
2240	whpx_vcpu_process_async_events(cpu);
2241	if (cpu->halted && !whpx_irqchip_in_kernel()) {
2242	cpu->exception_index = EXCP_HLT;
2243	qatomic_set(&cpu->exit_request, false);
2244	return 0;
2245	}
2246	}
2247
2248	bql_unlock();
2249
2250	if (exclusive_step_mode != WHPX_STEP_NONE) {
2251	start_exclusive();
2252	g_assert(cpu == current_cpu);
2253	g_assert(!cpu->running);
2254	cpu->running = true;
2255
2256	hr = whpx_set_exception_exit_bitmap(
2257	1UL << WHvX64ExceptionTypeDebugTrapOrFault);
2258	if (!SUCCEEDED(hr)) {
2259	error_report("WHPX: Failed to update exception exit mask, "
2260	"hr=%08lx.", hr);
2261	return 1;
2262	}
2263
2264	if (stepped_over_bp) {
2265	/* Temporarily disable the triggered breakpoint. */
2266	cpu_memory_rw_debug(cpu,
2267	stepped_over_bp->address,
2268	&stepped_over_bp->original_instruction,
2269	1,
2270	true);
2271	}
2272	} else {
2273	cpu_exec_start(cpu);
2274	}
2275
2276	do {
2277	if (cpu->vcpu_dirty) {
2278	whpx_set_registers(cpu, WHPX_LEVEL_RUNTIME_STATE);
2279	cpu->vcpu_dirty = false;
2280	}
2281
2282	if (exclusive_step_mode == WHPX_STEP_NONE) {
2283	whpx_vcpu_pre_run(cpu);
2284
2285	/* Corresponding store-release is in cpu_exit. */
2286	if (qatomic_load_acquire(&cpu->exit_request)) {
2287	whpx_vcpu_kick(cpu);
2288	}
2289	}
2290
2291	if (exclusive_step_mode != WHPX_STEP_NONE \|\| cpu_single_stepping(cpu)) {
2292	whpx_vcpu_configure_single_stepping(cpu, true, NULL);
2293	}
2294
2295	whpx_inject_exceptions(cpu);
2296
2297	hr = whp_dispatch.WHvRunVirtualProcessor(
2298	whpx->partition, cpu->cpu_index,
2299	&vcpu->exit_ctx, sizeof(vcpu->exit_ctx));
2300
2301	if (FAILED(hr)) {
2302	error_report("WHPX: Failed to exec a virtual processor,"
2303	" hr=%08lx", hr);
2304	ret = -1;
2305	break;
2306	}
2307
2308	if (exclusive_step_mode != WHPX_STEP_NONE \|\| cpu_single_stepping(cpu)) {
2309	whpx_vcpu_configure_single_stepping(cpu,
2310	false,
2311	&vcpu->exit_ctx.VpContext.Rflags);
2312	}
2313
2314	whpx_vcpu_post_run(cpu);
2315
2316	switch (vcpu->exit_ctx.ExitReason) {
2317	case WHvRunVpExitReasonMemoryAccess:
2318	ret = whpx_handle_mmio(cpu, &vcpu->exit_ctx);
2319	break;
2320
2321	case WHvRunVpExitReasonX64IoPortAccess:
2322	ret = whpx_handle_portio(cpu, &vcpu->exit_ctx);
2323	break;
2324
2325	case WHvRunVpExitReasonX64InterruptWindow:
2326	vcpu->ready_for_pic_interrupt = 1;
2327	vcpu->window_registered = 0;
2328	vcpu->window_priority = 0;
2329	ret = 0;
2330	break;
2331
2332	case WHvRunVpExitReasonX64ApicEoi:
2333	assert(whpx_irqchip_in_kernel());
2334	ioapic_eoi_broadcast(vcpu->exit_ctx.ApicEoi.InterruptVector);
2335	break;
2336
2337	case WHvRunVpExitReasonX64Halt:
2338	/*
2339	* Used for kernel-irqchip=off
2340	*/
2341	ret = whpx_handle_halt(cpu);
2342	break;
2343
2344	case WHvRunVpExitReasonCanceled:
2345	if (exclusive_step_mode != WHPX_STEP_NONE) {
2346	/*
2347	* We are trying to step over a single instruction, and
2348	* likely got a request to stop from another thread.
2349	* Delay it until we are done stepping
2350	* over.
2351	*/
2352	ret = 0;
2353	} else {
2354	cpu->exception_index = EXCP_INTERRUPT;
2355	ret = 1;
2356	}
2357	break;
2358	case WHvRunVpExitReasonX64MsrAccess: {
2359	WHV_REGISTER_VALUE reg_values[3] = {0};
2360	WHV_REGISTER_NAME reg_names[3];
2361	UINT32 reg_count;
2362	bool is_known_msr = 0;
2363	bool raises_gpf = false;
2364	uint64_t val;
2365
2366	if (vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite) {
2367	val = ((uint32_t)vcpu->exit_ctx.MsrAccess.Rax) \|
2368	((uint64_t)(vcpu->exit_ctx.MsrAccess.Rdx) << 32);
2369	} else {
2370	/*
2371	* Workaround for [-Werror=maybe-uninitialized]
2372	* with GCC. Not needed with Clang.
2373	*/
2374	val = 0;
2375	}
2376
2377	reg_names[0] = WHvX64RegisterRip;
2378	reg_names[1] = WHvX64RegisterRax;
2379	reg_names[2] = WHvX64RegisterRdx;
2380
2381	reg_values[0].Reg64 =
2382	vcpu->exit_ctx.VpContext.Rip +
2383	vcpu->exit_ctx.VpContext.InstructionLength;
2384
2385	if (vcpu->exit_ctx.MsrAccess.MsrNumber == HV_X64_MSR_APIC_FREQUENCY
2386	&& !vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite
2387	&& !whpx_irqchip_in_kernel()) {
2388	is_known_msr = 1;
2389	val = X86_CPU(cpu)->env.apic_bus_freq;
2390	}
2391
2392	if (vcpu->exit_ctx.MsrAccess.MsrNumber == MSR_IA32_APICBASE) {
2393	is_known_msr = 1;
2394	if (val & MSR_IA32_APICBASE_RESERVED) {
2395	x86_emul_raise_exception(&X86_CPU(cpu)->env, EXCP0D_GPF, 0);
2396	raises_gpf = true;
2397	}
2398	if (!vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite) {
2399	/* Read path unreachable on Hyper-V */
2400	abort();
2401	} else {
2402	WHV_REGISTER_VALUE reg = {.Reg64 = val};
2403	int msr_ret = cpu_set_apic_base(X86_CPU(cpu)->apic_state, val);
2404	if (msr_ret < 0) {
2405	x86_emul_raise_exception(&X86_CPU(cpu)->env, EXCP0D_GPF, 0);
2406	raises_gpf = true;
2407	} else {
2408	whpx_set_reg(cpu, WHvX64RegisterApicBase, reg);
2409	}
2410	}
2411	}
2412
2413	if (!whpx_irqchip_in_kernel() &&
2414	vcpu->exit_ctx.MsrAccess.MsrNumber >= MSR_APIC_START &&
2415	vcpu->exit_ctx.MsrAccess.MsrNumber <= MSR_APIC_END) {
2416	int index = vcpu->exit_ctx.MsrAccess.MsrNumber - MSR_APIC_START;
2417	int msr_ret;
2418	is_known_msr = 1;
2419	if (!vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite) {
2420	bql_lock();
2421	msr_ret = apic_msr_read(X86_CPU(cpu)->apic_state, index, &val);
2422	bql_unlock();
2423	reg_values[1].Reg64 = val;
2424	if (msr_ret < 0) {
2425	x86_emul_raise_exception(&X86_CPU(cpu)->env, EXCP0D_GPF, 0);
2426	raises_gpf = true;
2427	}
2428	} else {
2429	bql_lock();
2430	msr_ret = apic_msr_write(X86_CPU(cpu)->apic_state, index, val);
2431	bql_unlock();
2432	if (msr_ret < 0) {
2433	x86_emul_raise_exception(&X86_CPU(cpu)->env, EXCP0D_GPF, 0);
2434	raises_gpf = true;
2435	}
2436	}
2437	}
2438
2439	/*
2440	* Windows and Linux both use this MSR.
2441	* Windows 11 25H2 uses it even when not advertised.
2442	*/
2443	if (vcpu->exit_ctx.MsrAccess.MsrNumber == HV_X64_MSR_GUEST_IDLE
2444	&& !vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite
2445	&& !whpx_irqchip_in_kernel()
2446	&& whpx->hyperv_enlightenments_enabled) {
2447	is_known_msr = 1;
2448	whpx_bump_rip(cpu, &vcpu->exit_ctx);
2449	ret = whpx_handle_hyperv_guestidle(cpu);
2450	break;
2451	}
2452
2453	/*
2454	* Linux tries to use it anyway even when not exposed.
2455	* Ignore the write as the VP assist page is not used.
2456	*/
2457	if (vcpu->exit_ctx.MsrAccess.MsrNumber == HV_X64_MSR_VP_ASSIST_PAGE
2458	&& vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite
2459	&& !whpx_irqchip_in_kernel()
2460	&& whpx->hyperv_enlightenments_enabled) {
2461	is_known_msr = 1;
2462	}
2463
2464	/*
2465	* For all unsupported MSR access we:
2466	* ignore writes
2467	* return 0 on read.
2468	*/
2469	reg_count = vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite ?
2470	1 : 3;
2471
2472	if (!vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite) {
2473	reg_values[1].Reg32 = (uint32_t)val;
2474	reg_values[2].Reg32 = (uint32_t)(val >> 32);
2475	}
2476
2477	if (!is_known_msr) {
2478	trace_whpx_unsupported_msr_access(vcpu->exit_ctx.MsrAccess.MsrNumber,
2479	vcpu->exit_ctx.MsrAccess.AccessInfo.IsWrite);
2480	}
2481
2482	if (!is_known_msr && !whpx->ignore_unknown_msr) {
2483	x86_emul_raise_exception(&X86_CPU(cpu)->env, EXCP0D_GPF, 0);
2484	raises_gpf = true;
2485	}
2486
2487	/* When a GPF is raised, do not change Rip. */
2488	if (raises_gpf) {
2489	reg_values[0].Reg64 =
2490	vcpu->exit_ctx.VpContext.Rip;
2491	}
2492
2493	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
2494	whpx->partition,
2495	cpu->cpu_index,
2496	reg_names, reg_count,
2497	reg_values);
2498
2499	if (FAILED(hr)) {
2500	error_report("WHPX: Failed to set MsrAccess state "
2501	" registers, hr=%08lx", hr);
2502	}
2503	ret = 0;
2504	break;
2505	}
2506	case WHvRunVpExitReasonX64Cpuid: {
2507	WHV_REGISTER_VALUE reg_values[5] = {0};
2508	WHV_REGISTER_NAME reg_names[5];
2509	UINT32 reg_count = 5;
2510	X86CPU *x86_cpu = X86_CPU(cpu);
2511	CPUX86State *env = &x86_cpu->env;
2512
2513	reg_names[0] = WHvX64RegisterRip;
2514	reg_names[1] = WHvX64RegisterRax;
2515	reg_names[2] = WHvX64RegisterRcx;
2516	reg_names[3] = WHvX64RegisterRdx;
2517	reg_names[4] = WHvX64RegisterRbx;
2518
2519	reg_values[0].Reg64 =
2520	vcpu->exit_ctx.VpContext.Rip +
2521	vcpu->exit_ctx.VpContext.InstructionLength;
2522
2523	cpu_x86_cpuid(env, vcpu->exit_ctx.CpuidAccess.Rax,
2524	vcpu->exit_ctx.CpuidAccess.Rcx,
2525	(UINT32 *)&reg_values[1].Reg32,
2526	(UINT32 )&reg_values[4].Reg32, (UINT32 )&reg_values[2].Reg32,
2527	(UINT32 *)&reg_values[3].Reg32);
2528
2529	if (!whpx->hyperv_enlightenments_enabled) {
2530	switch (vcpu->exit_ctx.CpuidAccess.Rax) {
2531	case 1:
2532	reg_values[2].Reg64 \|= CPUID_EXT_HYPERVISOR;
2533	break;
2534	case 0x40000000:
2535	/*
2536	* Use vmware_cpuid_freq as a proxy to report VMware.
2537	* This is to get the TSC/APIC frequency query functionality
2538	* provided through vmport, as Linux doesn't use leaf
2539	* 0x40000010 for getting those frequencies.
2540	*/
2541	if (x86_cpu->vmware_cpuid_freq) {
2542	reg_values[1].Reg64 = 0x40000010;
2543	reg_values[4].Reg64 = 0x61774d56;
2544	reg_values[2].Reg64 = 0x4d566572;
2545	reg_values[3].Reg64 = 0x65726177;
2546	} else {
2547	/* report KVM otherwise if that's disabled */
2548	reg_values[1].Reg64 = 0x40000001;
2549	reg_values[4].Reg64 = 0x4b4d564b;
2550	reg_values[2].Reg64 = 0x564b4d56;
2551	reg_values[3].Reg64 = 0x4d;
2552	}
2553	break;
2554	case 0x40000001:
2555	if (!x86_cpu->vmware_cpuid_freq) {
2556	/* KVM reporting of X2APIC support */
2557	reg_values[1].Reg64 = reg_values[4].Reg64 =
2558	reg_values[2].Reg64 = 1 << 15;
2559	}
2560	break;
2561	case 0x40000010:
2562	if (x86_cpu->vmware_cpuid_freq) {
2563	reg_values[1].Reg64 = env->tsc_khz;
2564	reg_values[4].Reg64 = env->apic_bus_freq / 1000; /* Hz to KHz */
2565	}
2566	break;
2567	}
2568	} else {
2569	switch (vcpu->exit_ctx.CpuidAccess.Rax) {
2570	case 0x40000000:
2571	case 0x40000001:
2572	case 0x40000010:
2573	reg_values[1].Reg64 = vcpu->exit_ctx.CpuidAccess.DefaultResultRax;
2574	reg_values[2].Reg64 = vcpu->exit_ctx.CpuidAccess.DefaultResultRcx;
2575	reg_values[3].Reg64 = vcpu->exit_ctx.CpuidAccess.DefaultResultRdx;
2576	reg_values[4].Reg64 = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
2577	break;
2578	}
2579	}
2580
2581	if (vcpu->exit_ctx.CpuidAccess.Rax == 0x1) {
2582	if (cpu_has_x2apic_feature(env)) {
2583	reg_values[2].Reg64 \|= CPUID_EXT_X2APIC;
2584	} else {
2585	reg_values[2].Reg32 &= ~CPUID_EXT_X2APIC;
2586	}
2587
2588	/* CPUID[1:EDX].APIC is dynamic */
2589	if (env->features[FEAT_1_EDX] & CPUID_APIC) {
2590	reg_values[3].Reg32 \|= CPUID_APIC;
2591	} else {
2592	reg_values[3].Reg32 &= ~CPUID_APIC;
2593	}
2594	}
2595
2596	/* Dynamic depending on XCR0 and XSS, so query DefaultResult */
2597	if (vcpu->exit_ctx.CpuidAccess.Rax == 0x07
2598	&& vcpu->exit_ctx.CpuidAccess.Rcx == 0) {
2599	if (vcpu->exit_ctx.CpuidAccess.DefaultResultRdx
2600	& CPUID_7_0_EDX_CET_IBT) {
2601	reg_values[3].Reg32 \|= CPUID_7_0_EDX_CET_IBT;
2602	} else {
2603	reg_values[3].Reg32 &= ~CPUID_7_0_EDX_CET_IBT;
2604	}
2605
2606	if (vcpu->exit_ctx.CpuidAccess.DefaultResultRcx
2607	& CPUID_7_0_ECX_CET_SHSTK) {
2608	reg_values[2].Reg32 \|= CPUID_7_0_ECX_CET_SHSTK;
2609	} else {
2610	reg_values[2].Reg32 &= ~CPUID_7_0_ECX_CET_SHSTK;
2611	}
2612
2613	if (vcpu->exit_ctx.CpuidAccess.DefaultResultRcx
2614	& CPUID_7_0_ECX_OSPKE) {
2615	reg_values[2].Reg32 \|= CPUID_7_0_ECX_OSPKE;
2616	} else {
2617	reg_values[2].Reg32 &= ~CPUID_7_0_ECX_OSPKE;
2618	}
2619	}
2620
2621	/* CPUID[0xD,{1,2}].EBX are dynamic depending on guest features. */
2622	if (vcpu->exit_ctx.CpuidAccess.Rax == 0xd) {
2623	if (vcpu->exit_ctx.CpuidAccess.Rcx == 1
2624	\|\| vcpu->exit_ctx.CpuidAccess.Rcx == 2) {
2625	reg_values[4].Reg64 = vcpu->exit_ctx.CpuidAccess.DefaultResultRbx;
2626	}
2627	}
2628
2629	/* OSXSAVE is dynamic. Do this instead of syncing CR4 */
2630	if (vcpu->exit_ctx.CpuidAccess.Rax == 1) {
2631	if (vcpu->exit_ctx.CpuidAccess.DefaultResultRcx
2632	& CPUID_EXT_OSXSAVE) {
2633	reg_values[2].Reg32 \|= CPUID_EXT_OSXSAVE;
2634	} else {
2635	reg_values[2].Reg32 &= ~CPUID_EXT_OSXSAVE;
2636	}
2637	}
2638
2639	hr = whp_dispatch.WHvSetVirtualProcessorRegisters(
2640	whpx->partition,
2641	cpu->cpu_index,
2642	reg_names, reg_count,
2643	reg_values);
2644
2645	if (FAILED(hr)) {
2646	error_report("WHPX: Failed to set CpuidAccess state "
2647	" registers, hr=%08lx", hr);
2648	}
2649	ret = 0;
2650	break;
2651	}
2652	case WHvRunVpExitReasonException:
2653	if (vcpu->exit_ctx.VpException.ExceptionType ==
2654	WHvX64ExceptionTypeGeneralProtectionFault) {
2655	if (whpx_handle_msr_from_gpf(cpu)) {
2656	whpx_inject_back_gpf(cpu);
2657	}
2658	ret = 0;
2659	break;
2660	}
2661
2662	whpx_get_registers(cpu, WHPX_LEVEL_FULL_STATE);
2663
2664	if ((vcpu->exit_ctx.VpException.ExceptionType ==
2665	WHvX64ExceptionTypeDebugTrapOrFault) &&
2666	(vcpu->exit_ctx.VpException.InstructionByteCount >= 1) &&
2667	(vcpu->exit_ctx.VpException.InstructionBytes[0] ==
2668	whpx_breakpoint_instruction)) {
2669	/* Stopped at a software breakpoint. */
2670	cpu->exception_index = EXCP_DEBUG;
2671	} else if ((vcpu->exit_ctx.VpException.ExceptionType ==
2672	WHvX64ExceptionTypeDebugTrapOrFault) &&
2673	!cpu_single_stepping(cpu)) {
2674	whpx_inject_back_db(cpu);
2675	cpu->exception_index = EXCP_INTERRUPT;
2676	} else {
2677	/* Another exception or debug event. Report it to GDB. */
2678	cpu->exception_index = EXCP_DEBUG;
2679	}
2680
2681	ret = 1;
2682	break;
2683	case WHvRunVpExitReasonNone:
2684	case WHvRunVpExitReasonUnrecoverableException:
2685	case WHvRunVpExitReasonInvalidVpRegisterValue:
2686	case WHvRunVpExitReasonUnsupportedFeature:
2687	default:
2688	error_report("WHPX: Unexpected VP exit code %d",
2689	vcpu->exit_ctx.ExitReason);
2690	whpx_get_registers(cpu, WHPX_LEVEL_FULL_STATE);
2691	bql_lock();
2692	vm_stop(RUN_STATE_PAUSED);
2693	bql_unlock();
2694	break;
2695	}
2696
2697	} while (!ret);
2698
2699	if (stepped_over_bp) {
2700	/* Restore the breakpoint we stepped over */
2701	cpu_memory_rw_debug(cpu,
2702	stepped_over_bp->address,
2703	(void *)&whpx_breakpoint_instruction,
2704	1,
2705	true);
2706	}
2707
2708	if (exclusive_step_mode != WHPX_STEP_NONE) {
2709	g_assert(cpu_in_exclusive_context(cpu));
2710	cpu->running = false;
2711	end_exclusive();
2712
2713	exclusive_step_mode = WHPX_STEP_NONE;
2714	} else {
2715	cpu_exec_end(cpu);
2716	}
2717
2718	bql_lock();
2719	current_cpu = cpu;
2720
2721	if (--whpx->running_cpus == 0) {
2722	whpx_last_vcpu_stopping(cpu);
2723	}
2724
2725	return ret < 0;
2726	}
2727
2728	/*
2729	* Vcpu support.
2730	*/
2731
2732	static Error *whpx_migration_blocker;
2733
2734	static void whpx_cpu_update_state(void *opaque, bool running, RunState state)
2735	{
2736	CPUX86State *env = opaque;
2737
2738	if (running) {
2739	env->tsc_valid = false;
2740	}
2741	}
2742
2743	int whpx_init_vcpu(CPUState *cpu)
2744	{
2745	HRESULT hr;
2746	struct whpx_state *whpx = &whpx_global;
2747	AccelCPUState *vcpu = NULL;
2748	Error *local_error = NULL;
2749	X86CPU *x86_cpu = X86_CPU(cpu);
2750	CPUX86State *env = &x86_cpu->env;
2751	X86XSaveHeader *header;
2752	size_t page_size = qemu_real_host_page_size();
2753	size_t xsave_len;
2754	UINT64 freq = 0;
2755	int ret;
2756
2757	/* Add migration blockers for all unsupported features of the
2758	* Windows Hypervisor Platform
2759	*/
2760	if (whpx_migration_blocker == NULL) {
2761	error_setg(&whpx_migration_blocker,
2762	"State blocked due to missing dirty memory tracking support,"
2763	"And some system register/state save-restore ");
2764
2765	if (migrate_add_blocker(&whpx_migration_blocker, &local_error) < 0) {
2766	error_report_err(local_error);
2767	ret = -EINVAL;
2768	goto error;
2769	}
2770	}
2771
2772	vcpu = g_new0(AccelCPUState, 1);
2773
2774	hr = whp_dispatch.WHvCreateVirtualProcessor(
2775	whpx->partition, cpu->cpu_index, 0);
2776	if (FAILED(hr)) {
2777	error_report("WHPX: Failed to create a virtual processor,"
2778	" hr=%08lx", hr);
2779	ret = -EINVAL;
2780	goto error;
2781	}
2782
2783	if (!whpx_irqchip_in_kernel() && x86_cpu->apic_state != NULL) {
2784	WHV_REGISTER_VALUE apic_id = {.Reg64 = x86_cpu->apic_state->initial_apic_id};
2785	whpx_set_reg(cpu, WHvX64RegisterInitialApicId, apic_id);
2786	}
2787
2788	/*
2789	* vcpu's TSC frequency is either specified by user, or use the value
2790	* provided by Hyper-V if the former is not present. In the latter case, we
2791	* query it from Hyper-V and record in env->tsc_khz, so that vcpu's TSC
2792	* frequency can be migrated later via this field.
2793	*/
2794	if (!env->tsc_khz) {
2795	hr = whp_dispatch.WHvGetCapability(
2796	WHvCapabilityCodeProcessorClockFrequency, &freq, sizeof(freq),
2797	NULL);
2798	if (hr != WHV_E_UNKNOWN_CAPABILITY) {
2799	if (FAILED(hr)) {
2800	printf("WHPX: Failed to query tsc frequency, hr=0x%08lx\n", hr);
2801	} else {
2802	env->tsc_khz = freq / 1000; /* Hz to KHz */
2803	}
2804	}
2805	}
2806
2807	env->apic_bus_freq = HYPERV_APIC_BUS_FREQUENCY;
2808	hr = whp_dispatch.WHvGetCapability(
2809	WHvCapabilityCodeInterruptClockFrequency, &freq, sizeof(freq), NULL);
2810	if (hr != WHV_E_UNKNOWN_CAPABILITY) {
2811	if (FAILED(hr)) {
2812	printf("WHPX: Failed to query apic bus frequency hr=0x%08lx\n", hr);
2813	} else {
2814	env->apic_bus_freq = freq;
2815	}
2816	}
2817
2818	/* When not using the Hyper-V APIC, the frequency is 1 GHz */
2819	if (!whpx_irqchip_in_kernel()) {
2820	env->apic_bus_freq = 1000000000;
2821	}
2822
2823	vcpu->interruptable = true;
2824	cpu->vcpu_dirty = true;
2825	cpu->accel = vcpu;
2826	max_vcpu_index = max(max_vcpu_index, cpu->cpu_index);
2827	qemu_add_vm_change_state_handler(whpx_cpu_update_state, env);
2828
2829	env->emu_mmio_buf = g_new(char, 4096);
2830	/* Initialize XSAVE buffer page-aligned */
2831	xsave_len = whpx_get_xsave_max_len();
2832	env->xsave_buf = qemu_memalign(page_size, xsave_len);
2833	env->xsave_buf_len = xsave_len;
2834	memset(env->xsave_buf, 0, env->xsave_buf_len);
2835
2836	/* we need to set the compacted format bit in xsave header for Hyper-V */
2837	header = (X86XSaveHeader *)(env->xsave_buf + sizeof(X86LegacyXSaveArea));
2838	header->xcomp_bv = header->xstate_bv \| (1ULL << 63);
2839
2840	return 0;
2841
2842	error:
2843	g_free(vcpu);
2844
2845	return ret;
2846	}
2847
2848	static void whpx_cpu_xsave_init(void)
2849	{
2850	static bool first = true;
2851	int i;
2852
2853	if (!first) {
2854	return;
2855	}
2856	first = false;
2857
2858	/* x87 and SSE states are in the legacy region of the XSAVE area. */
2859	x86_ext_save_areas[XSTATE_FP_BIT].offset = 0;
2860	x86_ext_save_areas[XSTATE_SSE_BIT].offset = 0;
2861
2862	for (i = XSTATE_SSE_BIT + 1; i < XSAVE_STATE_AREA_COUNT; i++) {
2863	ExtSaveArea *esa = &x86_ext_save_areas[i];
2864
2865	if (esa->size) {
2866	int sz = whpx_get_supported_cpuid(0xd, i, R_EAX);
2867	if (sz != 0) {
2868	assert(esa->size == sz);
2869	esa->offset = whpx_get_supported_cpuid(0xd, i, R_EBX);
2870	}
2871	}
2872	}
2873	}
2874
2875	static void whpx_cpu_max_instance_init(X86CPU *cpu)
2876	{
2877	CPUX86State *env = &cpu->env;
2878
2879	env->cpuid_min_level =
2880	whpx_get_supported_cpuid(0x0, 0, R_EAX);
2881	env->cpuid_min_xlevel =
2882	whpx_get_supported_cpuid(0x80000000, 0, R_EAX);
2883	env->cpuid_min_xlevel2 =
2884	whpx_get_supported_cpuid(0xC0000000, 0, R_EAX);
2885	}
2886
2887	static PropValue whpx_default_props[] = {
2888	{ "x2apic", "on" },
2889	{ NULL, NULL },
2890	};
2891
2892
2893	void whpx_cpu_instance_init(CPUState *cs)
2894	{
2895	X86CPU *cpu = X86_CPU(cs);
2896	X86CPUClass *xcc = X86_CPU_GET_CLASS(cpu);
2897
2898	host_cpu_instance_init(cpu);
2899	x86_cpu_apply_props(cpu, whpx_default_props);
2900
2901	if (xcc->max_features) {
2902	whpx_cpu_max_instance_init(cpu);
2903	}
2904
2905	if (whpx_has_xsave()) {
2906	whpx_cpu_xsave_init();
2907	}
2908	}
2909
2910	/*
2911	* Partition support
2912	*/
2913
2914	static void whpx_set_unknown_msr(Object obj, Visitor v,
2915	const char name, void opaque,
2916	Error **errp)
2917	{
2918	struct whpx_state *whpx = &whpx_global;
2919	OnOffAuto mode;
2920
2921	if (!visit_type_OnOffAuto(v, name, &mode, errp)) {
2922	return;
2923	}
2924
2925	switch (mode) {
2926	case ON_OFF_AUTO_ON:
2927	whpx->ignore_unknown_msr = true;
2928	break;
2929
2930	case ON_OFF_AUTO_OFF:
2931	whpx->ignore_unknown_msr = false;
2932	break;
2933
2934	case ON_OFF_AUTO_AUTO:
2935	whpx->ignore_unknown_msr = true;
2936	break;
2937	default:
2938	/*
2939	* The value was checked in visit_type_OnOffAuto() above. If
2940	* we get here, then something is wrong in QEMU.
2941	*/
2942	abort();
2943	}
2944	}
2945
2946	static void whpx_set_intercept_msr_gp(Object obj, Visitor v,
2947	const char name, void opaque,
2948	Error **errp)
2949	{
2950	struct whpx_state *whpx = &whpx_global;
2951	OnOffAuto mode;
2952
2953	if (!visit_type_OnOffAuto(v, name, &mode, errp)) {
2954	return;
2955	}
2956
2957	switch (mode) {
2958	case ON_OFF_AUTO_ON:
2959	whpx->intercept_msr_gp = true;
2960	break;
2961
2962	case ON_OFF_AUTO_OFF:
2963	whpx->intercept_msr_gp = false;
2964	break;
2965
2966	case ON_OFF_AUTO_AUTO:
2967	whpx->intercept_msr_gp = false;
2968	break;
2969	default:
2970	/*
2971	* The value was checked in visit_type_OnOffAuto() above. If
2972	* we get here, then something is wrong in QEMU.
2973	*/
2974	abort();
2975	}
2976	}
2977
2978	static void whpx_set_ssd(Object obj, Visitor v,
2979	const char name, void opaque,
2980	Error **errp)
2981	{
2982	struct whpx_state *whpx = &whpx_global;
2983	OnOffAuto mode;
2984
2985	if (!visit_type_OnOffAuto(v, name, &mode, errp)) {
2986	return;
2987	}
2988
2989	switch (mode) {
2990	case ON_OFF_AUTO_ON:
2991	whpx->separate_security_domain = true;
2992	break;
2993
2994	case ON_OFF_AUTO_OFF:
2995	whpx->separate_security_domain = false;
2996	break;
2997
2998	case ON_OFF_AUTO_AUTO:
2999	whpx->separate_security_domain = true;
3000	break;
3001	default:
3002	/*
3003	* The value was checked in visit_type_OnOffAuto() above. If
3004	* we get here, then something is wrong in QEMU.
3005	*/
3006	abort();
3007	}
3008	}
3009
3010
3011	void whpx_arch_accel_class_init(ObjectClass *oc)
3012	{
3013	object_class_property_add(oc, "ignore-unknown-msr", "OnOffAuto",
3014	NULL, whpx_set_unknown_msr,
3015	NULL, NULL);
3016	object_class_property_set_description(oc, "ignore-unknown-msr",
3017	"Configure unknown MSR behavior");
3018	object_class_property_add(oc, "intercept-msr-gp", "OnOffAuto",
3019	NULL, whpx_set_intercept_msr_gp,
3020	NULL, NULL);
3021	object_class_property_set_description(oc, "intercept-msr-gp",
3022	"Intercept #GP to log erroring MSR accesses.");
3023	object_class_property_add(oc, "ssd", "OnOffAuto",
3024	NULL, whpx_set_ssd,
3025	NULL, NULL);
3026	object_class_property_set_description(oc, "ssd",
3027	"Separate security domain");
3028	}
3029
3030	int whpx_accel_init(AccelState as, MachineState ms)
3031	{
3032	struct whpx_state *whpx;
3033	int ret;
3034	HRESULT hr;
3035	WHV_CAPABILITY whpx_cap;
3036	UINT32 whpx_cap_size;
3037	WHV_PARTITION_PROPERTY prop;
3038	WHV_CAPABILITY_FEATURES features = {0};
3039	WHV_PROCESSOR_FEATURES_BANKS processor_features;
3040	WHV_PROCESSOR_PERFMON_FEATURES perfmon_features;
3041
3042	UINT32 cpuidExitList[] = {0x0, 0x1, 0x6, 0x7, 0xb, 0xd, 0x14, 0x24, 0x29, 0x1E,
3043	0x40000000, 0x40000001, 0x40000010, 0x80000000, 0x80000001,
3044	0x80000002, 0x80000003, 0x80000004, 0x80000007, 0x80000008,
3045	0x8000000A, 0x80000021, 0x80000022, 0xC0000000, 0xC0000001};
3046
3047	X86MachineState *x86ms = X86_MACHINE(ms);
3048	bool pic_enabled = false;
3049
3050	if (x86ms->pic == ON_OFF_AUTO_ON \|\| x86ms->pic == ON_OFF_AUTO_AUTO) {
3051	pic_enabled = true;
3052	}
3053
3054	whpx = &whpx_global;
3055
3056	if (!init_whp_dispatch()) {
3057	ret = -ENOSYS;
3058	goto error;
3059	}
3060
3061	/* for isapc, disable Hyper-V enlightenments and LAPIC */
3062	if (!strcmp(MACHINE_GET_CLASS(ms)->name, "isapc")) {
3063	whpx->kernel_irqchip_allowed = false;
3064	whpx->kernel_irqchip_required = false;
3065	whpx->hyperv_enlightenments_allowed = false;
3066	whpx->hyperv_enlightenments_required = false;
3067	}
3068
3069	whpx->mem_quota = ms->ram_size;
3070
3071	hr = whp_dispatch.WHvGetCapability(
3072	WHvCapabilityCodeHypervisorPresent, &whpx_cap,
3073	sizeof(whpx_cap), &whpx_cap_size);
3074	if (FAILED(hr) \|\| !whpx_cap.HypervisorPresent) {
3075	error_report("WHPX: No accelerator found, hr=%08lx", hr);
3076	ret = -ENOSPC;
3077	goto error;
3078	}
3079
3080	hr = whp_dispatch.WHvGetCapability(
3081	WHvCapabilityCodeFeatures, &features, sizeof(features), NULL);
3082	if (FAILED(hr)) {
3083	error_report("WHPX: Failed to query capabilities, hr=%08lx", hr);
3084	ret = -EINVAL;
3085	goto error;
3086	}
3087
3088	hr = whp_dispatch.WHvCreatePartition(&whpx->partition);
3089	if (FAILED(hr)) {
3090	error_report("WHPX: Failed to create partition, hr=%08lx", hr);
3091	ret = -EINVAL;
3092	goto error;
3093	}
3094
3095	/*
3096	* Query the XSAVE capability of the partition. Any error here is not
3097	* considered fatal.
3098	*/
3099	hr = whp_dispatch.WHvGetPartitionProperty(
3100	whpx->partition,
3101	WHvPartitionPropertyCodeProcessorXsaveFeatures,
3102	&whpx_xsave_cap,
3103	sizeof(whpx_xsave_cap),
3104	&whpx_cap_size);
3105
3106	/*
3107	* Windows version which don't support this property will return with the
3108	* specific error code.
3109	*/
3110	if (FAILED(hr) && hr != WHV_E_UNKNOWN_PROPERTY) {
3111	error_report("WHPX: Failed to query XSAVE capability, hr=%08lx", hr);
3112	}
3113
3114	memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
3115	prop.ProcessorCount = ms->smp.cpus;
3116	hr = whp_dispatch.WHvSetPartitionProperty(
3117	whpx->partition,
3118	WHvPartitionPropertyCodeProcessorCount,
3119	&prop,
3120	sizeof(WHV_PARTITION_PROPERTY));
3121
3122	if (FAILED(hr)) {
3123	error_report("WHPX: Failed to set partition processor count to %u,"
3124	" hr=%08lx", prop.ProcessorCount, hr);
3125	ret = -EINVAL;
3126	goto error;
3127	}
3128
3129	/* Enable supported performance monitoring capabilities */
3130	hr = whp_dispatch.WHvGetCapability(
3131	WHvCapabilityCodeProcessorPerfmonFeatures, &perfmon_features,
3132	sizeof(WHV_PROCESSOR_PERFMON_FEATURES), &whpx_cap_size);
3133	/*
3134	* Relying on this is a crutch to maintain Windows 10 support.
3135	*
3136	* WHvCapabilityCodeProcessorPerfmonFeatures and
3137	* WHvPartitionPropertyCodeSyntheticProcessorFeaturesBanks
3138	* are implemented starting from Windows Server 2022 (build 20348).
3139	*/
3140	if (FAILED(hr)) {
3141	warn_report("WHPX: Failed to get performance "
3142	"monitoring features, hr=%08lx", hr);
3143	is_modern_os = false;
3144	} else {
3145	hr = whp_dispatch.WHvSetPartitionProperty(
3146	whpx->partition,
3147	WHvPartitionPropertyCodeProcessorPerfmonFeatures,
3148	&perfmon_features,
3149	sizeof(WHV_PROCESSOR_PERFMON_FEATURES));
3150	if (FAILED(hr)) {
3151	error_report("WHPX: Failed to set performance "
3152	"monitoring features, hr=%08lx", hr);
3153	ret = -EINVAL;
3154	goto error;
3155	}
3156	}
3157
3158	/*
3159	* Error out if WHP doesn't support apic emulation and user is requiring
3160	* it.
3161	*/
3162	if (whpx->kernel_irqchip_required && (!features.LocalApicEmulation \|\|
3163	!whp_dispatch.WHvSetVirtualProcessorInterruptControllerState2)) {
3164	error_report("WHPX: kernel irqchip requested, but unavailable. "
3165	"Try without kernel-irqchip or with kernel-irqchip=off");
3166	ret = -EINVAL;
3167	goto error;
3168	}
3169
3170	if (whpx->kernel_irqchip_allowed && !(whpx_is_legacy_os() && pic_enabled
3171	&& !whpx->kernel_irqchip_required) && features.LocalApicEmulation
3172	&& whp_dispatch.WHvSetVirtualProcessorInterruptControllerState2) {
3173	WHV_X64_LOCAL_APIC_EMULATION_MODE mode =
3174	WHvX64LocalApicEmulationModeX2Apic;
3175	hr = whp_dispatch.WHvSetPartitionProperty(
3176	whpx->partition,
3177	WHvPartitionPropertyCodeLocalApicEmulationMode,
3178	&mode,
3179	sizeof(mode));
3180	if (FAILED(hr)) {
3181	error_report("WHPX: Failed to enable kernel irqchip hr=%08lx", hr);
3182	if (whpx->kernel_irqchip_required) {
3183	error_report("WHPX: kernel irqchip requested, but unavailable");
3184	ret = -EINVAL;
3185	goto error;
3186	}
3187	} else {
3188	whpx_irqchip_in_kernel = true;
3189	}
3190	}
3191
3192	/* Set all the supported features, to follow the MSHV example */
3193	memset(&processor_features, 0, sizeof(WHV_PROCESSOR_FEATURES_BANKS));
3194	processor_features.BanksCount = 2;
3195
3196	hr = whp_dispatch.WHvGetCapability(
3197	WHvCapabilityCodeProcessorFeaturesBanks, &processor_features,
3198	sizeof(WHV_PROCESSOR_FEATURES_BANKS), &whpx_cap_size);
3199	if (FAILED(hr)) {
3200	error_report("WHPX: Failed to get processor features, hr=%08lx", hr);
3201	ret = -ENOSPC;
3202	goto error;
3203	}
3204
3205	whpx_rdtsc_cap = processor_features.Bank0.RdtscpSupport;
3206	whpx_invpcid_cap = processor_features.Bank0.InvpcidSupport;
3207
3208	if (whpx_irqchip_in_kernel() && processor_features.Bank1.NestedVirtSupport) {
3209	memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
3210	prop.NestedVirtualization = 1;
3211	hr = whp_dispatch.WHvSetPartitionProperty(
3212	whpx->partition,
3213	WHvPartitionPropertyCodeNestedVirtualization,
3214	&prop,
3215	sizeof(WHV_PARTITION_PROPERTY));
3216	if (FAILED(hr)) {
3217	error_report("WHPX: Failed to enable nested virtualization, hr=%08lx", hr);
3218	ret = -EINVAL;
3219	goto error;
3220	}
3221	}
3222
3223	/*
3224	* The combination of separate security domain off
3225	* and disabling specifically these features results
3226	* in a significant vmexit performance improvement
3227	* by skipping speculative execution mitigations.
3228	*/
3229	if (!whpx->separate_security_domain) {
3230	processor_features.Bank0.IbrsSupport = 0;
3231	processor_features.Bank0.StibpSupport = 0;
3232	processor_features.Bank0.IbpbSupport = 0;
3233	processor_features.Bank0.SsbdSupport = 0;
3234	processor_features.Bank0.IbrsAllSupport = 0;
3235	processor_features.Bank1.PsfdSupport = 0;
3236	memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
3237	prop.SeparateSecurityDomain = 0;
3238	hr = whp_dispatch.WHvSetPartitionProperty(
3239	whpx->partition,
3240	WHvPartitionPropertyCodeSeparateSecurityDomain,
3241	&prop,
3242	sizeof(WHV_PARTITION_PROPERTY));
3243	if (FAILED(hr)) {
3244	error_report("WHPX: failed to unset separate security domain, hr=%08lx", hr);
3245	/* Some old Windows 10 releases didn't have this, so not fatal*/
3246	}
3247	}
3248
3249	hr = whp_dispatch.WHvSetPartitionProperty(
3250	whpx->partition,
3251	WHvPartitionPropertyCodeProcessorFeaturesBanks,
3252	&processor_features,
3253	sizeof(WHV_PROCESSOR_FEATURES_BANKS));
3254	if (FAILED(hr)) {
3255	error_report("WHPX: Failed to set processor features, hr=%08lx", hr);
3256	ret = -EINVAL;
3257	goto error;
3258	}
3259
3260
3261	/* Enable synthetic processor features */
3262	WHV_SYNTHETIC_PROCESSOR_FEATURES_BANKS synthetic_features;
3263	memset(&synthetic_features, 0, sizeof(WHV_SYNTHETIC_PROCESSOR_FEATURES_BANKS));
3264	synthetic_features.BanksCount = 1;
3265
3266	synthetic_features.Bank0.HypervisorPresent = 1;
3267	synthetic_features.Bank0.Hv1 = 1;
3268	synthetic_features.Bank0.FastHypercallOutput = 1;
3269	synthetic_features.Bank0.AccessVpRunTimeReg = 1;
3270	synthetic_features.Bank0.AccessPartitionReferenceCounter = 1;
3271	synthetic_features.Bank0.AccessPartitionReferenceTsc = 1;
3272	synthetic_features.Bank0.AccessHypercallRegs = 1;
3273	synthetic_features.Bank0.AccessFrequencyRegs = 1;
3274	synthetic_features.Bank0.AccessVpIndex = 1;
3275
3276	if (whpx_irqchip_in_kernel()) {
3277	synthetic_features.Bank0.AccessSynicRegs = 1;
3278	synthetic_features.Bank0.AccessSyntheticTimerRegs = 1;
3279	synthetic_features.Bank0.AccessIntrCtrlRegs = 1;
3280	synthetic_features.Bank0.SyntheticClusterIpi = 1;
3281	synthetic_features.Bank0.DirectSyntheticTimers = 1;
3282	synthetic_features.Bank0.AccessGuestIdleReg = 1;
3283	/*
3284	* These technically work without the Hyper-V LAPIC
3285	* but behave oddly for multi-core VMs.
3286	*/
3287	synthetic_features.Bank0.TbFlushHypercalls = 1;
3288	synthetic_features.Bank0.EnableExtendedGvaRangesForFlushVirtualAddressList = 1;
3289	}
3290
3291	if (is_modern_os && whpx->hyperv_enlightenments_allowed) {
3292	whpx->hyperv_enlightenments_enabled = true;
3293	hr = whp_dispatch.WHvSetPartitionProperty(
3294	whpx->partition,
3295	WHvPartitionPropertyCodeSyntheticProcessorFeaturesBanks,
3296	&synthetic_features,
3297	sizeof(WHV_SYNTHETIC_PROCESSOR_FEATURES_BANKS));
3298	if (FAILED(hr)) {
3299	error_report("WHPX: Failed to set synthetic features, hr=%08lx", hr);
3300	ret = -EINVAL;
3301	goto error;
3302	}
3303	} else if (!is_modern_os && whpx->hyperv_enlightenments_required) {
3304	error_report("Hyper-V enlightenments not available on legacy Windows");
3305	ret = -EINVAL;
3306	goto error;
3307	}
3308
3309	memset(&prop, 0, sizeof(WHV_PARTITION_PROPERTY));
3310	prop.X64MsrExitBitmap.UnhandledMsrs = 1;
3311	prop.X64MsrExitBitmap.ApicBaseMsrWrite = 1;
3312
3313	hr = whp_dispatch.WHvSetPartitionProperty(
3314	whpx->partition,
3315	WHvPartitionPropertyCodeX64MsrExitBitmap,
3316	&prop,
3317	sizeof(WHV_PARTITION_PROPERTY));
3318	if (FAILED(hr)) {
3319	error_report("WHPX: Failed to set MSR exit bitmap, hr=%08lx", hr);
3320	ret = -EINVAL;
3321	goto error;
3322	}
3323
3324	hr = whp_dispatch.WHvSetPartitionProperty(
3325	whpx->partition,
3326	WHvPartitionPropertyCodeCpuidExitList,
3327	cpuidExitList,
3328	RTL_NUMBER_OF(cpuidExitList) * sizeof(UINT32));
3329
3330	if (FAILED(hr)) {
3331	error_report("WHPX: Failed to set partition CpuidExitList hr=%08lx",
3332	hr);
3333	ret = -EINVAL;
3334	goto error;
3335	}
3336
3337	/*
3338	* We do not want to intercept any exceptions from the guest,
3339	* until we actually start debugging with gdb.
3340	*/
3341	whpx->exception_exit_bitmap = -1;
3342	hr = whpx_set_exception_exit_bitmap(0);
3343
3344	if (FAILED(hr)) {
3345	error_report("WHPX: Failed to set exception exit bitmap, hr=%08lx", hr);
3346	ret = -EINVAL;
3347	goto error;
3348	}
3349
3350	hr = whp_dispatch.WHvSetupPartition(whpx->partition);
3351	if (FAILED(hr)) {
3352	error_report("WHPX: Failed to setup partition, hr=%08lx", hr);
3353	ret = -EINVAL;
3354	goto error;
3355	}
3356
3357	whpx_memory_init();
3358	whpx_init_emu();
3359
3360	as->gdbstub.sstep_flags = SSTEP_ENABLE;
3361
3362	return 0;
3363
3364	error:
3365
3366	if (NULL != whpx->partition) {
3367	whp_dispatch.WHvDeletePartition(whpx->partition);
3368	whpx->partition = NULL;
3369	}
3370
3371	return ret;
3372	}