@samitouri / QOSamiQemu
Code Commits Issues Pulls Sessions CI/CD
master
master
c 2,375 lines 68.7 KB
Raw
1 /*
2 * QEMU MSHV support
3 *
4 * Copyright Microsoft, Corp. 2025
5 *
6 * Authors: Ziqiao Zhou <ziqiaozhou@microsoft.com>
7 * Magnus Kulke <magnuskulke@microsoft.com>
8 * Jinank Jain <jinankjain@microsoft.com>
9 *
10 * SPDX-License-Identifier: GPL-2.0-or-later
11 */
12
13 #include "qemu/osdep.h"
14 #include "qemu/error-report.h"
15 #include "qemu/memalign.h"
16
17 #include "system/mshv.h"
18 #include "system/mshv_int.h"
19 #include "system/address-spaces.h"
20 #include "linux/mshv.h"
21 #include "hw/hyperv/hvgdk.h"
22 #include "hw/hyperv/hvgdk_mini.h"
23 #include "hw/hyperv/hvhdk_mini.h"
24
25 #include "cpu.h"
26 #include "host-cpu.h"
27 #include "emulate/x86_decode.h"
28 #include "emulate/x86_emu.h"
29 #include "emulate/x86_flags.h"
30
31 #include "accel/accel-cpu-target.h"
32
33 #include "trace-accel_mshv.h"
34 #include "trace.h"
35
36 #include <sys/ioctl.h>
37
38 #define MSHV_MP_STATE_RUNNABLE 0
39 #define MSHV_MP_STATE_UNINITIALIZED 1
40 #define MSHV_MP_STATE_INIT_RECEIVED 2
41 #define MSHV_MP_STATE_HALTED 3
42
43 #define MAX_REGISTER_COUNT (MAX_CONST(ARRAY_SIZE(STANDARD_REGISTER_NAMES), \
44 MAX_CONST(ARRAY_SIZE(SPECIAL_REGISTER_NAMES), \
45 ARRAY_SIZE(FPU_REGISTER_NAMES))))
46
47 static enum hv_register_name STANDARD_REGISTER_NAMES[18] = {
48 HV_X64_REGISTER_RAX,
49 HV_X64_REGISTER_RBX,
50 HV_X64_REGISTER_RCX,
51 HV_X64_REGISTER_RDX,
52 HV_X64_REGISTER_RSI,
53 HV_X64_REGISTER_RDI,
54 HV_X64_REGISTER_RSP,
55 HV_X64_REGISTER_RBP,
56 HV_X64_REGISTER_R8,
57 HV_X64_REGISTER_R9,
58 HV_X64_REGISTER_R10,
59 HV_X64_REGISTER_R11,
60 HV_X64_REGISTER_R12,
61 HV_X64_REGISTER_R13,
62 HV_X64_REGISTER_R14,
63 HV_X64_REGISTER_R15,
64 HV_X64_REGISTER_RIP,
65 HV_X64_REGISTER_RFLAGS,
66 };
67
68 static enum hv_register_name SPECIAL_REGISTER_NAMES[17] = {
69 HV_X64_REGISTER_CS,
70 HV_X64_REGISTER_DS,
71 HV_X64_REGISTER_ES,
72 HV_X64_REGISTER_FS,
73 HV_X64_REGISTER_GS,
74 HV_X64_REGISTER_SS,
75 HV_X64_REGISTER_TR,
76 HV_X64_REGISTER_LDTR,
77 HV_X64_REGISTER_GDTR,
78 HV_X64_REGISTER_IDTR,
79 HV_X64_REGISTER_CR0,
80 HV_X64_REGISTER_CR2,
81 HV_X64_REGISTER_CR3,
82 HV_X64_REGISTER_CR4,
83 HV_X64_REGISTER_CR8,
84 HV_X64_REGISTER_EFER,
85 HV_X64_REGISTER_APIC_BASE,
86 };
87
88 static enum hv_register_name FPU_REGISTER_NAMES[26] = {
89 HV_X64_REGISTER_XMM0,
90 HV_X64_REGISTER_XMM1,
91 HV_X64_REGISTER_XMM2,
92 HV_X64_REGISTER_XMM3,
93 HV_X64_REGISTER_XMM4,
94 HV_X64_REGISTER_XMM5,
95 HV_X64_REGISTER_XMM6,
96 HV_X64_REGISTER_XMM7,
97 HV_X64_REGISTER_XMM8,
98 HV_X64_REGISTER_XMM9,
99 HV_X64_REGISTER_XMM10,
100 HV_X64_REGISTER_XMM11,
101 HV_X64_REGISTER_XMM12,
102 HV_X64_REGISTER_XMM13,
103 HV_X64_REGISTER_XMM14,
104 HV_X64_REGISTER_XMM15,
105 HV_X64_REGISTER_FP_MMX0,
106 HV_X64_REGISTER_FP_MMX1,
107 HV_X64_REGISTER_FP_MMX2,
108 HV_X64_REGISTER_FP_MMX3,
109 HV_X64_REGISTER_FP_MMX4,
110 HV_X64_REGISTER_FP_MMX5,
111 HV_X64_REGISTER_FP_MMX6,
112 HV_X64_REGISTER_FP_MMX7,
113 HV_X64_REGISTER_FP_CONTROL_STATUS,
114 HV_X64_REGISTER_XMM_CONTROL_STATUS,
115 };
116
117 static int set_special_regs(const CPUState *cpu);
118
119 static int get_synic_state(CPUState *cpu)
120 {
121 X86CPU *x86cpu = X86_CPU(cpu);
122 CPUX86State *env = &x86cpu->env;
123 int cpu_fd = mshv_vcpufd(cpu);
124 int ret;
125
126 ret = mshv_get_synthetic_timers(cpu_fd, env->hv_synthetic_timers_state);
127 if (ret < 0) {
128 error_report("failed to get synthetic timers");
129 return -1;
130 }
131
132 /* SIMP/SIEFP can only be read when SynIC is enabled */
133 if (!mshv_synic_enabled(cpu)) {
134 return 0;
135 }
136
137 ret = mshv_get_simp(cpu_fd, env->hv_simp_page);
138 if (ret < 0) {
139 error_report("failed to get simp state");
140 return -1;
141 }
142
143 ret = mshv_get_siefp(cpu_fd, env->hv_siefp_page);
144 if (ret < 0) {
145 error_report("failed to get siefp state");
146 return -1;
147 }
148
149 return 0;
150 }
151
152 static int get_xsave_state(CPUState *cpu)
153 {
154 X86CPU *x86cpu = X86_CPU(cpu);
155 CPUX86State *env = &x86cpu->env;
156 int cpu_fd = mshv_vcpufd(cpu);
157 int ret;
158 void *xsavec_buf;
159 const size_t page = HV_HYP_PAGE_SIZE;
160 size_t xsavec_buf_len = page;
161
162 /* TODO: should properly determine xsavec size based on CPUID */
163 xsavec_buf = qemu_memalign(page, xsavec_buf_len);
164 memset(xsavec_buf, 0, xsavec_buf_len);
165
166 struct mshv_get_set_vp_state args = {
167 .type = MSHV_VP_STATE_XSAVE,
168 .buf_sz = xsavec_buf_len,
169 .buf_ptr = (uintptr_t)xsavec_buf,
170 };
171
172 ret = ioctl(cpu_fd, MSHV_GET_VP_STATE, &args);
173 if (ret < 0) {
174 error_report("failed to get xsave state: %s", strerror(errno));
175 return -errno;
176 }
177
178 ret = decompact_xsave_area(xsavec_buf, xsavec_buf_len, env);
179 g_free(xsavec_buf);
180 if (ret < 0) {
181 error_report("failed to decompact xsave area");
182 return ret;
183 }
184 x86_cpu_xrstor_all_areas(x86cpu, env->xsave_buf, env->xsave_buf_len);
185
186 return 0;
187 }
188
189 static int set_xsave_state(const CPUState *cpu)
190 {
191 X86CPU *x86cpu = X86_CPU(cpu);
192 CPUX86State *env = &x86cpu->env;
193 int cpu_fd = mshv_vcpufd(cpu);
194 int ret;
195 void *xsavec_buf;
196 size_t page = HV_HYP_PAGE_SIZE, xsavec_buf_len;
197
198 /* allocate and populate compacted buffer */
199 xsavec_buf = qemu_memalign(page, page);
200 xsavec_buf_len = page;
201
202 /* save registers to standard format buffer */
203 x86_cpu_xsave_all_areas(x86cpu, env->xsave_buf, env->xsave_buf_len);
204
205 /* store compacted version of xsave area in xsavec_buf */
206 compact_xsave_area(env, xsavec_buf, xsavec_buf_len);
207
208 struct mshv_get_set_vp_state args = {
209 .type = MSHV_VP_STATE_XSAVE,
210 .buf_sz = xsavec_buf_len,
211 .buf_ptr = (uintptr_t)xsavec_buf,
212 };
213
214 ret = ioctl(cpu_fd, MSHV_SET_VP_STATE, &args);
215 g_free(xsavec_buf);
216 if (ret < 0) {
217 error_report("failed to set xsave state: %s", strerror(errno));
218 return -errno;
219 }
220
221 return 0;
222 }
223
224 static void populate_fpu(const hv_register_assoc *assocs, X86CPU *x86cpu)
225 {
226 union hv_register_value value;
227 const union hv_x64_fp_control_status_register *ctrl_status;
228 const union hv_x64_xmm_control_status_register *xmm_ctrl;
229 CPUX86State *env = &x86cpu->env;
230 size_t i, fp_i;
231 bool valid;
232
233 /* first 16 registers are xmm0-xmm15 */
234 for (i = 0; i < 16; i++) {
235 value = assocs[i].value;
236 env->xmm_regs[i].ZMM_Q(0) = value.reg128.low_part;
237 env->xmm_regs[i].ZMM_Q(1) = value.reg128.high_part;
238 }
239
240 /* next 8 registers are fp_mmx0-fp_mmx7 */
241 for (i = 16; i < 24; i++) {
242 fp_i = i - 16;
243 value = assocs[i].value;
244 env->fpregs[fp_i].d.low = value.fp.mantissa;
245 env->fpregs[fp_i].d.high = (value.fp.sign << 15)
246 | (value.fp.biased_exponent & 0x7FFF);
247 }
248
249 /* last two registers are fp_control_status and xmm_control_status */
250 ctrl_status = &assocs[24].value.fp_control_status;
251 env->fpuc = ctrl_status->fp_control;
252
253 env->fpus = ctrl_status->fp_status & ~0x3800;
254 /* bits 11,12,13 are the top of stack pointer */
255 env->fpstt = (ctrl_status->fp_status >> 11) & 0x7;
256
257 for (i = 0; i < 8; i++) {
258 valid = ctrl_status->fp_tag & (1 << i);
259 env->fptags[i] = valid ? 0 : 1;
260 }
261
262 env->fpop = ctrl_status->last_fp_op;
263 env->fpip = ctrl_status->last_fp_rip;
264
265 xmm_ctrl = &assocs[25].value.xmm_control_status;
266 env->mxcsr = xmm_ctrl->xmm_status_control;
267 env->fpdp = xmm_ctrl->last_fp_rdp;
268 }
269
270 static int get_fpu(CPUState *cpu)
271 {
272 struct hv_register_assoc assocs[ARRAY_SIZE(FPU_REGISTER_NAMES)];
273 int ret;
274 X86CPU *x86cpu = X86_CPU(cpu);
275 size_t n_regs = ARRAY_SIZE(FPU_REGISTER_NAMES);
276
277 for (size_t i = 0; i < n_regs; i++) {
278 assocs[i].name = FPU_REGISTER_NAMES[i];
279 }
280 ret = mshv_get_generic_regs(cpu, assocs, n_regs);
281 if (ret < 0) {
282 error_report("failed to get special registers");
283 return -errno;
284 }
285
286 populate_fpu(assocs, x86cpu);
287
288 return 0;
289 }
290
291 static int get_xc_reg(CPUState *cpu)
292 {
293 int ret;
294 X86CPU *x86cpu = X86_CPU(cpu);
295 CPUX86State *env = &x86cpu->env;
296 struct hv_register_assoc assocs[1];
297
298 assocs[0].name = HV_X64_REGISTER_XFEM;
299
300 ret = mshv_get_generic_regs(cpu, assocs, 1);
301 if (ret < 0) {
302 error_report("failed to get xcr0");
303 return -1;
304 }
305 env->xcr0 = assocs[0].value.reg64;
306
307 return 0;
308 }
309
310 static enum hv_register_name NON_VP_PAGE_REGISTER_NAMES[6] = {
311 HV_X64_REGISTER_TR,
312 HV_X64_REGISTER_LDTR,
313 HV_X64_REGISTER_GDTR,
314 HV_X64_REGISTER_IDTR,
315 HV_X64_REGISTER_CR2,
316 HV_X64_REGISTER_APIC_BASE,
317 };
318
319 static int translate_gva(const CPUState *cpu, uint64_t gva, uint64_t *gpa,
320 uint64_t flags)
321 {
322 int ret;
323 int cpu_fd = mshv_vcpufd(cpu);
324 int vp_index = cpu->cpu_index;
325
326 hv_input_translate_virtual_address in = { 0 };
327 hv_output_translate_virtual_address out = { 0 };
328 struct mshv_root_hvcall args = {0};
329 uint64_t gva_page = gva >> HV_HYP_PAGE_SHIFT;
330
331 in.vp_index = vp_index;
332 in.control_flags = flags;
333 in.gva_page = gva_page;
334
335 /* create the hvcall envelope */
336 args.code = HVCALL_TRANSLATE_VIRTUAL_ADDRESS;
337 args.in_sz = sizeof(in);
338 args.in_ptr = (uint64_t) &in;
339 args.out_sz = sizeof(out);
340 args.out_ptr = (uint64_t) &out;
341
342 /* perform the call */
343 ret = mshv_hvcall(cpu_fd, &args);
344 if (ret < 0) {
345 error_report("Failed to invoke gva->gpa translation");
346 return -errno;
347 }
348
349 if (out.translation_result.result_code != HV_TRANSLATE_GVA_SUCCESS) {
350 error_report("Failed to translate gva (" TARGET_FMT_lx ") to gpa", gva);
351 return -1;
352 }
353
354 *gpa = ((out.gpa_page << HV_HYP_PAGE_SHIFT)
355 | (gva & ~(uint64_t)HV_HYP_PAGE_MASK));
356
357 return 0;
358 }
359
360 int mshv_set_generic_regs(const CPUState *cpu, const hv_register_assoc *assocs,
361 size_t n_regs)
362 {
363 int cpu_fd = mshv_vcpufd(cpu);
364 int vp_index = cpu->cpu_index;
365 size_t in_sz, assocs_sz;
366 hv_input_set_vp_registers *in = cpu->accel->hvcall_args.input_page;
367 struct mshv_root_hvcall args = {0};
368 int ret;
369
370 /* find out the size of the struct w/ a flexible array at the tail */
371 assocs_sz = n_regs * sizeof(hv_register_assoc);
372 in_sz = sizeof(hv_input_set_vp_registers) + assocs_sz;
373
374 /* fill the input struct */
375 memset(in, 0, sizeof(hv_input_set_vp_registers));
376 in->vp_index = vp_index;
377 memcpy(in->elements, assocs, assocs_sz);
378
379 /* create the hvcall envelope */
380 args.code = HVCALL_SET_VP_REGISTERS;
381 args.in_sz = in_sz;
382 args.in_ptr = (uint64_t) in;
383 args.reps = (uint16_t) n_regs;
384
385 /* perform the call */
386 ret = mshv_hvcall(cpu_fd, &args);
387 if (ret < 0) {
388 error_report("Failed to set registers");
389 return -1;
390 }
391
392 /* assert we set all registers */
393 if (args.reps != n_regs) {
394 error_report("Failed to set registers: expected %zu elements"
395 ", got %u", n_regs, args.reps);
396 return -1;
397 }
398
399 return 0;
400 }
401
402 int mshv_get_generic_regs(CPUState *cpu, hv_register_assoc *assocs,
403 size_t n_regs)
404 {
405 int cpu_fd = mshv_vcpufd(cpu);
406 int vp_index = cpu->cpu_index;
407 hv_input_get_vp_registers *in = cpu->accel->hvcall_args.input_page;
408 hv_register_value *values = cpu->accel->hvcall_args.output_page;
409 size_t in_sz, names_sz, values_sz;
410 int i, ret;
411 struct mshv_root_hvcall args = {0};
412
413 /* find out the size of the struct w/ a flexible array at the tail */
414 names_sz = n_regs * sizeof(hv_register_name);
415 in_sz = sizeof(hv_input_get_vp_registers) + names_sz;
416
417 /* fill the input struct */
418 memset(in, 0, sizeof(hv_input_get_vp_registers));
419 in->vp_index = vp_index;
420 for (i = 0; i < n_regs; i++) {
421 in->names[i] = assocs[i].name;
422 }
423
424 /* determine size of value output buffer */
425 values_sz = n_regs * sizeof(union hv_register_value);
426
427 /* create the hvcall envelope */
428 args.code = HVCALL_GET_VP_REGISTERS;
429 args.in_sz = in_sz;
430 args.in_ptr = (uint64_t) in;
431 args.out_sz = values_sz;
432 args.out_ptr = (uint64_t) values;
433 args.reps = (uint16_t) n_regs;
434
435 /* perform the call */
436 ret = mshv_hvcall(cpu_fd, &args);
437 if (ret < 0) {
438 error_report("Failed to retrieve registers");
439 return -1;
440 }
441
442 /* assert we got all registers */
443 if (args.reps != n_regs) {
444 error_report("Failed to retrieve registers: expected %zu elements"
445 ", got %u", n_regs, args.reps);
446 return -1;
447 }
448
449 /* copy values into assoc */
450 for (i = 0; i < n_regs; i++) {
451 assocs[i].value = values[i];
452 }
453
454 return 0;
455 }
456
457 static int set_standard_regs(const CPUState *cpu)
458 {
459 X86CPU *x86cpu = X86_CPU(cpu);
460 CPUX86State *env = &x86cpu->env;
461 hv_register_assoc assocs[ARRAY_SIZE(STANDARD_REGISTER_NAMES)];
462 int ret;
463 size_t n_regs = ARRAY_SIZE(STANDARD_REGISTER_NAMES);
464
465 /* set names */
466 for (size_t i = 0; i < ARRAY_SIZE(STANDARD_REGISTER_NAMES); i++) {
467 assocs[i].name = STANDARD_REGISTER_NAMES[i];
468 }
469 assocs[0].value.reg64 = env->regs[R_EAX];
470 assocs[1].value.reg64 = env->regs[R_EBX];
471 assocs[2].value.reg64 = env->regs[R_ECX];
472 assocs[3].value.reg64 = env->regs[R_EDX];
473 assocs[4].value.reg64 = env->regs[R_ESI];
474 assocs[5].value.reg64 = env->regs[R_EDI];
475 assocs[6].value.reg64 = env->regs[R_ESP];
476 assocs[7].value.reg64 = env->regs[R_EBP];
477 assocs[8].value.reg64 = env->regs[R_R8];
478 assocs[9].value.reg64 = env->regs[R_R9];
479 assocs[10].value.reg64 = env->regs[R_R10];
480 assocs[11].value.reg64 = env->regs[R_R11];
481 assocs[12].value.reg64 = env->regs[R_R12];
482 assocs[13].value.reg64 = env->regs[R_R13];
483 assocs[14].value.reg64 = env->regs[R_R14];
484 assocs[15].value.reg64 = env->regs[R_R15];
485 assocs[16].value.reg64 = env->eip;
486 lflags_to_rflags(env);
487 assocs[17].value.reg64 = env->eflags;
488
489 ret = mshv_set_generic_regs(cpu, assocs, n_regs);
490 if (ret < 0) {
491 error_report("failed to set standard registers");
492 return -errno;
493 }
494 return 0;
495 }
496
497 static void mshv_set_standard_regs_vp_page(CPUState *cpu)
498 {
499 X86CPU *x86cpu = X86_CPU(cpu);
500 CPUX86State *env = &x86cpu->env;
501
502 env->regs_page->rax = env->regs[R_EAX];
503 env->regs_page->rbx = env->regs[R_EBX];
504 env->regs_page->rcx = env->regs[R_ECX];
505 env->regs_page->rdx = env->regs[R_EDX];
506 env->regs_page->rsi = env->regs[R_ESI];
507 env->regs_page->rdi = env->regs[R_EDI];
508 env->regs_page->rsp = env->regs[R_ESP];
509 env->regs_page->rbp = env->regs[R_EBP];
510 env->regs_page->r8 = env->regs[R_R8];
511 env->regs_page->r9 = env->regs[R_R9];
512 env->regs_page->r10 = env->regs[R_R10];
513 env->regs_page->r11 = env->regs[R_R11];
514 env->regs_page->r12 = env->regs[R_R12];
515 env->regs_page->r13 = env->regs[R_R13];
516 env->regs_page->r14 = env->regs[R_R14];
517 env->regs_page->r15 = env->regs[R_R15];
518 env->regs_page->rip = env->eip;
519 lflags_to_rflags(env);
520 env->regs_page->rflags = env->eflags;
521
522 env->regs_page->dirty |= (1u << HV_X64_REGISTER_CLASS_GENERAL)
523 | (1u << HV_X64_REGISTER_CLASS_IP)
524 | (1u << HV_X64_REGISTER_CLASS_FLAGS);
525 }
526
527 static int store_regs(CPUState *cpu)
528 {
529 X86CPU *x86cpu = X86_CPU(cpu);
530 CPUX86State *env = &x86cpu->env;
531 int ret;
532
533 /* Use register vp page to optimize registers access */
534 if (env->regs_page && env->regs_page->isvalid != 0) {
535 mshv_set_standard_regs_vp_page(cpu);
536 } else {
537 ret = set_standard_regs(cpu);
538 if (ret < 0) {
539 return ret;
540 }
541 }
542
543 ret = set_special_regs(cpu);
544 if (ret < 0) {
545 error_report("Failed to store speical registers");
546 return ret;
547 }
548
549 return 0;
550 }
551
552 static void populate_standard_regs(const hv_register_assoc *assocs,
553 CPUX86State *env)
554 {
555 env->regs[R_EAX] = assocs[0].value.reg64;
556 env->regs[R_EBX] = assocs[1].value.reg64;
557 env->regs[R_ECX] = assocs[2].value.reg64;
558 env->regs[R_EDX] = assocs[3].value.reg64;
559 env->regs[R_ESI] = assocs[4].value.reg64;
560 env->regs[R_EDI] = assocs[5].value.reg64;
561 env->regs[R_ESP] = assocs[6].value.reg64;
562 env->regs[R_EBP] = assocs[7].value.reg64;
563 env->regs[R_R8] = assocs[8].value.reg64;
564 env->regs[R_R9] = assocs[9].value.reg64;
565 env->regs[R_R10] = assocs[10].value.reg64;
566 env->regs[R_R11] = assocs[11].value.reg64;
567 env->regs[R_R12] = assocs[12].value.reg64;
568 env->regs[R_R13] = assocs[13].value.reg64;
569 env->regs[R_R14] = assocs[14].value.reg64;
570 env->regs[R_R15] = assocs[15].value.reg64;
571
572 env->eip = assocs[16].value.reg64;
573 env->eflags = assocs[17].value.reg64;
574 rflags_to_lflags(env);
575 }
576
577 static int get_standard_regs(CPUState *cpu)
578 {
579 struct hv_register_assoc assocs[ARRAY_SIZE(STANDARD_REGISTER_NAMES)];
580 int ret;
581 X86CPU *x86cpu = X86_CPU(cpu);
582 CPUX86State *env = &x86cpu->env;
583 size_t n_regs = ARRAY_SIZE(STANDARD_REGISTER_NAMES);
584
585 for (size_t i = 0; i < n_regs; i++) {
586 assocs[i].name = STANDARD_REGISTER_NAMES[i];
587 }
588 ret = mshv_get_generic_regs(cpu, assocs, n_regs);
589 if (ret < 0) {
590 error_report("failed to get standard registers");
591 return -1;
592 }
593
594 populate_standard_regs(assocs, env);
595 return 0;
596 }
597
598 static inline void populate_segment_reg(const hv_x64_segment_register *hv_seg,
599 SegmentCache *seg)
600 {
601 memset(seg, 0, sizeof(SegmentCache));
602
603 seg->base = hv_seg->base;
604 seg->limit = hv_seg->limit;
605 seg->selector = hv_seg->selector;
606
607 seg->flags = (hv_seg->segment_type << DESC_TYPE_SHIFT)
608 | (hv_seg->present * DESC_P_MASK)
609 | (hv_seg->descriptor_privilege_level << DESC_DPL_SHIFT)
610 | (hv_seg->_default << DESC_B_SHIFT)
611 | (hv_seg->non_system_segment * DESC_S_MASK)
612 | (hv_seg->_long << DESC_L_SHIFT)
613 | (hv_seg->granularity * DESC_G_MASK)
614 | (hv_seg->available * DESC_AVL_MASK);
615
616 }
617
618 static inline void populate_table_reg(const hv_x64_table_register *hv_seg,
619 SegmentCache *tbl)
620 {
621 memset(tbl, 0, sizeof(SegmentCache));
622
623 tbl->base = hv_seg->base;
624 tbl->limit = hv_seg->limit;
625 }
626
627 static void populate_special_regs(const hv_register_assoc *assocs,
628 X86CPU *x86cpu)
629 {
630 CPUX86State *env = &x86cpu->env;
631
632 populate_segment_reg(&assocs[0].value.segment, &env->segs[R_CS]);
633 populate_segment_reg(&assocs[1].value.segment, &env->segs[R_DS]);
634 populate_segment_reg(&assocs[2].value.segment, &env->segs[R_ES]);
635 populate_segment_reg(&assocs[3].value.segment, &env->segs[R_FS]);
636 populate_segment_reg(&assocs[4].value.segment, &env->segs[R_GS]);
637 populate_segment_reg(&assocs[5].value.segment, &env->segs[R_SS]);
638
639 populate_segment_reg(&assocs[6].value.segment, &env->tr);
640 populate_segment_reg(&assocs[7].value.segment, &env->ldt);
641
642 populate_table_reg(&assocs[8].value.table, &env->gdt);
643 populate_table_reg(&assocs[9].value.table, &env->idt);
644
645 env->cr[0] = assocs[10].value.reg64;
646 env->cr[2] = assocs[11].value.reg64;
647 env->cr[3] = assocs[12].value.reg64;
648 env->cr[4] = assocs[13].value.reg64;
649
650 cpu_set_apic_tpr(x86cpu->apic_state, assocs[14].value.reg64);
651 env->efer = assocs[15].value.reg64;
652 cpu_set_apic_base(x86cpu->apic_state, assocs[16].value.reg64);
653 }
654
655 static void mshv_get_standard_regs_vp_page(CPUState *cpu)
656 {
657 X86CPU *x86cpu = X86_CPU(cpu);
658 CPUX86State *env = &x86cpu->env;
659
660 /* General Purpose Registers */
661 env->regs[R_EAX] = env->regs_page->rax;
662 env->regs[R_EBX] = env->regs_page->rbx;
663 env->regs[R_ECX] = env->regs_page->rcx;
664 env->regs[R_EDX] = env->regs_page->rdx;
665 env->regs[R_ESI] = env->regs_page->rsi;
666 env->regs[R_EDI] = env->regs_page->rdi;
667 env->regs[R_ESP] = env->regs_page->rsp;
668 env->regs[R_EBP] = env->regs_page->rbp;
669 env->regs[R_R8] = env->regs_page->r8;
670 env->regs[R_R9] = env->regs_page->r9;
671 env->regs[R_R10] = env->regs_page->r10;
672 env->regs[R_R11] = env->regs_page->r11;
673 env->regs[R_R12] = env->regs_page->r12;
674 env->regs[R_R13] = env->regs_page->r13;
675 env->regs[R_R14] = env->regs_page->r14;
676 env->regs[R_R15] = env->regs_page->r15;
677
678 env->eip = env->regs_page->rip;
679 env->eflags = env->regs_page->rflags;
680 rflags_to_lflags(env);
681 }
682
683 static int mshv_get_special_regs_vp_page(CPUState *cpu)
684 {
685 X86CPU *x86cpu = X86_CPU(cpu);
686 CPUX86State *env = &x86cpu->env;
687 struct hv_register_assoc assocs[ARRAY_SIZE(NON_VP_PAGE_REGISTER_NAMES)];
688 int ret;
689 size_t n_regs = ARRAY_SIZE(NON_VP_PAGE_REGISTER_NAMES);
690 hv_x64_segment_register seg;
691
692 /* Populate special registers that are in the VP register page */
693 env->cr[0] = env->regs_page->cr0;
694 env->cr[3] = env->regs_page->cr3;
695 env->cr[4] = env->regs_page->cr4;
696 env->efer = env->regs_page->efer;
697 cpu_set_apic_tpr(x86cpu->apic_state, env->regs_page->cr8);
698
699 /* Segment Registers - copy from packed struct to avoid unaligned access */
700 memcpy(&seg, &env->regs_page->es, sizeof(hv_x64_segment_register));
701 populate_segment_reg(&seg, &env->segs[R_ES]);
702 memcpy(&seg, &env->regs_page->cs, sizeof(hv_x64_segment_register));
703 populate_segment_reg(&seg, &env->segs[R_CS]);
704 memcpy(&seg, &env->regs_page->ss, sizeof(hv_x64_segment_register));
705 populate_segment_reg(&seg, &env->segs[R_SS]);
706 memcpy(&seg, &env->regs_page->ds, sizeof(hv_x64_segment_register));
707 populate_segment_reg(&seg, &env->segs[R_DS]);
708 memcpy(&seg, &env->regs_page->fs, sizeof(hv_x64_segment_register));
709 populate_segment_reg(&seg, &env->segs[R_FS]);
710 memcpy(&seg, &env->regs_page->gs, sizeof(hv_x64_segment_register));
711 populate_segment_reg(&seg, &env->segs[R_GS]);
712
713 /* The rest of the special registers that are not in the VP register page */
714 for (size_t i = 0; i < n_regs; i++) {
715 assocs[i].name = NON_VP_PAGE_REGISTER_NAMES[i];
716 }
717
718 ret = mshv_get_generic_regs(cpu, assocs, n_regs);
719 if (ret < 0) {
720 error_report("failed to get non-vp-page special registers");
721 return -1;
722 }
723
724 /* Non-VP page registers - TR, LDTR, GDTR, IDTR, CR2, APIC_BASE */
725 populate_segment_reg(&assocs[0].value.segment, &env->tr);
726 populate_segment_reg(&assocs[1].value.segment, &env->ldt);
727
728 populate_table_reg(&assocs[2].value.table, &env->gdt);
729 populate_table_reg(&assocs[3].value.table, &env->idt);
730 env->cr[2] = assocs[4].value.reg64;
731
732 cpu_set_apic_base(x86cpu->apic_state, assocs[5].value.reg64);
733
734 return ret;
735 }
736
737 static int mshv_get_registers_vp_page(CPUState *cpu)
738 {
739 int ret;
740
741 /* General Purpose Registers */
742 mshv_get_standard_regs_vp_page(cpu);
743
744 /* Special Registers - makes a hypercall */
745 ret = mshv_get_special_regs_vp_page(cpu);
746 if (ret < 0) {
747 error_report("failed to get special registers for vp page");
748 return -1;
749 }
750
751 return 0;
752 }
753
754
755 static int get_special_regs(CPUState *cpu)
756 {
757 struct hv_register_assoc assocs[ARRAY_SIZE(SPECIAL_REGISTER_NAMES)];
758 int ret;
759 X86CPU *x86cpu = X86_CPU(cpu);
760 size_t n_regs = ARRAY_SIZE(SPECIAL_REGISTER_NAMES);
761
762 for (size_t i = 0; i < n_regs; i++) {
763 assocs[i].name = SPECIAL_REGISTER_NAMES[i];
764 }
765 ret = mshv_get_generic_regs(cpu, assocs, n_regs);
766 if (ret < 0) {
767 error_report("failed to get special registers");
768 return -errno;
769 }
770
771 populate_special_regs(assocs, x86cpu);
772 return 0;
773 }
774
775 static int load_regs(CPUState *cpu)
776 {
777 X86CPU *x86_cpu = X86_CPU(cpu);
778 CPUX86State *env = &x86_cpu->env;
779 int ret;
780
781 /* Use register vp page to optimize registers access */
782 if (env->regs_page && env->regs_page->isvalid != 0) {
783 ret = mshv_get_registers_vp_page(cpu);
784 return ret;
785 }
786
787 ret = get_standard_regs(cpu);
788 if (ret < 0) {
789 return ret;
790 }
791
792 ret = get_special_regs(cpu);
793 if (ret < 0) {
794 return ret;
795 }
796
797 return 0;
798 }
799
800 static int get_vcpu_events(CPUState *cpu)
801 {
802 X86CPU *x86cpu = X86_CPU(cpu);
803 CPUX86State *env = &x86cpu->env;
804 struct hv_register_assoc assocs[] = {
805 { .name = HV_REGISTER_PENDING_INTERRUPTION },
806 { .name = HV_REGISTER_INTERRUPT_STATE },
807 { .name = HV_REGISTER_PENDING_EVENT0 },
808 };
809 union hv_x64_pending_interruption_register pending_int;
810 union hv_x64_interrupt_state_register int_state;
811 union hv_x64_pending_exception_event pending_exc;
812 int ret;
813
814 ret = mshv_get_generic_regs(cpu, assocs, ARRAY_SIZE(assocs));
815 if (ret < 0) {
816 error_report("failed to get vcpu event registers");
817 return -1;
818 }
819
820 pending_int.as_uint64 = assocs[0].value.reg64;
821 int_state.as_uint64 = assocs[1].value.reg64;
822 pending_exc = assocs[2].value.pending_exception_event;
823
824 /* Clear previous state. injected ints/excs are blanked w/ -1 */
825 env->interrupt_injected = -1;
826 env->soft_interrupt = 0;
827 env->exception_injected = 0;
828 env->exception_pending = 0;
829 env->exception_nr = -1;
830 env->has_error_code = 0;
831 env->error_code = 0;
832 env->exception_has_payload = 0;
833 env->exception_payload = 0;
834 env->nmi_injected = 0;
835
836 if (pending_int.interruption_pending) {
837 switch (pending_int.interruption_type) {
838 case MSHV_HV_INTERRUPTION_TYPE_EXT_INT:
839 env->interrupt_injected = pending_int.interruption_vector;
840 break;
841 case MSHV_HV_INTERRUPTION_TYPE_NMI:
842 env->nmi_injected = 1;
843 break;
844 case MSHV_HV_INTERRUPTION_TYPE_HW_EXC:
845 env->exception_injected = 1;
846 env->exception_nr = pending_int.interruption_vector;
847 env->has_error_code = pending_int.deliver_error_code;
848 env->error_code = pending_int.error_code;
849 break;
850 case MSHV_HV_INTERRUPTION_TYPE_SW_INT:
851 env->interrupt_injected = pending_int.interruption_vector;
852 env->soft_interrupt = 1;
853 break;
854 case MSHV_HV_INTERRUPTION_TYPE_SW_EXC:
855 case MSHV_HV_INTERRUPTION_TYPE_PRIV_SW_EXC:
856 env->exception_injected = 1;
857 env->exception_nr = pending_int.interruption_vector;
858 env->has_error_code = pending_int.deliver_error_code;
859 env->error_code = pending_int.error_code;
860 break;
861 default:
862 error_report("unknown interruption type %u",
863 pending_int.interruption_type);
864 return -EINVAL;
865 }
866 }
867
868 /* disabled for one instr after STI, MOV/POP SS, see hvf_store_events() */
869 if (int_state.interrupt_shadow) {
870 env->hflags |= HF_INHIBIT_IRQ_MASK;
871 } else {
872 env->hflags &= ~HF_INHIBIT_IRQ_MASK;
873 }
874
875 /* see kvm_get_vcpu_events(), hvf_store_events() */
876 if (int_state.nmi_masked) {
877 env->hflags2 |= HF2_NMI_MASK;
878 } else {
879 env->hflags2 &= ~HF2_NMI_MASK;
880 }
881
882 /* HV_REGISTER_PENDING_EVENT0: pending exception not yet injected */
883 if (pending_exc.event_pending) {
884 env->exception_pending = 1;
885 env->exception_nr = pending_exc.vector;
886 env->has_error_code = pending_exc.deliver_error_code;
887 env->error_code = pending_exc.error_code;
888 env->exception_has_payload = (pending_exc.exception_parameter != 0);
889 env->exception_payload = pending_exc.exception_parameter;
890 }
891
892 /*
893 * Ignoring HV_REGISTER_PENDING_EVENT1, virtualization fault events, MSHV
894 * does not support nested virtualization.
895 */
896
897 return 0;
898 }
899
900 static int set_vcpu_events(const CPUState *cpu)
901 {
902 X86CPU *x86cpu = X86_CPU(cpu);
903 CPUX86State *env = &x86cpu->env;
904 union hv_x64_pending_interruption_register pending_int = { 0 };
905 union hv_x64_interrupt_state_register int_state = { 0 };
906 union hv_x64_pending_exception_event pending_exc = { 0 };
907 struct hv_register_assoc assocs[3];
908 int ret;
909
910 /* build pending_int from CPUX86State */
911 if (env->exception_injected) {
912 pending_int.interruption_pending = 1;
913 pending_int.interruption_type = MSHV_HV_INTERRUPTION_TYPE_HW_EXC;
914 pending_int.interruption_vector = env->exception_nr;
915 pending_int.deliver_error_code = env->has_error_code;
916 pending_int.error_code = env->error_code;
917 } else if (env->nmi_injected) {
918 pending_int.interruption_pending = 1;
919 pending_int.interruption_type = MSHV_HV_INTERRUPTION_TYPE_NMI;
920 pending_int.interruption_vector = EXCP02_NMI;
921 } else if (env->interrupt_injected >= 0) {
922 pending_int.interruption_pending = 1;
923 pending_int.interruption_type = env->soft_interrupt
924 ? MSHV_HV_INTERRUPTION_TYPE_SW_INT
925 : MSHV_HV_INTERRUPTION_TYPE_EXT_INT;
926 pending_int.interruption_vector = env->interrupt_injected;
927 }
928
929 /* build int_state, normalize to bool */
930 int_state.interrupt_shadow = !!(env->hflags & HF_INHIBIT_IRQ_MASK);
931 int_state.nmi_masked = !!(env->hflags2 & HF2_NMI_MASK);
932
933 /* build pending_exc */
934 if (env->exception_pending) {
935 pending_exc.event_pending = 1;
936 pending_exc.vector = env->exception_nr;
937 pending_exc.deliver_error_code = env->has_error_code;
938 pending_exc.error_code = env->error_code;
939 pending_exc.exception_parameter = env->exception_payload;
940 }
941
942 assocs[0].name = HV_REGISTER_PENDING_INTERRUPTION;
943 assocs[0].value.reg64 = pending_int.as_uint64;
944 assocs[1].name = HV_REGISTER_INTERRUPT_STATE;
945 assocs[1].value.reg64 = int_state.as_uint64;
946 assocs[2].name = HV_REGISTER_PENDING_EVENT0;
947 assocs[2].value.pending_exception_event = pending_exc;
948
949 ret = mshv_set_generic_regs(cpu, assocs, ARRAY_SIZE(assocs));
950 if (ret < 0) {
951 error_report("failed to set vcpu event registers");
952 return -1;
953 }
954
955 return 0;
956 }
957
958 static int get_mp_state(CPUState *cpu)
959 {
960 X86CPU *x86cpu = X86_CPU(cpu);
961 CPUX86State *env = &x86cpu->env;
962 struct hv_register_assoc assoc = {
963 .name = HV_REGISTER_INTERNAL_ACTIVITY_STATE,
964 };
965 union hv_internal_activity_register activity;
966 int ret;
967
968 ret = mshv_get_generic_regs(cpu, &assoc, 1);
969 if (ret < 0) {
970 error_report("failed to get internal activity state");
971 return -1;
972 }
973
974 activity.as_uint64 = assoc.value.reg64;
975
976 /*
977 * map MSHV activity state to KVM mp_state values, which are used as the
978 * shared representation in env->mp_state and serialized by vmstate_x86_cpu.
979 */
980
981 if (activity.startup_suspend) {
982 env->mp_state = MSHV_MP_STATE_UNINITIALIZED;
983 } else if (activity.halt_suspend) {
984 env->mp_state = MSHV_MP_STATE_HALTED;
985 } else {
986 env->mp_state = MSHV_MP_STATE_RUNNABLE;
987 }
988
989 cpu->halted = (env->mp_state == MSHV_MP_STATE_HALTED);
990
991 return 0;
992 }
993
994 int mshv_arch_set_mp_state(const CPUState *cpu)
995 {
996 X86CPU *x86cpu = X86_CPU(cpu);
997 CPUX86State *env = &x86cpu->env;
998 union hv_internal_activity_register activity = { 0 };
999 struct hv_register_assoc assoc = {
1000 .name = HV_REGISTER_INTERNAL_ACTIVITY_STATE,
1001 };
1002 int ret;
1003
1004 switch (env->mp_state) {
1005 case MSHV_MP_STATE_HALTED:
1006 activity.halt_suspend = 1;
1007 break;
1008 case MSHV_MP_STATE_UNINITIALIZED:
1009 case MSHV_MP_STATE_INIT_RECEIVED:
1010 activity.startup_suspend = 1;
1011 break;
1012 case MSHV_MP_STATE_RUNNABLE:
1013 default:
1014 break;
1015 }
1016
1017 assoc.value.reg64 = activity.as_uint64;
1018
1019 ret = mshv_set_generic_regs(cpu, &assoc, 1);
1020 if (ret < 0) {
1021 error_report("failed to set internal activity state");
1022 return -1;
1023 }
1024
1025 return 0;
1026 }
1027
1028 static int update_hflags(CPUState *cpu)
1029 {
1030 X86CPU *x86cpu = X86_CPU(cpu);
1031 CPUX86State *env = &x86cpu->env;
1032
1033 x86_update_hflags(env);
1034
1035 return 0;
1036 }
1037
1038 int mshv_arch_load_vcpu_state(CPUState *cpu)
1039 {
1040 int ret;
1041
1042 ret = get_standard_regs(cpu);
1043 if (ret < 0) {
1044 return ret;
1045 }
1046
1047 ret = get_special_regs(cpu);
1048 if (ret < 0) {
1049 return ret;
1050 }
1051
1052 /* INVARIANT: hflags are derived from regs+sregs, need to get both first */
1053 update_hflags(cpu);
1054
1055 ret = get_xc_reg(cpu);
1056 if (ret < 0) {
1057 return ret;
1058 }
1059
1060 ret = get_xsave_state(cpu);
1061 if (ret < 0) {
1062 return ret;
1063 }
1064
1065 ret = mshv_get_lapic(cpu);
1066 if (ret < 0) {
1067 return ret;
1068 }
1069
1070 ret = mshv_get_msrs(cpu);
1071 if (ret < 0) {
1072 return ret;
1073 }
1074
1075 ret = get_fpu(cpu);
1076 if (ret < 0) {
1077 return ret;
1078 }
1079
1080 ret = get_synic_state(cpu);
1081 if (ret < 0) {
1082 return ret;
1083 }
1084
1085 ret = get_vcpu_events(cpu);
1086 if (ret < 0) {
1087 return ret;
1088 }
1089
1090 ret = get_mp_state(cpu);
1091 if (ret < 0) {
1092 return ret;
1093 }
1094
1095 return 0;
1096 }
1097
1098 static void add_cpuid_entry(GList **cpuid_entries,
1099 uint32_t function, uint32_t index,
1100 uint32_t eax, uint32_t ebx,
1101 uint32_t ecx, uint32_t edx)
1102 {
1103 struct hv_cpuid_entry *entry;
1104
1105 entry = g_malloc0(sizeof(struct hv_cpuid_entry));
1106 entry->function = function;
1107 entry->index = index;
1108 entry->eax = eax;
1109 entry->ebx = ebx;
1110 entry->ecx = ecx;
1111 entry->edx = edx;
1112
1113 *cpuid_entries = g_list_append(*cpuid_entries, entry);
1114 }
1115
1116 static void collect_cpuid_entries(const CPUState *cpu, GList **cpuid_entries)
1117 {
1118 X86CPU *x86_cpu = X86_CPU(cpu);
1119 CPUX86State *env = &x86_cpu->env;
1120 uint32_t eax, ebx, ecx, edx;
1121 uint32_t leaf, subleaf;
1122 uint32_t max_basic_leaf, max_extended_leaf;
1123 uint32_t max_subleaf = 0x20;
1124 uint32_t leaves_with_subleaves[] = {0x04, 0x07, 0x0d, 0x0f, 0x10};
1125 int n_subleaf_leaves = ARRAY_SIZE(leaves_with_subleaves);
1126
1127 /* Get maximum basic and and extended CPUID leaves */
1128 cpu_x86_cpuid(env, 0, 0, &max_basic_leaf, &ebx, &ecx, &edx);
1129 cpu_x86_cpuid(env, 0x80000000, 0, &max_extended_leaf, &ebx, &ecx, &edx);
1130
1131 /* Collect basic leaves (0x0 to max_basic_leaf) */
1132 for (leaf = 0; leaf <= max_basic_leaf; leaf++) {
1133 bool has_subleaves = false;
1134 for (int i = 0; i < n_subleaf_leaves; i++) {
1135 if (leaf == leaves_with_subleaves[i]) {
1136 has_subleaves = true;
1137 break;
1138 }
1139 }
1140
1141 if (!has_subleaves) {
1142 cpu_x86_cpuid(env, leaf, 0, &eax, &ebx, &ecx, &edx);
1143 add_cpuid_entry(cpuid_entries, leaf, 0, eax, ebx, ecx, edx);
1144 continue;
1145 }
1146
1147 /*
1148 * Valid XSAVE components can exist at a higher index se we need to set
1149 * all subleaves for leaf 0x0d, even if we encounter an empty one.
1150 */
1151 if (leaf == 0x0d) {
1152 for (subleaf = 0; subleaf <= 63; subleaf++) {
1153 cpu_x86_cpuid(env, leaf, subleaf, &eax, &ebx, &ecx, &edx);
1154 add_cpuid_entry(cpuid_entries, leaf, subleaf,
1155 eax, ebx, ecx, edx);
1156 }
1157 continue;
1158 }
1159
1160 subleaf = 0;
1161 while (subleaf < max_subleaf) {
1162 cpu_x86_cpuid(env, leaf, subleaf, &eax, &ebx, &ecx, &edx);
1163
1164 if (eax == 0 && ebx == 0 && ecx == 0 && edx == 0) {
1165 break;
1166 }
1167 add_cpuid_entry(cpuid_entries, leaf, subleaf, eax, ebx, ecx, edx);
1168 subleaf++;
1169 }
1170 }
1171
1172 /* Collect extended leaves (0x80000000 to max_extended_leaf) */
1173 for (leaf = 0x80000000; leaf <= max_extended_leaf; leaf++) {
1174 cpu_x86_cpuid(env, leaf, 0, &eax, &ebx, &ecx, &edx);
1175 add_cpuid_entry(cpuid_entries, leaf, 0, eax, ebx, ecx, edx);
1176 }
1177 }
1178
1179 static int register_intercept_result_cpuid_entry(const CPUState *cpu,
1180 uint8_t subleaf_specific,
1181 uint8_t always_override,
1182 uint32_t ebx_mask,
1183 struct hv_cpuid_entry *entry)
1184 {
1185 int ret;
1186 int vp_index = cpu->cpu_index;
1187 int cpu_fd = mshv_vcpufd(cpu);
1188
1189 struct hv_register_x64_cpuid_result_parameters cpuid_params = {
1190 .input.eax = entry->function,
1191 .input.ecx = entry->index,
1192 .input.subleaf_specific = subleaf_specific,
1193 .input.always_override = always_override,
1194 .input.padding = 0,
1195 /*
1196 * Masks specify which bits to override. Set to 0xFFFFFFFF to
1197 * override all bits with the values from the QEMU CPU model.
1198 * A mask of 0 lets the hypervisor supply its own value.
1199 */
1200 .result.eax = entry->eax,
1201 .result.eax_mask = 0xFFFFFFFF,
1202 .result.ebx = entry->ebx,
1203 .result.ebx_mask = ebx_mask,
1204 .result.ecx = entry->ecx,
1205 .result.ecx_mask = 0xFFFFFFFF,
1206 .result.edx = entry->edx,
1207 .result.edx_mask = 0xFFFFFFFF,
1208 };
1209 union hv_register_intercept_result_parameters parameters = {
1210 .cpuid = cpuid_params,
1211 };
1212
1213 hv_input_register_intercept_result in = {0};
1214 in.vp_index = vp_index;
1215 in.intercept_type = HV_INTERCEPT_TYPE_X64_CPUID;
1216 in.parameters = parameters;
1217
1218 struct mshv_root_hvcall args = {0};
1219 args.code = HVCALL_REGISTER_INTERCEPT_RESULT;
1220 args.in_sz = sizeof(in);
1221 args.in_ptr = (uint64_t)&in;
1222
1223 ret = mshv_hvcall(cpu_fd, &args);
1224 if (ret < 0) {
1225 error_report("failed to register intercept result for cpuid");
1226 return -1;
1227 }
1228
1229 return 0;
1230 }
1231
1232 static int register_intercept_result_cpuid(const CPUState *cpu,
1233 struct hv_cpuid *cpuid)
1234 {
1235 int ret = 0, entry_ret;
1236 struct hv_cpuid_entry *entry;
1237 uint8_t subleaf_specific, always_override;
1238 uint32_t ebx_mask;
1239
1240 for (size_t i = 0; i < cpuid->nent; i++) {
1241 entry = &cpuid->entries[i];
1242
1243 /* set defaults */
1244 subleaf_specific = 0;
1245 always_override = 1;
1246 ebx_mask = 0xFFFFFFFF;
1247
1248 /*
1249 * Intel
1250 * 0xb - Extended Topology Enumeration Leaf
1251 * 0x1f - V2 Extended Topology Enumeration Leaf
1252 * AMD
1253 * 0x8000_001e - Processor Topology Information
1254 * 0x8000_0026 - Extended CPU Topology
1255 */
1256 if (entry->function == 0xb ||
1257 entry->function == 0x1f ||
1258 entry->function == 0x8000001e ||
1259 entry->function == 0x80000026) {
1260 subleaf_specific = 1;
1261 always_override = 1;
1262 /*
1263 * Feature enumeration leaves (subleaf-specific)
1264 * 0x04: Deterministic Cache Parameters
1265 * 0x07: Structured Extended Feature Flags
1266 * 0x0D: Processor Extended State Enumeration
1267 * 0x0F: Platform QoS Monitoring
1268 * 0x10: Platform QoS Enforcement
1269 */
1270 } else if (entry->function == 0x04 ||
1271 entry->function == 0x07 ||
1272 entry->function == 0x0d ||
1273 entry->function == 0x0f ||
1274 entry->function == 0x10) {
1275 subleaf_specific = 1;
1276 always_override = 1;
1277 /* Basic feature leaves (no subleaves) */
1278 } else if (entry->function == 0x00000001 ||
1279 entry->function == 0x80000000 ||
1280 entry->function == 0x80000001 ||
1281 entry->function == 0x80000008) {
1282 subleaf_specific = 0;
1283 always_override = 1;
1284 }
1285
1286 /*
1287 * CPUID[0xD,0].EBX and CPUID[0xD,1].EBX report the XSAVE area
1288 * size based on features currently enabled in XCR0/XSS. These
1289 * values are dynamic and must not be overridden with static
1290 * results from the QEMU CPU model. Setting ebx_mask to 0 lets
1291 * the hypervisor supply EBX based on the guest's actual state.
1292 */
1293 if (entry->function == 0x0d &&
1294 (entry->index == 0 || entry->index == 1)) {
1295 ebx_mask = 0;
1296 }
1297
1298 entry_ret = register_intercept_result_cpuid_entry(cpu,
1299 subleaf_specific,
1300 always_override,
1301 ebx_mask,
1302 entry);
1303 if ((entry_ret < 0) && (ret == 0)) {
1304 ret = entry_ret;
1305 }
1306 }
1307
1308 return ret;
1309 }
1310
1311 static int init_cpuid2(const CPUState *cpu)
1312 {
1313 int ret;
1314 size_t n_entries, cpuid_size;
1315 struct hv_cpuid *cpuid;
1316 struct hv_cpuid_entry *entry;
1317 GList *entries = NULL;
1318
1319 collect_cpuid_entries(cpu, &entries);
1320 n_entries = g_list_length(entries);
1321
1322 cpuid_size = sizeof(struct hv_cpuid)
1323 + n_entries * sizeof(struct hv_cpuid_entry);
1324
1325 cpuid = g_malloc0(cpuid_size);
1326 cpuid->nent = n_entries;
1327 cpuid->padding = 0;
1328
1329 for (size_t i = 0; i < n_entries; i++) {
1330 entry = g_list_nth_data(entries, i);
1331 cpuid->entries[i] = *entry;
1332 g_free(entry);
1333 }
1334 g_list_free(entries);
1335
1336 ret = register_intercept_result_cpuid(cpu, cpuid);
1337 g_free(cpuid);
1338 if (ret < 0) {
1339 return ret;
1340 }
1341
1342 return 0;
1343 }
1344
1345 static inline void populate_hv_segment_reg(SegmentCache *seg,
1346 hv_x64_segment_register *hv_reg)
1347 {
1348 uint32_t flags = seg->flags;
1349
1350 hv_reg->base = seg->base;
1351 hv_reg->limit = seg->limit;
1352 hv_reg->selector = seg->selector;
1353 hv_reg->segment_type = (flags >> DESC_TYPE_SHIFT) & 0xF;
1354 hv_reg->non_system_segment = (flags & DESC_S_MASK) != 0;
1355 hv_reg->descriptor_privilege_level = (flags >> DESC_DPL_SHIFT) & 0x3;
1356 hv_reg->present = (flags & DESC_P_MASK) != 0;
1357 hv_reg->reserved = 0;
1358 hv_reg->available = (flags & DESC_AVL_MASK) != 0;
1359 hv_reg->_long = (flags >> DESC_L_SHIFT) & 0x1;
1360 hv_reg->_default = (flags >> DESC_B_SHIFT) & 0x1;
1361 hv_reg->granularity = (flags & DESC_G_MASK) != 0;
1362 }
1363
1364 static inline void populate_hv_table_reg(const struct SegmentCache *seg,
1365 hv_x64_table_register *hv_reg)
1366 {
1367 memset(hv_reg, 0, sizeof(*hv_reg));
1368
1369 hv_reg->base = seg->base;
1370 hv_reg->limit = seg->limit;
1371 }
1372
1373 static int set_special_regs(const CPUState *cpu)
1374 {
1375 X86CPU *x86cpu = X86_CPU(cpu);
1376 CPUX86State *env = &x86cpu->env;
1377 struct hv_register_assoc assocs[ARRAY_SIZE(SPECIAL_REGISTER_NAMES)];
1378 size_t n_regs = ARRAY_SIZE(SPECIAL_REGISTER_NAMES);
1379 int ret;
1380
1381 /* set names */
1382 for (size_t i = 0; i < n_regs; i++) {
1383 assocs[i].name = SPECIAL_REGISTER_NAMES[i];
1384 }
1385 populate_hv_segment_reg(&env->segs[R_CS], &assocs[0].value.segment);
1386 populate_hv_segment_reg(&env->segs[R_DS], &assocs[1].value.segment);
1387 populate_hv_segment_reg(&env->segs[R_ES], &assocs[2].value.segment);
1388 populate_hv_segment_reg(&env->segs[R_FS], &assocs[3].value.segment);
1389 populate_hv_segment_reg(&env->segs[R_GS], &assocs[4].value.segment);
1390 populate_hv_segment_reg(&env->segs[R_SS], &assocs[5].value.segment);
1391 populate_hv_segment_reg(&env->tr, &assocs[6].value.segment);
1392 populate_hv_segment_reg(&env->ldt, &assocs[7].value.segment);
1393
1394 populate_hv_table_reg(&env->gdt, &assocs[8].value.table);
1395 populate_hv_table_reg(&env->idt, &assocs[9].value.table);
1396
1397 assocs[10].value.reg64 = env->cr[0];
1398 assocs[11].value.reg64 = env->cr[2];
1399 assocs[12].value.reg64 = env->cr[3];
1400 assocs[13].value.reg64 = env->cr[4];
1401 assocs[14].value.reg64 = cpu_get_apic_tpr(x86cpu->apic_state);
1402 assocs[15].value.reg64 = env->efer;
1403 assocs[16].value.reg64 = cpu_get_apic_base(x86cpu->apic_state);
1404
1405 ret = mshv_set_generic_regs(cpu, assocs, n_regs);
1406 if (ret < 0) {
1407 error_report("failed to set special registers");
1408 return -1;
1409 }
1410
1411 return 0;
1412 }
1413
1414 static int set_fpu(const CPUState *cpu)
1415 {
1416 struct hv_register_assoc assocs[ARRAY_SIZE(FPU_REGISTER_NAMES)];
1417 union hv_register_value *value;
1418 union hv_x64_fp_control_status_register *ctrl_status;
1419 union hv_x64_xmm_control_status_register *xmm_ctrl_status;
1420 int ret;
1421 size_t n_regs = ARRAY_SIZE(FPU_REGISTER_NAMES);
1422 X86CPU *x86cpu = X86_CPU(cpu);
1423 CPUX86State *env = &x86cpu->env;
1424 size_t i, fp_i;
1425 bool valid;
1426
1427 /* first 16 registers are xmm0-xmm15 */
1428 for (i = 0; i < 16; i++) {
1429 assocs[i].name = FPU_REGISTER_NAMES[i];
1430 value = &assocs[i].value;
1431 value->reg128.low_part = env->xmm_regs[i].ZMM_Q(0);
1432 value->reg128.high_part = env->xmm_regs[i].ZMM_Q(1);
1433 }
1434
1435 /* next 8 registers are fp_mmx0-fp_mmx7 */
1436 for (i = 16; i < 24; i++) {
1437 fp_i = (i - 16);
1438 assocs[i].name = FPU_REGISTER_NAMES[i];
1439 value = &assocs[i].value;
1440 value->fp.mantissa = env->fpregs[fp_i].d.low;
1441 value->fp.biased_exponent = env->fpregs[fp_i].d.high & 0x7FFF;
1442 value->fp.sign = (env->fpregs[fp_i].d.high >> 15) & 0x1;
1443 value->fp.reserved = 0;
1444 }
1445
1446 /* last two registers are fp_control_status and xmm_control_status */
1447 assocs[24].name = FPU_REGISTER_NAMES[24];
1448 value = &assocs[24].value;
1449 ctrl_status = &value->fp_control_status;
1450
1451 ctrl_status->fp_control = env->fpuc;
1452 /* bits 11,12,13 are the top of stack pointer */
1453 ctrl_status->fp_status = (env->fpus & ~0x3800) | ((env->fpstt & 0x7) << 11);
1454
1455 ctrl_status->fp_tag = 0;
1456 for (i = 0; i < 8; i++) {
1457 valid = (env->fptags[i] == 0);
1458 if (valid) {
1459 ctrl_status->fp_tag |= (1u << i);
1460 }
1461 }
1462
1463 ctrl_status->reserved = 0;
1464 ctrl_status->last_fp_op = env->fpop;
1465 ctrl_status->last_fp_rip = env->fpip;
1466
1467 assocs[25].name = FPU_REGISTER_NAMES[25];
1468 value = &assocs[25].value;
1469 xmm_ctrl_status = &value->xmm_control_status;
1470 xmm_ctrl_status->xmm_status_control = env->mxcsr;
1471 xmm_ctrl_status->xmm_status_control_mask = 0x0000ffff;
1472 xmm_ctrl_status->last_fp_rdp = env->fpdp;
1473
1474 ret = mshv_set_generic_regs(cpu, assocs, n_regs);
1475 if (ret < 0) {
1476 error_report("failed to set fpu registers");
1477 return -1;
1478 }
1479
1480 return 0;
1481 }
1482
1483 static int set_xc_reg(const CPUState *cpu)
1484 {
1485 int ret;
1486 X86CPU *x86cpu = X86_CPU(cpu);
1487 CPUX86State *env = &x86cpu->env;
1488
1489 struct hv_register_assoc assoc = {
1490 .name = HV_X64_REGISTER_XFEM,
1491 .value.reg64 = env->xcr0,
1492 };
1493
1494 ret = mshv_set_generic_regs(cpu, &assoc, 1);
1495 if (ret < 0) {
1496 error_report("failed to set xcr0");
1497 return -errno;
1498 }
1499 return 0;
1500 }
1501
1502 static int set_synic_state(const CPUState *cpu)
1503 {
1504 X86CPU *x86cpu = X86_CPU(cpu);
1505 CPUX86State *env = &x86cpu->env;
1506 int cpu_fd = mshv_vcpufd(cpu);
1507 int ret;
1508
1509 ret = mshv_set_synthetic_timers(cpu_fd, env->hv_synthetic_timers_state);
1510 if (ret < 0) {
1511 error_report("failed to set synthetic timers state");
1512 return -1;
1513 }
1514
1515 /* SIMP/SIEFP can only be written when SynIC is enabled */
1516 if (!mshv_synic_enabled(cpu)) {
1517 return 0;
1518 }
1519
1520 ret = mshv_set_simp(cpu_fd, env->hv_simp_page);
1521 if (ret < 0) {
1522 error_report("failed to set simp state");
1523 return -1;
1524 }
1525
1526 ret = mshv_set_siefp(cpu_fd, env->hv_siefp_page);
1527 if (ret < 0) {
1528 error_report("failed to set siefp state");
1529 return -1;
1530 }
1531
1532 return 0;
1533 }
1534
1535 int mshv_arch_store_vcpu_state(const CPUState *cpu)
1536 {
1537 int ret;
1538
1539 ret = set_standard_regs(cpu);
1540 if (ret < 0) {
1541 return ret;
1542 }
1543
1544 ret = set_special_regs(cpu);
1545 if (ret < 0) {
1546 return ret;
1547 }
1548
1549 ret = set_xc_reg(cpu);
1550 if (ret < 0) {
1551 return ret;
1552 }
1553
1554 ret = set_xsave_state(cpu);
1555 if (ret < 0) {
1556 return ret;
1557 }
1558
1559 /* INVARIANT: special regs (APIC_BASE) must be restored before LAPIC */
1560 ret = mshv_set_lapic(cpu);
1561 if (ret < 0) {
1562 return ret;
1563 }
1564
1565 ret = mshv_set_msrs(cpu);
1566 if (ret < 0) {
1567 return ret;
1568 }
1569
1570 /* INVARIANT: legacy FPU state must be restored after XSAVE */
1571 ret = set_fpu(cpu);
1572 if (ret < 0) {
1573 return ret;
1574 }
1575
1576 ret = set_synic_state(cpu);
1577 if (ret < 0) {
1578 return ret;
1579 }
1580
1581 ret = set_vcpu_events(cpu);
1582 if (ret < 0) {
1583 return ret;
1584 }
1585
1586 return 0;
1587 }
1588
1589 int mshv_arch_set_partition_msrs(const CPUState *cpu)
1590 {
1591 CPUX86State *env = &X86_CPU(cpu)->env;
1592 struct hv_register_assoc assocs[] = {
1593 { .name = HV_REGISTER_GUEST_OS_ID,
1594 .value.reg64 = env->msr_hv_guest_os_id },
1595 { .name = HV_REGISTER_REFERENCE_TSC,
1596 .value.reg64 = env->msr_hv_tsc },
1597 { .name = HV_X64_REGISTER_HYPERCALL,
1598 .value.reg64 = env->msr_hv_hypercall },
1599 };
1600
1601 return mshv_set_generic_regs(cpu, assocs, ARRAY_SIZE(assocs));
1602 }
1603
1604 void mshv_arch_amend_proc_features(
1605 union hv_partition_synthetic_processor_features *features)
1606 {
1607 features->access_guest_idle_reg = 1;
1608 }
1609
1610 void mshv_arch_disable_partition_proc_features(
1611 union hv_partition_processor_features *disabled_features)
1612 {
1613 disabled_features->la57_support = 1;
1614 }
1615
1616 static int set_memory_info(const struct hyperv_message *msg,
1617 struct hv_x64_memory_intercept_message *info)
1618 {
1619 if (msg->header.message_type != HVMSG_GPA_INTERCEPT
1620 && msg->header.message_type != HVMSG_UNMAPPED_GPA
1621 && msg->header.message_type != HVMSG_UNACCEPTED_GPA) {
1622 error_report("invalid message type");
1623 return -1;
1624 }
1625 memcpy(info, msg->payload, sizeof(*info));
1626
1627 return 0;
1628 }
1629
1630 static int emulate_instruction(CPUState *cpu,
1631 const uint8_t *insn_bytes, size_t insn_len,
1632 uint64_t gva, uint64_t gpa)
1633 {
1634 X86CPU *x86_cpu = X86_CPU(cpu);
1635 CPUX86State *env = &x86_cpu->env;
1636 struct x86_decode decode = { 0 };
1637 int ret;
1638 x86_insn_stream stream = { .bytes = insn_bytes, .len = insn_len };
1639
1640 ret = load_regs(cpu);
1641 if (ret < 0) {
1642 error_report("Failed to load registers");
1643 return -1;
1644 }
1645
1646 decode_instruction_stream(env, &decode, &stream);
1647 exec_instruction(env, &decode);
1648
1649 ret = store_regs(cpu);
1650 if (ret < 0) {
1651 error_report("failed to store registers");
1652 return -1;
1653 }
1654
1655 return 0;
1656 }
1657
1658 static int handle_mmio(CPUState *cpu, const struct hyperv_message *msg,
1659 MshvVmExit *exit_reason)
1660 {
1661 struct hv_x64_memory_intercept_message info = { 0 };
1662 size_t insn_len;
1663 uint8_t access_type;
1664 uint8_t *instruction_bytes;
1665 int ret;
1666
1667 ret = set_memory_info(msg, &info);
1668 if (ret < 0) {
1669 error_report("failed to convert message to memory info");
1670 return -1;
1671 }
1672 insn_len = info.instruction_byte_count;
1673 access_type = info.header.intercept_access_type;
1674
1675 if (access_type == HV_X64_INTERCEPT_ACCESS_TYPE_EXECUTE) {
1676 error_report("invalid intercept access type: execute");
1677 return -1;
1678 }
1679
1680 if (insn_len > 16) {
1681 error_report("invalid mmio instruction length: %zu", insn_len);
1682 return -1;
1683 }
1684
1685 trace_mshv_handle_mmio(info.guest_virtual_address,
1686 info.guest_physical_address,
1687 info.instruction_byte_count, access_type);
1688
1689 instruction_bytes = info.instruction_bytes;
1690
1691 ret = emulate_instruction(cpu, instruction_bytes, insn_len,
1692 info.guest_virtual_address,
1693 info.guest_physical_address);
1694 if (ret < 0) {
1695 error_report("failed to emulate mmio");
1696 return -1;
1697 }
1698
1699 *exit_reason = MshvVmExitIgnore;
1700
1701 return 0;
1702 }
1703
1704 static int set_ioport_info(const struct hyperv_message *msg,
1705 hv_x64_io_port_intercept_message *info)
1706 {
1707 if (msg->header.message_type != HVMSG_X64_IO_PORT_INTERCEPT) {
1708 error_report("Invalid message type");
1709 return -1;
1710 }
1711 memcpy(info, msg->payload, sizeof(*info));
1712
1713 return 0;
1714 }
1715
1716 static int set_x64_registers(const CPUState *cpu, const uint32_t *names,
1717 const uint64_t *values)
1718 {
1719
1720 hv_register_assoc assocs[2];
1721 int ret;
1722
1723 for (size_t i = 0; i < ARRAY_SIZE(assocs); i++) {
1724 assocs[i].name = names[i];
1725 assocs[i].value.reg64 = values[i];
1726 }
1727
1728 ret = mshv_set_generic_regs(cpu, assocs, ARRAY_SIZE(assocs));
1729 if (ret < 0) {
1730 error_report("failed to set x64 registers");
1731 return -1;
1732 }
1733
1734 return 0;
1735 }
1736
1737 static inline MemTxAttrs get_mem_attrs(bool is_secure_mode)
1738 {
1739 MemTxAttrs memattr = {0};
1740 memattr.secure = is_secure_mode;
1741 return memattr;
1742 }
1743
1744 static void pio_read(uint64_t port, uint8_t *data, uintptr_t size,
1745 bool is_secure_mode)
1746 {
1747 int ret = 0;
1748 MemTxAttrs memattr = get_mem_attrs(is_secure_mode);
1749 ret = address_space_rw(&address_space_io, port, memattr, (void *)data, size,
1750 false);
1751 if (ret != MEMTX_OK) {
1752 error_report("Failed to read from port %lx: %d", port, ret);
1753 abort();
1754 }
1755 }
1756
1757 static int pio_write(uint64_t port, const uint8_t *data, uintptr_t size,
1758 bool is_secure_mode)
1759 {
1760 int ret = 0;
1761 MemTxAttrs memattr = get_mem_attrs(is_secure_mode);
1762 ret = address_space_rw(&address_space_io, port, memattr, (void *)data, size,
1763 true);
1764 return ret;
1765 }
1766
1767 static int handle_pio_non_str(CPUState *cpu,
1768 hv_x64_io_port_intercept_message *info)
1769 {
1770 size_t len = info->access_info.access_size;
1771 uint8_t access_type = info->header.intercept_access_type;
1772 int ret;
1773 uint32_t val, eax;
1774 const uint32_t eax_mask = 0xffffffffu >> (32 - len * 8);
1775 size_t insn_len;
1776 uint64_t rip, rax;
1777 uint32_t reg_names[2];
1778 uint64_t reg_values[2];
1779 uint16_t port = info->port_number;
1780
1781 if (access_type == HV_X64_INTERCEPT_ACCESS_TYPE_WRITE) {
1782 union {
1783 uint32_t u32;
1784 uint8_t bytes[4];
1785 } conv;
1786
1787 /* convert the first 4 bytes of rax to bytes */
1788 conv.u32 = (uint32_t)info->rax;
1789 /* secure mode is set to false */
1790 ret = pio_write(port, conv.bytes, len, false);
1791 if (ret < 0) {
1792 error_report("Failed to write to io port");
1793 return -1;
1794 }
1795 } else {
1796 uint8_t data[4] = { 0 };
1797 /* secure mode is set to false */
1798 pio_read(info->port_number, data, len, false);
1799
1800 /* Preserve high bits in EAX, but clear out high bits in RAX */
1801 val = *(uint32_t *)data;
1802 eax = (((uint32_t)info->rax) & ~eax_mask) | (val & eax_mask);
1803 info->rax = (uint64_t)eax;
1804 }
1805
1806 insn_len = info->header.instruction_length;
1807
1808 /* Advance RIP and update RAX */
1809 rip = info->header.rip + insn_len;
1810 rax = info->rax;
1811
1812 reg_names[0] = HV_X64_REGISTER_RIP;
1813 reg_values[0] = rip;
1814 reg_names[1] = HV_X64_REGISTER_RAX;
1815 reg_values[1] = rax;
1816
1817 ret = set_x64_registers(cpu, reg_names, reg_values);
1818 if (ret < 0) {
1819 error_report("Failed to set x64 registers");
1820 return -1;
1821 }
1822
1823 cpu->vcpu_dirty = false;
1824
1825 return 0;
1826 }
1827
1828 static int read_memory(const CPUState *cpu, uint64_t initial_gva,
1829 uint64_t initial_gpa, uint64_t gva, uint8_t *data,
1830 size_t len)
1831 {
1832 int ret;
1833 uint64_t gpa, flags;
1834
1835 if (gva == initial_gva) {
1836 gpa = initial_gpa;
1837 } else {
1838 flags = HV_TRANSLATE_GVA_VALIDATE_READ;
1839 ret = translate_gva(cpu, gva, &gpa, flags);
1840 if (ret < 0) {
1841 return -1;
1842 }
1843
1844 ret = mshv_guest_mem_read(gpa, data, len, false, false);
1845 if (ret < 0) {
1846 error_report("failed to read guest mem");
1847 return -1;
1848 }
1849 }
1850
1851 return 0;
1852 }
1853
1854 static int write_memory(const CPUState *cpu, uint64_t gva, const uint8_t *data,
1855 size_t len)
1856 {
1857 int ret;
1858 uint64_t gpa, flags;
1859
1860 flags = HV_TRANSLATE_GVA_VALIDATE_WRITE;
1861 ret = translate_gva(cpu, gva, &gpa, flags);
1862 if (ret < 0) {
1863 error_report("failed to translate gva to gpa");
1864 return -1;
1865 }
1866
1867 ret = mshv_guest_mem_write(gpa, data, len, false);
1868 if (ret != MEMTX_OK) {
1869 error_report("failed to write to mmio");
1870 return -1;
1871 }
1872
1873 return 0;
1874 }
1875
1876 static int handle_pio_str_write(CPUState *cpu,
1877 hv_x64_io_port_intercept_message *info,
1878 size_t repeat, uint16_t port,
1879 bool direction_flag)
1880 {
1881 int ret;
1882 uint64_t src;
1883 uint8_t data[4] = { 0 };
1884 size_t len = info->access_info.access_size;
1885
1886 src = linear_addr(cpu, info->rsi, R_DS);
1887
1888 for (size_t i = 0; i < repeat; i++) {
1889 ret = read_memory(cpu, 0, 0, src, data, len);
1890 if (ret < 0) {
1891 error_report("Failed to read memory");
1892 return -1;
1893 }
1894 ret = pio_write(port, data, len, false);
1895 if (ret < 0) {
1896 error_report("Failed to write to io port");
1897 return -1;
1898 }
1899 src += direction_flag ? -len : len;
1900 info->rsi += direction_flag ? -len : len;
1901 }
1902
1903 return 0;
1904 }
1905
1906 static int handle_pio_str_read(CPUState *cpu,
1907 hv_x64_io_port_intercept_message *info,
1908 size_t repeat, uint16_t port,
1909 bool direction_flag)
1910 {
1911 int ret;
1912 uint64_t dst;
1913 size_t len = info->access_info.access_size;
1914 uint8_t data[4] = { 0 };
1915
1916 dst = linear_addr(cpu, info->rdi, R_ES);
1917
1918 for (size_t i = 0; i < repeat; i++) {
1919 pio_read(port, data, len, false);
1920
1921 ret = write_memory(cpu, dst, data, len);
1922 if (ret < 0) {
1923 error_report("Failed to write memory");
1924 return -1;
1925 }
1926 dst += direction_flag ? -len : len;
1927 info->rdi += direction_flag ? -len : len;
1928 }
1929
1930 return 0;
1931 }
1932
1933 static int handle_pio_str(CPUState *cpu, hv_x64_io_port_intercept_message *info)
1934 {
1935 uint8_t access_type = info->header.intercept_access_type;
1936 uint16_t port = info->port_number;
1937 bool repop = info->access_info.rep_prefix == 1;
1938 size_t repeat = repop ? info->rcx : 1;
1939 size_t insn_len = info->header.instruction_length;
1940 bool direction_flag;
1941 uint32_t reg_names[3];
1942 uint64_t reg_values[3];
1943 int ret;
1944 X86CPU *x86_cpu = X86_CPU(cpu);
1945 CPUX86State *env = &x86_cpu->env;
1946
1947 ret = load_regs(cpu);
1948 if (ret < 0) {
1949 error_report("Failed to load registers");
1950 return -1;
1951 }
1952
1953 direction_flag = (env->eflags & DESC_E_MASK) != 0;
1954
1955 if (access_type == HV_X64_INTERCEPT_ACCESS_TYPE_WRITE) {
1956 ret = handle_pio_str_write(cpu, info, repeat, port, direction_flag);
1957 if (ret < 0) {
1958 error_report("Failed to handle pio str write");
1959 return -1;
1960 }
1961 reg_names[0] = HV_X64_REGISTER_RSI;
1962 reg_values[0] = info->rsi;
1963 } else {
1964 ret = handle_pio_str_read(cpu, info, repeat, port, direction_flag);
1965 if (ret < 0) {
1966 error_report("Failed to handle pio str read");
1967 return -1;
1968 }
1969 reg_names[0] = HV_X64_REGISTER_RDI;
1970 reg_values[0] = info->rdi;
1971 }
1972
1973 reg_names[1] = HV_X64_REGISTER_RIP;
1974 reg_values[1] = info->header.rip + insn_len;
1975 reg_names[2] = HV_X64_REGISTER_RAX;
1976 reg_values[2] = info->rax;
1977
1978 ret = set_x64_registers(cpu, reg_names, reg_values);
1979 if (ret < 0) {
1980 error_report("Failed to set RIP and RAX registers");
1981 return -1;
1982 }
1983
1984 cpu->vcpu_dirty = false;
1985
1986 return 0;
1987 }
1988
1989 static int handle_pio(CPUState *cpu, const struct hyperv_message *msg)
1990 {
1991 struct hv_x64_io_port_intercept_message info = { 0 };
1992 int ret;
1993
1994 ret = set_ioport_info(msg, &info);
1995 if (ret < 0) {
1996 error_report("Failed to convert message to ioport info");
1997 return -1;
1998 }
1999
2000 if (info.access_info.string_op) {
2001 return handle_pio_str(cpu, &info);
2002 }
2003
2004 return handle_pio_non_str(cpu, &info);
2005 }
2006
2007 int mshv_run_vcpu(int vm_fd, CPUState *cpu, hv_message *msg, MshvVmExit *exit)
2008 {
2009 int ret;
2010 enum MshvVmExit exit_reason;
2011 int cpu_fd = mshv_vcpufd(cpu);
2012
2013 ret = ioctl(cpu_fd, MSHV_RUN_VP, msg);
2014 if (ret < 0) {
2015 return MshvVmExitShutdown;
2016 }
2017
2018 switch (msg->header.message_type) {
2019 case HVMSG_UNRECOVERABLE_EXCEPTION:
2020 return MshvVmExitShutdown;
2021 case HVMSG_UNMAPPED_GPA:
2022 case HVMSG_GPA_INTERCEPT:
2023 ret = handle_mmio(cpu, msg, &exit_reason);
2024 if (ret < 0) {
2025 error_report("failed to handle mmio");
2026 return -1;
2027 }
2028 return exit_reason;
2029 case HVMSG_X64_IO_PORT_INTERCEPT:
2030 ret = handle_pio(cpu, msg);
2031 if (ret < 0) {
2032 return MshvVmExitSpecial;
2033 }
2034 return MshvVmExitIgnore;
2035 default:
2036 break;
2037 }
2038
2039 *exit = MshvVmExitIgnore;
2040 return 0;
2041 }
2042
2043 void mshv_remove_vcpu(int vm_fd, int cpu_fd)
2044 {
2045 close(cpu_fd);
2046 }
2047
2048
2049 int mshv_create_vcpu(int vm_fd, uint8_t vp_index, int *cpu_fd)
2050 {
2051 int ret;
2052 struct mshv_create_vp vp_arg = {
2053 .vp_index = vp_index,
2054 };
2055 ret = ioctl(vm_fd, MSHV_CREATE_VP, &vp_arg);
2056 if (ret < 0) {
2057 error_report("failed to create mshv vcpu: %s", strerror(errno));
2058 return -1;
2059 }
2060
2061 *cpu_fd = ret;
2062
2063 return 0;
2064 }
2065
2066 static void read_segment_descriptor(CPUState *cpu,
2067 struct x86_segment_descriptor *desc,
2068 enum X86Seg seg_idx)
2069 {
2070 X86CPU *x86_cpu = X86_CPU(cpu);
2071 CPUX86State *env = &x86_cpu->env;
2072 SegmentCache *seg = &env->segs[seg_idx];
2073 uint32_t limit;
2074
2075 memset(desc, 0, sizeof(struct x86_segment_descriptor));
2076
2077 desc->type = (seg->flags & DESC_TYPE_MASK) >> DESC_TYPE_SHIFT;
2078 desc->s = (seg->flags & DESC_S_MASK) >> DESC_S_SHIFT;
2079 desc->dpl = (seg->flags & DESC_DPL_MASK) >> DESC_DPL_SHIFT;
2080 desc->p = (seg->flags & DESC_P_MASK) >> DESC_P_SHIFT;
2081 desc->avl = (seg->flags & DESC_AVL_MASK) >> DESC_AVL_SHIFT;
2082 desc->l = (seg->flags & DESC_L_MASK) >> DESC_L_SHIFT;
2083 desc->db = (seg->flags & DESC_B_MASK) >> DESC_B_SHIFT;
2084 desc->g = (seg->flags & DESC_G_MASK) >> DESC_G_SHIFT;
2085
2086 /*
2087 * SegmentCache stores the hypervisor-provided value verbatim (populated by
2088 * mshv_load_regs). We need to convert it to format expected by the
2089 * instruction emulator. We can have a limit value > 0xfffff with
2090 * granularity of 0 (byte granularity), which is not representable
2091 * in real x86_segment_descriptor. In this case we set granularity to 1
2092 * (4k granularity) and shift the limit accordingly.
2093 *
2094 * This quirk has been adopted from "whpx_segment_to_x86_description()"
2095 */
2096
2097 if (!desc->g && seg->limit <= 0xfffff) {
2098 limit = seg->limit;
2099 } else {
2100 limit = seg->limit >> 12;
2101 desc->g = 1;
2102 }
2103
2104 x86_set_segment_limit(desc, limit);
2105 x86_set_segment_base(desc, seg->base);
2106 }
2107
2108 static const struct x86_emul_ops mshv_x86_emul_ops = {
2109 .read_segment_descriptor = read_segment_descriptor,
2110 };
2111
2112 void mshv_init_mmio_emu(void)
2113 {
2114 init_decoder();
2115 init_emu(&mshv_x86_emul_ops);
2116 }
2117
2118 void mshv_arch_init_vcpu(CPUState *cpu)
2119 {
2120 X86CPU *x86_cpu = X86_CPU(cpu);
2121 CPUX86State *env = &x86_cpu->env;
2122 AccelCPUState *state = cpu->accel;
2123 size_t page = HV_HYP_PAGE_SIZE, xsave_len;
2124 void *mem = qemu_memalign(page, 2 * page);
2125 int ret;
2126 X86XSaveHeader *header;
2127
2128 /* sanity check, to make sure we don't overflow the page */
2129 QEMU_BUILD_BUG_ON((MAX_REGISTER_COUNT
2130 * sizeof(hv_register_assoc)
2131 + sizeof(hv_input_get_vp_registers)
2132 > HV_HYP_PAGE_SIZE));
2133
2134 /* mmap the registers page */
2135 void *rp = mmap(NULL, page, PROT_READ | PROT_WRITE,
2136 MAP_SHARED, mshv_vcpufd(cpu),
2137 MSHV_VP_MMAP_OFFSET_REGISTERS * page);
2138 if (rp == MAP_FAILED) {
2139 warn_report("register page mmap failed, falling back to hypercalls: %s",
2140 strerror(errno));
2141 env->regs_page = NULL;
2142 } else {
2143 env->regs_page = (struct hv_vp_register_page *) rp;
2144 }
2145
2146 state->hvcall_args.base = mem;
2147 state->hvcall_args.input_page = mem;
2148 state->hvcall_args.output_page = (uint8_t *)mem + page;
2149
2150 env->emu_mmio_buf = g_new(char, 4096);
2151
2152 /* Initialize XSAVE buffer page-aligned */
2153 /* TODO: pick proper size based on CPUID */
2154 xsave_len = page;
2155 env->xsave_buf = qemu_memalign(page, xsave_len);
2156 env->xsave_buf_len = xsave_len;
2157 memset(env->xsave_buf, 0, env->xsave_buf_len);
2158
2159 /* we need to set the compacted format bit in xsave header for mshv */
2160 header = (X86XSaveHeader *)(env->xsave_buf + sizeof(X86LegacyXSaveArea));
2161 header->xcomp_bv = header->xstate_bv | (1ULL << 63);
2162
2163 /*
2164 * TODO: populate topology info:
2165 * X86CPUTopoInfo *topo_info = &env->topo_info;
2166 */
2167
2168 ret = init_cpuid2(cpu);
2169 assert(ret == 0);
2170
2171 ret = mshv_init_msrs(cpu);
2172 assert(ret == 0);
2173
2174 ret = mshv_init_lint(cpu);
2175 assert(ret == 0);
2176 }
2177
2178 void mshv_arch_destroy_vcpu(CPUState *cpu)
2179 {
2180 X86CPU *x86_cpu = X86_CPU(cpu);
2181 CPUX86State *env = &x86_cpu->env;
2182 AccelCPUState *state = cpu->accel;
2183
2184 /* Unmap the register page */
2185 if (env->regs_page) {
2186 munmap(env->regs_page, HV_HYP_PAGE_SIZE);
2187 env->regs_page = NULL;
2188 }
2189 g_free(state->hvcall_args.base);
2190 state->hvcall_args = (MshvHvCallArgs){0};
2191 g_clear_pointer(&env->emu_mmio_buf, g_free);
2192
2193 qemu_vfree(env->xsave_buf);
2194 env->xsave_buf = NULL;
2195 env->xsave_buf_len = 0;
2196 }
2197
2198 uint32_t mshv_get_supported_cpuid(uint32_t func, uint32_t idx, int reg)
2199 {
2200 uint32_t eax, ebx, ecx, edx;
2201 uint32_t ret = 0;
2202
2203 host_cpuid(func, idx, &eax, &ebx, &ecx, &edx);
2204 switch (reg) {
2205 case R_EAX:
2206 ret = eax; break;
2207 case R_EBX:
2208 ret = ebx; break;
2209 case R_ECX:
2210 ret = ecx; break;
2211 case R_EDX:
2212 ret = edx; break;
2213 }
2214
2215 /* Disable nested virtualization features not yet supported by MSHV */
2216 if (func == 0x80000001 && reg == R_ECX) {
2217 ret &= ~CPUID_EXT3_SVM;
2218 }
2219 if (func == 0x01 && reg == R_ECX) {
2220 ret &= ~CPUID_EXT_VMX;
2221 }
2222
2223 if (func == 0x07 && idx == 0 && reg == R_ECX) {
2224 /*
2225 * LA57 (5-level paging) causes incorrect GVA=>GPA translations
2226 * in the instruction decoder/emulator. Disable until page table
2227 * walk in x86_mmu.c works w/ 5-level paging.
2228 */
2229 ret &= ~CPUID_7_0_ECX_LA57;
2230 }
2231 if (func == 0x07 && idx == 0 && reg == R_EDX) {
2232 /*
2233 * AMX TILE XSAVE state (XTILE_DATA) is 8KB, which exceeds the
2234 * current fixed 4KB XSAVE buffer size. Filter until buffer
2235 * sizing is computed dynamically from CPUID.
2236 */
2237 ret &= ~CPUID_7_0_EDX_AMX_TILE;
2238 ret &= ~CPUID_7_0_EDX_AMX_BF16;
2239 ret &= ~CPUID_7_0_EDX_AMX_INT8;
2240 }
2241 if (func == 0x07 && idx == 1 && reg == R_EAX) {
2242 ret &= ~CPUID_7_1_EAX_AMX_FP16;
2243 }
2244 if (func == 0x07 && idx == 1 && reg == R_EDX) {
2245 ret &= ~CPUID_7_1_EDX_AMX_COMPLEX;
2246 }
2247
2248 return ret;
2249 }
2250
2251 /*
2252 * Default Microsoft Hypervisor behavior for unimplemented MSR is to send a
2253 * fault to the guest if it tries to access it. It is possible to override
2254 * this behavior with a more suitable option i.e., ignore writes from the guest
2255 * and return zero in attempt to read unimplemented.
2256 */
2257 static int set_unimplemented_msr_action(int vm_fd)
2258 {
2259 struct hv_input_set_partition_property in = {0};
2260 struct mshv_root_hvcall args = {0};
2261
2262 in.property_code = HV_PARTITION_PROPERTY_UNIMPLEMENTED_MSR_ACTION;
2263 in.property_value = HV_UNIMPLEMENTED_MSR_ACTION_IGNORE_WRITE_READ_ZERO;
2264
2265 args.code = HVCALL_SET_PARTITION_PROPERTY;
2266 args.in_sz = sizeof(in);
2267 args.in_ptr = (uint64_t)&in;
2268
2269 trace_mshv_hvcall_args("unimplemented_msr_action", args.code, args.in_sz);
2270
2271 int ret = mshv_hvcall(vm_fd, &args);
2272 if (ret < 0) {
2273 error_report("Failed to set unimplemented MSR action");
2274 return -1;
2275 }
2276 return 0;
2277 }
2278
2279 int mshv_arch_post_init_vm(int vm_fd)
2280 {
2281 int ret;
2282
2283 ret = set_unimplemented_msr_action(vm_fd);
2284 if (ret < 0) {
2285 error_report("Failed to set unimplemented MSR action");
2286 }
2287
2288 return ret;
2289 }
2290
2291 static void mshv_cpu_xsave_init(void)
2292 {
2293 static bool first = true;
2294 uint32_t eax, ebx, ecx, edx;
2295 int i;
2296
2297 if (!first) {
2298 return;
2299 }
2300 first = false;
2301
2302 /* x87 and SSE states are in the legacy region of the XSAVE area. */
2303 x86_ext_save_areas[XSTATE_FP_BIT].offset = 0;
2304 x86_ext_save_areas[XSTATE_SSE_BIT].offset = 0;
2305
2306 for (i = XSTATE_SSE_BIT + 1; i < XSAVE_STATE_AREA_COUNT; i++) {
2307 ExtSaveArea *esa = &x86_ext_save_areas[i];
2308
2309 if (!esa->size) {
2310 continue;
2311 }
2312 host_cpuid(0xd, i, &eax, &ebx, &ecx, &edx);
2313 if (eax != 0) {
2314 assert(esa->size == eax);
2315 esa->offset = ebx;
2316 esa->ecx = ecx;
2317 }
2318 }
2319 }
2320
2321 int mshv_set_vp_state(int cpu_fd, const struct mshv_get_set_vp_state *state)
2322 {
2323 int ret;
2324
2325 ret = ioctl(cpu_fd, MSHV_SET_VP_STATE, state);
2326 if (ret < 0) {
2327 error_report("failed to set partition state: %s", strerror(errno));
2328 return -1;
2329 }
2330
2331 return 0;
2332 }
2333
2334
2335 int mshv_get_vp_state(int cpu_fd, struct mshv_get_set_vp_state *state)
2336 {
2337 int ret;
2338
2339 ret = ioctl(cpu_fd, MSHV_GET_VP_STATE, state);
2340 if (ret < 0) {
2341 error_report("failed to get partition state: %s", strerror(errno));
2342 return -1;
2343 }
2344
2345 return 0;
2346 }
2347
2348 static void mshv_cpu_instance_init(CPUState *cs)
2349 {
2350 X86CPU *cpu = X86_CPU(cs);
2351
2352 host_cpu_instance_init(cpu);
2353 mshv_cpu_xsave_init();
2354 }
2355
2356 static void mshv_cpu_accel_class_init(ObjectClass *oc, const void *data)
2357 {
2358 AccelCPUClass *acc = ACCEL_CPU_CLASS(oc);
2359
2360 acc->cpu_instance_init = mshv_cpu_instance_init;
2361 }
2362
2363 static const TypeInfo mshv_cpu_accel_type_info = {
2364 .name = ACCEL_CPU_NAME("mshv"),
2365 .parent = TYPE_ACCEL_CPU,
2366 .class_init = mshv_cpu_accel_class_init,
2367 .abstract = true,
2368 };
2369
2370 static void mshv_cpu_accel_register_types(void)
2371 {
2372 type_register_static(&mshv_cpu_accel_type_info);
2373 }
2374
2375 type_init(mshv_cpu_accel_register_types);